168 Register rnew_zaddress,
169 Register rnew_zpointer,
170 Register rtmp,
171 bool in_nmethod,
172 bool is_atomic,
173 Label& medium_path,
174 Label& medium_path_continuation) const {
175 assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
176 assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);
177
178 if (in_nmethod) {
179 if (is_atomic) {
180 __ lhu(rtmp, ref_addr);
181 // Atomic operations must ensure that the contents of memory are store-good before
182 // an atomic opertion can execute.
183 // A non-relocatable object could have spurious raw null pointers in its fields after
184 // getting promoted to the old generation.
185 __ relocate(barrier_Relocation::spec(), [&] {
186 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
187 }, ZBarrierRelocationFormatStoreGoodBits);
188 __ bne(rtmp, rnew_zpointer, medium_path, true /* is_far */);
189 } else {
190 __ ld(rtmp, ref_addr);
191 // Stores on relocatable objects never need to deal with raw null pointers in fields.
192 // Raw null pointers may only exists in the young generation, as they get pruned when
193 // the object is relocated to old. And no pre-write barrier needs to perform any action
194 // in the young generation.
195 __ relocate(barrier_Relocation::spec(), [&] {
196 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
197 }, ZBarrierRelocationFormatStoreBadMask);
198 __ andr(rtmp, rtmp, rnew_zpointer);
199 __ bnez(rtmp, medium_path, true /* is_far */);
200 }
201 __ bind(medium_path_continuation);
202 __ relocate(barrier_Relocation::spec(), [&] {
203 __ li16u(rtmp, barrier_Relocation::unpatched);
204 }, ZBarrierRelocationFormatStoreGoodBits);
205 __ slli(rnew_zpointer, rnew_zaddress, ZPointerLoadShift);
206 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
207 } else {
208 assert(!is_atomic, "atomic outside of nmethods not supported");
209 __ la(rtmp, ref_addr);
210 __ ld(rtmp, rtmp);
211 __ ld(rnew_zpointer, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
212 __ andr(rtmp, rtmp, rnew_zpointer);
213 __ bnez(rtmp, medium_path, true /* is_far */);
214 __ bind(medium_path_continuation);
215 if (rnew_zaddress == noreg) {
216 __ mv(rnew_zpointer, zr);
217 } else {
218 __ mv(rnew_zpointer, rnew_zaddress);
219 }
220
221 // Load the current good shift, and add the color bits
222 __ slli(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
223 __ ld(rtmp, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
224 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
225 }
226 }
227
228 static void store_barrier_buffer_add(MacroAssembler* masm,
229 Address ref_addr,
230 Register tmp1,
231 Register tmp2,
232 Label& slow_path) {
233 Address buffer(xthread, ZThreadLocalData::store_barrier_buffer_offset());
|
168 Register rnew_zaddress,
169 Register rnew_zpointer,
170 Register rtmp,
171 bool in_nmethod,
172 bool is_atomic,
173 Label& medium_path,
174 Label& medium_path_continuation) const {
175 assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
176 assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);
177
178 if (in_nmethod) {
179 if (is_atomic) {
180 __ lhu(rtmp, ref_addr);
181 // Atomic operations must ensure that the contents of memory are store-good before
182 // an atomic opertion can execute.
183 // A non-relocatable object could have spurious raw null pointers in its fields after
184 // getting promoted to the old generation.
185 __ relocate(barrier_Relocation::spec(), [&] {
186 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
187 }, ZBarrierRelocationFormatStoreGoodBits);
188 __ bne(rtmp, rnew_zpointer, medium_path, /* is_far */ true);
189 } else {
190 __ ld(rtmp, ref_addr);
191 // Stores on relocatable objects never need to deal with raw null pointers in fields.
192 // Raw null pointers may only exists in the young generation, as they get pruned when
193 // the object is relocated to old. And no pre-write barrier needs to perform any action
194 // in the young generation.
195 __ relocate(barrier_Relocation::spec(), [&] {
196 __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
197 }, ZBarrierRelocationFormatStoreBadMask);
198 __ andr(rtmp, rtmp, rnew_zpointer);
199 __ bnez(rtmp, medium_path, /* is_far */ true);
200 }
201 __ bind(medium_path_continuation);
202 __ relocate(barrier_Relocation::spec(), [&] {
203 __ li16u(rtmp, barrier_Relocation::unpatched);
204 }, ZBarrierRelocationFormatStoreGoodBits);
205 __ slli(rnew_zpointer, rnew_zaddress, ZPointerLoadShift);
206 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
207 } else {
208 assert(!is_atomic, "atomic outside of nmethods not supported");
209 __ la(rtmp, ref_addr);
210 __ ld(rtmp, rtmp);
211 __ ld(rnew_zpointer, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
212 __ andr(rtmp, rtmp, rnew_zpointer);
213 __ bnez(rtmp, medium_path, /* is_far */ true);
214 __ bind(medium_path_continuation);
215 if (rnew_zaddress == noreg) {
216 __ mv(rnew_zpointer, zr);
217 } else {
218 __ mv(rnew_zpointer, rnew_zaddress);
219 }
220
221 // Load the current good shift, and add the color bits
222 __ slli(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
223 __ ld(rtmp, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
224 __ orr(rnew_zpointer, rnew_zpointer, rtmp);
225 }
226 }
227
228 static void store_barrier_buffer_add(MacroAssembler* masm,
229 Address ref_addr,
230 Register tmp1,
231 Register tmp2,
232 Label& slow_path) {
233 Address buffer(xthread, ZThreadLocalData::store_barrier_buffer_offset());
|