1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "code/SCCache.hpp"
 28 #include "gc/g1/g1BarrierSet.hpp"
 29 #include "gc/g1/g1BarrierSetAssembler.hpp"
 30 #include "gc/g1/g1BarrierSetRuntime.hpp"
 31 #include "gc/g1/g1CardTable.hpp"
 32 #include "gc/g1/g1ThreadLocalData.hpp"
 33 #include "gc/g1/heapRegion.hpp"
 34 #include "interpreter/interp_masm.hpp"
 35 #include "runtime/sharedRuntime.hpp"
 36 #include "utilities/debug.hpp"
 37 #include "utilities/macros.hpp"
 38 #ifdef COMPILER1
 39 #include "c1/c1_LIRAssembler.hpp"
 40 #include "c1/c1_MacroAssembler.hpp"
 41 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 42 #endif
 43 
 44 #define __ masm->
 45 
 46 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 47                                                             Register addr, Register count) {
 48   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 49 
 50   if (!dest_uninitialized) {
 51     Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 52 #ifndef _LP64
 53     __ push(thread);
 54     __ get_thread(thread);
 55 #endif
 56 
 57     Label filtered;
 58     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 59     // Is marking active?
 60     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 61       __ cmpl(in_progress, 0);
 62     } else {
 63       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 64       __ cmpb(in_progress, 0);
 65     }
 66 
 67     NOT_LP64(__ pop(thread);)
 68 
 69     __ jcc(Assembler::equal, filtered);
 70 
 71     __ push_call_clobbered_registers(false /* save_fpu */);
 72 #ifdef _LP64
 73     if (count == c_rarg0) {
 74       if (addr == c_rarg1) {
 75         // exactly backwards!!
 76         __ xchgptr(c_rarg1, c_rarg0);
 77       } else {
 78         __ movptr(c_rarg1, count);
 79         __ movptr(c_rarg0, addr);
 80       }
 81     } else {
 82       __ movptr(c_rarg0, addr);
 83       __ movptr(c_rarg1, count);
 84     }
 85     if (UseCompressedOops) {
 86       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 87     } else {
 88       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 89     }
 90 #else
 91     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
 92                     addr, count);
 93 #endif
 94     __ pop_call_clobbered_registers(false /* save_fpu */);
 95 
 96     __ bind(filtered);
 97   }
 98 }
 99 
100 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
101                                                              Register addr, Register count, Register tmp) {
102   __ push_call_clobbered_registers(false /* save_fpu */);
103 #ifdef _LP64
104   if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
105     assert_different_registers(c_rarg1, addr);
106     __ mov(c_rarg1, count);
107     __ mov(c_rarg0, addr);
108   } else {
109     assert_different_registers(c_rarg0, count);
110     __ mov(c_rarg0, addr);
111     __ mov(c_rarg1, count);
112   }
113   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
114 #else
115   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
116                   addr, count);
117 #endif
118   __ pop_call_clobbered_registers(false /* save_fpu */);
119 }
120 
121 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
122                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
123   bool on_oop = is_reference_type(type);
124   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
125   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
126   bool on_reference = on_weak || on_phantom;
127   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
128   if (on_oop && on_reference) {
129     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
130 
131 #ifndef _LP64
132     // Work around the x86_32 bug that only manifests with Loom for some reason.
133     // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg.
134     if (thread == noreg) {
135       if (dst != rcx && tmp1 != rcx) {
136         thread = rcx;
137       } else if (dst != rdx && tmp1 != rdx) {
138         thread = rdx;
139       } else if (dst != rdi && tmp1 != rdi) {
140         thread = rdi;
141       }
142     }
143     assert_different_registers(dst, tmp1, thread);
144     __ push(thread);
145     __ get_thread(thread);
146 #endif
147 
148     // Generate the G1 pre-barrier code to log the value of
149     // the referent field in an SATB buffer.
150     g1_write_barrier_pre(masm /* masm */,
151                          noreg /* obj */,
152                          dst /* pre_val */,
153                          thread /* thread */,
154                          tmp1 /* tmp */,
155                          true /* tosca_live */,
156                          true /* expand_call */);
157 
158 #ifndef _LP64
159     __ pop(thread);
160 #endif
161   }
162 }
163 
164 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
165                                                  Register obj,
166                                                  Register pre_val,
167                                                  Register thread,
168                                                  Register tmp,
169                                                  bool tosca_live,
170                                                  bool expand_call) {
171   // If expand_call is true then we expand the call_VM_leaf macro
172   // directly to skip generating the check by
173   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
174 
175 #ifdef _LP64
176   assert(thread == r15_thread, "must be");
177 #endif // _LP64
178 
179   Label done;
180   Label runtime;
181 
182   assert(pre_val != noreg, "check this code");
183 
184   if (obj != noreg) {
185     assert_different_registers(obj, pre_val, tmp);
186     assert(pre_val != rax, "check this code");
187   }
188 
189   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
190   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
191   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
192 
193   // Is marking active?
194   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
195     __ cmpl(in_progress, 0);
196   } else {
197     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
198     __ cmpb(in_progress, 0);
199   }
200   __ jcc(Assembler::equal, done);
201 
202   // Do we need to load the previous value?
203   if (obj != noreg) {
204     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
205   }
206 
207   // Is the previous value null?
208   __ cmpptr(pre_val, NULL_WORD);
209   __ jcc(Assembler::equal, done);
210 
211   // Can we store original value in the thread's buffer?
212   // Is index == 0?
213   // (The index field is typed as size_t.)
214 
215   __ movptr(tmp, index);                   // tmp := *index_adr
216   __ cmpptr(tmp, 0);                       // tmp == 0?
217   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
218 
219   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
220   __ movptr(index, tmp);                   // *index_adr := tmp
221   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
222 
223   // Record the previous value
224   __ movptr(Address(tmp, 0), pre_val);
225   __ jmp(done);
226 
227   __ bind(runtime);
228 
229   // Determine and save the live input values
230   __ push_call_clobbered_registers();
231 
232   // Calling the runtime using the regular call_VM_leaf mechanism generates
233   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
234   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
235   //
236   // If we care generating the pre-barrier without a frame (e.g. in the
237   // intrinsified Reference.get() routine) then ebp might be pointing to
238   // the caller frame and so this check will most likely fail at runtime.
239   //
240   // Expanding the call directly bypasses the generation of the check.
241   // So when we do not have have a full interpreter frame on the stack
242   // expand_call should be passed true.
243 
244   if (expand_call) {
245     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
246 #ifdef _LP64
247     if (c_rarg1 != thread) {
248       __ mov(c_rarg1, thread);
249     }
250     if (c_rarg0 != pre_val) {
251       __ mov(c_rarg0, pre_val);
252     }
253 #else
254     __ push(thread);
255     __ push(pre_val);
256 #endif
257     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
258   } else {
259     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
260   }
261 
262   __ pop_call_clobbered_registers();
263 
264   __ bind(done);
265 }
266 
267 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
268                                                   Register store_addr,
269                                                   Register new_val,
270                                                   Register thread,
271                                                   Register tmp,
272                                                   Register tmp2) {
273   // Generated code assumes that buffer index is pointer sized.
274   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
275 #ifdef _LP64
276   assert(thread == r15_thread, "must be");
277 #endif // _LP64
278 
279   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
280   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
281 
282   CardTableBarrierSet* ct =
283     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
284 
285   Label done;
286   Label runtime;
287 
288   // Does store cross heap regions?
289 
290   __ movptr(tmp, store_addr);
291   __ xorptr(tmp, new_val);
292   __ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
293   __ jcc(Assembler::equal, done);
294 
295   // crosses regions, storing null?
296 
297   __ cmpptr(new_val, NULL_WORD);
298   __ jcc(Assembler::equal, done);
299 
300   // storing region crossing non-null, is card already dirty?
301 
302   const Register card_addr = tmp;
303   const Register cardtable = tmp2;
304 
305   __ movptr(card_addr, store_addr);
306   __ shrptr(card_addr, CardTable::card_shift());
307   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
308   // a valid address and therefore is not properly handled by the relocation code.
309   if (SCCache::is_on_for_write()) {
310     // SCA needs relocation info for this address
311     __ lea(cardtable, ExternalAddress((address)ct->card_table()->byte_map_base()));
312   } else {
313     __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
314   }
315   __ addptr(card_addr, cardtable);
316 
317   __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
318   __ jcc(Assembler::equal, done);
319 
320   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
321   __ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
322   __ jcc(Assembler::equal, done);
323 
324 
325   // storing a region crossing, non-null oop, card is clean.
326   // dirty card and log.
327 
328   __ movb(Address(card_addr, 0), G1CardTable::dirty_card_val());
329 
330   __ movptr(tmp2, queue_index);
331   __ testptr(tmp2, tmp2);
332   __ jcc(Assembler::zero, runtime);
333   __ subptr(tmp2, wordSize);
334   __ movptr(queue_index, tmp2);
335   __ addptr(tmp2, buffer);
336   __ movptr(Address(tmp2, 0), card_addr);
337   __ jmp(done);
338 
339   __ bind(runtime);
340   // save the live input values
341   RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
342   __ push_set(saved);
343   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
344   __ pop_set(saved);
345 
346   __ bind(done);
347 }
348 
349 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
350                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
351   bool in_heap = (decorators & IN_HEAP) != 0;
352   bool as_normal = (decorators & AS_NORMAL) != 0;
353 
354   bool needs_pre_barrier = as_normal;
355   bool needs_post_barrier = val != noreg && in_heap;
356 
357   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
358   // flatten object address if needed
359   // We do it regardless of precise because we need the registers
360   if (dst.index() == noreg && dst.disp() == 0) {
361     if (dst.base() != tmp1) {
362       __ movptr(tmp1, dst.base());
363     }
364   } else {
365     __ lea(tmp1, dst);
366   }
367 
368 #ifndef _LP64
369   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
370 #endif
371 
372   NOT_LP64(__ get_thread(rcx));
373   NOT_LP64(imasm->save_bcp());
374 
375   if (needs_pre_barrier) {
376     g1_write_barrier_pre(masm /*masm*/,
377                          tmp1 /* obj */,
378                          tmp2 /* pre_val */,
379                          rthread /* thread */,
380                          tmp3  /* tmp */,
381                          val != noreg /* tosca_live */,
382                          false /* expand_call */);
383   }
384   if (val == noreg) {
385     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
386   } else {
387     Register new_val = val;
388     if (needs_post_barrier) {
389       // G1 barrier needs uncompressed oop for region cross check.
390       if (UseCompressedOops) {
391         new_val = tmp2;
392         __ movptr(new_val, val);
393       }
394     }
395     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
396     if (needs_post_barrier) {
397       g1_write_barrier_post(masm /*masm*/,
398                             tmp1 /* store_adr */,
399                             new_val /* new_val */,
400                             rthread /* thread */,
401                             tmp3 /* tmp */,
402                             tmp2 /* tmp2 */);
403     }
404   }
405   NOT_LP64(imasm->restore_bcp());
406 }
407 
408 #ifdef COMPILER1
409 
410 #undef __
411 #define __ ce->masm()->
412 
413 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
414   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
415   // At this point we know that marking is in progress.
416   // If do_load() is true then we have to emit the
417   // load of the previous value; otherwise it has already
418   // been loaded into _pre_val.
419 
420   __ bind(*stub->entry());
421   assert(stub->pre_val()->is_register(), "Precondition.");
422 
423   Register pre_val_reg = stub->pre_val()->as_register();
424 
425   if (stub->do_load()) {
426     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
427   }
428 
429   __ cmpptr(pre_val_reg, NULL_WORD);
430   __ jcc(Assembler::equal, *stub->continuation());
431   ce->store_parameter(stub->pre_val()->as_register(), 0);
432   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
433   __ jmp(*stub->continuation());
434 
435 }
436 
437 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
438   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
439   __ bind(*stub->entry());
440   assert(stub->addr()->is_register(), "Precondition.");
441   assert(stub->new_val()->is_register(), "Precondition.");
442   Register new_val_reg = stub->new_val()->as_register();
443   __ cmpptr(new_val_reg, NULL_WORD);
444   __ jcc(Assembler::equal, *stub->continuation());
445   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
446   __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
447   __ jmp(*stub->continuation());
448 }
449 
450 #undef __
451 
452 #define __ sasm->
453 
454 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
455   // Generated code assumes that buffer index is pointer sized.
456   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
457 
458   __ prologue("g1_pre_barrier", false);
459   // arg0 : previous value of memory
460 
461   __ push(rax);
462   __ push(rdx);
463 
464   const Register pre_val = rax;
465   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
466   const Register tmp = rdx;
467 
468   NOT_LP64(__ get_thread(thread);)
469 
470   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
471   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
472   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
473 
474   Label done;
475   Label runtime;
476 
477   // Is marking still active?
478   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
479     __ cmpl(queue_active, 0);
480   } else {
481     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
482     __ cmpb(queue_active, 0);
483   }
484   __ jcc(Assembler::equal, done);
485 
486   // Can we store original value in the thread's buffer?
487 
488   __ movptr(tmp, queue_index);
489   __ testptr(tmp, tmp);
490   __ jcc(Assembler::zero, runtime);
491   __ subptr(tmp, wordSize);
492   __ movptr(queue_index, tmp);
493   __ addptr(tmp, buffer);
494 
495   // prev_val (rax)
496   __ load_parameter(0, pre_val);
497   __ movptr(Address(tmp, 0), pre_val);
498   __ jmp(done);
499 
500   __ bind(runtime);
501 
502   __ push_call_clobbered_registers();
503 
504   // load the pre-value
505   __ load_parameter(0, rcx);
506   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
507 
508   __ pop_call_clobbered_registers();
509 
510   __ bind(done);
511 
512   __ pop(rdx);
513   __ pop(rax);
514 
515   __ epilogue();
516 }
517 
518 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
519   __ prologue("g1_post_barrier", false);
520 
521   CardTableBarrierSet* ct =
522     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
523 
524   Label done;
525   Label enqueued;
526   Label runtime;
527 
528   // At this point we know new_value is non-null and the new_value crosses regions.
529   // Must check to see if card is already dirty
530 
531   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
532 
533   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
534   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
535 
536   __ push(rax);
537   __ push(rcx);
538 
539   const Register cardtable = rax;
540   const Register card_addr = rcx;
541 
542   __ load_parameter(0, card_addr);
543   __ shrptr(card_addr, CardTable::card_shift());
544   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
545   // a valid address and therefore is not properly handled by the relocation code.
546   if (SCCache::is_on()) {
547     // SCA needs relocation info for this address
548     __ lea(cardtable, ExternalAddress((address)ct->card_table()->byte_map_base()));
549   } else {
550     __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
551   }
552   __ addptr(card_addr, cardtable);
553 
554   NOT_LP64(__ get_thread(thread);)
555 
556   __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
557   __ jcc(Assembler::equal, done);
558 
559   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
560   __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
561   __ jcc(Assembler::equal, done);
562 
563   // storing region crossing non-null, card is clean.
564   // dirty card and log.
565 
566   __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
567 
568   const Register tmp = rdx;
569   __ push(rdx);
570 
571   __ movptr(tmp, queue_index);
572   __ testptr(tmp, tmp);
573   __ jcc(Assembler::zero, runtime);
574   __ subptr(tmp, wordSize);
575   __ movptr(queue_index, tmp);
576   __ addptr(tmp, buffer);
577   __ movptr(Address(tmp, 0), card_addr);
578   __ jmp(enqueued);
579 
580   __ bind(runtime);
581   __ push_call_clobbered_registers();
582 
583   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
584 
585   __ pop_call_clobbered_registers();
586 
587   __ bind(enqueued);
588   __ pop(rdx);
589 
590   __ bind(done);
591   __ pop(rcx);
592   __ pop(rax);
593 
594   __ epilogue();
595 }
596 
597 #undef __
598 
599 #endif // COMPILER1