1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "gc/g1/g1BarrierSet.hpp"
27 #include "gc/g1/g1BarrierSetAssembler.hpp"
28 #include "gc/g1/g1BarrierSetRuntime.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1HeapRegion.hpp"
31 #include "gc/g1/g1ThreadLocalData.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "utilities/debug.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
40 #endif // COMPILER1
41 #ifdef COMPILER2
42 #include "gc/g1/c2/g1BarrierSetC2.hpp"
43 #endif // COMPILER2
44
45 #define __ masm->
46
47 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
48 Register addr, Register count) {
49 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50
51 if (!dest_uninitialized) {
52 Register thread = r15_thread;
53
54 Label filtered;
55 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
56 // Is marking active?
57 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
58 __ cmpl(in_progress, 0);
59 } else {
60 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
61 __ cmpb(in_progress, 0);
62 }
63
64 __ jcc(Assembler::equal, filtered);
65
66 __ push_call_clobbered_registers(false /* save_fpu */);
67 if (count == c_rarg0) {
68 if (addr == c_rarg1) {
69 // exactly backwards!!
70 __ xchgptr(c_rarg1, c_rarg0);
71 } else {
72 __ movptr(c_rarg1, count);
73 __ movptr(c_rarg0, addr);
74 }
75 } else {
76 __ movptr(c_rarg0, addr);
77 __ movptr(c_rarg1, count);
78 }
79 if (UseCompressedOops) {
80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
81 } else {
82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
83 }
84 __ pop_call_clobbered_registers(false /* save_fpu */);
85
86 __ bind(filtered);
87 }
88 }
89
90 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
91 Register addr, Register count, Register tmp) {
92 __ push_call_clobbered_registers(false /* save_fpu */);
93 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
94 assert_different_registers(c_rarg1, addr);
95 __ mov(c_rarg1, count);
96 __ mov(c_rarg0, addr);
97 } else {
98 assert_different_registers(c_rarg0, count);
99 __ mov(c_rarg0, addr);
100 __ mov(c_rarg1, count);
101 }
102 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
103 __ pop_call_clobbered_registers(false /* save_fpu */);
104
105 }
106
107 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
108 Register dst, Address src, Register tmp1) {
109 bool on_oop = is_reference_type(type);
110 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
111 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
112 bool on_reference = on_weak || on_phantom;
113 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
114 if (on_oop && on_reference) {
115 // Generate the G1 pre-barrier code to log the value of
116 // the referent field in an SATB buffer.
117 g1_write_barrier_pre(masm /* masm */,
118 noreg /* obj */,
119 dst /* pre_val */,
120 tmp1 /* tmp */,
121 true /* tosca_live */,
122 true /* expand_call */);
123 }
124 }
125
126 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
127 const Register thread, const Register value, const Register temp) {
128 // This code assumes that buffer index is pointer sized.
129 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
130 // Can we store a value in the given thread's buffer?
131 // (The index field is typed as size_t.)
132 __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address)
133 __ testptr(temp, temp); // index == 0?
134 __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer)
135 // The buffer is not full, store value into it.
136 __ subptr(temp, wordSize); // temp := next index
137 __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index
138 __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index
139 __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value
140 }
141
142 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
143 const Register thread) {
144 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
145 // Is marking active?
146 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
147 __ cmpl(in_progress, 0);
148 } else {
149 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
150 __ cmpb(in_progress, 0);
151 }
152 }
153
154 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
155 const Register obj,
156 const Register pre_val,
157 const Register thread,
158 const Register tmp,
159 Label& done,
160 Label& runtime) {
161 // Do we need to load the previous value?
162 if (obj != noreg) {
163 __ load_heap_oop(pre_val, Address(obj, 0), noreg, AS_RAW);
164 }
165 // Is the previous value null?
166 __ cmpptr(pre_val, NULL_WORD);
167 __ jcc(Assembler::equal, done);
168 generate_queue_insertion(masm,
169 G1ThreadLocalData::satb_mark_queue_index_offset(),
170 G1ThreadLocalData::satb_mark_queue_buffer_offset(),
171 runtime,
172 thread, pre_val, tmp);
173 __ jmp(done);
174 }
175
176 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
177 Register obj,
178 Register pre_val,
179 Register tmp,
180 bool tosca_live,
181 bool expand_call) {
182 // If expand_call is true then we expand the call_VM_leaf macro
183 // directly to skip generating the check by
184 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
185
186 const Register thread = r15_thread;
187
188 Label done;
189 Label runtime;
190
191 assert(pre_val != noreg, "check this code");
192
193 if (obj != noreg) {
194 assert_different_registers(obj, pre_val, tmp);
195 assert(pre_val != rax, "check this code");
196 }
197
198 generate_pre_barrier_fast_path(masm, thread);
199 // If marking is not active (*(mark queue active address) == 0), jump to done
200 __ jcc(Assembler::equal, done);
201 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
202
203 __ bind(runtime);
204
205 // Determine and save the live input values
206 __ push_call_clobbered_registers();
207
208 // Calling the runtime using the regular call_VM_leaf mechanism generates
209 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
210 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
211 //
212 // If we care generating the pre-barrier without a frame (e.g. in the
213 // intrinsified Reference.get() routine) then ebp might be pointing to
214 // the caller frame and so this check will most likely fail at runtime.
215 //
216 // Expanding the call directly bypasses the generation of the check.
217 // So when we do not have have a full interpreter frame on the stack
218 // expand_call should be passed true.
219
220 if (expand_call) {
221 assert(pre_val != c_rarg1, "smashed arg");
222 if (c_rarg1 != thread) {
223 __ mov(c_rarg1, thread);
224 }
225 if (c_rarg0 != pre_val) {
226 __ mov(c_rarg0, pre_val);
227 }
228 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
229 } else {
230 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
231 }
232
233 __ pop_call_clobbered_registers();
234
235 __ bind(done);
236 }
237
238 static void generate_post_barrier_fast_path(MacroAssembler* masm,
239 const Register store_addr,
240 const Register new_val,
241 const Register tmp,
242 const Register tmp2,
243 Label& done,
244 bool new_val_may_be_null) {
245 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
246 // Does store cross heap regions?
247 __ movptr(tmp, store_addr); // tmp := store address
248 __ xorptr(tmp, new_val); // tmp := store address ^ new value
249 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
250 __ jcc(Assembler::equal, done);
251 // Crosses regions, storing null?
252 if (new_val_may_be_null) {
253 __ cmpptr(new_val, NULL_WORD); // new value == null?
254 __ jcc(Assembler::equal, done);
255 }
256 // Storing region crossing non-null, is card young?
257 __ movptr(tmp, store_addr); // tmp := store address
258 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
259 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
260 // a valid address and therefore is not properly handled by the relocation code.
261 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
262 __ addptr(tmp, tmp2); // tmp := card address
263 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
264 }
265
266 static void generate_post_barrier_slow_path(MacroAssembler* masm,
267 const Register thread,
268 const Register tmp,
269 const Register tmp2,
270 Label& done,
271 Label& runtime) {
272 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
273 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
274 __ jcc(Assembler::equal, done);
275 // Storing a region crossing, non-null oop, card is clean.
276 // Dirty card and log.
277 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
278 generate_queue_insertion(masm,
279 G1ThreadLocalData::dirty_card_queue_index_offset(),
280 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
281 runtime,
282 thread, tmp, tmp2);
283 __ jmp(done);
284 }
285
286 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
287 Register store_addr,
288 Register new_val,
289 Register tmp,
290 Register tmp2) {
291 const Register thread = r15_thread;
292
293 Label done;
294 Label runtime;
295
296 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
297 // If card is young, jump to done
298 __ jcc(Assembler::equal, done);
299 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
300
301 __ bind(runtime);
302 // save the live input values
303 RegSet saved = RegSet::of(store_addr);
304 __ push_set(saved);
305 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
306 __ pop_set(saved);
307
308 __ bind(done);
309 }
310
311 #if defined(COMPILER2)
312
313 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
314 SaveLiveRegisters save_registers(masm, stub);
315 if (c_rarg0 != arg) {
316 __ mov(c_rarg0, arg);
317 }
318 __ mov(c_rarg1, r15_thread);
319 // rax is a caller-saved, non-argument-passing register, so it does not
320 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
321 // entering this stub, it is saved at this point, and restored after the
322 // call. If it did not contain any live value, it is free to be used. In
323 // either case, it is safe to use it here as a call scratch register.
324 __ call(RuntimeAddress(runtime_path), rax);
325 }
326
327 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
328 Register obj,
329 Register pre_val,
330 Register tmp,
331 G1PreBarrierStubC2* stub) {
332 const Register thread = r15_thread;
333
334 assert(pre_val != noreg, "check this code");
335 if (obj != noreg) {
336 assert_different_registers(obj, pre_val, tmp);
337 }
338
339 stub->initialize_registers(obj, pre_val, thread, tmp);
340
341 generate_pre_barrier_fast_path(masm, thread);
342 // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
343 __ jcc(Assembler::notEqual, *stub->entry());
344
345 __ bind(*stub->continuation());
346 }
347
348 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
349 G1PreBarrierStubC2* stub) const {
350 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
351 Label runtime;
352 Register obj = stub->obj();
353 Register pre_val = stub->pre_val();
354 Register thread = stub->thread();
355 Register tmp = stub->tmp1();
356 assert(stub->tmp2() == noreg, "not needed in this platform");
357
358 __ bind(*stub->entry());
359 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
360
361 __ bind(runtime);
362 generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
363 __ jmp(*stub->continuation());
364 }
365
366 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
367 Register store_addr,
368 Register new_val,
369 Register tmp,
370 Register tmp2,
371 G1PostBarrierStubC2* stub) {
372 const Register thread = r15_thread;
373 stub->initialize_registers(thread, tmp, tmp2);
374
375 bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
376 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
377 // If card is not young, jump to stub (slow path)
378 __ jcc(Assembler::notEqual, *stub->entry());
379
380 __ bind(*stub->continuation());
381 }
382
383 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
384 G1PostBarrierStubC2* stub) const {
385 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
386 Label runtime;
387 Register thread = stub->thread();
388 Register tmp = stub->tmp1(); // tmp holds the card address.
389 Register tmp2 = stub->tmp2();
390 assert(stub->tmp3() == noreg, "not needed in this platform");
391
392 __ bind(*stub->entry());
393 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
394
395 __ bind(runtime);
396 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
397 __ jmp(*stub->continuation());
398 }
399
400 #endif // COMPILER2
401
402 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
403 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
404 bool in_heap = (decorators & IN_HEAP) != 0;
405 bool as_normal = (decorators & AS_NORMAL) != 0;
406
407 bool needs_pre_barrier = as_normal;
408 bool needs_post_barrier = val != noreg && in_heap;
409
410 // flatten object address if needed
411 // We do it regardless of precise because we need the registers
412 if (dst.index() == noreg && dst.disp() == 0) {
413 if (dst.base() != tmp1) {
414 __ movptr(tmp1, dst.base());
415 }
416 } else {
417 __ lea(tmp1, dst);
418 }
419
420 if (needs_pre_barrier) {
421 g1_write_barrier_pre(masm /*masm*/,
422 tmp1 /* obj */,
423 tmp2 /* pre_val */,
424 tmp3 /* tmp */,
425 val != noreg /* tosca_live */,
426 false /* expand_call */);
427 }
428 if (val == noreg) {
429 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
430 } else {
431 Register new_val = val;
432 if (needs_post_barrier) {
433 // G1 barrier needs uncompressed oop for region cross check.
434 if (UseCompressedOops) {
435 new_val = tmp2;
436 __ movptr(new_val, val);
437 }
438 }
439 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
440 if (needs_post_barrier) {
441 g1_write_barrier_post(masm /*masm*/,
442 tmp1 /* store_adr */,
443 new_val /* new_val */,
444 tmp3 /* tmp */,
445 tmp2 /* tmp2 */);
446 }
447 }
448 }
449
450 #ifdef COMPILER1
451
452 #undef __
453 #define __ ce->masm()->
454
455 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
456 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
457 // At this point we know that marking is in progress.
458 // If do_load() is true then we have to emit the
459 // load of the previous value; otherwise it has already
460 // been loaded into _pre_val.
461
462 __ bind(*stub->entry());
463 assert(stub->pre_val()->is_register(), "Precondition.");
464
465 Register pre_val_reg = stub->pre_val()->as_register();
466
467 if (stub->do_load()) {
468 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
469 }
470
471 __ cmpptr(pre_val_reg, NULL_WORD);
472 __ jcc(Assembler::equal, *stub->continuation());
473 ce->store_parameter(stub->pre_val()->as_register(), 0);
474 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
475 __ jmp(*stub->continuation());
476
477 }
478
479 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
480 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
481 __ bind(*stub->entry());
482 assert(stub->addr()->is_register(), "Precondition.");
483 assert(stub->new_val()->is_register(), "Precondition.");
484 Register new_val_reg = stub->new_val()->as_register();
485 __ cmpptr(new_val_reg, NULL_WORD);
486 __ jcc(Assembler::equal, *stub->continuation());
487 ce->store_parameter(stub->addr()->as_pointer_register(), 0);
488 __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
489 __ jmp(*stub->continuation());
490 }
491
492 #undef __
493
494 #define __ sasm->
495
496 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
497 // Generated code assumes that buffer index is pointer sized.
498 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
499
500 __ prologue("g1_pre_barrier", false);
501 // arg0 : previous value of memory
502
503 __ push(rax);
504 __ push(rdx);
505
506 const Register pre_val = rax;
507 const Register thread = r15_thread;
508 const Register tmp = rdx;
509
510 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
511 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
512 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
513
514 Label done;
515 Label runtime;
516
517 // Is marking still active?
518 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
519 __ cmpl(queue_active, 0);
520 } else {
521 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
522 __ cmpb(queue_active, 0);
523 }
524 __ jcc(Assembler::equal, done);
525
526 // Can we store original value in the thread's buffer?
527
528 __ movptr(tmp, queue_index);
529 __ testptr(tmp, tmp);
530 __ jcc(Assembler::zero, runtime);
531 __ subptr(tmp, wordSize);
532 __ movptr(queue_index, tmp);
533 __ addptr(tmp, buffer);
534
535 // prev_val (rax)
536 __ load_parameter(0, pre_val);
537 __ movptr(Address(tmp, 0), pre_val);
538 __ jmp(done);
539
540 __ bind(runtime);
541
542 __ push_call_clobbered_registers();
543
544 // load the pre-value
545 __ load_parameter(0, rcx);
546 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
547
548 __ pop_call_clobbered_registers();
549
550 __ bind(done);
551
552 __ pop(rdx);
553 __ pop(rax);
554
555 __ epilogue();
556 }
557
558 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
559 __ prologue("g1_post_barrier", false);
560
561 CardTableBarrierSet* ct =
562 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
563
564 Label done;
565 Label enqueued;
566 Label runtime;
567
568 // At this point we know new_value is non-null and the new_value crosses regions.
569 // Must check to see if card is already dirty
570
571 const Register thread = r15_thread;
572
573 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
574 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
575
576 __ push(rax);
577 __ push(rcx);
578
579 const Register cardtable = rax;
580 const Register card_addr = rcx;
581
582 __ load_parameter(0, card_addr);
583 __ shrptr(card_addr, CardTable::card_shift());
584 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
585 // a valid address and therefore is not properly handled by the relocation code.
586 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
587 __ addptr(card_addr, cardtable);
588
589 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
590 __ jcc(Assembler::equal, done);
591
592 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
593 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
594 __ jcc(Assembler::equal, done);
595
596 // storing region crossing non-null, card is clean.
597 // dirty card and log.
598
599 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
600
601 const Register tmp = rdx;
602 __ push(rdx);
603
604 __ movptr(tmp, queue_index);
605 __ testptr(tmp, tmp);
606 __ jcc(Assembler::zero, runtime);
607 __ subptr(tmp, wordSize);
608 __ movptr(queue_index, tmp);
609 __ addptr(tmp, buffer);
610 __ movptr(Address(tmp, 0), card_addr);
611 __ jmp(enqueued);
612
613 __ bind(runtime);
614 __ push_call_clobbered_registers();
615
616 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
617
618 __ pop_call_clobbered_registers();
619
620 __ bind(enqueued);
621 __ pop(rdx);
622
623 __ bind(done);
624 __ pop(rcx);
625 __ pop(rax);
626
627 __ epilogue();
628 }
629
630 #undef __
631
632 #endif // COMPILER1