1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #if INCLUDE_CDS
27 #include "code/SCCache.hpp"
28 #endif
29 #include "gc/g1/g1BarrierSet.hpp"
30 #include "gc/g1/g1BarrierSetAssembler.hpp"
31 #include "gc/g1/g1BarrierSetRuntime.hpp"
32 #include "gc/g1/g1CardTable.hpp"
33 #include "gc/g1/g1HeapRegion.hpp"
34 #include "gc/g1/g1ThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/debug.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/g1/c1/g1BarrierSetC1.hpp"
43 #endif // COMPILER1
44 #ifdef COMPILER2
45 #include "gc/g1/c2/g1BarrierSetC2.hpp"
46 #endif // COMPILER2
47
48 #define __ masm->
49
50 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
51 Register addr, Register count) {
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (!dest_uninitialized) {
55 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
56 #ifndef _LP64
57 __ push(thread);
58 __ get_thread(thread);
59 #endif
60
61 Label filtered;
62 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
63 // Is marking active?
64 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
65 __ cmpl(in_progress, 0);
66 } else {
67 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
68 __ cmpb(in_progress, 0);
69 }
70
71 NOT_LP64(__ pop(thread);)
72
73 __ jcc(Assembler::equal, filtered);
74
75 __ push_call_clobbered_registers(false /* save_fpu */);
76 #ifdef _LP64
77 if (count == c_rarg0) {
78 if (addr == c_rarg1) {
79 // exactly backwards!!
80 __ xchgptr(c_rarg1, c_rarg0);
81 } else {
82 __ movptr(c_rarg1, count);
83 __ movptr(c_rarg0, addr);
84 }
85 } else {
86 __ movptr(c_rarg0, addr);
87 __ movptr(c_rarg1, count);
88 }
89 if (UseCompressedOops) {
90 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
91 } else {
92 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
93 }
94 #else
95 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
96 addr, count);
97 #endif
98 __ pop_call_clobbered_registers(false /* save_fpu */);
99
100 __ bind(filtered);
101 }
102 }
103
104 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
105 Register addr, Register count, Register tmp) {
106 __ push_call_clobbered_registers(false /* save_fpu */);
107 #ifdef _LP64
108 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
109 assert_different_registers(c_rarg1, addr);
110 __ mov(c_rarg1, count);
111 __ mov(c_rarg0, addr);
112 } else {
113 assert_different_registers(c_rarg0, count);
114 __ mov(c_rarg0, addr);
115 __ mov(c_rarg1, count);
116 }
117 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
118 #else
119 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
120 addr, count);
121 #endif
122 __ pop_call_clobbered_registers(false /* save_fpu */);
123 }
124
125 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
126 Register dst, Address src, Register tmp1, Register tmp_thread) {
127 bool on_oop = is_reference_type(type);
128 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
129 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
130 bool on_reference = on_weak || on_phantom;
131 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
132 if (on_oop && on_reference) {
133 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
134
135 #ifndef _LP64
136 // Work around the x86_32 bug that only manifests with Loom for some reason.
137 // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg.
138 if (thread == noreg) {
139 if (dst != rcx && tmp1 != rcx) {
140 thread = rcx;
141 } else if (dst != rdx && tmp1 != rdx) {
142 thread = rdx;
143 } else if (dst != rdi && tmp1 != rdi) {
144 thread = rdi;
145 }
146 }
147 assert_different_registers(dst, tmp1, thread);
148 __ push(thread);
149 __ get_thread(thread);
150 #endif
151
152 // Generate the G1 pre-barrier code to log the value of
153 // the referent field in an SATB buffer.
154 g1_write_barrier_pre(masm /* masm */,
155 noreg /* obj */,
156 dst /* pre_val */,
157 thread /* thread */,
158 tmp1 /* tmp */,
159 true /* tosca_live */,
160 true /* expand_call */);
161
162 #ifndef _LP64
163 __ pop(thread);
164 #endif
165 }
166 }
167
168 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
169 const Register thread, const Register value, const Register temp) {
170 // This code assumes that buffer index is pointer sized.
171 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
172 // Can we store a value in the given thread's buffer?
173 // (The index field is typed as size_t.)
174 __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address)
175 __ testptr(temp, temp); // index == 0?
176 __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer)
177 // The buffer is not full, store value into it.
178 __ subptr(temp, wordSize); // temp := next index
179 __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index
180 __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index
181 __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value
182 }
183
184 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
185 const Register thread) {
186 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
187 // Is marking active?
188 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
189 __ cmpl(in_progress, 0);
190 } else {
191 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
192 __ cmpb(in_progress, 0);
193 }
194 }
195
196 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
197 const Register obj,
198 const Register pre_val,
199 const Register thread,
200 const Register tmp,
201 Label& done,
202 Label& runtime) {
203 // Do we need to load the previous value?
204 if (obj != noreg) {
205 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
206 }
207 // Is the previous value null?
208 __ cmpptr(pre_val, NULL_WORD);
209 __ jcc(Assembler::equal, done);
210 generate_queue_insertion(masm,
211 G1ThreadLocalData::satb_mark_queue_index_offset(),
212 G1ThreadLocalData::satb_mark_queue_buffer_offset(),
213 runtime,
214 thread, pre_val, tmp);
215 __ jmp(done);
216 }
217
218 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
219 Register obj,
220 Register pre_val,
221 Register thread,
222 Register tmp,
223 bool tosca_live,
224 bool expand_call) {
225 // If expand_call is true then we expand the call_VM_leaf macro
226 // directly to skip generating the check by
227 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
228
229 #ifdef _LP64
230 assert(thread == r15_thread, "must be");
231 #endif // _LP64
232
233 Label done;
234 Label runtime;
235
236 assert(pre_val != noreg, "check this code");
237
238 if (obj != noreg) {
239 assert_different_registers(obj, pre_val, tmp);
240 assert(pre_val != rax, "check this code");
241 }
242
243 generate_pre_barrier_fast_path(masm, thread);
244 // If marking is not active (*(mark queue active address) == 0), jump to done
245 __ jcc(Assembler::equal, done);
246 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
247
248 __ bind(runtime);
249
250 // Determine and save the live input values
251 __ push_call_clobbered_registers();
252
253 // Calling the runtime using the regular call_VM_leaf mechanism generates
254 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
255 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
256 //
257 // If we care generating the pre-barrier without a frame (e.g. in the
258 // intrinsified Reference.get() routine) then ebp might be pointing to
259 // the caller frame and so this check will most likely fail at runtime.
260 //
261 // Expanding the call directly bypasses the generation of the check.
262 // So when we do not have have a full interpreter frame on the stack
263 // expand_call should be passed true.
264
265 if (expand_call) {
266 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
267 #ifdef _LP64
268 if (c_rarg1 != thread) {
269 __ mov(c_rarg1, thread);
270 }
271 if (c_rarg0 != pre_val) {
272 __ mov(c_rarg0, pre_val);
273 }
274 #else
275 __ push(thread);
276 __ push(pre_val);
277 #endif
278 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
279 } else {
280 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
281 }
282
283 __ pop_call_clobbered_registers();
284
285 __ bind(done);
286 }
287
288 // return a register that differs from reg1, reg2, reg3 and is not rcx
289
290 static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
291 RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
292 RegSet::of(reg1, reg2, reg3, reg4));
293 return *(available.begin());
294 }
295
296 static void generate_post_barrier_fast_path(MacroAssembler* masm,
297 const Register store_addr,
298 const Register new_val,
299 const Register tmp,
300 const Register tmp2,
301 Label& done,
302 bool new_val_may_be_null) {
303 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
304 // Does store cross heap regions?
305 #if INCLUDE_CDS
306 // AOT code needs to load the barrier grain shift from the aot
307 // runtime constants area in the code cache otherwise we can compile
308 // it as an immediate operand
309
310 if (SCCache::is_on_for_write()) {
311 address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
312 Register save = pick_different_reg(rcx, tmp, new_val, store_addr);
313 __ push(save);
314 __ movptr(save, store_addr);
315 __ xorptr(save, new_val);
316 __ push(rcx);
317 __ lea(rcx, ExternalAddress(grain_shift_addr));
318 __ movptr(rcx, Address(rcx, 0));
319 __ shrptr(save);
320 __ pop(rcx);
321 __ mov(tmp, save);
322 __ pop(save);
323 __ jcc(Assembler::equal, done);
324 } else
325 #endif // INCLUDE_CDS
326 {
327 __ movptr(tmp, store_addr); // tmp := store address
328 __ xorptr(tmp, new_val); // tmp := store address ^ new value
329 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
330 __ jcc(Assembler::equal, done);
331 }
332
333 // Crosses regions, storing null?
334 if (new_val_may_be_null) {
335 __ cmpptr(new_val, NULL_WORD); // new value == null?
336 __ jcc(Assembler::equal, done);
337 }
338 // Storing region crossing non-null, is card young?
339 __ movptr(tmp, store_addr); // tmp := store address
340 #if INCLUDE_CDS
341 // AOT code needs to load the barrier card shift from the aot
342 // runtime constants area in the code cache otherwise we can compile
343 // it as an immediate operand
344 if (SCCache::is_on_for_write()) {
345 address card_shift_addr = AOTRuntimeConstants::card_shift_address();
346 Register save = pick_different_reg(rcx, tmp);
347 __ push(save);
348 __ mov(save, tmp);
349 __ push(rcx);
350 __ lea(rcx, ExternalAddress(card_shift_addr));
351 __ movptr(rcx, Address(rcx, 0));
352 __ shrptr(save);
353 __ pop(rcx);
354 __ mov(tmp, save);
355 __ pop(save);
356 } else
357 #endif // INCLUDE_CDS
358 {
359 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
360 }
361 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
362 // a valid address and therefore is not properly handled by the relocation code.
363 if (SCCache::is_on_for_write()) {
364 // SCA needs relocation info for this address
365 __ lea(tmp2, ExternalAddress((address)ct->card_table()->byte_map_base())); // tmp2 := card table base address
366 } else {
367 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
368 }
369 __ addptr(tmp, tmp2); // tmp := card address
370 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
371 }
372
373 static void generate_post_barrier_slow_path(MacroAssembler* masm,
374 const Register thread,
375 const Register tmp,
376 const Register tmp2,
377 Label& done,
378 Label& runtime) {
379 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
380 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
381 __ jcc(Assembler::equal, done);
382 // Storing a region crossing, non-null oop, card is clean.
383 // Dirty card and log.
384 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
385 generate_queue_insertion(masm,
386 G1ThreadLocalData::dirty_card_queue_index_offset(),
387 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
388 runtime,
389 thread, tmp, tmp2);
390 __ jmp(done);
391 }
392
393 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
394 Register store_addr,
395 Register new_val,
396 Register thread,
397 Register tmp,
398 Register tmp2) {
399 #ifdef _LP64
400 assert(thread == r15_thread, "must be");
401 #endif // _LP64
402
403 Label done;
404 Label runtime;
405
406 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
407 // If card is young, jump to done
408 __ jcc(Assembler::equal, done);
409 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
410
411 __ bind(runtime);
412 // save the live input values
413 RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
414 __ push_set(saved);
415 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
416 __ pop_set(saved);
417
418 __ bind(done);
419 }
420
421 #if defined(COMPILER2)
422
423 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
424 #ifdef _LP64
425 SaveLiveRegisters save_registers(masm, stub);
426 if (c_rarg0 != arg) {
427 __ mov(c_rarg0, arg);
428 }
429 __ mov(c_rarg1, r15_thread);
430 // rax is a caller-saved, non-argument-passing register, so it does not
431 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
432 // entering this stub, it is saved at this point, and restored after the
433 // call. If it did not contain any live value, it is free to be used. In
434 // either case, it is safe to use it here as a call scratch register.
435 __ call(RuntimeAddress(runtime_path), rax);
436 #else
437 Unimplemented();
438 #endif // _LP64
439 }
440
441 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
442 Register obj,
443 Register pre_val,
444 Register thread,
445 Register tmp,
446 G1PreBarrierStubC2* stub) {
447 #ifdef _LP64
448 assert(thread == r15_thread, "must be");
449 #endif // _LP64
450 assert(pre_val != noreg, "check this code");
451 if (obj != noreg) {
452 assert_different_registers(obj, pre_val, tmp);
453 }
454
455 stub->initialize_registers(obj, pre_val, thread, tmp);
456
457 generate_pre_barrier_fast_path(masm, thread);
458 // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
459 __ jcc(Assembler::notEqual, *stub->entry());
460
461 __ bind(*stub->continuation());
462 }
463
464 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
465 G1PreBarrierStubC2* stub) const {
466 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
467 Label runtime;
468 Register obj = stub->obj();
469 Register pre_val = stub->pre_val();
470 Register thread = stub->thread();
471 Register tmp = stub->tmp1();
472 assert(stub->tmp2() == noreg, "not needed in this platform");
473
474 __ bind(*stub->entry());
475 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
476
477 __ bind(runtime);
478 generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
479 __ jmp(*stub->continuation());
480 }
481
482 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
483 Register store_addr,
484 Register new_val,
485 Register thread,
486 Register tmp,
487 Register tmp2,
488 G1PostBarrierStubC2* stub) {
489 #ifdef _LP64
490 assert(thread == r15_thread, "must be");
491 #endif // _LP64
492
493 stub->initialize_registers(thread, tmp, tmp2);
494
495 bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
496 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
497 // If card is not young, jump to stub (slow path)
498 __ jcc(Assembler::notEqual, *stub->entry());
499
500 __ bind(*stub->continuation());
501 }
502
503 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
504 G1PostBarrierStubC2* stub) const {
505 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
506 Label runtime;
507 Register thread = stub->thread();
508 Register tmp = stub->tmp1(); // tmp holds the card address.
509 Register tmp2 = stub->tmp2();
510 assert(stub->tmp3() == noreg, "not needed in this platform");
511
512 __ bind(*stub->entry());
513 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
514
515 __ bind(runtime);
516 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
517 __ jmp(*stub->continuation());
518 }
519
520 #endif // COMPILER2
521
522 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
523 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
524 bool in_heap = (decorators & IN_HEAP) != 0;
525 bool as_normal = (decorators & AS_NORMAL) != 0;
526
527 bool needs_pre_barrier = as_normal;
528 bool needs_post_barrier = val != noreg && in_heap;
529
530 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
531 // flatten object address if needed
532 // We do it regardless of precise because we need the registers
533 if (dst.index() == noreg && dst.disp() == 0) {
534 if (dst.base() != tmp1) {
535 __ movptr(tmp1, dst.base());
536 }
537 } else {
538 __ lea(tmp1, dst);
539 }
540
541 #ifndef _LP64
542 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
543 #endif
544
545 NOT_LP64(__ get_thread(rcx));
546 NOT_LP64(imasm->save_bcp());
547
548 if (needs_pre_barrier) {
549 g1_write_barrier_pre(masm /*masm*/,
550 tmp1 /* obj */,
551 tmp2 /* pre_val */,
552 rthread /* thread */,
553 tmp3 /* tmp */,
554 val != noreg /* tosca_live */,
555 false /* expand_call */);
556 }
557 if (val == noreg) {
558 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
559 } else {
560 Register new_val = val;
561 if (needs_post_barrier) {
562 // G1 barrier needs uncompressed oop for region cross check.
563 if (UseCompressedOops) {
564 new_val = tmp2;
565 __ movptr(new_val, val);
566 }
567 }
568 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
569 if (needs_post_barrier) {
570 g1_write_barrier_post(masm /*masm*/,
571 tmp1 /* store_adr */,
572 new_val /* new_val */,
573 rthread /* thread */,
574 tmp3 /* tmp */,
575 tmp2 /* tmp2 */);
576 }
577 }
578 NOT_LP64(imasm->restore_bcp());
579 }
580
581 #ifdef COMPILER1
582
583 #undef __
584 #define __ ce->masm()->
585
586 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
587 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
588 // At this point we know that marking is in progress.
589 // If do_load() is true then we have to emit the
590 // load of the previous value; otherwise it has already
591 // been loaded into _pre_val.
592
593 __ bind(*stub->entry());
594 assert(stub->pre_val()->is_register(), "Precondition.");
595
596 Register pre_val_reg = stub->pre_val()->as_register();
597
598 if (stub->do_load()) {
599 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
600 }
601
602 __ cmpptr(pre_val_reg, NULL_WORD);
603 __ jcc(Assembler::equal, *stub->continuation());
604 ce->store_parameter(stub->pre_val()->as_register(), 0);
605 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
606 __ jmp(*stub->continuation());
607
608 }
609
610 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
611 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
612 __ bind(*stub->entry());
613 assert(stub->addr()->is_register(), "Precondition.");
614 assert(stub->new_val()->is_register(), "Precondition.");
615 Register new_val_reg = stub->new_val()->as_register();
616 __ cmpptr(new_val_reg, NULL_WORD);
617 __ jcc(Assembler::equal, *stub->continuation());
618 ce->store_parameter(stub->addr()->as_pointer_register(), 0);
619 __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
620 __ jmp(*stub->continuation());
621 }
622
623 #undef __
624
625 #define __ sasm->
626
627 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
628 // Generated code assumes that buffer index is pointer sized.
629 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
630
631 __ prologue("g1_pre_barrier", false);
632 // arg0 : previous value of memory
633
634 __ push(rax);
635 __ push(rdx);
636
637 const Register pre_val = rax;
638 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
639 const Register tmp = rdx;
640
641 NOT_LP64(__ get_thread(thread);)
642
643 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
644 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
645 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
646
647 Label done;
648 Label runtime;
649
650 // Is marking still active?
651 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
652 __ cmpl(queue_active, 0);
653 } else {
654 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
655 __ cmpb(queue_active, 0);
656 }
657 __ jcc(Assembler::equal, done);
658
659 // Can we store original value in the thread's buffer?
660
661 __ movptr(tmp, queue_index);
662 __ testptr(tmp, tmp);
663 __ jcc(Assembler::zero, runtime);
664 __ subptr(tmp, wordSize);
665 __ movptr(queue_index, tmp);
666 __ addptr(tmp, buffer);
667
668 // prev_val (rax)
669 __ load_parameter(0, pre_val);
670 __ movptr(Address(tmp, 0), pre_val);
671 __ jmp(done);
672
673 __ bind(runtime);
674
675 __ push_call_clobbered_registers();
676
677 // load the pre-value
678 __ load_parameter(0, rcx);
679 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
680
681 __ pop_call_clobbered_registers();
682
683 __ bind(done);
684
685 __ pop(rdx);
686 __ pop(rax);
687
688 __ epilogue();
689 }
690
691 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
692 __ prologue("g1_post_barrier", false);
693
694 CardTableBarrierSet* ct =
695 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
696
697 Label done;
698 Label enqueued;
699 Label runtime;
700
701 // At this point we know new_value is non-null and the new_value crosses regions.
702 // Must check to see if card is already dirty
703
704 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
705
706 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
707 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
708
709 __ push(rax);
710 __ push(rcx);
711
712 const Register cardtable = rax;
713 const Register card_addr = rcx;
714
715 __ load_parameter(0, card_addr);
716 __ shrptr(card_addr, CardTable::card_shift());
717 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
718 // a valid address and therefore is not properly handled by the relocation code.
719 if (SCCache::is_on()) {
720 // SCA needs relocation info for this address
721 __ lea(cardtable, ExternalAddress((address)ct->card_table()->byte_map_base()));
722 } else {
723 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
724 }
725 __ addptr(card_addr, cardtable);
726
727 NOT_LP64(__ get_thread(thread);)
728
729 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
730 __ jcc(Assembler::equal, done);
731
732 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
733 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
734 __ jcc(Assembler::equal, done);
735
736 // storing region crossing non-null, card is clean.
737 // dirty card and log.
738
739 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
740
741 const Register tmp = rdx;
742 __ push(rdx);
743
744 __ movptr(tmp, queue_index);
745 __ testptr(tmp, tmp);
746 __ jcc(Assembler::zero, runtime);
747 __ subptr(tmp, wordSize);
748 __ movptr(queue_index, tmp);
749 __ addptr(tmp, buffer);
750 __ movptr(Address(tmp, 0), card_addr);
751 __ jmp(enqueued);
752
753 __ bind(runtime);
754 __ push_call_clobbered_registers();
755
756 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
757
758 __ pop_call_clobbered_registers();
759
760 __ bind(enqueued);
761 __ pop(rdx);
762
763 __ bind(done);
764 __ pop(rcx);
765 __ pop(rax);
766
767 __ epilogue();
768 }
769
770 #undef __
771
772 #endif // COMPILER1