1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "gc/g1/g1BarrierSet.hpp"
27 #include "gc/g1/g1BarrierSetAssembler.hpp"
28 #include "gc/g1/g1BarrierSetRuntime.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1HeapRegion.hpp"
31 #include "gc/g1/g1ThreadLocalData.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "utilities/debug.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
40 #endif // COMPILER1
41 #ifdef COMPILER2
42 #include "gc/g1/c2/g1BarrierSetC2.hpp"
43 #endif // COMPILER2
44
45 #define __ masm->
46
47 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
48 Register addr, Register count) {
49 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50
51 if (!dest_uninitialized) {
52 Register thread = r15_thread;
53
54 Label filtered;
55 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
56 // Is marking active?
57 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
58 __ cmpl(in_progress, 0);
59 } else {
60 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
61 __ cmpb(in_progress, 0);
62 }
63
64 __ jcc(Assembler::equal, filtered);
65
66 __ push_call_clobbered_registers(false /* save_fpu */);
67 if (count == c_rarg0) {
68 if (addr == c_rarg1) {
69 // exactly backwards!!
70 __ xchgptr(c_rarg1, c_rarg0);
71 } else {
72 __ movptr(c_rarg1, count);
73 __ movptr(c_rarg0, addr);
74 }
75 } else {
76 __ movptr(c_rarg0, addr);
77 __ movptr(c_rarg1, count);
78 }
79 if (UseCompressedOops) {
80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
81 } else {
82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
83 }
84 __ pop_call_clobbered_registers(false /* save_fpu */);
85
86 __ bind(filtered);
87 }
88 }
89
90 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
91 Register addr, Register count, Register tmp) {
92 __ push_call_clobbered_registers(false /* save_fpu */);
93 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
94 assert_different_registers(c_rarg1, addr);
95 __ mov(c_rarg1, count);
96 __ mov(c_rarg0, addr);
97 } else {
98 assert_different_registers(c_rarg0, count);
99 __ mov(c_rarg0, addr);
100 __ mov(c_rarg1, count);
101 }
102 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
103 __ pop_call_clobbered_registers(false /* save_fpu */);
104
105 }
106
107 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
108 Register dst, Address src, Register tmp1) {
109 bool on_oop = is_reference_type(type);
110 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
111 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
112 bool on_reference = on_weak || on_phantom;
113 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
114 if (on_oop && on_reference) {
115 // Generate the G1 pre-barrier code to log the value of
116 // the referent field in an SATB buffer.
117 g1_write_barrier_pre(masm /* masm */,
118 noreg /* obj */,
119 dst /* pre_val */,
120 tmp1 /* tmp */,
121 true /* tosca_live */,
122 true /* expand_call */);
123 }
124 }
125
126 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
127 const Register thread, const Register value, const Register temp) {
128 // This code assumes that buffer index is pointer sized.
129 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
130 // Can we store a value in the given thread's buffer?
131 // (The index field is typed as size_t.)
132 __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address)
133 __ testptr(temp, temp); // index == 0?
134 __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer)
135 // The buffer is not full, store value into it.
136 __ subptr(temp, wordSize); // temp := next index
137 __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index
138 __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index
139 __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value
140 }
141
142 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
143 const Register thread) {
144 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
145 // Is marking active?
146 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
147 __ cmpl(in_progress, 0);
148 } else {
149 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
150 __ cmpb(in_progress, 0);
151 }
152 }
153
154 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
155 const Register obj,
156 const Register pre_val,
157 const Register thread,
158 const Register tmp,
159 Label& done,
160 Label& runtime) {
161 // Do we need to load the previous value?
162 if (obj != noreg) {
163 __ load_heap_oop(pre_val, Address(obj, 0), noreg, AS_RAW);
164 }
165 // Is the previous value null?
166 __ cmpptr(pre_val, NULL_WORD);
167 __ jcc(Assembler::equal, done);
168 generate_queue_insertion(masm,
169 G1ThreadLocalData::satb_mark_queue_index_offset(),
170 G1ThreadLocalData::satb_mark_queue_buffer_offset(),
171 runtime,
172 thread, pre_val, tmp);
173 __ jmp(done);
174 }
175
176 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
177 Register obj,
178 Register pre_val,
179 Register tmp,
180 bool tosca_live,
181 bool expand_call) {
182 // If expand_call is true then we expand the call_VM_leaf macro
183 // directly to skip generating the check by
184 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
185
186 const Register thread = r15_thread;
187
188 Label done;
189 Label runtime;
190
191 assert(pre_val != noreg, "check this code");
192
193 if (obj != noreg) {
194 assert_different_registers(obj, pre_val, tmp);
195 assert(pre_val != rax, "check this code");
196 }
197
198 generate_pre_barrier_fast_path(masm, thread);
199 // If marking is not active (*(mark queue active address) == 0), jump to done
200 __ jcc(Assembler::equal, done);
201 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
202
203 __ bind(runtime);
204
205 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
206 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
207 // types. Save all argument registers before calling into the runtime.
208 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
209 __ pusha();
210 __ subptr(rsp, 64);
211 __ movdbl(Address(rsp, 0), j_farg0);
212 __ movdbl(Address(rsp, 8), j_farg1);
213 __ movdbl(Address(rsp, 16), j_farg2);
214 __ movdbl(Address(rsp, 24), j_farg3);
215 __ movdbl(Address(rsp, 32), j_farg4);
216 __ movdbl(Address(rsp, 40), j_farg5);
217 __ movdbl(Address(rsp, 48), j_farg6);
218 __ movdbl(Address(rsp, 56), j_farg7);
219 } else {
220 // Determine and save the live input values
221 __ push_call_clobbered_registers();
222 }
223
224 // Calling the runtime using the regular call_VM_leaf mechanism generates
225 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
226 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
227 //
228 // If we care generating the pre-barrier without a frame (e.g. in the
229 // intrinsified Reference.get() routine) then ebp might be pointing to
230 // the caller frame and so this check will most likely fail at runtime.
231 //
232 // Expanding the call directly bypasses the generation of the check.
233 // So when we do not have have a full interpreter frame on the stack
234 // expand_call should be passed true.
235
236 if (expand_call) {
237 assert(pre_val != c_rarg1, "smashed arg");
238 if (c_rarg1 != thread) {
239 __ mov(c_rarg1, thread);
240 }
241 if (c_rarg0 != pre_val) {
242 __ mov(c_rarg0, pre_val);
243 }
244 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
245 } else {
246 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
247 }
248
249 if (EnableValhalla && InlineTypePassFieldsAsArgs) {
250 // Restore registers
251 __ movdbl(j_farg0, Address(rsp, 0));
252 __ movdbl(j_farg1, Address(rsp, 8));
253 __ movdbl(j_farg2, Address(rsp, 16));
254 __ movdbl(j_farg3, Address(rsp, 24));
255 __ movdbl(j_farg4, Address(rsp, 32));
256 __ movdbl(j_farg5, Address(rsp, 40));
257 __ movdbl(j_farg6, Address(rsp, 48));
258 __ movdbl(j_farg7, Address(rsp, 56));
259 __ addptr(rsp, 64);
260 __ popa();
261 } else {
262 __ pop_call_clobbered_registers();
263 }
264
265 __ bind(done);
266 }
267
268 static void generate_post_barrier_fast_path(MacroAssembler* masm,
269 const Register store_addr,
270 const Register new_val,
271 const Register tmp,
272 const Register tmp2,
273 Label& done,
274 bool new_val_may_be_null) {
275 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
276 // Does store cross heap regions?
277 __ movptr(tmp, store_addr); // tmp := store address
278 __ xorptr(tmp, new_val); // tmp := store address ^ new value
279 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
280 __ jcc(Assembler::equal, done);
281 // Crosses regions, storing null?
282 if (new_val_may_be_null) {
283 __ cmpptr(new_val, NULL_WORD); // new value == null?
284 __ jcc(Assembler::equal, done);
285 }
286 // Storing region crossing non-null, is card young?
287 __ movptr(tmp, store_addr); // tmp := store address
288 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
289 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
290 // a valid address and therefore is not properly handled by the relocation code.
291 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
292 __ addptr(tmp, tmp2); // tmp := card address
293 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
294 }
295
296 static void generate_post_barrier_slow_path(MacroAssembler* masm,
297 const Register thread,
298 const Register tmp,
299 const Register tmp2,
300 Label& done,
301 Label& runtime) {
302 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
303 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
304 __ jcc(Assembler::equal, done);
305 // Storing a region crossing, non-null oop, card is clean.
306 // Dirty card and log.
307 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
308 generate_queue_insertion(masm,
309 G1ThreadLocalData::dirty_card_queue_index_offset(),
310 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
311 runtime,
312 thread, tmp, tmp2);
313 __ jmp(done);
314 }
315
316 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
317 Register store_addr,
318 Register new_val,
319 Register tmp,
320 Register tmp2) {
321 const Register thread = r15_thread;
322
323 Label done;
324 Label runtime;
325
326 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
327 // If card is young, jump to done
328 __ jcc(Assembler::equal, done);
329 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
330
331 __ bind(runtime);
332 // Barriers might be emitted when converting between (scalarized) calling conventions for inline
333 // types. Save all argument registers before calling into the runtime.
334 // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
335 __ pusha();
336 __ subptr(rsp, 64);
337 __ movdbl(Address(rsp, 0), j_farg0);
338 __ movdbl(Address(rsp, 8), j_farg1);
339 __ movdbl(Address(rsp, 16), j_farg2);
340 __ movdbl(Address(rsp, 24), j_farg3);
341 __ movdbl(Address(rsp, 32), j_farg4);
342 __ movdbl(Address(rsp, 40), j_farg5);
343 __ movdbl(Address(rsp, 48), j_farg6);
344 __ movdbl(Address(rsp, 56), j_farg7);
345
346 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
347
348 // Restore registers
349 __ movdbl(j_farg0, Address(rsp, 0));
350 __ movdbl(j_farg1, Address(rsp, 8));
351 __ movdbl(j_farg2, Address(rsp, 16));
352 __ movdbl(j_farg3, Address(rsp, 24));
353 __ movdbl(j_farg4, Address(rsp, 32));
354 __ movdbl(j_farg5, Address(rsp, 40));
355 __ movdbl(j_farg6, Address(rsp, 48));
356 __ movdbl(j_farg7, Address(rsp, 56));
357 __ addptr(rsp, 64);
358 __ popa();
359
360 __ bind(done);
361 }
362
363 #if defined(COMPILER2)
364
365 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
366 SaveLiveRegisters save_registers(masm, stub);
367 if (c_rarg0 != arg) {
368 __ mov(c_rarg0, arg);
369 }
370 __ mov(c_rarg1, r15_thread);
371 // rax is a caller-saved, non-argument-passing register, so it does not
372 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
373 // entering this stub, it is saved at this point, and restored after the
374 // call. If it did not contain any live value, it is free to be used. In
375 // either case, it is safe to use it here as a call scratch register.
376 __ call(RuntimeAddress(runtime_path), rax);
377 }
378
379 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
380 Register obj,
381 Register pre_val,
382 Register tmp,
383 G1PreBarrierStubC2* stub) {
384 const Register thread = r15_thread;
385
386 assert(pre_val != noreg, "check this code");
387 if (obj != noreg) {
388 assert_different_registers(obj, pre_val, tmp);
389 }
390
391 stub->initialize_registers(obj, pre_val, thread, tmp);
392
393 generate_pre_barrier_fast_path(masm, thread);
394 // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
395 __ jcc(Assembler::notEqual, *stub->entry());
396
397 __ bind(*stub->continuation());
398 }
399
400 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
401 G1PreBarrierStubC2* stub) const {
402 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
403 Label runtime;
404 Register obj = stub->obj();
405 Register pre_val = stub->pre_val();
406 Register thread = stub->thread();
407 Register tmp = stub->tmp1();
408 assert(stub->tmp2() == noreg, "not needed in this platform");
409
410 __ bind(*stub->entry());
411 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
412
413 __ bind(runtime);
414 generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
415 __ jmp(*stub->continuation());
416 }
417
418 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
419 Register store_addr,
420 Register new_val,
421 Register tmp,
422 Register tmp2,
423 G1PostBarrierStubC2* stub) {
424 const Register thread = r15_thread;
425 stub->initialize_registers(thread, tmp, tmp2);
426
427 bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
428 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
429 // If card is not young, jump to stub (slow path)
430 __ jcc(Assembler::notEqual, *stub->entry());
431
432 __ bind(*stub->continuation());
433 }
434
435 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
436 G1PostBarrierStubC2* stub) const {
437 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
438 Label runtime;
439 Register thread = stub->thread();
440 Register tmp = stub->tmp1(); // tmp holds the card address.
441 Register tmp2 = stub->tmp2();
442 assert(stub->tmp3() == noreg, "not needed in this platform");
443
444 __ bind(*stub->entry());
445 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
446
447 __ bind(runtime);
448 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
449 __ jmp(*stub->continuation());
450 }
451
452 #endif // COMPILER2
453
454 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
455 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
456 bool in_heap = (decorators & IN_HEAP) != 0;
457 bool as_normal = (decorators & AS_NORMAL) != 0;
458 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
459
460 bool needs_pre_barrier = as_normal && !dest_uninitialized;
461 bool needs_post_barrier = val != noreg && in_heap;
462
463 // flatten object address if needed
464 // We do it regardless of precise because we need the registers
465 if (dst.index() == noreg && dst.disp() == 0) {
466 if (dst.base() != tmp1) {
467 __ movptr(tmp1, dst.base());
468 }
469 } else {
470 __ lea(tmp1, dst);
471 }
472
473 if (needs_pre_barrier) {
474 g1_write_barrier_pre(masm /*masm*/,
475 tmp1 /* obj */,
476 tmp2 /* pre_val */,
477 tmp3 /* tmp */,
478 val != noreg /* tosca_live */,
479 false /* expand_call */);
480 }
481 if (val == noreg) {
482 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
483 } else {
484 Register new_val = val;
485 if (needs_post_barrier) {
486 // G1 barrier needs uncompressed oop for region cross check.
487 if (UseCompressedOops) {
488 new_val = tmp2;
489 __ movptr(new_val, val);
490 }
491 }
492 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
493 if (needs_post_barrier) {
494 g1_write_barrier_post(masm /*masm*/,
495 tmp1 /* store_adr */,
496 new_val /* new_val */,
497 tmp3 /* tmp */,
498 tmp2 /* tmp2 */);
499 }
500 }
501 }
502
503 #ifdef COMPILER1
504
505 #undef __
506 #define __ ce->masm()->
507
508 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
509 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
510 // At this point we know that marking is in progress.
511 // If do_load() is true then we have to emit the
512 // load of the previous value; otherwise it has already
513 // been loaded into _pre_val.
514
515 __ bind(*stub->entry());
516 assert(stub->pre_val()->is_register(), "Precondition.");
517
518 Register pre_val_reg = stub->pre_val()->as_register();
519
520 if (stub->do_load()) {
521 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
522 }
523
524 __ cmpptr(pre_val_reg, NULL_WORD);
525 __ jcc(Assembler::equal, *stub->continuation());
526 ce->store_parameter(stub->pre_val()->as_register(), 0);
527 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
528 __ jmp(*stub->continuation());
529
530 }
531
532 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
533 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
534 __ bind(*stub->entry());
535 assert(stub->addr()->is_register(), "Precondition.");
536 assert(stub->new_val()->is_register(), "Precondition.");
537 Register new_val_reg = stub->new_val()->as_register();
538 __ cmpptr(new_val_reg, NULL_WORD);
539 __ jcc(Assembler::equal, *stub->continuation());
540 ce->store_parameter(stub->addr()->as_pointer_register(), 0);
541 __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
542 __ jmp(*stub->continuation());
543 }
544
545 #undef __
546
547 #define __ sasm->
548
549 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
550 // Generated code assumes that buffer index is pointer sized.
551 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
552
553 __ prologue("g1_pre_barrier", false);
554 // arg0 : previous value of memory
555
556 __ push(rax);
557 __ push(rdx);
558
559 const Register pre_val = rax;
560 const Register thread = r15_thread;
561 const Register tmp = rdx;
562
563 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
564 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
565 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
566
567 Label done;
568 Label runtime;
569
570 // Is marking still active?
571 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
572 __ cmpl(queue_active, 0);
573 } else {
574 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
575 __ cmpb(queue_active, 0);
576 }
577 __ jcc(Assembler::equal, done);
578
579 // Can we store original value in the thread's buffer?
580
581 __ movptr(tmp, queue_index);
582 __ testptr(tmp, tmp);
583 __ jcc(Assembler::zero, runtime);
584 __ subptr(tmp, wordSize);
585 __ movptr(queue_index, tmp);
586 __ addptr(tmp, buffer);
587
588 // prev_val (rax)
589 __ load_parameter(0, pre_val);
590 __ movptr(Address(tmp, 0), pre_val);
591 __ jmp(done);
592
593 __ bind(runtime);
594
595 __ push_call_clobbered_registers();
596
597 // load the pre-value
598 __ load_parameter(0, rcx);
599 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
600
601 __ pop_call_clobbered_registers();
602
603 __ bind(done);
604
605 __ pop(rdx);
606 __ pop(rax);
607
608 __ epilogue();
609 }
610
611 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
612 __ prologue("g1_post_barrier", false);
613
614 CardTableBarrierSet* ct =
615 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
616
617 Label done;
618 Label enqueued;
619 Label runtime;
620
621 // At this point we know new_value is non-null and the new_value crosses regions.
622 // Must check to see if card is already dirty
623
624 const Register thread = r15_thread;
625
626 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
627 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
628
629 __ push(rax);
630 __ push(rcx);
631
632 const Register cardtable = rax;
633 const Register card_addr = rcx;
634
635 __ load_parameter(0, card_addr);
636 __ shrptr(card_addr, CardTable::card_shift());
637 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
638 // a valid address and therefore is not properly handled by the relocation code.
639 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
640 __ addptr(card_addr, cardtable);
641
642 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
643 __ jcc(Assembler::equal, done);
644
645 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
646 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
647 __ jcc(Assembler::equal, done);
648
649 // storing region crossing non-null, card is clean.
650 // dirty card and log.
651
652 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
653
654 const Register tmp = rdx;
655 __ push(rdx);
656
657 __ movptr(tmp, queue_index);
658 __ testptr(tmp, tmp);
659 __ jcc(Assembler::zero, runtime);
660 __ subptr(tmp, wordSize);
661 __ movptr(queue_index, tmp);
662 __ addptr(tmp, buffer);
663 __ movptr(Address(tmp, 0), card_addr);
664 __ jmp(enqueued);
665
666 __ bind(runtime);
667 __ push_call_clobbered_registers();
668
669 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
670
671 __ pop_call_clobbered_registers();
672
673 __ bind(enqueued);
674 __ pop(rdx);
675
676 __ bind(done);
677 __ pop(rcx);
678 __ pop(rax);
679
680 __ epilogue();
681 }
682
683 #undef __
684
685 #endif // COMPILER1