1 /*
2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #if INCLUDE_CDS
28 #include "code/SCCache.hpp"
29 #endif
30 #include "code/vmreg.inline.hpp"
31 #include "gc/g1/c2/g1BarrierSetC2.hpp"
32 #include "gc/g1/g1BarrierSet.hpp"
33 #include "gc/g1/g1BarrierSetAssembler.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35 #include "gc/g1/g1CardTable.hpp"
36 #include "gc/g1/g1ThreadLocalData.hpp"
37 #include "gc/g1/g1HeapRegion.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/block.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/escape.hpp"
42 #include "opto/graphKit.hpp"
43 #include "opto/idealKit.hpp"
44 #include "opto/machnode.hpp"
45 #include "opto/macro.hpp"
46 #include "opto/memnode.hpp"
47 #include "opto/node.hpp"
48 #include "opto/output.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/runtime.hpp"
52 #include "opto/type.hpp"
53 #include "utilities/growableArray.hpp"
54 #include "utilities/macros.hpp"
55
56 /*
57 * Determine if the G1 pre-barrier can be removed. The pre-barrier is
58 * required by SATB to make sure all objects live at the start of the
59 * marking are kept alive, all reference updates need to any previous
60 * reference stored before writing.
61 *
62 * If the previous value is null there is no need to save the old value.
63 * References that are null are filtered during runtime by the barrier
64 * code to avoid unnecessary queuing.
65 *
66 * However in the case of newly allocated objects it might be possible to
67 * prove that the reference about to be overwritten is null during compile
68 * time and avoid adding the barrier code completely.
69 *
70 * The compiler needs to determine that the object in which a field is about
71 * to be written is newly allocated, and that no prior store to the same field
72 * has happened since the allocation.
73 */
74 bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
75 PhaseValues* phase,
76 Node* adr,
77 BasicType bt,
78 uint adr_idx) const {
79 intptr_t offset = 0;
80 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
81 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
82
83 if (offset == Type::OffsetBot) {
84 return false; // Cannot unalias unless there are precise offsets.
85 }
86 if (alloc == nullptr) {
87 return false; // No allocation found.
88 }
89
90 intptr_t size_in_bytes = type2aelembytes(bt);
91 Node* mem = kit->memory(adr_idx); // Start searching here.
92
93 for (int cnt = 0; cnt < 50; cnt++) {
94 if (mem->is_Store()) {
95 Node* st_adr = mem->in(MemNode::Address);
96 intptr_t st_offset = 0;
97 Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
98
99 if (st_base == nullptr) {
100 break; // Inscrutable pointer.
101 }
102 if (st_base == base && st_offset == offset) {
103 // We have found a store with same base and offset as ours.
104 break;
105 }
106 if (st_offset != offset && st_offset != Type::OffsetBot) {
107 const int MAX_STORE = BytesPerLong;
108 if (st_offset >= offset + size_in_bytes ||
109 st_offset <= offset - MAX_STORE ||
110 st_offset <= offset - mem->as_Store()->memory_size()) {
111 // Success: The offsets are provably independent.
112 // (You may ask, why not just test st_offset != offset and be done?
113 // The answer is that stores of different sizes can co-exist
114 // in the same sequence of RawMem effects. We sometimes initialize
115 // a whole 'tile' of array elements with a single jint or jlong.)
116 mem = mem->in(MemNode::Memory);
117 continue; // Advance through independent store memory.
118 }
119 }
120 if (st_base != base
121 && MemNode::detect_ptr_independence(base, alloc, st_base,
122 AllocateNode::Ideal_allocation(st_base),
123 phase)) {
124 // Success: the bases are provably independent.
125 mem = mem->in(MemNode::Memory);
126 continue; // Advance through independent store memory.
127 }
128 } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
129 InitializeNode* st_init = mem->in(0)->as_Initialize();
130 AllocateNode* st_alloc = st_init->allocation();
131
132 // Make sure that we are looking at the same allocation site.
133 // The alloc variable is guaranteed to not be null here from earlier check.
134 if (alloc == st_alloc) {
135 // Check that the initialization is storing null so that no previous store
136 // has been moved up and directly write a reference.
137 Node* captured_store = st_init->find_captured_store(offset,
138 type2aelembytes(T_OBJECT),
139 phase);
140 if (captured_store == nullptr || captured_store == st_init->zero_memory()) {
141 return true;
142 }
143 }
144 }
145 // Unless there is an explicit 'continue', we must bail out here,
146 // because 'mem' is an inscrutable memory state (e.g., a call).
147 break;
148 }
149 return false;
150 }
151
152 /*
153 * G1, similar to any GC with a Young Generation, requires a way to keep track
154 * of references from Old Generation to Young Generation to make sure all live
155 * objects are found. G1 also requires to keep track of object references
156 * between different regions to enable evacuation of old regions, which is done
157 * as part of mixed collections. References are tracked in remembered sets,
158 * which are continuously updated as references are written to with the help of
159 * the post-barrier.
160 *
161 * To reduce the number of updates to the remembered set, the post-barrier
162 * filters out updates to fields in objects located in the Young Generation, the
163 * same region as the reference, when null is being written, or if the card is
164 * already marked as dirty by an earlier write.
165 *
166 * Under certain circumstances it is possible to avoid generating the
167 * post-barrier completely, if it is possible during compile time to prove the
168 * object is newly allocated and that no safepoint exists between the allocation
169 * and the store. This can be seen as a compile-time version of the
170 * above-mentioned Young Generation filter.
171 *
172 * In the case of a slow allocation, the allocation code must handle the barrier
173 * as part of the allocation if the allocated object is not located in the
174 * nursery; this would happen for humongous objects.
175 */
176 bool G1BarrierSetC2::g1_can_remove_post_barrier(GraphKit* kit,
177 PhaseValues* phase, Node* store_ctrl,
178 Node* adr) const {
179 intptr_t offset = 0;
180 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
181 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
182
183 if (offset == Type::OffsetBot) {
184 return false; // Cannot unalias unless there are precise offsets.
185 }
186 if (alloc == nullptr) {
187 return false; // No allocation found.
188 }
189
190 Node* mem = store_ctrl; // Start search from Store node.
191 if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
192 InitializeNode* st_init = mem->in(0)->as_Initialize();
193 AllocateNode* st_alloc = st_init->allocation();
194 // Make sure we are looking at the same allocation
195 if (alloc == st_alloc) {
196 return true;
197 }
198 }
199
200 return false;
201 }
202
203 Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
204 DecoratorSet decorators = access.decorators();
205 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
206 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
207 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
208 // If we are reading the value of the referent field of a Reference object, we
209 // need to record the referent in an SATB log buffer using the pre-barrier
210 // mechanism. Also we need to add a memory barrier to prevent commoning reads
211 // from this field across safepoints, since GC can change its value.
212 bool need_read_barrier = ((on_weak || on_phantom) && !no_keepalive);
213 if (access.is_oop() && need_read_barrier) {
214 access.set_barrier_data(G1C2BarrierPre);
215 }
216 return CardTableBarrierSetC2::load_at_resolved(access, val_type);
217 }
218
219 void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
220 eliminate_gc_barrier_data(node);
221 }
222
223 void G1BarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
224 if (node->is_LoadStore()) {
225 LoadStoreNode* loadstore = node->as_LoadStore();
226 loadstore->set_barrier_data(0);
227 } else if (node->is_Mem()) {
228 MemNode* mem = node->as_Mem();
229 mem->set_barrier_data(0);
230 }
231 }
232
233 static void refine_barrier_by_new_val_type(const Node* n) {
234 if (n->Opcode() != Op_StoreP &&
235 n->Opcode() != Op_StoreN) {
236 return;
237 }
238 MemNode* store = n->as_Mem();
239 const Node* newval = n->in(MemNode::ValueIn);
240 assert(newval != nullptr, "");
241 const Type* newval_bottom = newval->bottom_type();
242 TypePtr::PTR newval_type = newval_bottom->make_ptr()->ptr();
243 uint8_t barrier_data = store->barrier_data();
244 if (!newval_bottom->isa_oopptr() &&
245 !newval_bottom->isa_narrowoop() &&
246 newval_type != TypePtr::Null) {
247 // newval is neither an OOP nor null, so there is no barrier to refine.
248 assert(barrier_data == 0, "non-OOP stores should have no barrier data");
249 return;
250 }
251 if (barrier_data == 0) {
252 // No barrier to refine.
253 return;
254 }
255 if (newval_type == TypePtr::Null) {
256 // Simply elide post-barrier if writing null.
257 barrier_data &= ~G1C2BarrierPost;
258 barrier_data &= ~G1C2BarrierPostNotNull;
259 } else if (((barrier_data & G1C2BarrierPost) != 0) &&
260 newval_type == TypePtr::NotNull) {
261 // If the post-barrier has not been elided yet (e.g. due to newval being
262 // freshly allocated), mark it as not-null (simplifies barrier tests and
263 // compressed OOPs logic).
264 barrier_data |= G1C2BarrierPostNotNull;
265 }
266 store->set_barrier_data(barrier_data);
267 return;
268 }
269
270 // Refine (not really expand) G1 barriers by looking at the new value type
271 // (whether it is necessarily null or necessarily non-null).
272 bool G1BarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
273 ResourceMark rm;
274 VectorSet visited;
275 Node_List worklist;
276 worklist.push(C->root());
277 while (worklist.size() > 0) {
278 Node* n = worklist.pop();
279 if (visited.test_set(n->_idx)) {
280 continue;
281 }
282 refine_barrier_by_new_val_type(n);
283 for (uint j = 0; j < n->req(); j++) {
284 Node* in = n->in(j);
285 if (in != nullptr) {
286 worklist.push(in);
287 }
288 }
289 }
290 return false;
291 }
292
293 uint G1BarrierSetC2::estimated_barrier_size(const Node* node) const {
294 // These Ideal node counts are extracted from the pre-matching Ideal graph
295 // generated when compiling the following method with early barrier expansion:
296 // static void write(MyObject obj1, Object o) {
297 // obj1.o1 = o;
298 // }
299 uint8_t barrier_data = MemNode::barrier_data(node);
300 uint nodes = 0;
301 if ((barrier_data & G1C2BarrierPre) != 0) {
302 nodes += 50;
303 }
304 if ((barrier_data & G1C2BarrierPost) != 0) {
305 nodes += 60;
306 }
307 return nodes;
308 }
309
310 bool G1BarrierSetC2::can_initialize_object(const StoreNode* store) const {
311 assert(store->Opcode() == Op_StoreP || store->Opcode() == Op_StoreN, "OOP store expected");
312 // It is OK to move the store across the object initialization boundary only
313 // if it does not have any barrier, or if it has barriers that can be safely
314 // elided (because of the compensation steps taken on the allocation slow path
315 // when ReduceInitialCardMarks is enabled).
316 return (MemNode::barrier_data(store) == 0) || use_ReduceInitialCardMarks();
317 }
318
319 void G1BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
320 if (ac->is_clone_inst() && !use_ReduceInitialCardMarks()) {
321 clone_in_runtime(phase, ac, G1BarrierSetRuntime::clone_addr(), "G1BarrierSetRuntime::clone");
322 return;
323 }
324 BarrierSetC2::clone_at_expansion(phase, ac);
325 }
326
327 Node* G1BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
328 DecoratorSet decorators = access.decorators();
329 bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
330 bool in_heap = (decorators & IN_HEAP) != 0;
331 bool tightly_coupled_alloc = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
332 bool need_store_barrier = !(tightly_coupled_alloc && use_ReduceInitialCardMarks()) && (in_heap || anonymous);
333 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
334 if (access.is_oop() && need_store_barrier) {
335 access.set_barrier_data(get_store_barrier(access));
336 if (tightly_coupled_alloc) {
337 assert(!use_ReduceInitialCardMarks(),
338 "post-barriers are only needed for tightly-coupled initialization stores when ReduceInitialCardMarks is disabled");
339 // Pre-barriers are unnecessary for tightly-coupled initialization stores.
340 access.set_barrier_data(access.barrier_data() & ~G1C2BarrierPre);
341 }
342 }
343 if (no_keepalive) {
344 // No keep-alive means no need for the pre-barrier.
345 access.set_barrier_data(access.barrier_data() & ~G1C2BarrierPre);
346 }
347 return BarrierSetC2::store_at_resolved(access, val);
348 }
349
350 Node* G1BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
351 Node* new_val, const Type* value_type) const {
352 GraphKit* kit = access.kit();
353 if (!access.is_oop()) {
354 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
355 }
356 access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
357 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
358 }
359
360 Node* G1BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
361 Node* new_val, const Type* value_type) const {
362 GraphKit* kit = access.kit();
363 if (!access.is_oop()) {
364 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
365 }
366 access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
367 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
368 }
369
370 Node* G1BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
371 GraphKit* kit = access.kit();
372 if (!access.is_oop()) {
373 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
374 }
375 access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
376 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
377 }
378
379 class G1BarrierSetC2State : public BarrierSetC2State {
380 private:
381 GrowableArray<G1BarrierStubC2*>* _stubs;
382
383 public:
384 G1BarrierSetC2State(Arena* arena)
385 : BarrierSetC2State(arena),
386 _stubs(new (arena) GrowableArray<G1BarrierStubC2*>(arena, 8, 0, nullptr)) {}
387
388 GrowableArray<G1BarrierStubC2*>* stubs() {
389 return _stubs;
390 }
391
392 bool needs_liveness_data(const MachNode* mach) const {
393 return G1PreBarrierStubC2::needs_barrier(mach) ||
394 G1PostBarrierStubC2::needs_barrier(mach);
395 }
396
397 bool needs_livein_data() const {
398 return false;
399 }
400 };
401
402 static G1BarrierSetC2State* barrier_set_state() {
403 return reinterpret_cast<G1BarrierSetC2State*>(Compile::current()->barrier_set_state());
404 }
405
406 G1BarrierStubC2::G1BarrierStubC2(const MachNode* node) : BarrierStubC2(node) {}
407
408 G1PreBarrierStubC2::G1PreBarrierStubC2(const MachNode* node) : G1BarrierStubC2(node) {}
409
410 bool G1PreBarrierStubC2::needs_barrier(const MachNode* node) {
411 return (node->barrier_data() & G1C2BarrierPre) != 0;
412 }
413
414 G1PreBarrierStubC2* G1PreBarrierStubC2::create(const MachNode* node) {
415 G1PreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) G1PreBarrierStubC2(node);
416 if (!Compile::current()->output()->in_scratch_emit_size()) {
417 barrier_set_state()->stubs()->append(stub);
418 }
419 return stub;
420 }
421
422 void G1PreBarrierStubC2::initialize_registers(Register obj, Register pre_val, Register thread, Register tmp1, Register tmp2) {
423 _obj = obj;
424 _pre_val = pre_val;
425 _thread = thread;
426 _tmp1 = tmp1;
427 _tmp2 = tmp2;
428 }
429
430 Register G1PreBarrierStubC2::obj() const {
431 return _obj;
432 }
433
434 Register G1PreBarrierStubC2::pre_val() const {
435 return _pre_val;
436 }
437
438 Register G1PreBarrierStubC2::thread() const {
439 return _thread;
440 }
441
442 Register G1PreBarrierStubC2::tmp1() const {
443 return _tmp1;
444 }
445
446 Register G1PreBarrierStubC2::tmp2() const {
447 return _tmp2;
448 }
449
450 void G1PreBarrierStubC2::emit_code(MacroAssembler& masm) {
451 G1BarrierSetAssembler* bs = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
452 bs->generate_c2_pre_barrier_stub(&masm, this);
453 }
454
455 G1PostBarrierStubC2::G1PostBarrierStubC2(const MachNode* node) : G1BarrierStubC2(node) {}
456
457 bool G1PostBarrierStubC2::needs_barrier(const MachNode* node) {
458 return (node->barrier_data() & G1C2BarrierPost) != 0;
459 }
460
461 G1PostBarrierStubC2* G1PostBarrierStubC2::create(const MachNode* node) {
462 G1PostBarrierStubC2* const stub = new (Compile::current()->comp_arena()) G1PostBarrierStubC2(node);
463 if (!Compile::current()->output()->in_scratch_emit_size()) {
464 barrier_set_state()->stubs()->append(stub);
465 }
466 return stub;
467 }
468
469 void G1PostBarrierStubC2::initialize_registers(Register thread, Register tmp1, Register tmp2, Register tmp3) {
470 _thread = thread;
471 _tmp1 = tmp1;
472 _tmp2 = tmp2;
473 _tmp3 = tmp3;
474 }
475
476 Register G1PostBarrierStubC2::thread() const {
477 return _thread;
478 }
479
480 Register G1PostBarrierStubC2::tmp1() const {
481 return _tmp1;
482 }
483
484 Register G1PostBarrierStubC2::tmp2() const {
485 return _tmp2;
486 }
487
488 Register G1PostBarrierStubC2::tmp3() const {
489 return _tmp3;
490 }
491
492 void G1PostBarrierStubC2::emit_code(MacroAssembler& masm) {
493 G1BarrierSetAssembler* bs = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
494 bs->generate_c2_post_barrier_stub(&masm, this);
495 }
496
497 void* G1BarrierSetC2::create_barrier_state(Arena* comp_arena) const {
498 return new (comp_arena) G1BarrierSetC2State(comp_arena);
499 }
500
501 int G1BarrierSetC2::get_store_barrier(C2Access& access) const {
502 if (!access.is_parse_access()) {
503 // Only support for eliding barriers at parse time for now.
504 return G1C2BarrierPre | G1C2BarrierPost;
505 }
506 GraphKit* kit = (static_cast<C2ParseAccess&>(access)).kit();
507 Node* ctl = kit->control();
508 Node* adr = access.addr().node();
509 uint adr_idx = kit->C->get_alias_index(access.addr().type());
510 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory");
511
512 bool can_remove_pre_barrier = g1_can_remove_pre_barrier(kit, &kit->gvn(), adr, access.type(), adr_idx);
513
514 // We can skip marks on a freshly-allocated object in Eden. Keep this code in
515 // sync with CardTableBarrierSet::on_slowpath_allocation_exit. That routine
516 // informs GC to take appropriate compensating steps, upon a slow-path
517 // allocation, so as to make this card-mark elision safe.
518 // The post-barrier can also be removed if null is written. This case is
519 // handled by G1BarrierSetC2::expand_barriers, which runs at the end of C2's
520 // platform-independent optimizations to exploit stronger type information.
521 bool can_remove_post_barrier = use_ReduceInitialCardMarks() &&
522 ((access.base() == kit->just_allocated_object(ctl)) ||
523 g1_can_remove_post_barrier(kit, &kit->gvn(), ctl, adr));
524
525 int barriers = 0;
526 if (!can_remove_pre_barrier) {
527 barriers |= G1C2BarrierPre;
528 }
529 if (!can_remove_post_barrier) {
530 barriers |= G1C2BarrierPost;
531 }
532
533 return barriers;
534 }
535
536 void G1BarrierSetC2::late_barrier_analysis() const {
537 compute_liveness_at_stubs();
538 }
539
540 void G1BarrierSetC2::emit_stubs(CodeBuffer& cb) const {
541 MacroAssembler masm(&cb);
542 GrowableArray<G1BarrierStubC2*>* const stubs = barrier_set_state()->stubs();
543 for (int i = 0; i < stubs->length(); i++) {
544 // Make sure there is enough space in the code buffer
545 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
546 ciEnv::current()->record_failure("CodeCache is full");
547 return;
548 }
549 stubs->at(i)->emit_code(masm);
550 }
551 masm.flush();
552 }
553
554 #ifndef PRODUCT
555 void G1BarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
556 if ((mach->barrier_data() & G1C2BarrierPre) != 0) {
557 st->print("pre ");
558 }
559 if ((mach->barrier_data() & G1C2BarrierPost) != 0) {
560 st->print("post ");
561 }
562 if ((mach->barrier_data() & G1C2BarrierPostNotNull) != 0) {
563 st->print("notnull ");
564 }
565 }
566 #endif // !PRODUCT