1 /*
2 * Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/escape.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/idealKit.hpp"
38 #include "opto/macro.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/output.hpp"
41 #include "opto/rootnode.hpp"
42 #include "opto/runtime.hpp"
43
44 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
45 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
46 }
47
48 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
49 BarrierSetC2State(comp_arena),
50 _stubs(new (comp_arena) GrowableArray<ShenandoahBarrierStubC2*>(comp_arena, 8, 0, nullptr)),
51 _trampoline_stubs_count(0),
52 _stubs_start_offset(0) {
53 }
54
55 static void set_barrier_data(C2Access& access, bool load, bool store) {
56 if (!access.is_oop()) {
57 return;
58 }
59
60 DecoratorSet decorators = access.decorators();
61 bool tightly_coupled = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
62 bool in_heap = (decorators & IN_HEAP) != 0;
63 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
64 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
65
66 if (tightly_coupled) {
67 access.set_barrier_data(ShenandoahBitElided);
68 return;
69 }
70
71 uint8_t barrier_data = 0;
72
73 if (load) {
74 if (ShenandoahLoadRefBarrier) {
75 if (on_phantom) {
76 barrier_data |= ShenandoahBitPhantom;
77 } else if (on_weak) {
78 barrier_data |= ShenandoahBitWeak;
79 } else {
80 barrier_data |= ShenandoahBitStrong;
81 }
82 }
83 }
84
85 if (store) {
86 if (ShenandoahSATBBarrier) {
87 barrier_data |= ShenandoahBitKeepAlive;
88 }
89 if (ShenandoahCardBarrier && in_heap) {
90 barrier_data |= ShenandoahBitCardMark;
91 }
92 }
93
94 if (!in_heap) {
95 barrier_data |= ShenandoahBitNative;
96 }
97
98 access.set_barrier_data(barrier_data);
99 }
100
101 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
102 // 1: Non-reference load, no additional barrier is needed
103 if (!access.is_oop()) {
104 return BarrierSetC2::load_at_resolved(access, val_type);
105 }
106
107 // 2. Set barrier data for load
108 set_barrier_data(access, /* load = */ true, /* store = */ false);
109
110 // 3. Correction: If we are reading the value of the referent field of
111 // a Reference object, we need to record the referent resurrection.
112 DecoratorSet decorators = access.decorators();
113 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
114 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
115 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
116 bool needs_keepalive = ((on_weak || on_phantom) && !no_keepalive);
117 if (needs_keepalive) {
118 uint8_t barriers = access.barrier_data() | (ShenandoahSATBBarrier ? ShenandoahBitKeepAlive : 0);
119 access.set_barrier_data(barriers);
120 }
121
122 return BarrierSetC2::load_at_resolved(access, val_type);
123 }
124
125 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
126 // 1: Non-reference store, no additional barrier is needed
127 if (!access.is_oop()) {
128 return BarrierSetC2::store_at_resolved(access, val);
129 }
130
131 // 2. Set barrier data for store
132 set_barrier_data(access, /* load = */ false, /* store = */ true);
133
134 // 3. Correction: avoid keep-alive barriers that should not do keep-alive.
135 DecoratorSet decorators = access.decorators();
136 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
137 if (no_keepalive) {
138 access.set_barrier_data(access.barrier_data() & ~ShenandoahBitKeepAlive);
139 }
140
141 return BarrierSetC2::store_at_resolved(access, val);
142 }
143
144 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
145 Node* new_val, const Type* value_type) const {
146 set_barrier_data(access, /* load = */ true, /* store = */ true);
147 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
148 }
149
150 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
151 Node* new_val, const Type* value_type) const {
152 set_barrier_data(access, /* load = */ true, /* store = */ true);
153 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
154 }
155
156 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
157 set_barrier_data(access, /* load = */ true, /* store = */ true);
158 return BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
159 }
160
161 void ShenandoahBarrierSetC2::refine_store(const Node* n) {
162 if (!ShenandoahElideBarriers) {
163 return;
164 }
165
166 MemNode* store = n->as_Store();
167 const Node* newval = n->in(MemNode::ValueIn);
168 assert(newval != nullptr, "");
169 const Type* newval_bottom = newval->bottom_type();
170 TypePtr::PTR newval_type = newval_bottom->make_ptr()->ptr();
171 uint8_t barrier_data = store->barrier_data();
172 if (!newval_bottom->isa_oopptr() &&
173 !newval_bottom->isa_narrowoop() &&
174 newval_type != TypePtr::Null) {
175 // newval is neither an OOP nor null, so there is no barrier to refine.
176 assert(barrier_data == 0, "non-OOP stores should have no barrier data");
177 return;
178 }
179 if (barrier_data == 0) {
180 // No barrier to refine.
181 return;
182 }
183 if (newval_type == TypePtr::Null) {
184 barrier_data &= ~ShenandoahBitNotNull;
185 // Simply elide post-barrier if writing null.
186 barrier_data &= ~ShenandoahBitCardMark;
187 } else if (newval_type == TypePtr::NotNull) {
188 barrier_data |= ShenandoahBitNotNull;
189 }
190 store->set_barrier_data(barrier_data);
191 }
192
193 bool ShenandoahBarrierSetC2::can_remove_load_barrier(Node* n) {
194 // Check if all outs feed into nodes that do not expose the oops to the rest
195 // of the runtime system. In this case, we can elide the LRB barrier. We bail
196 // out with false at the first sight of trouble.
197
198 ResourceMark rm;
199 VectorSet visited;
200 Node_List worklist;
201 worklist.push(n);
202
203 while (worklist.size() > 0) {
204 Node* n = worklist.pop();
205 if (visited.test_set(n->_idx)) {
206 continue;
207 }
208
209 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
210 Node* out = n->fast_out(i);
211 switch (out->Opcode()) {
212 case Op_CmpN: {
213 if (out->in(1) == n &&
214 out->in(2)->Opcode() == Op_ConN &&
215 out->in(2)->get_narrowcon() == 0) {
216 // Null check, no oop is exposed.
217 break;
218 } else {
219 return false;
220 }
221 }
222 case Op_CmpP: {
223 if (out->in(1) == n &&
224 out->in(2)->Opcode() == Op_ConP &&
225 out->in(2)->get_ptr() == 0) {
226 // Null check, no oop is exposed.
227 break;
228 } else {
229 return false;
230 }
231 }
232 case Op_DecodeN:
233 case Op_CastPP: {
234 // Check if any other outs are escaping.
235 worklist.push(out);
236 break;
237 }
238 case Op_CallStaticJava: {
239 if (out->as_CallStaticJava()->is_uncommon_trap()) {
240 // Local feeds into uncommon trap. Deopt machinery handles barriers itself.
241 break;
242 } else {
243 return false;
244 }
245 }
246
247 default: {
248 // Paranoidly distrust any other nodes.
249 // TODO: Check if there are other patterns that benefit from this elision.
250 return false;
251 }
252 }
253 }
254 }
255
256 // Nothing troublesome found.
257 return true;
258 }
259
260 void ShenandoahBarrierSetC2::refine_load(Node* n) {
261 if (!ShenandoahElideBarriers) {
262 return;
263 }
264
265 MemNode* load = n->as_Load();
266
267 uint8_t barrier_data = load->barrier_data();
268
269 // Do not touch weak LRBs at all: they are responsible for shielding from
270 // Reference.referent resurrection.
271 if ((barrier_data & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0) {
272 return;
273 }
274
275 if (can_remove_load_barrier(n)) {
276 barrier_data &= ~ShenandoahBitStrong;
277 barrier_data |= ShenandoahBitElided;
278 }
279
280 load->set_barrier_data(barrier_data);
281 }
282
283 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
284 ResourceMark rm;
285 VectorSet visited;
286 Node_List worklist;
287 worklist.push(C->root());
288 while (worklist.size() > 0) {
289 Node* n = worklist.pop();
290 if (visited.test_set(n->_idx)) {
291 continue;
292 }
293 switch(n->Opcode()) {
294 case Op_StoreP:
295 case Op_StoreN: {
296 refine_store(n);
297 break;
298 }
299 case Op_LoadN:
300 case Op_LoadP: {
301 refine_load(n);
302 break;
303 }
304 }
305
306 for (uint j = 0; j < n->req(); j++) {
307 Node* in = n->in(j);
308 if (in != nullptr) {
309 worklist.push(in);
310 }
311 }
312 }
313 return false;
314 }
315
316 // Support for macro expanded GC barriers
317 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
318 if (node->is_LoadStore()) {
319 LoadStoreNode* loadstore = node->as_LoadStore();
320 loadstore->set_barrier_data(0);
321 } else if (node->is_Mem()) {
322 MemNode* mem = node->as_Mem();
323 mem->set_barrier_data(0);
324 }
325 }
326
327 // If there are no real barrier flags on the node, strip away additional fluff.
328 // Matcher does not care about this, and we would like to avoid invoking "barrier_data() != 0"
329 // rules when the only flags are the irrelevant fluff.
330 void ShenandoahBarrierSetC2::strip_extra_data(const Node* n) const {
331 if (n->is_LoadStore()) {
332 LoadStoreNode* load_store = n->as_LoadStore();
333 uint8_t barrier_data = load_store->barrier_data();
334 if ((barrier_data & ShenandoahBitsReal) == 0) {
335 load_store->set_barrier_data(0);
336 }
337 } else if (n->is_Mem()) {
338 MemNode* mem = n->as_Mem();
339 uint8_t barrier_data = mem->barrier_data();
340 if ((barrier_data & ShenandoahBitsReal) == 0) {
341 mem->set_barrier_data(0);
342 }
343 }
344 }
345
346 void ShenandoahBarrierSetC2::strip_extra_data(Node_List& accesses) const {
347 for (uint c = 0; c < accesses.size(); c++) {
348 strip_extra_data(accesses.at(c));
349 }
350 }
351
352 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
353 eliminate_gc_barrier_data(node);
354 }
355
356 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* mach) const {
357 mach->set_barrier_data(0);
358 }
359
360 void ShenandoahBarrierSetC2::analyze_dominating_barriers() const {
361 if (!ShenandoahElideBarriers) {
362 return;
363 }
364
365 ResourceMark rm;
366 Compile* const C = Compile::current();
367 PhaseCFG* const cfg = C->cfg();
368
369 Node_List all_loads, loads, stores, atomics;
370 Node_List load_dominators, store_dominators, atomic_dominators;
371
372 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
373 const Block* const block = cfg->get_block(i);
374 for (uint j = 0; j < block->number_of_nodes(); ++j) {
375 Node* const node = block->get_node(j);
376
377 // Everything that happens in allocations does not need barriers.
378 if (node->is_Phi() && is_allocation(node)) {
379 load_dominators.push(node);
380 store_dominators.push(node);
381 atomic_dominators.push(node);
382 continue;
383 }
384
385 if (!node->is_Mach()) {
386 continue;
387 }
388
389 MachNode* const mach = node->as_Mach();
390 switch (mach->ideal_Opcode()) {
391
392 // Dominating loads have already passed through LRB and their load
393 // locations got fixed. Subsequent barriers are no longer required.
394 // The only exception are weak loads that have to go through LRB
395 // to deal with dying referents.
396 case Op_LoadP:
397 case Op_LoadN: {
398 if (mach->barrier_data() != 0) {
399 all_loads.push(mach);
400 }
401 if ((mach->barrier_data() & ShenandoahBitStrong) != 0) {
402 loads.push(mach);
403 load_dominators.push(mach);
404 }
405 break;
406 }
407
408 // Dominating stores have recorded the old value in SATB, and made the
409 // card table update for a location. Subsequent barriers are no longer
410 // required.
411 case Op_StoreP:
412 case Op_StoreN: {
413 if (mach->barrier_data() != 0) {
414 stores.push(mach);
415 load_dominators.push(mach);
416 store_dominators.push(mach);
417 atomic_dominators.push(mach);
418 }
419 break;
420 }
421
422 // Dominating atomics have dealt with everything as both loads and stores.
423 // Therefore, subsequent barriers are no longer required.
424 case Op_CompareAndExchangeN:
425 case Op_CompareAndExchangeP:
426 case Op_CompareAndSwapN:
427 case Op_CompareAndSwapP:
428 case Op_GetAndSetP:
429 case Op_GetAndSetN: {
430 if (mach->barrier_data() != 0) {
431 atomics.push(mach);
432 load_dominators.push(mach);
433 store_dominators.push(mach);
434 atomic_dominators.push(mach);
435 }
436 break;
437 }
438
439 default:
440 break;
441 }
442 }
443 }
444
445 elide_dominated_barriers(loads, load_dominators);
446 elide_dominated_barriers(stores, store_dominators);
447 elide_dominated_barriers(atomics, atomic_dominators);
448
449 // Also clean up extra metadata on these nodes. Dominance analysis likely left
450 // many non-elided barriers with extra metadata, which can be stripped away.
451 strip_extra_data(all_loads);
452 strip_extra_data(stores);
453 strip_extra_data(atomics);
454 }
455
456 uint ShenandoahBarrierSetC2::estimated_barrier_size(const Node* node) const {
457 // Barrier impact on fast-path is driven by GC state checks emitted very late.
458 // These checks are tight load-test-branch sequences, with no impact on C2 graph
459 // size. Limiting unrolling in presence of GC barriers might turn some loops
460 // tighter than with default unrolling, which may benefit performance due to denser
461 // code. Testing shows it is still counter-productive.
462 // Therefore, we report zero barrier size to let C2 do its normal thing.
463 return 0;
464 }
465
466 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const {
467 bool is_oop = is_reference_type(type);
468 if (!is_oop) {
469 return false;
470 }
471 if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
472 if (phase == Optimization) {
473 return false;
474 }
475 return !is_clone;
476 }
477 return true;
478 }
479
480 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
481 if (!ShenandoahCloneBarrier) {
482 return false;
483 }
484
485 if (src_type->isa_instptr() != nullptr) {
486 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
487 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
488 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
489 !ik->has_injected_fields() && !ik->has_object_fields()) {
490 if (!src_type->klass_is_exact()) {
491 // Class is *currently* the leaf in the hierarchy.
492 // Record the dependency so that we deopt if this does not hold in future.
493 Compile::current()->dependencies()->assert_leaf_type(ik);
494 }
495 return false;
496 }
497 } else if (src_type->isa_aryptr() != nullptr) {
498 // Array: need barrier only if array is oop-bearing.
499 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
500 if (is_reference_type(src_elem, true)) {
501 is_oop_array = true;
502 } else {
503 return false;
504 }
505 }
506
507 // Assume the worst.
508 return true;
509 }
510
511 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
512 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
513
514 bool is_oop_array = false;
515 if (!clone_needs_barrier(src_type, is_oop_array)) {
516 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
517 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
518 return;
519 }
520
521 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
522 // Looks like an instance? Prepare the instance clone. This would either
523 // be exploded into individual accesses or be left as runtime call.
524 // Common BarrierSetC2 prepares everything for both cases.
525 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
526 return;
527 }
528
529 // We are cloning the oop array. Prepare to call the normal arraycopy stub
530 // after the expansion. Normal stub takes the number of actual type-sized
531 // elements to copy after the base, compute the count here.
532 Node* offset = kit->MakeConX(arrayOopDesc::base_offset_in_bytes(UseCompressedOops ? T_NARROWOOP : T_OBJECT));
533 size = kit->gvn().transform(new SubXNode(size, offset));
534 size = kit->gvn().transform(new URShiftXNode(size, kit->intcon(LogBytesPerHeapOop)));
535 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, size, true, false);
536 ac->set_clone_array();
537 Node* n = kit->gvn().transform(ac);
538 if (n == ac) {
539 ac->set_adr_type(TypeRawPtr::BOTTOM);
540 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), TypeRawPtr::BOTTOM);
541 } else {
542 kit->set_all_memory(n);
543 }
544 }
545
546 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
547 Node* const ctrl = ac->in(TypeFunc::Control);
548 Node* const mem = ac->in(TypeFunc::Memory);
549 Node* const src = ac->in(ArrayCopyNode::Src);
550 Node* const src_offset = ac->in(ArrayCopyNode::SrcPos);
551 Node* const dest = ac->in(ArrayCopyNode::Dest);
552 Node* const dest_offset = ac->in(ArrayCopyNode::DestPos);
553 Node* length = ac->in(ArrayCopyNode::Length);
554
555 const TypeOopPtr* src_type = phase->igvn().type(src)->is_oopptr();
556
557 bool is_oop_array = false;
558 if (!clone_needs_barrier(src_type, is_oop_array)) {
559 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
560 BarrierSetC2::clone_at_expansion(phase, ac);
561 return;
562 }
563
564 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
565 // Still looks like an instance? Likely a large instance or reflective
566 // clone with unknown length. Go to runtime and handle it there.
567 clone_in_runtime(phase, ac, CAST_FROM_FN_PTR(address, ShenandoahRuntime::clone_addr()), "ShenandoahRuntime::clone");
568 return;
569 }
570
571 // We are cloning the oop array. Call into normal oop array copy stubs.
572 // Those stubs would call BarrierSetAssembler to handle GC barriers.
573
574 // This is the full clone, so offsets should equal each other and be at array base.
575 assert(src_offset == dest_offset, "should be equal");
576 const jlong offset = src_offset->get_long();
577 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
578 BasicType bt = ary_ptr->elem()->array_element_basic_type();
579 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
580
581 const char* copyfunc_name = "arraycopy";
582 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
583
584 Node* const call = phase->make_leaf_call(ctrl, mem,
585 OptoRuntime::fast_arraycopy_Type(),
586 copyfunc_addr, copyfunc_name,
587 TypeRawPtr::BOTTOM,
588 phase->basic_plus_adr(src, src_offset),
589 phase->basic_plus_adr(dest, dest_offset),
590 length,
591 phase->top()
592 );
593 phase->transform_later(call);
594
595 phase->igvn().replace_node(ac, call);
596 }
597
598 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
599 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
600 }
601
602 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
603 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
604 }
605
606 void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
607 os->print(" Node barriers: ");
608 if ((data & ShenandoahBitStrong) != 0) {
609 data &= ~ShenandoahBitStrong;
610 os->print("strong ");
611 }
612
613 if ((data & ShenandoahBitWeak) != 0) {
614 data &= ~ShenandoahBitWeak;
615 os->print("weak ");
616 }
617
618 if ((data & ShenandoahBitPhantom) != 0) {
619 data &= ~ShenandoahBitPhantom;
620 os->print("phantom ");
621 }
622
623 if ((data & ShenandoahBitElided) != 0) {
624 data &= ~ShenandoahBitElided;
625 os->print("elided ");
626 }
627
628 if ((data & ShenandoahBitKeepAlive) != 0) {
629 data &= ~ShenandoahBitKeepAlive;
630 os->print("keepalive ");
631 }
632
633 if ((data & ShenandoahBitCardMark) != 0) {
634 data &= ~ShenandoahBitCardMark;
635 os->print("cardmark ");
636 }
637
638 if ((data & ShenandoahBitNotNull) != 0) {
639 data &= ~ShenandoahBitNotNull;
640 os->print("not-null ");
641 }
642 os->cr();
643
644 if (data > 0) {
645 fatal("Unknown bit!");
646 }
647
648 os->print_cr(" GC configuration: %sLRB %sSATB %sCAS %sClone %sCard",
649 (ShenandoahLoadRefBarrier ? "+" : "-"),
650 (ShenandoahSATBBarrier ? "+" : "-"),
651 (ShenandoahCASBarrier ? "+" : "-"),
652 (ShenandoahCloneBarrier ? "+" : "-"),
653 (ShenandoahCardBarrier ? "+" : "-")
654 );
655 }
656
657 #ifdef ASSERT
658 void ShenandoahBarrierSetC2::verify_gc_barrier_assert(bool cond, const char* msg, uint8_t bd, Node* n) {
659 if (!cond) {
660 stringStream ss;
661 ss.print_cr("%s", msg);
662 ss.print_cr("-----------------");
663 print_barrier_data(&ss, bd);
664 ss.print_cr("-----------------");
665 n->dump_bfs(1, nullptr, "", &ss);
666 report_vm_error(__FILE__, __LINE__, ss.as_string());
667 }
668 }
669
670 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
671 if (!ShenandoahVerifyOptoBarriers) {
672 return;
673 }
674
675 // Optimizations might have removed the remaining auxiliary flags, making some accesses completely blank.
676 bool accept_blank = (phase == BeforeCodeGen);
677 bool expect_load_barriers = !accept_blank && ShenandoahLoadRefBarrier;
678 bool expect_store_barriers = !accept_blank && (ShenandoahSATBBarrier || ShenandoahCardBarrier);
679 bool expect_load_store_barriers = !accept_blank && ShenandoahCASBarrier;
680
681 Unique_Node_List wq;
682 Node_Stack phis(0);
683 VectorSet visited;
684
685 wq.push(compile->root());
686 for (uint next = 0; next < wq.size(); next++) {
687 Node *n = wq.at(next);
688 int opc = n->Opcode();
689
690 if (opc == Op_LoadP || opc == Op_LoadN) {
691 uint8_t bd = n->as_Load()->barrier_data();
692
693 const TypePtr* adr_type = n->as_Load()->adr_type();
694 if (adr_type->isa_oopptr() || adr_type->isa_narrowoop()) {
695 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Oop load should have barrier data", bd, n);
696
697 bool is_weak = ((bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0);
698 bool is_referent = adr_type->isa_instptr() &&
699 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
700 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
701
702 verify_gc_barrier_assert(!is_weak || is_referent, "Weak load only for Reference.referent", bd, n);
703 } else if (adr_type->isa_rawptr() || adr_type->isa_klassptr()) {
704 // Some LoadP-s are used for T_ADDRESS loads from raw pointers. These are not oops.
705 // Some LoadP-s are used to load class data.
706 // TODO: Verify their barrier data.
707 } else {
708 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
709 }
710 } else if (opc == Op_StoreP || opc == Op_StoreN) {
711 uint8_t bd = n->as_Store()->barrier_data();
712 const TypePtr* adr_type = n->as_Store()->adr_type();
713 if (adr_type->isa_oopptr() || adr_type->isa_narrowoop()) {
714 // Reference.clear stores null
715 bool is_referent = adr_type->isa_instptr() &&
716 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
717 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
718
719 const TypePtr* val_type = n->as_Store()->in(MemNode::Memory)->adr_type();
720 if (!is_referent && (val_type->isa_oopptr() || val_type->isa_narrowoop())) {
721 verify_gc_barrier_assert(!expect_store_barriers || (bd != 0), "Oop store should have barrier data", bd, n);
722 }
723 } else if (adr_type->isa_rawptr() || adr_type->isa_klassptr()) {
724 // Similar to LoadP-s, some of these accesses are raw, and some are handling oops.
725 // TODO: Verify their barrier data.
726 } else {
727 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
728 }
729 } else if (opc == Op_WeakCompareAndSwapP || opc == Op_WeakCompareAndSwapN ||
730 opc == Op_CompareAndExchangeP || opc == Op_CompareAndExchangeN ||
731 opc == Op_CompareAndSwapP || opc == Op_CompareAndSwapN ||
732 opc == Op_GetAndSetP || opc == Op_GetAndSetN) {
733 uint8_t bd = n->as_LoadStore()->barrier_data();
734 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Oop load-store should have barrier data", bd, n);
735 } else if (n->is_Mem()) {
736 uint8_t bd = MemNode::barrier_data(n); // FIXME: LOL HotSpot, why not n->as_Mem()? LoadStore is both is_Mem() and not as_Mem().
737 verify_gc_barrier_assert(bd == 0, "Other mem nodes should have no barrier data", bd, n);
738 }
739
740 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
741 Node* m = n->fast_out(i);
742 wq.push(m);
743 }
744 }
745 }
746 #endif
747
748 static ShenandoahBarrierSetC2State* barrier_set_state() {
749 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
750 }
751
752 int ShenandoahBarrierSetC2::estimate_stub_size() const {
753 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
754 assert(stubs->is_empty(), "Lifecycle: no stubs were yet created");
755 return 0;
756 }
757
758 void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
759 MacroAssembler masm(&cb);
760
761 PhaseOutput* const output = Compile::current()->output();
762 assert(masm.offset() <= output->buffer_sizing_data()->_code,
763 "Stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
764 barrier_set_state()->set_stubs_start_offset(masm.offset());
765 barrier_set_state()->set_save_slots_stack_offset(output->gc_barrier_save_slots_offset_in_bytes());
766
767 // Stub generation counts all stubs as skipped for the sake of inlining policy.
768 // This is critical for performance, check it.
769 #ifdef ASSERT
770 int offset_before = masm.offset();
771 int skipped_before = cb.total_skipped_instructions_size();
772 #endif
773
774 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
775 for (int i = 0; i < stubs->length(); i++) {
776 // Make sure there is enough space in the code buffer
777 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
778 ciEnv::current()->record_failure("CodeCache is full");
779 return;
780 }
781 stubs->at(i)->emit_code(masm);
782 }
783
784 #ifdef ASSERT
785 int offset_after = masm.offset();
786 int skipped_after = cb.total_skipped_instructions_size();
787 assert(offset_after - offset_before == skipped_after - skipped_before,
788 "All stubs are counted as skipped. masm: %d - %d = %d, cb: %d - %d = %d",
789 offset_after, offset_before, offset_after - offset_before,
790 skipped_after, skipped_before, skipped_after - skipped_before);
791 #endif
792
793 masm.flush();
794 }
795
796 void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
797 if (!Compile::current()->output()->in_scratch_emit_size()) {
798 barrier_set_state()->stubs()->append(stub);
799 }
800 }
801
802 void ShenandoahBarrierStubC2::inc_trampoline_stubs_count() {
803 if (!Compile::current()->output()->in_scratch_emit_size()) {
804 barrier_set_state()->inc_trampoline_stubs_count();
805 }
806 }
807
808 int ShenandoahBarrierStubC2::trampoline_stubs_count() {
809 return barrier_set_state()->trampoline_stubs_count();
810 }
811
812 int ShenandoahBarrierStubC2::stubs_start_offset() {
813 return barrier_set_state()->stubs_start_offset();
814 }
815
816 int ShenandoahBarrierStubC2::save_slots_stack_offset() {
817 return barrier_set_state()->save_slots_stack_offset();
818 }
819
820 int ShenandoahBarrierStubC2::push_save_slot() {
821 assert(_save_slots_idx < ShenandoahBarrierSetC2::bsc2()->reserved_slots(), "Enough slots are reserved");
822 return save_slots_stack_offset() + (_save_slots_idx++) * sizeof(address);
823 }
824
825 int ShenandoahBarrierStubC2::pop_save_slot() {
826 assert(_save_slots_idx > 0, "About to underflow");
827 return save_slots_stack_offset() + (--_save_slots_idx) * sizeof(address);
828 }
829
830 ShenandoahBarrierStubC2* ShenandoahBarrierStubC2::create(const MachNode* node, Register obj, Address addr, bool narrow, bool do_load, int offset) {
831 auto* stub = new (Compile::current()->comp_arena()) ShenandoahBarrierStubC2(node, obj, addr, narrow, do_load, offset);
832 ShenandoahBarrierStubC2::register_stub(stub);
833 return stub;
834 }
835
836 address ShenandoahBarrierStubC2::keepalive_runtime_entry_addr(SaveMode save_mode) {
837 switch (save_mode) {
838 case SaveMode::Nothing:
839 return SharedRuntime::shenandoah_keepalive_none();
840 case SaveMode::GP:
841 return SharedRuntime::shenandoah_keepalive_gp();
842 case SaveMode::All:
843 return SharedRuntime::shenandoah_keepalive_all();
844 }
845 ShouldNotReachHere();
846 return nullptr;
847 }
848
849 address ShenandoahBarrierStubC2::lrb_runtime_entry_addr(SaveMode save_mode) {
850 bool is_strong = (_node->barrier_data() & ShenandoahBitStrong) != 0;
851 bool is_weak = (_node->barrier_data() & ShenandoahBitWeak) != 0;
852 bool is_phantom = (_node->barrier_data() & ShenandoahBitPhantom) != 0;
853
854 switch (save_mode) {
855 case SaveMode::Nothing: {
856 if (_narrow) {
857 if (is_strong) {
858 return SharedRuntime::shenandoah_lrb_strong_narrow_none();
859 } else if (is_weak) {
860 return SharedRuntime::shenandoah_lrb_weak_narrow_none();
861 } else if (is_phantom) {
862 return SharedRuntime::shenandoah_lrb_phantom_narrow_none();
863 }
864 } else {
865 if (is_strong) {
866 return SharedRuntime::shenandoah_lrb_strong_none();
867 } else if (is_weak) {
868 return SharedRuntime::shenandoah_lrb_weak_none();
869 } else if (is_phantom) {
870 return SharedRuntime::shenandoah_lrb_phantom_none();
871 }
872 }
873 break;
874 }
875
876 case SaveMode::GP: {
877 if (_narrow) {
878 if (is_strong) {
879 return SharedRuntime::shenandoah_lrb_strong_narrow_gp();
880 } else if (is_weak) {
881 return SharedRuntime::shenandoah_lrb_weak_narrow_gp();
882 } else if (is_phantom) {
883 return SharedRuntime::shenandoah_lrb_phantom_narrow_gp();
884 }
885 } else {
886 if (is_strong) {
887 return SharedRuntime::shenandoah_lrb_strong_gp();
888 } else if (is_weak) {
889 return SharedRuntime::shenandoah_lrb_weak_gp();
890 } else if (is_phantom) {
891 return SharedRuntime::shenandoah_lrb_phantom_gp();
892 }
893 }
894 break;
895 }
896
897 case SaveMode::All: {
898 if (_narrow) {
899 if (is_strong) {
900 return SharedRuntime::shenandoah_lrb_strong_narrow_all();
901 } else if (is_weak) {
902 return SharedRuntime::shenandoah_lrb_weak_narrow_all();
903 } else if (is_phantom) {
904 return SharedRuntime::shenandoah_lrb_phantom_narrow_all();
905 }
906 } else {
907 if (is_strong) {
908 return SharedRuntime::shenandoah_lrb_strong_all();
909 } else if (is_weak) {
910 return SharedRuntime::shenandoah_lrb_weak_all();
911 } else if (is_phantom) {
912 return SharedRuntime::shenandoah_lrb_phantom_all();
913 }
914 }
915 break;
916 }
917 }
918
919 ShouldNotReachHere();
920 return nullptr;
921 }
922
923 bool ShenandoahBarrierSetC2State::needs_liveness_data(const MachNode* mach) const {
924 // Nodes that require slow-path stubs need liveness data.
925 return ShenandoahBarrierStubC2::needs_slow_barrier(mach);
926 }
927
928 bool ShenandoahBarrierSetC2State::needs_livein_data() const {
929 return true;
930 }