1 /*
2 * Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/escape.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/idealKit.hpp"
38 #include "opto/macro.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/output.hpp"
41 #include "opto/rootnode.hpp"
42 #include "opto/runtime.hpp"
43
44 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
45 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
46 }
47
48 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
49 BarrierSetC2State(comp_arena),
50 _stubs(new (comp_arena) GrowableArray<ShenandoahBarrierStubC2*>(comp_arena, 8, 0, nullptr)),
51 _trampoline_stubs_count(0),
52 _stubs_start_offset(0) {
53 }
54
55 static void set_barrier_data(C2Access& access, bool load, bool store) {
56 if (!access.is_oop()) {
57 return;
58 }
59
60 DecoratorSet decorators = access.decorators();
61 bool tightly_coupled = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
62 bool in_heap = (decorators & IN_HEAP) != 0;
63 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
64 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
65
66 if (tightly_coupled) {
67 access.set_barrier_data(ShenandoahBitElided);
68 return;
69 }
70
71 uint8_t barrier_data = 0;
72
73 if (load) {
74 if (ShenandoahLoadRefBarrier) {
75 if (on_phantom) {
76 barrier_data |= ShenandoahBitPhantom;
77 } else if (on_weak) {
78 barrier_data |= ShenandoahBitWeak;
79 } else {
80 barrier_data |= ShenandoahBitStrong;
81 }
82 }
83 }
84
85 if (store) {
86 if (ShenandoahSATBBarrier) {
87 barrier_data |= ShenandoahBitKeepAlive;
88 }
89 if (ShenandoahCardBarrier && in_heap) {
90 barrier_data |= ShenandoahBitCardMark;
91 }
92 }
93
94 if (!in_heap) {
95 barrier_data |= ShenandoahBitNative;
96 }
97
98 access.set_barrier_data(barrier_data);
99 }
100
101 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
102 // 1: Non-reference load, no additional barrier is needed
103 if (!access.is_oop()) {
104 return BarrierSetC2::load_at_resolved(access, val_type);
105 }
106
107 // 2. Set barrier data for load
108 set_barrier_data(access, /* load = */ true, /* store = */ false);
109
110 // 3. Correction: If we are reading the value of the referent field of
111 // a Reference object, we need to record the referent resurrection.
112 DecoratorSet decorators = access.decorators();
113 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
114 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
115 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
116 bool needs_keepalive = ((on_weak || on_phantom) && !no_keepalive);
117 if (needs_keepalive) {
118 uint8_t barriers = access.barrier_data() | (ShenandoahSATBBarrier ? ShenandoahBitKeepAlive : 0);
119 access.set_barrier_data(barriers);
120 }
121
122 return BarrierSetC2::load_at_resolved(access, val_type);
123 }
124
125 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
126 // 1: Non-reference store, no additional barrier is needed
127 if (!access.is_oop()) {
128 return BarrierSetC2::store_at_resolved(access, val);
129 }
130
131 // 2. Set barrier data for store
132 set_barrier_data(access, /* load = */ false, /* store = */ true);
133
134 // 3. Correction: avoid keep-alive barriers that should not do keep-alive.
135 DecoratorSet decorators = access.decorators();
136 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
137 if (no_keepalive) {
138 access.set_barrier_data(access.barrier_data() & ~ShenandoahBitKeepAlive);
139 }
140
141 return BarrierSetC2::store_at_resolved(access, val);
142 }
143
144 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
145 Node* new_val, const Type* value_type) const {
146 set_barrier_data(access, /* load = */ true, /* store = */ true);
147 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
148 }
149
150 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
151 Node* new_val, const Type* value_type) const {
152 set_barrier_data(access, /* load = */ true, /* store = */ true);
153 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
154 }
155
156 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
157 set_barrier_data(access, /* load = */ true, /* store = */ true);
158 return BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
159 }
160
161 bool ShenandoahBarrierSetC2::is_Load(int opcode) {
162 switch (opcode) {
163 case Op_LoadN:
164 case Op_LoadP:
165 return true;
166 default:
167 return false;
168 }
169 }
170
171 bool ShenandoahBarrierSetC2::is_Store(int opcode) {
172 switch (opcode) {
173 case Op_StoreN:
174 case Op_StoreP:
175 return true;
176 default:
177 return false;
178 }
179 }
180
181 bool ShenandoahBarrierSetC2::is_LoadStore(int opcode) {
182 switch (opcode) {
183 case Op_CompareAndExchangeN:
184 case Op_CompareAndExchangeP:
185 case Op_WeakCompareAndSwapN:
186 case Op_WeakCompareAndSwapP:
187 case Op_CompareAndSwapN:
188 case Op_CompareAndSwapP:
189 case Op_GetAndSetP:
190 case Op_GetAndSetN:
191 return true;
192 default:
193 return false;
194 }
195 }
196
197 bool ShenandoahBarrierSetC2::can_remove_load_barrier(Node* root) {
198 // Check if all outs feed into nodes that do not expose the oops to the rest
199 // of the runtime system. In this case, we can elide the LRB barrier. We bail
200 // out with false at the first sight of trouble.
201
202 ResourceMark rm;
203 VectorSet visited;
204 Node_List worklist;
205 worklist.push(root);
206
207 while (worklist.size() > 0) {
208 Node* n = worklist.pop();
209 if (visited.test_set(n->_idx)) {
210 continue;
211 }
212
213 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
214 Node* out = n->fast_out(i);
215 switch (out->Opcode()) {
216 case Op_Phi:
217 case Op_EncodeP:
218 case Op_DecodeN:
219 case Op_CastPP:
220 case Op_CheckCastPP:
221 case Op_AddP: {
222 // Transitive node, check if any other outs are doing anything troublesome.
223 worklist.push(out);
224 break;
225 }
226
227 case Op_LoadRange: {
228 // Array length is the same in all copies.
229 break;
230 }
231
232 case Op_LoadKlass: {
233 // Klass is the same in all copies.
234 // We would have liked to assert -UCOH, but there are legitimate klass
235 // loads from native Klass* instances, which are also safe under +UCOH.
236 break;
237 }
238
239 case Op_LoadNKlass: {
240 // Similar to above, but LoadNKlass is only safe without +UCOH.
241 // With +UCOH, it loads from mark word, which clashes with forwarding pointers.
242 if (!UseCompactObjectHeaders) {
243 break;
244 }
245 return false;
246 }
247
248 case Op_CmpN: {
249 if (out->in(1) == n &&
250 out->in(2)->Opcode() == Op_ConN &&
251 out->in(2)->get_narrowcon() == 0) {
252 // Null check, no oop is exposed.
253 break;
254 }
255 if (out->in(2) == n &&
256 out->in(1)->Opcode() == Op_ConN &&
257 out->in(1)->get_narrowcon() == 0) {
258 // Null check, no oop is exposed.
259 break;
260 }
261 return false;
262 }
263
264 case Op_CmpP: {
265 if (out->in(1) == n &&
266 out->in(2)->Opcode() == Op_ConP &&
267 out->in(2)->get_ptr() == 0) {
268 // Null check, no oop is exposed.
269 break;
270 }
271 if (out->in(2) == n &&
272 out->in(1)->Opcode() == Op_ConP &&
273 out->in(1)->get_ptr() == 0) {
274 // Null check, no oop is exposed.
275 break;
276 }
277 return false;
278 }
279
280 case Op_CallStaticJava: {
281 if (out->as_CallStaticJava()->is_uncommon_trap()) {
282 // Local feeds into uncommon trap. Deopt machinery handles barriers itself.
283 break;
284 }
285 return false;
286 }
287
288 default: {
289 // Paranoidly distrust any other nodes.
290 return false;
291 }
292 }
293 }
294 }
295
296 // Nothing troublesome found.
297 return true;
298 }
299
300 uint8_t ShenandoahBarrierSetC2::refine_load(Node* n, uint8_t bd) {
301 assert(ShenandoahElideIdealBarriers, "Checked by caller");
302 assert(bd != 0, "Checked by caller");
303
304 // Do not touch weak loads at all: they are responsible for shielding from
305 // Reference.referent resurrection.
306 if ((bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0) {
307 return bd;
308 }
309
310 if (((bd & ShenandoahBitStrong) != 0) && can_remove_load_barrier(n)) {
311 bd &= ~ShenandoahBitStrong;
312 }
313
314 return bd;
315 }
316
317 uint8_t ShenandoahBarrierSetC2::refine_store(Node* n, uint8_t bd) {
318 assert(ShenandoahElideIdealBarriers, "Checked by caller");
319 assert(bd != 0, "Checked by caller");
320 assert(n->is_Mem() || n->is_LoadStore(), "Sanity");
321
322 const Node* newval = n->in(MemNode::ValueIn);
323 assert(newval != nullptr, "Should be present");
324
325 // Type system tells us something about nullity?
326 const Type* newval_bottom = newval->bottom_type();
327 assert(newval_bottom->isa_oopptr() || newval_bottom->isa_narrowoop() ||
328 newval_bottom == TypePtr::NULL_PTR, "Should be an oop store");
329 const TypePtr* newval_type = newval_bottom->make_ptr();
330 assert(newval_type != nullptr, "Should have been filtered before");
331 TypePtr::PTR newval_type_ptr = newval_type->ptr();
332 if (newval_type_ptr == TypePtr::Null) {
333 bd &= ~ShenandoahBitNotNull;
334 // Card table barrier is not needed if we store null.
335 bd &= ~ShenandoahBitCardMark;
336 } else if (newval_type_ptr == TypePtr::NotNull) {
337 // Definitely not null.
338 bd |= ShenandoahBitNotNull;
339 }
340
341 return bd;
342 }
343
344 void ShenandoahBarrierSetC2::final_refinement(Compile* compile) const {
345 ResourceMark rm;
346 Unique_Node_List wq;
347
348 RootNode* root = compile->root();
349 wq.push(root);
350
351 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
352 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
353 Node* m = root->fast_out(i);
354 wq.push(m);
355 }
356
357 for (uint next = 0; next < wq.size(); next++) {
358 Node* n = wq.at(next);
359
360 assert(!n->is_Mach(), "No Mach nodes here yet");
361
362 int opc = n->Opcode();
363 bool is_load = is_Load(opc);
364 bool is_store = is_Store(opc);
365 bool is_load_store = is_LoadStore(opc);
366
367 uint8_t orig_bd = 0;
368 if (is_load_store) {
369 orig_bd = n->as_LoadStore()->barrier_data();
370 } else if (is_load || is_store) {
371 orig_bd = n->as_Mem()->barrier_data();
372 }
373
374 uint8_t bd = orig_bd;
375 if (ShenandoahElideIdealBarriers && bd != 0) {
376 // Note: we cannot apply load optimizations to LoadStores,
377 // because their load barriers are needed for fixups.
378 if (is_load) {
379 bd = refine_load(n, bd);
380 }
381 if (is_store || is_load_store) {
382 bd = refine_store(n, bd);
383 }
384 }
385
386 // If there are no real barrier flags on the node, strip away additional fluff.
387 // Matcher does not care about this, and we would like to avoid invoking "barrier_data() != 0"
388 // rules when the only flags are the irrelevant fluff.
389 if ((bd != 0) && (bd & ShenandoahBitsReal) == 0) {
390 bd = 0;
391 }
392
393 if (bd != orig_bd) {
394 if (is_load_store) {
395 n->as_LoadStore()->set_barrier_data(bd);
396 } else {
397 n->as_Mem()->set_barrier_data(bd);
398 }
399 }
400
401 for (uint j = 0; j < n->req(); j++) {
402 Node* in = n->in(j);
403 if (in != nullptr) {
404 wq.push(in);
405 }
406 }
407 }
408 }
409
410 // Support for macro expanded GC barriers
411 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
412 if (node->is_LoadStore()) {
413 LoadStoreNode* loadstore = node->as_LoadStore();
414 loadstore->set_barrier_data(0);
415 } else if (node->is_Mem()) {
416 MemNode* mem = node->as_Mem();
417 mem->set_barrier_data(0);
418 }
419 }
420
421 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
422 eliminate_gc_barrier_data(node);
423 }
424
425 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
426 uint8_t orig_bd = node->barrier_data();
427 if (orig_bd == 0) {
428 // Nothing to do.
429 return;
430 }
431
432 uint8_t bd = orig_bd;
433 int node_opcode = node->ideal_Opcode();
434
435 if (dominator == nullptr) {
436 // Must be allocation node.
437 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
438 // Loads from recent allocations do not need LRBs.
439 bd &= ~ShenandoahBitStrong;
440 }
441 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
442 // Stores to recent allocations do not need KA or CM.
443 bd &= ~ShenandoahBitKeepAlive;
444 bd &= ~ShenandoahBitCardMark;
445 }
446 } else {
447 // LoadStores do not get these optimizations, since their LRBs
448 // are required for fixups.
449 if (is_Load(node_opcode) || is_Store(node_opcode)) {
450 int dom_opcode = dominator->ideal_Opcode();
451 uint8_t dom_bd = dominator->barrier_data();
452
453 if (is_Load(dom_opcode) || is_LoadStore(dom_opcode)) {
454 // If dominating load is set up to perform LRB fixups, no further LRB is needed.
455 if ((dom_bd & ShenandoahBitStrong) != 0) {
456 bd &= ~ShenandoahBitStrong;
457 }
458 }
459 if (is_Store(dom_opcode)) {
460 // Dominating store has stored the good ref, no LRB is needed.
461 bd &= ~ShenandoahBitStrong;
462 }
463 }
464 }
465
466 if (orig_bd != bd) {
467 // We are already in final output.
468 // Strip the extra barrier data if no real bits are left.
469 if ((bd & ShenandoahBitsReal) != 0) {
470 node->set_barrier_data(bd);
471 } else {
472 node->set_barrier_data(0);
473 }
474 }
475 }
476
477 void ShenandoahBarrierSetC2::analyze_dominating_barriers() const {
478 if (!ShenandoahElideMachBarriers) {
479 return;
480 }
481
482 ResourceMark rm;
483 Node_List accesses, dominators;
484
485 PhaseCFG* const cfg = Compile::current()->cfg();
486 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
487 const Block* const block = cfg->get_block(i);
488 for (uint j = 0; j < block->number_of_nodes(); ++j) {
489 Node* const node = block->get_node(j);
490
491 // Everything that happens in allocations does not need barriers.
492 // Record them for dominance analysis.
493 if (node->is_Phi() && is_allocation(node)) {
494 dominators.push(node);
495 continue;
496 }
497
498 if (!node->is_Mach()) {
499 continue;
500 }
501
502 MachNode* const mach = node->as_Mach();
503 int opcode = mach->ideal_Opcode();
504 if (is_Load(opcode) || is_Store(opcode) || is_LoadStore(opcode)) {
505 if ((mach->barrier_data() & ShenandoahBitsReal) != 0) {
506 accesses.push(mach);
507 dominators.push(mach);
508 }
509 }
510 }
511 }
512
513 elide_dominated_barriers(accesses, dominators);
514 }
515
516 uint ShenandoahBarrierSetC2::estimated_barrier_size(const Node* node) const {
517 // Barrier impact on fast-path is driven by GC state checks emitted very late.
518 // These checks are tight load-test-branch sequences, with no impact on C2 graph
519 // size. Limiting unrolling in presence of GC barriers might turn some loops
520 // tighter than with default unrolling, which may benefit performance due to denser
521 // code. Testing shows it is still counter-productive.
522 // Therefore, we report zero barrier size to let C2 do its normal thing.
523 return 0;
524 }
525
526 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const {
527 bool is_oop = is_reference_type(type);
528 if (!is_oop) {
529 return false;
530 }
531 if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
532 if (phase == Optimization) {
533 return false;
534 }
535 return !is_clone;
536 }
537 return true;
538 }
539
540 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
541 if (!ShenandoahCloneBarrier) {
542 return false;
543 }
544
545 if (src_type->isa_instptr() != nullptr) {
546 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
547 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
548 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
549 !ik->has_injected_fields() && !ik->has_object_fields()) {
550 if (!src_type->klass_is_exact()) {
551 // Class is *currently* the leaf in the hierarchy.
552 // Record the dependency so that we deopt if this does not hold in future.
553 Compile::current()->dependencies()->assert_leaf_type(ik);
554 }
555 return false;
556 }
557 } else if (src_type->isa_aryptr() != nullptr) {
558 // Array: need barrier only if array is oop-bearing.
559 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
560 if (is_reference_type(src_elem, true)) {
561 is_oop_array = true;
562 } else {
563 return false;
564 }
565 }
566
567 // Assume the worst.
568 return true;
569 }
570
571 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
572 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
573
574 bool is_oop_array = false;
575 if (!clone_needs_barrier(src_type, is_oop_array)) {
576 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
577 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
578 return;
579 }
580
581 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
582 // Looks like an instance? Prepare the instance clone. This would either
583 // be exploded into individual accesses or be left as runtime call.
584 // Common BarrierSetC2 prepares everything for both cases.
585 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
586 return;
587 }
588
589 // We are cloning the oop array. Prepare to call the normal arraycopy stub
590 // after the expansion. Normal stub takes the number of actual type-sized
591 // elements to copy after the base, compute the count here.
592 Node* offset = kit->MakeConX(arrayOopDesc::base_offset_in_bytes(UseCompressedOops ? T_NARROWOOP : T_OBJECT));
593 size = kit->gvn().transform(new SubXNode(size, offset));
594 size = kit->gvn().transform(new URShiftXNode(size, kit->intcon(LogBytesPerHeapOop)));
595 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, size, true, false);
596 ac->set_clone_array();
597 Node* n = kit->gvn().transform(ac);
598 if (n == ac) {
599 ac->set_adr_type(TypeRawPtr::BOTTOM);
600 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), TypeRawPtr::BOTTOM);
601 } else {
602 kit->set_all_memory(n);
603 }
604 }
605
606 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
607 Node* const ctrl = ac->in(TypeFunc::Control);
608 Node* const mem = ac->in(TypeFunc::Memory);
609 Node* const src = ac->in(ArrayCopyNode::Src);
610 Node* const src_offset = ac->in(ArrayCopyNode::SrcPos);
611 Node* const dest = ac->in(ArrayCopyNode::Dest);
612 Node* const dest_offset = ac->in(ArrayCopyNode::DestPos);
613 Node* length = ac->in(ArrayCopyNode::Length);
614
615 const TypeOopPtr* src_type = phase->igvn().type(src)->is_oopptr();
616
617 bool is_oop_array = false;
618 if (!clone_needs_barrier(src_type, is_oop_array)) {
619 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
620 BarrierSetC2::clone_at_expansion(phase, ac);
621 return;
622 }
623
624 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
625 // Still looks like an instance? Likely a large instance or reflective
626 // clone with unknown length. Go to runtime and handle it there.
627 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
628 return;
629 }
630
631 // We are cloning the oop array. Call into normal oop array copy stubs.
632 // Those stubs would call BarrierSetAssembler to handle GC barriers.
633
634 // This is the full clone, so offsets should equal each other and be at array base.
635 assert(src_offset == dest_offset, "should be equal");
636 const jlong offset = src_offset->get_long();
637 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
638 BasicType bt = ary_ptr->elem()->array_element_basic_type();
639 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
640
641 const char* copyfunc_name = "arraycopy";
642 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
643
644 Node* const call = phase->make_leaf_call(ctrl, mem,
645 OptoRuntime::fast_arraycopy_Type(),
646 copyfunc_addr, copyfunc_name,
647 TypeRawPtr::BOTTOM,
648 phase->basic_plus_adr(src, src_offset),
649 phase->basic_plus_adr(dest, dest_offset),
650 length,
651 phase->top()
652 );
653 phase->transform_later(call);
654
655 phase->igvn().replace_node(ac, call);
656 }
657
658 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
659 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
660 }
661
662 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
663 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
664 }
665
666 void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
667 os->print(" Node barriers: ");
668 if ((data & ShenandoahBitStrong) != 0) {
669 data &= ~ShenandoahBitStrong;
670 os->print("strong ");
671 }
672
673 if ((data & ShenandoahBitWeak) != 0) {
674 data &= ~ShenandoahBitWeak;
675 os->print("weak ");
676 }
677
678 if ((data & ShenandoahBitPhantom) != 0) {
679 data &= ~ShenandoahBitPhantom;
680 os->print("phantom ");
681 }
682
683 if ((data & ShenandoahBitKeepAlive) != 0) {
684 data &= ~ShenandoahBitKeepAlive;
685 os->print("keepalive ");
686 }
687
688 if ((data & ShenandoahBitCardMark) != 0) {
689 data &= ~ShenandoahBitCardMark;
690 os->print("cardmark ");
691 }
692
693 if ((data & ShenandoahBitNative) != 0) {
694 data &= ~ShenandoahBitNative;
695 os->print("native ");
696 }
697
698 if ((data & ShenandoahBitNotNull) != 0) {
699 data &= ~ShenandoahBitNotNull;
700 os->print("not-null ");
701 }
702
703 if ((data & ShenandoahBitElided) != 0) {
704 data &= ~ShenandoahBitElided;
705 os->print("elided ");
706 }
707
708 os->cr();
709
710 if (data > 0) {
711 fatal("Unknown bit!");
712 }
713
714 os->print_cr(" GC configuration: %sLRB %sSATB %sCAS %sClone %sCard",
715 (ShenandoahLoadRefBarrier ? "+" : "-"),
716 (ShenandoahSATBBarrier ? "+" : "-"),
717 (ShenandoahCASBarrier ? "+" : "-"),
718 (ShenandoahCloneBarrier ? "+" : "-"),
719 (ShenandoahCardBarrier ? "+" : "-")
720 );
721 }
722
723
724 #ifdef ASSERT
725 void ShenandoahBarrierSetC2::verify_gc_barrier_assert(bool cond, const char* msg, uint8_t bd, Node* n) {
726 if (!cond) {
727 stringStream ss;
728 ss.print_cr("%s", msg);
729 ss.print_cr("-----------------");
730 print_barrier_data(&ss, bd);
731 ss.print_cr("-----------------");
732 n->dump_bfs(1, nullptr, "", &ss);
733 report_vm_error(__FILE__, __LINE__, ss.as_string());
734 }
735 }
736
737 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
738 if (!ShenandoahVerifyOptoBarriers) {
739 return;
740 }
741
742 // Verify depending on the barriers actually enabled, allowing verification in passive mode.
743 // Normally, we have _some_ bits set on all accesses. Optimizations may drop some bits,
744 // but only the last optimization step eliminates all remaining metadata flags. Only then
745 // the access data can be completely blank.
746 bool final_phase = (phase == BeforeCodeGen);
747 bool expect_load_barriers = !final_phase && ShenandoahLoadRefBarrier;
748 bool expect_store_barriers = !final_phase && (ShenandoahSATBBarrier || ShenandoahCardBarrier);
749 bool expect_load_store_barriers = expect_load_barriers || expect_store_barriers;
750 bool expect_some_real = final_phase;
751
752 Unique_Node_List wq;
753
754 RootNode* root = compile->root();
755 wq.push(root);
756
757 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
758 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
759 Node* m = root->fast_out(i);
760 wq.push(m);
761 }
762
763 for (uint next = 0; next < wq.size(); next++) {
764 Node *n = wq.at(next);
765 assert(!n->is_Mach(), "No Mach nodes here yet");
766
767 int opc = n->Opcode();
768
769 uint8_t bd = 0;
770 const TypePtr* adr_type = nullptr;
771 if (is_Load(opc)) {
772 bd = n->as_Load()->barrier_data();
773 adr_type = n->as_Load()->adr_type();
774 } else if (is_Store(opc)) {
775 bd = n->as_Store()->barrier_data();
776 adr_type = n->as_Store()->adr_type();
777 } else if (is_LoadStore(opc)) {
778 bd = n->as_LoadStore()->barrier_data();
779 adr_type = n->as_LoadStore()->adr_type();
780 } else if (n->is_Mem()) {
781 bd = MemNode::barrier_data(n);
782 verify_gc_barrier_assert(bd == 0, "Other mem nodes should have no barrier data", bd, n);
783 }
784
785 bool is_weak = (bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0;
786 bool is_native = (bd & ShenandoahBitNative) != 0;
787
788 bool is_referent = adr_type != nullptr &&
789 adr_type->isa_instptr() &&
790 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
791 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
792
793 bool is_oop_addr = (adr_type != nullptr) && (adr_type->isa_oopptr() || adr_type->isa_narrowoop());
794 bool is_raw_addr = (adr_type != nullptr) && (adr_type->isa_rawptr() || adr_type->isa_klassptr());
795
796 verify_gc_barrier_assert(!expect_some_real || (bd == 0) || (bd & ShenandoahBitsReal) != 0, "Without real barriers, metadata should be stripped at this point", bd, n);
797
798 if (is_oop_addr) {
799 if (is_Load(opc)) {
800 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Oop load should have barrier data", bd, n);
801 verify_gc_barrier_assert(!is_weak || is_referent, "Weak load only for Reference.referent", bd, n);
802 } else if (is_Store(opc)) {
803 // Reference.referent stores can be without barriers.
804 verify_gc_barrier_assert(!expect_store_barriers || is_referent || (bd != 0), "Oop store should have barrier data", bd, n);
805 } else if (is_LoadStore(opc)) {
806 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Oop load-store should have barrier data", bd, n);
807 }
808 } else if (is_raw_addr) {
809 if (is_native) {
810 if (is_Load(opc)) {
811 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Native oop load should have barrier data", bd, n);
812 }
813 if (is_Store(opc)) {
814 verify_gc_barrier_assert(!expect_store_barriers || (bd != 0), "Native oop store should have barrier data", bd, n);
815 }
816 if (is_LoadStore(opc)) {
817 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Native oop load-store should have barrier data", bd, n);
818 }
819 } else {
820 // Some Load/Stores are used for T_ADDRESS and/or raw stores, which are supposed not to have barriers.
821 // Some other Load/Stores are emitted for real oops, but on raw addresses via Unsafe.
822 // The distinction on this level is lost, so we cannot really verify this.
823 }
824 } else {
825 if (is_Load(opc) || is_Store(opc) || is_LoadStore(opc)) {
826 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
827 }
828 }
829
830 for (uint j = 0; j < n->req(); j++) {
831 Node* in = n->in(j);
832 if (in != nullptr) {
833 wq.push(in);
834 }
835 }
836 }
837 }
838 #endif
839
840 static ShenandoahBarrierSetC2State* barrier_set_state() {
841 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
842 }
843
844 int ShenandoahBarrierSetC2::estimate_stub_size() const {
845 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
846 assert(stubs->is_empty(), "Lifecycle: no stubs were yet created");
847 return 0;
848 }
849
850 void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
851 MacroAssembler masm(&cb);
852
853 PhaseOutput* const output = Compile::current()->output();
854 assert(masm.offset() <= output->buffer_sizing_data()->_code,
855 "Stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
856 barrier_set_state()->set_stubs_start_offset(masm.offset());
857
858 // Stub generation counts all stubs as skipped for the sake of inlining policy.
859 // This is critical for performance, check it.
860 #ifdef ASSERT
861 int offset_before = masm.offset();
862 int skipped_before = cb.total_skipped_instructions_size();
863 #endif
864
865 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
866 for (int i = 0; i < stubs->length(); i++) {
867 // Make sure there is enough space in the code buffer
868 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
869 ciEnv::current()->record_failure("CodeCache is full");
870 return;
871 }
872 stubs->at(i)->emit_code(masm);
873 }
874
875 #ifdef ASSERT
876 int offset_after = masm.offset();
877 int skipped_after = cb.total_skipped_instructions_size();
878 assert(offset_after - offset_before == skipped_after - skipped_before,
879 "All stubs are counted as skipped. masm: %d - %d = %d, cb: %d - %d = %d",
880 offset_after, offset_before, offset_after - offset_before,
881 skipped_after, skipped_before, skipped_after - skipped_before);
882 #endif
883
884 masm.flush();
885 }
886
887 void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
888 if (!Compile::current()->output()->in_scratch_emit_size()) {
889 barrier_set_state()->stubs()->append(stub);
890 }
891 }
892
893 ShenandoahBarrierStubC2* ShenandoahBarrierStubC2::create(const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow, bool do_load) {
894 auto* stub = new (Compile::current()->comp_arena()) ShenandoahBarrierStubC2(node, obj, addr, tmp1, tmp2, narrow, do_load);
895 register_stub(stub);
896 return stub;
897 }
898
899 void ShenandoahBarrierStubC2::load_post(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
900 // Load post-barrier:
901 // a. Satisfies the need for LRB for normal loads
902 // b. Passes a weak load through LRB-weak
903 // c. Keep-alives a weak load
904 if (needs_slow_barrier(node)) {
905 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ false);
906 char check = 0;
907 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
908 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
909 check |= needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
910 stub->enter_if_gc_state(*masm, check, tmp1);
911 }
912 }
913
914 void ShenandoahBarrierStubC2::store_pre(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
915 // Store pre-barrier: SATB, keep-alive the current memory value.
916 if (needs_slow_barrier(node)) {
917 assert(!needs_load_ref_barrier(node), "Should not be required for stores");
918 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ true);
919 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING, tmp1);
920 }
921 }
922
923 void ShenandoahBarrierStubC2::load_store_pre(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
924 // Load/Store pre-barrier:
925 // a. Avoids false positives from CAS encountering to-space memory values.
926 // b. Satisfies the need for LRB for the CAE result.
927 // c. Records old value for the sake of SATB.
928 //
929 // (a) and (b) are covered because load barrier does memory location fixup.
930 // (c) is covered by KA on the current memory value.
931 if (needs_slow_barrier(node)) {
932 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ true);
933 char check = 0;
934 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
935 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
936 assert(!needs_load_ref_barrier_weak(node), "Not supported for Load/Stores");
937 stub->enter_if_gc_state(*masm, check, tmp1);
938 }
939 }
940
941 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
942 if (needs_card_barrier(node)) {
943 cardtable(*masm, addr, tmp1, tmp2);
944 }
945 }
946
947 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
948 store_post(masm, node, addr, tmp1, tmp2);
949 }
950
951 bool ShenandoahBarrierStubC2::is_live_register(Register reg) {
952 return preserve_set().member(OptoReg::as_OptoReg(reg->as_VMReg()));
953 }
954
955 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Register skip_reg1) {
956 Register tmp = noreg;
957 Register fallback_live = noreg;
958
959 // Try to select non-live first:
960 for (int i = 0; i < available_gp_registers(); i++) {
961 Register r = as_Register(i);
962 if (r != _obj && r != _addr.base() && r != _addr.index() &&
963 r != skip_reg1 && !is_special_register(r)) {
964 if (!is_live_register(r)) {
965 tmp = r;
966 break;
967 } else if (fallback_live == noreg) {
968 fallback_live = r;
969 }
970 }
971 }
972
973 // If we could not find a non-live register, select the live fallback:
974 if (tmp == noreg) {
975 tmp = fallback_live;
976 selected_live = true;
977 } else {
978 selected_live = false;
979 }
980
981 assert(tmp != noreg, "successfully selected");
982 assert_different_registers(tmp, skip_reg1);
983 assert_different_registers(tmp, _obj);
984 assert_different_registers(tmp, _addr.base());
985 assert_different_registers(tmp, _addr.index());
986 return tmp;
987 }
988
989 address ShenandoahBarrierStubC2::keepalive_runtime_entry_addr() {
990 if (_narrow) {
991 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_narrow);
992 } else {
993 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre);
994 }
995 }
996
997 address ShenandoahBarrierStubC2::lrb_runtime_entry_addr() {
998 bool is_strong = (_node->barrier_data() & ShenandoahBitStrong) != 0;
999 bool is_weak = (_node->barrier_data() & ShenandoahBitWeak) != 0;
1000 bool is_phantom = (_node->barrier_data() & ShenandoahBitPhantom) != 0;
1001
1002 if (_narrow) {
1003 if (is_strong) {
1004 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow_narrow);
1005 } else if (is_weak) {
1006 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow_narrow);
1007 } else if (is_phantom) {
1008 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow_narrow);
1009 }
1010 } else {
1011 if (is_strong) {
1012 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1013 } else if (is_weak) {
1014 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1015 } else if (is_phantom) {
1016 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1017 }
1018 }
1019
1020 ShouldNotReachHere();
1021 return nullptr;
1022 }
1023
1024 bool ShenandoahBarrierSetC2State::needs_liveness_data(const MachNode* mach) const {
1025 // Nodes that require slow-path stubs need liveness data.
1026 return ShenandoahBarrierStubC2::needs_slow_barrier(mach);
1027 }
1028
1029 bool ShenandoahBarrierSetC2State::needs_livein_data() const {
1030 return true;
1031 }