1 /*
2 * Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "opto/arraycopynode.hpp"
34 #include "opto/escape.hpp"
35 #include "opto/graphKit.hpp"
36 #include "opto/idealKit.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/narrowptrnode.hpp"
39 #include "opto/output.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42
43 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
44 BarrierSetC2State(comp_arena),
45 _stubs(new (comp_arena) GrowableArray<ShenandoahBarrierStubC2*>(comp_arena, 8, 0, nullptr)),
46 _trampoline_stubs_count(0),
47 _stubs_start_offset(0),
48 _stubs_current_total_size(0) {
49 }
50
51 static void set_barrier_data(C2Access& access, bool load, bool store) {
52 if (!access.is_oop()) {
53 return;
54 }
55
56 DecoratorSet decorators = access.decorators();
57 bool tightly_coupled = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
58 bool in_heap = (decorators & IN_HEAP) != 0;
59 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
60 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
61
62 if (tightly_coupled) {
63 access.set_barrier_data(ShenandoahBitElided);
64 return;
65 }
66
67 uint8_t barrier_data = 0;
68
69 if (load) {
70 if (ShenandoahLoadRefBarrier) {
71 if (on_phantom) {
72 barrier_data |= ShenandoahBitPhantom;
73 } else if (on_weak) {
74 barrier_data |= ShenandoahBitWeak;
75 } else {
76 barrier_data |= ShenandoahBitStrong;
77 }
78 }
79 }
80
81 if (store) {
82 if (ShenandoahSATBBarrier) {
83 barrier_data |= ShenandoahBitKeepAlive;
84 }
85 if (ShenandoahCardBarrier && in_heap) {
86 barrier_data |= ShenandoahBitCardMark;
87 }
88 }
89
90 if (!in_heap) {
91 barrier_data |= ShenandoahBitNative;
92 }
93
94 access.set_barrier_data(barrier_data);
95 }
96
97 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
98 // 1: Non-reference load, no additional barrier is needed
99 if (!access.is_oop()) {
100 return BarrierSetC2::load_at_resolved(access, val_type);
101 }
102
103 // 2. Set barrier data for load
104 set_barrier_data(access, /* load = */ true, /* store = */ false);
105
106 // 3. Correction: If we are reading the value of the referent field of
107 // a Reference object, we need to record the referent resurrection.
108 DecoratorSet decorators = access.decorators();
109 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
110 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
111 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
112 bool needs_keepalive = ((on_weak || on_phantom) && !no_keepalive);
113 if (needs_keepalive) {
114 uint8_t barriers = access.barrier_data() | (ShenandoahSATBBarrier ? ShenandoahBitKeepAlive : 0);
115 access.set_barrier_data(barriers);
116 }
117
118 return BarrierSetC2::load_at_resolved(access, val_type);
119 }
120
121 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
122 // 1: Non-reference store, no additional barrier is needed
123 if (!access.is_oop()) {
124 return BarrierSetC2::store_at_resolved(access, val);
125 }
126
127 // 2. Set barrier data for store
128 set_barrier_data(access, /* load = */ false, /* store = */ true);
129
130 // 3. Correction: avoid keep-alive barriers that should not do keep-alive.
131 DecoratorSet decorators = access.decorators();
132 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
133 if (no_keepalive) {
134 access.set_barrier_data(access.barrier_data() & ~ShenandoahBitKeepAlive);
135 }
136
137 return BarrierSetC2::store_at_resolved(access, val);
138 }
139
140 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
141 Node* new_val, const Type* value_type) const {
142 set_barrier_data(access, /* load = */ true, /* store = */ true);
143 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
144 }
145
146 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
147 Node* new_val, const Type* value_type) const {
148 set_barrier_data(access, /* load = */ true, /* store = */ true);
149 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
150 }
151
152 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
153 set_barrier_data(access, /* load = */ true, /* store = */ true);
154 return BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
155 }
156
157 bool ShenandoahBarrierSetC2::is_Load(int opcode) {
158 switch (opcode) {
159 case Op_LoadN:
160 case Op_LoadP:
161 return true;
162 default:
163 return false;
164 }
165 }
166
167 bool ShenandoahBarrierSetC2::is_Store(int opcode) {
168 switch (opcode) {
169 case Op_StoreN:
170 case Op_StoreP:
171 return true;
172 default:
173 return false;
174 }
175 }
176
177 bool ShenandoahBarrierSetC2::is_LoadStore(int opcode) {
178 switch (opcode) {
179 case Op_CompareAndExchangeN:
180 case Op_CompareAndExchangeP:
181 case Op_WeakCompareAndSwapN:
182 case Op_WeakCompareAndSwapP:
183 case Op_CompareAndSwapN:
184 case Op_CompareAndSwapP:
185 case Op_GetAndSetP:
186 case Op_GetAndSetN:
187 return true;
188 default:
189 return false;
190 }
191 }
192
193 bool ShenandoahBarrierSetC2::can_remove_load_barrier(Node* root) {
194 // Check if all outs feed into nodes that do not expose the oops to the rest
195 // of the runtime system. In this case, we can elide the LRB barrier. We bail
196 // out with false at the first sight of trouble.
197
198 ResourceMark rm;
199 VectorSet visited;
200 Node_List worklist;
201 worklist.push(root);
202
203 while (worklist.size() > 0) {
204 Node* n = worklist.pop();
205 if (visited.test_set(n->_idx)) {
206 continue;
207 }
208
209 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
210 Node* out = n->fast_out(i);
211 switch (out->Opcode()) {
212 case Op_Phi:
213 case Op_EncodeP:
214 case Op_DecodeN:
215 case Op_CastPP:
216 case Op_CheckCastPP:
217 case Op_AddP: {
218 // Transitive node, check if any other outs are doing anything troublesome.
219 worklist.push(out);
220 break;
221 }
222
223 case Op_LoadRange: {
224 // Array length is the same in all copies.
225 break;
226 }
227
228 case Op_LoadKlass: {
229 // Klass is the same in all copies.
230 // We would have liked to assert -UCOH, but there are legitimate klass
231 // loads from native Klass* instances, which are also safe under +UCOH.
232 break;
233 }
234
235 case Op_LoadNKlass: {
236 // Similar to above, but LoadNKlass is only safe without +UCOH.
237 // With +UCOH, it loads from mark word, which clashes with forwarding pointers.
238 if (!UseCompactObjectHeaders) {
239 break;
240 }
241 return false;
242 }
243
244 case Op_CmpN: {
245 if (out->in(1) == n &&
246 out->in(2)->Opcode() == Op_ConN &&
247 out->in(2)->get_narrowcon() == 0) {
248 // Null check, no oop is exposed.
249 break;
250 }
251 if (out->in(2) == n &&
252 out->in(1)->Opcode() == Op_ConN &&
253 out->in(1)->get_narrowcon() == 0) {
254 // Null check, no oop is exposed.
255 break;
256 }
257 return false;
258 }
259
260 case Op_CmpP: {
261 if (out->in(1) == n &&
262 out->in(2)->Opcode() == Op_ConP &&
263 out->in(2)->get_ptr() == 0) {
264 // Null check, no oop is exposed.
265 break;
266 }
267 if (out->in(2) == n &&
268 out->in(1)->Opcode() == Op_ConP &&
269 out->in(1)->get_ptr() == 0) {
270 // Null check, no oop is exposed.
271 break;
272 }
273 return false;
274 }
275
276 case Op_CallStaticJava: {
277 if (out->as_CallStaticJava()->is_uncommon_trap()) {
278 // Local feeds into uncommon trap. Deopt machinery handles barriers itself.
279 break;
280 }
281 return false;
282 }
283
284 default: {
285 // Paranoidly distrust any other nodes.
286 return false;
287 }
288 }
289 }
290 }
291
292 // Nothing troublesome found.
293 return true;
294 }
295
296 uint8_t ShenandoahBarrierSetC2::refine_load(Node* n, uint8_t bd) {
297 assert(ShenandoahElideIdealBarriers, "Checked by caller");
298 assert(bd != 0, "Checked by caller");
299
300 // Do not touch weak loads at all: they are responsible for shielding from
301 // Reference.referent resurrection.
302 if ((bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0) {
303 return bd;
304 }
305
306 if (((bd & ShenandoahBitStrong) != 0) && can_remove_load_barrier(n)) {
307 bd &= ~ShenandoahBitStrong;
308 }
309
310 return bd;
311 }
312
313 uint8_t ShenandoahBarrierSetC2::refine_store(Node* n, uint8_t bd) {
314 assert(ShenandoahElideIdealBarriers, "Checked by caller");
315 assert(bd != 0, "Checked by caller");
316 assert(n->is_Mem() || n->is_LoadStore(), "Sanity");
317
318 const Node* newval = n->in(MemNode::ValueIn);
319 assert(newval != nullptr, "Should be present");
320
321 // Type system tells us something about nullity?
322 const Type* newval_bottom = newval->bottom_type();
323 assert(newval_bottom->isa_oopptr() || newval_bottom->isa_narrowoop() ||
324 newval_bottom == TypePtr::NULL_PTR, "Should be an oop store");
325 const TypePtr* newval_type = newval_bottom->make_ptr();
326 assert(newval_type != nullptr, "Should have been filtered before");
327 TypePtr::PTR newval_type_ptr = newval_type->ptr();
328 if (newval_type_ptr == TypePtr::Null) {
329 bd &= ~ShenandoahBitNotNull;
330 // Card table barrier is not needed if we store null.
331 bd &= ~ShenandoahBitCardMark;
332 } else if (newval_type_ptr == TypePtr::NotNull) {
333 // Definitely not null.
334 bd |= ShenandoahBitNotNull;
335 }
336
337 return bd;
338 }
339
340 void ShenandoahBarrierSetC2::final_refinement(Compile* compile) const {
341 ResourceMark rm;
342 Unique_Node_List wq;
343
344 RootNode* root = compile->root();
345 wq.push(root);
346
347 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
348 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
349 Node* m = root->fast_out(i);
350 wq.push(m);
351 }
352
353 for (uint next = 0; next < wq.size(); next++) {
354 Node* n = wq.at(next);
355
356 assert(!n->is_Mach(), "No Mach nodes here yet");
357
358 int opc = n->Opcode();
359 bool is_load = is_Load(opc);
360 bool is_store = is_Store(opc);
361 bool is_load_store = is_LoadStore(opc);
362
363 uint8_t orig_bd = 0;
364 if (is_load_store) {
365 orig_bd = n->as_LoadStore()->barrier_data();
366 } else if (is_load || is_store) {
367 orig_bd = n->as_Mem()->barrier_data();
368 }
369
370 uint8_t bd = orig_bd;
371 if (ShenandoahElideIdealBarriers && bd != 0) {
372 // Note: we cannot apply load optimizations to LoadStores,
373 // because their load barriers are needed for fixups.
374 if (is_load) {
375 bd = refine_load(n, bd);
376 }
377 if (is_store || is_load_store) {
378 bd = refine_store(n, bd);
379 }
380 }
381
382 // If there are no real barrier flags on the node, strip away additional fluff.
383 // Matcher does not care about this, and we would like to avoid invoking "barrier_data() != 0"
384 // rules when the only flags are the irrelevant fluff.
385 if ((bd != 0) && (bd & ShenandoahBitsReal) == 0) {
386 bd = 0;
387 }
388
389 if (bd != orig_bd) {
390 if (is_load_store) {
391 n->as_LoadStore()->set_barrier_data(bd);
392 } else {
393 n->as_Mem()->set_barrier_data(bd);
394 }
395 }
396
397 for (uint j = 0; j < n->req(); j++) {
398 Node* in = n->in(j);
399 if (in != nullptr) {
400 wq.push(in);
401 }
402 }
403 }
404 }
405
406 // Support for macro expanded GC barriers
407 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
408 if (node->is_LoadStore()) {
409 LoadStoreNode* loadstore = node->as_LoadStore();
410 loadstore->set_barrier_data(0);
411 } else if (node->is_Mem()) {
412 MemNode* mem = node->as_Mem();
413 mem->set_barrier_data(0);
414 }
415 }
416
417 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* macro, Node* node) const {
418 eliminate_gc_barrier_data(node);
419 }
420
421 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
422 uint8_t orig_bd = node->barrier_data();
423 if (orig_bd == 0) {
424 // Nothing to do.
425 return;
426 }
427
428 uint8_t bd = orig_bd;
429 int node_opcode = node->ideal_Opcode();
430
431 if (dominator == nullptr) {
432 // Must be allocation node.
433 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
434 // Loads from recent allocations do not need LRBs.
435 bd &= ~ShenandoahBitStrong;
436 }
437 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
438 // Stores to recent allocations do not need KA or CM.
439 bd &= ~ShenandoahBitKeepAlive;
440 bd &= ~ShenandoahBitCardMark;
441 }
442 } else {
443 // LoadStores do not get these optimizations, since their LRBs
444 // are required for fixups.
445 if (is_Load(node_opcode) || is_Store(node_opcode)) {
446 int dom_opcode = dominator->ideal_Opcode();
447 uint8_t dom_bd = dominator->barrier_data();
448
449 if (is_Load(dom_opcode) || is_LoadStore(dom_opcode)) {
450 // If dominating load is set up to perform LRB fixups, no further LRB is needed.
451 if ((dom_bd & ShenandoahBitStrong) != 0) {
452 bd &= ~ShenandoahBitStrong;
453 }
454 }
455 if (is_Store(dom_opcode)) {
456 // Dominating store has stored the good ref, no LRB is needed.
457 bd &= ~ShenandoahBitStrong;
458 }
459 }
460 }
461
462 if (orig_bd != bd) {
463 #ifdef ASSERT
464 PhaseRegAlloc* ra = Compile::current()->regalloc();
465 uint old_size = node->size(ra);
466 #endif
467 // We are already in final output. This means all nodes have already matched,
468 // and we are about to use Shenandoah match rules with stripped-down barriers.
469 // In this case, we must *not* strip non-real bits, because it would shift the
470 // encoding.
471 node->set_barrier_data(bd);
472 #ifdef ASSERT
473 uint new_size = node->size(ra);
474 assert(new_size <= old_size, "Node must not grow: %u -> %u", old_size, new_size);
475 #endif
476 }
477 }
478
479 void ShenandoahBarrierSetC2::analyze_dominating_barriers() const {
480 if (!ShenandoahElideMachBarriers) {
481 return;
482 }
483
484 ResourceMark rm;
485 Node_List accesses, dominators;
486
487 PhaseCFG* const cfg = Compile::current()->cfg();
488 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
489 const Block* const block = cfg->get_block(i);
490 for (uint j = 0; j < block->number_of_nodes(); ++j) {
491 Node* const node = block->get_node(j);
492
493 // Everything that happens in allocations does not need barriers.
494 // Record them for dominance analysis.
495 if (node->is_Phi() && is_allocation(node)) {
496 dominators.push(node);
497 continue;
498 }
499
500 if (!node->is_Mach()) {
501 continue;
502 }
503
504 MachNode* const mach = node->as_Mach();
505 int opcode = mach->ideal_Opcode();
506 if (is_Load(opcode) || is_Store(opcode) || is_LoadStore(opcode)) {
507 if ((mach->barrier_data() & ShenandoahBitsReal) != 0) {
508 accesses.push(mach);
509 dominators.push(mach);
510 }
511 }
512 }
513 }
514
515 elide_dominated_barriers(accesses, dominators);
516 }
517
518 uint ShenandoahBarrierSetC2::estimated_barrier_size(const Node* node) const {
519 // Barrier impact on fast-path is driven by GC state checks emitted very late.
520 // These checks are tight load-test-branch sequences, with no impact on C2 graph
521 // size. Limiting unrolling in presence of GC barriers might turn some loops
522 // tighter than with default unrolling, which may benefit performance due to denser
523 // code. Testing shows it is still counter-productive.
524 // Therefore, we report zero barrier size to let C2 do its normal thing.
525 return 0;
526 }
527
528 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const {
529 bool is_oop = is_reference_type(type);
530 if (!is_oop) {
531 return false;
532 }
533 if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
534 if (phase == Optimization) {
535 return false;
536 }
537 return !is_clone;
538 }
539 return true;
540 }
541
542 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
543 if (!ShenandoahCloneBarrier) {
544 return false;
545 }
546
547 if (src_type->isa_instptr() != nullptr) {
548 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
549 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
550 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
551 !ik->has_injected_fields() && !ik->has_object_fields()) {
552 if (!src_type->klass_is_exact()) {
553 // Class is *currently* the leaf in the hierarchy.
554 // Record the dependency so that we deopt if this does not hold in future.
555 Compile::current()->dependencies()->assert_leaf_type(ik);
556 }
557 return false;
558 }
559 } else if (src_type->isa_aryptr() != nullptr) {
560 // Array: need barrier only if array is oop-bearing.
561 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
562 if (is_reference_type(src_elem, true) && src_type->is_not_flat()) {
563 is_oop_array = true;
564 } else if (!src_type->is_not_flat()) {
565 // Maybe flat, assume the worst.
566 } else {
567 return false;
568 }
569 }
570
571 // Assume the worst.
572 return true;
573 }
574
575 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
576 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
577
578 bool is_oop_array = false;
579 if (!clone_needs_barrier(src_type, is_oop_array)) {
580 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
581 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
582 return;
583 }
584
585 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
586 // Looks like an instance? Prepare the instance clone. This would either
587 // be exploded into individual accesses or be left as runtime call.
588 // Common BarrierSetC2 prepares everything for both cases.
589 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
590 return;
591 }
592
593 // We are cloning the oop array. Prepare to call the normal arraycopy stub
594 // after the expansion. Normal stub takes the number of actual type-sized
595 // elements to copy after the base, compute the count here.
596 Node* offset = kit->MakeConX(arrayOopDesc::base_offset_in_bytes(UseCompressedOops ? T_NARROWOOP : T_OBJECT));
597 size = kit->gvn().transform(new SubXNode(size, offset));
598 size = kit->gvn().transform(new URShiftXNode(size, kit->intcon(LogBytesPerHeapOop)));
599 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, size, true, false);
600 ac->set_clone_array();
601 Node* n = kit->gvn().transform(ac);
602 if (n == ac) {
603 ac->set_adr_type(TypeRawPtr::BOTTOM);
604 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), TypeRawPtr::BOTTOM);
605 } else {
606 kit->set_all_memory(n);
607 }
608 }
609
610 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
611 Node* const ctrl = ac->in(TypeFunc::Control);
612 Node* const mem = ac->in(TypeFunc::Memory);
613 Node* const src = ac->in(ArrayCopyNode::Src);
614 Node* const src_offset = ac->in(ArrayCopyNode::SrcPos);
615 Node* const dest = ac->in(ArrayCopyNode::Dest);
616 Node* const dest_offset = ac->in(ArrayCopyNode::DestPos);
617 Node* length = ac->in(ArrayCopyNode::Length);
618
619 const TypeOopPtr* src_type = phase->igvn().type(src)->is_oopptr();
620
621 bool is_oop_array = false;
622 if (!clone_needs_barrier(src_type, is_oop_array)) {
623 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
624 BarrierSetC2::clone_at_expansion(phase, ac);
625 return;
626 }
627
628 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
629 // Still looks like an instance? Likely a large instance or reflective
630 // clone with unknown length. Go to runtime and handle it there.
631 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
632 return;
633 }
634
635 // We are cloning the oop array. Call into normal oop array copy stubs.
636 // Those stubs would call BarrierSetAssembler to handle GC barriers.
637
638 // This is the full clone, so offsets should equal each other and be at array base.
639 assert(src_offset == dest_offset, "should be equal");
640 const jlong offset = src_offset->get_long();
641 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
642 BasicType bt = ary_ptr->elem()->array_element_basic_type();
643 if (offset != arrayOopDesc::base_offset_in_bytes(bt)) {
644 // Something is off with flat arrays. Go to runtime instead.
645 // TODO: Figure this out.
646 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
647 return;
648 }
649 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
650
651 const char* copyfunc_name = "arraycopy";
652 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
653
654 Node* const call = phase->make_leaf_call(ctrl, mem,
655 OptoRuntime::fast_arraycopy_Type(),
656 copyfunc_addr, copyfunc_name,
657 TypeRawPtr::BOTTOM,
658 phase->basic_plus_adr(src, src_offset),
659 phase->basic_plus_adr(dest, dest_offset),
660 length,
661 phase->top()
662 );
663 phase->transform_later(call);
664
665 phase->igvn().replace_node(ac, call);
666 }
667
668 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
669 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
670 }
671
672 void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
673 os->print(" Node barriers: ");
674 if ((data & ShenandoahBitStrong) != 0) {
675 data &= ~ShenandoahBitStrong;
676 os->print("strong ");
677 }
678
679 if ((data & ShenandoahBitWeak) != 0) {
680 data &= ~ShenandoahBitWeak;
681 os->print("weak ");
682 }
683
684 if ((data & ShenandoahBitPhantom) != 0) {
685 data &= ~ShenandoahBitPhantom;
686 os->print("phantom ");
687 }
688
689 if ((data & ShenandoahBitKeepAlive) != 0) {
690 data &= ~ShenandoahBitKeepAlive;
691 os->print("keepalive ");
692 }
693
694 if ((data & ShenandoahBitCardMark) != 0) {
695 data &= ~ShenandoahBitCardMark;
696 os->print("cardmark ");
697 }
698
699 if ((data & ShenandoahBitNative) != 0) {
700 data &= ~ShenandoahBitNative;
701 os->print("native ");
702 }
703
704 if ((data & ShenandoahBitNotNull) != 0) {
705 data &= ~ShenandoahBitNotNull;
706 os->print("not-null ");
707 }
708
709 if ((data & ShenandoahBitElided) != 0) {
710 data &= ~ShenandoahBitElided;
711 os->print("elided ");
712 }
713
714 os->cr();
715
716 if (data > 0) {
717 fatal("Unknown bit!");
718 }
719
720 os->print_cr(" GC configuration: %sLRB %sSATB %sClone %sCard",
721 (ShenandoahLoadRefBarrier ? "+" : "-"),
722 (ShenandoahSATBBarrier ? "+" : "-"),
723 (ShenandoahCloneBarrier ? "+" : "-"),
724 (ShenandoahCardBarrier ? "+" : "-")
725 );
726 }
727
728
729 #ifdef ASSERT
730 void ShenandoahBarrierSetC2::verify_gc_barrier_assert(bool cond, const char* msg, uint8_t bd, Node* n) {
731 if (!cond) {
732 stringStream ss;
733 ss.print_cr("%s", msg);
734 ss.print_cr("-----------------");
735 print_barrier_data(&ss, bd);
736 ss.print_cr("-----------------");
737 n->dump_bfs(1, nullptr, "", &ss);
738 report_vm_error(__FILE__, __LINE__, ss.as_string());
739 }
740 }
741
742 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
743 if (!ShenandoahVerifyOptoBarriers) {
744 return;
745 }
746
747 // Verify depending on the barriers actually enabled, allowing verification in passive mode.
748 // Normally, we have _some_ bits set on all accesses. Optimizations may drop some bits,
749 // but only the last optimization step eliminates all remaining metadata flags. Only then
750 // the access data can be completely blank.
751 bool final_phase = (phase == BeforeCodeGen);
752 bool expect_load_barriers = !final_phase && ShenandoahLoadRefBarrier;
753 bool expect_store_barriers = !final_phase && (ShenandoahSATBBarrier || ShenandoahCardBarrier);
754 bool expect_load_store_barriers = expect_load_barriers || expect_store_barriers;
755 bool expect_some_real = final_phase;
756
757 Unique_Node_List wq;
758
759 RootNode* root = compile->root();
760 wq.push(root);
761
762 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
763 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
764 Node* m = root->fast_out(i);
765 wq.push(m);
766 }
767
768 for (uint next = 0; next < wq.size(); next++) {
769 Node *n = wq.at(next);
770 assert(!n->is_Mach(), "No Mach nodes here yet");
771
772 int opc = n->Opcode();
773
774 uint8_t bd = 0;
775 const TypePtr* adr_type = nullptr;
776 if (is_Load(opc)) {
777 bd = n->as_Load()->barrier_data();
778 adr_type = n->as_Load()->adr_type();
779 } else if (is_Store(opc)) {
780 bd = n->as_Store()->barrier_data();
781 adr_type = n->as_Store()->adr_type();
782 } else if (is_LoadStore(opc)) {
783 bd = n->as_LoadStore()->barrier_data();
784 adr_type = n->as_LoadStore()->adr_type();
785 } else if (n->is_Mem()) {
786 bd = MemNode::barrier_data(n);
787 verify_gc_barrier_assert(bd == 0, "Other mem nodes should have no barrier data", bd, n);
788 }
789
790 bool is_weak = (bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0;
791 bool is_native = (bd & ShenandoahBitNative) != 0;
792
793 bool is_referent = adr_type != nullptr &&
794 adr_type->isa_instptr() &&
795 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
796 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
797
798 bool is_oop_addr = (adr_type != nullptr) && (adr_type->isa_oopptr() || adr_type->isa_narrowoop());
799 bool is_raw_addr = (adr_type != nullptr) && (adr_type->isa_rawptr() || adr_type->isa_klassptr());
800
801 verify_gc_barrier_assert(!expect_some_real || (bd == 0) || (bd & ShenandoahBitsReal) != 0, "Without real barriers, metadata should be stripped at this point", bd, n);
802
803 if (is_oop_addr) {
804 if (is_Load(opc)) {
805 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Oop load should have barrier data", bd, n);
806 verify_gc_barrier_assert(!is_weak || is_referent, "Weak load only for Reference.referent", bd, n);
807 } else if (is_Store(opc)) {
808 // Reference.referent stores can be without barriers.
809 verify_gc_barrier_assert(!expect_store_barriers || is_referent || (bd != 0), "Oop store should have barrier data", bd, n);
810 } else if (is_LoadStore(opc)) {
811 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Oop load-store should have barrier data", bd, n);
812 }
813 } else if (is_raw_addr) {
814 if (is_native) {
815 if (is_Load(opc)) {
816 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Native oop load should have barrier data", bd, n);
817 }
818 if (is_Store(opc)) {
819 verify_gc_barrier_assert(!expect_store_barriers || (bd != 0), "Native oop store should have barrier data", bd, n);
820 }
821 if (is_LoadStore(opc)) {
822 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Native oop load-store should have barrier data", bd, n);
823 }
824 } else {
825 // Some Load/Stores are used for T_ADDRESS and/or raw stores, which are supposed not to have barriers.
826 // Some other Load/Stores are emitted for real oops, but on raw addresses via Unsafe.
827 // The distinction on this level is lost, so we cannot really verify this.
828 }
829 } else {
830 if (is_Load(opc) || is_Store(opc) || is_LoadStore(opc)) {
831 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
832 }
833 }
834
835 for (uint j = 0; j < n->req(); j++) {
836 Node* in = n->in(j);
837 if (in != nullptr) {
838 wq.push(in);
839 }
840 }
841 }
842 }
843 #endif
844
845 static ShenandoahBarrierSetC2State* barrier_set_state() {
846 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
847 }
848
849 int ShenandoahBarrierSetC2::estimate_stub_size() const {
850 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
851 assert(stubs->is_empty(), "Lifecycle: no stubs were yet created");
852 return 0;
853 }
854
855 void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
856 MacroAssembler masm(&cb);
857
858 PhaseOutput* const output = Compile::current()->output();
859 assert(masm.offset() <= output->buffer_sizing_data()->_code,
860 "Stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
861 barrier_set_state()->set_stubs_start_offset(masm.offset());
862
863 // Stub generation counts all stubs as skipped for the sake of inlining policy.
864 // This is critical for performance, check it.
865 #ifdef ASSERT
866 int offset_before = masm.offset();
867 int skipped_before = cb.total_skipped_instructions_size();
868 #endif
869
870 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
871 for (int i = 0; i < stubs->length(); i++) {
872 // Make sure there is enough space in the code buffer
873 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
874 ciEnv::current()->record_failure("CodeCache is full");
875 return;
876 }
877 stubs->at(i)->emit_code(masm);
878 }
879
880 #ifdef ASSERT
881 int offset_after = masm.offset();
882 int skipped_after = cb.total_skipped_instructions_size();
883 assert(offset_after - offset_before == skipped_after - skipped_before,
884 "All stubs are counted as skipped. masm: %d - %d = %d, cb: %d - %d = %d",
885 offset_after, offset_before, offset_after - offset_before,
886 skipped_after, skipped_before, skipped_after - skipped_before);
887 #endif
888
889 // Code will be copied. No ICache sync required.
890 }
891
892 void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
893 if (!Compile::current()->output()->in_scratch_emit_size()) {
894 barrier_set_state()->stubs()->append(stub);
895 }
896 }
897
898 ShenandoahBarrierStubC2* ShenandoahBarrierStubC2::create(const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow, bool do_load) {
899 auto* stub = new (Compile::current()->comp_arena()) ShenandoahBarrierStubC2(node, obj, addr, tmp1, tmp2, narrow, do_load);
900 register_stub(stub);
901 return stub;
902 }
903
904 void ShenandoahBarrierStubC2::load_post(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
905 // Load post-barrier:
906 // a. Satisfies the need for LRB for normal loads
907 // b. Passes a weak load through LRB-weak
908 // c. Keep-alives a weak load
909 if (needs_slow_barrier(node)) {
910 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ false);
911 char check = 0;
912 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
913 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
914 check |= needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
915 stub->enter_if_gc_state(*masm, check, tmp1, tmp2);
916 }
917 }
918
919 void ShenandoahBarrierStubC2::store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
920 // Store pre-barrier: SATB, keep-alive the current memory value.
921 if (needs_slow_barrier(node)) {
922 assert(!needs_load_ref_barrier(node), "Should not be required for stores");
923 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
924 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING, tmp1, tmp2);
925 }
926 }
927
928 void ShenandoahBarrierStubC2::load_store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
929 // Load/Store pre-barrier:
930 // a. Avoids false positives from CAS encountering to-space memory values.
931 // b. Satisfies the need for LRB for the CAE result.
932 // c. Records old value for the sake of SATB.
933 //
934 // (a) and (b) are covered because load barrier does memory location fixup.
935 // (c) is covered by KA on the current memory value.
936 if (needs_slow_barrier(node)) {
937 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
938 char check = 0;
939 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
940 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
941 assert(!needs_load_ref_barrier_weak(node), "Not supported for Load/Stores");
942 stub->enter_if_gc_state(*masm, check, tmp1, tmp2);
943 }
944 }
945
946 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
947 if (needs_card_barrier(node)) {
948 cardtable(*masm, addr, tmp1, tmp2);
949 }
950 }
951
952 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
953 store_post(masm, node, addr, tmp1, tmp2);
954 }
955
956 bool ShenandoahBarrierStubC2::is_live_register(Register reg) {
957 return preserve_set().member(OptoReg::as_OptoReg(reg->as_VMReg()));
958 }
959
960 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Register skip_reg1, Register skip_reg2) {
961 Register tmp = noreg;
962 Register fallback_live = noreg;
963
964 // Try to select non-live first:
965 for (int i = 0; i < available_gp_registers(); i++) {
966 Register r = as_Register(i);
967 if (r != _obj && r != _addr.base() && r != _addr.index() &&
968 r != skip_reg1 && r != skip_reg2 && !is_special_register(r)) {
969 if (!is_live_register(r)) {
970 tmp = r;
971 break;
972 } else if (fallback_live == noreg) {
973 fallback_live = r;
974 }
975 }
976 }
977
978 // If we could not find a non-live register, select the live fallback:
979 if (tmp == noreg) {
980 tmp = fallback_live;
981 selected_live = true;
982 } else {
983 selected_live = false;
984 }
985
986 assert(tmp != noreg, "successfully selected");
987 assert_different_registers(tmp, skip_reg1);
988 assert_different_registers(tmp, skip_reg2);
989 assert_different_registers(tmp, _obj);
990 assert_different_registers(tmp, _addr.base());
991 assert_different_registers(tmp, _addr.index());
992 return tmp;
993 }
994
995 address ShenandoahBarrierStubC2::keepalive_runtime_entry_addr() {
996 if (_narrow) {
997 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_narrow);
998 } else {
999 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre);
1000 }
1001 }
1002
1003 address ShenandoahBarrierStubC2::lrb_runtime_entry_addr() {
1004 bool is_strong = (_node->barrier_data() & ShenandoahBitStrong) != 0;
1005 bool is_weak = (_node->barrier_data() & ShenandoahBitWeak) != 0;
1006 bool is_phantom = (_node->barrier_data() & ShenandoahBitPhantom) != 0;
1007
1008 if (_narrow) {
1009 if (is_strong) {
1010 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow_narrow);
1011 } else if (is_weak) {
1012 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow_narrow);
1013 } else if (is_phantom) {
1014 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow_narrow);
1015 }
1016 } else {
1017 if (is_strong) {
1018 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1019 } else if (is_weak) {
1020 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1021 } else if (is_phantom) {
1022 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1023 }
1024 }
1025
1026 ShouldNotReachHere();
1027 return nullptr;
1028 }
1029
1030 bool ShenandoahBarrierSetC2State::needs_liveness_data(const MachNode* mach) const {
1031 // Nodes that require slow-path stubs need liveness data.
1032 return ShenandoahBarrierStubC2::needs_slow_barrier(mach);
1033 }
1034
1035 bool ShenandoahBarrierSetC2State::needs_livein_data() const {
1036 return true;
1037 }