1 /*
2 * Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "opto/arraycopynode.hpp"
34 #include "opto/escape.hpp"
35 #include "opto/graphKit.hpp"
36 #include "opto/idealKit.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/narrowptrnode.hpp"
39 #include "opto/output.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42
43 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
44 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
45 }
46
47 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
48 BarrierSetC2State(comp_arena),
49 _stubs(new (comp_arena) GrowableArray<ShenandoahBarrierStubC2*>(comp_arena, 8, 0, nullptr)),
50 _trampoline_stubs_count(0),
51 _stubs_start_offset(0),
52 _stubs_current_total_size(0) {
53 }
54
55 static void set_barrier_data(C2Access& access, bool load, bool store) {
56 if (!access.is_oop()) {
57 return;
58 }
59
60 DecoratorSet decorators = access.decorators();
61 bool tightly_coupled = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
62 bool in_heap = (decorators & IN_HEAP) != 0;
63 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
64 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
65
66 if (tightly_coupled) {
67 access.set_barrier_data(ShenandoahBitElided);
68 return;
69 }
70
71 uint8_t barrier_data = 0;
72
73 if (load) {
74 if (ShenandoahLoadRefBarrier) {
75 if (on_phantom) {
76 barrier_data |= ShenandoahBitPhantom;
77 } else if (on_weak) {
78 barrier_data |= ShenandoahBitWeak;
79 } else {
80 barrier_data |= ShenandoahBitStrong;
81 }
82 }
83 }
84
85 if (store) {
86 if (ShenandoahSATBBarrier) {
87 barrier_data |= ShenandoahBitKeepAlive;
88 }
89 if (ShenandoahCardBarrier && in_heap) {
90 barrier_data |= ShenandoahBitCardMark;
91 }
92 }
93
94 if (!in_heap) {
95 barrier_data |= ShenandoahBitNative;
96 }
97
98 access.set_barrier_data(barrier_data);
99 }
100
101 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
102 // 1: Non-reference load, no additional barrier is needed
103 if (!access.is_oop()) {
104 return BarrierSetC2::load_at_resolved(access, val_type);
105 }
106
107 // 2. Set barrier data for load
108 set_barrier_data(access, /* load = */ true, /* store = */ false);
109
110 // 3. Correction: If we are reading the value of the referent field of
111 // a Reference object, we need to record the referent resurrection.
112 DecoratorSet decorators = access.decorators();
113 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
114 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
115 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
116 bool needs_keepalive = ((on_weak || on_phantom) && !no_keepalive);
117 if (needs_keepalive) {
118 uint8_t barriers = access.barrier_data() | (ShenandoahSATBBarrier ? ShenandoahBitKeepAlive : 0);
119 access.set_barrier_data(barriers);
120 }
121
122 return BarrierSetC2::load_at_resolved(access, val_type);
123 }
124
125 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
126 // 1: Non-reference store, no additional barrier is needed
127 if (!access.is_oop()) {
128 return BarrierSetC2::store_at_resolved(access, val);
129 }
130
131 // 2. Set barrier data for store
132 set_barrier_data(access, /* load = */ false, /* store = */ true);
133
134 // 3. Correction: avoid keep-alive barriers that should not do keep-alive.
135 DecoratorSet decorators = access.decorators();
136 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
137 if (no_keepalive) {
138 access.set_barrier_data(access.barrier_data() & ~ShenandoahBitKeepAlive);
139 }
140
141 return BarrierSetC2::store_at_resolved(access, val);
142 }
143
144 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
145 Node* new_val, const Type* value_type) const {
146 set_barrier_data(access, /* load = */ true, /* store = */ true);
147 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
148 }
149
150 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
151 Node* new_val, const Type* value_type) const {
152 set_barrier_data(access, /* load = */ true, /* store = */ true);
153 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
154 }
155
156 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
157 set_barrier_data(access, /* load = */ true, /* store = */ true);
158 return BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
159 }
160
161 bool ShenandoahBarrierSetC2::is_Load(int opcode) {
162 switch (opcode) {
163 case Op_LoadN:
164 case Op_LoadP:
165 return true;
166 default:
167 return false;
168 }
169 }
170
171 bool ShenandoahBarrierSetC2::is_Store(int opcode) {
172 switch (opcode) {
173 case Op_StoreN:
174 case Op_StoreP:
175 return true;
176 default:
177 return false;
178 }
179 }
180
181 bool ShenandoahBarrierSetC2::is_LoadStore(int opcode) {
182 switch (opcode) {
183 case Op_CompareAndExchangeN:
184 case Op_CompareAndExchangeP:
185 case Op_WeakCompareAndSwapN:
186 case Op_WeakCompareAndSwapP:
187 case Op_CompareAndSwapN:
188 case Op_CompareAndSwapP:
189 case Op_GetAndSetP:
190 case Op_GetAndSetN:
191 return true;
192 default:
193 return false;
194 }
195 }
196
197 bool ShenandoahBarrierSetC2::can_remove_load_barrier(Node* root) {
198 // Check if all outs feed into nodes that do not expose the oops to the rest
199 // of the runtime system. In this case, we can elide the LRB barrier. We bail
200 // out with false at the first sight of trouble.
201
202 ResourceMark rm;
203 VectorSet visited;
204 Node_List worklist;
205 worklist.push(root);
206
207 while (worklist.size() > 0) {
208 Node* n = worklist.pop();
209 if (visited.test_set(n->_idx)) {
210 continue;
211 }
212
213 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
214 Node* out = n->fast_out(i);
215 switch (out->Opcode()) {
216 case Op_Phi:
217 case Op_EncodeP:
218 case Op_DecodeN:
219 case Op_CastPP:
220 case Op_CheckCastPP:
221 case Op_AddP: {
222 // Transitive node, check if any other outs are doing anything troublesome.
223 worklist.push(out);
224 break;
225 }
226
227 case Op_LoadRange: {
228 // Array length is the same in all copies.
229 break;
230 }
231
232 case Op_LoadKlass: {
233 // Klass is the same in all copies.
234 // We would have liked to assert -UCOH, but there are legitimate klass
235 // loads from native Klass* instances, which are also safe under +UCOH.
236 break;
237 }
238
239 case Op_LoadNKlass: {
240 // Similar to above, but LoadNKlass is only safe without +UCOH.
241 // With +UCOH, it loads from mark word, which clashes with forwarding pointers.
242 if (!UseCompactObjectHeaders) {
243 break;
244 }
245 return false;
246 }
247
248 case Op_CmpN: {
249 if (out->in(1) == n &&
250 out->in(2)->Opcode() == Op_ConN &&
251 out->in(2)->get_narrowcon() == 0) {
252 // Null check, no oop is exposed.
253 break;
254 }
255 if (out->in(2) == n &&
256 out->in(1)->Opcode() == Op_ConN &&
257 out->in(1)->get_narrowcon() == 0) {
258 // Null check, no oop is exposed.
259 break;
260 }
261 return false;
262 }
263
264 case Op_CmpP: {
265 if (out->in(1) == n &&
266 out->in(2)->Opcode() == Op_ConP &&
267 out->in(2)->get_ptr() == 0) {
268 // Null check, no oop is exposed.
269 break;
270 }
271 if (out->in(2) == n &&
272 out->in(1)->Opcode() == Op_ConP &&
273 out->in(1)->get_ptr() == 0) {
274 // Null check, no oop is exposed.
275 break;
276 }
277 return false;
278 }
279
280 case Op_CallStaticJava: {
281 if (out->as_CallStaticJava()->is_uncommon_trap()) {
282 // Local feeds into uncommon trap. Deopt machinery handles barriers itself.
283 break;
284 }
285 return false;
286 }
287
288 default: {
289 // Paranoidly distrust any other nodes.
290 return false;
291 }
292 }
293 }
294 }
295
296 // Nothing troublesome found.
297 return true;
298 }
299
300 uint8_t ShenandoahBarrierSetC2::refine_load(Node* n, uint8_t bd) {
301 assert(ShenandoahElideIdealBarriers, "Checked by caller");
302 assert(bd != 0, "Checked by caller");
303
304 // Do not touch weak loads at all: they are responsible for shielding from
305 // Reference.referent resurrection.
306 if ((bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0) {
307 return bd;
308 }
309
310 if (((bd & ShenandoahBitStrong) != 0) && can_remove_load_barrier(n)) {
311 bd &= ~ShenandoahBitStrong;
312 }
313
314 return bd;
315 }
316
317 uint8_t ShenandoahBarrierSetC2::refine_store(Node* n, uint8_t bd) {
318 assert(ShenandoahElideIdealBarriers, "Checked by caller");
319 assert(bd != 0, "Checked by caller");
320 assert(n->is_Mem() || n->is_LoadStore(), "Sanity");
321
322 const Node* newval = n->in(MemNode::ValueIn);
323 assert(newval != nullptr, "Should be present");
324
325 // Type system tells us something about nullity?
326 const Type* newval_bottom = newval->bottom_type();
327 assert(newval_bottom->isa_oopptr() || newval_bottom->isa_narrowoop() ||
328 newval_bottom == TypePtr::NULL_PTR, "Should be an oop store");
329 const TypePtr* newval_type = newval_bottom->make_ptr();
330 assert(newval_type != nullptr, "Should have been filtered before");
331 TypePtr::PTR newval_type_ptr = newval_type->ptr();
332 if (newval_type_ptr == TypePtr::Null) {
333 bd &= ~ShenandoahBitNotNull;
334 // Card table barrier is not needed if we store null.
335 bd &= ~ShenandoahBitCardMark;
336 } else if (newval_type_ptr == TypePtr::NotNull) {
337 // Definitely not null.
338 bd |= ShenandoahBitNotNull;
339 }
340
341 return bd;
342 }
343
344 void ShenandoahBarrierSetC2::final_refinement(Compile* compile) const {
345 ResourceMark rm;
346 Unique_Node_List wq;
347
348 RootNode* root = compile->root();
349 wq.push(root);
350
351 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
352 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
353 Node* m = root->fast_out(i);
354 wq.push(m);
355 }
356
357 for (uint next = 0; next < wq.size(); next++) {
358 Node* n = wq.at(next);
359
360 assert(!n->is_Mach(), "No Mach nodes here yet");
361
362 int opc = n->Opcode();
363 bool is_load = is_Load(opc);
364 bool is_store = is_Store(opc);
365 bool is_load_store = is_LoadStore(opc);
366
367 uint8_t orig_bd = 0;
368 if (is_load_store) {
369 orig_bd = n->as_LoadStore()->barrier_data();
370 } else if (is_load || is_store) {
371 orig_bd = n->as_Mem()->barrier_data();
372 }
373
374 uint8_t bd = orig_bd;
375 if (ShenandoahElideIdealBarriers && bd != 0) {
376 // Note: we cannot apply load optimizations to LoadStores,
377 // because their load barriers are needed for fixups.
378 if (is_load) {
379 bd = refine_load(n, bd);
380 }
381 if (is_store || is_load_store) {
382 bd = refine_store(n, bd);
383 }
384 }
385
386 // If there are no real barrier flags on the node, strip away additional fluff.
387 // Matcher does not care about this, and we would like to avoid invoking "barrier_data() != 0"
388 // rules when the only flags are the irrelevant fluff.
389 if ((bd != 0) && (bd & ShenandoahBitsReal) == 0) {
390 bd = 0;
391 }
392
393 if (bd != orig_bd) {
394 if (is_load_store) {
395 n->as_LoadStore()->set_barrier_data(bd);
396 } else {
397 n->as_Mem()->set_barrier_data(bd);
398 }
399 }
400
401 for (uint j = 0; j < n->req(); j++) {
402 Node* in = n->in(j);
403 if (in != nullptr) {
404 wq.push(in);
405 }
406 }
407 }
408 }
409
410 // Support for macro expanded GC barriers
411 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
412 if (node->is_LoadStore()) {
413 LoadStoreNode* loadstore = node->as_LoadStore();
414 loadstore->set_barrier_data(0);
415 } else if (node->is_Mem()) {
416 MemNode* mem = node->as_Mem();
417 mem->set_barrier_data(0);
418 }
419 }
420
421 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* macro, Node* node) const {
422 eliminate_gc_barrier_data(node);
423 }
424
425 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
426 uint8_t orig_bd = node->barrier_data();
427 if (orig_bd == 0) {
428 // Nothing to do.
429 return;
430 }
431
432 uint8_t bd = orig_bd;
433 int node_opcode = node->ideal_Opcode();
434
435 if (dominator == nullptr) {
436 // Must be allocation node.
437 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
438 // Loads from recent allocations do not need LRBs.
439 bd &= ~ShenandoahBitStrong;
440 }
441 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
442 // Stores to recent allocations do not need KA or CM.
443 bd &= ~ShenandoahBitKeepAlive;
444 bd &= ~ShenandoahBitCardMark;
445 }
446 } else {
447 // LoadStores do not get these optimizations, since their LRBs
448 // are required for fixups.
449 if (is_Load(node_opcode) || is_Store(node_opcode)) {
450 int dom_opcode = dominator->ideal_Opcode();
451 uint8_t dom_bd = dominator->barrier_data();
452
453 if (is_Load(dom_opcode) || is_LoadStore(dom_opcode)) {
454 // If dominating load is set up to perform LRB fixups, no further LRB is needed.
455 if ((dom_bd & ShenandoahBitStrong) != 0) {
456 bd &= ~ShenandoahBitStrong;
457 }
458 }
459 if (is_Store(dom_opcode)) {
460 // Dominating store has stored the good ref, no LRB is needed.
461 bd &= ~ShenandoahBitStrong;
462 }
463 }
464 }
465
466 if (orig_bd != bd) {
467 // We are already in final output.
468 // Strip the extra barrier data if no real bits are left.
469 if ((bd & ShenandoahBitsReal) != 0) {
470 node->set_barrier_data(bd);
471 } else {
472 node->set_barrier_data(0);
473 }
474 }
475 }
476
477 void ShenandoahBarrierSetC2::analyze_dominating_barriers() const {
478 if (!ShenandoahElideMachBarriers) {
479 return;
480 }
481
482 ResourceMark rm;
483 Node_List accesses, dominators;
484
485 PhaseCFG* const cfg = Compile::current()->cfg();
486 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
487 const Block* const block = cfg->get_block(i);
488 for (uint j = 0; j < block->number_of_nodes(); ++j) {
489 Node* const node = block->get_node(j);
490
491 // Everything that happens in allocations does not need barriers.
492 // Record them for dominance analysis.
493 if (node->is_Phi() && is_allocation(node)) {
494 dominators.push(node);
495 continue;
496 }
497
498 if (!node->is_Mach()) {
499 continue;
500 }
501
502 MachNode* const mach = node->as_Mach();
503 int opcode = mach->ideal_Opcode();
504 if (is_Load(opcode) || is_Store(opcode) || is_LoadStore(opcode)) {
505 if ((mach->barrier_data() & ShenandoahBitsReal) != 0) {
506 accesses.push(mach);
507 dominators.push(mach);
508 }
509 }
510 }
511 }
512
513 elide_dominated_barriers(accesses, dominators);
514 }
515
516 uint ShenandoahBarrierSetC2::estimated_barrier_size(const Node* node) const {
517 // Barrier impact on fast-path is driven by GC state checks emitted very late.
518 // These checks are tight load-test-branch sequences, with no impact on C2 graph
519 // size. Limiting unrolling in presence of GC barriers might turn some loops
520 // tighter than with default unrolling, which may benefit performance due to denser
521 // code. Testing shows it is still counter-productive.
522 // Therefore, we report zero barrier size to let C2 do its normal thing.
523 return 0;
524 }
525
526 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const {
527 bool is_oop = is_reference_type(type);
528 if (!is_oop) {
529 return false;
530 }
531 if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
532 if (phase == Optimization) {
533 return false;
534 }
535 return !is_clone;
536 }
537 return true;
538 }
539
540 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
541 if (!ShenandoahCloneBarrier) {
542 return false;
543 }
544
545 if (src_type->isa_instptr() != nullptr) {
546 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
547 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
548 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
549 !ik->has_injected_fields() && !ik->has_object_fields()) {
550 if (!src_type->klass_is_exact()) {
551 // Class is *currently* the leaf in the hierarchy.
552 // Record the dependency so that we deopt if this does not hold in future.
553 Compile::current()->dependencies()->assert_leaf_type(ik);
554 }
555 return false;
556 }
557 } else if (src_type->isa_aryptr() != nullptr) {
558 // Array: need barrier only if array is oop-bearing.
559 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
560 if (is_reference_type(src_elem, true) && src_type->is_not_flat()) {
561 is_oop_array = true;
562 } else if (!src_type->is_not_flat()) {
563 // Maybe flat, assume the worst.
564 } else {
565 return false;
566 }
567 }
568
569 // Assume the worst.
570 return true;
571 }
572
573 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
574 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
575
576 bool is_oop_array = false;
577 if (!clone_needs_barrier(src_type, is_oop_array)) {
578 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
579 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
580 return;
581 }
582
583 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
584 // Looks like an instance? Prepare the instance clone. This would either
585 // be exploded into individual accesses or be left as runtime call.
586 // Common BarrierSetC2 prepares everything for both cases.
587 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
588 return;
589 }
590
591 // We are cloning the oop array. Prepare to call the normal arraycopy stub
592 // after the expansion. Normal stub takes the number of actual type-sized
593 // elements to copy after the base, compute the count here.
594 Node* offset = kit->MakeConX(arrayOopDesc::base_offset_in_bytes(UseCompressedOops ? T_NARROWOOP : T_OBJECT));
595 size = kit->gvn().transform(new SubXNode(size, offset));
596 size = kit->gvn().transform(new URShiftXNode(size, kit->intcon(LogBytesPerHeapOop)));
597 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, size, true, false);
598 ac->set_clone_array();
599 Node* n = kit->gvn().transform(ac);
600 if (n == ac) {
601 ac->set_adr_type(TypeRawPtr::BOTTOM);
602 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), TypeRawPtr::BOTTOM);
603 } else {
604 kit->set_all_memory(n);
605 }
606 }
607
608 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
609 Node* const ctrl = ac->in(TypeFunc::Control);
610 Node* const mem = ac->in(TypeFunc::Memory);
611 Node* const src = ac->in(ArrayCopyNode::Src);
612 Node* const src_offset = ac->in(ArrayCopyNode::SrcPos);
613 Node* const dest = ac->in(ArrayCopyNode::Dest);
614 Node* const dest_offset = ac->in(ArrayCopyNode::DestPos);
615 Node* length = ac->in(ArrayCopyNode::Length);
616
617 const TypeOopPtr* src_type = phase->igvn().type(src)->is_oopptr();
618
619 bool is_oop_array = false;
620 if (!clone_needs_barrier(src_type, is_oop_array)) {
621 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
622 BarrierSetC2::clone_at_expansion(phase, ac);
623 return;
624 }
625
626 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
627 // Still looks like an instance? Likely a large instance or reflective
628 // clone with unknown length. Go to runtime and handle it there.
629 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
630 return;
631 }
632
633 // We are cloning the oop array. Call into normal oop array copy stubs.
634 // Those stubs would call BarrierSetAssembler to handle GC barriers.
635
636 // This is the full clone, so offsets should equal each other and be at array base.
637 assert(src_offset == dest_offset, "should be equal");
638 const jlong offset = src_offset->get_long();
639 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
640 BasicType bt = ary_ptr->elem()->array_element_basic_type();
641 if (offset != arrayOopDesc::base_offset_in_bytes(bt)) {
642 // Something is off with flat arrays. Go to runtime instead.
643 // TODO: Figure this out.
644 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
645 return;
646 }
647 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
648
649 const char* copyfunc_name = "arraycopy";
650 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
651
652 Node* const call = phase->make_leaf_call(ctrl, mem,
653 OptoRuntime::fast_arraycopy_Type(),
654 copyfunc_addr, copyfunc_name,
655 TypeRawPtr::BOTTOM,
656 phase->basic_plus_adr(src, src_offset),
657 phase->basic_plus_adr(dest, dest_offset),
658 length,
659 phase->top()
660 );
661 phase->transform_later(call);
662
663 phase->igvn().replace_node(ac, call);
664 }
665
666 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
667 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
668 }
669
670 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
671 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
672 }
673
674 void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
675 os->print(" Node barriers: ");
676 if ((data & ShenandoahBitStrong) != 0) {
677 data &= ~ShenandoahBitStrong;
678 os->print("strong ");
679 }
680
681 if ((data & ShenandoahBitWeak) != 0) {
682 data &= ~ShenandoahBitWeak;
683 os->print("weak ");
684 }
685
686 if ((data & ShenandoahBitPhantom) != 0) {
687 data &= ~ShenandoahBitPhantom;
688 os->print("phantom ");
689 }
690
691 if ((data & ShenandoahBitKeepAlive) != 0) {
692 data &= ~ShenandoahBitKeepAlive;
693 os->print("keepalive ");
694 }
695
696 if ((data & ShenandoahBitCardMark) != 0) {
697 data &= ~ShenandoahBitCardMark;
698 os->print("cardmark ");
699 }
700
701 if ((data & ShenandoahBitNative) != 0) {
702 data &= ~ShenandoahBitNative;
703 os->print("native ");
704 }
705
706 if ((data & ShenandoahBitNotNull) != 0) {
707 data &= ~ShenandoahBitNotNull;
708 os->print("not-null ");
709 }
710
711 if ((data & ShenandoahBitElided) != 0) {
712 data &= ~ShenandoahBitElided;
713 os->print("elided ");
714 }
715
716 os->cr();
717
718 if (data > 0) {
719 fatal("Unknown bit!");
720 }
721
722 os->print_cr(" GC configuration: %sLRB %sSATB %sClone %sCard",
723 (ShenandoahLoadRefBarrier ? "+" : "-"),
724 (ShenandoahSATBBarrier ? "+" : "-"),
725 (ShenandoahCloneBarrier ? "+" : "-"),
726 (ShenandoahCardBarrier ? "+" : "-")
727 );
728 }
729
730
731 #ifdef ASSERT
732 void ShenandoahBarrierSetC2::verify_gc_barrier_assert(bool cond, const char* msg, uint8_t bd, Node* n) {
733 if (!cond) {
734 stringStream ss;
735 ss.print_cr("%s", msg);
736 ss.print_cr("-----------------");
737 print_barrier_data(&ss, bd);
738 ss.print_cr("-----------------");
739 n->dump_bfs(1, nullptr, "", &ss);
740 report_vm_error(__FILE__, __LINE__, ss.as_string());
741 }
742 }
743
744 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
745 if (!ShenandoahVerifyOptoBarriers) {
746 return;
747 }
748
749 // Verify depending on the barriers actually enabled, allowing verification in passive mode.
750 // Normally, we have _some_ bits set on all accesses. Optimizations may drop some bits,
751 // but only the last optimization step eliminates all remaining metadata flags. Only then
752 // the access data can be completely blank.
753 bool final_phase = (phase == BeforeCodeGen);
754 bool expect_load_barriers = !final_phase && ShenandoahLoadRefBarrier;
755 bool expect_store_barriers = !final_phase && (ShenandoahSATBBarrier || ShenandoahCardBarrier);
756 bool expect_load_store_barriers = expect_load_barriers || expect_store_barriers;
757 bool expect_some_real = final_phase;
758
759 Unique_Node_List wq;
760
761 RootNode* root = compile->root();
762 wq.push(root);
763
764 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
765 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
766 Node* m = root->fast_out(i);
767 wq.push(m);
768 }
769
770 for (uint next = 0; next < wq.size(); next++) {
771 Node *n = wq.at(next);
772 assert(!n->is_Mach(), "No Mach nodes here yet");
773
774 int opc = n->Opcode();
775
776 uint8_t bd = 0;
777 const TypePtr* adr_type = nullptr;
778 if (is_Load(opc)) {
779 bd = n->as_Load()->barrier_data();
780 adr_type = n->as_Load()->adr_type();
781 } else if (is_Store(opc)) {
782 bd = n->as_Store()->barrier_data();
783 adr_type = n->as_Store()->adr_type();
784 } else if (is_LoadStore(opc)) {
785 bd = n->as_LoadStore()->barrier_data();
786 adr_type = n->as_LoadStore()->adr_type();
787 } else if (n->is_Mem()) {
788 bd = MemNode::barrier_data(n);
789 verify_gc_barrier_assert(bd == 0, "Other mem nodes should have no barrier data", bd, n);
790 }
791
792 bool is_weak = (bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0;
793 bool is_native = (bd & ShenandoahBitNative) != 0;
794
795 bool is_referent = adr_type != nullptr &&
796 adr_type->isa_instptr() &&
797 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
798 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
799
800 bool is_oop_addr = (adr_type != nullptr) && (adr_type->isa_oopptr() || adr_type->isa_narrowoop());
801 bool is_raw_addr = (adr_type != nullptr) && (adr_type->isa_rawptr() || adr_type->isa_klassptr());
802
803 verify_gc_barrier_assert(!expect_some_real || (bd == 0) || (bd & ShenandoahBitsReal) != 0, "Without real barriers, metadata should be stripped at this point", bd, n);
804
805 if (is_oop_addr) {
806 if (is_Load(opc)) {
807 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Oop load should have barrier data", bd, n);
808 verify_gc_barrier_assert(!is_weak || is_referent, "Weak load only for Reference.referent", bd, n);
809 } else if (is_Store(opc)) {
810 // Reference.referent stores can be without barriers.
811 verify_gc_barrier_assert(!expect_store_barriers || is_referent || (bd != 0), "Oop store should have barrier data", bd, n);
812 } else if (is_LoadStore(opc)) {
813 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Oop load-store should have barrier data", bd, n);
814 }
815 } else if (is_raw_addr) {
816 if (is_native) {
817 if (is_Load(opc)) {
818 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Native oop load should have barrier data", bd, n);
819 }
820 if (is_Store(opc)) {
821 verify_gc_barrier_assert(!expect_store_barriers || (bd != 0), "Native oop store should have barrier data", bd, n);
822 }
823 if (is_LoadStore(opc)) {
824 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Native oop load-store should have barrier data", bd, n);
825 }
826 } else {
827 // Some Load/Stores are used for T_ADDRESS and/or raw stores, which are supposed not to have barriers.
828 // Some other Load/Stores are emitted for real oops, but on raw addresses via Unsafe.
829 // The distinction on this level is lost, so we cannot really verify this.
830 }
831 } else {
832 if (is_Load(opc) || is_Store(opc) || is_LoadStore(opc)) {
833 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
834 }
835 }
836
837 for (uint j = 0; j < n->req(); j++) {
838 Node* in = n->in(j);
839 if (in != nullptr) {
840 wq.push(in);
841 }
842 }
843 }
844 }
845 #endif
846
847 static ShenandoahBarrierSetC2State* barrier_set_state() {
848 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
849 }
850
851 int ShenandoahBarrierSetC2::estimate_stub_size() const {
852 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
853 assert(stubs->is_empty(), "Lifecycle: no stubs were yet created");
854 return 0;
855 }
856
857 void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
858 MacroAssembler masm(&cb);
859
860 PhaseOutput* const output = Compile::current()->output();
861 assert(masm.offset() <= output->buffer_sizing_data()->_code,
862 "Stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
863 barrier_set_state()->set_stubs_start_offset(masm.offset());
864
865 // Stub generation counts all stubs as skipped for the sake of inlining policy.
866 // This is critical for performance, check it.
867 #ifdef ASSERT
868 int offset_before = masm.offset();
869 int skipped_before = cb.total_skipped_instructions_size();
870 #endif
871
872 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
873 for (int i = 0; i < stubs->length(); i++) {
874 // Make sure there is enough space in the code buffer
875 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
876 ciEnv::current()->record_failure("CodeCache is full");
877 return;
878 }
879 stubs->at(i)->emit_code(masm);
880 }
881
882 #ifdef ASSERT
883 int offset_after = masm.offset();
884 int skipped_after = cb.total_skipped_instructions_size();
885 assert(offset_after - offset_before == skipped_after - skipped_before,
886 "All stubs are counted as skipped. masm: %d - %d = %d, cb: %d - %d = %d",
887 offset_after, offset_before, offset_after - offset_before,
888 skipped_after, skipped_before, skipped_after - skipped_before);
889 #endif
890
891 masm.flush();
892 }
893
894 void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
895 if (!Compile::current()->output()->in_scratch_emit_size()) {
896 barrier_set_state()->stubs()->append(stub);
897 }
898 }
899
900 ShenandoahBarrierStubC2* ShenandoahBarrierStubC2::create(const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow, bool do_load) {
901 auto* stub = new (Compile::current()->comp_arena()) ShenandoahBarrierStubC2(node, obj, addr, tmp1, tmp2, narrow, do_load);
902 register_stub(stub);
903 return stub;
904 }
905
906 void ShenandoahBarrierStubC2::load_post(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
907 // Load post-barrier:
908 // a. Satisfies the need for LRB for normal loads
909 // b. Passes a weak load through LRB-weak
910 // c. Keep-alives a weak load
911 if (needs_slow_barrier(node)) {
912 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ false);
913 char check = 0;
914 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
915 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
916 check |= needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
917 stub->enter_if_gc_state(*masm, check, tmp1);
918 }
919 }
920
921 void ShenandoahBarrierStubC2::store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
922 // Store pre-barrier: SATB, keep-alive the current memory value.
923 if (needs_slow_barrier(node)) {
924 assert(!needs_load_ref_barrier(node), "Should not be required for stores");
925 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
926 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING, tmp1);
927 }
928 }
929
930 void ShenandoahBarrierStubC2::load_store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
931 // Load/Store pre-barrier:
932 // a. Avoids false positives from CAS encountering to-space memory values.
933 // b. Satisfies the need for LRB for the CAE result.
934 // c. Records old value for the sake of SATB.
935 //
936 // (a) and (b) are covered because load barrier does memory location fixup.
937 // (c) is covered by KA on the current memory value.
938 if (needs_slow_barrier(node)) {
939 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
940 char check = 0;
941 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
942 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
943 assert(!needs_load_ref_barrier_weak(node), "Not supported for Load/Stores");
944 stub->enter_if_gc_state(*masm, check, tmp1);
945 }
946 }
947
948 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
949 if (needs_card_barrier(node)) {
950 cardtable(*masm, addr, tmp1, tmp2);
951 }
952 }
953
954 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
955 store_post(masm, node, addr, tmp1, tmp2);
956 }
957
958 bool ShenandoahBarrierStubC2::is_live_register(Register reg) {
959 return preserve_set().member(OptoReg::as_OptoReg(reg->as_VMReg()));
960 }
961
962 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Register skip_reg1, Register skip_reg2) {
963 Register tmp = noreg;
964 Register fallback_live = noreg;
965
966 // Try to select non-live first:
967 for (int i = 0; i < available_gp_registers(); i++) {
968 Register r = as_Register(i);
969 if (r != _obj && r != _addr.base() && r != _addr.index() &&
970 r != skip_reg1 && r != skip_reg2 && !is_special_register(r)) {
971 if (!is_live_register(r)) {
972 tmp = r;
973 break;
974 } else if (fallback_live == noreg) {
975 fallback_live = r;
976 }
977 }
978 }
979
980 // If we could not find a non-live register, select the live fallback:
981 if (tmp == noreg) {
982 tmp = fallback_live;
983 selected_live = true;
984 } else {
985 selected_live = false;
986 }
987
988 assert(tmp != noreg, "successfully selected");
989 assert_different_registers(tmp, skip_reg1);
990 assert_different_registers(tmp, skip_reg2);
991 assert_different_registers(tmp, _obj);
992 assert_different_registers(tmp, _addr.base());
993 assert_different_registers(tmp, _addr.index());
994 return tmp;
995 }
996
997 address ShenandoahBarrierStubC2::keepalive_runtime_entry_addr() {
998 if (_narrow) {
999 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_narrow);
1000 } else {
1001 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre);
1002 }
1003 }
1004
1005 address ShenandoahBarrierStubC2::lrb_runtime_entry_addr() {
1006 bool is_strong = (_node->barrier_data() & ShenandoahBitStrong) != 0;
1007 bool is_weak = (_node->barrier_data() & ShenandoahBitWeak) != 0;
1008 bool is_phantom = (_node->barrier_data() & ShenandoahBitPhantom) != 0;
1009
1010 if (_narrow) {
1011 if (is_strong) {
1012 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow_narrow);
1013 } else if (is_weak) {
1014 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow_narrow);
1015 } else if (is_phantom) {
1016 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow_narrow);
1017 }
1018 } else {
1019 if (is_strong) {
1020 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1021 } else if (is_weak) {
1022 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1023 } else if (is_phantom) {
1024 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1025 }
1026 }
1027
1028 ShouldNotReachHere();
1029 return nullptr;
1030 }
1031
1032 bool ShenandoahBarrierSetC2State::needs_liveness_data(const MachNode* mach) const {
1033 // Nodes that require slow-path stubs need liveness data.
1034 return ShenandoahBarrierStubC2::needs_slow_barrier(mach);
1035 }
1036
1037 bool ShenandoahBarrierSetC2State::needs_livein_data() const {
1038 return true;
1039 }