1 /*
2 * Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "opto/arraycopynode.hpp"
34 #include "opto/escape.hpp"
35 #include "opto/graphKit.hpp"
36 #include "opto/idealKit.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/narrowptrnode.hpp"
39 #include "opto/output.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42
43 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
44 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
45 }
46
47 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
48 BarrierSetC2State(comp_arena),
49 _stubs(new (comp_arena) GrowableArray<ShenandoahBarrierStubC2*>(comp_arena, 8, 0, nullptr)),
50 _trampoline_stubs_count(0),
51 _stubs_start_offset(0),
52 _stubs_current_total_size(0) {
53 }
54
55 static void set_barrier_data(C2Access& access, bool load, bool store) {
56 if (!access.is_oop()) {
57 return;
58 }
59
60 DecoratorSet decorators = access.decorators();
61 bool tightly_coupled = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
62 bool in_heap = (decorators & IN_HEAP) != 0;
63 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
64 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
65
66 if (tightly_coupled) {
67 access.set_barrier_data(ShenandoahBitElided);
68 return;
69 }
70
71 uint8_t barrier_data = 0;
72
73 if (load) {
74 if (ShenandoahLoadRefBarrier) {
75 if (on_phantom) {
76 barrier_data |= ShenandoahBitPhantom;
77 } else if (on_weak) {
78 barrier_data |= ShenandoahBitWeak;
79 } else {
80 barrier_data |= ShenandoahBitStrong;
81 }
82 }
83 }
84
85 if (store) {
86 if (ShenandoahSATBBarrier) {
87 barrier_data |= ShenandoahBitKeepAlive;
88 }
89 if (ShenandoahCardBarrier && in_heap) {
90 barrier_data |= ShenandoahBitCardMark;
91 }
92 }
93
94 if (!in_heap) {
95 barrier_data |= ShenandoahBitNative;
96 }
97
98 access.set_barrier_data(barrier_data);
99 }
100
101 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
102 // 1: Non-reference load, no additional barrier is needed
103 if (!access.is_oop()) {
104 return BarrierSetC2::load_at_resolved(access, val_type);
105 }
106
107 // 2. Set barrier data for load
108 set_barrier_data(access, /* load = */ true, /* store = */ false);
109
110 // 3. Correction: If we are reading the value of the referent field of
111 // a Reference object, we need to record the referent resurrection.
112 DecoratorSet decorators = access.decorators();
113 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
114 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
115 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
116 bool needs_keepalive = ((on_weak || on_phantom) && !no_keepalive);
117 if (needs_keepalive) {
118 uint8_t barriers = access.barrier_data() | (ShenandoahSATBBarrier ? ShenandoahBitKeepAlive : 0);
119 access.set_barrier_data(barriers);
120 }
121
122 return BarrierSetC2::load_at_resolved(access, val_type);
123 }
124
125 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
126 // 1: Non-reference store, no additional barrier is needed
127 if (!access.is_oop()) {
128 return BarrierSetC2::store_at_resolved(access, val);
129 }
130
131 // 2. Set barrier data for store
132 set_barrier_data(access, /* load = */ false, /* store = */ true);
133
134 // 3. Correction: avoid keep-alive barriers that should not do keep-alive.
135 DecoratorSet decorators = access.decorators();
136 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
137 if (no_keepalive) {
138 access.set_barrier_data(access.barrier_data() & ~ShenandoahBitKeepAlive);
139 }
140
141 return BarrierSetC2::store_at_resolved(access, val);
142 }
143
144 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
145 Node* new_val, const Type* value_type) const {
146 set_barrier_data(access, /* load = */ true, /* store = */ true);
147 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
148 }
149
150 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
151 Node* new_val, const Type* value_type) const {
152 set_barrier_data(access, /* load = */ true, /* store = */ true);
153 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
154 }
155
156 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
157 set_barrier_data(access, /* load = */ true, /* store = */ true);
158 return BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
159 }
160
161 bool ShenandoahBarrierSetC2::is_Load(int opcode) {
162 switch (opcode) {
163 case Op_LoadN:
164 case Op_LoadP:
165 return true;
166 default:
167 return false;
168 }
169 }
170
171 bool ShenandoahBarrierSetC2::is_Store(int opcode) {
172 switch (opcode) {
173 case Op_StoreN:
174 case Op_StoreP:
175 return true;
176 default:
177 return false;
178 }
179 }
180
181 bool ShenandoahBarrierSetC2::is_LoadStore(int opcode) {
182 switch (opcode) {
183 case Op_CompareAndExchangeN:
184 case Op_CompareAndExchangeP:
185 case Op_WeakCompareAndSwapN:
186 case Op_WeakCompareAndSwapP:
187 case Op_CompareAndSwapN:
188 case Op_CompareAndSwapP:
189 case Op_GetAndSetP:
190 case Op_GetAndSetN:
191 return true;
192 default:
193 return false;
194 }
195 }
196
197 bool ShenandoahBarrierSetC2::can_remove_load_barrier(Node* root) {
198 // Check if all outs feed into nodes that do not expose the oops to the rest
199 // of the runtime system. In this case, we can elide the LRB barrier. We bail
200 // out with false at the first sight of trouble.
201
202 ResourceMark rm;
203 VectorSet visited;
204 Node_List worklist;
205 worklist.push(root);
206
207 while (worklist.size() > 0) {
208 Node* n = worklist.pop();
209 if (visited.test_set(n->_idx)) {
210 continue;
211 }
212
213 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
214 Node* out = n->fast_out(i);
215 switch (out->Opcode()) {
216 case Op_Phi:
217 case Op_EncodeP:
218 case Op_DecodeN:
219 case Op_CastPP:
220 case Op_CheckCastPP:
221 case Op_AddP: {
222 // Transitive node, check if any other outs are doing anything troublesome.
223 worklist.push(out);
224 break;
225 }
226
227 case Op_LoadRange: {
228 // Array length is normally the same in all copies, so it can be read
229 // from a from-space copy without a barrier. With +UCOH, however, the
230 // array length lives in the mark word, which is overwritten by the
231 // forwarding pointer during evacuation, so reading it from a from-space
232 // copy would yield garbage. Keep the barrier in that case.
233 // Mirrors Op_LoadNKlass below.
234 if (!UseCompactObjectHeaders) {
235 break;
236 }
237 return false;
238 }
239
240 case Op_LoadKlass: {
241 // Klass is the same in all copies.
242 // We would have liked to assert -UCOH, but there are legitimate klass
243 // loads from native Klass* instances, which are also safe under +UCOH.
244 break;
245 }
246
247 case Op_LoadNKlass: {
248 // Similar to above, but LoadNKlass is only safe without +UCOH.
249 // With +UCOH, it loads from mark word, which clashes with forwarding pointers.
250 if (!UseCompactObjectHeaders) {
251 break;
252 }
253 return false;
254 }
255
256 case Op_CmpN: {
257 if (out->in(1) == n &&
258 out->in(2)->Opcode() == Op_ConN &&
259 out->in(2)->get_narrowcon() == 0) {
260 // Null check, no oop is exposed.
261 break;
262 }
263 if (out->in(2) == n &&
264 out->in(1)->Opcode() == Op_ConN &&
265 out->in(1)->get_narrowcon() == 0) {
266 // Null check, no oop is exposed.
267 break;
268 }
269 return false;
270 }
271
272 case Op_CmpP: {
273 if (out->in(1) == n &&
274 out->in(2)->Opcode() == Op_ConP &&
275 out->in(2)->get_ptr() == 0) {
276 // Null check, no oop is exposed.
277 break;
278 }
279 if (out->in(2) == n &&
280 out->in(1)->Opcode() == Op_ConP &&
281 out->in(1)->get_ptr() == 0) {
282 // Null check, no oop is exposed.
283 break;
284 }
285 return false;
286 }
287
288 case Op_CallStaticJava: {
289 if (out->as_CallStaticJava()->is_uncommon_trap()) {
290 // Local feeds into uncommon trap. Deopt machinery handles barriers itself.
291 break;
292 }
293 return false;
294 }
295
296 default: {
297 // Paranoidly distrust any other nodes.
298 return false;
299 }
300 }
301 }
302 }
303
304 // Nothing troublesome found.
305 return true;
306 }
307
308 uint8_t ShenandoahBarrierSetC2::refine_load(Node* n, uint8_t bd) {
309 assert(ShenandoahElideIdealBarriers, "Checked by caller");
310 assert(bd != 0, "Checked by caller");
311
312 // Do not touch weak loads at all: they are responsible for shielding from
313 // Reference.referent resurrection.
314 if ((bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0) {
315 return bd;
316 }
317
318 if (((bd & ShenandoahBitStrong) != 0) && can_remove_load_barrier(n)) {
319 bd &= ~ShenandoahBitStrong;
320 }
321
322 return bd;
323 }
324
325 uint8_t ShenandoahBarrierSetC2::refine_store(Node* n, uint8_t bd) {
326 assert(ShenandoahElideIdealBarriers, "Checked by caller");
327 assert(bd != 0, "Checked by caller");
328 assert(n->is_Mem() || n->is_LoadStore(), "Sanity");
329
330 const Node* newval = n->in(MemNode::ValueIn);
331 assert(newval != nullptr, "Should be present");
332
333 // Type system tells us something about nullity?
334 const Type* newval_bottom = newval->bottom_type();
335 assert(newval_bottom->isa_oopptr() || newval_bottom->isa_narrowoop() ||
336 newval_bottom == TypePtr::NULL_PTR, "Should be an oop store");
337 const TypePtr* newval_type = newval_bottom->make_ptr();
338 assert(newval_type != nullptr, "Should have been filtered before");
339 TypePtr::PTR newval_type_ptr = newval_type->ptr();
340 if (newval_type_ptr == TypePtr::Null) {
341 bd &= ~ShenandoahBitNotNull;
342 // Card table barrier is not needed if we store null.
343 bd &= ~ShenandoahBitCardMark;
344 } else if (newval_type_ptr == TypePtr::NotNull) {
345 // Definitely not null.
346 bd |= ShenandoahBitNotNull;
347 }
348
349 return bd;
350 }
351
352 void ShenandoahBarrierSetC2::final_refinement(Compile* compile) const {
353 ResourceMark rm;
354 Unique_Node_List wq;
355
356 RootNode* root = compile->root();
357 wq.push(root);
358
359 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
360 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
361 Node* m = root->fast_out(i);
362 wq.push(m);
363 }
364
365 for (uint next = 0; next < wq.size(); next++) {
366 Node* n = wq.at(next);
367
368 assert(!n->is_Mach(), "No Mach nodes here yet");
369
370 int opc = n->Opcode();
371 bool is_load = is_Load(opc);
372 bool is_store = is_Store(opc);
373 bool is_load_store = is_LoadStore(opc);
374
375 uint8_t orig_bd = 0;
376 if (is_load_store) {
377 orig_bd = n->as_LoadStore()->barrier_data();
378 } else if (is_load || is_store) {
379 orig_bd = n->as_Mem()->barrier_data();
380 }
381
382 uint8_t bd = orig_bd;
383 if (ShenandoahElideIdealBarriers && bd != 0) {
384 // Note: we cannot apply load optimizations to LoadStores,
385 // because their load barriers are needed for fixups.
386 if (is_load) {
387 bd = refine_load(n, bd);
388 }
389 if (is_store || is_load_store) {
390 bd = refine_store(n, bd);
391 }
392 }
393
394 // If there are no real barrier flags on the node, strip away additional fluff.
395 // Matcher does not care about this, and we would like to avoid invoking "barrier_data() != 0"
396 // rules when the only flags are the irrelevant fluff.
397 if ((bd != 0) && (bd & ShenandoahBitsReal) == 0) {
398 bd = 0;
399 }
400
401 if (bd != orig_bd) {
402 if (is_load_store) {
403 n->as_LoadStore()->set_barrier_data(bd);
404 } else {
405 n->as_Mem()->set_barrier_data(bd);
406 }
407 }
408
409 for (uint j = 0; j < n->req(); j++) {
410 Node* in = n->in(j);
411 if (in != nullptr) {
412 wq.push(in);
413 }
414 }
415 }
416 }
417
418 // Support for macro expanded GC barriers
419 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
420 if (node->is_LoadStore()) {
421 LoadStoreNode* loadstore = node->as_LoadStore();
422 loadstore->set_barrier_data(0);
423 } else if (node->is_Mem()) {
424 MemNode* mem = node->as_Mem();
425 mem->set_barrier_data(0);
426 }
427 }
428
429 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
430 eliminate_gc_barrier_data(node);
431 }
432
433 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
434 uint8_t orig_bd = node->barrier_data();
435 if (orig_bd == 0) {
436 // Nothing to do.
437 return;
438 }
439
440 uint8_t bd = orig_bd;
441 int node_opcode = node->ideal_Opcode();
442
443 if (dominator == nullptr) {
444 // Must be allocation node.
445 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
446 // Loads from recent allocations do not need LRBs.
447 bd &= ~ShenandoahBitStrong;
448 }
449 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
450 // Stores to recent allocations do not need KA or CM.
451 bd &= ~ShenandoahBitKeepAlive;
452 bd &= ~ShenandoahBitCardMark;
453 }
454 } else {
455 // LoadStores do not get these optimizations, since their LRBs
456 // are required for fixups.
457 if (is_Load(node_opcode) || is_Store(node_opcode)) {
458 int dom_opcode = dominator->ideal_Opcode();
459 uint8_t dom_bd = dominator->barrier_data();
460
461 if (is_Load(dom_opcode) || is_LoadStore(dom_opcode)) {
462 // If dominating load is set up to perform LRB fixups, no further LRB is needed.
463 if ((dom_bd & ShenandoahBitStrong) != 0) {
464 bd &= ~ShenandoahBitStrong;
465 }
466 }
467 if (is_Store(dom_opcode)) {
468 // Dominating store has stored the good ref, no LRB is needed.
469 bd &= ~ShenandoahBitStrong;
470 }
471 }
472 }
473
474 if (orig_bd != bd) {
475 // We are already in final output.
476 // Strip the extra barrier data if no real bits are left.
477 if ((bd & ShenandoahBitsReal) != 0) {
478 node->set_barrier_data(bd);
479 } else {
480 node->set_barrier_data(0);
481 }
482 }
483 }
484
485 void ShenandoahBarrierSetC2::analyze_dominating_barriers() const {
486 if (!ShenandoahElideMachBarriers) {
487 return;
488 }
489
490 ResourceMark rm;
491 Node_List accesses, dominators;
492
493 PhaseCFG* const cfg = Compile::current()->cfg();
494 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
495 const Block* const block = cfg->get_block(i);
496 for (uint j = 0; j < block->number_of_nodes(); ++j) {
497 Node* const node = block->get_node(j);
498
499 // Everything that happens in allocations does not need barriers.
500 // Record them for dominance analysis.
501 if (node->is_Phi() && is_allocation(node)) {
502 dominators.push(node);
503 continue;
504 }
505
506 if (!node->is_Mach()) {
507 continue;
508 }
509
510 MachNode* const mach = node->as_Mach();
511 int opcode = mach->ideal_Opcode();
512 if (is_Load(opcode) || is_Store(opcode) || is_LoadStore(opcode)) {
513 if ((mach->barrier_data() & ShenandoahBitsReal) != 0) {
514 accesses.push(mach);
515 dominators.push(mach);
516 }
517 }
518 }
519 }
520
521 elide_dominated_barriers(accesses, dominators);
522 }
523
524 uint ShenandoahBarrierSetC2::estimated_barrier_size(const Node* node) const {
525 // Barrier impact on fast-path is driven by GC state checks emitted very late.
526 // These checks are tight load-test-branch sequences, with no impact on C2 graph
527 // size. Limiting unrolling in presence of GC barriers might turn some loops
528 // tighter than with default unrolling, which may benefit performance due to denser
529 // code. Testing shows it is still counter-productive.
530 // Therefore, we report zero barrier size to let C2 do its normal thing.
531 return 0;
532 }
533
534 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const {
535 bool is_oop = is_reference_type(type);
536 if (!is_oop) {
537 return false;
538 }
539 if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
540 if (phase == Optimization) {
541 return false;
542 }
543 return !is_clone;
544 }
545 return true;
546 }
547
548 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
549 if (!ShenandoahCloneBarrier) {
550 return false;
551 }
552
553 if (src_type->isa_instptr() != nullptr) {
554 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
555 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
556 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
557 !ik->has_injected_fields() && !ik->has_object_fields()) {
558 if (!src_type->klass_is_exact()) {
559 // Class is *currently* the leaf in the hierarchy.
560 // Record the dependency so that we deopt if this does not hold in future.
561 Compile::current()->dependencies()->assert_leaf_type(ik);
562 }
563 return false;
564 }
565 } else if (src_type->isa_aryptr() != nullptr) {
566 // Array: need barrier only if array is oop-bearing.
567 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
568 if (is_reference_type(src_elem, true)) {
569 is_oop_array = true;
570 } else {
571 return false;
572 }
573 }
574
575 // Assume the worst.
576 return true;
577 }
578
579 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
580 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
581
582 bool is_oop_array = false;
583 if (!clone_needs_barrier(src_type, is_oop_array)) {
584 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
585 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
586 return;
587 }
588
589 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
590 // Looks like an instance? Prepare the instance clone. This would either
591 // be exploded into individual accesses or be left as runtime call.
592 // Common BarrierSetC2 prepares everything for both cases.
593 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
594 return;
595 }
596
597 // We are cloning the oop array. Prepare to call the normal arraycopy stub
598 // after the expansion. Normal stub takes the number of actual type-sized
599 // elements to copy after the base, compute the count here.
600 Node* offset = kit->MakeConX(arrayOopDesc::base_offset_in_bytes(UseCompressedOops ? T_NARROWOOP : T_OBJECT));
601 size = kit->gvn().transform(new SubXNode(size, offset));
602 size = kit->gvn().transform(new URShiftXNode(size, kit->intcon(LogBytesPerHeapOop)));
603 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, size, true, false);
604 ac->set_clone_array();
605 Node* n = kit->gvn().transform(ac);
606 if (n == ac) {
607 ac->set_adr_type(TypeRawPtr::BOTTOM);
608 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), TypeRawPtr::BOTTOM);
609 } else {
610 kit->set_all_memory(n);
611 }
612 }
613
614 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
615 Node* const ctrl = ac->in(TypeFunc::Control);
616 Node* const mem = ac->in(TypeFunc::Memory);
617 Node* const src = ac->in(ArrayCopyNode::Src);
618 Node* const src_offset = ac->in(ArrayCopyNode::SrcPos);
619 Node* const dest = ac->in(ArrayCopyNode::Dest);
620 Node* const dest_offset = ac->in(ArrayCopyNode::DestPos);
621 Node* length = ac->in(ArrayCopyNode::Length);
622
623 const TypeOopPtr* src_type = phase->igvn().type(src)->is_oopptr();
624
625 bool is_oop_array = false;
626 if (!clone_needs_barrier(src_type, is_oop_array)) {
627 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
628 BarrierSetC2::clone_at_expansion(phase, ac);
629 return;
630 }
631
632 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
633 // Still looks like an instance? Likely a large instance or reflective
634 // clone with unknown length. Go to runtime and handle it there.
635 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
636 return;
637 }
638
639 // We are cloning the oop array. Call into normal oop array copy stubs.
640 // Those stubs would call BarrierSetAssembler to handle GC barriers.
641
642 // This is the full clone, so offsets should equal each other and be at array base.
643 assert(src_offset == dest_offset, "should be equal");
644 const jlong offset = src_offset->get_long();
645 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
646 BasicType bt = ary_ptr->elem()->array_element_basic_type();
647 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
648
649 const char* copyfunc_name = "arraycopy";
650 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
651
652 Node* const call = phase->make_leaf_call(ctrl, mem,
653 OptoRuntime::fast_arraycopy_Type(),
654 copyfunc_addr, copyfunc_name,
655 TypeRawPtr::BOTTOM,
656 phase->basic_plus_adr(src, src_offset),
657 phase->basic_plus_adr(dest, dest_offset),
658 length,
659 phase->top()
660 );
661 phase->transform_later(call);
662
663 phase->igvn().replace_node(ac, call);
664 }
665
666 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
667 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
668 }
669
670 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
671 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
672 }
673
674 void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
675 os->print(" Node barriers: ");
676 if ((data & ShenandoahBitStrong) != 0) {
677 data &= ~ShenandoahBitStrong;
678 os->print("strong ");
679 }
680
681 if ((data & ShenandoahBitWeak) != 0) {
682 data &= ~ShenandoahBitWeak;
683 os->print("weak ");
684 }
685
686 if ((data & ShenandoahBitPhantom) != 0) {
687 data &= ~ShenandoahBitPhantom;
688 os->print("phantom ");
689 }
690
691 if ((data & ShenandoahBitKeepAlive) != 0) {
692 data &= ~ShenandoahBitKeepAlive;
693 os->print("keepalive ");
694 }
695
696 if ((data & ShenandoahBitCardMark) != 0) {
697 data &= ~ShenandoahBitCardMark;
698 os->print("cardmark ");
699 }
700
701 if ((data & ShenandoahBitNative) != 0) {
702 data &= ~ShenandoahBitNative;
703 os->print("native ");
704 }
705
706 if ((data & ShenandoahBitNotNull) != 0) {
707 data &= ~ShenandoahBitNotNull;
708 os->print("not-null ");
709 }
710
711 if ((data & ShenandoahBitElided) != 0) {
712 data &= ~ShenandoahBitElided;
713 os->print("elided ");
714 }
715
716 os->cr();
717
718 if (data > 0) {
719 fatal("Unknown bit!");
720 }
721
722 os->print_cr(" GC configuration: %sLRB %sSATB %sClone %sCard",
723 (ShenandoahLoadRefBarrier ? "+" : "-"),
724 (ShenandoahSATBBarrier ? "+" : "-"),
725 (ShenandoahCloneBarrier ? "+" : "-"),
726 (ShenandoahCardBarrier ? "+" : "-")
727 );
728 }
729
730
731 #ifdef ASSERT
732 void ShenandoahBarrierSetC2::verify_gc_barrier_assert(bool cond, const char* msg, uint8_t bd, Node* n) {
733 if (!cond) {
734 stringStream ss;
735 ss.print_cr("%s", msg);
736 ss.print_cr("-----------------");
737 print_barrier_data(&ss, bd);
738 ss.print_cr("-----------------");
739 n->dump_bfs(1, nullptr, "", &ss);
740 report_vm_error(__FILE__, __LINE__, ss.as_string());
741 }
742 }
743
744 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
745 if (!ShenandoahVerifyOptoBarriers) {
746 return;
747 }
748
749 // Verify depending on the barriers actually enabled, allowing verification in passive mode.
750 // Normally, we have _some_ bits set on all accesses. Optimizations may drop some bits,
751 // but only the last optimization step eliminates all remaining metadata flags. Only then
752 // the access data can be completely blank.
753 bool final_phase = (phase == BeforeCodeGen);
754 bool expect_load_barriers = !final_phase && ShenandoahLoadRefBarrier;
755 bool expect_store_barriers = !final_phase && (ShenandoahSATBBarrier || ShenandoahCardBarrier);
756 bool expect_load_store_barriers = expect_load_barriers || expect_store_barriers;
757 bool expect_some_real = final_phase;
758
759 Unique_Node_List wq;
760
761 RootNode* root = compile->root();
762 wq.push(root);
763
764 // Also seed the outs to capture nodes are not reachable from in()-s, e.g. endless loops.
765 for (DUIterator_Fast imax, i = root->fast_outs(imax); i < imax; i++) {
766 Node* m = root->fast_out(i);
767 wq.push(m);
768 }
769
770 for (uint next = 0; next < wq.size(); next++) {
771 Node *n = wq.at(next);
772 assert(!n->is_Mach(), "No Mach nodes here yet");
773
774 int opc = n->Opcode();
775
776 uint8_t bd = 0;
777 const TypePtr* adr_type = nullptr;
778 if (is_Load(opc)) {
779 bd = n->as_Load()->barrier_data();
780 adr_type = n->as_Load()->adr_type();
781 } else if (is_Store(opc)) {
782 bd = n->as_Store()->barrier_data();
783 adr_type = n->as_Store()->adr_type();
784 } else if (is_LoadStore(opc)) {
785 bd = n->as_LoadStore()->barrier_data();
786 adr_type = n->as_LoadStore()->adr_type();
787 } else if (n->is_Mem()) {
788 bd = MemNode::barrier_data(n);
789 verify_gc_barrier_assert(bd == 0, "Other mem nodes should have no barrier data", bd, n);
790 }
791
792 bool is_weak = (bd & (ShenandoahBitWeak | ShenandoahBitPhantom)) != 0;
793 bool is_native = (bd & ShenandoahBitNative) != 0;
794
795 bool is_referent = adr_type != nullptr &&
796 adr_type->isa_instptr() &&
797 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
798 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset();
799
800 bool is_oop_addr = (adr_type != nullptr) && (adr_type->isa_oopptr() || adr_type->isa_narrowoop());
801 bool is_raw_addr = (adr_type != nullptr) && (adr_type->isa_rawptr() || adr_type->isa_klassptr());
802
803 verify_gc_barrier_assert(!expect_some_real || (bd == 0) || (bd & ShenandoahBitsReal) != 0, "Without real barriers, metadata should be stripped at this point", bd, n);
804
805 if (is_oop_addr) {
806 if (is_Load(opc)) {
807 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Oop load should have barrier data", bd, n);
808 verify_gc_barrier_assert(!is_weak || is_referent, "Weak load only for Reference.referent", bd, n);
809 } else if (is_Store(opc)) {
810 // Reference.referent stores can be without barriers.
811 verify_gc_barrier_assert(!expect_store_barriers || is_referent || (bd != 0), "Oop store should have barrier data", bd, n);
812 } else if (is_LoadStore(opc)) {
813 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Oop load-store should have barrier data", bd, n);
814 }
815 } else if (is_raw_addr) {
816 if (is_native) {
817 if (is_Load(opc)) {
818 verify_gc_barrier_assert(!expect_load_barriers || (bd != 0), "Native oop load should have barrier data", bd, n);
819 }
820 if (is_Store(opc)) {
821 verify_gc_barrier_assert(!expect_store_barriers || (bd != 0), "Native oop store should have barrier data", bd, n);
822 }
823 if (is_LoadStore(opc)) {
824 verify_gc_barrier_assert(!expect_load_store_barriers || (bd != 0), "Native oop load-store should have barrier data", bd, n);
825 }
826 } else {
827 // Some Load/Stores are used for T_ADDRESS and/or raw stores, which are supposed not to have barriers.
828 // Some other Load/Stores are emitted for real oops, but on raw addresses via Unsafe.
829 // The distinction on this level is lost, so we cannot really verify this.
830 }
831 } else {
832 if (is_Load(opc) || is_Store(opc) || is_LoadStore(opc)) {
833 verify_gc_barrier_assert(false, "Unclassified access type", bd, n);
834 }
835 }
836
837 for (uint j = 0; j < n->req(); j++) {
838 Node* in = n->in(j);
839 if (in != nullptr) {
840 wq.push(in);
841 }
842 }
843 }
844 }
845 #endif
846
847 static ShenandoahBarrierSetC2State* barrier_set_state() {
848 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
849 }
850
851 int ShenandoahBarrierSetC2::estimate_stub_size() const {
852 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
853 assert(stubs->is_empty(), "Lifecycle: no stubs were yet created");
854 return 0;
855 }
856
857 void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
858 MacroAssembler masm(&cb);
859
860 PhaseOutput* const output = Compile::current()->output();
861 assert(masm.offset() <= output->buffer_sizing_data()->_code,
862 "Stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
863 barrier_set_state()->set_stubs_start_offset(masm.offset());
864
865 // Stub generation counts all stubs as skipped for the sake of inlining policy.
866 // This is critical for performance, check it.
867 #ifdef ASSERT
868 int offset_before = masm.offset();
869 int skipped_before = cb.total_skipped_instructions_size();
870 #endif
871
872 GrowableArray<ShenandoahBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
873 for (int i = 0; i < stubs->length(); i++) {
874 // Make sure there is enough space in the code buffer
875 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
876 ciEnv::current()->record_failure("CodeCache is full");
877 return;
878 }
879 stubs->at(i)->emit_code(masm);
880 }
881
882 #ifdef ASSERT
883 int offset_after = masm.offset();
884 int skipped_after = cb.total_skipped_instructions_size();
885 assert(offset_after - offset_before == skipped_after - skipped_before,
886 "All stubs are counted as skipped. masm: %d - %d = %d, cb: %d - %d = %d",
887 offset_after, offset_before, offset_after - offset_before,
888 skipped_after, skipped_before, skipped_after - skipped_before);
889 #endif
890
891 masm.flush();
892 }
893
894 void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
895 if (!Compile::current()->output()->in_scratch_emit_size()) {
896 barrier_set_state()->stubs()->append(stub);
897 }
898 }
899
900 ShenandoahBarrierStubC2* ShenandoahBarrierStubC2::create(const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow, bool do_load) {
901 auto* stub = new (Compile::current()->comp_arena()) ShenandoahBarrierStubC2(node, obj, addr, tmp1, tmp2, narrow, do_load);
902 register_stub(stub);
903 return stub;
904 }
905
906 void ShenandoahBarrierStubC2::load_post(MacroAssembler* masm, const MachNode* node, Register obj, Address addr, Register tmp1, Register tmp2, bool narrow) {
907 // Load post-barrier:
908 // a. Satisfies the need for LRB for normal loads
909 // b. Passes a weak load through LRB-weak
910 // c. Keep-alives a weak load
911 if (needs_slow_barrier(node)) {
912 ShenandoahBarrierStubC2* const stub = create(node, obj, addr, tmp1, tmp2, narrow, /* do_load = */ false);
913 char check = 0;
914 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
915 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
916 check |= needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
917 stub->enter_if_gc_state(*masm, check, tmp1);
918 }
919 }
920
921 void ShenandoahBarrierStubC2::store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
922 // Store pre-barrier: SATB, keep-alive the current memory value.
923 if (needs_slow_barrier(node)) {
924 assert(!needs_load_ref_barrier(node), "Should not be required for stores");
925 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
926 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING, tmp1);
927 }
928 }
929
930 void ShenandoahBarrierStubC2::load_store_pre(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
931 // Load/Store pre-barrier:
932 // a. Avoids false positives from CAS encountering to-space memory values.
933 // b. Satisfies the need for LRB for the CAE result.
934 // c. Records old value for the sake of SATB.
935 //
936 // (a) and (b) are covered because load barrier does memory location fixup.
937 // (c) is covered by KA on the current memory value.
938 if (needs_slow_barrier(node)) {
939 ShenandoahBarrierStubC2* const stub = create(node, tmp1, addr, tmp2, tmp3, narrow, /* do_load = */ true);
940 char check = 0;
941 check |= needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
942 check |= needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
943 assert(!needs_load_ref_barrier_weak(node), "Not supported for Load/Stores");
944 stub->enter_if_gc_state(*masm, check, tmp1);
945 }
946 }
947
948 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
949 if (needs_card_barrier(node)) {
950 cardtable(*masm, addr, tmp1, tmp2);
951 }
952 }
953
954 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address addr, Register tmp1, Register tmp2) {
955 store_post(masm, node, addr, tmp1, tmp2);
956 }
957
958 bool ShenandoahBarrierStubC2::is_live_register(Register reg) {
959 return preserve_set().member(OptoReg::as_OptoReg(reg->as_VMReg()));
960 }
961
962 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Register skip_reg1, Register skip_reg2) {
963 Register tmp = noreg;
964 Register fallback_live = noreg;
965
966 // Try to select non-live first:
967 for (int i = 0; i < available_gp_registers(); i++) {
968 Register r = as_Register(i);
969 if (r != _obj && r != _addr.base() && r != _addr.index() &&
970 r != skip_reg1 && r != skip_reg2 && !is_special_register(r)) {
971 if (!is_live_register(r)) {
972 tmp = r;
973 break;
974 } else if (fallback_live == noreg) {
975 fallback_live = r;
976 }
977 }
978 }
979
980 // If we could not find a non-live register, select the live fallback:
981 if (tmp == noreg) {
982 tmp = fallback_live;
983 selected_live = true;
984 } else {
985 selected_live = false;
986 }
987
988 assert(tmp != noreg, "successfully selected");
989 assert_different_registers(tmp, skip_reg1);
990 assert_different_registers(tmp, skip_reg2);
991 assert_different_registers(tmp, _obj);
992 assert_different_registers(tmp, _addr.base());
993 assert_different_registers(tmp, _addr.index());
994 return tmp;
995 }
996
997 address ShenandoahBarrierStubC2::keepalive_runtime_entry_addr() {
998 if (_narrow) {
999 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_narrow);
1000 } else {
1001 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre);
1002 }
1003 }
1004
1005 address ShenandoahBarrierStubC2::lrb_runtime_entry_addr() {
1006 bool is_strong = (_node->barrier_data() & ShenandoahBitStrong) != 0;
1007 bool is_weak = (_node->barrier_data() & ShenandoahBitWeak) != 0;
1008 bool is_phantom = (_node->barrier_data() & ShenandoahBitPhantom) != 0;
1009
1010 if (_narrow) {
1011 if (is_strong) {
1012 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow_narrow);
1013 } else if (is_weak) {
1014 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow_narrow);
1015 } else if (is_phantom) {
1016 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow_narrow);
1017 }
1018 } else {
1019 if (is_strong) {
1020 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1021 } else if (is_weak) {
1022 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1023 } else if (is_phantom) {
1024 return CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1025 }
1026 }
1027
1028 ShouldNotReachHere();
1029 return nullptr;
1030 }
1031
1032 bool ShenandoahBarrierSetC2State::needs_liveness_data(const MachNode* mach) const {
1033 // Nodes that require slow-path stubs need liveness data.
1034 return ShenandoahBarrierStubC2::needs_slow_barrier(mach);
1035 }
1036
1037 bool ShenandoahBarrierSetC2State::needs_livein_data() const {
1038 return true;
1039 }