1 /*
2 * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "classfile/javaClasses.hpp"
26 #include "gc/x/c2/xBarrierSetC2.hpp"
27 #include "gc/x/xBarrierSet.hpp"
28 #include "gc/x/xBarrierSetAssembler.hpp"
29 #include "gc/x/xBarrierSetRuntime.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/block.hpp"
33 #include "opto/compile.hpp"
34 #include "opto/graphKit.hpp"
35 #include "opto/machnode.hpp"
36 #include "opto/macro.hpp"
37 #include "opto/memnode.hpp"
38 #include "opto/node.hpp"
39 #include "opto/output.hpp"
40 #include "opto/regalloc.hpp"
41 #include "opto/rootnode.hpp"
42 #include "opto/runtime.hpp"
43 #include "opto/type.hpp"
44 #include "utilities/growableArray.hpp"
45 #include "utilities/macros.hpp"
46
47 class XBarrierSetC2State : public ArenaObj {
48 private:
49 GrowableArray<XLoadBarrierStubC2*>* _stubs;
50 Node_Array _live;
51
52 public:
53 XBarrierSetC2State(Arena* arena) :
54 _stubs(new (arena) GrowableArray<XLoadBarrierStubC2*>(arena, 8, 0, nullptr)),
55 _live(arena) {}
56
57 GrowableArray<XLoadBarrierStubC2*>* stubs() {
58 return _stubs;
59 }
60
61 RegMask* live(const Node* node) {
62 if (!node->is_Mach()) {
63 // Don't need liveness for non-MachNodes
64 return nullptr;
65 }
66
67 const MachNode* const mach = node->as_Mach();
68 if (mach->barrier_data() == XLoadBarrierElided) {
69 // Don't need liveness data for nodes without barriers
70 return nullptr;
71 }
72
73 RegMask* live = (RegMask*)_live[node->_idx];
74 if (live == nullptr) {
75 live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
76 _live.map(node->_idx, (Node*)live);
77 }
78
79 return live;
80 }
81 };
82
83 static XBarrierSetC2State* barrier_set_state() {
84 return reinterpret_cast<XBarrierSetC2State*>(Compile::current()->barrier_set_state());
85 }
86
87 XLoadBarrierStubC2* XLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
88 XLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) XLoadBarrierStubC2(node, ref_addr, ref, tmp, barrier_data);
89 if (!Compile::current()->output()->in_scratch_emit_size()) {
90 barrier_set_state()->stubs()->append(stub);
91 }
92
93 return stub;
94 }
95
96 XLoadBarrierStubC2::XLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) :
97 _node(node),
98 _ref_addr(ref_addr),
99 _ref(ref),
100 _tmp(tmp),
101 _barrier_data(barrier_data),
102 _entry(),
103 _continuation() {
104 assert_different_registers(ref, ref_addr.base());
105 assert_different_registers(ref, ref_addr.index());
106 }
107
108 Address XLoadBarrierStubC2::ref_addr() const {
109 return _ref_addr;
110 }
111
112 Register XLoadBarrierStubC2::ref() const {
113 return _ref;
114 }
115
116 Register XLoadBarrierStubC2::tmp() const {
117 return _tmp;
118 }
119
120 address XLoadBarrierStubC2::slow_path() const {
121 DecoratorSet decorators = DECORATORS_NONE;
122 if (_barrier_data & XLoadBarrierStrong) {
123 decorators |= ON_STRONG_OOP_REF;
124 }
125 if (_barrier_data & XLoadBarrierWeak) {
126 decorators |= ON_WEAK_OOP_REF;
127 }
128 if (_barrier_data & XLoadBarrierPhantom) {
129 decorators |= ON_PHANTOM_OOP_REF;
130 }
131 if (_barrier_data & XLoadBarrierNoKeepalive) {
132 decorators |= AS_NO_KEEPALIVE;
133 }
134 return XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
135 }
136
137 RegMask& XLoadBarrierStubC2::live() const {
138 RegMask* mask = barrier_set_state()->live(_node);
139 assert(mask != nullptr, "must be mach-node with barrier");
140 return *mask;
141 }
142
143 Label* XLoadBarrierStubC2::entry() {
144 // The _entry will never be bound when in_scratch_emit_size() is true.
145 // However, we still need to return a label that is not bound now, but
146 // will eventually be bound. Any label will do, as it will only act as
147 // a placeholder, so we return the _continuation label.
148 return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
149 }
150
151 Label* XLoadBarrierStubC2::continuation() {
152 return &_continuation;
153 }
154
155 void* XBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
156 return new (comp_arena) XBarrierSetC2State(comp_arena);
157 }
158
159 void XBarrierSetC2::late_barrier_analysis() const {
160 analyze_dominating_barriers();
161 compute_liveness_at_stubs();
162 }
163
164 void XBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
165 MacroAssembler masm(&cb);
166 GrowableArray<XLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
167
168 for (int i = 0; i < stubs->length(); i++) {
169 // Make sure there is enough space in the code buffer
170 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
171 ciEnv::current()->record_failure("CodeCache is full");
172 return;
173 }
174
175 XBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
176 }
177
178 masm.flush();
179 }
180
181 int XBarrierSetC2::estimate_stub_size() const {
182 Compile* const C = Compile::current();
183 BufferBlob* const blob = C->output()->scratch_buffer_blob();
184 GrowableArray<XLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
185 int size = 0;
186
187 for (int i = 0; i < stubs->length(); i++) {
188 CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
189 MacroAssembler masm(&cb);
190 XBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
191 size += cb.insts_size();
192 }
193
194 return size;
195 }
196
197 static void set_barrier_data(C2Access& access) {
198 if (XBarrierSet::barrier_needed(access.decorators(), access.type())) {
199 uint8_t barrier_data = 0;
200
201 if (access.decorators() & ON_PHANTOM_OOP_REF) {
202 barrier_data |= XLoadBarrierPhantom;
203 } else if (access.decorators() & ON_WEAK_OOP_REF) {
204 barrier_data |= XLoadBarrierWeak;
205 } else {
206 barrier_data |= XLoadBarrierStrong;
207 }
208
209 if (access.decorators() & AS_NO_KEEPALIVE) {
210 barrier_data |= XLoadBarrierNoKeepalive;
211 }
212
213 access.set_barrier_data(barrier_data);
214 }
215 }
216
217 Node* XBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
218 set_barrier_data(access);
219 return BarrierSetC2::load_at_resolved(access, val_type);
220 }
221
222 Node* XBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
223 Node* new_val, const Type* val_type) const {
224 set_barrier_data(access);
225 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
226 }
227
228 Node* XBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
229 Node* new_val, const Type* value_type) const {
230 set_barrier_data(access);
231 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
232 }
233
234 Node* XBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
235 set_barrier_data(access);
236 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
237 }
238
239 bool XBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
240 bool is_clone, bool is_clone_instance,
241 ArrayCopyPhase phase) const {
242 if (phase == ArrayCopyPhase::Parsing) {
243 return false;
244 }
245 if (phase == ArrayCopyPhase::Optimization) {
246 return is_clone_instance;
247 }
248 // else ArrayCopyPhase::Expansion
249 return type == T_OBJECT || type == T_ARRAY;
250 }
251
252 // This TypeFunc assumes a 64bit system
253 static const TypeFunc* clone_type() {
254 // Create input type (domain)
255 const Type** domain_fields = TypeTuple::fields(4);
256 domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL; // src
257 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
258 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
259 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
260 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
261
262 // Create result type (range)
263 const Type** range_fields = TypeTuple::fields(0);
264 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
265
266 return TypeFunc::make(domain, range);
267 }
268
269 #define XTOP LP64_ONLY(COMMA phase->top())
270
271 void XBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
272 Node* const src = ac->in(ArrayCopyNode::Src);
273 const TypeAryPtr* ary_ptr = src->get_ptr_type()->isa_aryptr();
274
275 if (ac->is_clone_array() && ary_ptr != nullptr) {
276 BasicType bt = ary_ptr->elem()->array_element_basic_type();
277 if (is_reference_type(bt)) {
278 // Clone object array
279 bt = T_OBJECT;
280 } else {
281 // Clone primitive array
282 bt = T_LONG;
283 }
284
285 Node* ctrl = ac->in(TypeFunc::Control);
286 Node* mem = ac->in(TypeFunc::Memory);
287 Node* src = ac->in(ArrayCopyNode::Src);
288 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
289 Node* dest = ac->in(ArrayCopyNode::Dest);
290 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
291 Node* length = ac->in(ArrayCopyNode::Length);
292
293 if (bt == T_OBJECT) {
294 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
295 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
296 // to the first element in the array when cloning object arrays. Otherwise, load
297 // barriers are applied to parts of the header. Also adjust the length accordingly.
298 assert(src_offset == dest_offset, "should be equal");
299 jlong offset = src_offset->get_long();
300 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
301 assert(!UseCompressedClassPointers || UseCompactObjectHeaders, "should only happen without compressed class pointers");
302 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
303 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
304 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
305 dest_offset = src_offset;
306 }
307 }
308 Node* payload_src = phase->basic_plus_adr(src, src_offset);
309 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
310
311 const char* copyfunc_name = "arraycopy";
312 address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
313
314 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
315 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
316
317 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
318 phase->transform_later(call);
319
320 phase->igvn().replace_node(ac, call);
321 return;
322 }
323
324 // Clone instance
325 Node* const ctrl = ac->in(TypeFunc::Control);
326 Node* const mem = ac->in(TypeFunc::Memory);
327 Node* const dst = ac->in(ArrayCopyNode::Dest);
328 Node* const size = ac->in(ArrayCopyNode::Length);
329
330 assert(size->bottom_type()->is_long(), "Should be long");
331
332 // The native clone we are calling here expects the instance size in words
333 // Add header/offset size to payload size to get instance size.
334 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
335 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
336
337 Node* const call = phase->make_leaf_call(ctrl,
338 mem,
339 clone_type(),
340 XBarrierSetRuntime::clone_addr(),
341 "XBarrierSetRuntime::clone",
342 TypeRawPtr::BOTTOM,
343 src,
344 dst,
345 full_size,
346 phase->top());
347 phase->transform_later(call);
348 phase->igvn().replace_node(ac, call);
349 }
350
351 #undef XTOP
352
353 // == Dominating barrier elision ==
354
355 static bool block_has_safepoint(const Block* block, uint from, uint to) {
356 for (uint i = from; i < to; i++) {
357 if (block->get_node(i)->is_MachSafePoint()) {
358 // Safepoint found
359 return true;
360 }
361 }
362
363 // Safepoint not found
364 return false;
365 }
366
367 static bool block_has_safepoint(const Block* block) {
368 return block_has_safepoint(block, 0, block->number_of_nodes());
369 }
370
371 static uint block_index(const Block* block, const Node* node) {
372 for (uint j = 0; j < block->number_of_nodes(); ++j) {
373 if (block->get_node(j) == node) {
374 return j;
375 }
376 }
377 ShouldNotReachHere();
378 return 0;
379 }
380
381 void XBarrierSetC2::analyze_dominating_barriers() const {
382 ResourceMark rm;
383 Compile* const C = Compile::current();
384 PhaseCFG* const cfg = C->cfg();
385 Block_List worklist;
386 Node_List mem_ops;
387 Node_List barrier_loads;
388
389 // Step 1 - Find accesses, and track them in lists
390 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
391 const Block* const block = cfg->get_block(i);
392 for (uint j = 0; j < block->number_of_nodes(); ++j) {
393 const Node* const node = block->get_node(j);
394 if (!node->is_Mach()) {
395 continue;
396 }
397
398 MachNode* const mach = node->as_Mach();
399 switch (mach->ideal_Opcode()) {
400 case Op_LoadP:
401 if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
402 barrier_loads.push(mach);
403 }
404 if ((mach->barrier_data() & (XLoadBarrierStrong | XLoadBarrierNoKeepalive)) ==
405 XLoadBarrierStrong) {
406 mem_ops.push(mach);
407 }
408 break;
409 case Op_CompareAndExchangeP:
410 case Op_CompareAndSwapP:
411 case Op_GetAndSetP:
412 if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
413 barrier_loads.push(mach);
414 }
415 case Op_StoreP:
416 mem_ops.push(mach);
417 break;
418
419 default:
420 break;
421 }
422 }
423 }
424
425 // Step 2 - Find dominating accesses for each load
426 for (uint i = 0; i < barrier_loads.size(); i++) {
427 MachNode* const load = barrier_loads.at(i)->as_Mach();
428 const TypePtr* load_adr_type = nullptr;
429 intptr_t load_offset = 0;
430 const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
431 Block* const load_block = cfg->get_block_for_node(load);
432 const uint load_index = block_index(load_block, load);
433
434 for (uint j = 0; j < mem_ops.size(); j++) {
435 MachNode* mem = mem_ops.at(j)->as_Mach();
436 const TypePtr* mem_adr_type = nullptr;
437 intptr_t mem_offset = 0;
438 const Node* mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
439 Block* mem_block = cfg->get_block_for_node(mem);
440 uint mem_index = block_index(mem_block, mem);
441
442 if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
443 load_obj == nullptr || mem_obj == nullptr ||
444 load_offset < 0 || mem_offset < 0) {
445 continue;
446 }
447
448 if (mem_obj != load_obj || mem_offset != load_offset) {
449 // Not the same addresses, not a candidate
450 continue;
451 }
452
453 if (load_block == mem_block) {
454 // Earlier accesses in the same block
455 if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
456 load->set_barrier_data(XLoadBarrierElided);
457 }
458 } else if (mem_block->dominates(load_block)) {
459 // Dominating block? Look around for safepoints
460 ResourceMark rm;
461 Block_List stack;
462 VectorSet visited;
463 stack.push(load_block);
464 bool safepoint_found = block_has_safepoint(load_block);
465 while (!safepoint_found && stack.size() > 0) {
466 Block* block = stack.pop();
467 if (visited.test_set(block->_pre_order)) {
468 continue;
469 }
470 if (block_has_safepoint(block)) {
471 safepoint_found = true;
472 break;
473 }
474 if (block == mem_block) {
475 continue;
476 }
477
478 // Push predecessor blocks
479 for (uint p = 1; p < block->num_preds(); ++p) {
480 Block* pred = cfg->get_block_for_node(block->pred(p));
481 stack.push(pred);
482 }
483 }
484
485 if (!safepoint_found) {
486 load->set_barrier_data(XLoadBarrierElided);
487 }
488 }
489 }
490 }
491 }
492
493 // == Reduced spilling optimization ==
494
495 void XBarrierSetC2::compute_liveness_at_stubs() const {
496 ResourceMark rm;
497 Compile* const C = Compile::current();
498 Arena* const A = Thread::current()->resource_area();
499 PhaseCFG* const cfg = C->cfg();
500 PhaseRegAlloc* const regalloc = C->regalloc();
501 RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
502 XBarrierSetAssembler* const bs = XBarrierSet::assembler();
503 Block_List worklist;
504
505 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
506 new ((void*)(live + i)) RegMask();
507 worklist.push(cfg->get_block(i));
508 }
509
510 while (worklist.size() > 0) {
511 const Block* const block = worklist.pop();
512 RegMask& old_live = live[block->_pre_order];
513 RegMask new_live;
514
515 // Initialize to union of successors
516 for (uint i = 0; i < block->_num_succs; i++) {
517 const uint succ_id = block->_succs[i]->_pre_order;
518 new_live.OR(live[succ_id]);
519 }
520
521 // Walk block backwards, computing liveness
522 for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
523 const Node* const node = block->get_node(i);
524
525 // Remove def bits
526 const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
527 const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
528 if (first != OptoReg::Bad) {
529 new_live.Remove(first);
530 }
531 if (second != OptoReg::Bad) {
532 new_live.Remove(second);
533 }
534
535 // Add use bits
536 for (uint j = 1; j < node->req(); ++j) {
537 const Node* const use = node->in(j);
538 const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
539 const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
540 if (first != OptoReg::Bad) {
541 new_live.Insert(first);
542 }
543 if (second != OptoReg::Bad) {
544 new_live.Insert(second);
545 }
546 }
547
548 // If this node tracks liveness, update it
549 RegMask* const regs = barrier_set_state()->live(node);
550 if (regs != nullptr) {
551 regs->OR(new_live);
552 }
553 }
554
555 // Now at block top, see if we have any changes
556 new_live.SUBTRACT(old_live);
557 if (new_live.is_NotEmpty()) {
558 // Liveness has refined, update and propagate to prior blocks
559 old_live.OR(new_live);
560 for (uint i = 1; i < block->num_preds(); ++i) {
561 Block* const pred = cfg->get_block_for_node(block->pred(i));
562 worklist.push(pred);
563 }
564 }
565 }
566 }
567
568 #ifndef PRODUCT
569 void XBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
570 if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
571 st->print("strong ");
572 }
573 if ((mach->barrier_data() & XLoadBarrierWeak) != 0) {
574 st->print("weak ");
575 }
576 if ((mach->barrier_data() & XLoadBarrierPhantom) != 0) {
577 st->print("phantom ");
578 }
579 if ((mach->barrier_data() & XLoadBarrierNoKeepalive) != 0) {
580 st->print("nokeepalive ");
581 }
582 }
583 #endif // !PRODUCT