1 /*
2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/tlab_globals.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "opto/arraycopynode.hpp"
29 #include "opto/convertnode.hpp"
30 #include "opto/graphKit.hpp"
31 #include "opto/idealKit.hpp"
32 #include "opto/macro.hpp"
33 #include "opto/narrowptrnode.hpp"
34 #include "opto/runtime.hpp"
35 #include "utilities/macros.hpp"
36
37 // By default this is a no-op.
38 void BarrierSetC2::resolve_address(C2Access& access) const { }
39
40 void* C2ParseAccess::barrier_set_state() const {
41 return _kit->barrier_set_state();
42 }
43
44 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
45
46 Node* C2ParseAccess::control() const {
47 return _ctl == nullptr ? _kit->control() : _ctl;
48 }
49
50 bool C2Access::needs_cpu_membar() const {
51 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
52 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
53
54 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
55 bool in_heap = (_decorators & IN_HEAP) != 0;
56 bool in_native = (_decorators & IN_NATIVE) != 0;
57 bool is_mixed = !in_heap && !in_native;
58
59 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
60 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
61 bool is_atomic = is_read && is_write;
62
63 if (is_atomic) {
64 // Atomics always need to be wrapped in CPU membars
65 return true;
66 }
67
68 if (anonymous) {
69 // We will need memory barriers unless we can determine a unique
70 // alias category for this reference. (Note: If for some reason
71 // the barriers get omitted and the unsafe reference begins to "pollute"
72 // the alias analysis of the rest of the graph, either Compile::can_alias
73 // or Compile::must_alias will throw a diagnostic assert.)
74 if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
75 return true;
76 }
77 } else {
78 assert(!is_mixed, "not unsafe");
79 }
80
81 return false;
82 }
83
84 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
85 DecoratorSet decorators = access.decorators();
86
87 bool mismatched = (decorators & C2_MISMATCHED) != 0;
88 bool unaligned = (decorators & C2_UNALIGNED) != 0;
89 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
90 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
91
92 MemNode::MemOrd mo = access.mem_node_mo();
93
94 Node* store;
95 BasicType bt = access.type();
96 if (access.is_parse_access()) {
97 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
98
99 GraphKit* kit = parse_access.kit();
100 if (bt == T_DOUBLE) {
101 Node* new_val = kit->dprecision_rounding(val.node());
102 val.set_node(new_val);
103 }
104
105 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt,
106 access.addr().type(), mo, requires_atomic_access, unaligned,
107 mismatched, unsafe, access.barrier_data());
108 } else {
109 assert(access.is_opt_access(), "either parse or opt access");
110 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
111 Node* ctl = opt_access.ctl();
112 MergeMemNode* mm = opt_access.mem();
113 PhaseGVN& gvn = opt_access.gvn();
114 const TypePtr* adr_type = access.addr().type();
115 int alias = gvn.C->get_alias_index(adr_type);
116 Node* mem = mm->memory_at(alias);
117
118 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), bt, mo, requires_atomic_access);
119 if (unaligned) {
120 st->set_unaligned_access();
121 }
122 if (mismatched) {
123 st->set_mismatched_access();
124 }
125 st->set_barrier_data(access.barrier_data());
126 store = gvn.transform(st);
127 if (store == st) {
128 mm->set_memory_at(alias, st);
129 }
130 }
131 access.set_raw_access(store);
132
133 return store;
134 }
135
136 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
137 DecoratorSet decorators = access.decorators();
138
139 Node* adr = access.addr().node();
140 const TypePtr* adr_type = access.addr().type();
141
142 bool mismatched = (decorators & C2_MISMATCHED) != 0;
143 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
144 bool unaligned = (decorators & C2_UNALIGNED) != 0;
145 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
146 bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0;
147 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
148 bool immutable = (decorators & C2_IMMUTABLE_MEMORY) != 0;
149
150 MemNode::MemOrd mo = access.mem_node_mo();
151 LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest;
152
153 Node* load;
154 if (access.is_parse_access()) {
155 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
156 GraphKit* kit = parse_access.kit();
157 Node* control = control_dependent ? parse_access.control() : nullptr;
158
159 if (immutable) {
160 Compile* C = Compile::current();
161 Node* mem = kit->immutable_memory();
162 load = LoadNode::make(kit->gvn(), control, mem, adr,
163 adr_type, val_type, access.type(), mo, dep, requires_atomic_access,
164 unaligned, mismatched, unsafe, access.barrier_data());
165 load = kit->gvn().transform(load);
166 } else {
167 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
168 dep, requires_atomic_access, unaligned, mismatched, unsafe,
169 access.barrier_data());
170 }
171 } else {
172 assert(access.is_opt_access(), "either parse or opt access");
173 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
174 Node* control = control_dependent ? opt_access.ctl() : nullptr;
175 MergeMemNode* mm = opt_access.mem();
176 PhaseGVN& gvn = opt_access.gvn();
177 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
178 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep,
179 requires_atomic_access, unaligned, mismatched, unsafe, access.barrier_data());
180 load = gvn.transform(load);
181 }
182 access.set_raw_access(load);
183
184 return load;
185 }
186
187 class C2AccessFence: public StackObj {
188 C2Access& _access;
189 Node* _leading_membar;
190
191 public:
192 C2AccessFence(C2Access& access) :
193 _access(access), _leading_membar(nullptr) {
194 GraphKit* kit = nullptr;
195 if (access.is_parse_access()) {
196 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
197 kit = parse_access.kit();
198 }
199 DecoratorSet decorators = access.decorators();
200
201 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
202 bool is_read = (decorators & C2_READ_ACCESS) != 0;
203 bool is_atomic = is_read && is_write;
204
205 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
206 bool is_release = (decorators & MO_RELEASE) != 0;
207
208 if (is_atomic) {
209 assert(kit != nullptr, "unsupported at optimization time");
210 // Memory-model-wise, a LoadStore acts like a little synchronized
211 // block, so needs barriers on each side. These don't translate
212 // into actual barriers on most machines, but we still need rest of
213 // compiler to respect ordering.
214 if (is_release) {
215 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
216 } else if (is_volatile) {
217 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
218 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
219 } else {
220 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
221 }
222 }
223 } else if (is_write) {
224 // If reference is volatile, prevent following memory ops from
225 // floating down past the volatile write. Also prevents commoning
226 // another volatile read.
227 if (is_volatile || is_release) {
228 assert(kit != nullptr, "unsupported at optimization time");
229 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
230 }
231 } else {
232 // Memory barrier to prevent normal and 'unsafe' accesses from
233 // bypassing each other. Happens after null checks, so the
234 // exception paths do not take memory state from the memory barrier,
235 // so there's no problems making a strong assert about mixing users
236 // of safe & unsafe memory.
237 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
238 assert(kit != nullptr, "unsupported at optimization time");
239 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
240 }
241 }
242
243 if (access.needs_cpu_membar()) {
244 assert(kit != nullptr, "unsupported at optimization time");
245 kit->insert_mem_bar(Op_MemBarCPUOrder);
246 }
247
248 if (is_atomic) {
249 // 4984716: MemBars must be inserted before this
250 // memory node in order to avoid a false
251 // dependency which will confuse the scheduler.
252 access.set_memory();
253 }
254 }
255
256 ~C2AccessFence() {
257 GraphKit* kit = nullptr;
258 if (_access.is_parse_access()) {
259 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
260 kit = parse_access.kit();
261 }
262 DecoratorSet decorators = _access.decorators();
263
264 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
265 bool is_read = (decorators & C2_READ_ACCESS) != 0;
266 bool is_atomic = is_read && is_write;
267
268 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
269 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
270
271 // If reference is volatile, prevent following volatiles ops from
272 // floating up before the volatile access.
273 if (_access.needs_cpu_membar()) {
274 kit->insert_mem_bar(Op_MemBarCPUOrder);
275 }
276
277 if (is_atomic) {
278 assert(kit != nullptr, "unsupported at optimization time");
279 if (is_acquire || is_volatile) {
280 Node* n = _access.raw_access();
281 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
282 if (_leading_membar != nullptr) {
283 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
284 }
285 }
286 } else if (is_write) {
287 // If not multiple copy atomic, we do the MemBarVolatile before the load.
288 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
289 assert(kit != nullptr, "unsupported at optimization time");
290 Node* n = _access.raw_access();
291 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
292 if (_leading_membar != nullptr) {
293 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
294 }
295 }
296 } else {
297 if (is_volatile || is_acquire) {
298 assert(kit != nullptr, "unsupported at optimization time");
299 Node* n = _access.raw_access();
300 assert(_leading_membar == nullptr || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
301 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
302 mb->as_MemBar()->set_trailing_load();
303 }
304 }
305 }
306 };
307
308 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
309 C2AccessFence fence(access);
310 resolve_address(access);
311 return store_at_resolved(access, val);
312 }
313
314 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
315 C2AccessFence fence(access);
316 resolve_address(access);
317 return load_at_resolved(access, val_type);
318 }
319
320 MemNode::MemOrd C2Access::mem_node_mo() const {
321 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
322 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
323 if ((_decorators & MO_SEQ_CST) != 0) {
324 if (is_write && is_read) {
325 // For atomic operations
326 return MemNode::seqcst;
327 } else if (is_write) {
328 return MemNode::release;
329 } else {
330 assert(is_read, "what else?");
331 return MemNode::acquire;
332 }
333 } else if ((_decorators & MO_RELEASE) != 0) {
334 return MemNode::release;
335 } else if ((_decorators & MO_ACQUIRE) != 0) {
336 return MemNode::acquire;
337 } else if (is_write) {
338 // Volatile fields need releasing stores.
339 // Non-volatile fields also need releasing stores if they hold an
340 // object reference, because the object reference might point to
341 // a freshly created object.
342 // Conservatively release stores of object references.
343 return StoreNode::release_if_reference(_type);
344 } else {
345 return MemNode::unordered;
346 }
347 }
348
349 void C2Access::fixup_decorators() {
350 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
351 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
352 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
353
354 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
355 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
356
357 if (AlwaysAtomicAccesses && is_unordered) {
358 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
359 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
360 }
361
362 _decorators = AccessInternal::decorator_fixup(_decorators, _type);
363
364 if (is_read && !is_write && anonymous) {
365 // To be valid, unsafe loads may depend on other conditions than
366 // the one that guards them: pin the Load node
367 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
368 _decorators |= C2_UNKNOWN_CONTROL_LOAD;
369 const TypePtr* adr_type = _addr.type();
370 Node* adr = _addr.node();
371 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
372 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
373 intptr_t offset = Type::OffsetBot;
374 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
375 if (offset >= 0) {
376 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->instance_klass()->layout_helper());
377 if (offset < s) {
378 // Guaranteed to be a valid access, no need to pin it
379 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
380 _decorators ^= C2_UNKNOWN_CONTROL_LOAD;
381 }
382 }
383 }
384 }
385 }
386
387 //--------------------------- atomic operations---------------------------------
388
389 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
390 // SCMemProjNodes represent the memory state of a LoadStore. Their
391 // main role is to prevent LoadStore nodes from being optimized away
392 // when their results aren't used.
393 assert(access.is_parse_access(), "entry not supported at optimization time");
394 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
395 GraphKit* kit = parse_access.kit();
396 Node* load_store = access.raw_access();
397 assert(load_store != nullptr, "must pin atomic op");
398 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
399 kit->set_memory(proj, access.alias_idx());
400 }
401
402 void C2AtomicParseAccess::set_memory() {
403 Node *mem = _kit->memory(_alias_idx);
404 _memory = mem;
405 }
406
407 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
408 Node* new_val, const Type* value_type) const {
409 GraphKit* kit = access.kit();
410 MemNode::MemOrd mo = access.mem_node_mo();
411 Node* mem = access.memory();
412
413 Node* adr = access.addr().node();
414 const TypePtr* adr_type = access.addr().type();
415
416 Node* load_store = nullptr;
417
418 if (access.is_oop()) {
419 #ifdef _LP64
420 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
421 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
422 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
423 load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo);
424 } else
425 #endif
426 {
427 load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo);
428 }
429 } else {
430 switch (access.type()) {
431 case T_BYTE: {
432 load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
433 break;
434 }
435 case T_SHORT: {
436 load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
437 break;
438 }
439 case T_INT: {
440 load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
441 break;
442 }
443 case T_LONG: {
444 load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
445 break;
446 }
447 default:
448 ShouldNotReachHere();
449 }
450 }
451
452 load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
453 load_store = kit->gvn().transform(load_store);
454
455 access.set_raw_access(load_store);
456 pin_atomic_op(access);
457
458 #ifdef _LP64
459 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
460 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
461 }
462 #endif
463
464 return load_store;
465 }
466
467 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
468 Node* new_val, const Type* value_type) const {
469 GraphKit* kit = access.kit();
470 DecoratorSet decorators = access.decorators();
471 MemNode::MemOrd mo = access.mem_node_mo();
472 Node* mem = access.memory();
473 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
474 Node* load_store = nullptr;
475 Node* adr = access.addr().node();
476
477 if (access.is_oop()) {
478 #ifdef _LP64
479 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
480 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
481 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
482 if (is_weak_cas) {
483 load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo);
484 } else {
485 load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo);
486 }
487 } else
488 #endif
489 {
490 if (is_weak_cas) {
491 load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo);
492 } else {
493 load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo);
494 }
495 }
496 } else {
497 switch(access.type()) {
498 case T_BYTE: {
499 if (is_weak_cas) {
500 load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo);
501 } else {
502 load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo);
503 }
504 break;
505 }
506 case T_SHORT: {
507 if (is_weak_cas) {
508 load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo);
509 } else {
510 load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo);
511 }
512 break;
513 }
514 case T_INT: {
515 if (is_weak_cas) {
516 load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo);
517 } else {
518 load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo);
519 }
520 break;
521 }
522 case T_LONG: {
523 if (is_weak_cas) {
524 load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo);
525 } else {
526 load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo);
527 }
528 break;
529 }
530 default:
531 ShouldNotReachHere();
532 }
533 }
534
535 load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
536 load_store = kit->gvn().transform(load_store);
537
538 access.set_raw_access(load_store);
539 pin_atomic_op(access);
540
541 return load_store;
542 }
543
544 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
545 GraphKit* kit = access.kit();
546 Node* mem = access.memory();
547 Node* adr = access.addr().node();
548 const TypePtr* adr_type = access.addr().type();
549 Node* load_store = nullptr;
550
551 if (access.is_oop()) {
552 #ifdef _LP64
553 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
554 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
555 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
556 } else
557 #endif
558 {
559 load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr());
560 }
561 } else {
562 switch (access.type()) {
563 case T_BYTE:
564 load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type);
565 break;
566 case T_SHORT:
567 load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type);
568 break;
569 case T_INT:
570 load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type);
571 break;
572 case T_LONG:
573 load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type);
574 break;
575 default:
576 ShouldNotReachHere();
577 }
578 }
579
580 load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
581 load_store = kit->gvn().transform(load_store);
582
583 access.set_raw_access(load_store);
584 pin_atomic_op(access);
585
586 #ifdef _LP64
587 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
588 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
589 }
590 #endif
591
592 return load_store;
593 }
594
595 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
596 Node* load_store = nullptr;
597 GraphKit* kit = access.kit();
598 Node* adr = access.addr().node();
599 const TypePtr* adr_type = access.addr().type();
600 Node* mem = access.memory();
601
602 switch(access.type()) {
603 case T_BYTE:
604 load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type);
605 break;
606 case T_SHORT:
607 load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type);
608 break;
609 case T_INT:
610 load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type);
611 break;
612 case T_LONG:
613 load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type);
614 break;
615 default:
616 ShouldNotReachHere();
617 }
618
619 load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
620 load_store = kit->gvn().transform(load_store);
621
622 access.set_raw_access(load_store);
623 pin_atomic_op(access);
624
625 return load_store;
626 }
627
628 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
629 Node* new_val, const Type* value_type) const {
630 C2AccessFence fence(access);
631 resolve_address(access);
632 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
633 }
634
635 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
636 Node* new_val, const Type* value_type) const {
637 C2AccessFence fence(access);
638 resolve_address(access);
639 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
640 }
641
642 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
643 C2AccessFence fence(access);
644 resolve_address(access);
645 return atomic_xchg_at_resolved(access, new_val, value_type);
646 }
647
648 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
649 C2AccessFence fence(access);
650 resolve_address(access);
651 return atomic_add_at_resolved(access, new_val, value_type);
652 }
653
654 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
655 // Exclude the header but include array length to copy by 8 bytes words.
656 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
657 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
658 instanceOopDesc::base_offset_in_bytes();
659 // base_off:
660 // 8 - 32-bit VM
661 // 12 - 64-bit VM, compressed klass
662 // 16 - 64-bit VM, normal klass
663 if (base_off % BytesPerLong != 0) {
664 assert(UseCompressedClassPointers, "");
665 if (is_array) {
666 // Exclude length to copy by 8 bytes words.
667 base_off += sizeof(int);
668 } else {
669 // Include klass to copy by 8 bytes words.
670 base_off = instanceOopDesc::klass_offset_in_bytes();
671 }
672 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
673 }
674 return base_off;
675 }
676
677 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
678 int base_off = arraycopy_payload_base_offset(is_array);
679 Node* payload_size = size;
680 Node* offset = kit->MakeConX(base_off);
681 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
682 if (is_array) {
683 // Ensure the array payload size is rounded up to the next BytesPerLong
684 // multiple when converting to double-words. This is necessary because array
685 // size does not include object alignment padding, so it might not be a
686 // multiple of BytesPerLong for sub-long element types.
687 payload_size = kit->gvn().transform(new AddXNode(payload_size, kit->MakeConX(BytesPerLong - 1)));
688 }
689 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
690 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false);
691 if (is_array) {
692 ac->set_clone_array();
693 } else {
694 ac->set_clone_inst();
695 }
696 Node* n = kit->gvn().transform(ac);
697 if (n == ac) {
698 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
699 ac->set_adr_type(TypeRawPtr::BOTTOM);
700 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
701 } else {
702 kit->set_all_memory(n);
703 }
704 }
705
706 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* mem, Node* toobig_false, Node* size_in_bytes,
707 Node*& i_o, Node*& needgc_ctrl,
708 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
709 intx prefetch_lines) const {
710 assert(UseTLAB, "Only for TLAB enabled allocations");
711
712 Node* thread = macro->transform_later(new ThreadLocalNode());
713 Node* tlab_top_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_top_offset()));
714 Node* tlab_end_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_end_offset()));
715
716 // Load TLAB end.
717 //
718 // Note: We set the control input on "tlab_end" and "old_tlab_top" to work around
719 // a bug where these values were being moved across
720 // a safepoint. These are not oops, so they cannot be include in the oop
721 // map, but they can be changed by a GC. The proper way to fix this would
722 // be to set the raw memory state when generating a SafepointNode. However
723 // this will require extensive changes to the loop optimization in order to
724 // prevent a degradation of the optimization.
725 // See comment in memnode.hpp, around line 227 in class LoadPNode.
726 Node* tlab_end = macro->make_load(toobig_false, mem, tlab_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
727
728 // Load the TLAB top.
729 Node* old_tlab_top = new LoadPNode(toobig_false, mem, tlab_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
730 macro->transform_later(old_tlab_top);
731
732 // Add to heap top to get a new TLAB top
733 Node* new_tlab_top = new AddPNode(macro->top(), old_tlab_top, size_in_bytes);
734 macro->transform_later(new_tlab_top);
735
736 // Check against TLAB end
737 Node* tlab_full = new CmpPNode(new_tlab_top, tlab_end);
738 macro->transform_later(tlab_full);
739
740 Node* needgc_bol = new BoolNode(tlab_full, BoolTest::ge);
741 macro->transform_later(needgc_bol);
742 IfNode* needgc_iff = new IfNode(toobig_false, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
743 macro->transform_later(needgc_iff);
744
745 // Plug the failing-heap-space-need-gc test into the slow-path region
746 Node* needgc_true = new IfTrueNode(needgc_iff);
747 macro->transform_later(needgc_true);
748 needgc_ctrl = needgc_true;
749
750 // No need for a GC.
751 Node* needgc_false = new IfFalseNode(needgc_iff);
752 macro->transform_later(needgc_false);
753
754 // Fast path:
755 i_o = macro->prefetch_allocation(i_o, needgc_false, mem,
756 old_tlab_top, new_tlab_top, prefetch_lines);
757
758 // Store the modified TLAB top back down.
759 Node* store_tlab_top = new StorePNode(needgc_false, mem, tlab_top_adr,
760 TypeRawPtr::BOTTOM, new_tlab_top, MemNode::unordered);
761 macro->transform_later(store_tlab_top);
762
763 fast_oop_ctrl = needgc_false;
764 fast_oop_rawmem = store_tlab_top;
765 return old_tlab_top;
766 }
767
768 #define XTOP LP64_ONLY(COMMA phase->top())
769
770 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
771 Node* ctrl = ac->in(TypeFunc::Control);
772 Node* mem = ac->in(TypeFunc::Memory);
773 Node* src = ac->in(ArrayCopyNode::Src);
774 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
775 Node* dest = ac->in(ArrayCopyNode::Dest);
776 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
777 Node* length = ac->in(ArrayCopyNode::Length);
778
779 Node* payload_src = phase->basic_plus_adr(src, src_offset);
780 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
781
782 const char* copyfunc_name = "arraycopy";
783 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true);
784
785 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
786 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
787
788 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
789 phase->transform_later(call);
790
791 phase->replace_node(ac, call);
792 }
793
794 #undef XTOP