1 /* 2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/tlab_globals.hpp" 27 #include "gc/shared/c2/barrierSetC2.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/convertnode.hpp" 30 #include "opto/graphKit.hpp" 31 #include "opto/idealKit.hpp" 32 #include "opto/macro.hpp" 33 #include "opto/narrowptrnode.hpp" 34 #include "opto/runtime.hpp" 35 #include "utilities/macros.hpp" 36 37 // By default this is a no-op. 38 void BarrierSetC2::resolve_address(C2Access& access) const { } 39 40 void* C2ParseAccess::barrier_set_state() const { 41 return _kit->barrier_set_state(); 42 } 43 44 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } 45 46 bool C2Access::needs_cpu_membar() const { 47 bool mismatched = (_decorators & C2_MISMATCHED) != 0; 48 bool is_unordered = (_decorators & MO_UNORDERED) != 0; 49 50 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 51 bool in_heap = (_decorators & IN_HEAP) != 0; 52 bool in_native = (_decorators & IN_NATIVE) != 0; 53 bool is_mixed = !in_heap && !in_native; 54 55 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 56 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 57 bool is_atomic = is_read && is_write; 58 59 if (is_atomic) { 60 // Atomics always need to be wrapped in CPU membars 61 return true; 62 } 63 64 if (anonymous) { 65 // We will need memory barriers unless we can determine a unique 66 // alias category for this reference. (Note: If for some reason 67 // the barriers get omitted and the unsafe reference begins to "pollute" 68 // the alias analysis of the rest of the graph, either Compile::can_alias 69 // or Compile::must_alias will throw a diagnostic assert.) 70 if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) { 71 return true; 72 } 73 } else { 74 assert(!is_mixed, "not unsafe"); 75 } 76 77 return false; 78 } 79 80 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 81 DecoratorSet decorators = access.decorators(); 82 83 bool mismatched = (decorators & C2_MISMATCHED) != 0; 84 bool unaligned = (decorators & C2_UNALIGNED) != 0; 85 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 86 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 87 88 MemNode::MemOrd mo = access.mem_node_mo(); 89 90 Node* store; 91 BasicType bt = access.type(); 92 if (access.is_parse_access()) { 93 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 94 95 GraphKit* kit = parse_access.kit(); 96 if (bt == T_DOUBLE) { 97 Node* new_val = kit->dprecision_rounding(val.node()); 98 val.set_node(new_val); 99 } 100 101 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt, 102 access.addr().type(), mo, requires_atomic_access, unaligned, 103 mismatched, unsafe, access.barrier_data()); 104 } else { 105 assert(access.is_opt_access(), "either parse or opt access"); 106 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 107 Node* ctl = opt_access.ctl(); 108 MergeMemNode* mm = opt_access.mem(); 109 PhaseGVN& gvn = opt_access.gvn(); 110 const TypePtr* adr_type = access.addr().type(); 111 int alias = gvn.C->get_alias_index(adr_type); 112 Node* mem = mm->memory_at(alias); 113 114 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), bt, mo, requires_atomic_access); 115 if (unaligned) { 116 st->set_unaligned_access(); 117 } 118 if (mismatched) { 119 st->set_mismatched_access(); 120 } 121 st->set_barrier_data(access.barrier_data()); 122 store = gvn.transform(st); 123 if (store == st) { 124 mm->set_memory_at(alias, st); 125 } 126 } 127 access.set_raw_access(store); 128 129 return store; 130 } 131 132 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 133 DecoratorSet decorators = access.decorators(); 134 135 Node* adr = access.addr().node(); 136 const TypePtr* adr_type = access.addr().type(); 137 138 bool mismatched = (decorators & C2_MISMATCHED) != 0; 139 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 140 bool unaligned = (decorators & C2_UNALIGNED) != 0; 141 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0; 142 bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0; 143 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 144 bool immutable = (decorators & C2_IMMUTABLE_MEMORY) != 0; 145 146 MemNode::MemOrd mo = access.mem_node_mo(); 147 LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest; 148 149 Node* load; 150 if (access.is_parse_access()) { 151 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 152 GraphKit* kit = parse_access.kit(); 153 Node* control = control_dependent ? kit->control() : nullptr; 154 155 if (immutable) { 156 Compile* C = Compile::current(); 157 Node* mem = kit->immutable_memory(); 158 load = LoadNode::make(kit->gvn(), control, mem, adr, 159 adr_type, val_type, access.type(), mo, dep, requires_atomic_access, 160 unaligned, mismatched, unsafe, access.barrier_data()); 161 load = kit->gvn().transform(load); 162 } else { 163 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, 164 dep, requires_atomic_access, unaligned, mismatched, unsafe, 165 access.barrier_data()); 166 } 167 } else { 168 assert(access.is_opt_access(), "either parse or opt access"); 169 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 170 Node* control = control_dependent ? opt_access.ctl() : nullptr; 171 MergeMemNode* mm = opt_access.mem(); 172 PhaseGVN& gvn = opt_access.gvn(); 173 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); 174 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, 175 requires_atomic_access, unaligned, mismatched, unsafe, access.barrier_data()); 176 load = gvn.transform(load); 177 } 178 access.set_raw_access(load); 179 180 return load; 181 } 182 183 class C2AccessFence: public StackObj { 184 C2Access& _access; 185 Node* _leading_membar; 186 187 public: 188 C2AccessFence(C2Access& access) : 189 _access(access), _leading_membar(nullptr) { 190 GraphKit* kit = nullptr; 191 if (access.is_parse_access()) { 192 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 193 kit = parse_access.kit(); 194 } 195 DecoratorSet decorators = access.decorators(); 196 197 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 198 bool is_read = (decorators & C2_READ_ACCESS) != 0; 199 bool is_atomic = is_read && is_write; 200 201 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 202 bool is_release = (decorators & MO_RELEASE) != 0; 203 204 if (is_atomic) { 205 assert(kit != nullptr, "unsupported at optimization time"); 206 // Memory-model-wise, a LoadStore acts like a little synchronized 207 // block, so needs barriers on each side. These don't translate 208 // into actual barriers on most machines, but we still need rest of 209 // compiler to respect ordering. 210 if (is_release) { 211 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 212 } else if (is_volatile) { 213 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 214 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 215 } else { 216 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 217 } 218 } 219 } else if (is_write) { 220 // If reference is volatile, prevent following memory ops from 221 // floating down past the volatile write. Also prevents commoning 222 // another volatile read. 223 if (is_volatile || is_release) { 224 assert(kit != nullptr, "unsupported at optimization time"); 225 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 226 } 227 } else { 228 // Memory barrier to prevent normal and 'unsafe' accesses from 229 // bypassing each other. Happens after null checks, so the 230 // exception paths do not take memory state from the memory barrier, 231 // so there's no problems making a strong assert about mixing users 232 // of safe & unsafe memory. 233 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) { 234 assert(kit != nullptr, "unsupported at optimization time"); 235 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 236 } 237 } 238 239 if (access.needs_cpu_membar()) { 240 assert(kit != nullptr, "unsupported at optimization time"); 241 kit->insert_mem_bar(Op_MemBarCPUOrder); 242 } 243 244 if (is_atomic) { 245 // 4984716: MemBars must be inserted before this 246 // memory node in order to avoid a false 247 // dependency which will confuse the scheduler. 248 access.set_memory(); 249 } 250 } 251 252 ~C2AccessFence() { 253 GraphKit* kit = nullptr; 254 if (_access.is_parse_access()) { 255 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access); 256 kit = parse_access.kit(); 257 } 258 DecoratorSet decorators = _access.decorators(); 259 260 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 261 bool is_read = (decorators & C2_READ_ACCESS) != 0; 262 bool is_atomic = is_read && is_write; 263 264 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 265 bool is_acquire = (decorators & MO_ACQUIRE) != 0; 266 267 // If reference is volatile, prevent following volatiles ops from 268 // floating up before the volatile access. 269 if (_access.needs_cpu_membar()) { 270 kit->insert_mem_bar(Op_MemBarCPUOrder); 271 } 272 273 if (is_atomic) { 274 assert(kit != nullptr, "unsupported at optimization time"); 275 if (is_acquire || is_volatile) { 276 Node* n = _access.raw_access(); 277 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 278 if (_leading_membar != nullptr) { 279 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 280 } 281 } 282 } else if (is_write) { 283 // If not multiple copy atomic, we do the MemBarVolatile before the load. 284 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) { 285 assert(kit != nullptr, "unsupported at optimization time"); 286 Node* n = _access.raw_access(); 287 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar 288 if (_leading_membar != nullptr) { 289 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 290 } 291 } 292 } else { 293 if (is_volatile || is_acquire) { 294 assert(kit != nullptr, "unsupported at optimization time"); 295 Node* n = _access.raw_access(); 296 assert(_leading_membar == nullptr || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected"); 297 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 298 mb->as_MemBar()->set_trailing_load(); 299 } 300 } 301 } 302 }; 303 304 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { 305 C2AccessFence fence(access); 306 resolve_address(access); 307 return store_at_resolved(access, val); 308 } 309 310 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const { 311 C2AccessFence fence(access); 312 resolve_address(access); 313 return load_at_resolved(access, val_type); 314 } 315 316 MemNode::MemOrd C2Access::mem_node_mo() const { 317 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 318 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 319 if ((_decorators & MO_SEQ_CST) != 0) { 320 if (is_write && is_read) { 321 // For atomic operations 322 return MemNode::seqcst; 323 } else if (is_write) { 324 return MemNode::release; 325 } else { 326 assert(is_read, "what else?"); 327 return MemNode::acquire; 328 } 329 } else if ((_decorators & MO_RELEASE) != 0) { 330 return MemNode::release; 331 } else if ((_decorators & MO_ACQUIRE) != 0) { 332 return MemNode::acquire; 333 } else if (is_write) { 334 // Volatile fields need releasing stores. 335 // Non-volatile fields also need releasing stores if they hold an 336 // object reference, because the object reference might point to 337 // a freshly created object. 338 // Conservatively release stores of object references. 339 return StoreNode::release_if_reference(_type); 340 } else { 341 return MemNode::unordered; 342 } 343 } 344 345 void C2Access::fixup_decorators() { 346 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0; 347 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo; 348 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 349 350 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 351 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 352 353 if (AlwaysAtomicAccesses && is_unordered) { 354 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits 355 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess 356 } 357 358 _decorators = AccessInternal::decorator_fixup(_decorators, _type); 359 360 if (is_read && !is_write && anonymous) { 361 // To be valid, unsafe loads may depend on other conditions than 362 // the one that guards them: pin the Load node 363 _decorators |= C2_CONTROL_DEPENDENT_LOAD; 364 _decorators |= C2_UNKNOWN_CONTROL_LOAD; 365 const TypePtr* adr_type = _addr.type(); 366 Node* adr = _addr.node(); 367 if (!needs_cpu_membar() && adr_type->isa_instptr()) { 368 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); 369 intptr_t offset = Type::OffsetBot; 370 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); 371 if (offset >= 0) { 372 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->instance_klass()->layout_helper()); 373 if (offset < s) { 374 // Guaranteed to be a valid access, no need to pin it 375 _decorators ^= C2_CONTROL_DEPENDENT_LOAD; 376 _decorators ^= C2_UNKNOWN_CONTROL_LOAD; 377 } 378 } 379 } 380 } 381 } 382 383 //--------------------------- atomic operations--------------------------------- 384 385 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { 386 // SCMemProjNodes represent the memory state of a LoadStore. Their 387 // main role is to prevent LoadStore nodes from being optimized away 388 // when their results aren't used. 389 assert(access.is_parse_access(), "entry not supported at optimization time"); 390 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 391 GraphKit* kit = parse_access.kit(); 392 Node* load_store = access.raw_access(); 393 assert(load_store != nullptr, "must pin atomic op"); 394 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); 395 kit->set_memory(proj, access.alias_idx()); 396 } 397 398 void C2AtomicParseAccess::set_memory() { 399 Node *mem = _kit->memory(_alias_idx); 400 _memory = mem; 401 } 402 403 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 404 Node* new_val, const Type* value_type) const { 405 GraphKit* kit = access.kit(); 406 MemNode::MemOrd mo = access.mem_node_mo(); 407 Node* mem = access.memory(); 408 409 Node* adr = access.addr().node(); 410 const TypePtr* adr_type = access.addr().type(); 411 412 Node* load_store = nullptr; 413 414 if (access.is_oop()) { 415 #ifdef _LP64 416 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 417 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 418 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 419 load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); 420 } else 421 #endif 422 { 423 load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); 424 } 425 } else { 426 switch (access.type()) { 427 case T_BYTE: { 428 load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 429 break; 430 } 431 case T_SHORT: { 432 load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 433 break; 434 } 435 case T_INT: { 436 load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 437 break; 438 } 439 case T_LONG: { 440 load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 441 break; 442 } 443 default: 444 ShouldNotReachHere(); 445 } 446 } 447 448 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 449 load_store = kit->gvn().transform(load_store); 450 451 access.set_raw_access(load_store); 452 pin_atomic_op(access); 453 454 #ifdef _LP64 455 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 456 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 457 } 458 #endif 459 460 return load_store; 461 } 462 463 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 464 Node* new_val, const Type* value_type) const { 465 GraphKit* kit = access.kit(); 466 DecoratorSet decorators = access.decorators(); 467 MemNode::MemOrd mo = access.mem_node_mo(); 468 Node* mem = access.memory(); 469 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; 470 Node* load_store = nullptr; 471 Node* adr = access.addr().node(); 472 473 if (access.is_oop()) { 474 #ifdef _LP64 475 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 476 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 477 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 478 if (is_weak_cas) { 479 load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); 480 } else { 481 load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); 482 } 483 } else 484 #endif 485 { 486 if (is_weak_cas) { 487 load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); 488 } else { 489 load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); 490 } 491 } 492 } else { 493 switch(access.type()) { 494 case T_BYTE: { 495 if (is_weak_cas) { 496 load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); 497 } else { 498 load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); 499 } 500 break; 501 } 502 case T_SHORT: { 503 if (is_weak_cas) { 504 load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); 505 } else { 506 load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); 507 } 508 break; 509 } 510 case T_INT: { 511 if (is_weak_cas) { 512 load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); 513 } else { 514 load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); 515 } 516 break; 517 } 518 case T_LONG: { 519 if (is_weak_cas) { 520 load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); 521 } else { 522 load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); 523 } 524 break; 525 } 526 default: 527 ShouldNotReachHere(); 528 } 529 } 530 531 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 532 load_store = kit->gvn().transform(load_store); 533 534 access.set_raw_access(load_store); 535 pin_atomic_op(access); 536 537 return load_store; 538 } 539 540 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 541 GraphKit* kit = access.kit(); 542 Node* mem = access.memory(); 543 Node* adr = access.addr().node(); 544 const TypePtr* adr_type = access.addr().type(); 545 Node* load_store = nullptr; 546 547 if (access.is_oop()) { 548 #ifdef _LP64 549 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 550 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 551 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop())); 552 } else 553 #endif 554 { 555 load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); 556 } 557 } else { 558 switch (access.type()) { 559 case T_BYTE: 560 load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); 561 break; 562 case T_SHORT: 563 load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); 564 break; 565 case T_INT: 566 load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); 567 break; 568 case T_LONG: 569 load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); 570 break; 571 default: 572 ShouldNotReachHere(); 573 } 574 } 575 576 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 577 load_store = kit->gvn().transform(load_store); 578 579 access.set_raw_access(load_store); 580 pin_atomic_op(access); 581 582 #ifdef _LP64 583 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 584 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 585 } 586 #endif 587 588 return load_store; 589 } 590 591 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 592 Node* load_store = nullptr; 593 GraphKit* kit = access.kit(); 594 Node* adr = access.addr().node(); 595 const TypePtr* adr_type = access.addr().type(); 596 Node* mem = access.memory(); 597 598 switch(access.type()) { 599 case T_BYTE: 600 load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); 601 break; 602 case T_SHORT: 603 load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); 604 break; 605 case T_INT: 606 load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); 607 break; 608 case T_LONG: 609 load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); 610 break; 611 default: 612 ShouldNotReachHere(); 613 } 614 615 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 616 load_store = kit->gvn().transform(load_store); 617 618 access.set_raw_access(load_store); 619 pin_atomic_op(access); 620 621 return load_store; 622 } 623 624 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, 625 Node* new_val, const Type* value_type) const { 626 C2AccessFence fence(access); 627 resolve_address(access); 628 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 629 } 630 631 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, 632 Node* new_val, const Type* value_type) const { 633 C2AccessFence fence(access); 634 resolve_address(access); 635 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 636 } 637 638 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 639 C2AccessFence fence(access); 640 resolve_address(access); 641 return atomic_xchg_at_resolved(access, new_val, value_type); 642 } 643 644 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 645 C2AccessFence fence(access); 646 resolve_address(access); 647 return atomic_add_at_resolved(access, new_val, value_type); 648 } 649 650 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) { 651 // Exclude the header but include array length to copy by 8 bytes words. 652 // Can't use base_offset_in_bytes(bt) since basic type is unknown. 653 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : 654 instanceOopDesc::base_offset_in_bytes(); 655 // base_off: 656 // 8 - 32-bit VM 657 // 12 - 64-bit VM, compressed klass 658 // 16 - 64-bit VM, normal klass 659 if (base_off % BytesPerLong != 0) { 660 assert(UseCompressedClassPointers, ""); 661 if (is_array) { 662 // Exclude length to copy by 8 bytes words. 663 base_off += sizeof(int); 664 } else { 665 // Include klass to copy by 8 bytes words. 666 base_off = instanceOopDesc::klass_offset_in_bytes(); 667 } 668 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); 669 } 670 return base_off; 671 } 672 673 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const { 674 int base_off = arraycopy_payload_base_offset(is_array); 675 Node* payload_size = size; 676 Node* offset = kit->MakeConX(base_off); 677 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset)); 678 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong))); 679 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false); 680 if (is_array) { 681 ac->set_clone_array(); 682 } else { 683 ac->set_clone_inst(); 684 } 685 Node* n = kit->gvn().transform(ac); 686 if (n == ac) { 687 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 688 ac->set_adr_type(TypeRawPtr::BOTTOM); 689 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 690 } else { 691 kit->set_all_memory(n); 692 } 693 } 694 695 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* mem, Node* toobig_false, Node* size_in_bytes, 696 Node*& i_o, Node*& needgc_ctrl, 697 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 698 intx prefetch_lines) const { 699 assert(UseTLAB, "Only for TLAB enabled allocations"); 700 701 Node* thread = macro->transform_later(new ThreadLocalNode()); 702 Node* tlab_top_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_top_offset())); 703 Node* tlab_end_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_end_offset())); 704 705 // Load TLAB end. 706 // 707 // Note: We set the control input on "tlab_end" and "old_tlab_top" to work around 708 // a bug where these values were being moved across 709 // a safepoint. These are not oops, so they cannot be include in the oop 710 // map, but they can be changed by a GC. The proper way to fix this would 711 // be to set the raw memory state when generating a SafepointNode. However 712 // this will require extensive changes to the loop optimization in order to 713 // prevent a degradation of the optimization. 714 // See comment in memnode.hpp, around line 227 in class LoadPNode. 715 Node* tlab_end = macro->make_load(toobig_false, mem, tlab_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 716 717 // Load the TLAB top. 718 Node* old_tlab_top = new LoadPNode(toobig_false, mem, tlab_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered); 719 macro->transform_later(old_tlab_top); 720 721 // Add to heap top to get a new TLAB top 722 Node* new_tlab_top = new AddPNode(macro->top(), old_tlab_top, size_in_bytes); 723 macro->transform_later(new_tlab_top); 724 725 // Check against TLAB end 726 Node* tlab_full = new CmpPNode(new_tlab_top, tlab_end); 727 macro->transform_later(tlab_full); 728 729 Node* needgc_bol = new BoolNode(tlab_full, BoolTest::ge); 730 macro->transform_later(needgc_bol); 731 IfNode* needgc_iff = new IfNode(toobig_false, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); 732 macro->transform_later(needgc_iff); 733 734 // Plug the failing-heap-space-need-gc test into the slow-path region 735 Node* needgc_true = new IfTrueNode(needgc_iff); 736 macro->transform_later(needgc_true); 737 needgc_ctrl = needgc_true; 738 739 // No need for a GC. 740 Node* needgc_false = new IfFalseNode(needgc_iff); 741 macro->transform_later(needgc_false); 742 743 // Fast path: 744 i_o = macro->prefetch_allocation(i_o, needgc_false, mem, 745 old_tlab_top, new_tlab_top, prefetch_lines); 746 747 // Store the modified TLAB top back down. 748 Node* store_tlab_top = new StorePNode(needgc_false, mem, tlab_top_adr, 749 TypeRawPtr::BOTTOM, new_tlab_top, MemNode::unordered); 750 macro->transform_later(store_tlab_top); 751 752 fast_oop_ctrl = needgc_false; 753 fast_oop_rawmem = store_tlab_top; 754 return old_tlab_top; 755 } 756 757 #define XTOP LP64_ONLY(COMMA phase->top()) 758 759 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { 760 Node* ctrl = ac->in(TypeFunc::Control); 761 Node* mem = ac->in(TypeFunc::Memory); 762 Node* src = ac->in(ArrayCopyNode::Src); 763 Node* src_offset = ac->in(ArrayCopyNode::SrcPos); 764 Node* dest = ac->in(ArrayCopyNode::Dest); 765 Node* dest_offset = ac->in(ArrayCopyNode::DestPos); 766 Node* length = ac->in(ArrayCopyNode::Length); 767 768 Node* payload_src = phase->basic_plus_adr(src, src_offset); 769 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset); 770 771 const char* copyfunc_name = "arraycopy"; 772 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true); 773 774 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 775 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type(); 776 777 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP); 778 phase->transform_later(call); 779 780 phase->igvn().replace_node(ac, call); 781 } 782 783 #undef XTOP