1 /* 2 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/tlab_globals.hpp" 27 #include "gc/shared/c2/barrierSetC2.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/convertnode.hpp" 30 #include "opto/graphKit.hpp" 31 #include "opto/idealKit.hpp" 32 #include "opto/macro.hpp" 33 #include "opto/narrowptrnode.hpp" 34 #include "opto/runtime.hpp" 35 #include "utilities/macros.hpp" 36 37 // By default this is a no-op. 38 void BarrierSetC2::resolve_address(C2Access& access) const { } 39 40 void* C2ParseAccess::barrier_set_state() const { 41 return _kit->barrier_set_state(); 42 } 43 44 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } 45 46 bool C2Access::needs_cpu_membar() const { 47 bool mismatched = (_decorators & C2_MISMATCHED) != 0; 48 bool is_unordered = (_decorators & MO_UNORDERED) != 0; 49 50 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 51 bool in_heap = (_decorators & IN_HEAP) != 0; 52 bool in_native = (_decorators & IN_NATIVE) != 0; 53 bool is_mixed = !in_heap && !in_native; 54 55 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 56 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 57 bool is_atomic = is_read && is_write; 58 59 if (is_atomic) { 60 // Atomics always need to be wrapped in CPU membars 61 return true; 62 } 63 64 if (anonymous) { 65 // We will need memory barriers unless we can determine a unique 66 // alias category for this reference. (Note: If for some reason 67 // the barriers get omitted and the unsafe reference begins to "pollute" 68 // the alias analysis of the rest of the graph, either Compile::can_alias 69 // or Compile::must_alias will throw a diagnostic assert.) 70 if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) { 71 return true; 72 } 73 } else { 74 assert(!is_mixed, "not unsafe"); 75 } 76 77 return false; 78 } 79 80 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 81 DecoratorSet decorators = access.decorators(); 82 83 bool mismatched = (decorators & C2_MISMATCHED) != 0; 84 bool unaligned = (decorators & C2_UNALIGNED) != 0; 85 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 86 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 87 88 bool in_native = (decorators & IN_NATIVE) != 0; 89 assert(!in_native || (unsafe && !access.is_oop()), "not supported yet"); 90 91 MemNode::MemOrd mo = access.mem_node_mo(); 92 93 Node* store; 94 BasicType bt = access.type(); 95 if (access.is_parse_access()) { 96 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 97 98 GraphKit* kit = parse_access.kit(); 99 if (bt == T_DOUBLE) { 100 Node* new_val = kit->dprecision_rounding(val.node()); 101 val.set_node(new_val); 102 } 103 104 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt, 105 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe); 106 } else { 107 assert(access.is_opt_access(), "either parse or opt access"); 108 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 109 Node* ctl = opt_access.ctl(); 110 MergeMemNode* mm = opt_access.mem(); 111 PhaseGVN& gvn = opt_access.gvn(); 112 const TypePtr* adr_type = access.addr().type(); 113 int alias = gvn.C->get_alias_index(adr_type); 114 Node* mem = mm->memory_at(alias); 115 116 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), bt, mo, requires_atomic_access); 117 if (unaligned) { 118 st->set_unaligned_access(); 119 } 120 if (mismatched) { 121 st->set_mismatched_access(); 122 } 123 store = gvn.transform(st); 124 if (store == st) { 125 mm->set_memory_at(alias, st); 126 } 127 } 128 access.set_raw_access(store); 129 130 return store; 131 } 132 133 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 134 DecoratorSet decorators = access.decorators(); 135 136 Node* adr = access.addr().node(); 137 const TypePtr* adr_type = access.addr().type(); 138 139 bool mismatched = (decorators & C2_MISMATCHED) != 0; 140 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 141 bool unaligned = (decorators & C2_UNALIGNED) != 0; 142 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0; 143 bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0; 144 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 145 bool immutable = (decorators & C2_IMMUTABLE_MEMORY) != 0; 146 147 bool in_native = (decorators & IN_NATIVE) != 0; 148 149 MemNode::MemOrd mo = access.mem_node_mo(); 150 LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest; 151 152 Node* load; 153 if (access.is_parse_access()) { 154 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 155 GraphKit* kit = parse_access.kit(); 156 Node* control = control_dependent ? kit->control() : NULL; 157 158 if (immutable) { 159 Compile* C = Compile::current(); 160 Node* mem = kit->immutable_memory(); 161 load = LoadNode::make(kit->gvn(), control, mem, adr, 162 adr_type, val_type, access.type(), mo, dep, requires_atomic_access, 163 unaligned, mismatched, unsafe, access.barrier_data()); 164 load = kit->gvn().transform(load); 165 } else { 166 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, 167 dep, requires_atomic_access, unaligned, mismatched, unsafe, 168 access.barrier_data()); 169 } 170 } else { 171 assert(access.is_opt_access(), "either parse or opt access"); 172 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 173 Node* control = control_dependent ? opt_access.ctl() : NULL; 174 MergeMemNode* mm = opt_access.mem(); 175 PhaseGVN& gvn = opt_access.gvn(); 176 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); 177 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, 178 requires_atomic_access, unaligned, mismatched, unsafe, access.barrier_data()); 179 load = gvn.transform(load); 180 } 181 access.set_raw_access(load); 182 183 return load; 184 } 185 186 class C2AccessFence: public StackObj { 187 C2Access& _access; 188 Node* _leading_membar; 189 190 public: 191 C2AccessFence(C2Access& access) : 192 _access(access), _leading_membar(NULL) { 193 GraphKit* kit = NULL; 194 if (access.is_parse_access()) { 195 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 196 kit = parse_access.kit(); 197 } 198 DecoratorSet decorators = access.decorators(); 199 200 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 201 bool is_read = (decorators & C2_READ_ACCESS) != 0; 202 bool is_atomic = is_read && is_write; 203 204 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 205 bool is_release = (decorators & MO_RELEASE) != 0; 206 207 if (is_atomic) { 208 assert(kit != NULL, "unsupported at optimization time"); 209 // Memory-model-wise, a LoadStore acts like a little synchronized 210 // block, so needs barriers on each side. These don't translate 211 // into actual barriers on most machines, but we still need rest of 212 // compiler to respect ordering. 213 if (is_release) { 214 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 215 } else if (is_volatile) { 216 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 217 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 218 } else { 219 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 220 } 221 } 222 } else if (is_write) { 223 // If reference is volatile, prevent following memory ops from 224 // floating down past the volatile write. Also prevents commoning 225 // another volatile read. 226 if (is_volatile || is_release) { 227 assert(kit != NULL, "unsupported at optimization time"); 228 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 229 } 230 } else { 231 // Memory barrier to prevent normal and 'unsafe' accesses from 232 // bypassing each other. Happens after null checks, so the 233 // exception paths do not take memory state from the memory barrier, 234 // so there's no problems making a strong assert about mixing users 235 // of safe & unsafe memory. 236 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) { 237 assert(kit != NULL, "unsupported at optimization time"); 238 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 239 } 240 } 241 242 if (access.needs_cpu_membar()) { 243 assert(kit != NULL, "unsupported at optimization time"); 244 kit->insert_mem_bar(Op_MemBarCPUOrder); 245 } 246 247 if (is_atomic) { 248 // 4984716: MemBars must be inserted before this 249 // memory node in order to avoid a false 250 // dependency which will confuse the scheduler. 251 access.set_memory(); 252 } 253 } 254 255 ~C2AccessFence() { 256 GraphKit* kit = NULL; 257 if (_access.is_parse_access()) { 258 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access); 259 kit = parse_access.kit(); 260 } 261 DecoratorSet decorators = _access.decorators(); 262 263 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 264 bool is_read = (decorators & C2_READ_ACCESS) != 0; 265 bool is_atomic = is_read && is_write; 266 267 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 268 bool is_acquire = (decorators & MO_ACQUIRE) != 0; 269 270 // If reference is volatile, prevent following volatiles ops from 271 // floating up before the volatile access. 272 if (_access.needs_cpu_membar()) { 273 kit->insert_mem_bar(Op_MemBarCPUOrder); 274 } 275 276 if (is_atomic) { 277 assert(kit != NULL, "unsupported at optimization time"); 278 if (is_acquire || is_volatile) { 279 Node* n = _access.raw_access(); 280 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 281 if (_leading_membar != NULL) { 282 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 283 } 284 } 285 } else if (is_write) { 286 // If not multiple copy atomic, we do the MemBarVolatile before the load. 287 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) { 288 assert(kit != NULL, "unsupported at optimization time"); 289 Node* n = _access.raw_access(); 290 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar 291 if (_leading_membar != NULL) { 292 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 293 } 294 } 295 } else { 296 if (is_volatile || is_acquire) { 297 assert(kit != NULL, "unsupported at optimization time"); 298 Node* n = _access.raw_access(); 299 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected"); 300 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 301 mb->as_MemBar()->set_trailing_load(); 302 } 303 } 304 } 305 }; 306 307 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { 308 C2AccessFence fence(access); 309 resolve_address(access); 310 return store_at_resolved(access, val); 311 } 312 313 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const { 314 C2AccessFence fence(access); 315 resolve_address(access); 316 return load_at_resolved(access, val_type); 317 } 318 319 MemNode::MemOrd C2Access::mem_node_mo() const { 320 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 321 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 322 if ((_decorators & MO_SEQ_CST) != 0) { 323 if (is_write && is_read) { 324 // For atomic operations 325 return MemNode::seqcst; 326 } else if (is_write) { 327 return MemNode::release; 328 } else { 329 assert(is_read, "what else?"); 330 return MemNode::acquire; 331 } 332 } else if ((_decorators & MO_RELEASE) != 0) { 333 return MemNode::release; 334 } else if ((_decorators & MO_ACQUIRE) != 0) { 335 return MemNode::acquire; 336 } else if (is_write) { 337 // Volatile fields need releasing stores. 338 // Non-volatile fields also need releasing stores if they hold an 339 // object reference, because the object reference might point to 340 // a freshly created object. 341 // Conservatively release stores of object references. 342 return StoreNode::release_if_reference(_type); 343 } else { 344 return MemNode::unordered; 345 } 346 } 347 348 void C2Access::fixup_decorators() { 349 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0; 350 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo; 351 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 352 353 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 354 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 355 356 if (AlwaysAtomicAccesses && is_unordered) { 357 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits 358 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess 359 } 360 361 _decorators = AccessInternal::decorator_fixup(_decorators, _type); 362 363 if (is_read && !is_write && anonymous) { 364 // To be valid, unsafe loads may depend on other conditions than 365 // the one that guards them: pin the Load node 366 _decorators |= C2_CONTROL_DEPENDENT_LOAD; 367 _decorators |= C2_UNKNOWN_CONTROL_LOAD; 368 const TypePtr* adr_type = _addr.type(); 369 Node* adr = _addr.node(); 370 if (!needs_cpu_membar() && adr_type->isa_instptr()) { 371 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); 372 intptr_t offset = Type::OffsetBot; 373 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); 374 if (offset >= 0) { 375 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->instance_klass()->layout_helper()); 376 if (offset < s) { 377 // Guaranteed to be a valid access, no need to pin it 378 _decorators ^= C2_CONTROL_DEPENDENT_LOAD; 379 _decorators ^= C2_UNKNOWN_CONTROL_LOAD; 380 } 381 } 382 } 383 } 384 } 385 386 //--------------------------- atomic operations--------------------------------- 387 388 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { 389 // SCMemProjNodes represent the memory state of a LoadStore. Their 390 // main role is to prevent LoadStore nodes from being optimized away 391 // when their results aren't used. 392 assert(access.is_parse_access(), "entry not supported at optimization time"); 393 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 394 GraphKit* kit = parse_access.kit(); 395 Node* load_store = access.raw_access(); 396 assert(load_store != NULL, "must pin atomic op"); 397 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); 398 kit->set_memory(proj, access.alias_idx()); 399 } 400 401 void C2AtomicParseAccess::set_memory() { 402 Node *mem = _kit->memory(_alias_idx); 403 _memory = mem; 404 } 405 406 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 407 Node* new_val, const Type* value_type) const { 408 GraphKit* kit = access.kit(); 409 MemNode::MemOrd mo = access.mem_node_mo(); 410 Node* mem = access.memory(); 411 412 Node* adr = access.addr().node(); 413 const TypePtr* adr_type = access.addr().type(); 414 415 Node* load_store = NULL; 416 417 if (access.is_oop()) { 418 #ifdef _LP64 419 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 420 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 421 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 422 load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); 423 } else 424 #endif 425 { 426 load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); 427 } 428 } else { 429 switch (access.type()) { 430 case T_BYTE: { 431 load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 432 break; 433 } 434 case T_SHORT: { 435 load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 436 break; 437 } 438 case T_INT: { 439 load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 440 break; 441 } 442 case T_LONG: { 443 load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); 444 break; 445 } 446 default: 447 ShouldNotReachHere(); 448 } 449 } 450 451 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 452 load_store = kit->gvn().transform(load_store); 453 454 access.set_raw_access(load_store); 455 pin_atomic_op(access); 456 457 #ifdef _LP64 458 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 459 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 460 } 461 #endif 462 463 return load_store; 464 } 465 466 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 467 Node* new_val, const Type* value_type) const { 468 GraphKit* kit = access.kit(); 469 DecoratorSet decorators = access.decorators(); 470 MemNode::MemOrd mo = access.mem_node_mo(); 471 Node* mem = access.memory(); 472 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; 473 Node* load_store = NULL; 474 Node* adr = access.addr().node(); 475 476 if (access.is_oop()) { 477 #ifdef _LP64 478 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 479 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 480 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 481 if (is_weak_cas) { 482 load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); 483 } else { 484 load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); 485 } 486 } else 487 #endif 488 { 489 if (is_weak_cas) { 490 load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); 491 } else { 492 load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); 493 } 494 } 495 } else { 496 switch(access.type()) { 497 case T_BYTE: { 498 if (is_weak_cas) { 499 load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); 500 } else { 501 load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); 502 } 503 break; 504 } 505 case T_SHORT: { 506 if (is_weak_cas) { 507 load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); 508 } else { 509 load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); 510 } 511 break; 512 } 513 case T_INT: { 514 if (is_weak_cas) { 515 load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); 516 } else { 517 load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); 518 } 519 break; 520 } 521 case T_LONG: { 522 if (is_weak_cas) { 523 load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); 524 } else { 525 load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); 526 } 527 break; 528 } 529 default: 530 ShouldNotReachHere(); 531 } 532 } 533 534 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 535 load_store = kit->gvn().transform(load_store); 536 537 access.set_raw_access(load_store); 538 pin_atomic_op(access); 539 540 return load_store; 541 } 542 543 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 544 GraphKit* kit = access.kit(); 545 Node* mem = access.memory(); 546 Node* adr = access.addr().node(); 547 const TypePtr* adr_type = access.addr().type(); 548 Node* load_store = NULL; 549 550 if (access.is_oop()) { 551 #ifdef _LP64 552 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 553 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 554 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop())); 555 } else 556 #endif 557 { 558 load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); 559 } 560 } else { 561 switch (access.type()) { 562 case T_BYTE: 563 load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); 564 break; 565 case T_SHORT: 566 load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); 567 break; 568 case T_INT: 569 load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); 570 break; 571 case T_LONG: 572 load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); 573 break; 574 default: 575 ShouldNotReachHere(); 576 } 577 } 578 579 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 580 load_store = kit->gvn().transform(load_store); 581 582 access.set_raw_access(load_store); 583 pin_atomic_op(access); 584 585 #ifdef _LP64 586 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 587 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 588 } 589 #endif 590 591 return load_store; 592 } 593 594 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 595 Node* load_store = NULL; 596 GraphKit* kit = access.kit(); 597 Node* adr = access.addr().node(); 598 const TypePtr* adr_type = access.addr().type(); 599 Node* mem = access.memory(); 600 601 switch(access.type()) { 602 case T_BYTE: 603 load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); 604 break; 605 case T_SHORT: 606 load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); 607 break; 608 case T_INT: 609 load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); 610 break; 611 case T_LONG: 612 load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); 613 break; 614 default: 615 ShouldNotReachHere(); 616 } 617 618 load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); 619 load_store = kit->gvn().transform(load_store); 620 621 access.set_raw_access(load_store); 622 pin_atomic_op(access); 623 624 return load_store; 625 } 626 627 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, 628 Node* new_val, const Type* value_type) const { 629 C2AccessFence fence(access); 630 resolve_address(access); 631 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 632 } 633 634 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, 635 Node* new_val, const Type* value_type) const { 636 C2AccessFence fence(access); 637 resolve_address(access); 638 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 639 } 640 641 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 642 C2AccessFence fence(access); 643 resolve_address(access); 644 return atomic_xchg_at_resolved(access, new_val, value_type); 645 } 646 647 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 648 C2AccessFence fence(access); 649 resolve_address(access); 650 return atomic_add_at_resolved(access, new_val, value_type); 651 } 652 653 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) { 654 // Exclude the header but include array length to copy by 8 bytes words. 655 // Can't use base_offset_in_bytes(bt) since basic type is unknown. 656 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : 657 instanceOopDesc::base_offset_in_bytes(); 658 // base_off: 659 // 8 - 32-bit VM 660 // 12 - 64-bit VM, compressed klass 661 // 16 - 64-bit VM, normal klass 662 if (base_off % BytesPerLong != 0) { 663 assert(UseCompressedClassPointers, ""); 664 if (is_array) { 665 // Exclude length to copy by 8 bytes words. 666 base_off += sizeof(int); 667 } else { 668 // Include klass to copy by 8 bytes words. 669 base_off = instanceOopDesc::klass_offset_in_bytes(); 670 } 671 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); 672 } 673 return base_off; 674 } 675 676 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const { 677 int base_off = arraycopy_payload_base_offset(is_array); 678 Node* payload_size = size; 679 Node* offset = kit->MakeConX(base_off); 680 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset)); 681 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong))); 682 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false); 683 if (is_array) { 684 ac->set_clone_array(); 685 } else { 686 ac->set_clone_inst(); 687 } 688 Node* n = kit->gvn().transform(ac); 689 if (n == ac) { 690 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 691 ac->set_adr_type(TypeRawPtr::BOTTOM); 692 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 693 } else { 694 kit->set_all_memory(n); 695 } 696 } 697 698 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* mem, Node* toobig_false, Node* size_in_bytes, 699 Node*& i_o, Node*& needgc_ctrl, 700 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 701 intx prefetch_lines) const { 702 assert(UseTLAB, "Only for TLAB enabled allocations"); 703 704 Node* thread = macro->transform_later(new ThreadLocalNode()); 705 Node* tlab_top_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_top_offset())); 706 Node* tlab_end_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_end_offset())); 707 708 // Load TLAB end. 709 // 710 // Note: We set the control input on "tlab_end" and "old_tlab_top" to work around 711 // a bug where these values were being moved across 712 // a safepoint. These are not oops, so they cannot be include in the oop 713 // map, but they can be changed by a GC. The proper way to fix this would 714 // be to set the raw memory state when generating a SafepointNode. However 715 // this will require extensive changes to the loop optimization in order to 716 // prevent a degradation of the optimization. 717 // See comment in memnode.hpp, around line 227 in class LoadPNode. 718 Node* tlab_end = macro->make_load(toobig_false, mem, tlab_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 719 720 // Load the TLAB top. 721 Node* old_tlab_top = new LoadPNode(toobig_false, mem, tlab_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered); 722 macro->transform_later(old_tlab_top); 723 724 // Add to heap top to get a new TLAB top 725 Node* new_tlab_top = new AddPNode(macro->top(), old_tlab_top, size_in_bytes); 726 macro->transform_later(new_tlab_top); 727 728 // Check against TLAB end 729 Node* tlab_full = new CmpPNode(new_tlab_top, tlab_end); 730 macro->transform_later(tlab_full); 731 732 Node* needgc_bol = new BoolNode(tlab_full, BoolTest::ge); 733 macro->transform_later(needgc_bol); 734 IfNode* needgc_iff = new IfNode(toobig_false, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); 735 macro->transform_later(needgc_iff); 736 737 // Plug the failing-heap-space-need-gc test into the slow-path region 738 Node* needgc_true = new IfTrueNode(needgc_iff); 739 macro->transform_later(needgc_true); 740 needgc_ctrl = needgc_true; 741 742 // No need for a GC. 743 Node* needgc_false = new IfFalseNode(needgc_iff); 744 macro->transform_later(needgc_false); 745 746 // Fast path: 747 i_o = macro->prefetch_allocation(i_o, needgc_false, mem, 748 old_tlab_top, new_tlab_top, prefetch_lines); 749 750 // Store the modified TLAB top back down. 751 Node* store_tlab_top = new StorePNode(needgc_false, mem, tlab_top_adr, 752 TypeRawPtr::BOTTOM, new_tlab_top, MemNode::unordered); 753 macro->transform_later(store_tlab_top); 754 755 fast_oop_ctrl = needgc_false; 756 fast_oop_rawmem = store_tlab_top; 757 return old_tlab_top; 758 } 759 760 #define XTOP LP64_ONLY(COMMA phase->top()) 761 762 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { 763 Node* ctrl = ac->in(TypeFunc::Control); 764 Node* mem = ac->in(TypeFunc::Memory); 765 Node* src = ac->in(ArrayCopyNode::Src); 766 Node* src_offset = ac->in(ArrayCopyNode::SrcPos); 767 Node* dest = ac->in(ArrayCopyNode::Dest); 768 Node* dest_offset = ac->in(ArrayCopyNode::DestPos); 769 Node* length = ac->in(ArrayCopyNode::Length); 770 771 Node* payload_src = phase->basic_plus_adr(src, src_offset); 772 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset); 773 774 const char* copyfunc_name = "arraycopy"; 775 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, NULL, NULL, true, copyfunc_name, true); 776 777 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 778 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type(); 779 780 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP); 781 phase->transform_later(call); 782 783 phase->igvn().replace_node(ac, call); 784 } 785 786 #undef XTOP