1 /*
  2  * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/tlab_globals.hpp"
 27 #include "gc/shared/c2/barrierSetC2.hpp"
 28 #include "opto/arraycopynode.hpp"
 29 #include "opto/convertnode.hpp"
 30 #include "opto/graphKit.hpp"
 31 #include "opto/idealKit.hpp"
 32 #include "opto/macro.hpp"
 33 #include "opto/narrowptrnode.hpp"
 34 #include "opto/runtime.hpp"
 35 #include "utilities/macros.hpp"
 36 
 37 // By default this is a no-op.
 38 void BarrierSetC2::resolve_address(C2Access& access) const { }
 39 
 40 void* C2ParseAccess::barrier_set_state() const {
 41   return _kit->barrier_set_state();
 42 }
 43 
 44 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
 45 
 46 bool C2Access::needs_cpu_membar() const {
 47   bool mismatched   = (_decorators & C2_MISMATCHED) != 0;
 48   bool is_unordered = (_decorators & MO_UNORDERED) != 0;
 49 
 50   bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
 51   bool in_heap   = (_decorators & IN_HEAP) != 0;
 52   bool in_native = (_decorators & IN_NATIVE) != 0;
 53   bool is_mixed  = !in_heap && !in_native;
 54 
 55   bool is_write  = (_decorators & C2_WRITE_ACCESS) != 0;
 56   bool is_read   = (_decorators & C2_READ_ACCESS) != 0;
 57   bool is_atomic = is_read && is_write;
 58 
 59   if (is_atomic) {
 60     // Atomics always need to be wrapped in CPU membars
 61     return true;
 62   }
 63 
 64   if (anonymous) {
 65     // We will need memory barriers unless we can determine a unique
 66     // alias category for this reference.  (Note:  If for some reason
 67     // the barriers get omitted and the unsafe reference begins to "pollute"
 68     // the alias analysis of the rest of the graph, either Compile::can_alias
 69     // or Compile::must_alias will throw a diagnostic assert.)
 70     if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
 71       return true;
 72     }
 73   } else {
 74     assert(!is_mixed, "not unsafe");
 75   }
 76 
 77   return false;
 78 }
 79 
 80 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
 81   DecoratorSet decorators = access.decorators();
 82 
 83   bool mismatched = (decorators & C2_MISMATCHED) != 0;
 84   bool unaligned = (decorators & C2_UNALIGNED) != 0;
 85   bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
 86   bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
 87 
 88   bool in_native = (decorators & IN_NATIVE) != 0;
 89   assert(!in_native || (unsafe && !access.is_oop()), "not supported yet");
 90 
 91   MemNode::MemOrd mo = access.mem_node_mo();
 92 
 93   Node* store;
 94   BasicType bt = access.type();
 95   if (access.is_parse_access()) {
 96     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 97 
 98     GraphKit* kit = parse_access.kit();
 99     if (bt == T_DOUBLE) {
100       Node* new_val = kit->dprecision_rounding(val.node());
101       val.set_node(new_val);
102     }
103 
104     store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt,
105                                  access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe);
106   } else {
107     assert(access.is_opt_access(), "either parse or opt access");
108     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
109     Node* ctl = opt_access.ctl();
110     MergeMemNode* mm = opt_access.mem();
111     PhaseGVN& gvn = opt_access.gvn();
112     const TypePtr* adr_type = access.addr().type();
113     int alias = gvn.C->get_alias_index(adr_type);
114     Node* mem = mm->memory_at(alias);
115 
116     StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), bt, mo, requires_atomic_access);
117     if (unaligned) {
118       st->set_unaligned_access();
119     }
120     if (mismatched) {
121       st->set_mismatched_access();
122     }
123     store = gvn.transform(st);
124     if (store == st) {
125       mm->set_memory_at(alias, st);
126     }
127   }
128   access.set_raw_access(store);
129 
130   return store;
131 }
132 
133 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
134   DecoratorSet decorators = access.decorators();
135 
136   Node* adr = access.addr().node();
137   const TypePtr* adr_type = access.addr().type();
138 
139   bool mismatched = (decorators & C2_MISMATCHED) != 0;
140   bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
141   bool unaligned = (decorators & C2_UNALIGNED) != 0;
142   bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
143   bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0;
144   bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
145   bool immutable = (decorators & C2_IMMUTABLE_MEMORY) != 0;
146 
147   bool in_native = (decorators & IN_NATIVE) != 0;
148 
149   MemNode::MemOrd mo = access.mem_node_mo();
150   LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest;
151 
152   Node* load;
153   if (access.is_parse_access()) {
154     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
155     GraphKit* kit = parse_access.kit();
156     Node* control = control_dependent ? kit->control() : NULL;
157 
158     if (immutable) {
159       Compile* C = Compile::current();
160       Node* mem = kit->immutable_memory();
161       load = LoadNode::make(kit->gvn(), control, mem, adr,
162                             adr_type, val_type, access.type(), mo, dep, requires_atomic_access,
163                             unaligned, mismatched, unsafe, access.barrier_data());
164       load = kit->gvn().transform(load);
165     } else {
166       load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
167                             dep, requires_atomic_access, unaligned, mismatched, unsafe,
168                             access.barrier_data());
169     }
170   } else {
171     assert(access.is_opt_access(), "either parse or opt access");
172     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
173     Node* control = control_dependent ? opt_access.ctl() : NULL;
174     MergeMemNode* mm = opt_access.mem();
175     PhaseGVN& gvn = opt_access.gvn();
176     Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
177     load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep,
178                           requires_atomic_access, unaligned, mismatched, unsafe, access.barrier_data());
179     load = gvn.transform(load);
180   }
181   access.set_raw_access(load);
182 
183   return load;
184 }
185 
186 class C2AccessFence: public StackObj {
187   C2Access& _access;
188   Node* _leading_membar;
189 
190 public:
191   C2AccessFence(C2Access& access) :
192     _access(access), _leading_membar(NULL) {
193     GraphKit* kit = NULL;
194     if (access.is_parse_access()) {
195       C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
196       kit = parse_access.kit();
197     }
198     DecoratorSet decorators = access.decorators();
199 
200     bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
201     bool is_read = (decorators & C2_READ_ACCESS) != 0;
202     bool is_atomic = is_read && is_write;
203 
204     bool is_volatile = (decorators & MO_SEQ_CST) != 0;
205     bool is_release = (decorators & MO_RELEASE) != 0;
206 
207     if (is_atomic) {
208       assert(kit != NULL, "unsupported at optimization time");
209       // Memory-model-wise, a LoadStore acts like a little synchronized
210       // block, so needs barriers on each side.  These don't translate
211       // into actual barriers on most machines, but we still need rest of
212       // compiler to respect ordering.
213       if (is_release) {
214         _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
215       } else if (is_volatile) {
216         if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
217           _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
218         } else {
219           _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
220         }
221       }
222     } else if (is_write) {
223       // If reference is volatile, prevent following memory ops from
224       // floating down past the volatile write.  Also prevents commoning
225       // another volatile read.
226       if (is_volatile || is_release) {
227         assert(kit != NULL, "unsupported at optimization time");
228         _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
229       }
230     } else {
231       // Memory barrier to prevent normal and 'unsafe' accesses from
232       // bypassing each other.  Happens after null checks, so the
233       // exception paths do not take memory state from the memory barrier,
234       // so there's no problems making a strong assert about mixing users
235       // of safe & unsafe memory.
236       if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
237         assert(kit != NULL, "unsupported at optimization time");
238         _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
239       }
240     }
241 
242     if (access.needs_cpu_membar()) {
243       assert(kit != NULL, "unsupported at optimization time");
244       kit->insert_mem_bar(Op_MemBarCPUOrder);
245     }
246 
247     if (is_atomic) {
248       // 4984716: MemBars must be inserted before this
249       //          memory node in order to avoid a false
250       //          dependency which will confuse the scheduler.
251       access.set_memory();
252     }
253   }
254 
255   ~C2AccessFence() {
256     GraphKit* kit = NULL;
257     if (_access.is_parse_access()) {
258       C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
259       kit = parse_access.kit();
260     }
261     DecoratorSet decorators = _access.decorators();
262 
263     bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
264     bool is_read = (decorators & C2_READ_ACCESS) != 0;
265     bool is_atomic = is_read && is_write;
266 
267     bool is_volatile = (decorators & MO_SEQ_CST) != 0;
268     bool is_acquire = (decorators & MO_ACQUIRE) != 0;
269 
270     // If reference is volatile, prevent following volatiles ops from
271     // floating up before the volatile access.
272     if (_access.needs_cpu_membar()) {
273       kit->insert_mem_bar(Op_MemBarCPUOrder);
274     }
275 
276     if (is_atomic) {
277       assert(kit != NULL, "unsupported at optimization time");
278       if (is_acquire || is_volatile) {
279         Node* n = _access.raw_access();
280         Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
281         if (_leading_membar != NULL) {
282           MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
283         }
284       }
285     } else if (is_write) {
286       // If not multiple copy atomic, we do the MemBarVolatile before the load.
287       if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
288         assert(kit != NULL, "unsupported at optimization time");
289         Node* n = _access.raw_access();
290         Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
291         if (_leading_membar != NULL) {
292           MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
293         }
294       }
295     } else {
296       if (is_volatile || is_acquire) {
297         assert(kit != NULL, "unsupported at optimization time");
298         Node* n = _access.raw_access();
299         assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
300         Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
301         mb->as_MemBar()->set_trailing_load();
302       }
303     }
304   }
305 };
306 
307 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
308   C2AccessFence fence(access);
309   resolve_address(access);
310   return store_at_resolved(access, val);
311 }
312 
313 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
314   C2AccessFence fence(access);
315   resolve_address(access);
316   return load_at_resolved(access, val_type);
317 }
318 
319 MemNode::MemOrd C2Access::mem_node_mo() const {
320   bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
321   bool is_read = (_decorators & C2_READ_ACCESS) != 0;
322   if ((_decorators & MO_SEQ_CST) != 0) {
323     if (is_write && is_read) {
324       // For atomic operations
325       return MemNode::seqcst;
326     } else if (is_write) {
327       return MemNode::release;
328     } else {
329       assert(is_read, "what else?");
330       return MemNode::acquire;
331     }
332   } else if ((_decorators & MO_RELEASE) != 0) {
333     return MemNode::release;
334   } else if ((_decorators & MO_ACQUIRE) != 0) {
335     return MemNode::acquire;
336   } else if (is_write) {
337     // Volatile fields need releasing stores.
338     // Non-volatile fields also need releasing stores if they hold an
339     // object reference, because the object reference might point to
340     // a freshly created object.
341     // Conservatively release stores of object references.
342     return StoreNode::release_if_reference(_type);
343   } else {
344     return MemNode::unordered;
345   }
346 }
347 
348 void C2Access::fixup_decorators() {
349   bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
350   bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
351   bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
352 
353   bool is_read = (_decorators & C2_READ_ACCESS) != 0;
354   bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
355 
356   if (AlwaysAtomicAccesses && is_unordered) {
357     _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
358     _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
359   }
360 
361   _decorators = AccessInternal::decorator_fixup(_decorators, _type);
362 
363   if (is_read && !is_write && anonymous) {
364     // To be valid, unsafe loads may depend on other conditions than
365     // the one that guards them: pin the Load node
366     _decorators |= C2_CONTROL_DEPENDENT_LOAD;
367     _decorators |= C2_UNKNOWN_CONTROL_LOAD;
368     const TypePtr* adr_type = _addr.type();
369     Node* adr = _addr.node();
370     if (!needs_cpu_membar() && adr_type->isa_instptr()) {
371       assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
372       intptr_t offset = Type::OffsetBot;
373       AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
374       if (offset >= 0) {
375         int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->instance_klass()->layout_helper());
376         if (offset < s) {
377           // Guaranteed to be a valid access, no need to pin it
378           _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
379           _decorators ^= C2_UNKNOWN_CONTROL_LOAD;
380         }
381       }
382     }
383   }
384 }
385 
386 //--------------------------- atomic operations---------------------------------
387 
388 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
389   // SCMemProjNodes represent the memory state of a LoadStore. Their
390   // main role is to prevent LoadStore nodes from being optimized away
391   // when their results aren't used.
392   assert(access.is_parse_access(), "entry not supported at optimization time");
393   C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
394   GraphKit* kit = parse_access.kit();
395   Node* load_store = access.raw_access();
396   assert(load_store != NULL, "must pin atomic op");
397   Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
398   kit->set_memory(proj, access.alias_idx());
399 }
400 
401 void C2AtomicParseAccess::set_memory() {
402   Node *mem = _kit->memory(_alias_idx);
403   _memory = mem;
404 }
405 
406 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
407                                                    Node* new_val, const Type* value_type) const {
408   GraphKit* kit = access.kit();
409   MemNode::MemOrd mo = access.mem_node_mo();
410   Node* mem = access.memory();
411 
412   Node* adr = access.addr().node();
413   const TypePtr* adr_type = access.addr().type();
414 
415   Node* load_store = NULL;
416 
417   if (access.is_oop()) {
418 #ifdef _LP64
419     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
420       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
421       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
422       load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo);
423     } else
424 #endif
425     {
426       load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo);
427     }
428   } else {
429     switch (access.type()) {
430       case T_BYTE: {
431         load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
432         break;
433       }
434       case T_SHORT: {
435         load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
436         break;
437       }
438       case T_INT: {
439         load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
440         break;
441       }
442       case T_LONG: {
443         load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo);
444         break;
445       }
446       default:
447         ShouldNotReachHere();
448     }
449   }
450 
451   load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
452   load_store = kit->gvn().transform(load_store);
453 
454   access.set_raw_access(load_store);
455   pin_atomic_op(access);
456 
457 #ifdef _LP64
458   if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
459     return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
460   }
461 #endif
462 
463   return load_store;
464 }
465 
466 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
467                                                     Node* new_val, const Type* value_type) const {
468   GraphKit* kit = access.kit();
469   DecoratorSet decorators = access.decorators();
470   MemNode::MemOrd mo = access.mem_node_mo();
471   Node* mem = access.memory();
472   bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
473   Node* load_store = NULL;
474   Node* adr = access.addr().node();
475 
476   if (access.is_oop()) {
477 #ifdef _LP64
478     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
479       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
480       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
481       if (is_weak_cas) {
482         load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo);
483       } else {
484         load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo);
485       }
486     } else
487 #endif
488     {
489       if (is_weak_cas) {
490         load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo);
491       } else {
492         load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo);
493       }
494     }
495   } else {
496     switch(access.type()) {
497       case T_BYTE: {
498         if (is_weak_cas) {
499           load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo);
500         } else {
501           load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo);
502         }
503         break;
504       }
505       case T_SHORT: {
506         if (is_weak_cas) {
507           load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo);
508         } else {
509           load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo);
510         }
511         break;
512       }
513       case T_INT: {
514         if (is_weak_cas) {
515           load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo);
516         } else {
517           load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo);
518         }
519         break;
520       }
521       case T_LONG: {
522         if (is_weak_cas) {
523           load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo);
524         } else {
525           load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo);
526         }
527         break;
528       }
529       default:
530         ShouldNotReachHere();
531     }
532   }
533 
534   load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
535   load_store = kit->gvn().transform(load_store);
536 
537   access.set_raw_access(load_store);
538   pin_atomic_op(access);
539 
540   return load_store;
541 }
542 
543 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
544   GraphKit* kit = access.kit();
545   Node* mem = access.memory();
546   Node* adr = access.addr().node();
547   const TypePtr* adr_type = access.addr().type();
548   Node* load_store = NULL;
549 
550   if (access.is_oop()) {
551 #ifdef _LP64
552     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
553       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
554       load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
555     } else
556 #endif
557     {
558       load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr());
559     }
560   } else  {
561     switch (access.type()) {
562       case T_BYTE:
563         load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type);
564         break;
565       case T_SHORT:
566         load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type);
567         break;
568       case T_INT:
569         load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type);
570         break;
571       case T_LONG:
572         load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type);
573         break;
574       default:
575         ShouldNotReachHere();
576     }
577   }
578 
579   load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
580   load_store = kit->gvn().transform(load_store);
581 
582   access.set_raw_access(load_store);
583   pin_atomic_op(access);
584 
585 #ifdef _LP64
586   if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
587     return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
588   }
589 #endif
590 
591   return load_store;
592 }
593 
594 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
595   Node* load_store = NULL;
596   GraphKit* kit = access.kit();
597   Node* adr = access.addr().node();
598   const TypePtr* adr_type = access.addr().type();
599   Node* mem = access.memory();
600 
601   switch(access.type()) {
602     case T_BYTE:
603       load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type);
604       break;
605     case T_SHORT:
606       load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type);
607       break;
608     case T_INT:
609       load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type);
610       break;
611     case T_LONG:
612       load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type);
613       break;
614     default:
615       ShouldNotReachHere();
616   }
617 
618   load_store->as_LoadStore()->set_barrier_data(access.barrier_data());
619   load_store = kit->gvn().transform(load_store);
620 
621   access.set_raw_access(load_store);
622   pin_atomic_op(access);
623 
624   return load_store;
625 }
626 
627 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
628                                           Node* new_val, const Type* value_type) const {
629   C2AccessFence fence(access);
630   resolve_address(access);
631   return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
632 }
633 
634 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
635                                            Node* new_val, const Type* value_type) const {
636   C2AccessFence fence(access);
637   resolve_address(access);
638   return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
639 }
640 
641 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
642   C2AccessFence fence(access);
643   resolve_address(access);
644   return atomic_xchg_at_resolved(access, new_val, value_type);
645 }
646 
647 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
648   C2AccessFence fence(access);
649   resolve_address(access);
650   return atomic_add_at_resolved(access, new_val, value_type);
651 }
652 
653 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
654   // Exclude the header but include array length to copy by 8 bytes words.
655   // Can't use base_offset_in_bytes(bt) since basic type is unknown.
656   int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
657                             instanceOopDesc::base_offset_in_bytes();
658   // base_off:
659   // 8  - 32-bit VM
660   // 12 - 64-bit VM, compressed klass
661   // 16 - 64-bit VM, normal klass
662   if (base_off % BytesPerLong != 0) {
663     assert(UseCompressedClassPointers, "");
664     if (is_array) {
665       // Exclude length to copy by 8 bytes words.
666       base_off += sizeof(int);
667     } else {
668       // Include klass to copy by 8 bytes words.
669       base_off = instanceOopDesc::klass_offset_in_bytes();
670     }
671     assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
672   }
673   return base_off;
674 }
675 
676 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
677   int base_off = arraycopy_payload_base_offset(is_array);
678   Node* payload_size = size;
679   Node* offset = kit->MakeConX(base_off);
680   payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
681   payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
682   ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset,  dst_base, offset, payload_size, true, false);
683   if (is_array) {
684     ac->set_clone_array();
685   } else {
686     ac->set_clone_inst();
687   }
688   Node* n = kit->gvn().transform(ac);
689   if (n == ac) {
690     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
691     ac->set_adr_type(TypeRawPtr::BOTTOM);
692     kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
693   } else {
694     kit->set_all_memory(n);
695   }
696 }
697 
698 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* mem, Node* toobig_false, Node* size_in_bytes,
699                                  Node*& i_o, Node*& needgc_ctrl,
700                                  Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
701                                  intx prefetch_lines) const {
702   assert(UseTLAB, "Only for TLAB enabled allocations");
703 
704   Node* thread = macro->transform_later(new ThreadLocalNode());
705   Node* tlab_top_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_top_offset()));
706   Node* tlab_end_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, in_bytes(JavaThread::tlab_end_offset()));
707 
708   // Load TLAB end.
709   //
710   // Note: We set the control input on "tlab_end" and "old_tlab_top" to work around
711   //       a bug where these values were being moved across
712   //       a safepoint.  These are not oops, so they cannot be include in the oop
713   //       map, but they can be changed by a GC.   The proper way to fix this would
714   //       be to set the raw memory state when generating a  SafepointNode.  However
715   //       this will require extensive changes to the loop optimization in order to
716   //       prevent a degradation of the optimization.
717   //       See comment in memnode.hpp, around line 227 in class LoadPNode.
718   Node* tlab_end = macro->make_load(toobig_false, mem, tlab_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
719 
720   // Load the TLAB top.
721   Node* old_tlab_top = new LoadPNode(toobig_false, mem, tlab_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
722   macro->transform_later(old_tlab_top);
723 
724   // Add to heap top to get a new TLAB top
725   Node* new_tlab_top = new AddPNode(macro->top(), old_tlab_top, size_in_bytes);
726   macro->transform_later(new_tlab_top);
727 
728   // Check against TLAB end
729   Node* tlab_full = new CmpPNode(new_tlab_top, tlab_end);
730   macro->transform_later(tlab_full);
731 
732   Node* needgc_bol = new BoolNode(tlab_full, BoolTest::ge);
733   macro->transform_later(needgc_bol);
734   IfNode* needgc_iff = new IfNode(toobig_false, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
735   macro->transform_later(needgc_iff);
736 
737   // Plug the failing-heap-space-need-gc test into the slow-path region
738   Node* needgc_true = new IfTrueNode(needgc_iff);
739   macro->transform_later(needgc_true);
740   needgc_ctrl = needgc_true;
741 
742   // No need for a GC.
743   Node* needgc_false = new IfFalseNode(needgc_iff);
744   macro->transform_later(needgc_false);
745 
746   // Fast path:
747   i_o = macro->prefetch_allocation(i_o, needgc_false, mem,
748                                    old_tlab_top, new_tlab_top, prefetch_lines);
749 
750   // Store the modified TLAB top back down.
751   Node* store_tlab_top = new StorePNode(needgc_false, mem, tlab_top_adr,
752                    TypeRawPtr::BOTTOM, new_tlab_top, MemNode::unordered);
753   macro->transform_later(store_tlab_top);
754 
755   fast_oop_ctrl = needgc_false;
756   fast_oop_rawmem = store_tlab_top;
757   return old_tlab_top;
758 }
759 
760 #define XTOP LP64_ONLY(COMMA phase->top())
761 
762 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
763   Node* ctrl = ac->in(TypeFunc::Control);
764   Node* mem = ac->in(TypeFunc::Memory);
765   Node* src = ac->in(ArrayCopyNode::Src);
766   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
767   Node* dest = ac->in(ArrayCopyNode::Dest);
768   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
769   Node* length = ac->in(ArrayCopyNode::Length);
770 
771   Node* payload_src = phase->basic_plus_adr(src, src_offset);
772   Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
773 
774   const char* copyfunc_name = "arraycopy";
775   address     copyfunc_addr = phase->basictype2arraycopy(T_LONG, NULL, NULL, true, copyfunc_name, true);
776 
777   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
778   const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
779 
780   Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
781   phase->transform_later(call);
782 
783   phase->igvn().replace_node(ac, call);
784 }
785 
786 #undef XTOP