< prev index next >

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

Print this page

 43 
 44 inline PSPromotionManager* PSPromotionManager::manager_array(uint index) {
 45   assert(_manager_array != NULL, "access of NULL manager_array");
 46   assert(index <= ParallelGCThreads, "out of range manager_array access");
 47   return &_manager_array[index];
 48 }
 49 
 50 inline void PSPromotionManager::push_depth(ScannerTask task) {
 51   claimed_stack_depth()->push(task);
 52 }
 53 
 54 template <class T>
 55 inline void PSPromotionManager::claim_or_forward_depth(T* p) {
 56   assert(should_scavenge(p, true), "revisiting object?");
 57   assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap");
 58   oop obj = RawAccess<IS_NOT_NULL>::oop_load(p);
 59   Prefetch::write(obj->mark_addr(), 0);
 60   push_depth(ScannerTask(p));
 61 }
 62 
 63 inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
 64                                                       size_t obj_size,
 65                                                       uint age, bool tenured,
 66                                                       const PSPromotionLAB* lab) {
 67   // Skip if memory allocation failed
 68   if (new_obj != NULL) {
 69     const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer();
 70 
 71     if (lab != NULL) {
 72       // Promotion of object through newly allocated PLAB
 73       if (gc_tracer->should_report_promotion_in_new_plab_event()) {
 74         size_t obj_bytes = obj_size * HeapWordSize;
 75         size_t lab_size = lab->capacity();
 76         gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes,
 77                                                       age, tenured, lab_size);
 78       }
 79     } else {
 80       // Promotion of object directly to heap
 81       if (gc_tracer->should_report_promotion_outside_plab_event()) {
 82         size_t obj_bytes = obj_size * HeapWordSize;
 83         gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes,
 84                                                        age, tenured);
 85       }
 86     }
 87   }
 88 }
 89 
 90 class PSPushContentsClosure: public BasicOopIterateClosure {
 91   PSPromotionManager* _pm;
 92  public:
 93   PSPushContentsClosure(PSPromotionManager* pm) : BasicOopIterateClosure(PSScavenge::reference_processor()), _pm(pm) {}
 94 
 95   template <typename T> void do_oop_nv(T* p) {
 96     if (PSScavenge::should_scavenge(p)) {
 97       _pm->claim_or_forward_depth(p);
 98     }
 99   }
100 
101   virtual void do_oop(oop* p)       { do_oop_nv(p); }
102   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
103 };

127     obj->oop_iterate_backwards(&pcc);
128   }
129 }
130 
131 template<bool promote_immediately>
132 inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
133   assert(should_scavenge(&o), "Sanity");
134 
135   // NOTE! We must be very careful with any methods that access the mark
136   // in o. There may be multiple threads racing on it, and it may be forwarded
137   // at any time.
138   markWord m = o->mark();
139   if (!m.is_marked()) {
140     return copy_unmarked_to_survivor_space<promote_immediately>(o, m);
141   } else {
142     // Ensure any loads from the forwardee follow all changes that precede
143     // the release-cmpxchg that performed the forwarding, possibly in some
144     // other thread.
145     OrderAccess::acquire();
146     // Return the already installed forwardee.
147     return cast_to_oop(m.decode_pointer());
148   }
149 }
150 
151 //
152 // This method is pretty bulky. It would be nice to split it up
153 // into smaller submethods, but we need to be careful not to hurt
154 // performance.
155 //
156 template<bool promote_immediately>
157 inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
158                                                                markWord test_mark) {
159   assert(should_scavenge(&o), "Sanity");
160 
161   oop new_obj = NULL;
162   bool new_obj_is_tenured = false;
163   size_t new_obj_size = o->size();









164 
165   // Find the objects age, MT safe.
166   uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
167       test_mark.displaced_mark_helper().age() : test_mark.age();
168 
169   if (!promote_immediately) {
170     // Try allocating obj in to-space (unless too old)
171     if (age < PSScavenge::tenuring_threshold()) {
172       new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
173       if (new_obj == NULL && !_young_gen_is_full) {
174         // Do we allocate directly, or flush and refill?
175         if (new_obj_size > (YoungPLABSize / 2)) {
176           // Allocate this object directly
177           new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size));
178           promotion_trace_event(new_obj, o, new_obj_size, age, false, NULL);
179         } else {
180           // Flush and fill
181           _young_lab.flush();
182 
183           HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
184           if (lab_base != NULL) {
185             _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
186             // Try the young lab allocation again.
187             new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
188             promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab);
189           } else {
190             _young_gen_is_full = true;
191           }
192         }
193       }
194     }
195   }
196 
197   // Otherwise try allocating obj tenured
198   if (new_obj == NULL) {
199 #ifndef PRODUCT
200     if (ParallelScavengeHeap::heap()->promotion_should_fail()) {
201       return oop_promotion_failed(o, test_mark);
202     }
203 #endif  // #ifndef PRODUCT
204 
205     new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
206     new_obj_is_tenured = true;
207 
208     if (new_obj == NULL) {
209       if (!_old_gen_is_full) {
210         // Do we allocate directly, or flush and refill?
211         if (new_obj_size > (OldPLABSize / 2)) {
212           // Allocate this object directly
213           new_obj = cast_to_oop(old_gen()->allocate(new_obj_size));
214           promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL);
215         } else {
216           // Flush and fill
217           _old_lab.flush();
218 
219           HeapWord* lab_base = old_gen()->allocate(OldPLABSize);
220           if(lab_base != NULL) {
221 #ifdef ASSERT
222             // Delay the initialization of the promotion lab (plab).
223             // This exposes uninitialized plabs to card table processing.
224             if (GCWorkerDelayMillis > 0) {
225               os::naked_sleep(GCWorkerDelayMillis);
226             }
227 #endif
228             _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
229             // Try the old lab allocation again.
230             new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
231             promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab);
232           }
233         }
234       }
235 
236       // This is the promotion failed test, and code handling.
237       // The code belongs here for two reasons. It is slightly
238       // different than the code below, and cannot share the
239       // CAS testing code. Keeping the code here also minimizes
240       // the impact on the common case fast path code.
241 
242       if (new_obj == NULL) {
243         _old_gen_is_full = true;
244         return oop_promotion_failed(o, test_mark);
245       }
246     }
247   }
248 
249   assert(new_obj != NULL, "allocation should have succeeded");
250 
251   // Copy obj

 43 
 44 inline PSPromotionManager* PSPromotionManager::manager_array(uint index) {
 45   assert(_manager_array != NULL, "access of NULL manager_array");
 46   assert(index <= ParallelGCThreads, "out of range manager_array access");
 47   return &_manager_array[index];
 48 }
 49 
 50 inline void PSPromotionManager::push_depth(ScannerTask task) {
 51   claimed_stack_depth()->push(task);
 52 }
 53 
 54 template <class T>
 55 inline void PSPromotionManager::claim_or_forward_depth(T* p) {
 56   assert(should_scavenge(p, true), "revisiting object?");
 57   assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap");
 58   oop obj = RawAccess<IS_NOT_NULL>::oop_load(p);
 59   Prefetch::write(obj->mark_addr(), 0);
 60   push_depth(ScannerTask(p));
 61 }
 62 
 63 inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj, Klass* klass,
 64                                                       size_t obj_size,
 65                                                       uint age, bool tenured,
 66                                                       const PSPromotionLAB* lab) {
 67   // Skip if memory allocation failed
 68   if (new_obj != NULL) {
 69     const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer();
 70 
 71     if (lab != NULL) {
 72       // Promotion of object through newly allocated PLAB
 73       if (gc_tracer->should_report_promotion_in_new_plab_event()) {
 74         size_t obj_bytes = obj_size * HeapWordSize;
 75         size_t lab_size = lab->capacity();
 76         gc_tracer->report_promotion_in_new_plab_event(klass, obj_bytes,
 77                                                       age, tenured, lab_size);
 78       }
 79     } else {
 80       // Promotion of object directly to heap
 81       if (gc_tracer->should_report_promotion_outside_plab_event()) {
 82         size_t obj_bytes = obj_size * HeapWordSize;
 83         gc_tracer->report_promotion_outside_plab_event(klass, obj_bytes,
 84                                                        age, tenured);
 85       }
 86     }
 87   }
 88 }
 89 
 90 class PSPushContentsClosure: public BasicOopIterateClosure {
 91   PSPromotionManager* _pm;
 92  public:
 93   PSPushContentsClosure(PSPromotionManager* pm) : BasicOopIterateClosure(PSScavenge::reference_processor()), _pm(pm) {}
 94 
 95   template <typename T> void do_oop_nv(T* p) {
 96     if (PSScavenge::should_scavenge(p)) {
 97       _pm->claim_or_forward_depth(p);
 98     }
 99   }
100 
101   virtual void do_oop(oop* p)       { do_oop_nv(p); }
102   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
103 };

127     obj->oop_iterate_backwards(&pcc);
128   }
129 }
130 
131 template<bool promote_immediately>
132 inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
133   assert(should_scavenge(&o), "Sanity");
134 
135   // NOTE! We must be very careful with any methods that access the mark
136   // in o. There may be multiple threads racing on it, and it may be forwarded
137   // at any time.
138   markWord m = o->mark();
139   if (!m.is_marked()) {
140     return copy_unmarked_to_survivor_space<promote_immediately>(o, m);
141   } else {
142     // Ensure any loads from the forwardee follow all changes that precede
143     // the release-cmpxchg that performed the forwarding, possibly in some
144     // other thread.
145     OrderAccess::acquire();
146     // Return the already installed forwardee.
147     return o->forwardee(m);
148   }
149 }
150 
151 //
152 // This method is pretty bulky. It would be nice to split it up
153 // into smaller submethods, but we need to be careful not to hurt
154 // performance.
155 //
156 template<bool promote_immediately>
157 inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
158                                                                markWord test_mark) {
159   assert(should_scavenge(&o), "Sanity");
160 
161   oop new_obj = NULL;
162   bool new_obj_is_tenured = false;
163   Klass* klass;
164 #ifdef _LP64
165   if (UseCompactObjectHeaders) {
166     klass = test_mark.safe_klass();
167   } else
168 #endif
169   {
170     klass = o->klass();
171   }
172   size_t new_obj_size = o->size_given_klass(klass);
173 
174   // Find the objects age, MT safe.
175   uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
176       test_mark.displaced_mark_helper().age() : test_mark.age();
177 
178   if (!promote_immediately) {
179     // Try allocating obj in to-space (unless too old)
180     if (age < PSScavenge::tenuring_threshold()) {
181       new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
182       if (new_obj == NULL && !_young_gen_is_full) {
183         // Do we allocate directly, or flush and refill?
184         if (new_obj_size > (YoungPLABSize / 2)) {
185           // Allocate this object directly
186           new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size));
187           promotion_trace_event(new_obj, o, klass, new_obj_size, age, false, NULL);
188         } else {
189           // Flush and fill
190           _young_lab.flush();
191 
192           HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
193           if (lab_base != NULL) {
194             _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
195             // Try the young lab allocation again.
196             new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
197             promotion_trace_event(new_obj, o, klass, new_obj_size, age, false, &_young_lab);
198           } else {
199             _young_gen_is_full = true;
200           }
201         }
202       }
203     }
204   }
205 
206   // Otherwise try allocating obj tenured
207   if (new_obj == NULL) {
208 #ifndef PRODUCT
209     if (ParallelScavengeHeap::heap()->promotion_should_fail()) {
210       return oop_promotion_failed(o, test_mark);
211     }
212 #endif  // #ifndef PRODUCT
213 
214     new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
215     new_obj_is_tenured = true;
216 
217     if (new_obj == NULL) {
218       if (!_old_gen_is_full) {
219         // Do we allocate directly, or flush and refill?
220         if (new_obj_size > (OldPLABSize / 2)) {
221           // Allocate this object directly
222           new_obj = cast_to_oop(old_gen()->allocate(new_obj_size));
223           promotion_trace_event(new_obj, o, klass, new_obj_size, age, true, NULL);
224         } else {
225           // Flush and fill
226           _old_lab.flush();
227 
228           HeapWord* lab_base = old_gen()->allocate(OldPLABSize);
229           if(lab_base != NULL) {
230 #ifdef ASSERT
231             // Delay the initialization of the promotion lab (plab).
232             // This exposes uninitialized plabs to card table processing.
233             if (GCWorkerDelayMillis > 0) {
234               os::naked_sleep(GCWorkerDelayMillis);
235             }
236 #endif
237             _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
238             // Try the old lab allocation again.
239             new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
240             promotion_trace_event(new_obj, o, klass, new_obj_size, age, true, &_old_lab);
241           }
242         }
243       }
244 
245       // This is the promotion failed test, and code handling.
246       // The code belongs here for two reasons. It is slightly
247       // different than the code below, and cannot share the
248       // CAS testing code. Keeping the code here also minimizes
249       // the impact on the common case fast path code.
250 
251       if (new_obj == NULL) {
252         _old_gen_is_full = true;
253         return oop_promotion_failed(o, test_mark);
254       }
255     }
256   }
257 
258   assert(new_obj != NULL, "allocation should have succeeded");
259 
260   // Copy obj
< prev index next >