1 /*
  2  * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP
 26 #define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP
 27 
 28 #include "jfr/recorder/storage/jfrMemorySpace.hpp"
 29 
 30 #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
 31 #include "runtime/atomic.hpp"
 32 #include "runtime/os.hpp"
 33 
 34 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 35 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::
 36 JfrMemorySpace(size_t min_element_size, size_t free_list_cache_count_limit, Client* client) :
 37   _free_list(),
 38   _live_list_epoch_0(),
 39   _live_list_epoch_1(),
 40   _client(client),
 41   _min_element_size(min_element_size),
 42   _free_list_cache_count_limit(free_list_cache_count_limit),
 43   _free_list_cache_count(0) {}
 44 
 45 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 46 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::~JfrMemorySpace() {
 47   while (_live_list_epoch_0.is_nonempty()) {
 48     deallocate(_live_list_epoch_0.remove());
 49   }
 50   while (_live_list_epoch_1.is_nonempty()) {
 51     deallocate(_live_list_epoch_1.remove());
 52   }
 53   while (_free_list.is_nonempty()) {
 54     deallocate(_free_list.remove());
 55   }
 56 }
 57 
 58 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 59 bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::initialize(size_t cache_prealloc_count, bool prealloc_to_free_list) {
 60   if (!(_free_list.initialize() && _live_list_epoch_0.initialize() && _live_list_epoch_1.initialize())) {
 61     return false;
 62   }
 63   // pre-allocate elements to be cached in the requested list
 64   for (size_t i = 0; i < cache_prealloc_count; ++i) {
 65     NodePtr const node = allocate(_min_element_size);
 66     if (node == NULL) {
 67       return false;
 68     }
 69     if (prealloc_to_free_list) {
 70       add_to_free_list(node);
 71     } else {
 72       add_to_live_list(node);
 73     }
 74   }
 75   return true;
 76 }
 77 
 78 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 79 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::should_populate_free_list_cache() const {
 80   return !is_free_list_cache_limited() || _free_list_cache_count < _free_list_cache_count_limit;
 81 }
 82 
 83 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 84 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::is_free_list_cache_limited() const {
 85   return _free_list_cache_count_limit != JFR_MSPACE_UNLIMITED_CACHE_SIZE;
 86 }
 87 
 88 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 89 inline size_t JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::min_element_size() const {
 90   return _min_element_size;
 91 }
 92 
 93 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 94 inline FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() {
 95   return _free_list;
 96 }
 97 
 98 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
 99 inline const FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() const {
100   return _free_list;
101 }
102 
103 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
104 inline FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) {
105   if (epoch_aware) {
106     return previous_epoch ? previous_epoch_list() : current_epoch_list();
107   }
108   return _live_list_epoch_0;
109 }
110 
111 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
112 inline const FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) const {
113   if (epoch_aware) {
114     return previous_epoch ? previous_epoch_list() : current_epoch_list();
115   }
116   return _live_list_epoch_0;
117 }
118 
119 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
120 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_empty() const {
121   return _free_list.is_empty();
122 }
123 
124 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
125 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_nonempty() const {
126   return !free_list_is_empty();
127 }
128 
129 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
130 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_empty(bool previous_epoch) const {
131   return live_list().is_empty();
132 }
133 
134 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
135 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_nonempty(bool previous_epoch) const {
136   return live_list().is_nonempty();
137 }
138 
139 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
140 bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_free_list(const typename FreeListType::Node* node) const {
141   return _free_list.in_list(node);
142 }
143 
144 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
145 inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
146 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) const {
147   assert(epoch_aware, "invariant");
148   return epoch == 0 ? _live_list_epoch_0 : _live_list_epoch_1;
149 }
150 
151 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
152 inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
153 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) {
154   return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
155     const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->epoch_list_selector(epoch));
156 }
157 
158 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
159 inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
160 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() const {
161   assert(epoch_aware, "invariant");
162   return epoch_list_selector(JfrTraceIdEpoch::current());
163 }
164 
165 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
166 inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
167 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() {
168   return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
169     const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->current_epoch_list());
170 }
171 
172 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
173 inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
174 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() const {
175   assert(epoch_aware, "invariant");
176   return epoch_list_selector(JfrTraceIdEpoch::previous());
177 }
178 
179 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
180 inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
181 JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() {
182   return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
183     const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->previous_epoch_list());
184 }
185 
186 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
187 bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_live_list(const typename FreeListType::Node* node, bool previous_epoch) const {
188   return live_list(previous_epoch).in_list(node);
189 }
190 
191 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
192 inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_current_epoch_list(const typename FreeListType::Node* node) const {
193   assert(epoch_aware, "invariant");
194   return current_epoch_list().in_list(node);
195 }
196 
197 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
198 inline bool JfrMemorySpace< Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_previous_epoch_list(const typename FreeListType::Node* node) const {
199   assert(epoch_aware, "invariant");
200   return previous_epoch_list().in_list(node);
201 }
202 
203 // allocations are even multiples of the mspace min size
204 static inline size_t align_allocation_size(size_t requested_size, size_t min_element_size) {
205   u8 alloc_size_bytes = min_element_size;
206   while (requested_size > alloc_size_bytes) {
207     alloc_size_bytes <<= 1;
208   }
209   return (size_t)alloc_size_bytes;
210 }
211 
212 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
213 inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::allocate(size_t size) {
214   const size_t aligned_size_bytes = align_allocation_size(size, _min_element_size);
215   void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(Node));
216   if (allocation == NULL) {
217     return NULL;
218   }
219   NodePtr node = new (allocation) Node();
220   assert(node != NULL, "invariant");
221   if (!node->initialize(sizeof(Node), aligned_size_bytes)) {
222     JfrCHeapObj::free(node, aligned_size_bytes + sizeof(Node));
223     return NULL;
224   }
225   return node;
226 }
227 
228 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
229 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::deallocate(typename FreeListType::NodePtr node) {
230   assert(node != NULL, "invariant");
231   assert(!in_free_list(node), "invariant");
232   assert(!_live_list_epoch_0.in_list(node), "invariant");
233   assert(!_live_list_epoch_1.in_list(node), "invariant");
234   assert(node != NULL, "invariant");
235   JfrCHeapObj::free(node, node->total_size());
236 }
237 
238 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
239 inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::acquire(size_t size, bool free_list, Thread* thread, bool previous_epoch) {
240   return RetrievalPolicy<JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware> >::acquire(this, free_list, thread, size, previous_epoch);
241 }
242 
243 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
244 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::release(typename FreeListType::NodePtr node) {
245   assert(node != NULL, "invariant");
246   if (node->transient()) {
247     deallocate(node);
248     return;
249   }
250   assert(node->empty(), "invariant");
251   assert(!node->retired(), "invariant");
252   assert(node->identity() == NULL, "invariant");
253   if (should_populate_free_list_cache()) {
254     add_to_free_list(node);
255   } else {
256     deallocate(node);
257   }
258 }
259 
260 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
261 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_free_list(typename FreeListType::NodePtr node) {
262   assert(node != NULL, "invariant");
263   _free_list.add(node);
264   if (is_free_list_cache_limited()) {
265     Atomic::inc(&_free_list_cache_count);
266   }
267 }
268 
269 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
270 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_live_list(typename FreeListType::NodePtr node, bool previous_epoch) {
271   assert(node != NULL, "invariant");
272   live_list(previous_epoch).add(node);
273 }
274 
275 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
276 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::decrement_free_list_count() {
277   if (is_free_list_cache_limited()) {
278     Atomic::dec(&_free_list_cache_count);
279   }
280 }
281 
282 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
283 template <typename Callback>
284 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_free_list(Callback& callback) {
285   return _free_list.iterate(callback);
286 }
287 
288 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
289 template <typename Callback>
290 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_live_list(Callback& callback, bool previous_epoch) {
291   if (epoch_aware) {
292     live_list(previous_epoch).iterate(callback);
293     return;
294   }
295   _live_list_epoch_0.iterate(callback);
296 }
297 
298 template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
299 inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::register_full(typename FreeListType::NodePtr node, Thread* thread) {
300   _client->register_full(node, thread);
301 }
302 
303 template <typename Mspace, typename Client>
304 static inline Mspace* create_mspace(size_t min_element_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count, bool prealloc_to_free_list, Client* cb) {
305   Mspace* const mspace = new Mspace(min_element_size, free_list_cache_count_limit, cb);
306   if (mspace != NULL) {
307     mspace->initialize(cache_prealloc_count, prealloc_to_free_list);
308   }
309   return mspace;
310 }
311 
312 template <typename Mspace>
313 inline typename Mspace::NodePtr mspace_allocate(size_t size, Mspace* mspace) {
314   return mspace->allocate(size);
315 }
316 
317 template <typename Mspace>
318 inline typename Mspace::NodePtr mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) {
319   typename Mspace::NodePtr node = mspace_allocate(size, mspace);
320   if (node == NULL) return NULL;
321   node->set_identity(thread);
322   return node;
323 }
324 
325 template <typename Mspace>
326 inline typename Mspace::NodePtr mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) {
327   typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
328   if (node == NULL) return NULL;
329   assert(node->acquired_by_self(), "invariant");
330   node->set_transient();
331   return node;
332 }
333 
334 template <typename Mspace>
335 inline typename Mspace::NodePtr mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) {
336   typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
337   if (node == NULL) return NULL;
338   assert(node->transient(), "invariant");
339   node->set_lease();
340   return node;
341 }
342 
343 template <typename Mspace>
344 inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) {
345   typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
346   if (node == NULL) return NULL;
347   assert(node->lease(), "invariant");
348   mspace->add_to_free_list(node);
349   return node;
350 }
351 
352 template <typename Mspace>
353 inline typename Mspace::NodePtr mspace_acquire_free(size_t size, Mspace* mspace, Thread* thread) {
354   return mspace->acquire(size, true, thread);
355 }
356 
357 template <typename Mspace>
358 inline typename Mspace::NodePtr mspace_acquire_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
359   assert(size <= mspace->min_element_size(), "invariant");
360   for (size_t i = 0; i < retry_count; ++i) {
361     typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread);
362     if (node != NULL) {
363       return node;
364     }
365   }
366   return NULL;
367 }
368 
369 template <typename Mspace>
370 inline typename Mspace::NodePtr mspace_allocate_to_live_list(size_t size, Mspace* mspace, Thread* thread) {
371   typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
372   if (node == NULL) return NULL;
373   assert(node->acquired_by_self(), "invariant");
374   mspace->add_to_live_list(node);
375   return node;
376 }
377 
378 template <typename Mspace>
379 inline typename Mspace::NodePtr mspace_allocate_transient_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
380   typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
381   if (node == NULL) return NULL;
382   assert(node->transient(), "invariant");
383   mspace->add_to_live_list(node, previous_epoch);
384   return node;
385 }
386 
387 template <typename Mspace>
388 inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
389   typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
390   if (node == NULL) return NULL;
391   assert(node->lease(), "invariant");
392   mspace->add_to_live_list(node, previous_epoch);
393   return node;
394 }
395 
396 template <typename Mspace>
397 inline typename Mspace::NodePtr mspace_acquire_free_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
398   assert(size <= mspace->min_element_size(), "invariant");
399   typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread);
400   if (node == NULL) {
401     return NULL;
402   }
403   assert(node->acquired_by_self(), "invariant");
404   mspace->add_to_live_list(node, previous_epoch);
405   return node;
406 }
407 
408 template <typename Mspace>
409 inline typename Mspace::NodePtr mspace_acquire_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
410   if (size <= mspace->min_element_size()) {
411     typename Mspace::NodePtr node = mspace_acquire_free_to_live_list(size, mspace, thread, previous_epoch);
412     if (node != NULL) {
413       return node;
414     }
415   }
416   return mspace_allocate_to_live_list(size, mspace, thread);
417 }
418 
419 template <typename Mspace>
420 inline typename Mspace::NodePtr mspace_acquire_live(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
421   return mspace->acquire(size, false, thread, previous_epoch);
422 }
423 
424 template <typename Mspace>
425 inline typename Mspace::NodePtr mspace_acquire_live_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) {
426   assert(size <= mspace->min_element_size(), "invariant");
427   for (size_t i = 0; i < retry_count; ++i) {
428     typename Mspace::NodePtr const node = mspace_acquire_live(size, mspace, thread, previous_epoch);
429     if (node != NULL) {
430       return node;
431     }
432   }
433   return NULL;
434 }
435 
436 template <typename Mspace>
437 inline typename Mspace::NodePtr mspace_acquire_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) {
438   typename Mspace::NodePtr node = mspace_acquire_live_with_retry(size, mspace, retry_count, thread, previous_epoch);
439   if (node != NULL) {
440     node->set_lease();
441   }
442   return node;
443 }
444 
445 template <typename Mspace>
446 inline void mspace_release(typename Mspace::NodePtr node, Mspace* mspace) {
447   assert(node != NULL, "invariant");
448   assert(node->unflushed_size() == 0, "invariant");
449   assert(mspace != NULL, "invariant");
450   mspace->release(node);
451 }
452 
453 template <typename Callback, typename Mspace>
454 inline void process_live_list(Callback& callback, Mspace* mspace, bool previous_epoch = false) {
455   assert(mspace != NULL, "invariant");
456   mspace->iterate_live_list(callback, previous_epoch);
457 }
458 
459 template <typename Callback, typename Mspace>
460 inline void process_free_list(Callback& callback, Mspace* mspace) {
461   assert(mspace != NULL, "invariant");
462   assert(mspace->free_list_is_nonempty(), "invariant");
463   mspace->iterate_free_list(callback);
464 }
465 
466 template <typename Mspace>
467 class ReleaseOp : public StackObj {
468  private:
469   Mspace* _mspace;
470   bool _previous_epoch;
471  public:
472   typedef typename Mspace::Node Node;
473   ReleaseOp(Mspace* mspace) : _mspace(mspace) {}
474   bool process(typename Mspace::NodePtr node);
475   size_t processed() const { return 0; }
476 };
477 
478 template <typename Mspace>
479 inline bool ReleaseOp<Mspace>::process(typename Mspace::NodePtr node) {
480   assert(node != NULL, "invariant");
481   // assumes some means of exclusive access to the node
482   if (node->transient()) {
483     // make sure the transient node is already detached
484     _mspace->release(node);
485     return true;
486   }
487   node->reinitialize();
488   if (node->identity() != NULL) {
489     assert(node->empty(), "invariant");
490     assert(!node->retired(), "invariant");
491     node->release(); // publish
492   }
493   return true;
494 }
495 
496 template <typename Mspace, typename List>
497 class ReleaseWithExcisionOp : public ReleaseOp<Mspace> {
498  private:
499   List& _list;
500   typename List::NodePtr _prev;
501   size_t _count;
502   size_t _amount;
503  public:
504   ReleaseWithExcisionOp(Mspace* mspace, List& list) :
505     ReleaseOp<Mspace>(mspace), _list(list), _prev(NULL), _count(0), _amount(0) {}
506   bool process(typename List::NodePtr node);
507   size_t processed() const { return _count; }
508   size_t amount() const { return _amount; }
509 };
510 
511 template <typename Mspace, typename List>
512 inline bool ReleaseWithExcisionOp<Mspace, List>::process(typename List::NodePtr node) {
513   assert(node != NULL, "invariant");
514   if (node->transient()) {
515     _prev = _list.excise(_prev, node);
516   } else {
517     _prev = node;
518   }
519   return ReleaseOp<Mspace>::process(node);
520 }
521 
522 template <typename Mspace, typename List>
523 class ScavengingReleaseOp : public StackObj {
524  protected:
525   Mspace* _mspace;
526   List& _list;
527   typename List::NodePtr _prev;
528   size_t _count;
529   size_t _amount;
530     bool excise_with_release(typename List::NodePtr node);
531  public:
532   typedef typename List::Node Node;
533   ScavengingReleaseOp(Mspace* mspace, List& list) :
534     _mspace(mspace), _list(list), _prev(NULL), _count(0), _amount(0) {}
535   bool process(typename List::NodePtr node);
536   size_t processed() const { return _count; }
537   size_t amount() const { return _amount; }
538 };
539 
540 template <typename Mspace, typename List>
541 inline bool ScavengingReleaseOp<Mspace, List>::process(typename List::NodePtr node) {
542   assert(node != NULL, "invariant");
543   assert(!node->transient(), "invariant");
544   if (node->retired()) {
545     return excise_with_release(node);
546   }
547   _prev = node;
548   return true;
549 }
550 
551 template <typename Mspace, typename List>
552 inline bool ScavengingReleaseOp<Mspace, List>::excise_with_release(typename List::NodePtr node) {
553   assert(node != NULL, "invariant");
554   assert(node->retired(), "invariant");
555   _prev = _list.excise(_prev, node);
556   if (node->transient()) {
557     _mspace->deallocate(node);
558     return true;
559   }
560   assert(node->identity() != NULL, "invariant");
561   assert(node->empty(), "invariant");
562   assert(!node->lease(), "invariant");
563   assert(!node->excluded(), "invariant");
564   ++_count;
565   _amount += node->total_size();
566   node->clear_retired();
567   node->release();
568   mspace_release(node, _mspace);
569   return true;
570 }
571 
572 template <typename Functor, typename Mspace, typename FromList>
573 class ReleaseRetiredOp : public StackObj {
574 private:
575   Functor& _functor;
576   Mspace* _mspace;
577   FromList& _list;
578   typename Mspace::NodePtr _prev;
579 public:
580   typedef typename Mspace::Node Node;
581   ReleaseRetiredOp(Functor& functor, Mspace* mspace, FromList& list) :
582     _functor(functor), _mspace(mspace), _list(list), _prev(NULL) {}
583   bool process(Node* node);
584 };
585 
586 template <typename Functor, typename Mspace, typename FromList>
587 inline bool ReleaseRetiredOp<Functor, Mspace, FromList>::process(typename Mspace::Node* node) {
588   assert(node != NULL, "invariant");
589   const bool is_retired = node->retired();
590   const bool result = _functor.process(node);
591   if (is_retired) {
592     assert(node->unflushed_size() == 0, "invariant");
593     _prev = _list.excise(_prev, node);
594     node->reinitialize();
595     assert(node->empty(), "invariant");
596     assert(!node->retired(), "invariant");
597     node->release();
598     mspace_release(node, _mspace);
599   } else {
600     _prev = node;
601   }
602   return result;
603 }
604 
605 template <typename Mspace, typename FromList>
606 class ReinitializeAllReleaseRetiredOp : public StackObj {
607 private:
608   Mspace* _mspace;
609   FromList& _list;
610   typename Mspace::NodePtr _prev;
611  public:
612   typedef typename Mspace::Node Node;
613   ReinitializeAllReleaseRetiredOp(Mspace* mspace, FromList& list) :
614     _mspace(mspace), _list(list), _prev(NULL) {}
615   bool process(Node* node);
616 };
617 
618 template <typename Mspace, typename FromList>
619 inline bool ReinitializeAllReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) {
620   assert(node != NULL, "invariant");
621   // assumes some means of exclusive access to node
622   const bool retired = node->retired();
623   node->reinitialize();
624   assert(node->empty(), "invariant");
625   assert(!node->retired(), "invariant");
626   if (retired) {
627     _prev = _list.excise(_prev, node);
628     node->release();
629     mspace_release(node, _mspace);
630   } else {
631     _prev = node;
632   }
633   return true;
634 }
635 
636 #ifdef ASSERT
637 template <typename Node>
638 inline void assert_migration_state(const Node* old, const Node* new_node, size_t used, size_t requested) {
639   assert(old != NULL, "invariant");
640   assert(new_node != NULL, "invariant");
641   assert(old->pos() >= old->start(), "invariant");
642   assert(old->pos() + used <= old->end(), "invariant");
643   assert(new_node->free_size() >= (used + requested), "invariant");
644 }
645 #endif // ASSERT
646 
647 template <typename Node>
648 inline void migrate_outstanding_writes(const Node* old, Node* new_node, size_t used, size_t requested) {
649   DEBUG_ONLY(assert_migration_state(old, new_node, used, requested);)
650   if (used > 0) {
651     memcpy(new_node->pos(), old->pos(), used);
652   }
653 }
654 
655 #endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP
--- EOF ---