1 /*
  2  * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/parallel/mutableNUMASpace.hpp"
 27 #include "gc/shared/collectedHeap.hpp"
 28 #include "gc/shared/gc_globals.hpp"
 29 #include "gc/shared/spaceDecorator.hpp"
 30 #include "gc/shared/workgroup.hpp"
 31 #include "memory/allocation.inline.hpp"
 32 #include "oops/oop.inline.hpp"
 33 #include "oops/typeArrayOop.hpp"
 34 #include "runtime/atomic.hpp"
 35 #include "runtime/java.hpp"
 36 #include "runtime/os.inline.hpp"
 37 #include "runtime/thread.inline.hpp"
 38 #include "runtime/threadSMR.hpp"
 39 #include "utilities/align.hpp"
 40 
 41 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
 42   _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, mtGC);
 43   _page_size = os::vm_page_size();
 44   _adaptation_cycles = 0;
 45   _samples_count = 0;
 46 
 47 #ifdef LINUX
 48   // Changing the page size can lead to freeing of memory. When using large pages
 49   // and the memory has been both reserved and committed, Linux does not support
 50   // freeing parts of it.
 51     if (UseLargePages && !os::can_commit_large_page_memory()) {
 52       _must_use_large_pages = true;
 53     }
 54 #endif // LINUX
 55 
 56   update_layout(true);
 57 }
 58 
 59 MutableNUMASpace::~MutableNUMASpace() {
 60   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 61     delete lgrp_spaces()->at(i);
 62   }
 63   delete lgrp_spaces();
 64 }
 65 
 66 #ifndef PRODUCT
 67 void MutableNUMASpace::mangle_unused_area() {
 68   // This method should do nothing.
 69   // It can be called on a numa space during a full compaction.
 70 }
 71 void MutableNUMASpace::mangle_unused_area_complete() {
 72   // This method should do nothing.
 73   // It can be called on a numa space during a full compaction.
 74 }
 75 void MutableNUMASpace::mangle_region(MemRegion mr) {
 76   // This method should do nothing because numa spaces are not mangled.
 77 }
 78 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
 79   assert(false, "Do not mangle MutableNUMASpace's");
 80 }
 81 void MutableNUMASpace::set_top_for_allocations() {
 82   // This method should do nothing.
 83 }
 84 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
 85   // This method should do nothing.
 86 }
 87 void MutableNUMASpace::check_mangled_unused_area_complete() {
 88   // This method should do nothing.
 89 }
 90 #endif  // NOT_PRODUCT
 91 
 92 // There may be unallocated holes in the middle chunks
 93 // that should be filled with dead objects to ensure parsability.
 94 void MutableNUMASpace::ensure_parsability() {
 95   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 96     LGRPSpace *ls = lgrp_spaces()->at(i);
 97     MutableSpace *s = ls->space();
 98     if (s->top() < top()) { // For all spaces preceding the one containing top()
 99       if (s->free_in_words() > 0) {
100         HeapWord* cur_top = s->top();
101         size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
102         while (words_left_to_fill > 0) {
103           size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
104           assert(words_to_fill >= CollectedHeap::min_fill_size(),
105                  "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
106                  words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
107           CollectedHeap::fill_with_object(cur_top, words_to_fill);
108           if (!os::numa_has_static_binding()) {
109             size_t touched_words = words_to_fill;
110 #ifndef ASSERT
111             if (!ZapUnusedHeapArea) {
112               touched_words = MIN2((size_t)align_object_size(align_up(typeArrayOopDesc::base_offset_in_bytes(T_INT), HeapWordSize) / HeapWordSize),
113                 touched_words);
114             }
115 #endif
116             MemRegion invalid;
117             HeapWord *crossing_start = align_up(cur_top, os::vm_page_size());
118             HeapWord *crossing_end = align_down(cur_top + touched_words, os::vm_page_size());
119             if (crossing_start != crossing_end) {
120               // If object header crossed a small page boundary we mark the area
121               // as invalid rounding it to a page_size().
122               HeapWord *start = MAX2(align_down(cur_top, page_size()), s->bottom());
123               HeapWord *end = MIN2(align_up(cur_top + touched_words, page_size()), s->end());
124               invalid = MemRegion(start, end);
125             }
126 
127             ls->add_invalid_region(invalid);
128           }
129           cur_top += words_to_fill;
130           words_left_to_fill -= words_to_fill;
131         }
132       }
133     } else {
134       if (!os::numa_has_static_binding()) {
135 #ifdef ASSERT
136         MemRegion invalid(s->top(), s->end());
137         ls->add_invalid_region(invalid);
138 #else
139         if (ZapUnusedHeapArea) {
140           MemRegion invalid(s->top(), s->end());
141           ls->add_invalid_region(invalid);
142         } else {
143           return;
144         }
145 #endif
146       } else {
147           return;
148       }
149     }
150   }
151 }
152 
153 size_t MutableNUMASpace::used_in_words() const {
154   size_t s = 0;
155   for (int i = 0; i < lgrp_spaces()->length(); i++) {
156     s += lgrp_spaces()->at(i)->space()->used_in_words();
157   }
158   return s;
159 }
160 
161 size_t MutableNUMASpace::free_in_words() const {
162   size_t s = 0;
163   for (int i = 0; i < lgrp_spaces()->length(); i++) {
164     s += lgrp_spaces()->at(i)->space()->free_in_words();
165   }
166   return s;
167 }
168 
169 
170 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
171   guarantee(thr != NULL, "No thread");
172   int lgrp_id = thr->lgrp_id();
173   if (lgrp_id == -1) {
174     // This case can occur after the topology of the system has
175     // changed. Thread can change their location, the new home
176     // group will be determined during the first allocation
177     // attempt. For now we can safely assume that all spaces
178     // have equal size because the whole space will be reinitialized.
179     if (lgrp_spaces()->length() > 0) {
180       return capacity_in_bytes() / lgrp_spaces()->length();
181     } else {
182       assert(false, "There should be at least one locality group");
183       return 0;
184     }
185   }
186   // That's the normal case, where we know the locality group of the thread.
187   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
188   if (i == -1) {
189     return 0;
190   }
191   return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
192 }
193 
194 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
195   // Please see the comments for tlab_capacity().
196   guarantee(thr != NULL, "No thread");
197   int lgrp_id = thr->lgrp_id();
198   if (lgrp_id == -1) {
199     if (lgrp_spaces()->length() > 0) {
200       return (used_in_bytes()) / lgrp_spaces()->length();
201     } else {
202       assert(false, "There should be at least one locality group");
203       return 0;
204     }
205   }
206   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
207   if (i == -1) {
208     return 0;
209   }
210   return lgrp_spaces()->at(i)->space()->used_in_bytes();
211 }
212 
213 
214 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
215   // Please see the comments for tlab_capacity().
216   guarantee(thr != NULL, "No thread");
217   int lgrp_id = thr->lgrp_id();
218   if (lgrp_id == -1) {
219     if (lgrp_spaces()->length() > 0) {
220       return free_in_bytes() / lgrp_spaces()->length();
221     } else {
222       assert(false, "There should be at least one locality group");
223       return 0;
224     }
225   }
226   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
227   if (i == -1) {
228     return 0;
229   }
230   return lgrp_spaces()->at(i)->space()->free_in_bytes();
231 }
232 
233 
234 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
235   guarantee(thr != NULL, "No thread");
236   int lgrp_id = thr->lgrp_id();
237   if (lgrp_id == -1) {
238     if (lgrp_spaces()->length() > 0) {
239       return capacity_in_words() / lgrp_spaces()->length();
240     } else {
241       assert(false, "There should be at least one locality group");
242       return 0;
243     }
244   }
245   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
246   if (i == -1) {
247     return 0;
248   }
249   return lgrp_spaces()->at(i)->space()->capacity_in_words();
250 }
251 
252 // Check if the NUMA topology has changed. Add and remove spaces if needed.
253 // The update can be forced by setting the force parameter equal to true.
254 bool MutableNUMASpace::update_layout(bool force) {
255   // Check if the topology had changed.
256   bool changed = os::numa_topology_changed();
257   if (force || changed) {
258     // Compute lgrp intersection. Add/remove spaces.
259     int lgrp_limit = (int)os::numa_get_groups_num();
260     int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
261     int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
262     assert(lgrp_num > 0, "There should be at least one locality group");
263     // Add new spaces for the new nodes
264     for (int i = 0; i < lgrp_num; i++) {
265       bool found = false;
266       for (int j = 0; j < lgrp_spaces()->length(); j++) {
267         if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
268           found = true;
269           break;
270         }
271       }
272       if (!found) {
273         lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
274       }
275     }
276 
277     // Remove spaces for the removed nodes.
278     for (int i = 0; i < lgrp_spaces()->length();) {
279       bool found = false;
280       for (int j = 0; j < lgrp_num; j++) {
281         if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
282           found = true;
283           break;
284         }
285       }
286       if (!found) {
287         delete lgrp_spaces()->at(i);
288         lgrp_spaces()->remove_at(i);
289       } else {
290         i++;
291       }
292     }
293 
294     FREE_C_HEAP_ARRAY(int, lgrp_ids);
295 
296     if (changed) {
297       for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
298         thread->set_lgrp_id(-1);
299       }
300     }
301     return true;
302   }
303   return false;
304 }
305 
306 // Bias region towards the first-touching lgrp. Set the right page sizes.
307 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
308   HeapWord *start = align_up(mr.start(), page_size());
309   HeapWord *end = align_down(mr.end(), page_size());
310   if (end > start) {
311     MemRegion aligned_region(start, end);
312     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
313            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
314     assert(region().contains(aligned_region), "Sanity");
315     // First we tell the OS which page size we want in the given range. The underlying
316     // large page can be broken down if we require small pages.
317     os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
318     // Then we uncommit the pages in the range.
319     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
320     // And make them local/first-touch biased.
321     os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
322   }
323 }
324 
325 // Free all pages in the region.
326 void MutableNUMASpace::free_region(MemRegion mr) {
327   HeapWord *start = align_up(mr.start(), page_size());
328   HeapWord *end = align_down(mr.end(), page_size());
329   if (end > start) {
330     MemRegion aligned_region(start, end);
331     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
332            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
333     assert(region().contains(aligned_region), "Sanity");
334     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
335   }
336 }
337 
338 // Update space layout. Perform adaptation.
339 void MutableNUMASpace::update() {
340   if (update_layout(false)) {
341     // If the topology has changed, make all chunks zero-sized.
342     // And clear the alloc-rate statistics.
343     // In future we may want to handle this more gracefully in order
344     // to avoid the reallocation of the pages as much as possible.
345     for (int i = 0; i < lgrp_spaces()->length(); i++) {
346       LGRPSpace *ls = lgrp_spaces()->at(i);
347       MutableSpace *s = ls->space();
348       s->set_end(s->bottom());
349       s->set_top(s->bottom());
350       ls->clear_alloc_rate();
351     }
352     // A NUMA space is never mangled
353     initialize(region(),
354                SpaceDecorator::Clear,
355                SpaceDecorator::DontMangle);
356   } else {
357     bool should_initialize = false;
358     if (!os::numa_has_static_binding()) {
359       for (int i = 0; i < lgrp_spaces()->length(); i++) {
360         if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
361           should_initialize = true;
362           break;
363         }
364       }
365     }
366 
367     if (should_initialize ||
368         (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
369       // A NUMA space is never mangled
370       initialize(region(),
371                  SpaceDecorator::Clear,
372                  SpaceDecorator::DontMangle);
373     }
374   }
375 
376   if (NUMAStats) {
377     for (int i = 0; i < lgrp_spaces()->length(); i++) {
378       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
379     }
380   }
381 
382   scan_pages(NUMAPageScanRate);
383 }
384 
385 // Scan pages. Free pages that have smaller size or wrong placement.
386 void MutableNUMASpace::scan_pages(size_t page_count)
387 {
388   size_t pages_per_chunk = page_count / lgrp_spaces()->length();
389   if (pages_per_chunk > 0) {
390     for (int i = 0; i < lgrp_spaces()->length(); i++) {
391       LGRPSpace *ls = lgrp_spaces()->at(i);
392       ls->scan_pages(page_size(), pages_per_chunk);
393     }
394   }
395 }
396 
397 // Accumulate statistics about the allocation rate of each lgrp.
398 void MutableNUMASpace::accumulate_statistics() {
399   if (UseAdaptiveNUMAChunkSizing) {
400     for (int i = 0; i < lgrp_spaces()->length(); i++) {
401       lgrp_spaces()->at(i)->sample();
402     }
403     increment_samples_count();
404   }
405 
406   if (NUMAStats) {
407     for (int i = 0; i < lgrp_spaces()->length(); i++) {
408       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
409     }
410   }
411 }
412 
413 // Get the current size of a chunk.
414 // This function computes the size of the chunk based on the
415 // difference between chunk ends. This allows it to work correctly in
416 // case the whole space is resized and during the process of adaptive
417 // chunk resizing.
418 size_t MutableNUMASpace::current_chunk_size(int i) {
419   HeapWord *cur_end, *prev_end;
420   if (i == 0) {
421     prev_end = bottom();
422   } else {
423     prev_end = lgrp_spaces()->at(i - 1)->space()->end();
424   }
425   if (i == lgrp_spaces()->length() - 1) {
426     cur_end = end();
427   } else {
428     cur_end = lgrp_spaces()->at(i)->space()->end();
429   }
430   if (cur_end > prev_end) {
431     return pointer_delta(cur_end, prev_end, sizeof(char));
432   }
433   return 0;
434 }
435 
436 // Return the default chunk size by equally diving the space.
437 // page_size() aligned.
438 size_t MutableNUMASpace::default_chunk_size() {
439   return base_space_size() / lgrp_spaces()->length() * page_size();
440 }
441 
442 // Produce a new chunk size. page_size() aligned.
443 // This function is expected to be called on sequence of i's from 0 to
444 // lgrp_spaces()->length().
445 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
446   size_t pages_available = base_space_size();
447   for (int j = 0; j < i; j++) {
448     pages_available -= align_down(current_chunk_size(j), page_size()) / page_size();
449   }
450   pages_available -= lgrp_spaces()->length() - i - 1;
451   assert(pages_available > 0, "No pages left");
452   float alloc_rate = 0;
453   for (int j = i; j < lgrp_spaces()->length(); j++) {
454     alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
455   }
456   size_t chunk_size = 0;
457   if (alloc_rate > 0) {
458     LGRPSpace *ls = lgrp_spaces()->at(i);
459     chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
460   }
461   chunk_size = MAX2(chunk_size, page_size());
462 
463   if (limit > 0) {
464     limit = align_down(limit, page_size());
465     if (chunk_size > current_chunk_size(i)) {
466       size_t upper_bound = pages_available * page_size();
467       if (upper_bound > limit &&
468           current_chunk_size(i) < upper_bound - limit) {
469         // The resulting upper bound should not exceed the available
470         // amount of memory (pages_available * page_size()).
471         upper_bound = current_chunk_size(i) + limit;
472       }
473       chunk_size = MIN2(chunk_size, upper_bound);
474     } else {
475       size_t lower_bound = page_size();
476       if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
477         lower_bound = current_chunk_size(i) - limit;
478       }
479       chunk_size = MAX2(chunk_size, lower_bound);
480     }
481   }
482   assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
483   return chunk_size;
484 }
485 
486 
487 // Return the bottom_region and the top_region. Align them to page_size() boundary.
488 // |------------------new_region---------------------------------|
489 // |----bottom_region--|---intersection---|------top_region------|
490 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
491                                     MemRegion* bottom_region, MemRegion *top_region) {
492   // Is there bottom?
493   if (new_region.start() < intersection.start()) { // Yes
494     // Try to coalesce small pages into a large one.
495     if (UseLargePages && page_size() >= alignment()) {
496       HeapWord* p = align_up(intersection.start(), alignment());
497       if (new_region.contains(p)
498           && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
499         if (intersection.contains(p)) {
500           intersection = MemRegion(p, intersection.end());
501         } else {
502           intersection = MemRegion(p, p);
503         }
504       }
505     }
506     *bottom_region = MemRegion(new_region.start(), intersection.start());
507   } else {
508     *bottom_region = MemRegion();
509   }
510 
511   // Is there top?
512   if (intersection.end() < new_region.end()) { // Yes
513     // Try to coalesce small pages into a large one.
514     if (UseLargePages && page_size() >= alignment()) {
515       HeapWord* p = align_down(intersection.end(), alignment());
516       if (new_region.contains(p)
517           && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
518         if (intersection.contains(p)) {
519           intersection = MemRegion(intersection.start(), p);
520         } else {
521           intersection = MemRegion(p, p);
522         }
523       }
524     }
525     *top_region = MemRegion(intersection.end(), new_region.end());
526   } else {
527     *top_region = MemRegion();
528   }
529 }
530 
531 // Try to merge the invalid region with the bottom or top region by decreasing
532 // the intersection area. Return the invalid_region aligned to the page_size()
533 // boundary if it's inside the intersection. Return non-empty invalid_region
534 // if it lies inside the intersection (also page-aligned).
535 // |------------------new_region---------------------------------|
536 // |----------------|-------invalid---|--------------------------|
537 // |----bottom_region--|---intersection---|------top_region------|
538 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
539                                      MemRegion *invalid_region) {
540   if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
541     *intersection = MemRegion(invalid_region->end(), intersection->end());
542     *invalid_region = MemRegion();
543   } else
544     if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
545       *intersection = MemRegion(intersection->start(), invalid_region->start());
546       *invalid_region = MemRegion();
547     } else
548       if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
549         *intersection = MemRegion(new_region.start(), new_region.start());
550         *invalid_region = MemRegion();
551       } else
552         if (intersection->contains(invalid_region)) {
553             // That's the only case we have to make an additional bias_region() call.
554             HeapWord* start = invalid_region->start();
555             HeapWord* end = invalid_region->end();
556             if (UseLargePages && page_size() >= alignment()) {
557               HeapWord *p = align_down(start, alignment());
558               if (new_region.contains(p)) {
559                 start = p;
560               }
561               p = align_up(end, alignment());
562               if (new_region.contains(end)) {
563                 end = p;
564               }
565             }
566             if (intersection->start() > start) {
567               *intersection = MemRegion(start, intersection->end());
568             }
569             if (intersection->end() < end) {
570               *intersection = MemRegion(intersection->start(), end);
571             }
572             *invalid_region = MemRegion(start, end);
573         }
574 }
575 
576 void MutableNUMASpace::initialize(MemRegion mr,
577                                   bool clear_space,
578                                   bool mangle_space,
579                                   bool setup_pages,
580                                   WorkGang* pretouch_gang) {
581   assert(clear_space, "Reallocation will destroy data!");
582   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
583 
584   MemRegion old_region = region(), new_region;
585   set_bottom(mr.start());
586   set_end(mr.end());
587   // Must always clear the space
588   clear(SpaceDecorator::DontMangle);
589 
590   // Compute chunk sizes
591   size_t prev_page_size = page_size();
592   set_page_size(UseLargePages ? alignment() : os::vm_page_size());
593   HeapWord* rounded_bottom = align_up(bottom(), page_size());
594   HeapWord* rounded_end = align_down(end(), page_size());
595   size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
596 
597   // Try small pages if the chunk size is too small
598   if (base_space_size_pages / lgrp_spaces()->length() == 0
599       && page_size() > (size_t)os::vm_page_size()) {
600     // Changing the page size below can lead to freeing of memory. So we fail initialization.
601     if (_must_use_large_pages) {
602       vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
603     }
604     set_page_size(os::vm_page_size());
605     rounded_bottom = align_up(bottom(), page_size());
606     rounded_end = align_down(end(), page_size());
607     base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
608   }
609   guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
610   set_base_space_size(base_space_size_pages);
611 
612   // Handle space resize
613   MemRegion top_region, bottom_region;
614   if (!old_region.equals(region())) {
615     new_region = MemRegion(rounded_bottom, rounded_end);
616     MemRegion intersection = new_region.intersection(old_region);
617     if (intersection.start() == NULL ||
618         intersection.end() == NULL   ||
619         prev_page_size > page_size()) { // If the page size got smaller we have to change
620                                         // the page size preference for the whole space.
621       intersection = MemRegion(new_region.start(), new_region.start());
622     }
623     select_tails(new_region, intersection, &bottom_region, &top_region);
624     bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
625     bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
626   }
627 
628   // Check if the space layout has changed significantly?
629   // This happens when the space has been resized so that either head or tail
630   // chunk became less than a page.
631   bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
632                       current_chunk_size(0) > page_size() &&
633                       current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
634 
635 
636   for (int i = 0; i < lgrp_spaces()->length(); i++) {
637     LGRPSpace *ls = lgrp_spaces()->at(i);
638     MutableSpace *s = ls->space();
639     old_region = s->region();
640 
641     size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
642     if (i < lgrp_spaces()->length() - 1) {
643       if (!UseAdaptiveNUMAChunkSizing                                ||
644           (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
645            samples_count() < AdaptiveSizePolicyReadyThreshold) {
646         // No adaptation. Divide the space equally.
647         chunk_byte_size = default_chunk_size();
648       } else
649         if (!layout_valid || NUMASpaceResizeRate == 0) {
650           // Fast adaptation. If no space resize rate is set, resize
651           // the chunks instantly.
652           chunk_byte_size = adaptive_chunk_size(i, 0);
653         } else {
654           // Slow adaptation. Resize the chunks moving no more than
655           // NUMASpaceResizeRate bytes per collection.
656           size_t limit = NUMASpaceResizeRate /
657                          (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
658           chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
659         }
660 
661       assert(chunk_byte_size >= page_size(), "Chunk size too small");
662       assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
663     }
664 
665     if (i == 0) { // Bottom chunk
666       if (i != lgrp_spaces()->length() - 1) {
667         new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
668       } else {
669         new_region = MemRegion(bottom(), end());
670       }
671     } else
672       if (i < lgrp_spaces()->length() - 1) { // Middle chunks
673         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
674         new_region = MemRegion(ps->end(),
675                                ps->end() + (chunk_byte_size >> LogHeapWordSize));
676       } else { // Top chunk
677         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
678         new_region = MemRegion(ps->end(), end());
679       }
680     guarantee(region().contains(new_region), "Region invariant");
681 
682 
683     // The general case:
684     // |---------------------|--invalid---|--------------------------|
685     // |------------------new_region---------------------------------|
686     // |----bottom_region--|---intersection---|------top_region------|
687     //                     |----old_region----|
688     // The intersection part has all pages in place we don't need to migrate them.
689     // Pages for the top and bottom part should be freed and then reallocated.
690 
691     MemRegion intersection = old_region.intersection(new_region);
692 
693     if (intersection.start() == NULL || intersection.end() == NULL) {
694       intersection = MemRegion(new_region.start(), new_region.start());
695     }
696 
697     if (!os::numa_has_static_binding()) {
698       MemRegion invalid_region = ls->invalid_region().intersection(new_region);
699       // Invalid region is a range of memory that could've possibly
700       // been allocated on the other node. That's relevant only on Solaris where
701       // there is no static memory binding.
702       if (!invalid_region.is_empty()) {
703         merge_regions(new_region, &intersection, &invalid_region);
704         free_region(invalid_region);
705         ls->set_invalid_region(MemRegion());
706       }
707     }
708 
709     select_tails(new_region, intersection, &bottom_region, &top_region);
710 
711     if (!os::numa_has_static_binding()) {
712       // If that's a system with the first-touch policy then it's enough
713       // to free the pages.
714       free_region(bottom_region);
715       free_region(top_region);
716     } else {
717       // In a system with static binding we have to change the bias whenever
718       // we reshape the heap.
719       bias_region(bottom_region, ls->lgrp_id());
720       bias_region(top_region, ls->lgrp_id());
721     }
722 
723     // Clear space (set top = bottom) but never mangle.
724     s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
725 
726     set_adaptation_cycles(samples_count());
727   }
728 }
729 
730 // Set the top of the whole space.
731 // Mark the the holes in chunks below the top() as invalid.
732 void MutableNUMASpace::set_top(HeapWord* value) {
733   bool found_top = false;
734   for (int i = 0; i < lgrp_spaces()->length();) {
735     LGRPSpace *ls = lgrp_spaces()->at(i);
736     MutableSpace *s = ls->space();
737     HeapWord *top = MAX2(align_down(s->top(), page_size()), s->bottom());
738 
739     if (s->contains(value)) {
740       // Check if setting the chunk's top to a given value would create a hole less than
741       // a minimal object; assuming that's not the last chunk in which case we don't care.
742       if (i < lgrp_spaces()->length() - 1) {
743         size_t remainder = pointer_delta(s->end(), value);
744         const size_t min_fill_size = CollectedHeap::min_fill_size();
745         if (remainder < min_fill_size && remainder > 0) {
746           // Add a minimum size filler object; it will cross the chunk boundary.
747           CollectedHeap::fill_with_object(value, min_fill_size);
748           value += min_fill_size;
749           assert(!s->contains(value), "Should be in the next chunk");
750           // Restart the loop from the same chunk, since the value has moved
751           // to the next one.
752           continue;
753         }
754       }
755 
756       if (!os::numa_has_static_binding() && top < value && top < s->end()) {
757         ls->add_invalid_region(MemRegion(top, value));
758       }
759       s->set_top(value);
760       found_top = true;
761     } else {
762         if (found_top) {
763             s->set_top(s->bottom());
764         } else {
765           if (!os::numa_has_static_binding() && top < s->end()) {
766             ls->add_invalid_region(MemRegion(top, s->end()));
767           }
768           s->set_top(s->end());
769         }
770     }
771     i++;
772   }
773   MutableSpace::set_top(value);
774 }
775 
776 void MutableNUMASpace::clear(bool mangle_space) {
777   MutableSpace::set_top(bottom());
778   for (int i = 0; i < lgrp_spaces()->length(); i++) {
779     // Never mangle NUMA spaces because the mangling will
780     // bind the memory to a possibly unwanted lgroup.
781     lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
782   }
783 }
784 
785 /*
786    Linux supports static memory binding, therefore the most part of the
787    logic dealing with the possible invalid page allocation is effectively
788    disabled. Besides there is no notion of the home node in Linux. A
789    thread is allowed to migrate freely. Although the scheduler is rather
790    reluctant to move threads between the nodes. We check for the current
791    node every allocation. And with a high probability a thread stays on
792    the same node for some time allowing local access to recently allocated
793    objects.
794  */
795 
796 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
797   Thread* thr = Thread::current();
798   int lgrp_id = thr->lgrp_id();
799   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
800     lgrp_id = os::numa_get_group_id();
801     thr->set_lgrp_id(lgrp_id);
802   }
803 
804   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
805   // It is possible that a new CPU has been hotplugged and
806   // we haven't reshaped the space accordingly.
807   if (i == -1) {
808     i = os::random() % lgrp_spaces()->length();
809   }
810   LGRPSpace *ls = lgrp_spaces()->at(i);
811   MutableSpace *s = ls->space();
812   HeapWord *p = s->cas_allocate(size);
813   if (p != NULL) {
814     size_t remainder = pointer_delta(s->end(), p + size);
815     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
816       if (s->cas_deallocate(p, size)) {
817         // We were the last to allocate and created a fragment less than
818         // a minimal object.
819         p = NULL;
820       } else {
821         guarantee(false, "Deallocation should always succeed");
822       }
823     }
824   }
825   if (p != NULL) {
826     HeapWord* cur_top, *cur_chunk_top = p + size;
827     while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
828       if (Atomic::cmpxchg(top_addr(), cur_top, cur_chunk_top) == cur_top) {
829         break;
830       }
831     }
832   }
833 
834   // Make the page allocation happen here if there is no static binding.
835   if (p != NULL && !os::numa_has_static_binding() ) {
836     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
837       *(int*)i = 0;
838     }
839   }
840   if (p == NULL) {
841     ls->set_allocation_failed();
842   }
843   return p;
844 }
845 
846 void MutableNUMASpace::print_short_on(outputStream* st) const {
847   MutableSpace::print_short_on(st);
848   st->print(" (");
849   for (int i = 0; i < lgrp_spaces()->length(); i++) {
850     st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
851     lgrp_spaces()->at(i)->space()->print_short_on(st);
852     if (i < lgrp_spaces()->length() - 1) {
853       st->print(", ");
854     }
855   }
856   st->print(")");
857 }
858 
859 void MutableNUMASpace::print_on(outputStream* st) const {
860   MutableSpace::print_on(st);
861   for (int i = 0; i < lgrp_spaces()->length(); i++) {
862     LGRPSpace *ls = lgrp_spaces()->at(i);
863     st->print("    lgrp %d", ls->lgrp_id());
864     ls->space()->print_on(st);
865     if (NUMAStats) {
866       for (int i = 0; i < lgrp_spaces()->length(); i++) {
867         lgrp_spaces()->at(i)->accumulate_statistics(page_size());
868       }
869       st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
870                 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
871                 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
872                 ls->space_stats()->_local_space / K,
873                 ls->space_stats()->_remote_space / K,
874                 ls->space_stats()->_unbiased_space / K,
875                 ls->space_stats()->_uncommited_space / K,
876                 ls->space_stats()->_large_pages,
877                 ls->space_stats()->_small_pages);
878     }
879   }
880 }
881 
882 void MutableNUMASpace::verify() {
883   // This can be called after setting an arbitrary value to the space's top,
884   // so an object can cross the chunk boundary. We ensure the parsability
885   // of the space and just walk the objects in linear fashion.
886   ensure_parsability();
887   MutableSpace::verify();
888 }
889 
890 // Scan pages and gather stats about page placement and size.
891 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
892   clear_space_stats();
893   char *start = (char*)align_up(space()->bottom(), page_size);
894   char* end = (char*)align_down(space()->end(), page_size);
895   if (start < end) {
896     for (char *p = start; p < end;) {
897       os::page_info info;
898       if (os::get_page_info(p, &info)) {
899         if (info.size > 0) {
900           if (info.size > (size_t)os::vm_page_size()) {
901             space_stats()->_large_pages++;
902           } else {
903             space_stats()->_small_pages++;
904           }
905           if (info.lgrp_id == lgrp_id()) {
906             space_stats()->_local_space += info.size;
907           } else {
908             space_stats()->_remote_space += info.size;
909           }
910           p += info.size;
911         } else {
912           p += os::vm_page_size();
913           space_stats()->_uncommited_space += os::vm_page_size();
914         }
915       } else {
916         return;
917       }
918     }
919   }
920   space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
921                                    pointer_delta(space()->end(), end, sizeof(char));
922 
923 }
924 
925 // Scan page_count pages and verify if they have the right size and right placement.
926 // If invalid pages are found they are freed in hope that subsequent reallocation
927 // will be more successful.
928 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
929 {
930   char* range_start = (char*)align_up(space()->bottom(), page_size);
931   char* range_end = (char*)align_down(space()->end(), page_size);
932 
933   if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
934     set_last_page_scanned(range_start);
935   }
936 
937   char *scan_start = last_page_scanned();
938   char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
939 
940   os::page_info page_expected, page_found;
941   page_expected.size = page_size;
942   page_expected.lgrp_id = lgrp_id();
943 
944   char *s = scan_start;
945   while (s < scan_end) {
946     char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
947     if (e == NULL) {
948       break;
949     }
950     if (e != scan_end) {
951       assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
952 
953       if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
954           && page_expected.size != 0) {
955         os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
956       }
957       page_expected = page_found;
958     }
959     s = e;
960   }
961 
962   set_last_page_scanned(scan_end);
963 }