61 #include "gc/shared/oopStorageSetParState.inline.hpp"
62 #include "gc/shared/parallelCleaning.hpp"
63 #include "gc/shared/preservedMarks.inline.hpp"
64 #include "gc/shared/referencePolicy.hpp"
65 #include "gc/shared/referenceProcessor.hpp"
66 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
67 #include "gc/shared/spaceDecorator.hpp"
68 #include "gc/shared/taskTerminator.hpp"
69 #include "gc/shared/weakProcessor.inline.hpp"
70 #include "gc/shared/workerPolicy.hpp"
71 #include "gc/shared/workerThread.hpp"
72 #include "gc/shared/workerUtils.hpp"
73 #include "logging/log.hpp"
74 #include "memory/iterator.inline.hpp"
75 #include "memory/memoryReserver.hpp"
76 #include "memory/metaspaceUtils.hpp"
77 #include "memory/resourceArea.hpp"
78 #include "memory/universe.hpp"
79 #include "nmt/memTracker.hpp"
80 #include "oops/access.inline.hpp"
81 #include "oops/instanceClassLoaderKlass.inline.hpp"
82 #include "oops/instanceKlass.inline.hpp"
83 #include "oops/instanceMirrorKlass.inline.hpp"
84 #include "oops/methodData.hpp"
85 #include "oops/objArrayKlass.inline.hpp"
86 #include "oops/oop.inline.hpp"
87 #include "runtime/handles.inline.hpp"
88 #include "runtime/java.hpp"
89 #include "runtime/safepoint.hpp"
90 #include "runtime/threads.hpp"
91 #include "runtime/vmThread.hpp"
92 #include "services/memoryService.hpp"
93 #include "utilities/align.hpp"
94 #include "utilities/debug.hpp"
95 #include "utilities/events.hpp"
96 #include "utilities/formatBuffer.hpp"
97 #include "utilities/macros.hpp"
98 #include "utilities/stack.inline.hpp"
99 #if INCLUDE_JVMCI
100 #include "jvmci/jvmci.hpp"
101 #endif
102
103 #include <math.h>
104
105 // All sizes are in HeapWords.
106 const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words
1428
1429 // Split [start, end) evenly for a number of workers and return the
1430 // range for worker_id.
1431 static void split_regions_for_worker(size_t start, size_t end,
1432 uint worker_id, uint num_workers,
1433 size_t* worker_start, size_t* worker_end) {
1434 assert(start < end, "precondition");
1435 assert(num_workers > 0, "precondition");
1436 assert(worker_id < num_workers, "precondition");
1437
1438 size_t num_regions = end - start;
1439 size_t num_regions_per_worker = num_regions / num_workers;
1440 size_t remainder = num_regions % num_workers;
1441 // The first few workers will get one extra.
1442 *worker_start = start + worker_id * num_regions_per_worker
1443 + MIN2(checked_cast<size_t>(worker_id), remainder);
1444 *worker_end = *worker_start + num_regions_per_worker
1445 + (worker_id < remainder ? 1 : 0);
1446 }
1447
1448 void PSParallelCompact::forward_to_new_addr() {
1449 GCTraceTime(Info, gc, phases) tm("Forward", &_gc_timer);
1450 uint nworkers = ParallelScavengeHeap::heap()->workers().active_workers();
1451
1452 struct ForwardTask final : public WorkerTask {
1453 uint _num_workers;
1454
1455 explicit ForwardTask(uint num_workers) :
1456 WorkerTask("PSForward task"),
1457 _num_workers(num_workers) {}
1458
1459 static void forward_objs_in_range(ParCompactionManager* cm,
1460 HeapWord* start,
1461 HeapWord* end,
1462 HeapWord* destination) {
1463 HeapWord* cur_addr = start;
1464 HeapWord* new_addr = destination;
1465
1466 while (cur_addr < end) {
1467 cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end);
1468 if (cur_addr >= end) {
1469 return;
1470 }
1471 assert(mark_bitmap()->is_marked(cur_addr), "inv");
1472 oop obj = cast_to_oop(cur_addr);
1473 if (new_addr != cur_addr) {
1474 cm->preserved_marks()->push_if_necessary(obj, obj->mark());
1475 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
1476 }
1477 size_t obj_size = obj->size();
1478 new_addr += obj_size;
1479 cur_addr += obj_size;
1480 }
1481 }
1482
1483 void work(uint worker_id) override {
1484 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
1485 for (uint id = old_space_id; id < last_space_id; ++id) {
1486 MutableSpace* sp = PSParallelCompact::space(SpaceId(id));
1487 HeapWord* dense_prefix_addr = dense_prefix(SpaceId(id));
1488 HeapWord* top = sp->top();
1489
1490 if (dense_prefix_addr == top) {
1491 // Empty space
1492 continue;
1493 }
1494
2095
2096 HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) {
2097 ParallelCompactData& sd = summary_data();
2098 assert(sd.is_region_aligned(region_start_addr), "precondition");
2099
2100 // Use per-region partial_obj_size to locate the end of the obj, that extends
2101 // to region_start_addr.
2102 size_t start_region_idx = sd.addr_to_region_idx(region_start_addr);
2103 size_t end_region_idx = sd.region_count();
2104 size_t accumulated_size = 0;
2105 for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) {
2106 size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size();
2107 accumulated_size += cur_partial_obj_size;
2108 if (cur_partial_obj_size != ParallelCompactData::RegionSize) {
2109 break;
2110 }
2111 }
2112 return region_start_addr + accumulated_size;
2113 }
2114
2115 // Use region_idx as the destination region, and evacuate all live objs on its
2116 // source regions to this destination region.
2117 void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx)
2118 {
2119 ParMarkBitMap* const bitmap = mark_bitmap();
2120 ParallelCompactData& sd = summary_data();
2121 RegionData* const region_ptr = sd.region(region_idx);
2122
2123 // Get the source region and related info.
2124 size_t src_region_idx = region_ptr->source_region();
2125 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx));
2126 HeapWord* src_space_top = _space_info[src_space_id].space()->top();
2127 HeapWord* dest_addr = sd.region_to_addr(region_idx);
2128
2129 closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx));
2130
2131 // Adjust src_region_idx to prepare for decrementing destination counts (the
2132 // destination count is not decremented when a region is copied to itself).
2133 if (src_region_idx == region_idx) {
2134 src_region_idx += 1;
2210 HeapWord* cur_addr = closure.source();
2211 HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1),
2212 src_space_top);
2213 // To handle the case where the final obj in source region extends to next region.
2214 HeapWord* final_obj_start = (end_addr == src_space_top)
2215 ? nullptr
2216 : sd.addr_to_region_ptr(end_addr)->partial_obj_addr();
2217 // Apply closure on objs inside [cur_addr, end_addr)
2218 do {
2219 cur_addr = bitmap->find_obj_beg(cur_addr, end_addr);
2220 if (cur_addr == end_addr) {
2221 break;
2222 }
2223 size_t obj_size;
2224 if (final_obj_start == cur_addr) {
2225 obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr);
2226 } else {
2227 // This obj doesn't extend into next region; size() is safe to use.
2228 obj_size = cast_to_oop(cur_addr)->size();
2229 }
2230 closure.do_addr(cur_addr, obj_size);
2231 cur_addr += obj_size;
2232 } while (cur_addr < end_addr && !closure.is_full());
2233
2234 if (closure.is_full()) {
2235 decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source());
2236 closure.complete_region(dest_addr, region_ptr);
2237 return;
2238 }
2239
2240 decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr);
2241
2242 // Move to the next source region, possibly switching spaces as well. All
2243 // args except end_addr may be modified.
2244 src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr);
2245 } while (true);
2246 }
2247
2248 void PSParallelCompact::fill_and_update_region(ParCompactionManager* cm, size_t region_idx)
2249 {
2250 MoveAndUpdateClosure cl(mark_bitmap(), region_idx);
2330 }
2331
2332 void MoveAndUpdateClosure::copy_partial_obj(size_t partial_obj_size)
2333 {
2334 size_t words = MIN2(partial_obj_size, words_remaining());
2335
2336 // This test is necessary; if omitted, the pointer updates to a partial object
2337 // that crosses the dense prefix boundary could be overwritten.
2338 if (source() != copy_destination()) {
2339 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2340 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2341 }
2342 update_state(words);
2343 }
2344
2345 void MoveAndUpdateClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2346 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::NormalRegion, "Region should be finished");
2347 region_ptr->set_completed();
2348 }
2349
2350 void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) {
2351 assert(destination() != nullptr, "sanity");
2352 _source = addr;
2353
2354 // The start_array must be updated even if the object is not moving.
2355 if (_start_array != nullptr) {
2356 _start_array->update_for_block(destination(), destination() + words);
2357 }
2358
2359 // Avoid overflow
2360 words = MIN2(words, words_remaining());
2361 assert(words > 0, "inv");
2362
2363 if (copy_destination() != source()) {
2364 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2365 assert(source() != destination(), "inv");
2366 assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
2367 assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
2368 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2369 cast_to_oop(copy_destination())->init_mark();
2370 }
2371
2372 update_state(words);
2373 }
2374
2375 void MoveAndUpdateShadowClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2376 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::ShadowRegion, "Region should be shadow");
2377 // Record the shadow region index
2378 region_ptr->set_shadow_region(_shadow);
2379 // Mark the shadow region as filled to indicate the data is ready to be
2380 // copied back
2381 region_ptr->mark_filled();
2382 // Try to copy the content of the shadow region back to its corresponding
2383 // heap region if available; the GC thread that decreases the destination
2384 // count to zero will do the copying otherwise (see
2385 // PSParallelCompact::decrement_destination_counts).
2386 if (((region_ptr->available() && region_ptr->claim()) || region_ptr->claimed()) && region_ptr->mark_copied()) {
2387 region_ptr->set_completed();
2388 PSParallelCompact::copy_back(PSParallelCompact::summary_data().region_to_addr(_shadow), dest_addr);
2389 ParCompactionManager::push_shadow_region_mt_safe(_shadow);
|
61 #include "gc/shared/oopStorageSetParState.inline.hpp"
62 #include "gc/shared/parallelCleaning.hpp"
63 #include "gc/shared/preservedMarks.inline.hpp"
64 #include "gc/shared/referencePolicy.hpp"
65 #include "gc/shared/referenceProcessor.hpp"
66 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
67 #include "gc/shared/spaceDecorator.hpp"
68 #include "gc/shared/taskTerminator.hpp"
69 #include "gc/shared/weakProcessor.inline.hpp"
70 #include "gc/shared/workerPolicy.hpp"
71 #include "gc/shared/workerThread.hpp"
72 #include "gc/shared/workerUtils.hpp"
73 #include "logging/log.hpp"
74 #include "memory/iterator.inline.hpp"
75 #include "memory/memoryReserver.hpp"
76 #include "memory/metaspaceUtils.hpp"
77 #include "memory/resourceArea.hpp"
78 #include "memory/universe.hpp"
79 #include "nmt/memTracker.hpp"
80 #include "oops/access.inline.hpp"
81 #include "oops/flatArrayKlass.inline.hpp"
82 #include "oops/instanceClassLoaderKlass.inline.hpp"
83 #include "oops/instanceKlass.inline.hpp"
84 #include "oops/instanceMirrorKlass.inline.hpp"
85 #include "oops/methodData.hpp"
86 #include "oops/objArrayKlass.inline.hpp"
87 #include "oops/oop.inline.hpp"
88 #include "runtime/arguments.hpp"
89 #include "runtime/handles.inline.hpp"
90 #include "runtime/java.hpp"
91 #include "runtime/safepoint.hpp"
92 #include "runtime/threads.hpp"
93 #include "runtime/vmThread.hpp"
94 #include "services/memoryService.hpp"
95 #include "utilities/align.hpp"
96 #include "utilities/debug.hpp"
97 #include "utilities/events.hpp"
98 #include "utilities/formatBuffer.hpp"
99 #include "utilities/macros.hpp"
100 #include "utilities/stack.inline.hpp"
101 #if INCLUDE_JVMCI
102 #include "jvmci/jvmci.hpp"
103 #endif
104
105 #include <math.h>
106
107 // All sizes are in HeapWords.
108 const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words
1430
1431 // Split [start, end) evenly for a number of workers and return the
1432 // range for worker_id.
1433 static void split_regions_for_worker(size_t start, size_t end,
1434 uint worker_id, uint num_workers,
1435 size_t* worker_start, size_t* worker_end) {
1436 assert(start < end, "precondition");
1437 assert(num_workers > 0, "precondition");
1438 assert(worker_id < num_workers, "precondition");
1439
1440 size_t num_regions = end - start;
1441 size_t num_regions_per_worker = num_regions / num_workers;
1442 size_t remainder = num_regions % num_workers;
1443 // The first few workers will get one extra.
1444 *worker_start = start + worker_id * num_regions_per_worker
1445 + MIN2(checked_cast<size_t>(worker_id), remainder);
1446 *worker_end = *worker_start + num_regions_per_worker
1447 + (worker_id < remainder ? 1 : 0);
1448 }
1449
1450 static bool safe_to_read_header(size_t words) {
1451 precond(words > 0);
1452
1453 // Safe to read if we have enough words for the full header, i.e., both
1454 // markWord and Klass pointer.
1455 const bool safe = words >= (size_t)oopDesc::header_size();
1456
1457 // If using Compact Object Headers, the full header is inside the markWord,
1458 // so will always be safe to read
1459 assert(!UseCompactObjectHeaders || safe, "Compact Object Headers should always be safe");
1460
1461 return safe;
1462 }
1463
1464 void PSParallelCompact::forward_to_new_addr() {
1465 GCTraceTime(Info, gc, phases) tm("Forward", &_gc_timer);
1466 uint nworkers = ParallelScavengeHeap::heap()->workers().active_workers();
1467
1468 struct ForwardTask final : public WorkerTask {
1469 uint _num_workers;
1470
1471 explicit ForwardTask(uint num_workers) :
1472 WorkerTask("PSForward task"),
1473 _num_workers(num_workers) {}
1474
1475 static bool should_preserve_mark(oop obj, HeapWord* end_addr) {
1476 size_t remaining_words = pointer_delta(end_addr, cast_from_oop<HeapWord*>(obj));
1477
1478 if (Arguments::is_valhalla_enabled() && !safe_to_read_header(remaining_words)) {
1479 // When using Valhalla, it might be necessary to preserve the Valhalla-
1480 // specific bits in the markWord. If the entire object header is
1481 // copied, the correct markWord (with the appropriate Valhalla bits)
1482 // can be safely read from the Klass. However, if the full header is
1483 // not copied, we cannot safely read the Klass to obtain this information.
1484 // In such cases, we always preserve the markWord to ensure that all
1485 // relevant bits, including Valhalla-specific ones, are retained.
1486 return true;
1487 } else {
1488 return obj->mark().must_be_preserved();
1489 }
1490 }
1491
1492 static void forward_objs_in_range(ParCompactionManager* cm,
1493 HeapWord* start,
1494 HeapWord* end,
1495 HeapWord* destination) {
1496 HeapWord* cur_addr = start;
1497 HeapWord* new_addr = destination;
1498
1499 while (cur_addr < end) {
1500 cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end);
1501 if (cur_addr >= end) {
1502 return;
1503 }
1504 assert(mark_bitmap()->is_marked(cur_addr), "inv");
1505 oop obj = cast_to_oop(cur_addr);
1506
1507 if (new_addr != cur_addr) {
1508 if (should_preserve_mark(obj, end)) {
1509 cm->preserved_marks()->push_always(obj, obj->mark());
1510 }
1511
1512 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
1513 }
1514 size_t obj_size = obj->size();
1515 new_addr += obj_size;
1516 cur_addr += obj_size;
1517 }
1518 }
1519
1520 void work(uint worker_id) override {
1521 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
1522 for (uint id = old_space_id; id < last_space_id; ++id) {
1523 MutableSpace* sp = PSParallelCompact::space(SpaceId(id));
1524 HeapWord* dense_prefix_addr = dense_prefix(SpaceId(id));
1525 HeapWord* top = sp->top();
1526
1527 if (dense_prefix_addr == top) {
1528 // Empty space
1529 continue;
1530 }
1531
2132
2133 HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) {
2134 ParallelCompactData& sd = summary_data();
2135 assert(sd.is_region_aligned(region_start_addr), "precondition");
2136
2137 // Use per-region partial_obj_size to locate the end of the obj, that extends
2138 // to region_start_addr.
2139 size_t start_region_idx = sd.addr_to_region_idx(region_start_addr);
2140 size_t end_region_idx = sd.region_count();
2141 size_t accumulated_size = 0;
2142 for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) {
2143 size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size();
2144 accumulated_size += cur_partial_obj_size;
2145 if (cur_partial_obj_size != ParallelCompactData::RegionSize) {
2146 break;
2147 }
2148 }
2149 return region_start_addr + accumulated_size;
2150 }
2151
2152 static markWord safe_mark_word_prototype(HeapWord* cur_addr, HeapWord* end_addr) {
2153 // If the original markWord contains bits that cannot be reconstructed because
2154 // the header cannot be safely read, a placeholder is used. In this case,
2155 // the correct markWord is preserved before compaction and restored after
2156 // compaction completes.
2157 size_t remaining_words = pointer_delta(end_addr, cur_addr);
2158
2159 if (UseCompactObjectHeaders || (Arguments::is_valhalla_enabled() && safe_to_read_header(remaining_words))) {
2160 return cast_to_oop(cur_addr)->klass()->prototype_header();
2161 } else {
2162 return markWord::prototype();
2163 }
2164 }
2165
2166 // Use region_idx as the destination region, and evacuate all live objs on its
2167 // source regions to this destination region.
2168 void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx)
2169 {
2170 ParMarkBitMap* const bitmap = mark_bitmap();
2171 ParallelCompactData& sd = summary_data();
2172 RegionData* const region_ptr = sd.region(region_idx);
2173
2174 // Get the source region and related info.
2175 size_t src_region_idx = region_ptr->source_region();
2176 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx));
2177 HeapWord* src_space_top = _space_info[src_space_id].space()->top();
2178 HeapWord* dest_addr = sd.region_to_addr(region_idx);
2179
2180 closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx));
2181
2182 // Adjust src_region_idx to prepare for decrementing destination counts (the
2183 // destination count is not decremented when a region is copied to itself).
2184 if (src_region_idx == region_idx) {
2185 src_region_idx += 1;
2261 HeapWord* cur_addr = closure.source();
2262 HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1),
2263 src_space_top);
2264 // To handle the case where the final obj in source region extends to next region.
2265 HeapWord* final_obj_start = (end_addr == src_space_top)
2266 ? nullptr
2267 : sd.addr_to_region_ptr(end_addr)->partial_obj_addr();
2268 // Apply closure on objs inside [cur_addr, end_addr)
2269 do {
2270 cur_addr = bitmap->find_obj_beg(cur_addr, end_addr);
2271 if (cur_addr == end_addr) {
2272 break;
2273 }
2274 size_t obj_size;
2275 if (final_obj_start == cur_addr) {
2276 obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr);
2277 } else {
2278 // This obj doesn't extend into next region; size() is safe to use.
2279 obj_size = cast_to_oop(cur_addr)->size();
2280 }
2281
2282 markWord mark = safe_mark_word_prototype(cur_addr, end_addr);
2283
2284 // Perform the move and update of the object
2285 closure.do_addr(cur_addr, obj_size, mark);
2286
2287 cur_addr += obj_size;
2288 } while (cur_addr < end_addr && !closure.is_full());
2289
2290 if (closure.is_full()) {
2291 decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source());
2292 closure.complete_region(dest_addr, region_ptr);
2293 return;
2294 }
2295
2296 decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr);
2297
2298 // Move to the next source region, possibly switching spaces as well. All
2299 // args except end_addr may be modified.
2300 src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr);
2301 } while (true);
2302 }
2303
2304 void PSParallelCompact::fill_and_update_region(ParCompactionManager* cm, size_t region_idx)
2305 {
2306 MoveAndUpdateClosure cl(mark_bitmap(), region_idx);
2386 }
2387
2388 void MoveAndUpdateClosure::copy_partial_obj(size_t partial_obj_size)
2389 {
2390 size_t words = MIN2(partial_obj_size, words_remaining());
2391
2392 // This test is necessary; if omitted, the pointer updates to a partial object
2393 // that crosses the dense prefix boundary could be overwritten.
2394 if (source() != copy_destination()) {
2395 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2396 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2397 }
2398 update_state(words);
2399 }
2400
2401 void MoveAndUpdateClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2402 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::NormalRegion, "Region should be finished");
2403 region_ptr->set_completed();
2404 }
2405
2406 void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words, markWord mark) {
2407 assert(destination() != nullptr, "sanity");
2408 _source = addr;
2409
2410 // The start_array must be updated even if the object is not moving.
2411 if (_start_array != nullptr) {
2412 _start_array->update_for_block(destination(), destination() + words);
2413 }
2414
2415 // Avoid overflow
2416 words = MIN2(words, words_remaining());
2417 assert(words > 0, "inv");
2418
2419 if (copy_destination() != source()) {
2420 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2421 assert(source() != destination(), "inv");
2422 assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
2423 assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
2424 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2425 cast_to_oop(copy_destination())->set_mark(mark);
2426 }
2427
2428 update_state(words);
2429 }
2430
2431 void MoveAndUpdateShadowClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2432 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::ShadowRegion, "Region should be shadow");
2433 // Record the shadow region index
2434 region_ptr->set_shadow_region(_shadow);
2435 // Mark the shadow region as filled to indicate the data is ready to be
2436 // copied back
2437 region_ptr->mark_filled();
2438 // Try to copy the content of the shadow region back to its corresponding
2439 // heap region if available; the GC thread that decreases the destination
2440 // count to zero will do the copying otherwise (see
2441 // PSParallelCompact::decrement_destination_counts).
2442 if (((region_ptr->available() && region_ptr->claim()) || region_ptr->claimed()) && region_ptr->mark_copied()) {
2443 region_ptr->set_completed();
2444 PSParallelCompact::copy_back(PSParallelCompact::summary_data().region_to_addr(_shadow), dest_addr);
2445 ParCompactionManager::push_shadow_region_mt_safe(_shadow);
|