11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "cds/aotMappedHeapWriter.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "gc/shared/classUnloadingContext.hpp"
31 #include "gc/shared/fullGCForwarding.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "gc/shared/gcArguments.hpp"
34 #include "gc/shared/gcTimer.hpp"
35 #include "gc/shared/gcTraceTime.inline.hpp"
36 #include "gc/shared/locationPrinter.inline.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "gc/shared/plab.hpp"
39 #include "gc/shared/tlab_globals.hpp"
40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
42 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
43 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
44 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
45 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
46 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
47 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
48 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
49 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
50 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
51 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
1297 Handshake::execute(&composite);
1298 }
1299 }
1300
1301 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1302 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1303
1304 ShenandoahHeapRegion* r = heap_region_containing(p);
1305 assert(!r->is_humongous(), "never evacuate humongous objects");
1306
1307 ShenandoahAffiliation target_gen = r->affiliation();
1308 return try_evacuate_object(p, thread, r, target_gen);
1309 }
1310
1311 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1312 ShenandoahAffiliation target_gen) {
1313 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1314 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1315 bool alloc_from_lab = true;
1316 HeapWord* copy = nullptr;
1317 size_t size = ShenandoahForwarding::size(p);
1318
1319 #ifdef ASSERT
1320 if (ShenandoahOOMDuringEvacALot &&
1321 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1322 copy = nullptr;
1323 } else {
1324 #endif
1325 if (UseTLAB) {
1326 copy = allocate_from_gclab(thread, size);
1327 }
1328 if (copy == nullptr) {
1329 // If we failed to allocate in LAB, we'll try a shared allocation.
1330 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1331 copy = allocate_memory(req);
1332 alloc_from_lab = false;
1333 }
1334 #ifdef ASSERT
1335 }
1336 #endif
1337
1344 // (they succeeded where we failed) or self-forwarded first.
1345 markWord old_mark = p->mark();
1346 if (old_mark.is_forwarded()) {
1347 return ShenandoahForwarding::get_forwardee(p);
1348 }
1349 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
1350 if (winner == nullptr) {
1351 // We own the self-forwarding. Flag the region so the degen/full GC
1352 // entry drain knows to scan it for self_fwd bits to clear.
1353 from_region->set_has_self_forwards();
1354 return p;
1355 }
1356 return winner;
1357 }
1358
1359 if (ShenandoahEvacTracking) {
1360 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1361 }
1362
1363 // Copy the object:
1364 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1365
1366 oop copy_val = cast_to_oop(copy);
1367
1368 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
1369 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
1370 // relativize derived pointers and set gc_mode before that happens. Skip if the
1371 // copy's mark word is already a forwarding pointer (another thread won the race
1372 // and overwrote the original's header before we copied it).
1373 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
1374 ContinuationGCSupport::relativize_stack_chunk(copy_val);
1375 }
1376
1377 // Try to install the new forwarding pointer.
1378 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1379 if (result == copy_val) {
1380 // Successfully evacuated. Our copy is now the public one!
1381 shenandoah_assert_correct(nullptr, copy_val);
1382 if (ShenandoahEvacTracking) {
1383 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1384 }
1385 return copy_val;
1386 } else {
1387 // Failed to evacuate. We need to deal with the object that is left behind. Since this
|
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "cds/aotMappedHeapWriter.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "gc/shared/classUnloadingContext.hpp"
31 #include "gc/shared/fullGCForwarding.inline.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "gc/shared/gcArguments.hpp"
34 #include "gc/shared/gcTimer.hpp"
35 #include "gc/shared/gcTraceTime.inline.hpp"
36 #include "gc/shared/locationPrinter.inline.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "gc/shared/plab.hpp"
39 #include "gc/shared/tlab_globals.hpp"
40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
42 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
43 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
44 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
45 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
46 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
47 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
48 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
49 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
50 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
51 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
1297 Handshake::execute(&composite);
1298 }
1299 }
1300
1301 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1302 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1303
1304 ShenandoahHeapRegion* r = heap_region_containing(p);
1305 assert(!r->is_humongous(), "never evacuate humongous objects");
1306
1307 ShenandoahAffiliation target_gen = r->affiliation();
1308 return try_evacuate_object(p, thread, r, target_gen);
1309 }
1310
1311 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1312 ShenandoahAffiliation target_gen) {
1313 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1314 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1315 bool alloc_from_lab = true;
1316 HeapWord* copy = nullptr;
1317
1318 markWord mark = p->mark();
1319 if (ShenandoahForwarding::is_forwarded(mark)) {
1320 return ShenandoahForwarding::get_forwardee(p);
1321 }
1322 size_t old_size = ShenandoahForwarding::size(p);
1323 size_t size = p->copy_size(old_size, mark);
1324
1325 #ifdef ASSERT
1326 if (ShenandoahOOMDuringEvacALot &&
1327 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1328 copy = nullptr;
1329 } else {
1330 #endif
1331 if (UseTLAB) {
1332 copy = allocate_from_gclab(thread, size);
1333 }
1334 if (copy == nullptr) {
1335 // If we failed to allocate in LAB, we'll try a shared allocation.
1336 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1337 copy = allocate_memory(req);
1338 alloc_from_lab = false;
1339 }
1340 #ifdef ASSERT
1341 }
1342 #endif
1343
1350 // (they succeeded where we failed) or self-forwarded first.
1351 markWord old_mark = p->mark();
1352 if (old_mark.is_forwarded()) {
1353 return ShenandoahForwarding::get_forwardee(p);
1354 }
1355 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
1356 if (winner == nullptr) {
1357 // We own the self-forwarding. Flag the region so the degen/full GC
1358 // entry drain knows to scan it for self_fwd bits to clear.
1359 from_region->set_has_self_forwards();
1360 return p;
1361 }
1362 return winner;
1363 }
1364
1365 if (ShenandoahEvacTracking) {
1366 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1367 }
1368
1369 // Copy the object:
1370 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
1371 oop copy_val = cast_to_oop(copy);
1372
1373 // Initialize the identity hash on the copy before installing the forwarding
1374 // pointer, using the mark word we captured earlier. We must do this before
1375 // the CAS so that the copy is fully initialized when it becomes visible to
1376 // other threads. Using the captured mark (rather than re-reading the copy's
1377 // mark) avoids races with other threads that may have evacuated p and
1378 // installed a forwarding pointer in the meantime.
1379 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
1380 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
1381 }
1382
1383 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
1384 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
1385 // relativize derived pointers and set gc_mode before that happens. Skip if the
1386 // copy's mark word is already a forwarding pointer (another thread won the race
1387 // and overwrote the original's header before we copied it).
1388 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
1389 ContinuationGCSupport::relativize_stack_chunk(copy_val);
1390 }
1391
1392 // Try to install the new forwarding pointer.
1393 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1394 if (result == copy_val) {
1395 // Successfully evacuated. Our copy is now the public one!
1396 shenandoah_assert_correct(nullptr, copy_val);
1397 if (ShenandoahEvacTracking) {
1398 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1399 }
1400 return copy_val;
1401 } else {
1402 // Failed to evacuate. We need to deal with the object that is left behind. Since this
|