11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "cds/aotMappedHeapWriter.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "gc/shared/classUnloadingContext.hpp"
31 #include "gc/shared/fullGCForwarding.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "gc/shared/gcArguments.hpp"
34 #include "gc/shared/gcTimer.hpp"
35 #include "gc/shared/gcTraceTime.inline.hpp"
36 #include "gc/shared/locationPrinter.inline.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "gc/shared/plab.hpp"
39 #include "gc/shared/tlab_globals.hpp"
40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
42 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
43 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
44 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
45 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
46 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
47 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
48 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
49 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
50 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
51 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
1300 // This thread went through the OOM during evac protocol. It is safe to return
1301 // the forward pointer. It must not attempt to evacuate any other objects.
1302 return ShenandoahBarrierSet::resolve_forwarded(p);
1303 }
1304
1305 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1306
1307 ShenandoahHeapRegion* r = heap_region_containing(p);
1308 assert(!r->is_humongous(), "never evacuate humongous objects");
1309
1310 ShenandoahAffiliation target_gen = r->affiliation();
1311 return try_evacuate_object(p, thread, r, target_gen);
1312 }
1313
1314 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1315 ShenandoahAffiliation target_gen) {
1316 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1317 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1318 bool alloc_from_lab = true;
1319 HeapWord* copy = nullptr;
1320 size_t size = ShenandoahForwarding::size(p);
1321
1322 #ifdef ASSERT
1323 if (ShenandoahOOMDuringEvacALot &&
1324 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1325 copy = nullptr;
1326 } else {
1327 #endif
1328 if (UseTLAB) {
1329 copy = allocate_from_gclab(thread, size);
1330 }
1331 if (copy == nullptr) {
1332 // If we failed to allocate in LAB, we'll try a shared allocation.
1333 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1334 copy = allocate_memory(req);
1335 alloc_from_lab = false;
1336 }
1337 #ifdef ASSERT
1338 }
1339 #endif
1340
1341 if (copy == nullptr) {
1342 control_thread()->handle_alloc_failure_evac(size);
1343
1344 _oom_evac_handler.handle_out_of_memory_during_evacuation();
1345
1346 return ShenandoahBarrierSet::resolve_forwarded(p);
1347 }
1348
1349 if (ShenandoahEvacTracking) {
1350 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1351 }
1352
1353 // Copy the object:
1354 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1355
1356 // Try to install the new forwarding pointer.
1357 oop copy_val = cast_to_oop(copy);
1358 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1359 if (result == copy_val) {
1360 // Successfully evacuated. Our copy is now the public one!
1361 ContinuationGCSupport::relativize_stack_chunk(copy_val);
1362 shenandoah_assert_correct(nullptr, copy_val);
1363 if (ShenandoahEvacTracking) {
1364 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1365 }
1366 return copy_val;
1367 } else {
1368 // Failed to evacuate. We need to deal with the object that is left behind. Since this
1369 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1370 // But if it happens to contain references to evacuated regions, those references would
1371 // not get updated for this stale copy during this cycle, and we will crash while scanning
1372 // it the next cycle.
1373 if (alloc_from_lab) {
1374 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1375 // object will overwrite this stale copy, or the filler object on LAB retirement will
1376 // do this.
1377 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
|
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "cds/aotMappedHeapWriter.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "gc/shared/classUnloadingContext.hpp"
31 #include "gc/shared/fullGCForwarding.inline.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "gc/shared/gcArguments.hpp"
34 #include "gc/shared/gcTimer.hpp"
35 #include "gc/shared/gcTraceTime.inline.hpp"
36 #include "gc/shared/locationPrinter.inline.hpp"
37 #include "gc/shared/memAllocator.hpp"
38 #include "gc/shared/plab.hpp"
39 #include "gc/shared/tlab_globals.hpp"
40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
42 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
43 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
44 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
45 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
46 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
47 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
48 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
49 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
50 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
51 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
1300 // This thread went through the OOM during evac protocol. It is safe to return
1301 // the forward pointer. It must not attempt to evacuate any other objects.
1302 return ShenandoahBarrierSet::resolve_forwarded(p);
1303 }
1304
1305 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1306
1307 ShenandoahHeapRegion* r = heap_region_containing(p);
1308 assert(!r->is_humongous(), "never evacuate humongous objects");
1309
1310 ShenandoahAffiliation target_gen = r->affiliation();
1311 return try_evacuate_object(p, thread, r, target_gen);
1312 }
1313
1314 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1315 ShenandoahAffiliation target_gen) {
1316 assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1317 assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1318 bool alloc_from_lab = true;
1319 HeapWord* copy = nullptr;
1320
1321 markWord mark = p->mark();
1322 if (ShenandoahForwarding::is_forwarded(mark)) {
1323 return ShenandoahForwarding::get_forwardee(p);
1324 }
1325 size_t old_size = ShenandoahForwarding::size(p);
1326 size_t size = p->copy_size(old_size, mark);
1327
1328 #ifdef ASSERT
1329 if (ShenandoahOOMDuringEvacALot &&
1330 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1331 copy = nullptr;
1332 } else {
1333 #endif
1334 if (UseTLAB) {
1335 copy = allocate_from_gclab(thread, size);
1336 }
1337 if (copy == nullptr) {
1338 // If we failed to allocate in LAB, we'll try a shared allocation.
1339 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1340 copy = allocate_memory(req);
1341 alloc_from_lab = false;
1342 }
1343 #ifdef ASSERT
1344 }
1345 #endif
1346
1347 if (copy == nullptr) {
1348 control_thread()->handle_alloc_failure_evac(size);
1349
1350 _oom_evac_handler.handle_out_of_memory_during_evacuation();
1351
1352 return ShenandoahBarrierSet::resolve_forwarded(p);
1353 }
1354
1355 if (ShenandoahEvacTracking) {
1356 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1357 }
1358
1359 // Copy the object:
1360 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
1361 oop copy_val = cast_to_oop(copy);
1362
1363 // Initialize the identity hash on the copy before installing the forwarding
1364 // pointer, using the mark word we captured earlier. We must do this before
1365 // the CAS so that the copy is fully initialized when it becomes visible to
1366 // other threads. Using the captured mark (rather than re-reading the copy's
1367 // mark) avoids races with other threads that may have evacuated p and
1368 // installed a forwarding pointer in the meantime.
1369 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
1370 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
1371 }
1372
1373 // Try to install the new forwarding pointer.
1374 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1375 if (result == copy_val) {
1376 // Successfully evacuated. Our copy is now the public one!
1377 ContinuationGCSupport::relativize_stack_chunk(copy_val);
1378 shenandoah_assert_correct(nullptr, copy_val);
1379 if (ShenandoahEvacTracking) {
1380 evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
1381 }
1382 return copy_val;
1383 } else {
1384 // Failed to evacuate. We need to deal with the object that is left behind. Since this
1385 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1386 // But if it happens to contain references to evacuated regions, those references would
1387 // not get updated for this stale copy during this cycle, and we will crash while scanning
1388 // it the next cycle.
1389 if (alloc_from_lab) {
1390 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1391 // object will overwrite this stale copy, or the filler object on LAB retirement will
1392 // do this.
1393 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
|