< prev index next > src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
Print this page
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* questions.
*
*/
+ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/continuation.hpp"
#include "runtime/safepointVerifiers.hpp"
- ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) :
- _nm(nm), _oops(nullptr), _oops_count(0), _unregistered(false), _lock(), _ic_lock() {
-
- if (!oops.is_empty()) {
- _oops_count = oops.length();
- _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
- for (int c = 0; c < _oops_count; c++) {
- _oops[c] = oops.at(c);
- }
- }
- _has_non_immed_oops = non_immediate_oops;
-
- assert_same_oops();
+ ShenandoahNMethod::ShenandoahNMethod(nmethod* nm) :
+ _nm(nm), _oops(nullptr), _oops_count(0), _barriers(nullptr), _barriers_count(0), _unregistered(false), _lock(), _ic_lock() {
+ init_from(nm);
}
ShenandoahNMethod::~ShenandoahNMethod() {
if (_oops != nullptr) {
FREE_C_HEAP_ARRAY(_oops);
}
+ if (_barriers != nullptr) {
+ FREE_C_HEAP_ARRAY(_barriers);
+ }
}
void ShenandoahNMethod::update() {
+ init_from(nm());
+ }
+
+ void ShenandoahNMethod::init_from(nmethod* nm) {
ResourceMark rm;
bool non_immediate_oops = false;
GrowableArray<oop*> oops;
+ GrowableArray<ShenandoahNMethodBarrier> barriers;
- detect_reloc_oops(nm(), oops, non_immediate_oops);
- if (oops.length() != _oops_count) {
+ parse(nm, oops, non_immediate_oops, barriers);
+
+ int new_oops_count = oops.length();
+ if (_oops_count != new_oops_count) {
if (_oops != nullptr) {
FREE_C_HEAP_ARRAY(_oops);
_oops = nullptr;
}
-
- _oops_count = oops.length();
- if (_oops_count > 0) {
- _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC);
+ if (new_oops_count > 0) {
+ _oops = NEW_C_HEAP_ARRAY(oop*, new_oops_count, mtGC);
}
}
+ _oops_count = new_oops_count;
+ for (int c = 0; c < _oops_count; c++) {
+ _oops[c] = oops.at(c);
+ }
+ assert_same_oops();
- for (int index = 0; index < _oops_count; index ++) {
- _oops[index] = oops.at(index);
+ int new_barriers_count = barriers.length();
+ if (_barriers_count != new_barriers_count) {
+ if (_barriers != nullptr) {
+ FREE_C_HEAP_ARRAY(_barriers);
+ _barriers = nullptr;
+ }
+ if (new_barriers_count > 0) {
+ _barriers = NEW_C_HEAP_ARRAY(ShenandoahNMethodBarrier, new_barriers_count, mtGC);
+ }
+ }
+ _barriers_count = new_barriers_count;
+ for (int c = 0; c < _barriers_count; c++) {
+ _barriers[c] = barriers.at(c);
}
- _has_non_immed_oops = non_immediate_oops;
- assert_same_oops();
+ _has_non_immed_oops = non_immediate_oops;
}
- void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) {
+ void ShenandoahNMethod::parse(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops, GrowableArray<ShenandoahNMethodBarrier>& barriers) {
has_non_immed_oops = false;
- // Find all oops relocations
+ address code_begin = nm->code_begin();
RelocIterator iter(nm);
while (iter.next()) {
- if (iter.type() != relocInfo::oop_type) {
- // Not an oop
- continue;
- }
-
- oop_Relocation* r = iter.oop_reloc();
- if (!r->oop_is_immediate()) {
- // Non-immediate oop found
- has_non_immed_oops = true;
- continue;
- }
-
- oop value = r->oop_value();
- if (value != nullptr) {
- oop* addr = r->oop_addr();
- shenandoah_assert_correct(addr, value);
- shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
- shenandoah_assert_not_forwarded(addr, value);
- // Non-null immediate oop found. null oops can safely be
- // ignored since the method will be re-registered if they
- // are later patched to be non-null.
- oops.push(addr);
+ switch (iter.type()) {
+ case relocInfo::oop_type: {
+ oop_Relocation* r = iter.oop_reloc();
+ if (!r->oop_is_immediate()) {
+ // Non-immediate oop found
+ has_non_immed_oops = true;
+ break;
+ }
+
+ oop value = r->oop_value();
+ if (value != nullptr) {
+ oop* addr = r->oop_addr();
+ shenandoah_assert_correct(addr, value);
+ shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
+ shenandoah_assert_not_forwarded(addr, value);
+ // Non-null immediate oop found. null oops can safely be
+ // ignored since the method will be re-registered if they
+ // are later patched to be non-null.
+ oops.push(addr);
+ }
+ break;
+ }
+ case relocInfo::patchable_barrier_type: {
+ patchable_barrier_Relocation* r = iter.patchable_barrier_reloc();
+
+ ShenandoahNMethodBarrier b;
+ b._rel_pc = checked_cast<int32_t>(pointer_delta(r->addr(), code_begin, 1));
+ b._rel_target_pc = r->target_offset();
+ b._gc_state = decode_reloc_gc_state(r->metadata());
+ b._jump_when_state = decode_reloc_jump_when_state(r->metadata());
+ barriers.push(b);
+ break;
+ }
+ default:
+ // We do not care about other relocations.
+ break;
}
}
}
ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
- ResourceMark rm;
- bool non_immediate_oops = false;
- GrowableArray<oop*> oops;
-
- detect_reloc_oops(nm, oops, non_immediate_oops);
- return new ShenandoahNMethod(nm, oops, non_immediate_oops);
+ return new ShenandoahNMethod(nm);
}
- void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
+ bool ShenandoahNMethod::handle_oops(nmethod* nm) {
ShenandoahNMethod* data = gc_data(nm);
assert(data != nullptr, "Sanity");
assert(data->lock()->owned_by_self(), "Must hold the lock");
ShenandoahHeap* const heap = ShenandoahHeap::heap();
if ((heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) ||
heap->is_concurrent_strong_root_in_progress()) {
heal_nmethod_metadata(data);
+ // Assume healing changed the code.
+ return true;
} else if (heap->is_concurrent_mark_in_progress()) {
ShenandoahKeepAliveClosure cl;
data->oops_do(&cl);
} else {
// There is possibility that GC is cancelled when it arrives final mark.
// In this case, concurrent root phase is skipped and degenerated GC should be
// followed, where nmethods are disarmed.
}
+
+ // No code modifications happened
+ return false;
+ }
+
+ bool ShenandoahNMethod::handle_barriers(nmethod* nm) {
+ ShenandoahNMethod* data = gc_data(nm);
+ assert(data != nullptr, "Sanity");
+ assert(data->lock()->owned_by_self(), "Must hold the lock");
+
+ char gc_state = ShenandoahHeap::heap()->gc_state();
+ address code_begin = nm->code_begin();
+
+ bool changed = false;
+ for (int c = 0; c < data->_barriers_count; c++) {
+ ShenandoahNMethodBarrier& b = data->_barriers[c];
+ changed |= patch_barrier(code_begin + b._rel_pc,
+ code_begin + b._rel_target_pc,
+ ((gc_state & b._gc_state) != 0) == b._jump_when_state);
+ }
+ return changed;
+ }
+
+ bool ShenandoahNMethod::patch_barrier(address pc, address target_pc, bool should_jump) {
+ // Use precise instruction rewrite code, and only when it recognizes the current insns.
+ // This patching code is non-atomic, but it runs in two safe contexts:
+ // a) For new nmethods that are not yet executing;
+ // b) For existing methods in the nmethod entry barrier context. The nmethod entry barriers
+ // are armed along with stack watermark machinery activation, which together guarantee
+ // the nmethod updates are not interleaved with execution.
+ // The icache flushing is also handled on both paths.
+ bool patched = true;
+ if (should_jump && ShenandoahBarrierSetAssembler::is_patchable_nop(pc)) {
+ ShenandoahBarrierSetAssembler::insert_patchable_jump(pc, target_pc);
+ } else if (!should_jump && ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc)) {
+ ShenandoahBarrierSetAssembler::insert_patchable_nop(pc);
+ } else {
+ patched = false;
+ }
+
+ // Failing to change the barrier is catastrophic for correctness,
+ // so prefer to crash hard even in product.
+ if (should_jump) {
+ guarantee(ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc),
+ "Should be jump to the same address");
+ assert(ShenandoahBarrierSetAssembler::parse_jump_address(pc) == target_pc,
+ "Cross-checking, jump should be to the same address");
+ } else {
+ guarantee(ShenandoahBarrierSetAssembler::is_patchable_nop(pc),
+ "Should be patchable nop");
+ }
+ return patched;
}
#ifdef ASSERT
void ShenandoahNMethod::assert_correct() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
debug_stream.print_cr("recorded oops: %d", _oops_count);
for (int i = 0; i < _oops_count; i++) {
debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i]));
}
GrowableArray<oop*> check;
+ GrowableArray<ShenandoahNMethodBarrier> barriers;
bool non_immed;
- detect_reloc_oops(nm(), check, non_immed);
+ parse(nm(), check, non_immed, barriers);
debug_stream.print_cr("check oops: %d", check.length());
for (int i = 0; i < check.length(); i++) {
debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i)));
}
fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s",
assert(_index >= 0 && _index <= _list->size(), "Sanity");
ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
if (data != nullptr) {
+ // Re-registering the existing nmethod. This is the C1 oop patching path.
+ // We expect no barriers here, as only oops can change in C1 case.
assert(contain(nm), "Must have been registered");
assert(nm == data->nm(), "Must be same nmethod");
+ assert(nm->is_compiled_by_c1(), "Must be compiled by C1");
+ assert(!data->has_barriers(), "Must not have barriers");
// Prevent updating a nmethod while concurrent iteration is in progress.
wait_until_concurrent_iteration_done();
ShenandoahNMethodLocker data_locker(data->lock());
data->update();
} else {
- // For a new nmethod, we can safely append it to the list, because
- // concurrent iteration will not touch it.
+ // New nmethod, not yet executing. We can safely append it to the list,
+ // because concurrent iteration will not touch it. Ditto we do barrier
+ // fixups right here, without relying on nmethod entry barrier to be armed
+ // for new nmethods.
data = ShenandoahNMethod::for_nmethod(nm);
assert(data != nullptr, "Sanity");
ShenandoahNMethod::attach_gc_data(nm, data);
ShenandoahLocker locker(&_lock);
log_register_nmethod(nm);
append(data);
+ ShenandoahNMethodLocker data_locker(data->lock());
+ if (ShenandoahNMethod::handle_barriers(nm)) {
+ ICache::invalidate_range(nm->code_begin(), nm->code_size());
+ }
+ ShenandoahNMethod::disarm_nmethod(nm);
}
- // Disarm new nmethod
- ShenandoahNMethod::disarm_nmethod(nm);
}
void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
assert_locked_or_safepoint(CodeCache_lock);
< prev index next >