< prev index next > src/hotspot/share/runtime/continuationFreezeThaw.cpp
Print this page
#include "runtime/frame.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaThread.inline.hpp"
#include "runtime/jniHandles.inline.hpp"
#include "runtime/keepStackGCProcessed.hpp"
+ #include "runtime/objectMonitor.inline.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/smallRegisterMap.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stackChunkFrameStream.inline.hpp"
#include "runtime/stackOverflow.hpp"
#include "runtime/stackWatermarkSet.inline.hpp"
#include "utilities/debug.hpp"
#include "utilities/exceptions.hpp"
#include "utilities/macros.hpp"
+ #include "utilities/vmError.hpp"
#if INCLUDE_ZGC
#include "gc/z/zStackChunkGCData.inline.hpp"
#endif
#include <type_traits>
#else
static void verify_continuation(oop continuation) { }
#define assert_pfl(p, ...)
#endif
- // should match Continuation.preemptStatus() in Continuation.java
- enum freeze_result {
- freeze_ok = 0,
- freeze_ok_bottom = 1,
- freeze_pinned_cs = 2,
- freeze_pinned_native = 3,
- freeze_pinned_monitor = 4,
- freeze_exception = 5
- };
-
- const char* freeze_result_names[6] = {
- "freeze_ok",
- "freeze_ok_bottom",
- "freeze_pinned_cs",
- "freeze_pinned_native",
- "freeze_pinned_monitor",
- "freeze_exception"
- };
-
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint);
! template<typename ConfigT> static inline int freeze_internal(JavaThread* current, intptr_t* const sp);
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier);
template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind);
#else
static void verify_continuation(oop continuation) { }
#define assert_pfl(p, ...)
#endif
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint);
! template<typename ConfigT, bool preempt> static inline int freeze_internal(JavaThread* thread, intptr_t* const sp);
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier);
template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind);
template<typename ConfigT>
static JRT_LEAF(intptr_t*, thaw(JavaThread* thread, int kind))
// TODO: JRT_LEAF and NoHandleMark is problematic for JFR events.
// vFrameStreamCommon allocates Handles in RegisterMap for continuations.
+ // Also the preemption case with JVMTI events enabled might safepoint so
+ // undo the NoSafepointVerifier here and rely on handling by ContinuationWrapper.
// JRT_ENTRY instead?
ResetNoHandleMark rnhm;
+ debug_only(PauseNoSafepointVerifier pnsv(&__nsv);)
// we might modify the code cache via BarrierSetNMethod::nmethod_entry_barrier
MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread));
return ConfigT::thaw(thread, (Continuation::thaw_kind)kind);
JRT_END
public:
typedef Config<oops, BarrierSetT> SelfT;
using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>;
static int freeze(JavaThread* thread, intptr_t* const sp) {
! return freeze_internal<SelfT>(thread, sp);
}
static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) {
return thaw_internal<SelfT>(thread, kind);
}
public:
typedef Config<oops, BarrierSetT> SelfT;
using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>;
static int freeze(JavaThread* thread, intptr_t* const sp) {
! return freeze_internal<SelfT, false>(thread, sp);
+ }
+
+ static int freeze_preempt(JavaThread* thread, intptr_t* const sp) {
+ return freeze_internal<SelfT, true>(thread, sp);
}
static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) {
return thaw_internal<SelfT>(thread, kind);
}
static oop get_continuation(JavaThread* thread) {
assert(thread != nullptr, "");
assert(thread->threadObj() != nullptr, "");
return java_lang_Thread::continuation(thread->threadObj());
}
inline void clear_anchor(JavaThread* thread) {
thread->frame_anchor()->clear();
}
! static void set_anchor(JavaThread* thread, intptr_t* sp) {
! address pc = ContinuationHelper::return_address_at(
! sp - frame::sender_sp_ret_address_offset());
assert(pc != nullptr, "");
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(sp);
anchor->set_last_Java_pc(pc);
ContinuationHelper::set_anchor_pd(anchor, sp);
assert(thread->has_last_Java_frame(), "");
assert(thread->last_frame().cb() != nullptr, "");
}
- #endif // ASSERT
static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(entry->entry_sp());
anchor->set_last_Java_pc(entry->entry_pc());
static oop get_continuation(JavaThread* thread) {
assert(thread != nullptr, "");
assert(thread->threadObj() != nullptr, "");
return java_lang_Thread::continuation(thread->threadObj());
}
+ #endif // ASSERT
inline void clear_anchor(JavaThread* thread) {
thread->frame_anchor()->clear();
}
! static void set_anchor(JavaThread* thread, intptr_t* sp, address pc = nullptr) {
! if (pc == nullptr) {
! pc = ContinuationHelper::return_address_at(
+ sp - frame::sender_sp_ret_address_offset());
+ }
assert(pc != nullptr, "");
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(sp);
anchor->set_last_Java_pc(pc);
ContinuationHelper::set_anchor_pd(anchor, sp);
assert(thread->has_last_Java_frame(), "");
assert(thread->last_frame().cb() != nullptr, "");
}
static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(entry->entry_sp());
anchor->set_last_Java_pc(entry->entry_pc());
assert(thread->has_last_Java_frame(), "");
assert(thread->last_frame().cb() != nullptr, "");
}
+ #ifdef ASSERT
+ static int monitors_to_fix_on_stack(JavaThread* thread) {
+ ResourceMark rm(JavaThread::current());
+ ContinuationEntry* ce = thread->last_continuation();
+ RegisterMap map(thread,
+ RegisterMap::UpdateMap::include,
+ RegisterMap::ProcessFrames::include,
+ RegisterMap::WalkContinuation::include);
+ map.set_include_argument_oops(false);
+ ResourceHashtable<oopDesc*, bool> rhtable;
+ int monitor_count = 0;
+ for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(thread, f); f = f.sender(&map)) {
+ if (f.is_interpreted_frame()) {
+ frame abs = !f.is_heap_frame() ? f : map.stack_chunk()->derelativize(f);
+ monitor_count += ContinuationHelper::InterpretedFrame::monitors_to_fix(thread, abs, rhtable, map.stack_chunk()());
+ } else if (f.is_compiled_frame()) {
+ monitor_count += ContinuationHelper::CompiledFrame::monitors_to_fix(thread, &map, f, rhtable);
+ } else if (f.is_native_frame()) {
+ monitor_count += ContinuationHelper::NativeFrame::monitors_to_fix(thread, f, rhtable);
+ }
+ }
+ return monitor_count;
+ }
+ #endif
+
#if CONT_JFR
class FreezeThawJfrInfo : public StackObj {
short _e_size;
short _e_num_interpreted_frames;
public:
class FreezeBase : public StackObj {
protected:
JavaThread* const _thread;
ContinuationWrapper& _cont;
bool _barriers; // only set when we allocate a chunk
- const bool _preempt; // used only on the slow path
- const intptr_t * const _frame_sp; // Top frame sp for this freeze
intptr_t* _bottom_address;
int _freeze_size; // total size of all frames plus metadata in words.
int _total_align_size;
intptr_t* _cont_stack_top;
intptr_t* _cont_stack_bottom;
class FreezeBase : public StackObj {
protected:
JavaThread* const _thread;
ContinuationWrapper& _cont;
bool _barriers; // only set when we allocate a chunk
intptr_t* _bottom_address;
+ const bool _preempt;
+ // Used on preemption only
+ frame _last_frame;
+ oop _monitorenter_obj;
+
+ // Used to support freezing with held monitors
+ int _monitors_to_fix;
+ int _monitors_in_lockstack;
+
int _freeze_size; // total size of all frames plus metadata in words.
int _total_align_size;
intptr_t* _cont_stack_top;
intptr_t* _cont_stack_bottom;
JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector;
NOT_PRODUCT(int _frames;)
DEBUG_ONLY(intptr_t* _last_write;)
! inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp);
public:
NOINLINE freeze_result freeze_slow();
void freeze_fast_existing_chunk();
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; }
inline int size_if_fast_freeze_available();
#ifdef ASSERT
bool check_valid_fast_path();
#endif
protected:
JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector;
NOT_PRODUCT(int _frames;)
DEBUG_ONLY(intptr_t* _last_write;)
! inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempt);
public:
NOINLINE freeze_result freeze_slow();
void freeze_fast_existing_chunk();
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; }
inline int size_if_fast_freeze_available();
+ inline frame& last_frame() { return _last_frame; }
+ inline void set_last_frame() { _last_frame = _thread->last_frame(); }
+
#ifdef ASSERT
bool check_valid_fast_path();
#endif
protected:
int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); }
private:
// slow path
frame freeze_start_frame();
! frame freeze_start_frame_safepoint_stub(frame f);
NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top);
! inline frame freeze_start_frame_yield_stub(frame f);
template<typename FKind>
inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); }
private:
// slow path
frame freeze_start_frame();
! frame freeze_start_frame_on_preempt();
NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top);
! inline frame freeze_start_frame_yield_stub();
template<typename FKind>
inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
NOINLINE void finish_freeze(const frame& f, const frame& top);
+ void fix_monitors_in_interpreted_frame(frame& f);
+ template <typename RegisterMapT>
+ void fix_monitors_in_compiled_frame(frame& f, RegisterMapT* map);
+ void fix_monitors_in_fast_path();
+
inline bool stack_overflow();
static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
: sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
template<typename FKind> static inline frame sender(const frame& f);
template<typename FKind> frame new_heap_frame(frame& f, frame& caller);
inline void set_top_frame_metadata_pd(const frame& hf);
inline void patch_pd(frame& callee, const frame& caller);
void adjust_interpreted_frame_unextended_sp(frame& f);
+ static inline void prepare_freeze_interpreted_top_frame(const frame& f);
static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
protected:
void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
bool freeze_fast_new_chunk(stackChunkOop chunk);
class Freeze : public FreezeBase {
private:
stackChunkOop allocate_chunk(size_t stack_size);
public:
! inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp)
! : FreezeBase(thread, cont, frame_sp) {}
freeze_result try_freeze_fast();
protected:
virtual stackChunkOop allocate_chunk_slow(size_t stack_size) override { return allocate_chunk(stack_size); }
};
! FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) :
! _thread(thread), _cont(cont), _barriers(false), _preempt(false), _frame_sp(frame_sp) {
DEBUG_ONLY(_jvmti_event_collector = nullptr;)
assert(_thread != nullptr, "");
assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), "");
class Freeze : public FreezeBase {
private:
stackChunkOop allocate_chunk(size_t stack_size);
public:
! inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt)
! : FreezeBase(thread, cont, frame_sp, preempt) {}
freeze_result try_freeze_fast();
protected:
virtual stackChunkOop allocate_chunk_slow(size_t stack_size) override { return allocate_chunk(stack_size); }
};
! FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt) :
! _thread(thread), _cont(cont), _barriers(false), _preempt(preempt), _last_frame(false /* no initialization */) {
DEBUG_ONLY(_jvmti_event_collector = nullptr;)
assert(_thread != nullptr, "");
assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), "");
#if !defined(PPC64) || defined(ZERO)
static const int doYield_stub_frame_size = frame::metadata_words;
#else
static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord;
#endif
! assert(SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, "");
// properties of the continuation on the stack; all sizes are in words
! _cont_stack_top = frame_sp + doYield_stub_frame_size; // we don't freeze the doYield stub frame
_cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0)
- ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw
log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
assert(cont_size() > 0, "");
}
void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling
_freeze_size = 0;
_total_align_size = 0;
NOT_PRODUCT(_frames = 0;)
}
void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) {
stackChunkOop chunk = _cont.tail();
chunk->copy_from_stack_to_chunk(from, to, size);
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
#if !defined(PPC64) || defined(ZERO)
static const int doYield_stub_frame_size = frame::metadata_words;
#else
static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord;
#endif
! // With preemption doYield() might not have been resolved yet
+ assert(_preempt || SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, "");
// properties of the continuation on the stack; all sizes are in words
! _cont_stack_top = frame_sp + (!preempt ? doYield_stub_frame_size : 0); // we don't freeze the doYield stub frame
_cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0)
- ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw
log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
assert(cont_size() > 0, "");
+
+ if (LockingMode == LM_LEGACY) {
+ _monitors_to_fix = thread->held_monitor_count();
+ assert(_monitors_to_fix >= 0, "invariant");
+ _monitors_in_lockstack = 0;
+ assert(_thread->lock_stack().monitor_count() == 0, "should be set only for LM_LIGHTWEIGHT");
+ } else {
+ _monitors_in_lockstack = _thread->lock_stack().monitor_count();
+ assert(_monitors_in_lockstack >= 0 && _monitors_in_lockstack <= 8, "_monitors_in_lockstack=%d", _monitors_in_lockstack);
+ _monitors_to_fix = _monitors_in_lockstack;
+ assert(thread->held_monitor_count() == 0, "should be set only for LM_LEGACY");
+ }
+ DEBUG_ONLY(int verified_cnt = monitors_to_fix_on_stack(_thread);)
+ assert(verified_cnt == _monitors_to_fix || thread->obj_locker_count() > 0 ||
+ (LockingMode == LM_LIGHTWEIGHT && verified_cnt == _thread->lock_stack().unique_count()),
+ "wrong monitor count. Found %d in the stack but counter is %d", verified_cnt, _monitors_to_fix);
}
void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling
_freeze_size = 0;
_total_align_size = 0;
NOT_PRODUCT(_frames = 0;)
}
+ void FreezeBase::fix_monitors_in_interpreted_frame(frame& f) {
+ assert(LockingMode == LM_LEGACY, "invariant");
+ BasicObjectLock* first_mon = f.interpreter_frame_monitor_begin();
+ BasicObjectLock* last_mon = f.interpreter_frame_monitor_end();
+ assert(last_mon <= first_mon, "must be");
+
+ if (first_mon == last_mon) {
+ // No monitors in this frame
+ return;
+ }
+
+ for (BasicObjectLock* current = f.previous_monitor_in_interpreter_frame(first_mon);
+ current >= last_mon; current = f.previous_monitor_in_interpreter_frame(current)) {
+ oop obj = current->obj();
+ if (obj == nullptr) {
+ continue;
+ }
+ if (obj == _monitorenter_obj) {
+ assert(_preempt, "should be preemption on monitorenter case");
+ assert(obj->mark().monitor() != nullptr, "failed to acquire the lock but its not inflated");
+ continue;
+ }
+ markWord mark = obj->mark();
+ if (mark.has_monitor() && !mark.monitor()->is_owner_anonymous()) {
+ // Good for freezing, nothing to do.
+ assert(mark.monitor()->is_owner(_thread), "invariant");
+ continue;
+ }
+ // Found locked monitor that needs fixing. Inflate will fix the owner for stack-locked case or inflated but anonymously owned.
+ ObjectMonitor* om = ObjectSynchronizer::inflate(_thread, obj, ObjectSynchronizer::InflateCause::inflate_cause_cont_freeze);
+ assert(om != nullptr, "invariant");
+ assert(om->is_owner(_thread), "invariant");
+ if (--_monitors_to_fix == 0) break;
+ }
+ assert(_monitors_to_fix >= 0, "invariant");
+ }
+
+ template <typename RegisterMapT>
+ void FreezeBase::fix_monitors_in_compiled_frame(frame& f, RegisterMapT* map) {
+ assert(LockingMode == LM_LEGACY, "invariant");
+ assert(!f.is_interpreted_frame(), "");
+ assert(ContinuationHelper::CompiledFrame::is_instance(f), "");
+
+ nmethod* nm = f.cb()->as_nmethod();
+
+ if (!nm->has_monitors()) {
+ // No monitors in this frame
+ return;
+ }
+
+ for (ScopeDesc* scope = nm->scope_desc_at(f.pc()); scope != nullptr; scope = scope->sender()) {
+ GrowableArray<MonitorValue*>* mons = scope->monitors();
+ if (mons == nullptr || mons->is_empty()) {
+ continue;
+ }
+
+ for (int index = (mons->length()-1); index >= 0; index--) { // see compiledVFrame::monitors()
+ MonitorValue* mon = mons->at(index);
+ if (mon->eliminated()) {
+ continue; // we ignore eliminated monitors
+ }
+ ScopeValue* ov = mon->owner();
+ StackValue* owner_sv = StackValue::create_stack_value(&f, map, ov); // it is an oop
+ oop obj = owner_sv->get_obj()();
+ if (obj == nullptr) {
+ continue;
+ }
+ if (obj == _monitorenter_obj) {
+ assert(_preempt, "should be preemption on monitorenter case");
+ assert(obj->mark().monitor() != nullptr, "failed to acquire the lock but its not inflated");
+ continue;
+ }
+ markWord mark = obj->mark();
+ if (mark.has_monitor() && !mark.monitor()->is_owner_anonymous()) {
+ // Good for freezing, nothing to do.
+ assert(mark.monitor()->is_owner(_thread), "invariant");
+ continue;
+ }
+ // Found locked monitor that needs fixing. Inflate will fix the owner for stack-locked case or inflated but anonymously owned.
+ ObjectMonitor* om = ObjectSynchronizer::inflate(_thread, obj, ObjectSynchronizer::InflateCause::inflate_cause_cont_freeze);
+ assert(om != nullptr, "invariant");
+ assert(om->is_owner(_thread), "invariant");
+ if (--_monitors_to_fix == 0) break;
+ }
+ }
+ assert(_monitors_to_fix >= 0, "invariant");
+ }
+
+ void FreezeBase::fix_monitors_in_fast_path() {
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ stackChunkOop chunk = _cont.tail();
+ assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "no room for lockstack");
+ _thread->lock_stack().move_to_address((oop*)chunk->start_address());
+
+ chunk->set_lockStackSize((uint8_t)_monitors_in_lockstack);
+ chunk->set_has_lockStack(true);
+
+ DEBUG_ONLY(_monitors_to_fix -= _monitors_in_lockstack;)
+ } else {
+ assert(LockingMode == LM_LEGACY, "invariant");
+ _monitorenter_obj = _thread->is_on_monitorenter() ? _thread->current_pending_monitor()->object() : nullptr;
+
+ ResourceMark rm(_thread);
+ RegisterMap map(JavaThread::current(),
+ RegisterMap::UpdateMap::include,
+ RegisterMap::ProcessFrames::skip, // already processed in unwind_frames()
+ RegisterMap::WalkContinuation::skip);
+ map.set_include_argument_oops(false);
+ frame freezed_top(_cont_stack_top);
+ frame first = !_preempt ? freezed_top : freezed_top.sender(&map);
+ for (frame f = first; _monitors_to_fix > 0 && Continuation::is_frame_in_continuation(_cont.entry(), f); f = f.sender(&map)) {
+ fix_monitors_in_compiled_frame(f, &map);
+ }
+ }
+ assert(_monitors_to_fix == 0, "missing monitors in stack");
+ assert(_thread->held_monitor_count() == 0, "should be 0 now");
+ }
+
void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) {
stackChunkOop chunk = _cont.tail();
chunk->copy_from_stack_to_chunk(from, to, size);
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
assert(_thread->cont_fastpath(), "");
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
assert(_fast_freeze_size == 0, "");
! stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words);
if (freeze_fast_new_chunk(chunk)) {
return freeze_ok;
}
if (_thread->has_pending_exception()) {
return freeze_exception;
assert(_thread->cont_fastpath(), "");
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
assert(_fast_freeze_size == 0, "");
! stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words + _monitors_in_lockstack);
if (freeze_fast_new_chunk(chunk)) {
return freeze_ok;
}
if (_thread->has_pending_exception()) {
return freeze_exception;
// although that would require changing stackChunkOopDesc::is_empty
if (chunk_sp < chunk->stack_size()) {
total_size_needed -= _cont.argsize() + frame::metadata_words_at_top;
}
+ total_size_needed += _monitors_in_lockstack;
+
int chunk_free_room = chunk_sp - frame::metadata_words_at_bottom;
bool available = chunk_free_room >= total_size_needed;
log_develop_trace(continuations)("chunk available: %s size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
available ? "yes" : "no" , total_size_needed, _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
return available ? total_size_needed : 0;
chunk->set_max_thawing_size(cont_size());
chunk->set_argsize(_cont.argsize());
// in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments.
// They'll then be stored twice: in the chunk and in the parent chunk's top frame
! const int chunk_start_sp = cont_size() + frame::metadata_words;
assert(chunk_start_sp == chunk->stack_size(), "");
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true));
chunk->set_max_thawing_size(cont_size());
chunk->set_argsize(_cont.argsize());
// in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments.
// They'll then be stored twice: in the chunk and in the parent chunk's top frame
! const int chunk_start_sp = cont_size() + frame::metadata_words + _monitors_in_lockstack;
assert(chunk_start_sp == chunk->stack_size(), "");
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true));
p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize());
assert(chunk_start_sp <= chunk->stack_size(), "");
assert(chunk_start_sp >= cont_size(), "no room in the chunk");
const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze
! assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, "");
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
#ifdef ASSERT
if (!_empty) {
intptr_t* retaddr_slot = (_orig_chunk_sp
p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize());
assert(chunk_start_sp <= chunk->stack_size(), "");
assert(chunk_start_sp >= cont_size(), "no room in the chunk");
const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze
! assert(!(_fast_freeze_size > 0) || (_orig_chunk_sp - (chunk->start_address() + chunk_new_sp)) == (_fast_freeze_size - _monitors_in_lockstack), "");
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
#ifdef ASSERT
if (!_empty) {
intptr_t* retaddr_slot = (_orig_chunk_sp
chunk->set_sp(chunk_new_sp);
// set chunk->pc to the return address of the topmost frame in the chunk
chunk->set_pc(ContinuationHelper::return_address_at(
_cont_stack_top - frame::sender_sp_ret_address_offset()));
+ // Fix monitors after unwinding frames to make sure oops are valid.
+ if (_monitors_to_fix > 0) {
+ fix_monitors_in_fast_path();
+ }
+
_cont.write();
log_develop_trace(continuations)("FREEZE CHUNK #" INTPTR_FORMAT " (young)", _cont.hash());
LogTarget(Trace, continuations) lt;
if (lt.develop_is_enabled()) {
freeze_result res = recurse_freeze(f, caller, 0, false, true);
if (res == freeze_ok) {
finish_freeze(f, caller);
_cont.write();
}
return res;
}
frame FreezeBase::freeze_start_frame() {
- frame f = _thread->last_frame();
if (LIKELY(!_preempt)) {
! return freeze_start_frame_yield_stub(f);
} else {
! return freeze_start_frame_safepoint_stub(f);
}
}
! frame FreezeBase::freeze_start_frame_yield_stub(frame f) {
assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be");
f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f);
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
return f;
}
! frame FreezeBase::freeze_start_frame_safepoint_stub(frame f) {
! #if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
! f.set_fp(f.real_fp()); // f.set_fp(*Frame::callee_link_address(f)); // ????
! #else
- Unimplemented();
- #endif
- if (!Interpreter::contains(f.pc())) {
- assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be");
- assert(f.oop_map() != nullptr, "must be");
-
- if (Interpreter::contains(ContinuationHelper::StubFrame::return_pc(f))) {
- f = sender<ContinuationHelper::StubFrame>(f); // Safepoint stub in interpreter
- }
- }
- assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
- return f;
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) {
assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame
! assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())), "");
if (stack_overflow()) {
return freeze_exception;
}
freeze_result res = recurse_freeze(f, caller, 0, false, true);
if (res == freeze_ok) {
finish_freeze(f, caller);
_cont.write();
+ assert(_monitors_to_fix == 0, "missing monitors in stack");
+ assert(_thread->held_monitor_count() == 0, "should be 0 now");
}
return res;
}
frame FreezeBase::freeze_start_frame() {
if (LIKELY(!_preempt)) {
! return freeze_start_frame_yield_stub();
} else {
! return freeze_start_frame_on_preempt();
}
}
! frame FreezeBase::freeze_start_frame_yield_stub() {
+ frame f = _thread->last_frame();
assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be");
f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f);
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
return f;
}
! frame FreezeBase::freeze_start_frame_on_preempt() {
! assert(_last_frame.sp() == _thread->last_frame().sp(), "_last_frame should be already initialized");
! assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), _last_frame), "");
! return _last_frame;
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) {
assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame
! assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb()))
+ || ((top && _preempt) == f.is_native_frame()), "");
if (stack_overflow()) {
return freeze_exception;
}
// special native frame
return freeze_pinned_native;
}
return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted);
} else if (f.is_interpreted_frame()) {
! assert((_preempt && top) || !f.interpreter_frame_method()->is_native(), "");
if (_preempt && top && f.interpreter_frame_method()->is_native()) {
! // int native entry
return freeze_pinned_native;
}
-
return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted);
! } else if (_preempt && top && ContinuationHelper::Frame::is_stub(f.cb())) {
! return recurse_freeze_stub_frame(f, caller);
} else {
return freeze_pinned_native;
}
}
// special native frame
return freeze_pinned_native;
}
return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted);
} else if (f.is_interpreted_frame()) {
! assert(!f.interpreter_frame_method()->is_native() || (_preempt && top && _thread->is_on_monitorenter()), "");
if (_preempt && top && f.interpreter_frame_method()->is_native()) {
! // TODO: Allow preemption for this case too
return freeze_pinned_native;
}
return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted);
! } else if (_preempt && top) {
! assert(ContinuationHelper::Frame::is_stub(f.cb()) || (f.is_native_frame() && _thread->is_on_monitorenter()), "invariant");
+ if (f.is_native_frame()) {
+ // TODO: Allow preemption for this case too
+ return freeze_pinned_native;
+ } else {
+ return recurse_freeze_stub_frame(f, caller);
+ }
} else {
return freeze_pinned_native;
}
}
// The parameter argsize_md includes metadata that has to be part of caller/callee overlap.
// See also StackChunkFrameStream<frame_kind>::frame_size()
freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, int argsize_md) {
int argsize = argsize_md - frame::metadata_words_at_top;
assert(callee.is_interpreted_frame()
+ || ContinuationHelper::Frame::is_stub(callee.cb())
|| callee.cb()->as_nmethod()->is_osr_method()
|| argsize == _cont.argsize(), "argsize: %d cont.argsize: %d", argsize, _cont.argsize());
log_develop_trace(continuations)("bottom: " INTPTR_FORMAT " count %d size: %d argsize: %d",
p2i(_bottom_address), _frames, _freeze_size << LogBytesPerWord, argsize);
assert(chunk == nullptr || chunk->is_empty()
|| unextended_sp == chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp()), "");
assert(chunk != nullptr || unextended_sp < _freeze_size, "");
+ _freeze_size += _monitors_in_lockstack;
+
// _barriers can be set to true by an allocation in freeze_fast, in which case the chunk is available
bool allocated_old_in_freeze_fast = _barriers;
assert(!allocated_old_in_freeze_fast || (unextended_sp >= _freeze_size && chunk->is_empty()),
"Chunk allocated in freeze_fast is of insufficient size "
"unextended_sp: %d size: %d is_empty: %d", unextended_sp, _freeze_size, chunk->is_empty());
assert(!_barriers || is_empty(chunk), "");
assert(!is_empty(chunk) || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), "");
assert(!is_empty(chunk) || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), "");
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
// will either see no continuation or a consistent chunk.
unwind_frames();
! chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - frame::metadata_words);
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top chunk:");
chunk->print_on(&ls);
}
// The topmost existing frame in the chunk; or an empty frame if the chunk is empty
caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame();
DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);)
assert(!_barriers || is_empty(chunk), "");
assert(!is_empty(chunk) || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), "");
assert(!is_empty(chunk) || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), "");
+ if (_preempt) {
+ frame f = _thread->last_frame();
+ if (f.is_interpreted_frame()) {
+ // Do it now that we know freezing will be successful.
+ prepare_freeze_interpreted_top_frame(f);
+ }
+ }
+
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
// will either see no continuation or a consistent chunk.
unwind_frames();
! chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - _monitors_in_lockstack - frame::metadata_words);
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top chunk:");
chunk->print_on(&ls);
}
+ // Fix monitors after unwinding frames to make sure oops are valid. Also do it now to avoid unnecessary
+ // calls to fix_monitors_in_interpreted_frame/fix_monitors_in_compiled_frame() later.
+ if (_monitors_to_fix > 0) {
+ if (_monitors_in_lockstack > 0) {
+ assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "no room for lockstack");
+ _thread->lock_stack().move_to_address((oop*)chunk->start_address());
+
+ chunk->set_lockStackSize((uint8_t)_monitors_in_lockstack);
+ chunk->set_has_lockStack(true);
+
+ _monitors_to_fix -= _monitors_in_lockstack;
+ assert(_monitors_to_fix == 0, "invariant");
+ }
+ _monitorenter_obj = _thread->is_on_monitorenter() ? _thread->current_pending_monitor()->object() : nullptr;
+ }
+
// The topmost existing frame in the chunk; or an empty frame if the chunk is empty
caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame();
DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);)
assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), "");
frame entry = sender(callee);
! assert(Continuation::is_return_barrier_entry(entry.pc()) || Continuation::is_continuation_enterSpecial(entry), "");
assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
#endif
return freeze_ok_bottom;
}
assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), "");
frame entry = sender(callee);
! assert((!empty && Continuation::is_return_barrier_entry(entry.pc())) || (empty && Continuation::is_continuation_enterSpecial(entry)), "");
assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
#endif
return freeze_ok_bottom;
}
// including metadata between f and its args
const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top;
log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d",
frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize);
! // we'd rather not yield inside methods annotated with @JvmtiMountTransition
! assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
// including metadata between f and its args
const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top;
log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d",
frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize);
! // we'd rather not yield inside methods annotated with @JvmtiMountTransition. In the preempt case
! // we already checked it is safe to do so in Continuation::is_safe_vthread_to_preempt().
+ assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition() || _preempt, "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
caller = hf;
// Mark frame_method's GC epoch for class redefinition on_stack calculation.
frame_method->record_gc_epoch();
+ if (_monitors_to_fix > 0) {
+ // Check if we have monitors in this frame
+ fix_monitors_in_interpreted_frame(f);
+ }
+
return freeze_ok;
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
// See also StackChunkFrameStream<frame_kind>::frame_size()
log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d",
ContinuationHelper::Frame::frame_method(f) != nullptr ?
ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "",
_freeze_size, fsize, argsize);
! // we'd rather not yield inside methods annotated with @JvmtiMountTransition
! assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d",
ContinuationHelper::Frame::frame_method(f) != nullptr ?
ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "",
_freeze_size, fsize, argsize);
! // we'd rather not yield inside methods annotated with @JvmtiMountTransition. In the preempt case
! // we already checked it is safe to do so in Continuation::is_safe_vthread_to_preempt().
+ assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition() || _preempt, "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
caller = hf;
return freeze_ok;
}
NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) {
intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0);
const int fsize = f.cb()->frame_size();
log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
! // recurse_freeze_java_frame and freeze inlined here because we need to use a full RegisterMap for lock ownership
- NOT_PRODUCT(_frames++;)
- _freeze_size += fsize;
-
- RegisterMap map(_cont.thread(),
- RegisterMap::UpdateMap::include,
- RegisterMap::ProcessFrames::skip,
- RegisterMap::WalkContinuation::skip);
- map.set_include_argument_oops(false);
- ContinuationHelper::update_register_map<ContinuationHelper::StubFrame>(f, &map);
- f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case
- frame senderf = sender<ContinuationHelper::StubFrame>(f);
- assert(senderf.unextended_sp() < _bottom_address - 1, "");
- assert(senderf.is_compiled_frame(), "");
-
- if (UNLIKELY(senderf.oop_map() == nullptr)) {
- // native frame
- return freeze_pinned_native;
- }
-
- freeze_result result = recurse_freeze_compiled_frame(senderf, caller, 0, 0); // This might be deoptimized
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
- assert(result != freeze_ok_bottom, "");
- assert(!caller.is_interpreted_frame(), "");
! DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false);)
frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller);
intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0);
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
! DEBUG_ONLY(after_freeze_java_frame(hf, false);)
caller = hf;
return freeze_ok;
}
assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
caller = hf;
+
+ if (_monitors_to_fix > 0) {
+ // Check if we have monitors in this frame
+ fix_monitors_in_compiled_frame(f, SmallRegisterMap::instance);
+ }
+
return freeze_ok;
}
NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) {
+ DEBUG_ONLY(frame fsender = sender(f);)
+ assert(!fsender.is_native_frame() || (Continuation::is_continuation_enterSpecial(fsender) && !_cont.is_empty()), "sender should't be native except for enterSpecial case");
+
intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0);
const int fsize = f.cb()->frame_size();
log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
! freeze_result result = recurse_freeze_java_frame<ContinuationHelper::StubFrame>(f, caller, fsize, 0);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
! bool is_bottom_frame = result == freeze_ok_bottom;
+ assert(!caller.is_empty() || (is_bottom_frame && !_cont.is_empty()), "");
+
+ DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, is_bottom_frame);)
+
frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller);
intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0);
+
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
!
+ if (caller.is_interpreted_frame()) {
+ _total_align_size += frame::align_wiggle;
+ }
+
+ patch(f, hf, caller, is_bottom_frame);
+
+ DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
caller = hf;
return freeze_ok;
}
chunk->set_sp(chunk->to_offset(top.sp()));
chunk->set_pc(top.pc());
chunk->set_max_thawing_size(chunk->max_thawing_size() + _total_align_size);
+ assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "clash with lockstack");
+
// At this point the chunk is consistent
if (UNLIKELY(_barriers)) {
log_develop_trace(continuations)("do barriers on old chunk");
// Serial and Parallel GC can allocate objects directly into the old generation.
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top hframe after (freeze):");
assert(_cont.last_frame().is_heap_frame(), "should be");
_cont.last_frame().print_on(&ls);
+ DEBUG_ONLY(print_frame_layout(top, false, &ls);)
}
assert(_cont.chunk_invariant(), "");
}
class StackChunkAllocator : public MemAllocator {
const size_t _stack_size;
ContinuationWrapper& _continuation_wrapper;
JvmtiSampledObjectAllocEventCollector* const _jvmti_event_collector;
+ JavaThread* const _target;
mutable bool _took_slow_path;
// Does the minimal amount of initialization needed for a TLAB allocation.
// We don't need to do a full initialization, as such an allocation need not be immediately walkable.
virtual oop initialize(HeapWord* mem) const override {
assert(_stack_size <= max_jint, "");
assert(_word_size > _stack_size, "");
// zero out fields (but not the stack)
const size_t hs = oopDesc::header_size();
+ oopDesc::set_klass_gap(mem, 0);
Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs);
jdk_internal_vm_StackChunk::set_size(mem, (int)_stack_size);
jdk_internal_vm_StackChunk::set_sp(mem, (int)_stack_size);
oop obj = initialize(mem);
return stackChunkOopDesc::cast(obj);
}
public:
StackChunkAllocator(Klass* klass,
size_t word_size,
Thread* thread,
size_t stack_size,
ContinuationWrapper& continuation_wrapper,
! JvmtiSampledObjectAllocEventCollector* jvmti_event_collector)
: MemAllocator(klass, word_size, thread),
_stack_size(stack_size),
_continuation_wrapper(continuation_wrapper),
_jvmti_event_collector(jvmti_event_collector),
_took_slow_path(false) {}
// Provides it's own, specialized allocation which skips instrumentation
// if the memory can be allocated without going to a slow-path.
stackChunkOop allocate() const {
oop obj = initialize(mem);
return stackChunkOopDesc::cast(obj);
}
+ bool is_preempt() const { return _thread != _target; }
+
public:
StackChunkAllocator(Klass* klass,
size_t word_size,
Thread* thread,
size_t stack_size,
ContinuationWrapper& continuation_wrapper,
! JvmtiSampledObjectAllocEventCollector* jvmti_event_collector,
+ JavaThread* target)
: MemAllocator(klass, word_size, thread),
_stack_size(stack_size),
_continuation_wrapper(continuation_wrapper),
_jvmti_event_collector(jvmti_event_collector),
+ _target(target),
_took_slow_path(false) {}
// Provides it's own, specialized allocation which skips instrumentation
// if the memory can be allocated without going to a slow-path.
stackChunkOop allocate() const {
//
// This might safepoint while allocating, but all safepointing due to
// instrumentation have been deferred. This property is important for
// some GCs, as this ensures that the allocated object is in the young
// generation / newly allocated memory.
! StackChunkAllocator allocator(klass, size_in_words, current, stack_size, _cont, _jvmti_event_collector);
stackChunkOop chunk = allocator.allocate();
if (chunk == nullptr) {
return nullptr; // OOME
}
//
// This might safepoint while allocating, but all safepointing due to
// instrumentation have been deferred. This property is important for
// some GCs, as this ensures that the allocated object is in the young
// generation / newly allocated memory.
! StackChunkAllocator allocator(klass, size_in_words, current, stack_size, _cont, _jvmti_event_collector, _thread);
stackChunkOop chunk = allocator.allocate();
if (chunk == nullptr) {
return nullptr; // OOME
}
assert(chunk->max_thawing_size() == 0, "");
assert(chunk->pc() == nullptr, "");
assert(chunk->argsize() == 0, "");
assert(chunk->flags() == 0, "");
assert(chunk->is_gc_mode() == false, "");
+ assert(chunk->lockStackSize() == 0, "");
+ assert(chunk->objectMonitor() == nullptr, "");
// fields are uninitialized
chunk->set_parent_access<IS_DEST_UNINITIALIZED>(_cont.last_nonempty_chunk());
chunk->set_cont_access<IS_DEST_UNINITIALIZED>(_cont.continuation());
ContinuationWrapper::SafepointOp so(Thread::current(), cont);
JvmtiExport::continuation_yield_cleanup(JavaThread::current(), num_frames);
}
invalidate_jvmti_stack(thread);
}
- #endif // INCLUDE_JVMTI
! #ifdef ASSERT
! static bool monitors_on_stack(JavaThread* thread) {
! ContinuationEntry* ce = thread->last_continuation();
! RegisterMap map(thread,
! RegisterMap::UpdateMap::include,
! RegisterMap::ProcessFrames::include,
! RegisterMap::WalkContinuation::skip);
! map.set_include_argument_oops(false);
! for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
! if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) ||
! (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f))) {
! return true;
}
! }
! return false;
}
// There are no interpreted frames if we're not called from the interpreter and we haven't ancountered an i2c
// adapter or called Deoptimization::unpack_frames. As for native frames, upcalls from JNI also go through the
// interpreter (see JavaCalls::call_helper), while the UpcallLinker explicitly sets cont_fastpath.
bool FreezeBase::check_valid_fast_path() {
ContinuationWrapper::SafepointOp so(Thread::current(), cont);
JvmtiExport::continuation_yield_cleanup(JavaThread::current(), num_frames);
}
invalidate_jvmti_stack(thread);
}
! static void jvmti_mount_end(JavaThread* current, ContinuationWrapper& cont, frame top, ObjectMonitor* mon, bool post_mount_event) {
! assert(current->vthread() != nullptr, "must be");
!
! HandleMarkCleaner hm(current);
! Handle vth(current, current->vthread());
!
! ContinuationWrapper::SafepointOp so(current, cont);
!
! // Since we might safepoint set the anchor so that the stack can we walked.
! set_anchor(current, top.sp());
!
! JRT_BLOCK
+ current->rebind_to_jvmti_thread_state_of(vth());
+ {
+ MutexLocker mu(JvmtiThreadState_lock);
+ JvmtiThreadState* state = current->jvmti_thread_state();
+ if (state != NULL && state->is_pending_interp_only_mode()) {
+ JvmtiEventController::enter_interp_only_mode();
+ }
}
! assert(current->is_in_VTMS_transition(), "sanity check");
! assert(!current->is_in_tmp_VTMS_transition(), "sanity check");
+ JvmtiVTMSTransitionDisabler::finish_VTMS_transition((jthread)vth.raw_value(), /* is_mount */ true);
+
+ if (post_mount_event && JvmtiExport::should_post_vthread_mount()) {
+ JvmtiExport::post_vthread_mount((jthread)vth.raw_value());
+ } else if (!post_mount_event) {
+ // Preemption cancelled case. Clear the unmount event pending flag. The
+ // flag might actually not be set if _VTMS_notify_jvmti_events was enabled
+ // after preemption happened (late binding agents). But since this would
+ // be a rare case we just do it unconditionally.
+ current->set_jvmti_unmount_event_pending(false);
+ }
+
+ if (mon != nullptr) {
+ JvmtiExport::post_monitor_contended_entered(current, mon);
+ }
+ JRT_BLOCK_END
+
+ clear_anchor(current);
}
+ #endif // INCLUDE_JVMTI
+
+ #ifdef ASSERT
// There are no interpreted frames if we're not called from the interpreter and we haven't ancountered an i2c
// adapter or called Deoptimization::unpack_frames. As for native frames, upcalls from JNI also go through the
// interpreter (see JavaCalls::call_helper), while the UpcallLinker explicitly sets cont_fastpath.
bool FreezeBase::check_valid_fast_path() {
RegisterMap map(_thread,
RegisterMap::UpdateMap::skip,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
! for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
! if (!f.is_compiled_frame() || f.is_deoptimized_frame()) {
return false;
}
}
return true;
}
#endif // ASSERT
! static inline int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont) {
verify_continuation(cont.continuation());
assert(!cont.is_empty(), "");
- // This is done for the sake of the enterSpecial frame
- StackWatermarkSet::after_unwind(thread);
log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash());
-
return 0;
}
static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) {
if (UNLIKELY(res != freeze_ok)) {
RegisterMap map(_thread,
RegisterMap::UpdateMap::skip,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
! int i = 0;
! for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map), i++) {
+ if (!((f.is_compiled_frame() && !f.is_deoptimized_frame()) || (i == 0 && f.is_runtime_frame()))) {
return false;
}
}
return true;
}
#endif // ASSERT
! static inline int freeze_epilog(ContinuationWrapper& cont) {
verify_continuation(cont.continuation());
assert(!cont.is_empty(), "");
log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash());
return 0;
}
static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) {
if (UNLIKELY(res != freeze_ok)) {
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint
! return freeze_epilog(thread, cont);
}
! template<typename ConfigT>
static inline int freeze_internal(JavaThread* current, intptr_t* const sp) {
assert(!current->has_pending_exception(), "");
#ifdef ASSERT
! log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()));
log_frames(current);
#endif
CONT_JFR_ONLY(EventContinuationFreeze event;)
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint
! return freeze_epilog(cont);
}
! static int preempt_epilog(ContinuationWrapper& cont, freeze_result res, frame& old_last_frame) {
+ if (UNLIKELY(res != freeze_ok)) {
+ verify_continuation(cont.continuation());
+ log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
+ return res;
+ }
+
+ patch_return_pc_with_preempt_stub(old_last_frame);
+ cont.set_preempted(true);
+ cont.tail()->set_is_preempted(true);
+
+ if (cont.thread()->is_on_monitorenter()) {
+ cont.tail()->set_objectMonitor(cont.thread()->current_pending_monitor());
+ }
+
+ return freeze_epilog(cont);
+ }
+
+ template<typename ConfigT, bool preempt>
static inline int freeze_internal(JavaThread* current, intptr_t* const sp) {
assert(!current->has_pending_exception(), "");
#ifdef ASSERT
! log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT "JavaThread: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()), p2i(current));
log_frames(current);
#endif
CONT_JFR_ONLY(EventContinuationFreeze event;)
ContinuationWrapper cont(current, oopCont);
log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), "");
! assert(monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0),
! "Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count());
-
- if (entry->is_pinned() || current->held_monitor_count() > 0) {
log_develop_debug(continuations)("PINNED due to critical section/hold monitor");
verify_continuation(cont.continuation());
freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor;
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
! Freeze<ConfigT> freeze(current, cont, sp);
assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), "");
bool fast = UseContinuationFastPath && current->cont_fastpath();
if (fast && freeze.size_if_fast_freeze_available() > 0) {
freeze.freeze_fast_existing_chunk();
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
! freeze_epilog(current, cont);
! return 0;
}
log_develop_trace(continuations)("chunk unavailable; transitioning to VM");
! assert(current == JavaThread::current(), "must be current thread except for preempt");
JRT_BLOCK
// delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk
JvmtiSampledObjectAllocEventCollector jsoaec(false);
freeze.set_jvmti_event_collector(&jsoaec);
ContinuationWrapper cont(current, oopCont);
log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), "");
! const bool pinned_monitor = NOT_LOOM_MONITOR_SUPPORT(current->held_monitor_count() > 0) LOOM_MONITOR_SUPPORT_ONLY(current->jni_monitor_count() > 0);
! if (entry->is_pinned() || pinned_monitor) {
log_develop_debug(continuations)("PINNED due to critical section/hold monitor");
verify_continuation(cont.continuation());
freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor;
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
! Freeze<ConfigT> freeze(current, cont, sp, preempt);
+
+ if (preempt) {
+ freeze.set_last_frame(); // remember last frame
+ #ifdef AARCH64
+ JvmtiSampledObjectAllocEventCollector jsoaec(false);
+ freeze.set_jvmti_event_collector(&jsoaec);
+
+ // Force aarch64 to slow path always for now. It needs extra instructions to correctly set
+ // the last pc we copy into the stackChunk, since it will not necessarily be at sp[-1]).
+ freeze_result res = freeze.freeze_slow();
+ CONT_JFR_ONLY(cont.post_jfr_event(&event, oopCont, current);)
+ return preempt_epilog(cont, res, freeze.last_frame());
+ #endif
+ }
assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), "");
bool fast = UseContinuationFastPath && current->cont_fastpath();
if (fast && freeze.size_if_fast_freeze_available() > 0) {
freeze.freeze_fast_existing_chunk();
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
! return !preempt ? freeze_epilog(cont) : preempt_epilog(cont, freeze_ok, freeze.last_frame());
! }
+
+ if (preempt) {
+ JvmtiSampledObjectAllocEventCollector jsoaec(false);
+ freeze.set_jvmti_event_collector(&jsoaec);
+
+ freeze_result res = fast ? freeze.try_freeze_fast() : freeze.freeze_slow();
+
+ CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
+ preempt_epilog(cont, res, freeze.last_frame());
+ return res;
}
log_develop_trace(continuations)("chunk unavailable; transitioning to VM");
! assert(current == JavaThread::current(), "must be current thread");
JRT_BLOCK
// delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk
JvmtiSampledObjectAllocEventCollector jsoaec(false);
freeze.set_jvmti_event_collector(&jsoaec);
static int thaw_size(stackChunkOop chunk) {
int size = chunk->max_thawing_size();
size += frame::metadata_words; // For the top pc+fp in push_return_frame or top = stack_sp - frame::metadata_words in thaw_fast
size += 2*frame::align_wiggle; // in case of alignments at the top and bottom
+ size += frame::metadata_words; // for preemption case (see push_preempt_rerun_adapter)
return size;
}
// make room on the stack for thaw
// returns the size in bytes, or 0 on failure
// fast path
inline void prefetch_chunk_pd(void* start, int size_words);
void patch_return(intptr_t* sp, bool is_last);
! // slow path
! NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, bool return_barrier);
- private:
void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top);
template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
void finalize_thaw(frame& entry, int argsize);
inline bool seen_by_gc();
// fast path
inline void prefetch_chunk_pd(void* start, int size_words);
void patch_return(intptr_t* sp, bool is_last);
! intptr_t* handle_preempted_continuation(stackChunkOop original_chunk, intptr_t* sp, bool fast_case);
! inline intptr_t* push_preempt_rerun_adapter(frame top, bool is_interpreted_frame);
+ inline intptr_t* push_preempt_monitorenter_redo(stackChunkOop chunk);
void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top);
+ void finish_thaw(frame& f);
+
+ private:
template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
void finalize_thaw(frame& entry, int argsize);
inline bool seen_by_gc();
void clear_bitmap_bits(address start, address end);
NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames);
void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
- void finish_thaw(frame& f);
void push_return_frame(frame& f);
inline frame new_entry_frame();
template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
inline void patch_pd(frame& f, const frame& sender);
inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
void clear_bitmap_bits(address start, address end);
NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames);
void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
void push_return_frame(frame& f);
inline frame new_entry_frame();
template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
inline void patch_pd(frame& f, const frame& sender);
+ inline void patch_pd(frame& f, intptr_t* caller_sp);
inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
&& !PreserveFramePointer;
}
inline intptr_t* thaw(Continuation::thaw_kind kind);
NOINLINE intptr_t* thaw_fast(stackChunkOop chunk);
+ NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind);
inline void patch_caller_links(intptr_t* sp, intptr_t* bottom);
};
template <typename ConfigT>
inline intptr_t* Thaw<ConfigT>::thaw(Continuation::thaw_kind kind) {
assert(chunk != nullptr, "guaranteed by prepare_thaw");
assert(!chunk->is_empty(), "guaranteed by prepare_thaw");
_barriers = chunk->requires_barriers();
return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk)
! : thaw_slow(chunk, kind != Continuation::thaw_top);
}
class ReconstructedStack : public StackObj {
intptr_t* _base; // _cont.entrySP(); // top of the entry frame
int _thaw_size;
assert(chunk != nullptr, "guaranteed by prepare_thaw");
assert(!chunk->is_empty(), "guaranteed by prepare_thaw");
_barriers = chunk->requires_barriers();
return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk)
! : thaw_slow(chunk, kind);
}
class ReconstructedStack : public StackObj {
intptr_t* _base; // _cont.entrySP(); // top of the entry frame
int _thaw_size;
StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
assert(chunk_sp == f.sp(), "");
assert(chunk_sp == f.unextended_sp(), "");
! const int frame_size = f.cb()->frame_size();
argsize = f.stack_argsize();
f.next(SmallRegisterMap::instance, true /* stop */);
empty = f.is_done();
assert(!empty || argsize == chunk->argsize(), "");
if (empty) {
clear_chunk(chunk);
} else {
chunk->set_sp(chunk->sp() + frame_size);
chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
assert(chunk_sp == f.sp(), "");
assert(chunk_sp == f.unextended_sp(), "");
! int frame_size = f.cb()->frame_size();
argsize = f.stack_argsize();
+ bool is_stub = f.is_stub();
f.next(SmallRegisterMap::instance, true /* stop */);
empty = f.is_done();
assert(!empty || argsize == chunk->argsize(), "");
+ assert(!is_stub || !empty, "runtime stub should have caller frame");
+ if (is_stub) {
+ // If we don't thaw the top compiled frame too, after restoring the saved
+ // registers back in Java, we would hit the return barrier to thaw one more
+ // frame effectively overwritting the restored registers during that call.
+ f.get_cb();
+ frame_size += f.cb()->frame_size();
+ argsize = f.stack_argsize();
+ f.next(SmallRegisterMap::instance, true /* stop */);
+ empty = f.is_done();
+ assert(!empty || argsize == chunk->argsize(), "");
+ }
+
if (empty) {
clear_chunk(chunk);
} else {
chunk->set_sp(chunk->sp() + frame_size);
chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
inline bool ThawBase::seen_by_gc() {
return _barriers || _cont.tail()->is_gc_mode();
}
! NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) {
LogTarget(Trace, continuations) lt;
if (lt.develop_is_enabled()) {
LogStream ls(lt);
! ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, return_barrier, p2i(chunk));
chunk->print_on(true, &ls);
}
#if CONT_JFR
EventContinuationThawSlow e;
inline bool ThawBase::seen_by_gc() {
return _barriers || _cont.tail()->is_gc_mode();
}
! template <typename ConfigT>
+ NOINLINE intptr_t* Thaw<ConfigT>::thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind) {
+ bool retry_fast_path = false;
+
+ bool preempted_case = chunk->is_preempted();
+ if (preempted_case) {
+ assert(_cont.is_preempted(), "must be");
+ assert(chunk->objectMonitor() != nullptr, "must be");
+
+ ObjectMonitor* mon = chunk->objectMonitor();
+ if (!mon->is_owner(_thread)) {
+ return push_preempt_monitorenter_redo(chunk);
+ }
+ chunk->set_is_preempted(false);
+ retry_fast_path = true;
+ }
+
+ #if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC
+ if (UseZGC || UseShenandoahGC) {
+ _cont.tail()->relativize_derived_pointers_concurrently();
+ }
+ #endif
+
+ // First thaw after freeze. If there were oops in the stacklock
+ // during freeze, restore them now.
+ if (chunk->lockStackSize() > 0) {
+ int lockStackSize = chunk->lockStackSize();
+ assert(lockStackSize > 0, "should be");
+
+ oop tmp_lockstack[8];
+ chunk->copy_lockstack(tmp_lockstack);
+ _thread->lock_stack().move_from_address(tmp_lockstack, lockStackSize);
+
+ chunk->set_lockStackSize(0);
+ chunk->set_has_lockStack(false);
+ retry_fast_path = true;
+ }
+
+ // Retry the fast path now that we possibly cleared the FLAG_HAS_LOCKSTACK
+ // and FLAG_PREEMPTED flags from the stackChunk.
+ if (retry_fast_path && can_thaw_fast(chunk)) {
+ intptr_t* sp = thaw_fast(chunk);
+ if (preempted_case) {
+ assert(_cont.is_preempted(), "must be");
+ return handle_preempted_continuation(chunk, sp, true /* fast_case */);
+ }
+ return sp;
+ }
+
LogTarget(Trace, continuations) lt;
if (lt.develop_is_enabled()) {
LogStream ls(lt);
! ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, kind, p2i(chunk));
chunk->print_on(true, &ls);
}
#if CONT_JFR
EventContinuationThawSlow e;
}
#endif
DEBUG_ONLY(_frames = 0;)
_align_size = 0;
! int num_frames = (return_barrier ? 1 : 2);
_stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk);
_top_unextended_sp_before_thaw = _stream.unextended_sp();
frame heap_frame = _stream.to_frame();
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top hframe before (thaw):");
assert(heap_frame.is_heap_frame(), "should have created a relative frame");
heap_frame.print_value_on(&ls, nullptr);
}
- #if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC
- if (UseZGC || UseShenandoahGC) {
- _cont.tail()->relativize_derived_pointers_concurrently();
- }
- #endif
-
frame caller; // the thawed caller on the stack
recurse_thaw(heap_frame, caller, num_frames, true);
finish_thaw(caller); // caller is now the topmost thawed frame
_cont.write();
assert(_cont.chunk_invariant(), "");
! JVMTI_ONLY(if (!return_barrier) invalidate_jvmti_stack(_thread));
_thread->set_cont_fastpath(_fastpath);
intptr_t* sp = caller.sp();
return sp;
}
void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) {
log_develop_debug(continuations)("thaw num_frames: %d", num_frames);
assert(!_cont.is_empty(), "no more frames");
assert(num_frames > 0, "");
assert(!heap_frame.is_empty(), "");
! if (top && heap_frame.is_safepoint_blob_frame()) {
- assert(ContinuationHelper::Frame::is_stub(heap_frame.cb()), "cb: %s", heap_frame.cb()->name());
recurse_thaw_stub_frame(heap_frame, caller, num_frames);
} else if (!heap_frame.is_interpreted_frame()) {
recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
} else {
recurse_thaw_interpreted_frame(heap_frame, caller, num_frames);
}
#endif
DEBUG_ONLY(_frames = 0;)
_align_size = 0;
! bool is_return_barrier = kind != Continuation::thaw_top;
+ int num_frames = (is_return_barrier ? 1 : 2);
_stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk);
_top_unextended_sp_before_thaw = _stream.unextended_sp();
+ stackChunkOop original_chunk = chunk;
+
frame heap_frame = _stream.to_frame();
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top hframe before (thaw):");
assert(heap_frame.is_heap_frame(), "should have created a relative frame");
heap_frame.print_value_on(&ls, nullptr);
}
frame caller; // the thawed caller on the stack
recurse_thaw(heap_frame, caller, num_frames, true);
finish_thaw(caller); // caller is now the topmost thawed frame
_cont.write();
assert(_cont.chunk_invariant(), "");
! JVMTI_ONLY(if (!is_return_barrier) invalidate_jvmti_stack(_thread));
_thread->set_cont_fastpath(_fastpath);
intptr_t* sp = caller.sp();
+
+ if (preempted_case) {
+ assert(_cont.is_preempted(), "must be");
+ return handle_preempted_continuation(original_chunk, sp, false /* fast_case */);
+ }
return sp;
}
void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) {
log_develop_debug(continuations)("thaw num_frames: %d", num_frames);
assert(!_cont.is_empty(), "no more frames");
assert(num_frames > 0, "");
assert(!heap_frame.is_empty(), "");
! if (top && ContinuationHelper::Frame::is_stub(heap_frame.cb())) {
recurse_thaw_stub_frame(heap_frame, caller, num_frames);
} else if (!heap_frame.is_interpreted_frame()) {
recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
} else {
recurse_thaw_interpreted_frame(heap_frame, caller, num_frames);
stackChunkOop chunk = _cont.tail();
chunk->bitmap().clear_range(chunk->bit_index_for(start), chunk->bit_index_for(effective_end));
assert(effective_end == end || !chunk->bitmap().at(chunk->bit_index_for(effective_end)), "bit should not be set");
}
+ intptr_t* ThawBase::handle_preempted_continuation(stackChunkOop original_chunk, intptr_t* sp, bool fast_case) {
+ frame top(sp);
+ assert(top.pc() == *(address*)(sp - frame::sender_sp_ret_address_offset()), "");
+ bool top_is_interpreted = Interpreter::contains(top.pc());
+
+ if (fast_case) {
+ assert(ContinuationHelper::Frame::is_stub(top.cb()), "invariant");
+ int fsize = ContinuationHelper::StubFrame::size(top);
+ patch_pd(top, sp + fsize);
+ }
+
+ _cont.set_preempted(false);
+ bool same_chunk = original_chunk == _cont.tail();
+
+ ObjectMonitor* mon = original_chunk->objectMonitor();
+ if (same_chunk && mon != nullptr) {
+ original_chunk->set_objectMonitor(nullptr);
+ }
+
+ bool post_mount_event = true;
+ if (_thread->preemption_cancelled()) {
+ // Since we never actually unmounted don't post the mount event.
+ post_mount_event = false;
+ _thread->set_preemption_cancelled(false);
+ }
+
+ #if INCLUDE_JVMTI
+ bool is_vthread = Continuation::continuation_scope(_cont.continuation()) == java_lang_VirtualThread::vthread_scope();
+ if (is_vthread) {
+ if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
+ jvmti_mount_end(_thread, _cont, top, mon, post_mount_event);
+ } else {
+ _thread->set_is_in_VTMS_transition(false);
+ java_lang_Thread::set_is_in_VTMS_transition(_thread->vthread(), false);
+ }
+ }
+ #endif
+
+ if (!top_is_interpreted) {
+ assert(ContinuationHelper::Frame::is_stub(top.cb()), "invariant");
+ // The continuation might now run on a different platform thread than the previous time so
+ // we need to adjust the current thread saved in the stub frame before restoring registers.
+ JavaThread** thread_addr = frame::saved_thread_address(top);
+ if (thread_addr != nullptr) *thread_addr = _thread;
+ }
+ sp = push_preempt_rerun_adapter(top, top_is_interpreted /* is_interpreted_frame */);
+ return sp;
+ }
+
NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) {
assert(hf.is_interpreted_frame(), "");
if (UNLIKELY(seen_by_gc())) {
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance);
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
caller = f;
}
void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
! assert(!hf.is_interpreted_frame(), "");
assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption");
if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance);
}
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
caller = f;
}
void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
! assert(hf.is_compiled_frame(), "");
assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption");
if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance);
}
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
assert(caller.sp() == caller.unextended_sp(), "");
if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
! _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in freeze_interpreted_frame
}
// new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
// yet laid out in the stack, and so the original_pc is not stored in it.
// As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
assert(caller.sp() == caller.unextended_sp(), "");
if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
! _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_compiled_frame
}
// new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
// yet laid out in the stack, and so the original_pc is not stored in it.
// As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
maybe_set_fastpath(f.sp());
} else if (_thread->is_interp_only_mode()
|| (_cont.is_preempted() && f.cb()->as_nmethod()->is_marked_for_deoptimization())) {
// The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so
// cannot rely on nmethod patching for deopt.
- assert(_thread->is_interp_only_mode() || stub_caller, "expected a stub-caller");
log_develop_trace(continuations)("Deoptimizing thawed frame");
DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr));
f.deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
caller = f;
}
void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) {
DEBUG_ONLY(_frames++;)
! {
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
_stream.next(&map);
! assert(!_stream.is_done(), "");
- if (UNLIKELY(seen_by_gc())) { // we're now doing this on the stub's caller
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map);
}
! assert(!_stream.is_done(), "");
}
! recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); // this could be deoptimized
!
! DEBUG_ONLY(before_thaw_java_frame(hf, caller, false, num_frames);)
! assert(ContinuationHelper::Frame::is_stub(hf.cb()), "");
assert(caller.sp() == caller.unextended_sp(), "");
! assert(!caller.is_interpreted_frame(), "");
! int fsize = ContinuationHelper::StubFrame::size(hf);
frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false);
intptr_t* stack_frame_top = f.sp();
intptr_t* heap_frame_top = hf.sp();
copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words,
fsize + frame::metadata_words);
! { // can only fix caller once this frame is thawed (due to callee saved regs)
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
! RegisterMap::WalkContinuation::skip); // map.clear();
map.set_include_argument_oops(false);
f.oop_map()->update_register_map(&f, &map);
ContinuationHelper::update_register_map_with_callee(caller, &map);
_cont.tail()->fix_thawed_frame(caller, &map);
}
! DEBUG_ONLY(after_thaw_java_frame(f, false);)
caller = f;
}
void ThawBase::finish_thaw(frame& f) {
stackChunkOop chunk = _cont.tail();
caller = f;
}
void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) {
DEBUG_ONLY(_frames++;)
+ bool is_bottom_frame = false;
! if (UNLIKELY(seen_by_gc())) {
+ // Process the stub's caller here since we might need the full map (if the stub was
+ // generated on a poll on return we shouldn't need a full map).
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
_stream.next(&map);
! if (!_stream.is_done()) {
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map);
}
! } else {
+ _stream.next(SmallRegisterMap::instance);
}
! if (_stream.is_done()) {
! finalize_thaw(caller, 0);
! is_bottom_frame = true;
+ } else {
+ frame f = _stream.to_frame();
+ if (f.is_interpreted_frame()) {
+ recurse_thaw_interpreted_frame(f, caller, num_frames);
+ } else {
+ recurse_thaw_compiled_frame(f, caller, num_frames, true);
+ }
+ }
! assert(!is_bottom_frame || !_cont.is_empty(), "");
assert(caller.sp() == caller.unextended_sp(), "");
! assert(!caller.is_native_frame() || (is_bottom_frame && Continuation::is_continuation_enterSpecial(caller)), "caller should't be native except for enterSpecial case");
! DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
+
+ if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
+ _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_stub_frame
+ }
frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false);
intptr_t* stack_frame_top = f.sp();
intptr_t* heap_frame_top = hf.sp();
+ int fsize = ContinuationHelper::StubFrame::size(hf);
copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words,
fsize + frame::metadata_words);
! patch(f, caller, is_bottom_frame);
+
+ if (!is_bottom_frame) {
+ // can only fix caller once this frame is thawed (due to callee saved regs)
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
! RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
f.oop_map()->update_register_map(&f, &map);
ContinuationHelper::update_register_map_with_callee(caller, &map);
_cont.tail()->fix_thawed_frame(caller, &map);
}
! DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
caller = f;
}
void ThawBase::finish_thaw(frame& f) {
stackChunkOop chunk = _cont.tail();
Thaw<ConfigT> thw(thread, cont);
intptr_t* const sp = thw.thaw(kind);
assert(is_aligned(sp, frame::frame_alignment), "");
- // All the frames have been thawed so we know they don't hold any monitors
- assert(thread->held_monitor_count() == 0, "Must be");
-
#ifdef ASSERT
intptr_t* sp0 = sp;
! set_anchor(thread, sp0);
log_frames(thread);
if (LoomVerifyAfterThaw) {
assert(do_verify_after_thaw(thread, cont.tail(), tty), "");
}
assert(ContinuationEntry::assert_entry_frame_laid_out(thread), "");
Thaw<ConfigT> thw(thread, cont);
intptr_t* const sp = thw.thaw(kind);
assert(is_aligned(sp, frame::frame_alignment), "");
#ifdef ASSERT
intptr_t* sp0 = sp;
! address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset());
+
+ bool sent_entry_pc = false;
+ if (pc0 == Interpreter::cont_preempt_rerun_interpreter_adapter() ||
+ pc0 == StubRoutines::cont_preempt_rerun_compiler_adapter()) {
+ sp0 += frame::metadata_words; // see push_preempt_rerun_adapter
+ #ifdef AARCH64
+ if (pc0 == StubRoutines::cont_preempt_rerun_compiler_adapter()) {
+ address pc1 = *(address*)(sp0 - frame::sender_sp_ret_address_offset());
+ CodeBlob* cb = CodeCache::find_blob(pc1);
+ assert(cb != nullptr, "should be either c1 or c2 runtime stub");
+ if (cb->frame_size() == 2) {
+ sp0 += frame::metadata_words;
+ }
+ }
+ #endif
+ } else if (pc0 == StubRoutines::cont_preempt_monitorenter_redo()) {
+ sp0 += 2 * frame::metadata_words; // see push_preempt_monitorenter_redo
+ sent_entry_pc = true;
+ }
+ set_anchor(thread, sp0, sent_entry_pc ? cont.entryPC() : nullptr);
log_frames(thread);
if (LoomVerifyAfterThaw) {
assert(do_verify_after_thaw(thread, cont.tail(), tty), "");
}
assert(ContinuationEntry::assert_entry_frame_laid_out(thread), "");
}
return true;
}
static void log_frames(JavaThread* thread) {
! const static int show_entry_callers = 3;
LogTarget(Trace, continuations) lt;
if (!lt.develop_is_enabled()) {
return;
}
LogStream ls(lt);
! ls.print_cr("------- frames ---------");
if (!thread->has_last_Java_frame()) {
ls.print_cr("NO ANCHOR!");
}
RegisterMap map(thread,
}
return true;
}
static void log_frames(JavaThread* thread) {
! const static int show_entry_callers = 100;
LogTarget(Trace, continuations) lt;
if (!lt.develop_is_enabled()) {
return;
}
LogStream ls(lt);
! ls.print_cr("------- frames --------- for thread " INTPTR_FORMAT, p2i(thread));
if (!thread->has_last_Java_frame()) {
ls.print_cr("NO ANCHOR!");
}
RegisterMap map(thread,
HandleMark hm(Thread::current());
FrameValues values;
int i = 0;
int post_entry = -1;
! for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) {
! f.describe(values, i++, &map);
if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f))
post_entry++;
if (post_entry >= show_entry_callers)
break;
}
HandleMark hm(Thread::current());
FrameValues values;
int i = 0;
int post_entry = -1;
! for (frame f = thread->last_frame(); !f.is_first_frame(); f = f.sender(&map), i++) {
! f.describe(values, i, &map, i == 0);
if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f))
post_entry++;
if (post_entry >= show_entry_callers)
break;
}
map.set_include_argument_oops(false);
map.set_skip_missing(true);
if (callee_complete) {
frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f));
}
! const_cast<frame&>(f).describe(values, 0, &map);
values.print_on(static_cast<JavaThread*>(nullptr), st);
}
#endif
static address thaw_entry = nullptr;
static address freeze_entry = nullptr;
address Continuation::thaw_entry() {
return ::thaw_entry;
}
address Continuation::freeze_entry() {
return ::freeze_entry;
}
class ConfigResolve {
public:
static void resolve() { resolve_compressed(); }
static void resolve_compressed() {
map.set_include_argument_oops(false);
map.set_skip_missing(true);
if (callee_complete) {
frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f));
}
! const_cast<frame&>(f).describe(values, 0, &map, true);
values.print_on(static_cast<JavaThread*>(nullptr), st);
}
#endif
static address thaw_entry = nullptr;
static address freeze_entry = nullptr;
+ static address freeze_preempt_entry = nullptr;
address Continuation::thaw_entry() {
return ::thaw_entry;
}
address Continuation::freeze_entry() {
return ::freeze_entry;
}
+ address Continuation::freeze_preempt_entry() {
+ return ::freeze_preempt_entry;
+ }
+
class ConfigResolve {
public:
static void resolve() { resolve_compressed(); }
static void resolve_compressed() {
template <bool use_compressed, typename BarrierSetT>
static void resolve() {
typedef Config<use_compressed ? oop_kind::NARROW : oop_kind::WIDE, BarrierSetT> SelectedConfigT;
freeze_entry = (address)freeze<SelectedConfigT>;
+ freeze_preempt_entry = (address)SelectedConfigT::freeze_preempt;
// If we wanted, we could templatize by kind and have three different thaw entries
thaw_entry = (address)thaw<SelectedConfigT>;
}
};
< prev index next >