< prev index next > src/hotspot/share/runtime/continuationFreezeThaw.cpp
Print this page
#include "runtime/frame.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaThread.inline.hpp"
#include "runtime/jniHandles.inline.hpp"
#include "runtime/keepStackGCProcessed.hpp"
+ #include "runtime/objectMonitor.inline.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/smallRegisterMap.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stackChunkFrameStream.inline.hpp"
#include "runtime/stackOverflow.hpp"
#include "runtime/stackWatermarkSet.inline.hpp"
#include "utilities/debug.hpp"
#include "utilities/exceptions.hpp"
#include "utilities/macros.hpp"
+ #include "utilities/vmError.hpp"
#if INCLUDE_ZGC
#include "gc/z/zStackChunkGCData.inline.hpp"
#endif
#include <type_traits>
static void verify_continuation(oop continuation) { Continuation::debug_verify_continuation(continuation); }
static void do_deopt_after_thaw(JavaThread* thread);
static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st);
static void log_frames(JavaThread* thread);
+ static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted);
static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st = tty);
#define assert_pfl(p, ...) \
do { \
if (!(p)) { \
#else
static void verify_continuation(oop continuation) { }
#define assert_pfl(p, ...)
#endif
- // should match Continuation.preemptStatus() in Continuation.java
- enum freeze_result {
- freeze_ok = 0,
- freeze_ok_bottom = 1,
- freeze_pinned_cs = 2,
- freeze_pinned_native = 3,
- freeze_pinned_monitor = 4,
- freeze_exception = 5
- };
-
- const char* freeze_result_names[6] = {
- "freeze_ok",
- "freeze_ok_bottom",
- "freeze_pinned_cs",
- "freeze_pinned_native",
- "freeze_pinned_monitor",
- "freeze_exception"
- };
-
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint);
! template<typename ConfigT> static inline int freeze_internal(JavaThread* current, intptr_t* const sp);
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier);
template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind);
#else
static void verify_continuation(oop continuation) { }
#define assert_pfl(p, ...)
#endif
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint);
! template<typename ConfigT, bool preempt> static inline int freeze_internal(JavaThread* current, intptr_t* const sp);
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier);
template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind);
template<typename ConfigT>
static JRT_LEAF(intptr_t*, thaw(JavaThread* thread, int kind))
// TODO: JRT_LEAF and NoHandleMark is problematic for JFR events.
// vFrameStreamCommon allocates Handles in RegisterMap for continuations.
+ // Also the preemption case with JVMTI events enabled might safepoint so
+ // undo the NoSafepointVerifier here and rely on handling by ContinuationWrapper.
// JRT_ENTRY instead?
ResetNoHandleMark rnhm;
+ DEBUG_ONLY(PauseNoSafepointVerifier pnsv(&__nsv);)
// we might modify the code cache via BarrierSetNMethod::nmethod_entry_barrier
MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread));
return ConfigT::thaw(thread, (Continuation::thaw_kind)kind);
JRT_END
public:
typedef Config<oops, BarrierSetT> SelfT;
using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>;
static int freeze(JavaThread* thread, intptr_t* const sp) {
! return freeze_internal<SelfT>(thread, sp);
}
static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) {
return thaw_internal<SelfT>(thread, kind);
}
public:
typedef Config<oops, BarrierSetT> SelfT;
using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>;
static int freeze(JavaThread* thread, intptr_t* const sp) {
! return freeze_internal<SelfT, false>(thread, sp);
+ }
+
+ static int freeze_preempt(JavaThread* thread, intptr_t* const sp) {
+ return freeze_internal<SelfT, true>(thread, sp);
}
static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) {
return thaw_internal<SelfT>(thread, kind);
}
static oop get_continuation(JavaThread* thread) {
assert(thread != nullptr, "");
assert(thread->threadObj() != nullptr, "");
return java_lang_Thread::continuation(thread->threadObj());
}
inline void clear_anchor(JavaThread* thread) {
thread->frame_anchor()->clear();
}
! static void set_anchor(JavaThread* thread, intptr_t* sp) {
! address pc = ContinuationHelper::return_address_at(
! sp - frame::sender_sp_ret_address_offset());
assert(pc != nullptr, "");
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(sp);
anchor->set_last_Java_pc(pc);
ContinuationHelper::set_anchor_pd(anchor, sp);
assert(thread->has_last_Java_frame(), "");
assert(thread->last_frame().cb() != nullptr, "");
}
- #endif // ASSERT
static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(entry->entry_sp());
anchor->set_last_Java_pc(entry->entry_pc());
static oop get_continuation(JavaThread* thread) {
assert(thread != nullptr, "");
assert(thread->threadObj() != nullptr, "");
return java_lang_Thread::continuation(thread->threadObj());
}
+ #endif // ASSERT
inline void clear_anchor(JavaThread* thread) {
thread->frame_anchor()->clear();
}
! static void set_anchor(JavaThread* thread, intptr_t* sp, address pc = nullptr) {
! if (pc == nullptr) {
! pc = ContinuationHelper::return_address_at(
+ sp - frame::sender_sp_ret_address_offset());
+ }
assert(pc != nullptr, "");
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(sp);
anchor->set_last_Java_pc(pc);
ContinuationHelper::set_anchor_pd(anchor, sp);
assert(thread->has_last_Java_frame(), "");
assert(thread->last_frame().cb() != nullptr, "");
}
static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
JavaFrameAnchor* anchor = thread->frame_anchor();
anchor->set_last_Java_sp(entry->entry_sp());
anchor->set_last_Java_pc(entry->entry_pc());
class FreezeBase : public StackObj {
protected:
JavaThread* const _thread;
ContinuationWrapper& _cont;
bool _barriers; // only set when we allocate a chunk
- const bool _preempt; // used only on the slow path
- const intptr_t * const _frame_sp; // Top frame sp for this freeze
intptr_t* _bottom_address;
int _freeze_size; // total size of all frames plus metadata in words.
int _total_align_size;
intptr_t* _cont_stack_top;
intptr_t* _cont_stack_bottom;
class FreezeBase : public StackObj {
protected:
JavaThread* const _thread;
ContinuationWrapper& _cont;
bool _barriers; // only set when we allocate a chunk
intptr_t* _bottom_address;
+ // Used for preemption only
+ const bool _preempt;
+ frame _last_frame;
+
+ // Used to support freezing with held monitors
+ int _monitors_in_lockstack;
+
int _freeze_size; // total size of all frames plus metadata in words.
int _total_align_size;
intptr_t* _cont_stack_top;
intptr_t* _cont_stack_bottom;
JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector;
NOT_PRODUCT(int _frames;)
DEBUG_ONLY(intptr_t* _last_write;)
! inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp);
public:
NOINLINE freeze_result freeze_slow();
void freeze_fast_existing_chunk();
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; }
inline int size_if_fast_freeze_available();
#ifdef ASSERT
bool check_valid_fast_path();
#endif
protected:
JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector;
NOT_PRODUCT(int _frames;)
DEBUG_ONLY(intptr_t* _last_write;)
! inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempt);
public:
NOINLINE freeze_result freeze_slow();
void freeze_fast_existing_chunk();
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; }
inline int size_if_fast_freeze_available();
+ inline frame& last_frame() { return _last_frame; }
+
#ifdef ASSERT
bool check_valid_fast_path();
#endif
protected:
int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); }
private:
// slow path
frame freeze_start_frame();
! frame freeze_start_frame_safepoint_stub(frame f);
NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top);
! inline frame freeze_start_frame_yield_stub(frame f);
template<typename FKind>
inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame);
NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
NOINLINE void finish_freeze(const frame& f, const frame& top);
inline bool stack_overflow();
static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
: sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
template<typename FKind> static inline frame sender(const frame& f);
template<typename FKind> frame new_heap_frame(frame& f, frame& caller);
inline void set_top_frame_metadata_pd(const frame& hf);
inline void patch_pd(frame& callee, const frame& caller);
void adjust_interpreted_frame_unextended_sp(frame& f);
static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
protected:
void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
bool freeze_fast_new_chunk(stackChunkOop chunk);
int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); }
private:
// slow path
frame freeze_start_frame();
! frame freeze_start_frame_on_preempt();
NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top);
! inline frame freeze_start_frame_yield_stub();
template<typename FKind>
inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame);
NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
+ NOINLINE freeze_result recurse_freeze_native_frame(frame& f, frame& caller);
NOINLINE void finish_freeze(const frame& f, const frame& top);
+ void freeze_lockstack(stackChunkOop chunk);
+
inline bool stack_overflow();
static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
: sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
template<typename FKind> static inline frame sender(const frame& f);
template<typename FKind> frame new_heap_frame(frame& f, frame& caller);
inline void set_top_frame_metadata_pd(const frame& hf);
inline void patch_pd(frame& callee, const frame& caller);
void adjust_interpreted_frame_unextended_sp(frame& f);
+ static inline void prepare_freeze_interpreted_top_frame(frame& f);
static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
protected:
void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
bool freeze_fast_new_chunk(stackChunkOop chunk);
class Freeze : public FreezeBase {
private:
stackChunkOop allocate_chunk(size_t stack_size, int argsize_md);
public:
! inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp)
! : FreezeBase(thread, cont, frame_sp) {}
freeze_result try_freeze_fast();
protected:
virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) override { return allocate_chunk(stack_size, argsize_md); }
};
! FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) :
! _thread(thread), _cont(cont), _barriers(false), _preempt(false), _frame_sp(frame_sp) {
DEBUG_ONLY(_jvmti_event_collector = nullptr;)
assert(_thread != nullptr, "");
assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), "");
class Freeze : public FreezeBase {
private:
stackChunkOop allocate_chunk(size_t stack_size, int argsize_md);
public:
! inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt)
! : FreezeBase(thread, cont, frame_sp, preempt) {}
freeze_result try_freeze_fast();
protected:
virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) override { return allocate_chunk(stack_size, argsize_md); }
};
! FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt) :
! _thread(thread), _cont(cont), _barriers(false), _preempt(preempt), _last_frame(false /* no initialization */) {
DEBUG_ONLY(_jvmti_event_collector = nullptr;)
assert(_thread != nullptr, "");
assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), "");
#if !defined(PPC64) || defined(ZERO)
static const int doYield_stub_frame_size = frame::metadata_words;
#else
static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord;
#endif
! assert(SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, "");
// properties of the continuation on the stack; all sizes are in words
! _cont_stack_top = frame_sp + doYield_stub_frame_size; // we don't freeze the doYield stub frame
_cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0)
- ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw
log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
assert(cont_size() > 0, "");
}
void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling
_freeze_size = 0;
_total_align_size = 0;
NOT_PRODUCT(_frames = 0;)
}
void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) {
stackChunkOop chunk = _cont.tail();
chunk->copy_from_stack_to_chunk(from, to, size);
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
#if !defined(PPC64) || defined(ZERO)
static const int doYield_stub_frame_size = frame::metadata_words;
#else
static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord;
#endif
! // With preemption doYield() might not have been resolved yet
+ assert(_preempt || SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, "");
+
+ if (preempt) {
+ _last_frame = _thread->last_frame();
+ }
// properties of the continuation on the stack; all sizes are in words
! _cont_stack_top = frame_sp + (!preempt ? doYield_stub_frame_size : 0); // we don't freeze the doYield stub frame
_cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0)
- ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw
log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
assert(cont_size() > 0, "");
+
+ if (LockingMode != LM_LIGHTWEIGHT) {
+ _monitors_in_lockstack = 0;
+ } else {
+ _monitors_in_lockstack = _thread->lock_stack().monitor_count();
+ }
}
void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling
_freeze_size = 0;
_total_align_size = 0;
NOT_PRODUCT(_frames = 0;)
}
+ void FreezeBase::freeze_lockstack(stackChunkOop chunk) {
+ assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "no room for lockstack");
+
+ _thread->lock_stack().move_to_address((oop*)chunk->start_address());
+ chunk->set_lockstack_size(checked_cast<uint8_t>(_monitors_in_lockstack));
+ chunk->set_has_lockstack(true);
+ }
+
void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) {
stackChunkOop chunk = _cont.tail();
chunk->copy_from_stack_to_chunk(from, to, size);
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
assert(_thread->cont_fastpath(), "");
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
assert(_fast_freeze_size == 0, "");
! stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words, _cont.argsize() + frame::metadata_words_at_top);
if (freeze_fast_new_chunk(chunk)) {
return freeze_ok;
}
if (_thread->has_pending_exception()) {
return freeze_exception;
assert(_thread->cont_fastpath(), "");
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
assert(_fast_freeze_size == 0, "");
! stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words + _monitors_in_lockstack, _cont.argsize() + frame::metadata_words_at_top);
if (freeze_fast_new_chunk(chunk)) {
return freeze_ok;
}
if (_thread->has_pending_exception()) {
return freeze_exception;
// although that would require changing stackChunkOopDesc::is_empty
if (!chunk->is_empty()) {
total_size_needed -= _cont.argsize() + frame::metadata_words_at_top;
}
+ total_size_needed += _monitors_in_lockstack;
+
int chunk_free_room = chunk_sp - frame::metadata_words_at_bottom;
bool available = chunk_free_room >= total_size_needed;
log_develop_trace(continuations)("chunk available: %s size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
available ? "yes" : "no" , total_size_needed, _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
return available ? total_size_needed : 0;
chunk->set_max_thawing_size(cont_size());
// in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments.
// They'll then be stored twice: in the chunk and in the parent chunk's top frame
! const int chunk_start_sp = cont_size() + frame::metadata_words;
assert(chunk_start_sp == chunk->stack_size(), "");
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true));
chunk->set_max_thawing_size(cont_size());
// in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments.
// They'll then be stored twice: in the chunk and in the parent chunk's top frame
! const int chunk_start_sp = cont_size() + frame::metadata_words + _monitors_in_lockstack;
assert(chunk_start_sp == chunk->stack_size(), "");
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true));
p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize());
assert(chunk_start_sp <= chunk->stack_size(), "");
assert(chunk_start_sp >= cont_size(), "no room in the chunk");
const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze
! assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, "");
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
#ifdef ASSERT
if (!_empty) {
intptr_t* retaddr_slot = (_orig_chunk_sp
p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize());
assert(chunk_start_sp <= chunk->stack_size(), "");
assert(chunk_start_sp >= cont_size(), "no room in the chunk");
const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze
! assert(!(_fast_freeze_size > 0) || (_orig_chunk_sp - (chunk->start_address() + chunk_new_sp)) == (_fast_freeze_size - _monitors_in_lockstack), "");
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
#ifdef ASSERT
if (!_empty) {
intptr_t* retaddr_slot = (_orig_chunk_sp
ContinuationHelper::patch_return_address_at(chunk_bottom_retaddr_slot,
chunk->pc());
// We're always writing to a young chunk, so the GC can't see it until the next safepoint.
chunk->set_sp(chunk_new_sp);
// set chunk->pc to the return address of the topmost frame in the chunk
! chunk->set_pc(ContinuationHelper::return_address_at(
_cont_stack_top - frame::sender_sp_ret_address_offset()));
_cont.write();
log_develop_trace(continuations)("FREEZE CHUNK #" INTPTR_FORMAT " (young)", _cont.hash());
LogTarget(Trace, continuations) lt;
ContinuationHelper::patch_return_address_at(chunk_bottom_retaddr_slot,
chunk->pc());
// We're always writing to a young chunk, so the GC can't see it until the next safepoint.
chunk->set_sp(chunk_new_sp);
+
// set chunk->pc to the return address of the topmost frame in the chunk
! if (_preempt) {
+ // On aarch64/riscv64, the return pc of the top frame won't necessarily be at sp[-1].
+ // Also, on x64, if the top frame is the native wrapper frame, sp[-1] will not
+ // be the pc we used when creating the oopmap. Get the top's frame last pc from
+ // the anchor instead.
+ address last_pc = _last_frame.pc();
+ ContinuationHelper::patch_return_address_at(chunk_top - frame::sender_sp_ret_address_offset(), last_pc);
+ chunk->set_pc(last_pc);
+ } else {
+ chunk->set_pc(ContinuationHelper::return_address_at(
_cont_stack_top - frame::sender_sp_ret_address_offset()));
+ }
+
+ if (_monitors_in_lockstack > 0) {
+ freeze_lockstack(chunk);
+ }
_cont.write();
log_develop_trace(continuations)("FREEZE CHUNK #" INTPTR_FORMAT " (young)", _cont.hash());
LogTarget(Trace, continuations) lt;
return res;
}
frame FreezeBase::freeze_start_frame() {
- frame f = _thread->last_frame();
if (LIKELY(!_preempt)) {
! return freeze_start_frame_yield_stub(f);
} else {
! return freeze_start_frame_safepoint_stub(f);
}
}
! frame FreezeBase::freeze_start_frame_yield_stub(frame f) {
assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be");
f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f);
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
return f;
}
! frame FreezeBase::freeze_start_frame_safepoint_stub(frame f) {
! #if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
! f.set_fp(f.real_fp()); // f.set_fp(*Frame::callee_link_address(f)); // ????
! #else
- Unimplemented();
- #endif
- if (!Interpreter::contains(f.pc())) {
- assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be");
- assert(f.oop_map() != nullptr, "must be");
-
- if (Interpreter::contains(ContinuationHelper::StubFrame::return_pc(f))) {
- f = sender<ContinuationHelper::StubFrame>(f); // Safepoint stub in interpreter
- }
- }
- assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
- return f;
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) {
assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame
! assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())), "");
if (stack_overflow()) {
return freeze_exception;
}
return res;
}
frame FreezeBase::freeze_start_frame() {
if (LIKELY(!_preempt)) {
! return freeze_start_frame_yield_stub();
} else {
! return freeze_start_frame_on_preempt();
}
}
! frame FreezeBase::freeze_start_frame_yield_stub() {
+ frame f = _thread->last_frame();
assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be");
f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f);
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
return f;
}
! frame FreezeBase::freeze_start_frame_on_preempt() {
! assert(_last_frame.sp() == _thread->last_frame().sp(), "_last_frame should be already initialized");
! assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), _last_frame), "");
! return _last_frame;
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) {
assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame
! assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb()))
+ || ((top && _preempt) == f.is_native_frame()), "");
if (stack_overflow()) {
return freeze_exception;
}
// special native frame
return freeze_pinned_native;
}
return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted);
} else if (f.is_interpreted_frame()) {
! assert((_preempt && top) || !f.interpreter_frame_method()->is_native(), "");
- if (_preempt && top && f.interpreter_frame_method()->is_native()) {
- // int native entry
- return freeze_pinned_native;
- }
-
return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted);
! } else if (_preempt && top && ContinuationHelper::Frame::is_stub(f.cb())) {
! return recurse_freeze_stub_frame(f, caller);
} else {
return freeze_pinned_native;
}
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
// special native frame
return freeze_pinned_native;
}
return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted);
} else if (f.is_interpreted_frame()) {
! assert(!f.interpreter_frame_method()->is_native() || (top && _preempt), "");
return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted);
! } else if (top && _preempt) {
! assert(f.is_native_frame() || f.is_runtime_frame(), "");
+ return f.is_native_frame() ? recurse_freeze_native_frame(f, caller) : recurse_freeze_stub_frame(f, caller);
} else {
+ // frame can't be freezed. Most likely the call_stub or upcall_stub
+ // which indicates there are further natives frames up the stack.
return freeze_pinned_native;
}
}
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
// The parameter argsize_md includes metadata that has to be part of caller/callee overlap.
// See also StackChunkFrameStream<frame_kind>::frame_size()
freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, int argsize_md) {
int argsize = argsize_md - frame::metadata_words_at_top;
assert(callee.is_interpreted_frame()
+ || ContinuationHelper::Frame::is_stub(callee.cb())
|| callee.cb()->as_nmethod()->is_osr_method()
|| argsize == _cont.argsize(), "argsize: %d cont.argsize: %d", argsize, _cont.argsize());
log_develop_trace(continuations)("bottom: " INTPTR_FORMAT " count %d size: %d argsize: %d",
p2i(_bottom_address), _frames, _freeze_size << LogBytesPerWord, argsize);
assert(chunk == nullptr || chunk->is_empty()
|| unextended_sp == chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp()), "");
assert(chunk != nullptr || unextended_sp < _freeze_size, "");
+ _freeze_size += _monitors_in_lockstack;
+
// _barriers can be set to true by an allocation in freeze_fast, in which case the chunk is available
bool allocated_old_in_freeze_fast = _barriers;
assert(!allocated_old_in_freeze_fast || (unextended_sp >= _freeze_size && chunk->is_empty()),
"Chunk allocated in freeze_fast is of insufficient size "
"unextended_sp: %d size: %d is_empty: %d", unextended_sp, _freeze_size, chunk->is_empty());
assert(!_barriers || chunk->is_empty(), "");
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), "");
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), "");
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
// will either see no continuation or a consistent chunk.
unwind_frames();
! chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - frame::metadata_words);
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top chunk:");
chunk->print_on(&ls);
}
// The topmost existing frame in the chunk; or an empty frame if the chunk is empty
caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame();
DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);)
assert(!_barriers || chunk->is_empty(), "");
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), "");
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), "");
+ if (_preempt) {
+ frame f = _thread->last_frame();
+ if (f.is_interpreted_frame()) {
+ // Some platforms do not save the last_sp in the top interpreter frame on VM calls.
+ // We need it so that on resume we can restore the sp to the right place, since
+ // thawing might add an alignment word to the expression stack (see finish_thaw()).
+ // We do it now that we know freezing will be successful.
+ prepare_freeze_interpreted_top_frame(f);
+ }
+ }
+
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
// will either see no continuation or a consistent chunk.
unwind_frames();
! chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - _monitors_in_lockstack - frame::metadata_words);
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top chunk:");
chunk->print_on(&ls);
}
+ if (_monitors_in_lockstack > 0) {
+ freeze_lockstack(chunk);
+ }
+
// The topmost existing frame in the chunk; or an empty frame if the chunk is empty
caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame();
DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);)
assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), "");
frame entry = sender(callee);
! assert(Continuation::is_return_barrier_entry(entry.pc()) || Continuation::is_continuation_enterSpecial(entry), "");
assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
#endif
return freeze_ok_bottom;
}
void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) {
if (is_bottom_frame) {
// If we're the bottom frame, we need to replace the return barrier with the real
// caller's pc.
address last_pc = caller.pc();
assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), "");
frame entry = sender(callee);
! assert((!empty && Continuation::is_return_barrier_entry(entry.pc())) || (empty && Continuation::is_continuation_enterSpecial(entry)), "");
assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
#endif
return freeze_ok_bottom;
}
+ // After freezing a frame we need to possibly adjust some values related to the caller frame.
void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) {
if (is_bottom_frame) {
// If we're the bottom frame, we need to replace the return barrier with the real
// caller's pc.
address last_pc = caller.pc();
Method* frame_method = ContinuationHelper::Frame::frame_method(f);
// including metadata between f and its args
const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top;
! log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d",
! frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize);
// we'd rather not yield inside methods annotated with @JvmtiMountTransition
assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
Method* frame_method = ContinuationHelper::Frame::frame_method(f);
// including metadata between f and its args
const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top;
! log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d callee_interpreted: %d",
! frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize, callee_interpreted);
// we'd rather not yield inside methods annotated with @JvmtiMountTransition
assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
if (caller.is_interpreted_frame()) {
! _total_align_size += frame::align_wiggle; // See Thaw::align
}
patch(f, hf, caller, is_bottom_frame);
assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
if (caller.is_interpreted_frame()) {
! // When thawing the frame we might need to add alignment (see Thaw::align)
+ _total_align_size += frame::align_wiggle;
}
patch(f, hf, caller, is_bottom_frame);
assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
caller = hf;
return freeze_ok;
}
NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) {
! intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0);
const int fsize = f.cb()->frame_size();
log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
! // recurse_freeze_java_frame and freeze inlined here because we need to use a full RegisterMap for lock ownership
! NOT_PRODUCT(_frames++;)
! _freeze_size += fsize;
! RegisterMap map(_cont.thread(),
! RegisterMap::UpdateMap::include,
! RegisterMap::ProcessFrames::skip,
! RegisterMap::WalkContinuation::skip);
! map.set_include_argument_oops(false);
! ContinuationHelper::update_register_map<ContinuationHelper::StubFrame>(f, &map);
! f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case
! frame senderf = sender<ContinuationHelper::StubFrame>(f);
! assert(senderf.unextended_sp() < _bottom_address - 1, "");
! assert(senderf.is_compiled_frame(), "");
!
! if (UNLIKELY(senderf.oop_map() == nullptr)) {
! // native frame
return freeze_pinned_native;
}
! freeze_result result = recurse_freeze_compiled_frame(senderf, caller, 0, 0); // This might be deoptimized
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
- assert(result != freeze_ok_bottom, "");
- assert(!caller.is_interpreted_frame(), "");
! DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false);)
! frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller);
! intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0);
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
! DEBUG_ONLY(after_freeze_java_frame(hf, false);)
caller = hf;
return freeze_ok;
}
caller = hf;
return freeze_ok;
}
NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) {
! DEBUG_ONLY(frame fsender = sender(f);)
+ assert(fsender.is_compiled_frame(), "sender should be compiled frame");
+
+ intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f);
const int fsize = f.cb()->frame_size();
log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
! freeze_result result = recurse_freeze_java_frame<ContinuationHelper::StubFrame>(f, caller, fsize, 0);
! if (UNLIKELY(result > freeze_ok_bottom)) {
! return result;
+ }
! assert(result == freeze_ok, "should have caller");
! DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false /*is_bottom_frame*/);)
!
! frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller);
! intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf);
!
! copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
!
! patch(f, hf, caller, false /*is_bottom_frame*/);
!
! DEBUG_ONLY(after_freeze_java_frame(hf, false /*is_bottom_frame*/);)
!
! caller = hf;
+ return freeze_ok;
+ }
+
+ NOINLINE freeze_result FreezeBase::recurse_freeze_native_frame(frame& f, frame& caller) {
+ if (!f.cb()->as_nmethod()->method()->is_object_wait0()) {
+ assert(f.cb()->as_nmethod()->method()->is_synchronized(), "");
+ // Synchronized native method case. Unlike the interpreter native wrapper, the compiled
+ // native wrapper tries to acquire the monitor after marshalling the arguments from the
+ // caller into the native convention. This is so that we have a valid oopMap in case of
+ // having to block in the slow path. But that would require freezing those registers too
+ // and then fixing them back on thaw in case of oops. To avoid complicating things and
+ // given that this would be a rare case anyways just pin the vthread to the carrier.
return freeze_pinned_native;
}
! intptr_t* const stack_frame_top = ContinuationHelper::NativeFrame::frame_top(f);
+ // There are no stackargs but argsize must include the metadata
+ const int argsize = frame::metadata_words_at_top;
+ const int fsize = f.cb()->frame_size() + argsize;
+
+ log_develop_trace(continuations)("recurse_freeze_native_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
+ f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
+
+ freeze_result result = recurse_freeze_java_frame<ContinuationHelper::NativeFrame>(f, caller, fsize, argsize);
if (UNLIKELY(result > freeze_ok_bottom)) {
return result;
}
! assert(result == freeze_ok, "should have caller frame");
! DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, false /* is_bottom_frame */);)
!
+ frame hf = new_heap_frame<ContinuationHelper::NativeFrame>(f, caller);
+ intptr_t* heap_frame_top = ContinuationHelper::NativeFrame::frame_top(hf);
+
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
!
+ if (caller.is_interpreted_frame()) {
+ // When thawing the frame we might need to add alignment (see Thaw::align)
+ _total_align_size += frame::align_wiggle;
+ }
+
+ patch(f, hf, caller, false /* is_bottom_frame */);
+
+ DEBUG_ONLY(after_freeze_java_frame(hf, false /* is_bottom_frame */);)
caller = hf;
return freeze_ok;
}
chunk->set_sp(chunk->to_offset(top.sp()));
chunk->set_pc(top.pc());
chunk->set_max_thawing_size(chunk->max_thawing_size() + _total_align_size);
+ assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "clash with lockstack");
+
// At this point the chunk is consistent
if (UNLIKELY(_barriers)) {
log_develop_trace(continuations)("do barriers on old chunk");
// Serial and Parallel GC can allocate objects directly into the old generation.
if (lt.develop_is_enabled()) {
LogStream ls(lt);
ls.print_cr("top hframe after (freeze):");
assert(_cont.last_frame().is_heap_frame(), "should be");
_cont.last_frame().print_on(&ls);
+ DEBUG_ONLY(print_frame_layout(top, false, &ls);)
}
assert(_cont.chunk_invariant(), "");
}
assert(_stack_size <= max_jint, "");
assert(_word_size > _stack_size, "");
// zero out fields (but not the stack)
const size_t hs = oopDesc::header_size();
+ oopDesc::set_klass_gap(mem, 0);
Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs);
int bottom = (int)_stack_size - _argsize_md;
jdk_internal_vm_StackChunk::set_size(mem, (int)_stack_size);
assert(chunk->max_thawing_size() == 0, "");
assert(chunk->pc() == nullptr, "");
assert(chunk->is_empty(), "");
assert(chunk->flags() == 0, "");
assert(chunk->is_gc_mode() == false, "");
+ assert(chunk->lockstack_size() == 0, "");
+ assert(chunk->object_waiter() == nullptr, "");
// fields are uninitialized
chunk->set_parent_access<IS_DEST_UNINITIALIZED>(_cont.last_nonempty_chunk());
chunk->set_cont_access<IS_DEST_UNINITIALIZED>(_cont.continuation());
ContinuationWrapper::SafepointOp so(Thread::current(), cont);
JvmtiExport::continuation_yield_cleanup(JavaThread::current(), num_frames);
}
invalidate_jvmti_stack(thread);
}
+
+ static void jvmti_mount_end(JavaThread* current, ContinuationWrapper& cont, frame top) {
+ assert(current->vthread() != nullptr, "must be");
+
+ HandleMarkCleaner hm(current);
+ Handle vth(current, current->vthread());
+
+ ContinuationWrapper::SafepointOp so(current, cont);
+
+ // Since we might safepoint set the anchor so that the stack can be walked.
+ set_anchor(current, top.sp());
+
+ JRT_BLOCK
+ current->rebind_to_jvmti_thread_state_of(vth());
+ JvmtiVTMSTransitionDisabler::finish_VTMS_transition((jthread)vth.raw_value(), /* is_mount */ true);
+
+ // If pending_jvmti_unmount_event() is true here we are in the preemption
+ // cancelled case. Since we never unmounted we don't post the mount event
+ // and just clear the pending unmount flag.
+ if (current->pending_jvmti_unmount_event()) {
+ current->set_pending_jvmti_unmount_event(false);
+ } else if (JvmtiExport::should_post_vthread_mount()) {
+ JvmtiExport::post_vthread_mount((jthread)vth.raw_value());
+ }
+
+ if (current->pending_contended_entered_event()) {
+ JvmtiExport::post_monitor_contended_entered(current, current->contended_entered_monitor());
+ current->set_contended_entered_monitor(nullptr);
+ }
+ JRT_BLOCK_END
+
+ clear_anchor(current);
+ }
#endif // INCLUDE_JVMTI
#ifdef ASSERT
static bool monitors_on_stack(JavaThread* thread) {
ContinuationEntry* ce = thread->last_continuation();
RegisterMap::ProcessFrames::include,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) ||
! (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f))) {
return true;
}
}
return false;
}
RegisterMap::ProcessFrames::include,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) ||
! (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f)) ||
+ (f.is_native_frame() && ContinuationHelper::NativeFrame::is_owning_locks(map.thread(), f))) {
return true;
}
}
return false;
}
RegisterMap map(_thread,
RegisterMap::UpdateMap::skip,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
! for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
! if (!f.is_compiled_frame() || f.is_deoptimized_frame()) {
return false;
}
}
return true;
}
#endif // ASSERT
! static inline int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont) {
verify_continuation(cont.continuation());
assert(!cont.is_empty(), "");
- // This is done for the sake of the enterSpecial frame
- StackWatermarkSet::after_unwind(thread);
log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash());
-
return 0;
}
static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) {
if (UNLIKELY(res != freeze_ok)) {
RegisterMap map(_thread,
RegisterMap::UpdateMap::skip,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
! int i = 0;
! for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map), i++) {
+ if (!((f.is_compiled_frame() && !f.is_deoptimized_frame()) || (i == 0 && (f.is_runtime_frame() || f.is_native_frame())))) {
return false;
}
}
return true;
}
#endif // ASSERT
! static inline int freeze_epilog(ContinuationWrapper& cont) {
verify_continuation(cont.continuation());
assert(!cont.is_empty(), "");
log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash());
return 0;
}
static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) {
if (UNLIKELY(res != freeze_ok)) {
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint
! return freeze_epilog(thread, cont);
}
! template<typename ConfigT>
static inline int freeze_internal(JavaThread* current, intptr_t* const sp) {
assert(!current->has_pending_exception(), "");
#ifdef ASSERT
! log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()));
log_frames(current);
#endif
CONT_JFR_ONLY(EventContinuationFreeze event;)
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
return res;
}
JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint
! return freeze_epilog(cont);
}
! static int preempt_epilog(ContinuationWrapper& cont, freeze_result res, frame& old_last_frame) {
+ if (UNLIKELY(res != freeze_ok)) {
+ verify_continuation(cont.continuation());
+ log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
+ return res;
+ }
+
+ patch_return_pc_with_preempt_stub(old_last_frame);
+ cont.tail()->set_preempted(true);
+
+ return freeze_epilog(cont);
+ }
+
+ template<typename ConfigT, bool preempt>
static inline int freeze_internal(JavaThread* current, intptr_t* const sp) {
assert(!current->has_pending_exception(), "");
#ifdef ASSERT
! log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT "JavaThread: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()), p2i(current));
log_frames(current);
#endif
CONT_JFR_ONLY(EventContinuationFreeze event;)
ContinuationWrapper cont(current, oopCont);
log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), "");
! assert(monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0),
"Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count());
if (entry->is_pinned() || current->held_monitor_count() > 0) {
log_develop_debug(continuations)("PINNED due to critical section/hold monitor");
verify_continuation(cont.continuation());
freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor;
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
// Avoid Thread.yield() loops without safepoint polls.
! if (SafepointMechanism::should_process(current)) {
cont.done(); // allow safepoint
ThreadInVMfromJava tivmfj(current);
}
return res;
}
! Freeze<ConfigT> freeze(current, cont, sp);
assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), "");
bool fast = UseContinuationFastPath && current->cont_fastpath();
if (fast && freeze.size_if_fast_freeze_available() > 0) {
freeze.freeze_fast_existing_chunk();
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
! freeze_epilog(current, cont);
! return 0;
}
log_develop_trace(continuations)("chunk unavailable; transitioning to VM");
! assert(current == JavaThread::current(), "must be current thread except for preempt");
JRT_BLOCK
// delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk
JvmtiSampledObjectAllocEventCollector jsoaec(false);
freeze.set_jvmti_event_collector(&jsoaec);
ContinuationWrapper cont(current, oopCont);
log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), "");
! assert(LockingMode != LM_LEGACY || (monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0)),
"Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count());
+ assert(LockingMode == LM_LEGACY || (current->held_monitor_count() == 0 && current->jni_monitor_count() == 0),
+ "Held monitor count should only be used for LM_LEGACY: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count());
if (entry->is_pinned() || current->held_monitor_count() > 0) {
log_develop_debug(continuations)("PINNED due to critical section/hold monitor");
verify_continuation(cont.continuation());
freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor;
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
// Avoid Thread.yield() loops without safepoint polls.
! if (SafepointMechanism::should_process(current) && !preempt) {
cont.done(); // allow safepoint
ThreadInVMfromJava tivmfj(current);
}
return res;
}
! Freeze<ConfigT> freeze(current, cont, sp, preempt);
assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), "");
bool fast = UseContinuationFastPath && current->cont_fastpath();
if (fast && freeze.size_if_fast_freeze_available() > 0) {
freeze.freeze_fast_existing_chunk();
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
! return !preempt ? freeze_epilog(cont) : preempt_epilog(cont, freeze_ok, freeze.last_frame());
! }
+
+ if (preempt) {
+ JvmtiSampledObjectAllocEventCollector jsoaec(false);
+ freeze.set_jvmti_event_collector(&jsoaec);
+
+ freeze_result res = fast ? freeze.try_freeze_fast() : freeze.freeze_slow();
+
+ CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
+ preempt_epilog(cont, res, freeze.last_frame());
+ return res;
}
log_develop_trace(continuations)("chunk unavailable; transitioning to VM");
! assert(current == JavaThread::current(), "must be current thread");
JRT_BLOCK
// delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk
JvmtiSampledObjectAllocEventCollector jsoaec(false);
freeze.set_jvmti_event_collector(&jsoaec);
static int thaw_size(stackChunkOop chunk) {
int size = chunk->max_thawing_size();
size += frame::metadata_words; // For the top pc+fp in push_return_frame or top = stack_sp - frame::metadata_words in thaw_fast
size += 2*frame::align_wiggle; // in case of alignments at the top and bottom
+ size += frame::metadata_words; // for preemption case (see possibly_adjust_frame)
return size;
}
// make room on the stack for thaw
// returns the size in bytes, or 0 on failure
ContinuationWrapper& _cont;
CONT_JFR_ONLY(FreezeThawJfrInfo _jfr_info;)
intptr_t* _fastpath;
bool _barriers;
+ bool _preempted_case;
intptr_t* _top_unextended_sp_before_thaw;
int _align_size;
DEBUG_ONLY(intptr_t* _top_stack_address);
StackChunkFrameStream<ChunkFrames::Mixed> _stream;
assert (cont.tail() != nullptr, "no last chunk");
DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());)
}
void clear_chunk(stackChunkOop chunk);
int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize);
void copy_from_chunk(intptr_t* from, intptr_t* to, int size);
// fast path
inline void prefetch_chunk_pd(void* start, int size_words);
void patch_return(intptr_t* sp, bool is_last);
! // slow path
! NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, bool return_barrier);
private:
- void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top);
template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
void finalize_thaw(frame& entry, int argsize);
inline bool seen_by_gc();
assert (cont.tail() != nullptr, "no last chunk");
DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());)
}
void clear_chunk(stackChunkOop chunk);
+ template<bool check_stub>
int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize);
void copy_from_chunk(intptr_t* from, intptr_t* to, int size);
+ void thaw_lockstack(stackChunkOop chunk);
+
// fast path
inline void prefetch_chunk_pd(void* start, int size_words);
void patch_return(intptr_t* sp, bool is_last);
! intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case);
! inline intptr_t* possibly_adjust_frame(frame& top);
+ inline intptr_t* push_cleanup_continuation();
+ void throw_interrupted_exception(JavaThread* current, frame& top);
+
+ void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case);
+ void finish_thaw(frame& f);
private:
template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
void finalize_thaw(frame& entry, int argsize);
inline bool seen_by_gc();
void clear_bitmap_bits(address start, address end);
NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames);
void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
! void finish_thaw(frame& f);
void push_return_frame(frame& f);
inline frame new_entry_frame();
template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
inline void patch_pd(frame& f, const frame& sender);
inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
void clear_bitmap_bits(address start, address end);
NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames);
void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
! void recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames);
void push_return_frame(frame& f);
inline frame new_entry_frame();
template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
inline void patch_pd(frame& f, const frame& sender);
+ inline void patch_pd(frame& f, intptr_t* caller_sp);
inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
&& !chunk->has_thaw_slowpath_condition()
&& !PreserveFramePointer;
}
inline intptr_t* thaw(Continuation::thaw_kind kind);
+ template<bool check_stub = false>
NOINLINE intptr_t* thaw_fast(stackChunkOop chunk);
+ NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind);
inline void patch_caller_links(intptr_t* sp, intptr_t* bottom);
};
template <typename ConfigT>
inline intptr_t* Thaw<ConfigT>::thaw(Continuation::thaw_kind kind) {
assert(chunk != nullptr, "guaranteed by prepare_thaw");
assert(!chunk->is_empty(), "guaranteed by prepare_thaw");
_barriers = chunk->requires_barriers();
return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk)
! : thaw_slow(chunk, kind != Continuation::thaw_top);
}
class ReconstructedStack : public StackObj {
intptr_t* _base; // _cont.entrySP(); // top of the entry frame
int _thaw_size;
assert(chunk != nullptr, "guaranteed by prepare_thaw");
assert(!chunk->is_empty(), "guaranteed by prepare_thaw");
_barriers = chunk->requires_barriers();
return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk)
! : thaw_slow(chunk, kind);
}
class ReconstructedStack : public StackObj {
intptr_t* _base; // _cont.entrySP(); // top of the entry frame
int _thaw_size;
inline void ThawBase::clear_chunk(stackChunkOop chunk) {
chunk->set_sp(chunk->bottom());
chunk->set_max_thawing_size(0);
}
int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) {
bool empty = false;
StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
assert(chunk_sp == f.sp(), "");
assert(chunk_sp == f.unextended_sp(), "");
! const int frame_size = f.cb()->frame_size();
argsize = f.stack_argsize();
f.next(SmallRegisterMap::instance(), true /* stop */);
empty = f.is_done();
assert(!empty || argsize == chunk->argsize(), "");
if (empty) {
inline void ThawBase::clear_chunk(stackChunkOop chunk) {
chunk->set_sp(chunk->bottom());
chunk->set_max_thawing_size(0);
}
+ template<bool check_stub>
int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) {
bool empty = false;
StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
assert(chunk_sp == f.sp(), "");
assert(chunk_sp == f.unextended_sp(), "");
! int frame_size = f.cb()->frame_size();
argsize = f.stack_argsize();
+ assert(!f.is_stub() || check_stub, "");
+ if (check_stub && f.is_stub()) {
+ // If we don't thaw the top compiled frame too, after restoring the saved
+ // registers back in Java, we would hit the return barrier to thaw one more
+ // frame effectively overwriting the restored registers during that call.
+ f.next(SmallRegisterMap::instance(), true /* stop */);
+ assert(!f.is_done(), "");
+
+ f.get_cb();
+ assert(f.is_compiled(), "");
+ frame_size += f.cb()->frame_size();
+ argsize = f.stack_argsize();
+
+ if (f.cb()->as_nmethod()->is_marked_for_deoptimization()) {
+ // The caller of the runtime stub when the continuation is preempted is not at a
+ // Java call instruction, and so cannot rely on nmethod patching for deopt.
+ log_develop_trace(continuations)("Deoptimizing runtime stub caller");
+ f.to_frame().deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
+ }
+ }
+
f.next(SmallRegisterMap::instance(), true /* stop */);
empty = f.is_done();
assert(!empty || argsize == chunk->argsize(), "");
if (empty) {
// returns the size required to store the frame on stack, and because it is a
// compiled frame, it must include a copy of the arguments passed by the caller
return frame_size + argsize + frame::metadata_words_at_top;
}
+ void ThawBase::thaw_lockstack(stackChunkOop chunk) {
+ int lockStackSize = chunk->lockstack_size();
+ assert(lockStackSize > 0 && lockStackSize <= LockStack::CAPACITY, "");
+
+ oop tmp_lockstack[LockStack::CAPACITY];
+ chunk->transfer_lockstack(tmp_lockstack);
+ _thread->lock_stack().move_from_address(tmp_lockstack, lockStackSize);
+
+ chunk->set_lockstack_size(0);
+ chunk->set_has_lockstack(false);
+ }
+
void ThawBase::copy_from_chunk(intptr_t* from, intptr_t* to, int size) {
assert(to >= _top_stack_address, "overwrote past thawing space"
" to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(to), p2i(_top_stack_address));
assert(to + size <= _cont.entrySP(), "overwrote past thawing space");
_cont.tail()->copy_from_chunk_to_stack(from, to, size);
sp - frame::sender_sp_ret_address_offset(),
pc);
}
template <typename ConfigT>
+ template<bool check_stub>
NOINLINE intptr_t* Thaw<ConfigT>::thaw_fast(stackChunkOop chunk) {
assert(chunk == _cont.tail(), "");
assert(!chunk->has_mixed_frames(), "");
assert(!chunk->requires_barriers(), "");
assert(!chunk->has_bitmap(), "");
clear_chunk(chunk);
thaw_size = full_chunk_size;
empty = true;
} else { // thaw a single frame
partial = true;
! thaw_size = remove_top_compiled_frame_from_chunk(chunk, argsize);
empty = chunk->is_empty();
}
// Are we thawing the last frame(s) in the continuation
const bool is_last = empty && chunk->parent() == nullptr;
clear_chunk(chunk);
thaw_size = full_chunk_size;
empty = true;
} else { // thaw a single frame
partial = true;
! thaw_size = remove_top_compiled_frame_from_chunk<check_stub>(chunk, argsize);
empty = chunk->is_empty();
}
// Are we thawing the last frame(s) in the continuation
const bool is_last = empty && chunk->parent() == nullptr;
inline bool ThawBase::seen_by_gc() {
return _barriers || _cont.tail()->is_gc_mode();
}
! NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) {
LogTarget(Trace, continuations) lt;
if (lt.develop_is_enabled()) {
LogStream ls(lt);
! ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, return_barrier, p2i(chunk));
chunk->print_on(true, &ls);
}
#if CONT_JFR
EventContinuationThawSlow e;
inline bool ThawBase::seen_by_gc() {
return _barriers || _cont.tail()->is_gc_mode();
}
! static inline void relativize_chunk_concurrently(stackChunkOop chunk) {
+ #if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC
+ if (UseZGC || UseShenandoahGC) {
+ chunk->relativize_derived_pointers_concurrently();
+ }
+ #endif
+ }
+
+ template <typename ConfigT>
+ NOINLINE intptr_t* Thaw<ConfigT>::thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind) {
+ Continuation::preempt_kind preempt_kind;
+ bool retry_fast_path = false;
+
+ _preempted_case = chunk->preempted();
+ if (_preempted_case) {
+ if (chunk->object_waiter() != nullptr) {
+ // Mounted again after preemption. Resume the pending monitor operation,
+ // which will be either a monitorenter or Object.wait() call.
+ assert(chunk->current_pending_monitor() != nullptr || chunk->current_waiting_monitor() != nullptr, "");
+ ObjectWaiter* waiter = chunk->object_waiter();
+ ObjectMonitor* mon = waiter->monitor();
+ preempt_kind = waiter->is_wait() ? Continuation::freeze_on_wait : Continuation::freeze_on_monitorenter;
+
+ bool mon_acquired = mon->resume_operation(_thread, waiter, _cont);
+ assert(!mon_acquired || mon->has_owner(_thread), "invariant");
+ if (!mon_acquired) {
+ // Failed to acquire monitor. Return to enterSpecial to unmount again.
+ return push_cleanup_continuation();
+ }
+ chunk = _cont.tail(); // reload oop in case of safepoint in resume_operation (if posting JVMTI events).
+ } else {
+ // Preemption cancelled in moniterenter case. We actually acquired
+ // the monitor after freezing all frames so nothing to do.
+ preempt_kind = Continuation::freeze_on_monitorenter;
+ }
+ // Call this first to avoid racing with GC threads later when modifying the chunk flags.
+ relativize_chunk_concurrently(chunk);
+ chunk->set_preempted(false);
+ retry_fast_path = true;
+ } else {
+ relativize_chunk_concurrently(chunk);
+ }
+
+ // On first thaw after freeze restore oops to the lockstack if any.
+ assert(chunk->lockstack_size() == 0 || kind == Continuation::thaw_top, "");
+ if (kind == Continuation::thaw_top && chunk->lockstack_size() > 0) {
+ thaw_lockstack(chunk);
+ retry_fast_path = true;
+ }
+
+ // Retry the fast path now that we possibly cleared the FLAG_HAS_LOCKSTACK
+ // and FLAG_PREEMPTED flags from the stackChunk.
+ if (retry_fast_path && can_thaw_fast(chunk)) {
+ intptr_t* sp = thaw_fast<true>(chunk);
+ if (_preempted_case) {
+ return handle_preempted_continuation(sp, preempt_kind, true /* fast_case */);
+ }
+ return sp;
+ }
+
LogTarget(Trace, continuations) lt;
if (lt.develop_is_enabled()) {
LogStream ls(lt);
! ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, kind, p2i(chunk));
chunk->print_on(true, &ls);
}
#if CONT_JFR
EventContinuationThawSlow e;
}
#endif
DEBUG_ONLY(_frames = 0;)
_align_size = 0;
! int num_frames = (return_barrier ? 1 : 2);
_stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk);
_top_unextended_sp_before_thaw = _stream.unextended_sp();
frame heap_frame = _stream.to_frame();
}
#endif
DEBUG_ONLY(_frames = 0;)
_align_size = 0;
! int num_frames = kind == Continuation::thaw_top ? 2 : 1;
_stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk);
_top_unextended_sp_before_thaw = _stream.unextended_sp();
frame heap_frame = _stream.to_frame();
ls.print_cr("top hframe before (thaw):");
assert(heap_frame.is_heap_frame(), "should have created a relative frame");
heap_frame.print_value_on(&ls);
}
- #if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC
- if (UseZGC || UseShenandoahGC) {
- _cont.tail()->relativize_derived_pointers_concurrently();
- }
- #endif
-
frame caller; // the thawed caller on the stack
! recurse_thaw(heap_frame, caller, num_frames, true);
finish_thaw(caller); // caller is now the topmost thawed frame
_cont.write();
assert(_cont.chunk_invariant(), "");
JVMTI_ONLY(invalidate_jvmti_stack(_thread));
_thread->set_cont_fastpath(_fastpath);
intptr_t* sp = caller.sp();
return sp;
}
! void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) {
log_develop_debug(continuations)("thaw num_frames: %d", num_frames);
assert(!_cont.is_empty(), "no more frames");
assert(num_frames > 0, "");
assert(!heap_frame.is_empty(), "");
! if (top && heap_frame.is_safepoint_blob_frame()) {
! assert(ContinuationHelper::Frame::is_stub(heap_frame.cb()), "cb: %s", heap_frame.cb()->name());
- recurse_thaw_stub_frame(heap_frame, caller, num_frames);
} else if (!heap_frame.is_interpreted_frame()) {
recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
} else {
recurse_thaw_interpreted_frame(heap_frame, caller, num_frames);
}
ls.print_cr("top hframe before (thaw):");
assert(heap_frame.is_heap_frame(), "should have created a relative frame");
heap_frame.print_value_on(&ls);
}
frame caller; // the thawed caller on the stack
! recurse_thaw(heap_frame, caller, num_frames, _preempted_case);
finish_thaw(caller); // caller is now the topmost thawed frame
_cont.write();
assert(_cont.chunk_invariant(), "");
JVMTI_ONLY(invalidate_jvmti_stack(_thread));
_thread->set_cont_fastpath(_fastpath);
intptr_t* sp = caller.sp();
+
+ if (_preempted_case) {
+ return handle_preempted_continuation(sp, preempt_kind, false /* fast_case */);
+ }
return sp;
}
! void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case) {
log_develop_debug(continuations)("thaw num_frames: %d", num_frames);
assert(!_cont.is_empty(), "no more frames");
assert(num_frames > 0, "");
assert(!heap_frame.is_empty(), "");
! if (top_on_preempt_case && (heap_frame.is_native_frame() || heap_frame.is_runtime_frame())) {
! heap_frame.is_native_frame() ? recurse_thaw_native_frame(heap_frame, caller, 2) : recurse_thaw_stub_frame(heap_frame, caller, 2);
} else if (!heap_frame.is_interpreted_frame()) {
recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
} else {
recurse_thaw_interpreted_frame(heap_frame, caller, num_frames);
}
if (num_frames == 1 || _stream.is_done()) { // end recursion
finalize_thaw(caller, FKind::interpreted ? 0 : argsize);
return true; // bottom
} else { // recurse
! recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false);
return false;
}
}
void ThawBase::finalize_thaw(frame& entry, int argsize) {
if (num_frames == 1 || _stream.is_done()) { // end recursion
finalize_thaw(caller, FKind::interpreted ? 0 : argsize);
return true; // bottom
} else { // recurse
! recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false /* top_on_preempt_case */);
return false;
}
}
void ThawBase::finalize_thaw(frame& entry, int argsize) {
stackChunkOop chunk = _cont.tail();
chunk->bitmap().clear_range(chunk->bit_index_for(start), chunk->bit_index_for(effective_end));
assert(effective_end == end || !chunk->bitmap().at(chunk->bit_index_for(effective_end)), "bit should not be set");
}
+ intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case) {
+ assert(preempt_kind == Continuation::freeze_on_wait || preempt_kind == Continuation::freeze_on_monitorenter, "");
+ frame top(sp);
+ assert(top.pc() == *(address*)(sp - frame::sender_sp_ret_address_offset()), "");
+
+ #if INCLUDE_JVMTI
+ // Finish the VTMS transition.
+ assert(_thread->is_in_VTMS_transition(), "must be");
+ bool is_vthread = Continuation::continuation_scope(_cont.continuation()) == java_lang_VirtualThread::vthread_scope();
+ if (is_vthread) {
+ if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
+ jvmti_mount_end(_thread, _cont, top);
+ } else {
+ _thread->set_is_in_VTMS_transition(false);
+ java_lang_Thread::set_is_in_VTMS_transition(_thread->vthread(), false);
+ }
+ }
+ #endif
+
+ if (fast_case) {
+ // If we thawed in the slow path the runtime stub/native wrapper frame already
+ // has the correct fp (see ThawBase::new_stack_frame). On the fast path though,
+ // we copied the original fp at the time of freeze which now will have to be fixed.
+ assert(top.is_runtime_frame() || top.is_native_frame(), "");
+ int fsize = top.cb()->frame_size();
+ patch_pd(top, sp + fsize);
+ }
+
+ if (preempt_kind == Continuation::freeze_on_wait) {
+ // Check now if we need to throw IE exception.
+ if (_thread->pending_interrupted_exception()) {
+ throw_interrupted_exception(_thread, top);
+ _thread->set_pending_interrupted_exception(false);
+ }
+ } else if (top.is_runtime_frame()) {
+ // The continuation might now run on a different platform thread than the previous time so
+ // we need to adjust the current thread saved in the stub frame before restoring registers.
+ JavaThread** thread_addr = frame::saved_thread_address(top);
+ if (thread_addr != nullptr) *thread_addr = _thread;
+ // Some platforms require the size of the runtime frame to be adjusted.
+ sp = possibly_adjust_frame(top);
+ }
+ return sp;
+ }
+
+ void ThawBase::throw_interrupted_exception(JavaThread* current, frame& top) {
+ ContinuationWrapper::SafepointOp so(current, _cont);
+ // Since we might safepoint set the anchor so that the stack can be walked.
+ set_anchor(current, top.sp());
+ JRT_BLOCK
+ THROW(vmSymbols::java_lang_InterruptedException());
+ JRT_BLOCK_END
+ clear_anchor(current);
+ }
+
NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) {
assert(hf.is_interpreted_frame(), "");
if (UNLIKELY(seen_by_gc())) {
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance());
CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
maybe_set_fastpath(f.sp());
! const int locals = hf.interpreter_frame_method()->max_locals();
if (!is_bottom_frame) {
// can only fix caller once this frame is thawed (due to callee saved regs)
_cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance());
} else if (_cont.tail()->has_bitmap() && locals > 0) {
CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
maybe_set_fastpath(f.sp());
! Method* m = hf.interpreter_frame_method();
+ assert(!m->is_native() || !is_bottom_frame, "should be top frame of thaw_top case; missing caller frame");
+ const int locals = m->max_locals();
if (!is_bottom_frame) {
// can only fix caller once this frame is thawed (due to callee saved regs)
_cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance());
} else if (_cont.tail()->has_bitmap() && locals > 0) {
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
caller = f;
}
void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
! assert(!hf.is_interpreted_frame(), "");
! assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption");
if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance());
}
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
caller = f;
}
void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
! assert(hf.is_compiled_frame(), "");
! assert(_preempted_case || !stub_caller, "stub caller not at preemption");
if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance());
}
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
assert(caller.sp() == caller.unextended_sp(), "");
if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
! _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in freeze_interpreted_frame
}
// new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
// yet laid out in the stack, and so the original_pc is not stored in it.
// As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
assert(caller.sp() == caller.unextended_sp(), "");
if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
! _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_compiled_frame
}
// new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
// yet laid out in the stack, and so the original_pc is not stored in it.
// As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
// f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
assert(!f.is_deoptimized_frame(), "");
if (hf.is_deoptimized_frame()) {
maybe_set_fastpath(f.sp());
} else if (_thread->is_interp_only_mode()
! || (_cont.is_preempted() && f.cb()->as_nmethod()->is_marked_for_deoptimization())) {
// The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so
// cannot rely on nmethod patching for deopt.
- assert(_thread->is_interp_only_mode() || stub_caller, "expected a stub-caller");
log_develop_trace(continuations)("Deoptimizing thawed frame");
DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr));
f.deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
// f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
assert(!f.is_deoptimized_frame(), "");
if (hf.is_deoptimized_frame()) {
maybe_set_fastpath(f.sp());
} else if (_thread->is_interp_only_mode()
! || (stub_caller && f.cb()->as_nmethod()->is_marked_for_deoptimization())) {
// The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so
// cannot rely on nmethod patching for deopt.
log_develop_trace(continuations)("Deoptimizing thawed frame");
DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr));
f.deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
}
void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) {
DEBUG_ONLY(_frames++;)
! {
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
_stream.next(&map);
assert(!_stream.is_done(), "");
! if (UNLIKELY(seen_by_gc())) { // we're now doing this on the stub's caller
! _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map);
! }
assert(!_stream.is_done(), "");
}
! recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); // this could be deoptimized
-
- DEBUG_ONLY(before_thaw_java_frame(hf, caller, false, num_frames);)
! assert(ContinuationHelper::Frame::is_stub(hf.cb()), "");
assert(caller.sp() == caller.unextended_sp(), "");
- assert(!caller.is_interpreted_frame(), "");
! int fsize = ContinuationHelper::StubFrame::size(hf);
frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false);
intptr_t* stack_frame_top = f.sp();
intptr_t* heap_frame_top = hf.sp();
copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words,
fsize + frame::metadata_words);
! { // can only fix caller once this frame is thawed (due to callee saved regs)
! RegisterMap map(nullptr,
! RegisterMap::UpdateMap::include,
! RegisterMap::ProcessFrames::skip,
! RegisterMap::WalkContinuation::skip); // map.clear();
! map.set_include_argument_oops(false);
! f.oop_map()->update_register_map(&f, &map);
! ContinuationHelper::update_register_map_with_callee(caller, &map);
! _cont.tail()->fix_thawed_frame(caller, &map);
}
! DEBUG_ONLY(after_thaw_java_frame(f, false);)
caller = f;
}
void ThawBase::finish_thaw(frame& f) {
stackChunkOop chunk = _cont.tail();
}
void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) {
DEBUG_ONLY(_frames++;)
! if (UNLIKELY(seen_by_gc())) {
+ // Process the stub's caller here since we might need the full map.
RegisterMap map(nullptr,
RegisterMap::UpdateMap::include,
RegisterMap::ProcessFrames::skip,
RegisterMap::WalkContinuation::skip);
map.set_include_argument_oops(false);
_stream.next(&map);
assert(!_stream.is_done(), "");
! _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map);
! } else {
! _stream.next(SmallRegisterMap::instance());
assert(!_stream.is_done(), "");
}
! recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true);
! assert(caller.is_compiled_frame(), "");
assert(caller.sp() == caller.unextended_sp(), "");
! DEBUG_ONLY(before_thaw_java_frame(hf, caller, false /*is_bottom_frame*/, num_frames);)
frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false);
intptr_t* stack_frame_top = f.sp();
intptr_t* heap_frame_top = hf.sp();
+ int fsize = ContinuationHelper::StubFrame::size(hf);
copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words,
fsize + frame::metadata_words);
! patch(f, caller, false /*is_bottom_frame*/);
!
! // can only fix caller once this frame is thawed (due to callee saved regs)
! RegisterMap map(nullptr,
! RegisterMap::UpdateMap::include,
! RegisterMap::ProcessFrames::skip,
! RegisterMap::WalkContinuation::skip);
! map.set_include_argument_oops(false);
! f.oop_map()->update_register_map(&f, &map);
+ ContinuationHelper::update_register_map_with_callee(caller, &map);
+ _cont.tail()->fix_thawed_frame(caller, &map);
+
+ DEBUG_ONLY(after_thaw_java_frame(f, false /*is_bottom_frame*/);)
+ caller = f;
+ }
+
+ void ThawBase::recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames) {
+ assert(hf.is_native_frame(), "");
+ assert(_preempted_case && hf.cb()->as_nmethod()->method()->is_object_wait0(), "");
+
+ if (UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
+ _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance());
}
! const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::NativeFrame>(caller, num_frames);
+ assert(!is_bottom_frame, "");
+
+ DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
+
+ assert(caller.sp() == caller.unextended_sp(), "");
+
+ if (caller.is_interpreted_frame()) {
+ _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_native_frame
+ }
+
+ // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
+ // yet laid out in the stack, and so the original_pc is not stored in it.
+ // As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
+ frame f = new_stack_frame<ContinuationHelper::NativeFrame>(hf, caller, false /* bottom */);
+ intptr_t* const stack_frame_top = f.sp();
+ intptr_t* const heap_frame_top = hf.unextended_sp();
+
+ int fsize = ContinuationHelper::NativeFrame::size(hf);
+ assert(fsize <= (int)(caller.unextended_sp() - f.unextended_sp()), "");
+
+ intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom;
+ intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom;
+ int sz = fsize + frame::metadata_words_at_bottom;
+
+ copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above
+
+ patch(f, caller, false /* bottom */);
+
+ // f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
+ assert(!f.is_deoptimized_frame(), "");
+ assert(!hf.is_deoptimized_frame(), "");
+ assert(!f.cb()->as_nmethod()->is_marked_for_deoptimization(), "");
+
+ // can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack
+ _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance());
+
+ DEBUG_ONLY(after_thaw_java_frame(f, false /* bottom */);)
caller = f;
}
void ThawBase::finish_thaw(frame& f) {
stackChunkOop chunk = _cont.tail();
set_anchor_to_entry(thread, cont.entry());
log_frames(thread);
clear_anchor(thread);
#endif
Thaw<ConfigT> thw(thread, cont);
intptr_t* const sp = thw.thaw(kind);
assert(is_aligned(sp, frame::frame_alignment), "");
!
- // All or part of the frames have been thawed so we know they don't hold any monitors except JNI monitors.
- assert(thread->held_monitor_count() == thread->jni_monitor_count(), "Must be");
-
- #ifdef ASSERT
- intptr_t* sp0 = sp;
- set_anchor(thread, sp0);
- log_frames(thread);
- if (LoomVerifyAfterThaw) {
- assert(do_verify_after_thaw(thread, cont.tail(), tty), "");
- }
- assert(ContinuationEntry::assert_entry_frame_laid_out(thread), "");
- clear_anchor(thread);
-
- LogTarget(Trace, continuations) lt;
- if (lt.develop_is_enabled()) {
- LogStream ls(lt);
- ls.print_cr("Jumping to frame (thaw):");
- frame(sp).print_value_on(&ls);
- }
- #endif
CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);)
verify_continuation(cont.continuation());
log_develop_debug(continuations)("=== End of thaw #" INTPTR_FORMAT, cont.hash());
set_anchor_to_entry(thread, cont.entry());
log_frames(thread);
clear_anchor(thread);
#endif
+ DEBUG_ONLY(bool preempted = cont.tail()->preempted();)
Thaw<ConfigT> thw(thread, cont);
intptr_t* const sp = thw.thaw(kind);
assert(is_aligned(sp, frame::frame_alignment), "");
! DEBUG_ONLY(log_frames_after_thaw(thread, cont, sp, preempted);)
CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);)
verify_continuation(cont.continuation());
log_develop_debug(continuations)("=== End of thaw #" INTPTR_FORMAT, cont.hash());
if (!lt.develop_is_enabled()) {
return;
}
LogStream ls(lt);
! ls.print_cr("------- frames ---------");
if (!thread->has_last_Java_frame()) {
ls.print_cr("NO ANCHOR!");
}
RegisterMap map(thread,
if (!lt.develop_is_enabled()) {
return;
}
LogStream ls(lt);
! ls.print_cr("------- frames --------- for thread " INTPTR_FORMAT, p2i(thread));
if (!thread->has_last_Java_frame()) {
ls.print_cr("NO ANCHOR!");
}
RegisterMap map(thread,
HandleMark hm(Thread::current());
FrameValues values;
int i = 0;
int post_entry = -1;
! for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) {
! f.describe(values, i++, &map);
if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f))
post_entry++;
if (post_entry >= show_entry_callers)
break;
}
values.print_on(thread, &ls);
}
ls.print_cr("======= end frames =========");
}
#endif // ASSERT
#include CPU_HEADER_INLINE(continuationFreezeThaw)
#ifdef ASSERT
HandleMark hm(Thread::current());
FrameValues values;
int i = 0;
int post_entry = -1;
! for (frame f = thread->last_frame(); !f.is_first_frame(); f = f.sender(&map), i++) {
! f.describe(values, i, &map, i == 0);
if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f))
post_entry++;
if (post_entry >= show_entry_callers)
break;
}
values.print_on(thread, &ls);
}
ls.print_cr("======= end frames =========");
}
+
+ static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted) {
+ intptr_t* sp0 = sp;
+ address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset());
+ bool use_cont_entry = false;
+
+ // Some preemption cases need to use and adjusted version of sp.
+ if (preempted) {
+ if (sp0 == cont.entrySP()) {
+ // Still preempted (monitor not acquired) so no frames were thawed.
+ assert(cont.tail()->preempted(), "");
+ use_cont_entry = true;
+ }
+ #if defined (AARCH64) || defined (RISCV64)
+ else {
+ CodeBlob* cb = CodeCache::find_blob(pc0);
+ if (cb->frame_size() == 2) {
+ assert(cb->is_runtime_stub(), "");
+ // Returning to c2 runtime stub requires extra adjustment on aarch64
+ // and riscv64 (see possibly_adjust_frame()).
+ sp0 += frame::metadata_words;
+ }
+ }
+ #endif
+ }
+
+ set_anchor(thread, use_cont_entry ? cont.entrySP() : sp0, use_cont_entry ? cont.entryPC() : nullptr);
+ log_frames(thread);
+ if (LoomVerifyAfterThaw) {
+ assert(do_verify_after_thaw(thread, cont.tail(), tty), "");
+ }
+ assert(ContinuationEntry::assert_entry_frame_laid_out(thread), "");
+ clear_anchor(thread);
+
+ LogTarget(Trace, continuations) lt;
+ if (lt.develop_is_enabled()) {
+ LogStream ls(lt);
+ ls.print_cr("Jumping to frame (thaw):");
+ frame(sp).print_value_on(&ls);
+ }
+ }
#endif // ASSERT
#include CPU_HEADER_INLINE(continuationFreezeThaw)
#ifdef ASSERT
map.set_include_argument_oops(false);
map.set_skip_missing(true);
if (callee_complete) {
frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f));
}
! const_cast<frame&>(f).describe(values, 0, &map);
values.print_on(static_cast<JavaThread*>(nullptr), st);
}
#endif
static address thaw_entry = nullptr;
static address freeze_entry = nullptr;
address Continuation::thaw_entry() {
return ::thaw_entry;
}
address Continuation::freeze_entry() {
return ::freeze_entry;
}
class ConfigResolve {
public:
static void resolve() { resolve_compressed(); }
static void resolve_compressed() {
map.set_include_argument_oops(false);
map.set_skip_missing(true);
if (callee_complete) {
frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f));
}
! const_cast<frame&>(f).describe(values, 0, &map, true);
values.print_on(static_cast<JavaThread*>(nullptr), st);
}
#endif
static address thaw_entry = nullptr;
static address freeze_entry = nullptr;
+ static address freeze_preempt_entry = nullptr;
address Continuation::thaw_entry() {
return ::thaw_entry;
}
address Continuation::freeze_entry() {
return ::freeze_entry;
}
+ address Continuation::freeze_preempt_entry() {
+ return ::freeze_preempt_entry;
+ }
+
class ConfigResolve {
public:
static void resolve() { resolve_compressed(); }
static void resolve_compressed() {
template <bool use_compressed, typename BarrierSetT>
static void resolve() {
typedef Config<use_compressed ? oop_kind::NARROW : oop_kind::WIDE, BarrierSetT> SelectedConfigT;
freeze_entry = (address)freeze<SelectedConfigT>;
+ freeze_preempt_entry = (address)SelectedConfigT::freeze_preempt;
// If we wanted, we could templatize by kind and have three different thaw entries
thaw_entry = (address)thaw<SelectedConfigT>;
}
};
< prev index next >