1 /* 2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/javaClasses.inline.hpp" 27 #include "classfile/vmSymbols.hpp" 28 #include "code/codeCache.inline.hpp" 29 #include "code/nmethod.inline.hpp" 30 #include "code/vmreg.inline.hpp" 31 #include "compiler/oopMap.inline.hpp" 32 #include "gc/shared/continuationGCSupport.inline.hpp" 33 #include "gc/shared/gc_globals.hpp" 34 #include "gc/shared/barrierSet.hpp" 35 #include "gc/shared/memAllocator.hpp" 36 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "jfr/jfrEvents.hpp" 39 #include "logging/log.hpp" 40 #include "logging/logStream.hpp" 41 #include "oops/access.inline.hpp" 42 #include "oops/method.inline.hpp" 43 #include "oops/oopsHierarchy.hpp" 44 #include "oops/objArrayOop.inline.hpp" 45 #include "oops/stackChunkOop.inline.hpp" 46 #include "prims/jvmtiThreadState.hpp" 47 #include "runtime/arguments.hpp" 48 #include "runtime/continuation.hpp" 49 #include "runtime/continuationEntry.inline.hpp" 50 #include "runtime/continuationHelper.inline.hpp" 51 #include "runtime/continuationJavaClasses.inline.hpp" 52 #include "runtime/continuationWrapper.inline.hpp" 53 #include "runtime/frame.inline.hpp" 54 #include "runtime/interfaceSupport.inline.hpp" 55 #include "runtime/javaThread.inline.hpp" 56 #include "runtime/jniHandles.inline.hpp" 57 #include "runtime/keepStackGCProcessed.hpp" 58 #include "runtime/orderAccess.hpp" 59 #include "runtime/prefetch.inline.hpp" 60 #include "runtime/smallRegisterMap.inline.hpp" 61 #include "runtime/sharedRuntime.hpp" 62 #include "runtime/stackChunkFrameStream.inline.hpp" 63 #include "runtime/stackFrameStream.inline.hpp" 64 #include "runtime/stackOverflow.hpp" 65 #include "runtime/stackWatermarkSet.inline.hpp" 66 #include "utilities/debug.hpp" 67 #include "utilities/exceptions.hpp" 68 #include "utilities/macros.hpp" 69 #if INCLUDE_ZGC 70 #include "gc/z/zStackChunkGCData.inline.hpp" 71 #endif 72 73 #include <type_traits> 74 75 /* 76 * This file contains the implementation of continuation freezing (yield) and thawing (run). 77 * 78 * This code is very latency-critical and very hot. An ordinary and well-behaved server application 79 * would likely call these operations many thousands of times per second second, on every core. 80 * 81 * Freeze might be called every time the application performs any I/O operation, every time it 82 * acquires a j.u.c. lock, every time it takes a message from a queue, and thaw can be called 83 * multiple times in each of those cases, as it is called by the return barrier, which may be 84 * invoked on method return. 85 * 86 * The amortized budget for each of those two operations is ~100-150ns. That is why, for 87 * example, every effort is made to avoid Java-VM transitions as much as possible. 88 * 89 * On the fast path, all frames are known to be compiled, and the chunk requires no barriers 90 * and so frames simply copied, and the bottom-most one is patched. 91 * On the slow path, internal pointers in interpreted frames are de/relativized to/from offsets 92 * and absolute pointers, and barriers invoked. 93 */ 94 95 /************************************************ 96 97 Thread-stack layout on freeze/thaw. 98 See corresponding stack-chunk layout in instanceStackChunkKlass.hpp 99 100 +----------------------------+ 101 | . | 102 | . | 103 | . | 104 | carrier frames | 105 | | 106 |----------------------------| 107 | | 108 | Continuation.run | 109 | | 110 |============================| 111 | enterSpecial frame | 112 | pc | 113 | rbp | 114 | ----- | 115 ^ | int argsize | = ContinuationEntry 116 | | oopDesc* cont | 117 | | oopDesc* chunk | 118 | | ContinuationEntry* parent | 119 | | ... | 120 | |============================| <------ JavaThread::_cont_entry = entry->sp() 121 | | ? alignment word ? | 122 | |----------------------------| <--\ 123 | | | | 124 | | ? caller stack args ? | | argsize (might not be 2-word aligned) words 125 Address | | | | Caller is still in the chunk. 126 | |----------------------------| | 127 | | pc (? return barrier ?) | | This pc contains the return barrier when the bottom-most frame 128 | | rbp | | isn't the last one in the continuation. 129 | | | | 130 | | frame | | 131 | | | | 132 +----------------------------| \__ Continuation frames to be frozen/thawed 133 | | / 134 | frame | | 135 | | | 136 |----------------------------| | 137 | | | 138 | frame | | 139 | | | 140 |----------------------------| <--/ 141 | | 142 | doYield/safepoint stub | When preempting forcefully, we could have a safepoint stub 143 | | instead of a doYield stub 144 |============================| <- the sp passed to freeze 145 | | 146 | Native freeze/thaw frames | 147 | . | 148 | . | 149 | . | 150 +----------------------------+ 151 152 ************************************************/ 153 154 static const bool TEST_THAW_ONE_CHUNK_FRAME = false; // force thawing frames one-at-a-time for testing 155 156 #define CONT_JFR false // emit low-level JFR events that count slow/fast path for continuation performance debugging only 157 #if CONT_JFR 158 #define CONT_JFR_ONLY(code) code 159 #else 160 #define CONT_JFR_ONLY(code) 161 #endif 162 163 // TODO: See AbstractAssembler::generate_stack_overflow_check, 164 // Compile::bang_size_in_bytes(), m->as_SafePoint()->jvms()->interpreter_frame_size() 165 // when we stack-bang, we need to update a thread field with the lowest (farthest) bang point. 166 167 // Data invariants are defined by Continuation::debug_verify_continuation and Continuation::debug_verify_stack_chunk 168 169 // Used to just annotatate cold/hot branches 170 #define LIKELY(condition) (condition) 171 #define UNLIKELY(condition) (condition) 172 173 // debugging functions 174 #ifdef ASSERT 175 extern "C" bool dbg_is_safe(const void* p, intptr_t errvalue); // address p is readable and *(intptr_t*)p != errvalue 176 177 static void verify_continuation(oop continuation) { Continuation::debug_verify_continuation(continuation); } 178 179 static void do_deopt_after_thaw(JavaThread* thread); 180 static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st); 181 static void log_frames(JavaThread* thread); 182 static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st = tty); 183 184 #define assert_pfl(p, ...) \ 185 do { \ 186 if (!(p)) { \ 187 JavaThread* t = JavaThread::active(); \ 188 if (t->has_last_Java_frame()) { \ 189 tty->print_cr("assert(" #p ") failed:"); \ 190 t->print_frame_layout(); \ 191 } \ 192 } \ 193 vmassert(p, __VA_ARGS__); \ 194 } while(0) 195 196 #else 197 static void verify_continuation(oop continuation) { } 198 #define assert_pfl(p, ...) 199 #endif 200 201 // should match Continuation.preemptStatus() in Continuation.java 202 enum freeze_result { 203 freeze_ok = 0, 204 freeze_ok_bottom = 1, 205 freeze_pinned_cs = 2, 206 freeze_pinned_native = 3, 207 freeze_pinned_monitor = 4, 208 freeze_exception = 5 209 }; 210 211 const char* freeze_result_names[6] = { 212 "freeze_ok", 213 "freeze_ok_bottom", 214 "freeze_pinned_cs", 215 "freeze_pinned_native", 216 "freeze_pinned_monitor", 217 "freeze_exception" 218 }; 219 220 static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint); 221 template<typename ConfigT> static inline int freeze_internal(JavaThread* current, intptr_t* const sp); 222 223 static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier); 224 template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind); 225 226 227 // Entry point to freeze. Transitions are handled manually 228 // Called from gen_continuation_yield() in sharedRuntime_<cpu>.cpp through Continuation::freeze_entry(); 229 template<typename ConfigT> 230 static JRT_BLOCK_ENTRY(int, freeze(JavaThread* current, intptr_t* sp)) 231 assert(sp == current->frame_anchor()->last_Java_sp(), ""); 232 233 if (current->raw_cont_fastpath() > current->last_continuation()->entry_sp() || current->raw_cont_fastpath() < sp) { 234 current->set_cont_fastpath(nullptr); 235 } 236 237 return ConfigT::freeze(current, sp); 238 JRT_END 239 240 JRT_LEAF(int, Continuation::prepare_thaw(JavaThread* thread, bool return_barrier)) 241 return prepare_thaw_internal(thread, return_barrier); 242 JRT_END 243 244 template<typename ConfigT> 245 static JRT_LEAF(intptr_t*, thaw(JavaThread* thread, int kind)) 246 // TODO: JRT_LEAF and NoHandleMark is problematic for JFR events. 247 // vFrameStreamCommon allocates Handles in RegisterMap for continuations. 248 // JRT_ENTRY instead? 249 ResetNoHandleMark rnhm; 250 251 // we might modify the code cache via BarrierSetNMethod::nmethod_entry_barrier 252 MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread)); 253 return ConfigT::thaw(thread, (Continuation::thaw_kind)kind); 254 JRT_END 255 256 JVM_ENTRY(jint, CONT_isPinned0(JNIEnv* env, jobject cont_scope)) { 257 JavaThread* thread = JavaThread::thread_from_jni_environment(env); 258 return is_pinned0(thread, JNIHandles::resolve(cont_scope), false); 259 } 260 JVM_END 261 262 /////////// 263 264 enum class oop_kind { NARROW, WIDE }; 265 template <oop_kind oops, typename BarrierSetT> 266 class Config { 267 public: 268 typedef Config<oops, BarrierSetT> SelfT; 269 using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>; 270 271 static int freeze(JavaThread* thread, intptr_t* const sp) { 272 return freeze_internal<SelfT>(thread, sp); 273 } 274 275 static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) { 276 return thaw_internal<SelfT>(thread, kind); 277 } 278 }; 279 280 static bool stack_overflow_check(JavaThread* thread, size_t size, address sp) { 281 const size_t page_size = os::vm_page_size(); 282 if (size > page_size) { 283 if (sp - size < thread->stack_overflow_state()->shadow_zone_safe_limit()) { 284 return false; 285 } 286 } 287 return true; 288 } 289 290 #ifdef ASSERT 291 static oop get_continuation(JavaThread* thread) { 292 assert(thread != nullptr, ""); 293 assert(thread->threadObj() != nullptr, ""); 294 return java_lang_Thread::continuation(thread->threadObj()); 295 } 296 297 inline void clear_anchor(JavaThread* thread) { 298 thread->frame_anchor()->clear(); 299 } 300 301 static void set_anchor(JavaThread* thread, intptr_t* sp) { 302 address pc = ContinuationHelper::return_address_at( 303 sp - frame::sender_sp_ret_address_offset()); 304 assert(pc != nullptr, ""); 305 306 JavaFrameAnchor* anchor = thread->frame_anchor(); 307 anchor->set_last_Java_sp(sp); 308 anchor->set_last_Java_pc(pc); 309 ContinuationHelper::set_anchor_pd(anchor, sp); 310 311 assert(thread->has_last_Java_frame(), ""); 312 assert(thread->last_frame().cb() != nullptr, ""); 313 } 314 #endif // ASSERT 315 316 static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) { 317 JavaFrameAnchor* anchor = thread->frame_anchor(); 318 anchor->set_last_Java_sp(entry->entry_sp()); 319 anchor->set_last_Java_pc(entry->entry_pc()); 320 ContinuationHelper::set_anchor_to_entry_pd(anchor, entry); 321 322 assert(thread->has_last_Java_frame(), ""); 323 assert(thread->last_frame().cb() != nullptr, ""); 324 } 325 326 #if CONT_JFR 327 class FreezeThawJfrInfo : public StackObj { 328 short _e_size; 329 short _e_num_interpreted_frames; 330 public: 331 332 FreezeThawJfrInfo() : _e_size(0), _e_num_interpreted_frames(0) {} 333 inline void record_interpreted_frame() { _e_num_interpreted_frames++; } 334 inline void record_size_copied(int size) { _e_size += size << LogBytesPerWord; } 335 template<typename Event> void post_jfr_event(Event *e, oop continuation, JavaThread* jt); 336 }; 337 338 template<typename Event> void FreezeThawJfrInfo::post_jfr_event(Event* e, oop continuation, JavaThread* jt) { 339 if (e->should_commit()) { 340 log_develop_trace(continuations)("JFR event: iframes: %d size: %d", _e_num_interpreted_frames, _e_size); 341 e->set_carrierThread(JFR_JVM_THREAD_ID(jt)); 342 e->set_continuationClass(continuation->klass()); 343 e->set_interpretedFrames(_e_num_interpreted_frames); 344 e->set_size(_e_size); 345 e->commit(); 346 } 347 } 348 #endif // CONT_JFR 349 350 /////////////// FREEZE //// 351 352 class FreezeBase : public StackObj { 353 protected: 354 JavaThread* const _thread; 355 ContinuationWrapper& _cont; 356 bool _barriers; // only set when we allocate a chunk 357 const bool _preempt; // used only on the slow path 358 const intptr_t * const _frame_sp; // Top frame sp for this freeze 359 360 intptr_t* _bottom_address; 361 362 int _freeze_size; // total size of all frames plus metadata in words. 363 int _total_align_size; 364 365 intptr_t* _cont_stack_top; 366 intptr_t* _cont_stack_bottom; 367 368 CONT_JFR_ONLY(FreezeThawJfrInfo _jfr_info;) 369 370 #ifdef ASSERT 371 intptr_t* _orig_chunk_sp; 372 int _fast_freeze_size; 373 bool _empty; 374 #endif 375 376 JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector; 377 378 NOT_PRODUCT(int _frames;) 379 DEBUG_ONLY(intptr_t* _last_write;) 380 381 inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp); 382 383 public: 384 NOINLINE freeze_result freeze_slow(); 385 void freeze_fast_existing_chunk(); 386 387 CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; }) 388 void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; } 389 390 inline int size_if_fast_freeze_available(); 391 392 #ifdef ASSERT 393 bool check_valid_fast_path(); 394 #endif 395 396 protected: 397 inline void init_rest(); 398 void throw_stack_overflow_on_humongous_chunk(); 399 400 // fast path 401 inline void copy_to_chunk(intptr_t* from, intptr_t* to, int size); 402 inline void unwind_frames(); 403 inline void patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp); 404 405 // slow path 406 virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) = 0; 407 408 int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); } 409 410 private: 411 // slow path 412 frame freeze_start_frame(); 413 frame freeze_start_frame_safepoint_stub(frame f); 414 NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top); 415 inline frame freeze_start_frame_yield_stub(frame f); 416 template<typename FKind> 417 inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize); 418 inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame); 419 inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame); 420 freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize); 421 void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame); 422 NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted); 423 freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted); 424 NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller); 425 NOINLINE void finish_freeze(const frame& f, const frame& top); 426 427 inline bool stack_overflow(); 428 429 static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f) 430 : sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); } 431 template<typename FKind> static inline frame sender(const frame& f); 432 template<typename FKind> frame new_heap_frame(frame& f, frame& caller); 433 inline void set_top_frame_metadata_pd(const frame& hf); 434 inline void patch_pd(frame& callee, const frame& caller); 435 void adjust_interpreted_frame_unextended_sp(frame& f); 436 static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf); 437 438 protected: 439 void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated)); 440 bool freeze_fast_new_chunk(stackChunkOop chunk); 441 }; 442 443 template <typename ConfigT> 444 class Freeze : public FreezeBase { 445 private: 446 stackChunkOop allocate_chunk(size_t stack_size, int argsize_md); 447 448 public: 449 inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) 450 : FreezeBase(thread, cont, frame_sp) {} 451 452 freeze_result try_freeze_fast(); 453 454 protected: 455 virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) override { return allocate_chunk(stack_size, argsize_md); } 456 }; 457 458 FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) : 459 _thread(thread), _cont(cont), _barriers(false), _preempt(false), _frame_sp(frame_sp) { 460 DEBUG_ONLY(_jvmti_event_collector = nullptr;) 461 462 assert(_thread != nullptr, ""); 463 assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), ""); 464 465 DEBUG_ONLY(_cont.entry()->verify_cookie();) 466 467 assert(!Interpreter::contains(_cont.entryPC()), ""); 468 469 _bottom_address = _cont.entrySP() - _cont.entry_frame_extension(); 470 #ifdef _LP64 471 if (((intptr_t)_bottom_address & 0xf) != 0) { 472 _bottom_address--; 473 } 474 assert(is_aligned(_bottom_address, frame::frame_alignment), ""); 475 #endif 476 477 log_develop_trace(continuations)("bottom_address: " INTPTR_FORMAT " entrySP: " INTPTR_FORMAT " argsize: " PTR_FORMAT, 478 p2i(_bottom_address), p2i(_cont.entrySP()), (_cont.entrySP() - _bottom_address) << LogBytesPerWord); 479 assert(_bottom_address != nullptr, ""); 480 assert(_bottom_address <= _cont.entrySP(), ""); 481 DEBUG_ONLY(_last_write = nullptr;) 482 483 assert(_cont.chunk_invariant(), ""); 484 assert(!Interpreter::contains(_cont.entryPC()), ""); 485 #if !defined(PPC64) || defined(ZERO) 486 static const int doYield_stub_frame_size = frame::metadata_words; 487 #else 488 static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord; 489 #endif 490 assert(SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, ""); 491 492 // properties of the continuation on the stack; all sizes are in words 493 _cont_stack_top = frame_sp + doYield_stub_frame_size; // we don't freeze the doYield stub frame 494 _cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0) 495 - ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw 496 497 log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT, 498 cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom)); 499 assert(cont_size() > 0, ""); 500 } 501 502 void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling 503 _freeze_size = 0; 504 _total_align_size = 0; 505 NOT_PRODUCT(_frames = 0;) 506 } 507 508 void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) { 509 stackChunkOop chunk = _cont.tail(); 510 chunk->copy_from_stack_to_chunk(from, to, size); 511 CONT_JFR_ONLY(_jfr_info.record_size_copied(size);) 512 513 #ifdef ASSERT 514 if (_last_write != nullptr) { 515 assert(_last_write == to + size, "Missed a spot: _last_write: " INTPTR_FORMAT " to+size: " INTPTR_FORMAT 516 " stack_size: %d _last_write offset: " PTR_FORMAT " to+size: " PTR_FORMAT, p2i(_last_write), p2i(to+size), 517 chunk->stack_size(), _last_write-chunk->start_address(), to+size-chunk->start_address()); 518 _last_write = to; 519 } 520 #endif 521 } 522 523 // Called _after_ the last possible safepoint during the freeze operation (chunk allocation) 524 void FreezeBase::unwind_frames() { 525 ContinuationEntry* entry = _cont.entry(); 526 entry->flush_stack_processing(_thread); 527 set_anchor_to_entry(_thread, entry); 528 } 529 530 template <typename ConfigT> 531 freeze_result Freeze<ConfigT>::try_freeze_fast() { 532 assert(_thread->thread_state() == _thread_in_vm, ""); 533 assert(_thread->cont_fastpath(), ""); 534 535 DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();) 536 assert(_fast_freeze_size == 0, ""); 537 538 stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words, _cont.argsize() + frame::metadata_words_at_top); 539 if (freeze_fast_new_chunk(chunk)) { 540 return freeze_ok; 541 } 542 if (_thread->has_pending_exception()) { 543 return freeze_exception; 544 } 545 546 // TODO R REMOVE when deopt change is fixed 547 assert(!_thread->cont_fastpath() || _barriers, ""); 548 log_develop_trace(continuations)("-- RETRYING SLOW --"); 549 return freeze_slow(); 550 } 551 552 // Returns size needed if the continuation fits, otherwise 0. 553 int FreezeBase::size_if_fast_freeze_available() { 554 stackChunkOop chunk = _cont.tail(); 555 if (chunk == nullptr || chunk->is_gc_mode() || chunk->requires_barriers() || chunk->has_mixed_frames()) { 556 log_develop_trace(continuations)("chunk available %s", chunk == nullptr ? "no chunk" : "chunk requires barriers"); 557 return 0; 558 } 559 560 int total_size_needed = cont_size(); 561 const int chunk_sp = chunk->sp(); 562 563 // argsize can be nonzero if we have a caller, but the caller could be in a non-empty parent chunk, 564 // so we subtract it only if we overlap with the caller, i.e. the current chunk isn't empty. 565 // Consider leaving the chunk's argsize set when emptying it and removing the following branch, 566 // although that would require changing stackChunkOopDesc::is_empty 567 if (!chunk->is_empty()) { 568 total_size_needed -= _cont.argsize() + frame::metadata_words_at_top; 569 } 570 571 int chunk_free_room = chunk_sp - frame::metadata_words_at_bottom; 572 bool available = chunk_free_room >= total_size_needed; 573 log_develop_trace(continuations)("chunk available: %s size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT, 574 available ? "yes" : "no" , total_size_needed, _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom)); 575 return available ? total_size_needed : 0; 576 } 577 578 void FreezeBase::freeze_fast_existing_chunk() { 579 stackChunkOop chunk = _cont.tail(); 580 581 DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();) 582 assert(_fast_freeze_size > 0, ""); 583 584 if (!chunk->is_empty()) { // we are copying into a non-empty chunk 585 DEBUG_ONLY(_empty = false;) 586 DEBUG_ONLY(_orig_chunk_sp = chunk->sp_address();) 587 #ifdef ASSERT 588 { 589 intptr_t* retaddr_slot = (chunk->sp_address() 590 - frame::sender_sp_ret_address_offset()); 591 assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(), 592 "unexpected saved return address"); 593 } 594 #endif 595 596 // the chunk's sp before the freeze, adjusted to point beyond the stack-passed arguments in the topmost frame 597 // we overlap; we'll overwrite the chunk's top frame's callee arguments 598 const int chunk_start_sp = chunk->sp() + _cont.argsize() + frame::metadata_words_at_top; 599 assert(chunk_start_sp <= chunk->stack_size(), "sp not pointing into stack"); 600 601 // increase max_size by what we're freezing minus the overlap 602 chunk->set_max_thawing_size(chunk->max_thawing_size() + cont_size() - _cont.argsize() - frame::metadata_words_at_top); 603 604 intptr_t* const bottom_sp = _cont_stack_bottom - _cont.argsize() - frame::metadata_words_at_top; 605 assert(bottom_sp == _bottom_address, ""); 606 // Because the chunk isn't empty, we know there's a caller in the chunk, therefore the bottom-most frame 607 // should have a return barrier (installed back when we thawed it). 608 #ifdef ASSERT 609 { 610 intptr_t* retaddr_slot = (bottom_sp 611 - frame::sender_sp_ret_address_offset()); 612 assert(ContinuationHelper::return_address_at(retaddr_slot) 613 == StubRoutines::cont_returnBarrier(), 614 "should be the continuation return barrier"); 615 } 616 #endif 617 // We copy the fp from the chunk back to the stack because it contains some caller data, 618 // including, possibly, an oop that might have gone stale since we thawed. 619 patch_stack_pd(bottom_sp, chunk->sp_address()); 620 // we don't patch the return pc at this time, so as not to make the stack unwalkable for async walks 621 622 freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA false)); 623 } else { // the chunk is empty 624 const int chunk_start_sp = chunk->stack_size(); 625 626 DEBUG_ONLY(_empty = true;) 627 DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;) 628 629 chunk->set_max_thawing_size(cont_size()); 630 chunk->set_bottom(chunk_start_sp - _cont.argsize() - frame::metadata_words_at_top); 631 chunk->set_sp(chunk->bottom()); 632 633 freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA false)); 634 } 635 } 636 637 bool FreezeBase::freeze_fast_new_chunk(stackChunkOop chunk) { 638 DEBUG_ONLY(_empty = true;) 639 640 // Install new chunk 641 _cont.set_tail(chunk); 642 643 if (UNLIKELY(chunk == nullptr || !_thread->cont_fastpath() || _barriers)) { // OOME/probably humongous 644 log_develop_trace(continuations)("Retrying slow. Barriers: %d", _barriers); 645 return false; 646 } 647 648 chunk->set_max_thawing_size(cont_size()); 649 650 // in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments. 651 // They'll then be stored twice: in the chunk and in the parent chunk's top frame 652 const int chunk_start_sp = cont_size() + frame::metadata_words; 653 assert(chunk_start_sp == chunk->stack_size(), ""); 654 655 DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;) 656 657 freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true)); 658 659 return true; 660 } 661 662 void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated)) { 663 assert(chunk != nullptr, ""); 664 assert(!chunk->has_mixed_frames(), ""); 665 assert(!chunk->is_gc_mode(), ""); 666 assert(!chunk->has_bitmap(), ""); 667 assert(!chunk->requires_barriers(), ""); 668 assert(chunk == _cont.tail(), ""); 669 670 // We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before 671 // writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here 672 // will either see no continuation on the stack, or a consistent chunk. 673 unwind_frames(); 674 675 log_develop_trace(continuations)("freeze_fast start: chunk " INTPTR_FORMAT " size: %d orig sp: %d argsize: %d", 676 p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize()); 677 assert(chunk_start_sp <= chunk->stack_size(), ""); 678 assert(chunk_start_sp >= cont_size(), "no room in the chunk"); 679 680 const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze 681 assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, ""); 682 683 intptr_t* chunk_top = chunk->start_address() + chunk_new_sp; 684 #ifdef ASSERT 685 if (!_empty) { 686 intptr_t* retaddr_slot = (_orig_chunk_sp 687 - frame::sender_sp_ret_address_offset()); 688 assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(), 689 "unexpected saved return address"); 690 } 691 #endif 692 693 log_develop_trace(continuations)("freeze_fast start: " INTPTR_FORMAT " sp: %d chunk_top: " INTPTR_FORMAT, 694 p2i(chunk->start_address()), chunk_new_sp, p2i(chunk_top)); 695 intptr_t* from = _cont_stack_top - frame::metadata_words_at_bottom; 696 intptr_t* to = chunk_top - frame::metadata_words_at_bottom; 697 copy_to_chunk(from, to, cont_size() + frame::metadata_words_at_bottom); 698 // Because we're not patched yet, the chunk is now in a bad state 699 700 // patch return pc of the bottom-most frozen frame (now in the chunk) 701 // with the actual caller's return address 702 intptr_t* chunk_bottom_retaddr_slot = (chunk_top + cont_size() 703 - _cont.argsize() 704 - frame::metadata_words_at_top 705 - frame::sender_sp_ret_address_offset()); 706 #ifdef ASSERT 707 if (!_empty) { 708 assert(ContinuationHelper::return_address_at(chunk_bottom_retaddr_slot) 709 == StubRoutines::cont_returnBarrier(), 710 "should be the continuation return barrier"); 711 } 712 #endif 713 ContinuationHelper::patch_return_address_at(chunk_bottom_retaddr_slot, 714 chunk->pc()); 715 716 // We're always writing to a young chunk, so the GC can't see it until the next safepoint. 717 chunk->set_sp(chunk_new_sp); 718 // set chunk->pc to the return address of the topmost frame in the chunk 719 chunk->set_pc(ContinuationHelper::return_address_at( 720 _cont_stack_top - frame::sender_sp_ret_address_offset())); 721 722 _cont.write(); 723 724 log_develop_trace(continuations)("FREEZE CHUNK #" INTPTR_FORMAT " (young)", _cont.hash()); 725 LogTarget(Trace, continuations) lt; 726 if (lt.develop_is_enabled()) { 727 LogStream ls(lt); 728 chunk->print_on(true, &ls); 729 } 730 731 // Verification 732 assert(_cont.chunk_invariant(), ""); 733 chunk->verify(); 734 735 #if CONT_JFR 736 EventContinuationFreezeFast e; 737 if (e.should_commit()) { 738 e.set_id(cast_from_oop<u8>(chunk)); 739 DEBUG_ONLY(e.set_allocate(chunk_is_allocated);) 740 e.set_size(cont_size() << LogBytesPerWord); 741 e.commit(); 742 } 743 #endif 744 } 745 746 NOINLINE freeze_result FreezeBase::freeze_slow() { 747 #ifdef ASSERT 748 ResourceMark rm; 749 #endif 750 751 log_develop_trace(continuations)("freeze_slow #" INTPTR_FORMAT, _cont.hash()); 752 assert(_thread->thread_state() == _thread_in_vm || _thread->thread_state() == _thread_blocked, ""); 753 754 #if CONT_JFR 755 EventContinuationFreezeSlow e; 756 if (e.should_commit()) { 757 e.set_id(cast_from_oop<u8>(_cont.continuation())); 758 e.commit(); 759 } 760 #endif 761 762 init_rest(); 763 764 HandleMark hm(Thread::current()); 765 766 frame f = freeze_start_frame(); 767 768 LogTarget(Debug, continuations) lt; 769 if (lt.develop_is_enabled()) { 770 LogStream ls(lt); 771 f.print_on(&ls); 772 } 773 774 frame caller; // the frozen caller in the chunk 775 freeze_result res = recurse_freeze(f, caller, 0, false, true); 776 777 if (res == freeze_ok) { 778 finish_freeze(f, caller); 779 _cont.write(); 780 } 781 782 return res; 783 } 784 785 frame FreezeBase::freeze_start_frame() { 786 frame f = _thread->last_frame(); 787 if (LIKELY(!_preempt)) { 788 return freeze_start_frame_yield_stub(f); 789 } else { 790 return freeze_start_frame_safepoint_stub(f); 791 } 792 } 793 794 frame FreezeBase::freeze_start_frame_yield_stub(frame f) { 795 assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be"); 796 f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); 797 assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), ""); 798 return f; 799 } 800 801 frame FreezeBase::freeze_start_frame_safepoint_stub(frame f) { 802 #if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO) 803 f.set_fp(f.real_fp()); // f.set_fp(*Frame::callee_link_address(f)); // ???? 804 #else 805 Unimplemented(); 806 #endif 807 if (!Interpreter::contains(f.pc())) { 808 assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be"); 809 assert(f.oop_map() != nullptr, "must be"); 810 811 if (Interpreter::contains(ContinuationHelper::StubFrame::return_pc(f))) { 812 f = sender<ContinuationHelper::StubFrame>(f); // Safepoint stub in interpreter 813 } 814 } 815 assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), ""); 816 return f; 817 } 818 819 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap. 820 NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) { 821 assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame 822 assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())), ""); 823 824 if (stack_overflow()) { 825 return freeze_exception; 826 } 827 828 if (f.is_compiled_frame()) { 829 if (UNLIKELY(f.oop_map() == nullptr)) { 830 // special native frame 831 return freeze_pinned_native; 832 } 833 return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted); 834 } else if (f.is_interpreted_frame()) { 835 assert((_preempt && top) || !f.interpreter_frame_method()->is_native(), ""); 836 if (_preempt && top && f.interpreter_frame_method()->is_native()) { 837 // int native entry 838 return freeze_pinned_native; 839 } 840 841 return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted); 842 } else if (_preempt && top && ContinuationHelper::Frame::is_stub(f.cb())) { 843 return recurse_freeze_stub_frame(f, caller); 844 } else { 845 return freeze_pinned_native; 846 } 847 } 848 849 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap. 850 // See also StackChunkFrameStream<frame_kind>::frame_size() 851 template<typename FKind> 852 inline freeze_result FreezeBase::recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize) { 853 assert(FKind::is_instance(f), ""); 854 855 assert(fsize > 0, ""); 856 assert(argsize >= 0, ""); 857 _freeze_size += fsize; 858 NOT_PRODUCT(_frames++;) 859 860 assert(FKind::frame_bottom(f) <= _bottom_address, ""); 861 862 // We don't use FKind::frame_bottom(f) == _bottom_address because on x64 there's sometimes an extra word between 863 // enterSpecial and an interpreted frame 864 if (FKind::frame_bottom(f) >= _bottom_address - 1) { 865 return finalize_freeze(f, caller, argsize); // recursion end 866 } else { 867 frame senderf = sender<FKind>(f); 868 assert(FKind::interpreted || senderf.sp() == senderf.unextended_sp(), ""); 869 freeze_result result = recurse_freeze(senderf, caller, argsize, FKind::interpreted, false); // recursive call 870 return result; 871 } 872 } 873 874 inline void FreezeBase::before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame) { 875 LogTarget(Trace, continuations) lt; 876 if (lt.develop_is_enabled()) { 877 LogStream ls(lt); 878 ls.print_cr("======== FREEZING FRAME interpreted: %d bottom: %d", f.is_interpreted_frame(), is_bottom_frame); 879 ls.print_cr("fsize: %d argsize: %d", fsize, argsize); 880 f.print_value_on(&ls, nullptr); 881 } 882 assert(caller.is_interpreted_frame() == Interpreter::contains(caller.pc()), ""); 883 } 884 885 inline void FreezeBase::after_freeze_java_frame(const frame& hf, bool is_bottom_frame) { 886 LogTarget(Trace, continuations) lt; 887 if (lt.develop_is_enabled()) { 888 LogStream ls(lt); 889 DEBUG_ONLY(hf.print_value_on(&ls, nullptr);) 890 assert(hf.is_heap_frame(), "should be"); 891 DEBUG_ONLY(print_frame_layout(hf, false, &ls);) 892 if (is_bottom_frame) { 893 ls.print_cr("bottom h-frame:"); 894 hf.print_on(&ls); 895 } 896 } 897 } 898 899 // The parameter argsize_md includes metadata that has to be part of caller/callee overlap. 900 // See also StackChunkFrameStream<frame_kind>::frame_size() 901 freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, int argsize_md) { 902 int argsize = argsize_md - frame::metadata_words_at_top; 903 assert(callee.is_interpreted_frame() 904 || callee.cb()->as_nmethod()->is_osr_method() 905 || argsize == _cont.argsize(), "argsize: %d cont.argsize: %d", argsize, _cont.argsize()); 906 log_develop_trace(continuations)("bottom: " INTPTR_FORMAT " count %d size: %d argsize: %d", 907 p2i(_bottom_address), _frames, _freeze_size << LogBytesPerWord, argsize); 908 909 LogTarget(Trace, continuations) lt; 910 911 #ifdef ASSERT 912 bool empty = _cont.is_empty(); 913 log_develop_trace(continuations)("empty: %d", empty); 914 #endif 915 916 stackChunkOop chunk = _cont.tail(); 917 918 assert(chunk == nullptr || (chunk->max_thawing_size() == 0) == chunk->is_empty(), ""); 919 920 _freeze_size += frame::metadata_words; // for top frame's metadata 921 922 int overlap = 0; // the args overlap the caller -- if there is one in this chunk and is of the same kind 923 int unextended_sp = -1; 924 if (chunk != nullptr) { 925 if (!chunk->is_empty()) { 926 StackChunkFrameStream<ChunkFrames::Mixed> last(chunk); 927 unextended_sp = chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp()); 928 bool top_interpreted = Interpreter::contains(chunk->pc()); 929 if (callee.is_interpreted_frame() == top_interpreted) { 930 overlap = argsize_md; 931 } 932 } else { 933 unextended_sp = chunk->stack_size() - frame::metadata_words_at_top; 934 } 935 } 936 937 log_develop_trace(continuations)("finalize _size: %d overlap: %d unextended_sp: %d", _freeze_size, overlap, unextended_sp); 938 939 _freeze_size -= overlap; 940 assert(_freeze_size >= 0, ""); 941 942 assert(chunk == nullptr || chunk->is_empty() 943 || unextended_sp == chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp()), ""); 944 assert(chunk != nullptr || unextended_sp < _freeze_size, ""); 945 946 // _barriers can be set to true by an allocation in freeze_fast, in which case the chunk is available 947 bool allocated_old_in_freeze_fast = _barriers; 948 assert(!allocated_old_in_freeze_fast || (unextended_sp >= _freeze_size && chunk->is_empty()), 949 "Chunk allocated in freeze_fast is of insufficient size " 950 "unextended_sp: %d size: %d is_empty: %d", unextended_sp, _freeze_size, chunk->is_empty()); 951 assert(!allocated_old_in_freeze_fast || (!UseZGC && !UseG1GC), "Unexpected allocation"); 952 953 DEBUG_ONLY(bool empty_chunk = true); 954 if (unextended_sp < _freeze_size || chunk->is_gc_mode() || (!allocated_old_in_freeze_fast && chunk->requires_barriers())) { 955 // ALLOCATE NEW CHUNK 956 957 if (lt.develop_is_enabled()) { 958 LogStream ls(lt); 959 if (chunk == nullptr) { 960 ls.print_cr("no chunk"); 961 } else { 962 ls.print_cr("chunk barriers: %d _size: %d free size: %d", 963 chunk->requires_barriers(), _freeze_size, chunk->sp() - frame::metadata_words); 964 chunk->print_on(&ls); 965 } 966 } 967 968 _freeze_size += overlap; // we're allocating a new chunk, so no overlap 969 // overlap = 0; 970 971 chunk = allocate_chunk_slow(_freeze_size, argsize_md); 972 if (chunk == nullptr) { 973 return freeze_exception; 974 } 975 976 // Install new chunk 977 _cont.set_tail(chunk); 978 assert(chunk->is_empty(), ""); 979 } else { 980 // REUSE EXISTING CHUNK 981 log_develop_trace(continuations)("Reusing chunk mixed: %d empty: %d", chunk->has_mixed_frames(), chunk->is_empty()); 982 if (chunk->is_empty()) { 983 int sp = chunk->stack_size() - argsize_md; 984 chunk->set_sp(sp); 985 chunk->set_bottom(sp); 986 _freeze_size += overlap; 987 assert(chunk->max_thawing_size() == 0, ""); 988 } DEBUG_ONLY(else empty_chunk = false;) 989 } 990 assert(!chunk->is_gc_mode(), ""); 991 assert(!chunk->has_bitmap(), ""); 992 chunk->set_has_mixed_frames(true); 993 994 assert(chunk->requires_barriers() == _barriers, ""); 995 assert(!_barriers || chunk->is_empty(), ""); 996 997 assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), ""); 998 assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), ""); 999 1000 // We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before 1001 // writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here 1002 // will either see no continuation or a consistent chunk. 1003 unwind_frames(); 1004 1005 chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - frame::metadata_words); 1006 1007 if (lt.develop_is_enabled()) { 1008 LogStream ls(lt); 1009 ls.print_cr("top chunk:"); 1010 chunk->print_on(&ls); 1011 } 1012 1013 // The topmost existing frame in the chunk; or an empty frame if the chunk is empty 1014 caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame(); 1015 1016 DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);) 1017 1018 assert(chunk->is_in_chunk(_last_write - _freeze_size), 1019 "last_write-size: " INTPTR_FORMAT " start: " INTPTR_FORMAT, p2i(_last_write-_freeze_size), p2i(chunk->start_address())); 1020 #ifdef ASSERT 1021 if (lt.develop_is_enabled()) { 1022 LogStream ls(lt); 1023 ls.print_cr("top hframe before (freeze):"); 1024 assert(caller.is_heap_frame(), "should be"); 1025 caller.print_on(&ls); 1026 } 1027 1028 assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), ""); 1029 1030 frame entry = sender(callee); 1031 1032 assert(Continuation::is_return_barrier_entry(entry.pc()) || Continuation::is_continuation_enterSpecial(entry), ""); 1033 assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), ""); 1034 #endif 1035 1036 return freeze_ok_bottom; 1037 } 1038 1039 void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) { 1040 if (is_bottom_frame) { 1041 // If we're the bottom frame, we need to replace the return barrier with the real 1042 // caller's pc. 1043 address last_pc = caller.pc(); 1044 assert((last_pc == nullptr) == _cont.tail()->is_empty(), ""); 1045 ContinuationHelper::Frame::patch_pc(caller, last_pc); 1046 } else { 1047 assert(!caller.is_empty(), ""); 1048 } 1049 1050 patch_pd(hf, caller); 1051 1052 if (f.is_interpreted_frame()) { 1053 assert(hf.is_heap_frame(), "should be"); 1054 ContinuationHelper::InterpretedFrame::patch_sender_sp(hf, caller); 1055 } 1056 1057 #ifdef ASSERT 1058 if (hf.is_compiled_frame()) { 1059 if (f.is_deoptimized_frame()) { // TODO DEOPT: long term solution: unroll on freeze and patch pc 1060 log_develop_trace(continuations)("Freezing deoptimized frame"); 1061 assert(f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), ""); 1062 assert(f.cb()->as_nmethod()->is_deopt_pc(ContinuationHelper::Frame::real_pc(f)), ""); 1063 } 1064 } 1065 #endif 1066 } 1067 1068 #ifdef ASSERT 1069 static void verify_frame_top(const frame& f, intptr_t* top) { 1070 ResourceMark rm; 1071 InterpreterOopMap mask; 1072 f.interpreted_frame_oop_map(&mask); 1073 assert(top <= ContinuationHelper::InterpretedFrame::frame_top(f, &mask), 1074 "frame_top: " INTPTR_FORMAT " Interpreted::frame_top: " INTPTR_FORMAT, 1075 p2i(top), p2i(ContinuationHelper::InterpretedFrame::frame_top(f, &mask))); 1076 } 1077 #endif // ASSERT 1078 1079 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap. 1080 // See also StackChunkFrameStream<frame_kind>::frame_size() 1081 NOINLINE freeze_result FreezeBase::recurse_freeze_interpreted_frame(frame& f, frame& caller, 1082 int callee_argsize /* incl. metadata */, 1083 bool callee_interpreted) { 1084 adjust_interpreted_frame_unextended_sp(f); 1085 1086 // The frame's top never includes the stack arguments to the callee 1087 intptr_t* const stack_frame_top = ContinuationHelper::InterpretedFrame::frame_top(f, callee_argsize, callee_interpreted); 1088 intptr_t* const stack_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(f); 1089 const int fsize = pointer_delta_as_int(stack_frame_bottom, stack_frame_top); 1090 1091 DEBUG_ONLY(verify_frame_top(f, stack_frame_top)); 1092 1093 Method* frame_method = ContinuationHelper::Frame::frame_method(f); 1094 // including metadata between f and its args 1095 const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top; 1096 1097 log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d", 1098 frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize); 1099 // we'd rather not yield inside methods annotated with @JvmtiMountTransition 1100 assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), ""); 1101 1102 freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize); 1103 if (UNLIKELY(result > freeze_ok_bottom)) { 1104 return result; 1105 } 1106 1107 bool is_bottom_frame = result == freeze_ok_bottom; 1108 assert(!caller.is_empty() || is_bottom_frame, ""); 1109 1110 DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, is_bottom_frame);) 1111 1112 frame hf = new_heap_frame<ContinuationHelper::InterpretedFrame>(f, caller); 1113 _total_align_size += frame::align_wiggle; // add alignment room for internal interpreted frame alignment on AArch64/PPC64 1114 1115 intptr_t* heap_frame_top = ContinuationHelper::InterpretedFrame::frame_top(hf, callee_argsize, callee_interpreted); 1116 intptr_t* heap_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(hf); 1117 assert(heap_frame_bottom == heap_frame_top + fsize, ""); 1118 1119 // Some architectures (like AArch64/PPC64/RISC-V) add padding between the locals and the fixed_frame to keep the fp 16-byte-aligned. 1120 // On those architectures we freeze the padding in order to keep the same fp-relative offsets in the fixed_frame. 1121 copy_to_chunk(stack_frame_top, heap_frame_top, fsize); 1122 assert(!is_bottom_frame || !caller.is_interpreted_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), ""); 1123 1124 relativize_interpreted_frame_metadata(f, hf); 1125 1126 patch(f, hf, caller, is_bottom_frame); 1127 1128 CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();) 1129 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);) 1130 caller = hf; 1131 1132 // Mark frame_method's GC epoch for class redefinition on_stack calculation. 1133 frame_method->record_gc_epoch(); 1134 1135 return freeze_ok; 1136 } 1137 1138 // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap. 1139 // See also StackChunkFrameStream<frame_kind>::frame_size() 1140 freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller, 1141 int callee_argsize /* incl. metadata */, 1142 bool callee_interpreted) { 1143 // The frame's top never includes the stack arguments to the callee 1144 intptr_t* const stack_frame_top = ContinuationHelper::CompiledFrame::frame_top(f, callee_argsize, callee_interpreted); 1145 intptr_t* const stack_frame_bottom = ContinuationHelper::CompiledFrame::frame_bottom(f); 1146 // including metadata between f and its stackargs 1147 const int argsize = ContinuationHelper::CompiledFrame::stack_argsize(f) + frame::metadata_words_at_top; 1148 const int fsize = pointer_delta_as_int(stack_frame_bottom + argsize, stack_frame_top); 1149 1150 log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d", 1151 ContinuationHelper::Frame::frame_method(f) != nullptr ? 1152 ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "", 1153 _freeze_size, fsize, argsize); 1154 // we'd rather not yield inside methods annotated with @JvmtiMountTransition 1155 assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), ""); 1156 1157 freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize); 1158 if (UNLIKELY(result > freeze_ok_bottom)) { 1159 return result; 1160 } 1161 1162 bool is_bottom_frame = result == freeze_ok_bottom; 1163 assert(!caller.is_empty() || is_bottom_frame, ""); 1164 1165 DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, is_bottom_frame);) 1166 1167 frame hf = new_heap_frame<ContinuationHelper::CompiledFrame>(f, caller); 1168 1169 intptr_t* heap_frame_top = ContinuationHelper::CompiledFrame::frame_top(hf, callee_argsize, callee_interpreted); 1170 1171 copy_to_chunk(stack_frame_top, heap_frame_top, fsize); 1172 assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), ""); 1173 1174 if (caller.is_interpreted_frame()) { 1175 _total_align_size += frame::align_wiggle; // See Thaw::align 1176 } 1177 1178 patch(f, hf, caller, is_bottom_frame); 1179 1180 assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), ""); 1181 1182 DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);) 1183 caller = hf; 1184 return freeze_ok; 1185 } 1186 1187 NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) { 1188 intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0); 1189 const int fsize = f.cb()->frame_size(); 1190 1191 log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT, 1192 f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize)); 1193 1194 // recurse_freeze_java_frame and freeze inlined here because we need to use a full RegisterMap for lock ownership 1195 NOT_PRODUCT(_frames++;) 1196 _freeze_size += fsize; 1197 1198 RegisterMap map(_cont.thread(), 1199 RegisterMap::UpdateMap::include, 1200 RegisterMap::ProcessFrames::skip, 1201 RegisterMap::WalkContinuation::skip); 1202 map.set_include_argument_oops(false); 1203 ContinuationHelper::update_register_map<ContinuationHelper::StubFrame>(f, &map); 1204 f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case 1205 frame senderf = sender<ContinuationHelper::StubFrame>(f); 1206 assert(senderf.unextended_sp() < _bottom_address - 1, ""); 1207 assert(senderf.is_compiled_frame(), ""); 1208 1209 if (UNLIKELY(senderf.oop_map() == nullptr)) { 1210 // native frame 1211 return freeze_pinned_native; 1212 } 1213 1214 freeze_result result = recurse_freeze_compiled_frame(senderf, caller, 0, 0); // This might be deoptimized 1215 if (UNLIKELY(result > freeze_ok_bottom)) { 1216 return result; 1217 } 1218 assert(result != freeze_ok_bottom, ""); 1219 assert(!caller.is_interpreted_frame(), ""); 1220 1221 DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false);) 1222 frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller); 1223 intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0); 1224 copy_to_chunk(stack_frame_top, heap_frame_top, fsize); 1225 DEBUG_ONLY(after_freeze_java_frame(hf, false);) 1226 1227 caller = hf; 1228 return freeze_ok; 1229 } 1230 1231 NOINLINE void FreezeBase::finish_freeze(const frame& f, const frame& top) { 1232 stackChunkOop chunk = _cont.tail(); 1233 1234 LogTarget(Trace, continuations) lt; 1235 if (lt.develop_is_enabled()) { 1236 LogStream ls(lt); 1237 assert(top.is_heap_frame(), "should be"); 1238 top.print_on(&ls); 1239 } 1240 1241 set_top_frame_metadata_pd(top); 1242 1243 chunk->set_sp(chunk->to_offset(top.sp())); 1244 chunk->set_pc(top.pc()); 1245 1246 chunk->set_max_thawing_size(chunk->max_thawing_size() + _total_align_size); 1247 1248 // At this point the chunk is consistent 1249 1250 if (UNLIKELY(_barriers)) { 1251 log_develop_trace(continuations)("do barriers on old chunk"); 1252 // Serial and Parallel GC can allocate objects directly into the old generation. 1253 // Then we want to relativize the derived pointers eagerly so that 1254 // old chunks are all in GC mode. 1255 assert(!UseG1GC, "G1 can not deal with allocating outside of eden"); 1256 assert(!UseZGC, "ZGC can not deal with allocating chunks visible to marking"); 1257 if (UseShenandoahGC) { 1258 _cont.tail()->relativize_derived_pointers_concurrently(); 1259 } else { 1260 ContinuationGCSupport::transform_stack_chunk(_cont.tail()); 1261 } 1262 // For objects in the old generation we must maintain the remembered set 1263 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(); 1264 } 1265 1266 log_develop_trace(continuations)("finish_freeze: has_mixed_frames: %d", chunk->has_mixed_frames()); 1267 if (lt.develop_is_enabled()) { 1268 LogStream ls(lt); 1269 chunk->print_on(true, &ls); 1270 } 1271 1272 if (lt.develop_is_enabled()) { 1273 LogStream ls(lt); 1274 ls.print_cr("top hframe after (freeze):"); 1275 assert(_cont.last_frame().is_heap_frame(), "should be"); 1276 _cont.last_frame().print_on(&ls); 1277 } 1278 1279 assert(_cont.chunk_invariant(), ""); 1280 } 1281 1282 inline bool FreezeBase::stack_overflow() { // detect stack overflow in recursive native code 1283 JavaThread* t = !_preempt ? _thread : JavaThread::current(); 1284 assert(t == JavaThread::current(), ""); 1285 if (os::current_stack_pointer() < t->stack_overflow_state()->shadow_zone_safe_limit()) { 1286 if (!_preempt) { 1287 ContinuationWrapper::SafepointOp so(t, _cont); // could also call _cont.done() instead 1288 Exceptions::_throw_msg(t, __FILE__, __LINE__, vmSymbols::java_lang_StackOverflowError(), "Stack overflow while freezing"); 1289 } 1290 return true; 1291 } 1292 return false; 1293 } 1294 1295 class StackChunkAllocator : public MemAllocator { 1296 const size_t _stack_size; 1297 int _argsize_md; 1298 ContinuationWrapper& _continuation_wrapper; 1299 JvmtiSampledObjectAllocEventCollector* const _jvmti_event_collector; 1300 mutable bool _took_slow_path; 1301 1302 // Does the minimal amount of initialization needed for a TLAB allocation. 1303 // We don't need to do a full initialization, as such an allocation need not be immediately walkable. 1304 virtual oop initialize(HeapWord* mem) const override { 1305 assert(_stack_size > 0, ""); 1306 assert(_stack_size <= max_jint, ""); 1307 assert(_word_size > _stack_size, ""); 1308 1309 // zero out fields (but not the stack) 1310 const size_t hs = oopDesc::header_size(); 1311 Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs); 1312 1313 int bottom = (int)_stack_size - _argsize_md; 1314 1315 jdk_internal_vm_StackChunk::set_size(mem, (int)_stack_size); 1316 jdk_internal_vm_StackChunk::set_bottom(mem, bottom); 1317 jdk_internal_vm_StackChunk::set_sp(mem, bottom); 1318 1319 return finish(mem); 1320 } 1321 1322 stackChunkOop allocate_fast() const { 1323 if (!UseTLAB) { 1324 return nullptr; 1325 } 1326 1327 HeapWord* const mem = MemAllocator::mem_allocate_inside_tlab_fast(); 1328 if (mem == nullptr) { 1329 return nullptr; 1330 } 1331 1332 oop obj = initialize(mem); 1333 return stackChunkOopDesc::cast(obj); 1334 } 1335 1336 public: 1337 StackChunkAllocator(Klass* klass, 1338 size_t word_size, 1339 Thread* thread, 1340 size_t stack_size, 1341 int argsize_md, 1342 ContinuationWrapper& continuation_wrapper, 1343 JvmtiSampledObjectAllocEventCollector* jvmti_event_collector) 1344 : MemAllocator(klass, word_size, thread), 1345 _stack_size(stack_size), 1346 _argsize_md(argsize_md), 1347 _continuation_wrapper(continuation_wrapper), 1348 _jvmti_event_collector(jvmti_event_collector), 1349 _took_slow_path(false) {} 1350 1351 // Provides it's own, specialized allocation which skips instrumentation 1352 // if the memory can be allocated without going to a slow-path. 1353 stackChunkOop allocate() const { 1354 // First try to allocate without any slow-paths or instrumentation. 1355 stackChunkOop obj = allocate_fast(); 1356 if (obj != nullptr) { 1357 return obj; 1358 } 1359 1360 // Now try full-blown allocation with all expensive operations, 1361 // including potentially safepoint operations. 1362 _took_slow_path = true; 1363 1364 // Protect unhandled Loom oops 1365 ContinuationWrapper::SafepointOp so(_thread, _continuation_wrapper); 1366 1367 // Can safepoint 1368 _jvmti_event_collector->start(); 1369 1370 // Can safepoint 1371 return stackChunkOopDesc::cast(MemAllocator::allocate()); 1372 } 1373 1374 bool took_slow_path() const { 1375 return _took_slow_path; 1376 } 1377 }; 1378 1379 template <typename ConfigT> 1380 stackChunkOop Freeze<ConfigT>::allocate_chunk(size_t stack_size, int argsize_md) { 1381 log_develop_trace(continuations)("allocate_chunk allocating new chunk"); 1382 1383 InstanceStackChunkKlass* klass = InstanceStackChunkKlass::cast(vmClasses::StackChunk_klass()); 1384 size_t size_in_words = klass->instance_size(stack_size); 1385 1386 if (CollectedHeap::stack_chunk_max_size() > 0 && size_in_words >= CollectedHeap::stack_chunk_max_size()) { 1387 if (!_preempt) { 1388 throw_stack_overflow_on_humongous_chunk(); 1389 } 1390 return nullptr; 1391 } 1392 1393 JavaThread* current = _preempt ? JavaThread::current() : _thread; 1394 assert(current == JavaThread::current(), "should be current"); 1395 1396 // Allocate the chunk. 1397 // 1398 // This might safepoint while allocating, but all safepointing due to 1399 // instrumentation have been deferred. This property is important for 1400 // some GCs, as this ensures that the allocated object is in the young 1401 // generation / newly allocated memory. 1402 StackChunkAllocator allocator(klass, size_in_words, current, stack_size, argsize_md, _cont, _jvmti_event_collector); 1403 stackChunkOop chunk = allocator.allocate(); 1404 1405 if (chunk == nullptr) { 1406 return nullptr; // OOME 1407 } 1408 1409 // assert that chunk is properly initialized 1410 assert(chunk->stack_size() == (int)stack_size, ""); 1411 assert(chunk->size() >= stack_size, "chunk->size(): %zu size: %zu", chunk->size(), stack_size); 1412 assert(chunk->sp() == chunk->bottom(), ""); 1413 assert((intptr_t)chunk->start_address() % 8 == 0, ""); 1414 assert(chunk->max_thawing_size() == 0, ""); 1415 assert(chunk->pc() == nullptr, ""); 1416 assert(chunk->is_empty(), ""); 1417 assert(chunk->flags() == 0, ""); 1418 assert(chunk->is_gc_mode() == false, ""); 1419 1420 // fields are uninitialized 1421 chunk->set_parent_access<IS_DEST_UNINITIALIZED>(_cont.last_nonempty_chunk()); 1422 chunk->set_cont_access<IS_DEST_UNINITIALIZED>(_cont.continuation()); 1423 1424 #if INCLUDE_ZGC 1425 if (UseZGC) { 1426 if (ZGenerational) { 1427 ZStackChunkGCData::initialize(chunk); 1428 } 1429 assert(!chunk->requires_barriers(), "ZGC always allocates in the young generation"); 1430 _barriers = false; 1431 } else 1432 #endif 1433 #if INCLUDE_SHENANDOAHGC 1434 if (UseShenandoahGC) { 1435 _barriers = chunk->requires_barriers(); 1436 } else 1437 #endif 1438 { 1439 if (!allocator.took_slow_path()) { 1440 // Guaranteed to be in young gen / newly allocated memory 1441 assert(!chunk->requires_barriers(), "Unfamiliar GC requires barriers on TLAB allocation"); 1442 _barriers = false; 1443 } else { 1444 // Some GCs could put direct allocations in old gen for slow-path 1445 // allocations; need to explicitly check if that was the case. 1446 _barriers = chunk->requires_barriers(); 1447 } 1448 } 1449 1450 if (_barriers) { 1451 log_develop_trace(continuations)("allocation requires barriers"); 1452 } 1453 1454 assert(chunk->parent() == nullptr || chunk->parent()->is_stackChunk(), ""); 1455 1456 return chunk; 1457 } 1458 1459 void FreezeBase::throw_stack_overflow_on_humongous_chunk() { 1460 ContinuationWrapper::SafepointOp so(_thread, _cont); // could also call _cont.done() instead 1461 Exceptions::_throw_msg(_thread, __FILE__, __LINE__, vmSymbols::java_lang_StackOverflowError(), "Humongous stack chunk"); 1462 } 1463 1464 #if INCLUDE_JVMTI 1465 static int num_java_frames(ContinuationWrapper& cont) { 1466 ResourceMark rm; // used for scope traversal in num_java_frames(nmethod*, address) 1467 int count = 0; 1468 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 1469 count += chunk->num_java_frames(); 1470 } 1471 return count; 1472 } 1473 1474 static void invalidate_jvmti_stack(JavaThread* thread) { 1475 if (thread->is_interp_only_mode()) { 1476 JvmtiThreadState *state = thread->jvmti_thread_state(); 1477 if (state != nullptr) 1478 state->invalidate_cur_stack_depth(); 1479 } 1480 } 1481 1482 static void jvmti_yield_cleanup(JavaThread* thread, ContinuationWrapper& cont) { 1483 if (JvmtiExport::can_post_frame_pop()) { 1484 int num_frames = num_java_frames(cont); 1485 1486 ContinuationWrapper::SafepointOp so(Thread::current(), cont); 1487 JvmtiExport::continuation_yield_cleanup(JavaThread::current(), num_frames); 1488 } 1489 invalidate_jvmti_stack(thread); 1490 } 1491 #endif // INCLUDE_JVMTI 1492 1493 #ifdef ASSERT 1494 static bool monitors_on_stack(JavaThread* thread) { 1495 ContinuationEntry* ce = thread->last_continuation(); 1496 RegisterMap map(thread, 1497 RegisterMap::UpdateMap::include, 1498 RegisterMap::ProcessFrames::include, 1499 RegisterMap::WalkContinuation::skip); 1500 map.set_include_argument_oops(false); 1501 for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) { 1502 if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) || 1503 (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f))) { 1504 return true; 1505 } 1506 } 1507 return false; 1508 } 1509 1510 // There are no interpreted frames if we're not called from the interpreter and we haven't ancountered an i2c 1511 // adapter or called Deoptimization::unpack_frames. As for native frames, upcalls from JNI also go through the 1512 // interpreter (see JavaCalls::call_helper), while the UpcallLinker explicitly sets cont_fastpath. 1513 bool FreezeBase::check_valid_fast_path() { 1514 ContinuationEntry* ce = _thread->last_continuation(); 1515 RegisterMap map(_thread, 1516 RegisterMap::UpdateMap::skip, 1517 RegisterMap::ProcessFrames::skip, 1518 RegisterMap::WalkContinuation::skip); 1519 map.set_include_argument_oops(false); 1520 for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) { 1521 if (!f.is_compiled_frame() || f.is_deoptimized_frame()) { 1522 return false; 1523 } 1524 } 1525 return true; 1526 } 1527 #endif // ASSERT 1528 1529 static inline int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont) { 1530 verify_continuation(cont.continuation()); 1531 assert(!cont.is_empty(), ""); 1532 // This is done for the sake of the enterSpecial frame 1533 StackWatermarkSet::after_unwind(thread); 1534 1535 log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash()); 1536 1537 return 0; 1538 } 1539 1540 static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) { 1541 if (UNLIKELY(res != freeze_ok)) { 1542 verify_continuation(cont.continuation()); 1543 log_develop_trace(continuations)("=== end of freeze (fail %d)", res); 1544 return res; 1545 } 1546 1547 JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint 1548 return freeze_epilog(thread, cont); 1549 } 1550 1551 template<typename ConfigT> 1552 static inline int freeze_internal(JavaThread* current, intptr_t* const sp) { 1553 assert(!current->has_pending_exception(), ""); 1554 1555 #ifdef ASSERT 1556 log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp())); 1557 log_frames(current); 1558 #endif 1559 1560 CONT_JFR_ONLY(EventContinuationFreeze event;) 1561 1562 ContinuationEntry* entry = current->last_continuation(); 1563 1564 oop oopCont = entry->cont_oop(current); 1565 assert(oopCont == current->last_continuation()->cont_oop(current), ""); 1566 assert(ContinuationEntry::assert_entry_frame_laid_out(current), ""); 1567 1568 verify_continuation(oopCont); 1569 ContinuationWrapper cont(current, oopCont); 1570 log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont)); 1571 1572 assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), ""); 1573 1574 assert(monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0), 1575 "Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count()); 1576 1577 if (entry->is_pinned() || current->held_monitor_count() > 0) { 1578 log_develop_debug(continuations)("PINNED due to critical section/hold monitor"); 1579 verify_continuation(cont.continuation()); 1580 freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor; 1581 log_develop_trace(continuations)("=== end of freeze (fail %d)", res); 1582 // Avoid Thread.yield() loops without safepoint polls. 1583 if (SafepointMechanism::should_process(current)) { 1584 cont.done(); // allow safepoint 1585 ThreadInVMfromJava tivmfj(current); 1586 } 1587 return res; 1588 } 1589 1590 Freeze<ConfigT> freeze(current, cont, sp); 1591 1592 assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), ""); 1593 bool fast = UseContinuationFastPath && current->cont_fastpath(); 1594 if (fast && freeze.size_if_fast_freeze_available() > 0) { 1595 freeze.freeze_fast_existing_chunk(); 1596 CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);) 1597 freeze_epilog(current, cont); 1598 return 0; 1599 } 1600 1601 log_develop_trace(continuations)("chunk unavailable; transitioning to VM"); 1602 assert(current == JavaThread::current(), "must be current thread except for preempt"); 1603 JRT_BLOCK 1604 // delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk 1605 JvmtiSampledObjectAllocEventCollector jsoaec(false); 1606 freeze.set_jvmti_event_collector(&jsoaec); 1607 1608 freeze_result res = fast ? freeze.try_freeze_fast() : freeze.freeze_slow(); 1609 1610 CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);) 1611 freeze_epilog(current, cont, res); 1612 cont.done(); // allow safepoint in the transition back to Java 1613 return res; 1614 JRT_BLOCK_END 1615 } 1616 1617 static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint) { 1618 ContinuationEntry* entry = thread->last_continuation(); 1619 if (entry == nullptr) { 1620 return freeze_ok; 1621 } 1622 if (entry->is_pinned()) { 1623 return freeze_pinned_cs; 1624 } else if (thread->held_monitor_count() > 0) { 1625 return freeze_pinned_monitor; 1626 } 1627 1628 RegisterMap map(thread, 1629 RegisterMap::UpdateMap::include, 1630 RegisterMap::ProcessFrames::skip, 1631 RegisterMap::WalkContinuation::skip); 1632 map.set_include_argument_oops(false); 1633 frame f = thread->last_frame(); 1634 1635 if (!safepoint) { 1636 f = f.sender(&map); // this is the yield frame 1637 } else { // safepoint yield 1638 #if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO) 1639 f.set_fp(f.real_fp()); // Instead of this, maybe in ContinuationWrapper::set_last_frame always use the real_fp? 1640 #else 1641 Unimplemented(); 1642 #endif 1643 if (!Interpreter::contains(f.pc())) { 1644 assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be"); 1645 assert(f.oop_map() != nullptr, "must be"); 1646 f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case 1647 } 1648 } 1649 1650 while (true) { 1651 if ((f.is_interpreted_frame() && f.interpreter_frame_method()->is_native()) || f.is_native_frame()) { 1652 return freeze_pinned_native; 1653 } 1654 1655 f = f.sender(&map); 1656 if (!Continuation::is_frame_in_continuation(entry, f)) { 1657 oop scope = jdk_internal_vm_Continuation::scope(entry->cont_oop(thread)); 1658 if (scope == cont_scope) { 1659 break; 1660 } 1661 intx monitor_count = entry->parent_held_monitor_count(); 1662 entry = entry->parent(); 1663 if (entry == nullptr) { 1664 break; 1665 } 1666 if (entry->is_pinned()) { 1667 return freeze_pinned_cs; 1668 } else if (monitor_count > 0) { 1669 return freeze_pinned_monitor; 1670 } 1671 } 1672 } 1673 return freeze_ok; 1674 } 1675 1676 /////////////// THAW //// 1677 1678 static int thaw_size(stackChunkOop chunk) { 1679 int size = chunk->max_thawing_size(); 1680 size += frame::metadata_words; // For the top pc+fp in push_return_frame or top = stack_sp - frame::metadata_words in thaw_fast 1681 size += 2*frame::align_wiggle; // in case of alignments at the top and bottom 1682 return size; 1683 } 1684 1685 // make room on the stack for thaw 1686 // returns the size in bytes, or 0 on failure 1687 static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier) { 1688 log_develop_trace(continuations)("~~~~ prepare_thaw return_barrier: %d", return_barrier); 1689 1690 assert(thread == JavaThread::current(), ""); 1691 1692 ContinuationEntry* ce = thread->last_continuation(); 1693 assert(ce != nullptr, ""); 1694 oop continuation = ce->cont_oop(thread); 1695 assert(continuation == get_continuation(thread), ""); 1696 verify_continuation(continuation); 1697 1698 stackChunkOop chunk = jdk_internal_vm_Continuation::tail(continuation); 1699 assert(chunk != nullptr, ""); 1700 1701 // The tail can be empty because it might still be available for another freeze. 1702 // However, here we want to thaw, so we get rid of it (it will be GCed). 1703 if (UNLIKELY(chunk->is_empty())) { 1704 chunk = chunk->parent(); 1705 assert(chunk != nullptr, ""); 1706 assert(!chunk->is_empty(), ""); 1707 jdk_internal_vm_Continuation::set_tail(continuation, chunk); 1708 } 1709 1710 // Verification 1711 chunk->verify(); 1712 assert(chunk->max_thawing_size() > 0, "chunk invariant violated; expected to not be empty"); 1713 1714 // Only make space for the last chunk because we only thaw from the last chunk 1715 int size = thaw_size(chunk) << LogBytesPerWord; 1716 1717 const address bottom = (address)thread->last_continuation()->entry_sp(); 1718 // 300 is an estimate for stack size taken for this native code, in addition to StackShadowPages 1719 // for the Java frames in the check below. 1720 if (!stack_overflow_check(thread, size + 300, bottom)) { 1721 return 0; 1722 } 1723 1724 log_develop_trace(continuations)("prepare_thaw bottom: " INTPTR_FORMAT " top: " INTPTR_FORMAT " size: %d", 1725 p2i(bottom), p2i(bottom - size), size); 1726 return size; 1727 } 1728 1729 class ThawBase : public StackObj { 1730 protected: 1731 JavaThread* _thread; 1732 ContinuationWrapper& _cont; 1733 CONT_JFR_ONLY(FreezeThawJfrInfo _jfr_info;) 1734 1735 intptr_t* _fastpath; 1736 bool _barriers; 1737 intptr_t* _top_unextended_sp_before_thaw; 1738 int _align_size; 1739 DEBUG_ONLY(intptr_t* _top_stack_address); 1740 1741 StackChunkFrameStream<ChunkFrames::Mixed> _stream; 1742 1743 NOT_PRODUCT(int _frames;) 1744 1745 protected: 1746 ThawBase(JavaThread* thread, ContinuationWrapper& cont) : 1747 _thread(thread), _cont(cont), 1748 _fastpath(nullptr) { 1749 DEBUG_ONLY(_top_unextended_sp_before_thaw = nullptr;) 1750 assert (cont.tail() != nullptr, "no last chunk"); 1751 DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());) 1752 } 1753 1754 void clear_chunk(stackChunkOop chunk); 1755 int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize); 1756 void copy_from_chunk(intptr_t* from, intptr_t* to, int size); 1757 1758 // fast path 1759 inline void prefetch_chunk_pd(void* start, int size_words); 1760 void patch_return(intptr_t* sp, bool is_last); 1761 1762 // slow path 1763 NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, bool return_barrier); 1764 1765 private: 1766 void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top); 1767 template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames); 1768 void finalize_thaw(frame& entry, int argsize); 1769 1770 inline bool seen_by_gc(); 1771 1772 inline void before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame); 1773 inline void after_thaw_java_frame(const frame& f, bool bottom); 1774 inline void patch(frame& f, const frame& caller, bool bottom); 1775 void clear_bitmap_bits(address start, address end); 1776 1777 NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames); 1778 void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller); 1779 void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames); 1780 void finish_thaw(frame& f); 1781 1782 void push_return_frame(frame& f); 1783 inline frame new_entry_frame(); 1784 template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom); 1785 inline void patch_pd(frame& f, const frame& sender); 1786 inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom); 1787 1788 void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; } 1789 1790 static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f); 1791 1792 public: 1793 CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; }) 1794 }; 1795 1796 template <typename ConfigT> 1797 class Thaw : public ThawBase { 1798 public: 1799 Thaw(JavaThread* thread, ContinuationWrapper& cont) : ThawBase(thread, cont) {} 1800 1801 inline bool can_thaw_fast(stackChunkOop chunk) { 1802 return !_barriers 1803 && _thread->cont_fastpath_thread_state() 1804 && !chunk->has_thaw_slowpath_condition() 1805 && !PreserveFramePointer; 1806 } 1807 1808 inline intptr_t* thaw(Continuation::thaw_kind kind); 1809 NOINLINE intptr_t* thaw_fast(stackChunkOop chunk); 1810 inline void patch_caller_links(intptr_t* sp, intptr_t* bottom); 1811 }; 1812 1813 template <typename ConfigT> 1814 inline intptr_t* Thaw<ConfigT>::thaw(Continuation::thaw_kind kind) { 1815 verify_continuation(_cont.continuation()); 1816 assert(!jdk_internal_vm_Continuation::done(_cont.continuation()), ""); 1817 assert(!_cont.is_empty(), ""); 1818 1819 stackChunkOop chunk = _cont.tail(); 1820 assert(chunk != nullptr, "guaranteed by prepare_thaw"); 1821 assert(!chunk->is_empty(), "guaranteed by prepare_thaw"); 1822 1823 _barriers = chunk->requires_barriers(); 1824 return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk) 1825 : thaw_slow(chunk, kind != Continuation::thaw_top); 1826 } 1827 1828 class ReconstructedStack : public StackObj { 1829 intptr_t* _base; // _cont.entrySP(); // top of the entry frame 1830 int _thaw_size; 1831 int _argsize; 1832 public: 1833 ReconstructedStack(intptr_t* base, int thaw_size, int argsize) 1834 : _base(base), _thaw_size(thaw_size - (argsize == 0 ? frame::metadata_words_at_top : 0)), _argsize(argsize) { 1835 // The only possible source of misalignment is stack-passed arguments b/c compiled frames are 16-byte aligned. 1836 assert(argsize != 0 || (_base - _thaw_size) == ContinuationHelper::frame_align_pointer(_base - _thaw_size), ""); 1837 // We're at most one alignment word away from entrySP 1838 assert(_base - 1 <= top() + total_size() + frame::metadata_words_at_bottom, "missed entry frame"); 1839 } 1840 1841 int entry_frame_extension() const { return _argsize + (_argsize > 0 ? frame::metadata_words_at_top : 0); } 1842 1843 // top and bottom stack pointers 1844 intptr_t* sp() const { return ContinuationHelper::frame_align_pointer(_base - _thaw_size); } 1845 intptr_t* bottom_sp() const { return ContinuationHelper::frame_align_pointer(_base - entry_frame_extension()); } 1846 1847 // several operations operate on the totality of the stack being reconstructed, 1848 // including the metadata words 1849 intptr_t* top() const { return sp() - frame::metadata_words_at_bottom; } 1850 int total_size() const { return _thaw_size + frame::metadata_words_at_bottom; } 1851 }; 1852 1853 inline void ThawBase::clear_chunk(stackChunkOop chunk) { 1854 chunk->set_sp(chunk->bottom()); 1855 chunk->set_max_thawing_size(0); 1856 } 1857 1858 int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) { 1859 bool empty = false; 1860 StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk); 1861 DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();) 1862 assert(chunk_sp == f.sp(), ""); 1863 assert(chunk_sp == f.unextended_sp(), ""); 1864 1865 const int frame_size = f.cb()->frame_size(); 1866 argsize = f.stack_argsize(); 1867 1868 f.next(SmallRegisterMap::instance, true /* stop */); 1869 empty = f.is_done(); 1870 assert(!empty || argsize == chunk->argsize(), ""); 1871 1872 if (empty) { 1873 clear_chunk(chunk); 1874 } else { 1875 chunk->set_sp(chunk->sp() + frame_size); 1876 chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size); 1877 // We set chunk->pc to the return pc into the next frame 1878 chunk->set_pc(f.pc()); 1879 #ifdef ASSERT 1880 { 1881 intptr_t* retaddr_slot = (chunk_sp 1882 + frame_size 1883 - frame::sender_sp_ret_address_offset()); 1884 assert(f.pc() == ContinuationHelper::return_address_at(retaddr_slot), 1885 "unexpected pc"); 1886 } 1887 #endif 1888 } 1889 assert(empty == chunk->is_empty(), ""); 1890 // returns the size required to store the frame on stack, and because it is a 1891 // compiled frame, it must include a copy of the arguments passed by the caller 1892 return frame_size + argsize + frame::metadata_words_at_top; 1893 } 1894 1895 void ThawBase::copy_from_chunk(intptr_t* from, intptr_t* to, int size) { 1896 assert(to >= _top_stack_address, "overwrote past thawing space" 1897 " to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(to), p2i(_top_stack_address)); 1898 assert(to + size <= _cont.entrySP(), "overwrote past thawing space"); 1899 _cont.tail()->copy_from_chunk_to_stack(from, to, size); 1900 CONT_JFR_ONLY(_jfr_info.record_size_copied(size);) 1901 } 1902 1903 void ThawBase::patch_return(intptr_t* sp, bool is_last) { 1904 log_develop_trace(continuations)("thaw_fast patching -- sp: " INTPTR_FORMAT, p2i(sp)); 1905 1906 address pc = !is_last ? StubRoutines::cont_returnBarrier() : _cont.entryPC(); 1907 ContinuationHelper::patch_return_address_at( 1908 sp - frame::sender_sp_ret_address_offset(), 1909 pc); 1910 } 1911 1912 template <typename ConfigT> 1913 NOINLINE intptr_t* Thaw<ConfigT>::thaw_fast(stackChunkOop chunk) { 1914 assert(chunk == _cont.tail(), ""); 1915 assert(!chunk->has_mixed_frames(), ""); 1916 assert(!chunk->requires_barriers(), ""); 1917 assert(!chunk->has_bitmap(), ""); 1918 assert(!_thread->is_interp_only_mode(), ""); 1919 1920 LogTarget(Trace, continuations) lt; 1921 if (lt.develop_is_enabled()) { 1922 LogStream ls(lt); 1923 ls.print_cr("thaw_fast"); 1924 chunk->print_on(true, &ls); 1925 } 1926 1927 // Below this heuristic, we thaw the whole chunk, above it we thaw just one frame. 1928 static const int threshold = 500; // words 1929 1930 const int full_chunk_size = chunk->stack_size() - chunk->sp(); // this initial size could be reduced if it's a partial thaw 1931 int argsize, thaw_size; 1932 1933 intptr_t* const chunk_sp = chunk->start_address() + chunk->sp(); 1934 1935 bool partial, empty; 1936 if (LIKELY(!TEST_THAW_ONE_CHUNK_FRAME && (full_chunk_size < threshold))) { 1937 prefetch_chunk_pd(chunk->start_address(), full_chunk_size); // prefetch anticipating memcpy starting at highest address 1938 1939 partial = false; 1940 argsize = chunk->argsize(); // must be called *before* clearing the chunk 1941 clear_chunk(chunk); 1942 thaw_size = full_chunk_size; 1943 empty = true; 1944 } else { // thaw a single frame 1945 partial = true; 1946 thaw_size = remove_top_compiled_frame_from_chunk(chunk, argsize); 1947 empty = chunk->is_empty(); 1948 } 1949 1950 // Are we thawing the last frame(s) in the continuation 1951 const bool is_last = empty && chunk->parent() == nullptr; 1952 assert(!is_last || argsize == 0, ""); 1953 1954 log_develop_trace(continuations)("thaw_fast partial: %d is_last: %d empty: %d size: %d argsize: %d entrySP: " PTR_FORMAT, 1955 partial, is_last, empty, thaw_size, argsize, p2i(_cont.entrySP())); 1956 1957 ReconstructedStack rs(_cont.entrySP(), thaw_size, argsize); 1958 1959 // also copy metadata words at frame bottom 1960 copy_from_chunk(chunk_sp - frame::metadata_words_at_bottom, rs.top(), rs.total_size()); 1961 1962 // update the ContinuationEntry 1963 _cont.set_argsize(argsize); 1964 log_develop_trace(continuations)("setting entry argsize: %d", _cont.argsize()); 1965 assert(rs.bottom_sp() == _cont.entry()->bottom_sender_sp(), ""); 1966 1967 // install the return barrier if not last frame, or the entry's pc if last 1968 patch_return(rs.bottom_sp(), is_last); 1969 1970 // insert the back links from callee to caller frames 1971 patch_caller_links(rs.top(), rs.top() + rs.total_size()); 1972 1973 assert(is_last == _cont.is_empty(), ""); 1974 assert(_cont.chunk_invariant(), ""); 1975 1976 #if CONT_JFR 1977 EventContinuationThawFast e; 1978 if (e.should_commit()) { 1979 e.set_id(cast_from_oop<u8>(chunk)); 1980 e.set_size(thaw_size << LogBytesPerWord); 1981 e.set_full(!partial); 1982 e.commit(); 1983 } 1984 #endif 1985 1986 #ifdef ASSERT 1987 set_anchor(_thread, rs.sp()); 1988 log_frames(_thread); 1989 if (LoomDeoptAfterThaw) { 1990 do_deopt_after_thaw(_thread); 1991 } 1992 clear_anchor(_thread); 1993 #endif 1994 1995 return rs.sp(); 1996 } 1997 1998 inline bool ThawBase::seen_by_gc() { 1999 return _barriers || _cont.tail()->is_gc_mode(); 2000 } 2001 2002 NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) { 2003 LogTarget(Trace, continuations) lt; 2004 if (lt.develop_is_enabled()) { 2005 LogStream ls(lt); 2006 ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, return_barrier, p2i(chunk)); 2007 chunk->print_on(true, &ls); 2008 } 2009 2010 #if CONT_JFR 2011 EventContinuationThawSlow e; 2012 if (e.should_commit()) { 2013 e.set_id(cast_from_oop<u8>(_cont.continuation())); 2014 e.commit(); 2015 } 2016 #endif 2017 2018 DEBUG_ONLY(_frames = 0;) 2019 _align_size = 0; 2020 int num_frames = (return_barrier ? 1 : 2); 2021 2022 _stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk); 2023 _top_unextended_sp_before_thaw = _stream.unextended_sp(); 2024 2025 frame heap_frame = _stream.to_frame(); 2026 if (lt.develop_is_enabled()) { 2027 LogStream ls(lt); 2028 ls.print_cr("top hframe before (thaw):"); 2029 assert(heap_frame.is_heap_frame(), "should have created a relative frame"); 2030 heap_frame.print_value_on(&ls, nullptr); 2031 } 2032 2033 #if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC 2034 if (UseZGC || UseShenandoahGC) { 2035 _cont.tail()->relativize_derived_pointers_concurrently(); 2036 } 2037 #endif 2038 2039 frame caller; // the thawed caller on the stack 2040 recurse_thaw(heap_frame, caller, num_frames, true); 2041 finish_thaw(caller); // caller is now the topmost thawed frame 2042 _cont.write(); 2043 2044 assert(_cont.chunk_invariant(), ""); 2045 2046 JVMTI_ONLY(if (!return_barrier) invalidate_jvmti_stack(_thread)); 2047 2048 _thread->set_cont_fastpath(_fastpath); 2049 2050 intptr_t* sp = caller.sp(); 2051 return sp; 2052 } 2053 2054 void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) { 2055 log_develop_debug(continuations)("thaw num_frames: %d", num_frames); 2056 assert(!_cont.is_empty(), "no more frames"); 2057 assert(num_frames > 0, ""); 2058 assert(!heap_frame.is_empty(), ""); 2059 2060 if (top && heap_frame.is_safepoint_blob_frame()) { 2061 assert(ContinuationHelper::Frame::is_stub(heap_frame.cb()), "cb: %s", heap_frame.cb()->name()); 2062 recurse_thaw_stub_frame(heap_frame, caller, num_frames); 2063 } else if (!heap_frame.is_interpreted_frame()) { 2064 recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false); 2065 } else { 2066 recurse_thaw_interpreted_frame(heap_frame, caller, num_frames); 2067 } 2068 } 2069 2070 template<typename FKind> 2071 bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) { 2072 assert(num_frames > 0, ""); 2073 2074 DEBUG_ONLY(_frames++;) 2075 2076 int argsize = _stream.stack_argsize(); 2077 2078 _stream.next(SmallRegisterMap::instance); 2079 assert(_stream.to_frame().is_empty() == _stream.is_done(), ""); 2080 2081 // we never leave a compiled caller of an interpreted frame as the top frame in the chunk 2082 // as it makes detecting that situation and adjusting unextended_sp tricky 2083 if (num_frames == 1 && !_stream.is_done() && FKind::interpreted && _stream.is_compiled()) { 2084 log_develop_trace(continuations)("thawing extra compiled frame to not leave a compiled interpreted-caller at top"); 2085 num_frames++; 2086 } 2087 2088 if (num_frames == 1 || _stream.is_done()) { // end recursion 2089 finalize_thaw(caller, FKind::interpreted ? 0 : argsize); 2090 return true; // bottom 2091 } else { // recurse 2092 recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false); 2093 return false; 2094 } 2095 } 2096 2097 void ThawBase::finalize_thaw(frame& entry, int argsize) { 2098 stackChunkOop chunk = _cont.tail(); 2099 2100 if (!_stream.is_done()) { 2101 assert(_stream.sp() >= chunk->sp_address(), ""); 2102 chunk->set_sp(chunk->to_offset(_stream.sp())); 2103 chunk->set_pc(_stream.pc()); 2104 } else { 2105 chunk->set_sp(chunk->bottom()); 2106 chunk->set_pc(nullptr); 2107 } 2108 assert(_stream.is_done() == chunk->is_empty(), ""); 2109 2110 int total_thawed = pointer_delta_as_int(_stream.unextended_sp(), _top_unextended_sp_before_thaw); 2111 chunk->set_max_thawing_size(chunk->max_thawing_size() - total_thawed); 2112 2113 _cont.set_argsize(argsize); 2114 entry = new_entry_frame(); 2115 2116 assert(entry.sp() == _cont.entrySP(), ""); 2117 assert(Continuation::is_continuation_enterSpecial(entry), ""); 2118 assert(_cont.is_entry_frame(entry), ""); 2119 } 2120 2121 inline void ThawBase::before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame) { 2122 LogTarget(Trace, continuations) lt; 2123 if (lt.develop_is_enabled()) { 2124 LogStream ls(lt); 2125 ls.print_cr("======== THAWING FRAME: %d", num_frame); 2126 assert(hf.is_heap_frame(), "should be"); 2127 hf.print_value_on(&ls, nullptr); 2128 } 2129 assert(bottom == _cont.is_entry_frame(caller), "bottom: %d is_entry_frame: %d", bottom, _cont.is_entry_frame(hf)); 2130 } 2131 2132 inline void ThawBase::after_thaw_java_frame(const frame& f, bool bottom) { 2133 #ifdef ASSERT 2134 LogTarget(Trace, continuations) lt; 2135 if (lt.develop_is_enabled()) { 2136 LogStream ls(lt); 2137 ls.print_cr("thawed frame:"); 2138 print_frame_layout(f, false, &ls); // f.print_on(&ls); 2139 } 2140 #endif 2141 } 2142 2143 inline void ThawBase::patch(frame& f, const frame& caller, bool bottom) { 2144 assert(!bottom || caller.fp() == _cont.entryFP(), ""); 2145 if (bottom) { 2146 ContinuationHelper::Frame::patch_pc(caller, _cont.is_empty() ? caller.pc() 2147 : StubRoutines::cont_returnBarrier()); 2148 } else { 2149 // caller might have been deoptimized during thaw but we've overwritten the return address when copying f from the heap. 2150 // If the caller is not deoptimized, pc is unchanged. 2151 ContinuationHelper::Frame::patch_pc(caller, caller.raw_pc()); 2152 } 2153 2154 patch_pd(f, caller); 2155 2156 if (f.is_interpreted_frame()) { 2157 ContinuationHelper::InterpretedFrame::patch_sender_sp(f, caller); 2158 } 2159 2160 assert(!bottom || !_cont.is_empty() || Continuation::is_continuation_entry_frame(f, nullptr), ""); 2161 assert(!bottom || (_cont.is_empty() != Continuation::is_cont_barrier_frame(f)), ""); 2162 } 2163 2164 void ThawBase::clear_bitmap_bits(address start, address end) { 2165 assert(is_aligned(start, wordSize), "should be aligned: " PTR_FORMAT, p2i(start)); 2166 assert(is_aligned(end, VMRegImpl::stack_slot_size), "should be aligned: " PTR_FORMAT, p2i(end)); 2167 2168 // we need to clear the bits that correspond to arguments as they reside in the caller frame 2169 // or they will keep objects that are otherwise unreachable alive. 2170 2171 // Align `end` if UseCompressedOops is not set to avoid UB when calculating the bit index, since 2172 // `end` could be at an odd number of stack slots from `start`, i.e might not be oop aligned. 2173 // If that's the case the bit range corresponding to the last stack slot should not have bits set 2174 // anyways and we assert that before returning. 2175 address effective_end = UseCompressedOops ? end : align_down(end, wordSize); 2176 log_develop_trace(continuations)("clearing bitmap for " INTPTR_FORMAT " - " INTPTR_FORMAT, p2i(start), p2i(effective_end)); 2177 stackChunkOop chunk = _cont.tail(); 2178 chunk->bitmap().clear_range(chunk->bit_index_for(start), chunk->bit_index_for(effective_end)); 2179 assert(effective_end == end || !chunk->bitmap().at(chunk->bit_index_for(effective_end)), "bit should not be set"); 2180 } 2181 2182 NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) { 2183 assert(hf.is_interpreted_frame(), ""); 2184 2185 if (UNLIKELY(seen_by_gc())) { 2186 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance); 2187 } 2188 2189 const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::InterpretedFrame>(caller, num_frames); 2190 2191 DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);) 2192 2193 _align_size += frame::align_wiggle; // possible added alignment for internal interpreted frame alignment om AArch64 2194 2195 frame f = new_stack_frame<ContinuationHelper::InterpretedFrame>(hf, caller, is_bottom_frame); 2196 2197 intptr_t* const stack_frame_top = f.sp() + frame::metadata_words_at_top; 2198 intptr_t* const stack_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(f); 2199 intptr_t* const heap_frame_top = hf.unextended_sp() + frame::metadata_words_at_top; 2200 intptr_t* const heap_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(hf); 2201 2202 assert(hf.is_heap_frame(), "should be"); 2203 assert(!f.is_heap_frame(), "should not be"); 2204 2205 const int fsize = pointer_delta_as_int(heap_frame_bottom, heap_frame_top); 2206 assert((stack_frame_bottom == stack_frame_top + fsize), ""); 2207 2208 // Some architectures (like AArch64/PPC64/RISC-V) add padding between the locals and the fixed_frame to keep the fp 16-byte-aligned. 2209 // On those architectures we freeze the padding in order to keep the same fp-relative offsets in the fixed_frame. 2210 copy_from_chunk(heap_frame_top, stack_frame_top, fsize); 2211 2212 // Make sure the relativized locals is already set. 2213 assert(f.interpreter_frame_local_at(0) == stack_frame_bottom - 1, "invalid frame bottom"); 2214 2215 derelativize_interpreted_frame_metadata(hf, f); 2216 patch(f, caller, is_bottom_frame); 2217 2218 assert(f.is_interpreted_frame_valid(_cont.thread()), "invalid thawed frame"); 2219 assert(stack_frame_bottom <= ContinuationHelper::Frame::frame_top(caller), ""); 2220 2221 CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();) 2222 2223 maybe_set_fastpath(f.sp()); 2224 2225 const int locals = hf.interpreter_frame_method()->max_locals(); 2226 2227 if (!is_bottom_frame) { 2228 // can only fix caller once this frame is thawed (due to callee saved regs) 2229 _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance); 2230 } else if (_cont.tail()->has_bitmap() && locals > 0) { 2231 assert(hf.is_heap_frame(), "should be"); 2232 address start = (address)(heap_frame_bottom - locals); 2233 address end = (address)heap_frame_bottom; 2234 clear_bitmap_bits(start, end); 2235 } 2236 2237 DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);) 2238 caller = f; 2239 } 2240 2241 void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) { 2242 assert(!hf.is_interpreted_frame(), ""); 2243 assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption"); 2244 2245 if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap 2246 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance); 2247 } 2248 2249 const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::CompiledFrame>(caller, num_frames); 2250 2251 DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);) 2252 2253 assert(caller.sp() == caller.unextended_sp(), ""); 2254 2255 if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) { 2256 _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in freeze_interpreted_frame 2257 } 2258 2259 // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not 2260 // yet laid out in the stack, and so the original_pc is not stored in it. 2261 // As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized. 2262 frame f = new_stack_frame<ContinuationHelper::CompiledFrame>(hf, caller, is_bottom_frame); 2263 intptr_t* const stack_frame_top = f.sp(); 2264 intptr_t* const heap_frame_top = hf.unextended_sp(); 2265 2266 const int added_argsize = (is_bottom_frame || caller.is_interpreted_frame()) ? hf.compiled_frame_stack_argsize() : 0; 2267 int fsize = ContinuationHelper::CompiledFrame::size(hf) + added_argsize; 2268 assert(fsize <= (int)(caller.unextended_sp() - f.unextended_sp()), ""); 2269 2270 intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom; 2271 intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom; 2272 // copy metadata, except the metadata at the top of the (unextended) entry frame 2273 int sz = fsize + frame::metadata_words_at_bottom + (is_bottom_frame && added_argsize == 0 ? 0 : frame::metadata_words_at_top); 2274 2275 // If we're the bottom-most thawed frame, we're writing to within one word from entrySP 2276 // (we might have one padding word for alignment) 2277 assert(!is_bottom_frame || (_cont.entrySP() - 1 <= to + sz && to + sz <= _cont.entrySP()), ""); 2278 assert(!is_bottom_frame || hf.compiled_frame_stack_argsize() != 0 || (to + sz && to + sz == _cont.entrySP()), ""); 2279 2280 copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above 2281 2282 patch(f, caller, is_bottom_frame); 2283 2284 // f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above) 2285 assert(!f.is_deoptimized_frame(), ""); 2286 if (hf.is_deoptimized_frame()) { 2287 maybe_set_fastpath(f.sp()); 2288 } else if (_thread->is_interp_only_mode() 2289 || (_cont.is_preempted() && f.cb()->as_nmethod()->is_marked_for_deoptimization())) { 2290 // The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so 2291 // cannot rely on nmethod patching for deopt. 2292 assert(_thread->is_interp_only_mode() || stub_caller, "expected a stub-caller"); 2293 2294 log_develop_trace(continuations)("Deoptimizing thawed frame"); 2295 DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr)); 2296 2297 f.deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for 2298 assert(f.is_deoptimized_frame(), ""); 2299 assert(ContinuationHelper::Frame::is_deopt_return(f.raw_pc(), f), ""); 2300 maybe_set_fastpath(f.sp()); 2301 } 2302 2303 if (!is_bottom_frame) { 2304 // can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack 2305 _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance); 2306 } else if (_cont.tail()->has_bitmap() && added_argsize > 0) { 2307 address start = (address)(heap_frame_top + ContinuationHelper::CompiledFrame::size(hf) + frame::metadata_words_at_top); 2308 int stack_args_slots = f.cb()->as_nmethod()->num_stack_arg_slots(false /* rounded */); 2309 int argsize_in_bytes = stack_args_slots * VMRegImpl::stack_slot_size; 2310 clear_bitmap_bits(start, start + argsize_in_bytes); 2311 } 2312 2313 DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);) 2314 caller = f; 2315 } 2316 2317 void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) { 2318 DEBUG_ONLY(_frames++;) 2319 2320 { 2321 RegisterMap map(nullptr, 2322 RegisterMap::UpdateMap::include, 2323 RegisterMap::ProcessFrames::skip, 2324 RegisterMap::WalkContinuation::skip); 2325 map.set_include_argument_oops(false); 2326 _stream.next(&map); 2327 assert(!_stream.is_done(), ""); 2328 if (UNLIKELY(seen_by_gc())) { // we're now doing this on the stub's caller 2329 _cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map); 2330 } 2331 assert(!_stream.is_done(), ""); 2332 } 2333 2334 recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); // this could be deoptimized 2335 2336 DEBUG_ONLY(before_thaw_java_frame(hf, caller, false, num_frames);) 2337 2338 assert(ContinuationHelper::Frame::is_stub(hf.cb()), ""); 2339 assert(caller.sp() == caller.unextended_sp(), ""); 2340 assert(!caller.is_interpreted_frame(), ""); 2341 2342 int fsize = ContinuationHelper::StubFrame::size(hf); 2343 2344 frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false); 2345 intptr_t* stack_frame_top = f.sp(); 2346 intptr_t* heap_frame_top = hf.sp(); 2347 2348 copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words, 2349 fsize + frame::metadata_words); 2350 2351 { // can only fix caller once this frame is thawed (due to callee saved regs) 2352 RegisterMap map(nullptr, 2353 RegisterMap::UpdateMap::include, 2354 RegisterMap::ProcessFrames::skip, 2355 RegisterMap::WalkContinuation::skip); // map.clear(); 2356 map.set_include_argument_oops(false); 2357 f.oop_map()->update_register_map(&f, &map); 2358 ContinuationHelper::update_register_map_with_callee(caller, &map); 2359 _cont.tail()->fix_thawed_frame(caller, &map); 2360 } 2361 2362 DEBUG_ONLY(after_thaw_java_frame(f, false);) 2363 caller = f; 2364 } 2365 2366 void ThawBase::finish_thaw(frame& f) { 2367 stackChunkOop chunk = _cont.tail(); 2368 2369 if (chunk->is_empty()) { 2370 // Only remove chunk from list if it can't be reused for another freeze 2371 if (seen_by_gc()) { 2372 _cont.set_tail(chunk->parent()); 2373 } else { 2374 chunk->set_has_mixed_frames(false); 2375 } 2376 chunk->set_max_thawing_size(0); 2377 } else { 2378 chunk->set_max_thawing_size(chunk->max_thawing_size() - _align_size); 2379 } 2380 assert(chunk->is_empty() == (chunk->max_thawing_size() == 0), ""); 2381 2382 if (!is_aligned(f.sp(), frame::frame_alignment)) { 2383 assert(f.is_interpreted_frame(), ""); 2384 f.set_sp(align_down(f.sp(), frame::frame_alignment)); 2385 } 2386 push_return_frame(f); 2387 chunk->fix_thawed_frame(f, SmallRegisterMap::instance); // can only fix caller after push_return_frame (due to callee saved regs) 2388 2389 assert(_cont.is_empty() == _cont.last_frame().is_empty(), ""); 2390 2391 log_develop_trace(continuations)("thawed %d frames", _frames); 2392 2393 LogTarget(Trace, continuations) lt; 2394 if (lt.develop_is_enabled()) { 2395 LogStream ls(lt); 2396 ls.print_cr("top hframe after (thaw):"); 2397 _cont.last_frame().print_value_on(&ls, nullptr); 2398 } 2399 } 2400 2401 void ThawBase::push_return_frame(frame& f) { // see generate_cont_thaw 2402 assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), ""); 2403 assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == (f.pc() != f.raw_pc()), ""); 2404 2405 LogTarget(Trace, continuations) lt; 2406 if (lt.develop_is_enabled()) { 2407 LogStream ls(lt); 2408 ls.print_cr("push_return_frame"); 2409 f.print_value_on(&ls, nullptr); 2410 } 2411 2412 assert(f.sp() - frame::metadata_words_at_bottom >= _top_stack_address, "overwrote past thawing space" 2413 " to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(f.sp() - frame::metadata_words), p2i(_top_stack_address)); 2414 ContinuationHelper::Frame::patch_pc(f, f.raw_pc()); // in case we want to deopt the frame in a full transition, this is checked. 2415 ContinuationHelper::push_pd(f); 2416 2417 assert(ContinuationHelper::Frame::assert_frame_laid_out(f), ""); 2418 } 2419 2420 // returns new top sp 2421 // called after preparations (stack overflow check and making room) 2422 template<typename ConfigT> 2423 static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind) { 2424 assert(thread == JavaThread::current(), "Must be current thread"); 2425 2426 CONT_JFR_ONLY(EventContinuationThaw event;) 2427 2428 log_develop_trace(continuations)("~~~~ thaw kind: %d sp: " INTPTR_FORMAT, kind, p2i(thread->last_continuation()->entry_sp())); 2429 2430 ContinuationEntry* entry = thread->last_continuation(); 2431 assert(entry != nullptr, ""); 2432 oop oopCont = entry->cont_oop(thread); 2433 2434 assert(!jdk_internal_vm_Continuation::done(oopCont), ""); 2435 assert(oopCont == get_continuation(thread), ""); 2436 verify_continuation(oopCont); 2437 2438 assert(entry->is_virtual_thread() == (entry->scope(thread) == java_lang_VirtualThread::vthread_scope()), ""); 2439 2440 ContinuationWrapper cont(thread, oopCont); 2441 log_develop_debug(continuations)("THAW #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont)); 2442 2443 #ifdef ASSERT 2444 set_anchor_to_entry(thread, cont.entry()); 2445 log_frames(thread); 2446 clear_anchor(thread); 2447 #endif 2448 2449 Thaw<ConfigT> thw(thread, cont); 2450 intptr_t* const sp = thw.thaw(kind); 2451 assert(is_aligned(sp, frame::frame_alignment), ""); 2452 2453 // All or part of the frames have been thawed so we know they don't hold any monitors except JNI monitors. 2454 assert(thread->held_monitor_count() == thread->jni_monitor_count(), "Must be"); 2455 2456 #ifdef ASSERT 2457 intptr_t* sp0 = sp; 2458 set_anchor(thread, sp0); 2459 log_frames(thread); 2460 if (LoomVerifyAfterThaw) { 2461 assert(do_verify_after_thaw(thread, cont.tail(), tty), ""); 2462 } 2463 assert(ContinuationEntry::assert_entry_frame_laid_out(thread), ""); 2464 clear_anchor(thread); 2465 2466 LogTarget(Trace, continuations) lt; 2467 if (lt.develop_is_enabled()) { 2468 LogStream ls(lt); 2469 ls.print_cr("Jumping to frame (thaw):"); 2470 frame(sp).print_value_on(&ls, nullptr); 2471 } 2472 #endif 2473 2474 CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);) 2475 2476 verify_continuation(cont.continuation()); 2477 log_develop_debug(continuations)("=== End of thaw #" INTPTR_FORMAT, cont.hash()); 2478 2479 return sp; 2480 } 2481 2482 #ifdef ASSERT 2483 static void do_deopt_after_thaw(JavaThread* thread) { 2484 int i = 0; 2485 StackFrameStream fst(thread, true, false); 2486 fst.register_map()->set_include_argument_oops(false); 2487 ContinuationHelper::update_register_map_with_callee(*fst.current(), fst.register_map()); 2488 for (; !fst.is_done(); fst.next()) { 2489 if (fst.current()->cb()->is_nmethod()) { 2490 nmethod* nm = fst.current()->cb()->as_nmethod(); 2491 if (!nm->method()->is_continuation_native_intrinsic()) { 2492 nm->make_deoptimized(); 2493 } 2494 } 2495 } 2496 } 2497 2498 class ThawVerifyOopsClosure: public OopClosure { 2499 intptr_t* _p; 2500 outputStream* _st; 2501 bool is_good_oop(oop o) { 2502 return dbg_is_safe(o, -1) && dbg_is_safe(o->klass(), -1) && oopDesc::is_oop(o) && o->klass()->is_klass(); 2503 } 2504 public: 2505 ThawVerifyOopsClosure(outputStream* st) : _p(nullptr), _st(st) {} 2506 intptr_t* p() { return _p; } 2507 void reset() { _p = nullptr; } 2508 2509 virtual void do_oop(oop* p) { 2510 oop o = *p; 2511 if (o == nullptr || is_good_oop(o)) { 2512 return; 2513 } 2514 _p = (intptr_t*)p; 2515 _st->print_cr("*** non-oop " PTR_FORMAT " found at " PTR_FORMAT, p2i(*p), p2i(p)); 2516 } 2517 virtual void do_oop(narrowOop* p) { 2518 oop o = RawAccess<>::oop_load(p); 2519 if (o == nullptr || is_good_oop(o)) { 2520 return; 2521 } 2522 _p = (intptr_t*)p; 2523 _st->print_cr("*** (narrow) non-oop %x found at " PTR_FORMAT, (int)(*p), p2i(p)); 2524 } 2525 }; 2526 2527 static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st) { 2528 assert(thread->has_last_Java_frame(), ""); 2529 2530 ResourceMark rm; 2531 ThawVerifyOopsClosure cl(st); 2532 NMethodToOopClosure cf(&cl, false); 2533 2534 StackFrameStream fst(thread, true, false); 2535 fst.register_map()->set_include_argument_oops(false); 2536 ContinuationHelper::update_register_map_with_callee(*fst.current(), fst.register_map()); 2537 for (; !fst.is_done() && !Continuation::is_continuation_enterSpecial(*fst.current()); fst.next()) { 2538 if (fst.current()->cb()->is_nmethod() && fst.current()->cb()->as_nmethod()->is_marked_for_deoptimization()) { 2539 st->print_cr(">>> do_verify_after_thaw deopt"); 2540 fst.current()->deoptimize(nullptr); 2541 fst.current()->print_on(st); 2542 } 2543 2544 fst.current()->oops_do(&cl, &cf, fst.register_map()); 2545 if (cl.p() != nullptr) { 2546 frame fr = *fst.current(); 2547 st->print_cr("Failed for frame barriers: %d",chunk->requires_barriers()); 2548 fr.print_on(st); 2549 if (!fr.is_interpreted_frame()) { 2550 st->print_cr("size: %d argsize: %d", 2551 ContinuationHelper::NonInterpretedUnknownFrame::size(fr), 2552 ContinuationHelper::NonInterpretedUnknownFrame::stack_argsize(fr)); 2553 } 2554 VMReg reg = fst.register_map()->find_register_spilled_here(cl.p(), fst.current()->sp()); 2555 if (reg != nullptr) { 2556 st->print_cr("Reg %s %d", reg->name(), reg->is_stack() ? (int)reg->reg2stack() : -99); 2557 } 2558 cl.reset(); 2559 DEBUG_ONLY(thread->print_frame_layout();) 2560 if (chunk != nullptr) { 2561 chunk->print_on(true, st); 2562 } 2563 return false; 2564 } 2565 } 2566 return true; 2567 } 2568 2569 static void log_frames(JavaThread* thread) { 2570 const static int show_entry_callers = 3; 2571 LogTarget(Trace, continuations) lt; 2572 if (!lt.develop_is_enabled()) { 2573 return; 2574 } 2575 LogStream ls(lt); 2576 2577 ls.print_cr("------- frames ---------"); 2578 if (!thread->has_last_Java_frame()) { 2579 ls.print_cr("NO ANCHOR!"); 2580 } 2581 2582 RegisterMap map(thread, 2583 RegisterMap::UpdateMap::include, 2584 RegisterMap::ProcessFrames::include, 2585 RegisterMap::WalkContinuation::skip); 2586 map.set_include_argument_oops(false); 2587 2588 if (false) { 2589 for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) { 2590 f.print_on(&ls); 2591 } 2592 } else { 2593 map.set_skip_missing(true); 2594 ResetNoHandleMark rnhm; 2595 ResourceMark rm; 2596 HandleMark hm(Thread::current()); 2597 FrameValues values; 2598 2599 int i = 0; 2600 int post_entry = -1; 2601 for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) { 2602 f.describe(values, i++, &map); 2603 if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f)) 2604 post_entry++; 2605 if (post_entry >= show_entry_callers) 2606 break; 2607 } 2608 values.print_on(thread, &ls); 2609 } 2610 2611 ls.print_cr("======= end frames ========="); 2612 } 2613 #endif // ASSERT 2614 2615 #include CPU_HEADER_INLINE(continuationFreezeThaw) 2616 2617 #ifdef ASSERT 2618 static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st) { 2619 ResourceMark rm; 2620 FrameValues values; 2621 assert(f.get_cb() != nullptr, ""); 2622 RegisterMap map(f.is_heap_frame() ? 2623 nullptr : 2624 JavaThread::current(), 2625 RegisterMap::UpdateMap::include, 2626 RegisterMap::ProcessFrames::skip, 2627 RegisterMap::WalkContinuation::skip); 2628 map.set_include_argument_oops(false); 2629 map.set_skip_missing(true); 2630 if (callee_complete) { 2631 frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f)); 2632 } 2633 const_cast<frame&>(f).describe(values, 0, &map); 2634 values.print_on(static_cast<JavaThread*>(nullptr), st); 2635 } 2636 #endif 2637 2638 static address thaw_entry = nullptr; 2639 static address freeze_entry = nullptr; 2640 2641 address Continuation::thaw_entry() { 2642 return ::thaw_entry; 2643 } 2644 2645 address Continuation::freeze_entry() { 2646 return ::freeze_entry; 2647 } 2648 2649 class ConfigResolve { 2650 public: 2651 static void resolve() { resolve_compressed(); } 2652 2653 static void resolve_compressed() { 2654 UseCompressedOops ? resolve_gc<true>() 2655 : resolve_gc<false>(); 2656 } 2657 2658 private: 2659 template <bool use_compressed> 2660 static void resolve_gc() { 2661 BarrierSet* bs = BarrierSet::barrier_set(); 2662 assert(bs != nullptr, "freeze/thaw invoked before BarrierSet is set"); 2663 switch (bs->kind()) { 2664 #define BARRIER_SET_RESOLVE_BARRIER_CLOSURE(bs_name) \ 2665 case BarrierSet::bs_name: { \ 2666 resolve<use_compressed, typename BarrierSet::GetType<BarrierSet::bs_name>::type>(); \ 2667 } \ 2668 break; 2669 FOR_EACH_CONCRETE_BARRIER_SET_DO(BARRIER_SET_RESOLVE_BARRIER_CLOSURE) 2670 #undef BARRIER_SET_RESOLVE_BARRIER_CLOSURE 2671 2672 default: 2673 fatal("BarrierSet resolving not implemented"); 2674 }; 2675 } 2676 2677 template <bool use_compressed, typename BarrierSetT> 2678 static void resolve() { 2679 typedef Config<use_compressed ? oop_kind::NARROW : oop_kind::WIDE, BarrierSetT> SelectedConfigT; 2680 2681 freeze_entry = (address)freeze<SelectedConfigT>; 2682 2683 // If we wanted, we could templatize by kind and have three different thaw entries 2684 thaw_entry = (address)thaw<SelectedConfigT>; 2685 } 2686 }; 2687 2688 void Continuation::init() { 2689 ConfigResolve::resolve(); 2690 }