1 /* 2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/vmSymbols.hpp" 27 #include "gc/shared/barrierSetNMethod.hpp" 28 #include "oops/method.inline.hpp" 29 #include "oops/oop.inline.hpp" 30 #include "prims/jvmtiThreadState.inline.hpp" 31 #include "runtime/continuation.hpp" 32 #include "runtime/continuationEntry.inline.hpp" 33 #include "runtime/continuationHelper.inline.hpp" 34 #include "runtime/continuationJavaClasses.inline.hpp" 35 #include "runtime/continuationWrapper.inline.hpp" 36 #include "runtime/interfaceSupport.inline.hpp" 37 #include "runtime/javaThread.inline.hpp" 38 #include "runtime/jniHandles.inline.hpp" 39 #include "runtime/osThread.hpp" 40 #include "runtime/vframe.inline.hpp" 41 #include "runtime/vframe_hp.hpp" 42 43 // defined in continuationFreezeThaw.cpp 44 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope); 45 46 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) { 47 if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) { 48 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow"); 49 } 50 } 51 JVM_END 52 53 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) { 54 if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) { 55 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow"); 56 } 57 } 58 JVM_END 59 60 #if INCLUDE_JVMTI 61 class JvmtiUnmountBeginMark : public StackObj { 62 Handle _vthread; 63 JavaThread* _target; 64 int _preempt_result; 65 bool _failed; 66 67 public: 68 JvmtiUnmountBeginMark(JavaThread* t) : 69 _vthread(t, t->vthread()), _target(t), _preempt_result(freeze_pinned_native), _failed(false) { 70 assert(!_target->is_in_any_VTMS_transition(), "must be"); 71 72 if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) { 73 JvmtiVTMSTransitionDisabler::start_VTMS_transition((jthread)_vthread.raw_value(), /* is_mount */ false); 74 75 // Don't preempt if there is a pending popframe or earlyret operation. This can 76 // be installed in start_VTMS_transition() so we need to check it here. 77 if (JvmtiExport::can_pop_frame() || JvmtiExport::can_force_early_return()) { 78 JvmtiThreadState* state = _target->jvmti_thread_state(); 79 if (_target->has_pending_popframe() || (state != nullptr && state->is_earlyret_pending())) { 80 _failed = true; 81 } 82 } 83 84 // Don't preempt in case there is an async exception installed since 85 // we would incorrectly throw it during the unmount logic in the carrier. 86 if (_target->has_async_exception_condition()) { 87 _failed = true; 88 } 89 } else { 90 _target->set_is_in_VTMS_transition(true); 91 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), true); 92 } 93 } 94 ~JvmtiUnmountBeginMark() { 95 assert(!_target->is_suspended(), "must be"); 96 97 assert(_target->is_in_VTMS_transition(), "must be"); 98 assert(java_lang_Thread::is_in_VTMS_transition(_vthread()), "must be"); 99 100 // Read it again since for late binding agents the flag could have 101 // been set while blocked in the allocation path during freeze. 102 bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events(); 103 104 if (_preempt_result != freeze_ok) { 105 // Undo transition 106 if (jvmti_present) { 107 JvmtiVTMSTransitionDisabler::finish_VTMS_transition((jthread)_vthread.raw_value(), false); 108 } else { 109 _target->set_is_in_VTMS_transition(false); 110 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), false); 111 } 112 } else { 113 if (jvmti_present) { 114 _target->rebind_to_jvmti_thread_state_of(_target->threadObj()); 115 if (JvmtiExport::should_post_vthread_unmount()) { 116 // We are inside the VTMS transition already so we will post the event 117 // once we finish it in JvmtiVTMSTransitionDisabler::VTMS_unmount_end(). 118 _target->set_pending_jvmti_unmount_event(true); 119 } 120 } 121 } 122 } 123 void set_preempt_result(int res) { _preempt_result = res; } 124 bool failed() { return _failed; } 125 }; 126 127 static bool is_vthread_safe_to_preempt_for_jvmti(JavaThread* target, oop vthread) { 128 if (target->is_in_any_VTMS_transition()) { 129 // We caught target at the end of a mount transition (is_in_VTMS_transition()) or at the 130 // beginning or end of a temporary switch to carrier thread (is_in_tmp_VTMS_transition()). 131 return false; 132 } 133 return true; 134 } 135 #endif // INCLUDE_JVMTI 136 137 static bool is_vthread_safe_to_preempt(JavaThread* target, oop vthread) { 138 if (!java_lang_VirtualThread::is_instance(vthread) || // inside tmp transition 139 java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) { // inside transition 140 return false; 141 } 142 return JVMTI_ONLY(is_vthread_safe_to_preempt_for_jvmti(target, vthread)) NOT_JVMTI(true); 143 } 144 145 typedef int (*FreezeContFnT)(JavaThread*, intptr_t*); 146 147 static void verify_preempt_preconditions(JavaThread* target, oop continuation) { 148 assert(target == JavaThread::current(), "no support for external preemption"); 149 assert(target->has_last_Java_frame(), ""); 150 assert(!target->preempting(), ""); 151 assert(target->last_continuation() != nullptr, ""); 152 assert(target->last_continuation()->cont_oop(target) == continuation, ""); 153 assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), ""); 154 assert(!target->has_pending_exception(), ""); 155 } 156 157 int Continuation::try_preempt(JavaThread* target, oop continuation) { 158 verify_preempt_preconditions(target, continuation); 159 160 if (LockingMode == LM_LEGACY) { 161 return freeze_unsupported; 162 } 163 164 if (!is_vthread_safe_to_preempt(target, target->vthread())) { 165 return freeze_pinned_native; 166 } 167 168 JVMTI_ONLY(JvmtiUnmountBeginMark jubm(target);) 169 JVMTI_ONLY(if (jubm.failed()) return freeze_pinned_native;) 170 int res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(target, target->last_Java_sp()); 171 log_trace(continuations, preempt)("try_preempt: %d", res); 172 JVMTI_ONLY(jubm.set_preempt_result(res);) 173 return res; 174 } 175 176 #ifndef PRODUCT 177 static jlong java_tid(JavaThread* thread) { 178 return java_lang_Thread::thread_id(thread->threadObj()); 179 } 180 #endif 181 182 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) { 183 if (thread == nullptr || continuation == nullptr) { 184 return nullptr; 185 } 186 187 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) { 188 if (continuation == entry->cont_oop(thread)) { 189 return entry; 190 } 191 } 192 return nullptr; 193 } 194 195 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) { 196 if (entry == nullptr) { 197 return false; 198 } 199 200 assert(thread->is_in_full_stack((address)entry), ""); 201 return true; 202 // return false if called when transitioning to Java on return from freeze 203 // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp(); 204 } 205 206 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) { 207 return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation)); 208 } 209 210 // When walking the virtual stack, this method returns true 211 // iff the frame is a thawed continuation frame whose 212 // caller is still frozen on the h-stack. 213 // The continuation object can be extracted from the thread. 214 bool Continuation::is_cont_barrier_frame(const frame& f) { 215 assert(f.is_interpreted_frame() || f.cb() != nullptr, ""); 216 if (!Continuations::enabled()) return false; 217 return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f) 218 : ContinuationHelper::CompiledFrame::return_pc(f)); 219 } 220 221 bool Continuation::is_return_barrier_entry(const address pc) { 222 if (!Continuations::enabled()) return false; 223 return pc == StubRoutines::cont_returnBarrier(); 224 } 225 226 bool Continuation::is_continuation_enterSpecial(const frame& f) { 227 if (f.cb() == nullptr || !f.cb()->is_nmethod()) { 228 return false; 229 } 230 Method* m = f.cb()->as_nmethod()->method(); 231 return (m != nullptr && m->is_continuation_enter_intrinsic()); 232 } 233 234 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) { 235 // we can do this because the entry frame is never inlined 236 Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame()) 237 ? map->stack_chunk()->interpreter_frame_method(f) 238 : ContinuationHelper::Frame::frame_method(f); 239 return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter; 240 } 241 242 // The parameter `sp` should be the actual sp and not the unextended sp because at 243 // least on PPC64 unextended_sp < sp is possible as interpreted frames are trimmed 244 // to the actual size of the expression stack before calls. The problem there is 245 // that even unextended_sp < entry_sp < sp is possible for an interpreted frame. 246 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) { 247 // entry_sp() returns the unextended_sp which is always greater or equal to the actual sp 248 return entry->entry_sp() > sp; 249 } 250 251 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) { 252 return is_sp_in_continuation(entry, f.sp()); 253 } 254 255 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) { 256 assert(thread != nullptr, ""); 257 ContinuationEntry* entry = thread->last_continuation(); 258 while (entry != nullptr && !is_sp_in_continuation(entry, sp)) { 259 entry = entry->parent(); 260 } 261 return entry; 262 } 263 264 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) { 265 assert(is_continuation_enterSpecial(f), ""); 266 ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp(); 267 assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry"); 268 return entry; 269 } 270 271 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) { 272 return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.sp()) != nullptr); 273 } 274 275 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) { 276 stackChunkOop chunk = cont.last_nonempty_chunk(); 277 map->set_stack_chunk(chunk); 278 return chunk != nullptr ? chunk->top_frame(map) : frame(); 279 } 280 281 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) { 282 ContinuationWrapper cont(continuation); 283 if (!cont.is_empty()) { 284 *frame = continuation_top_frame(cont, map); 285 return true; 286 } else { 287 return false; 288 } 289 } 290 291 frame Continuation::last_frame(oop continuation, RegisterMap *map) { 292 assert(map != nullptr, "a map must be given"); 293 return continuation_top_frame(ContinuationWrapper(continuation), map); 294 } 295 296 frame Continuation::top_frame(const frame& callee, RegisterMap* map) { 297 assert(map != nullptr, ""); 298 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp()); 299 assert(ce != nullptr, ""); 300 oop continuation = ce->cont_oop(map->thread()); 301 ContinuationWrapper cont(continuation); 302 return continuation_top_frame(cont, map); 303 } 304 305 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) { 306 assert(map != nullptr, "a map must be given"); 307 if (!ContinuationWrapper(continuation()).is_empty()) { 308 frame f = last_frame(continuation(), map); 309 for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) { 310 if (vf->is_java_frame()) { 311 return javaVFrame::cast(vf); 312 } 313 } 314 } 315 return nullptr; 316 } 317 318 frame Continuation::continuation_parent_frame(RegisterMap* map) { 319 assert(map->in_cont(), ""); 320 ContinuationWrapper cont(map); 321 assert(map->thread() != nullptr || !cont.is_mounted(), ""); 322 323 log_develop_trace(continuations)("continuation_parent_frame"); 324 if (map->update_map()) { 325 // we need to register the link address for the entry frame 326 if (cont.entry() != nullptr) { 327 cont.entry()->update_register_map(map); 328 } else { 329 map->clear(); 330 } 331 } 332 333 if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end 334 oop parent = jdk_internal_vm_Continuation::parent(cont.continuation()); 335 stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr; 336 if (chunk != nullptr) { 337 return chunk->top_frame(map); 338 } 339 340 map->set_stack_chunk(nullptr); 341 return frame(); 342 } 343 344 map->set_stack_chunk(nullptr); 345 346 #if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO) 347 frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC()); 348 #else 349 frame sender = frame(); 350 Unimplemented(); 351 #endif 352 353 return sender; 354 } 355 356 oop Continuation::continuation_scope(oop continuation) { 357 return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr; 358 } 359 360 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) { 361 if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) { 362 return false; 363 } 364 365 oop continuation; 366 if (map->in_cont()) { 367 continuation = map->cont(); 368 } else { 369 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp()); 370 if (ce == nullptr) { 371 return false; 372 } 373 continuation = ce->cont_oop(map->thread()); 374 } 375 if (continuation == nullptr) { 376 return false; 377 } 378 379 oop sc = continuation_scope(continuation); 380 assert(sc != nullptr, ""); 381 return sc == cont_scope; 382 } 383 384 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) { 385 ContinuationWrapper cont(map); 386 stackChunkOop chunk = cont.find_chunk_by_address(addr); 387 return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false; 388 } 389 390 bool Continuation::pin(JavaThread* current) { 391 ContinuationEntry* ce = current->last_continuation(); 392 if (ce == nullptr) { 393 return true; // no continuation mounted 394 } 395 return ce->pin(); 396 } 397 398 bool Continuation::unpin(JavaThread* current) { 399 ContinuationEntry* ce = current->last_continuation(); 400 if (ce == nullptr) { 401 return true; // no continuation mounted 402 } 403 return ce->unpin(); 404 } 405 406 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) { 407 assert (thread != nullptr, ""); 408 ContinuationEntry* ce = get_continuation_entry_for_sp(thread, callee.sp()); 409 assert(ce != nullptr, "callee.sp(): " INTPTR_FORMAT, p2i(callee.sp())); 410 411 log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT 412 " sender_sp: " INTPTR_FORMAT, 413 java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp)); 414 415 frame entry = ce->to_frame(); 416 if (callee.is_interpreted_frame()) { 417 entry.set_sp(sender_sp); // sp != unextended_sp 418 } 419 return entry; 420 } 421 422 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) { 423 ContinuationEntry* ce; 424 if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) { 425 return ce->entry_pc(); 426 } 427 return pc; 428 } 429 430 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) { 431 assert(thread != nullptr, ""); 432 bool fast = !thread->is_interp_only_mode(); 433 thread->set_cont_fastpath_thread_state(fast); 434 } 435 436 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) { 437 ContinuationEntry* entry = thread->last_continuation(); 438 439 if (entry == nullptr) { 440 return; 441 } 442 443 if (is_sp_in_continuation(entry, sp)) { 444 thread->push_cont_fastpath(sp); 445 return; 446 } 447 448 ContinuationEntry* prev; 449 do { 450 prev = entry; 451 entry = entry->parent(); 452 } while (entry != nullptr && !is_sp_in_continuation(entry, sp)); 453 454 if (entry == nullptr) { 455 return; 456 } 457 assert(is_sp_in_continuation(entry, sp), ""); 458 if (sp > prev->parent_cont_fastpath()) { 459 prev->set_parent_cont_fastpath(sp); 460 } 461 } 462 463 #ifndef PRODUCT 464 void Continuation::describe(FrameValues &values) { 465 JavaThread* thread = JavaThread::active(); 466 if (thread != nullptr) { 467 for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) { 468 intptr_t* bottom = ce->entry_sp(); 469 if (bottom != nullptr) { 470 values.describe(-1, bottom, "continuation entry"); 471 } 472 } 473 } 474 } 475 #endif 476 477 #ifdef ASSERT 478 void Continuation::debug_verify_continuation(oop contOop) { 479 if (!VerifyContinuations) { 480 return; 481 } 482 assert(contOop != nullptr, ""); 483 assert(oopDesc::is_oop(contOop), ""); 484 ContinuationWrapper cont(contOop); 485 486 assert(oopDesc::is_oop_or_null(cont.tail()), ""); 487 assert(cont.chunk_invariant(), ""); 488 489 bool nonempty_chunk = false; 490 size_t max_size = 0; 491 int num_chunks = 0; 492 int num_frames = 0; 493 int num_interpreted_frames = 0; 494 int num_oops = 0; 495 496 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 497 log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks); 498 chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames); 499 if (!chunk->is_empty()) { 500 nonempty_chunk = true; 501 } 502 num_chunks++; 503 } 504 505 const bool is_empty = cont.is_empty(); 506 assert(!nonempty_chunk || !is_empty, ""); 507 assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), ""); 508 } 509 510 void Continuation::print(oop continuation) { print_on(tty, continuation); } 511 512 void Continuation::print_on(outputStream* st, oop continuation) { 513 ContinuationWrapper cont(continuation); 514 515 st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d", 516 continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation)); 517 st->print_cr("CHUNKS:"); 518 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 519 st->print("* "); 520 chunk->print_on(true, st); 521 } 522 } 523 #endif // ASSERT 524 525 526 void continuations_init() { Continuations::init(); } 527 528 void Continuations::init() { 529 Continuation::init(); 530 } 531 532 bool Continuations::enabled() { 533 return VMContinuations; 534 } 535 536 #define CC (char*) /*cast a literal from (const char*)*/ 537 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) 538 539 static JNINativeMethod CONT_methods[] = { 540 {CC"pin", CC"()V", FN_PTR(CONT_pin)}, 541 {CC"unpin", CC"()V", FN_PTR(CONT_unpin)}, 542 {CC"isPinned0", CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)}, 543 }; 544 545 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) { 546 JavaThread* thread = JavaThread::current(); 547 ThreadToNativeFromVM trans(thread); 548 int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod)); 549 guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives"); 550 guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives"); 551 }