1 /* 2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/vmSymbols.hpp" 27 #include "gc/shared/barrierSetNMethod.hpp" 28 #include "oops/method.inline.hpp" 29 #include "oops/oop.inline.hpp" 30 #include "prims/jvmtiThreadState.inline.hpp" 31 #include "runtime/continuation.hpp" 32 #include "runtime/continuationEntry.inline.hpp" 33 #include "runtime/continuationHelper.inline.hpp" 34 #include "runtime/continuationJavaClasses.inline.hpp" 35 #include "runtime/continuationWrapper.inline.hpp" 36 #include "runtime/interfaceSupport.inline.hpp" 37 #include "runtime/javaThread.inline.hpp" 38 #include "runtime/jniHandles.inline.hpp" 39 #include "runtime/osThread.hpp" 40 #include "runtime/vframe.inline.hpp" 41 #include "runtime/vframe_hp.hpp" 42 43 // defined in continuationFreezeThaw.cpp 44 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope); 45 46 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) { 47 if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) { 48 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow"); 49 } 50 } 51 JVM_END 52 53 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) { 54 if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) { 55 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow"); 56 } 57 } 58 JVM_END 59 60 #if INCLUDE_JVMTI 61 class JvmtiUnmountBeginMark : public StackObj { 62 JavaThread* _target; 63 int _preempt_result; 64 65 public: 66 JvmtiUnmountBeginMark(JavaThread* t) : 67 _target(t), _preempt_result(freeze_pinned_native) { 68 assert(!_target->is_in_any_VTMS_transition(), "must be"); 69 70 if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) { 71 JvmtiVTMSTransitionDisabler::start_VTMS_transition(JNIHandles::make_local(_target->vthread()), /* is_mount */ false); 72 } else { 73 _target->set_is_in_VTMS_transition(true); 74 java_lang_Thread::set_is_in_VTMS_transition(_target->vthread(), true); 75 } 76 } 77 ~JvmtiUnmountBeginMark() { 78 assert(!_target->is_suspended(), "must be"); 79 80 assert(_target->is_in_VTMS_transition(), "must be"); 81 assert(java_lang_Thread::is_in_VTMS_transition(_target->vthread()), "must be"); 82 83 // Read it again since for late binding agents the flag could have 84 // been set while blocked in the allocation path during freeze. 85 bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events(); 86 87 if (_preempt_result != freeze_ok) { 88 // Undo transition 89 if (jvmti_present) { 90 JvmtiVTMSTransitionDisabler::finish_VTMS_transition(JNIHandles::make_local(_target->vthread()), false); 91 } else { 92 _target->set_is_in_VTMS_transition(false); 93 java_lang_Thread::set_is_in_VTMS_transition(_target->vthread(), false); 94 } 95 } else { 96 if (jvmti_present) { 97 _target->rebind_to_jvmti_thread_state_of(_target->threadObj()); 98 _target->set_jvmti_unmount_event_pending(true); 99 } 100 } 101 } 102 void set_preempt_result(int res) { _preempt_result = res; } 103 }; 104 105 static bool is_safe_vthread_to_preempt_for_jvmti(JavaThread* target, oop vthread) { 106 assert(!target->has_pending_popframe(), "should be true; no support for vthreads yet"); 107 JvmtiThreadState* state = target->jvmti_thread_state(); 108 assert(state == nullptr || !state->is_earlyret_pending(), "should be true; no support for vthreads yet"); 109 110 if (target->is_in_any_VTMS_transition()) { 111 // We caught target at the end of a mount transition (is_in_VTMS_transition()) or at the 112 // beginning or end of a temporary switch to carrier thread (is_in_tmp_VTMS_transition()). 113 return false; 114 } 115 return true; 116 } 117 #endif 118 119 static bool is_safe_vthread_to_preempt(JavaThread* target, oop vthread) { 120 if (!java_lang_VirtualThread::is_instance(vthread) || // inside transition 121 java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) { // inside transition 122 return false; 123 } 124 return JVMTI_ONLY(is_safe_vthread_to_preempt_for_jvmti(target, vthread)) NOT_JVMTI(true); 125 } 126 127 typedef int (*FreezeContFnT)(JavaThread*, intptr_t*); 128 129 int Continuation::try_preempt(JavaThread* target, oop continuation, bool set_state_yielding) { 130 assert(target->is_on_monitorenter(), "must be"); 131 assert(target->has_last_Java_frame(), "must be"); 132 assert(!target->preempting(), "must be"); 133 assert(!target->is_suspended() || target->is_disable_suspend(), "must be"); 134 assert(!target->has_pending_exception(), "must be"); 135 assert(target->last_continuation() != nullptr, "must be"); 136 assert(target->last_continuation()->cont_oop(target) == continuation, "must be"); 137 assert(!is_continuation_preempted(continuation), "shouldn't be"); 138 assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), "must be"); 139 140 if (!VM_Version::supports_cont_preemption()) { 141 return unsupported; 142 } 143 144 if (is_continuation_done(continuation)) { 145 return freeze_not_mounted; 146 } 147 148 // Continuation is mounted and it's not done so check if it's safe to preempt. 149 if (!is_safe_vthread_to_preempt(target, target->vthread())) { 150 return freeze_pinned_native; 151 } 152 153 #if INCLUDE_JVMTI 154 JvmtiUnmountBeginMark jubm(target); 155 #endif 156 target->set_preempting(true); 157 int res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(target, target->last_Java_sp()); 158 log_trace(continuations, preempt)("try_preempt: %d", res); 159 JVMTI_ONLY(jubm.set_preempt_result(res);) 160 if (res != freeze_ok) { 161 target->set_preempting(false); 162 } else if (set_state_yielding) { 163 java_lang_VirtualThread::set_state(target->vthread(), java_lang_VirtualThread::YIELDING); 164 } 165 return res; 166 } 167 168 bool Continuation::is_continuation_preempted(oop cont) { 169 return jdk_internal_vm_Continuation::is_preempted(cont); 170 } 171 172 bool Continuation::is_continuation_done(oop cont) { 173 return jdk_internal_vm_Continuation::done(cont); 174 } 175 176 #ifndef PRODUCT 177 static jlong java_tid(JavaThread* thread) { 178 return java_lang_Thread::thread_id(thread->threadObj()); 179 } 180 #endif 181 182 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) { 183 if (thread == nullptr || continuation == nullptr) { 184 return nullptr; 185 } 186 187 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) { 188 if (continuation == entry->cont_oop(thread)) { 189 return entry; 190 } 191 } 192 return nullptr; 193 } 194 195 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) { 196 if (entry == nullptr) { 197 return false; 198 } 199 200 assert(thread->is_in_full_stack((address)entry), ""); 201 return true; 202 // return false if called when transitioning to Java on return from freeze 203 // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp(); 204 } 205 206 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) { 207 return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation)); 208 } 209 210 // When walking the virtual stack, this method returns true 211 // iff the frame is a thawed continuation frame whose 212 // caller is still frozen on the h-stack. 213 // The continuation object can be extracted from the thread. 214 bool Continuation::is_cont_barrier_frame(const frame& f) { 215 assert(f.is_interpreted_frame() || f.cb() != nullptr, ""); 216 if (!Continuations::enabled()) return false; 217 return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f) 218 : ContinuationHelper::CompiledFrame::return_pc(f)); 219 } 220 221 bool Continuation::is_return_barrier_entry(const address pc) { 222 if (!Continuations::enabled()) return false; 223 return pc == StubRoutines::cont_returnBarrier(); 224 } 225 226 bool Continuation::is_continuation_enterSpecial(const frame& f) { 227 if (f.cb() == nullptr || !f.cb()->is_nmethod()) { 228 return false; 229 } 230 Method* m = f.cb()->as_nmethod()->method(); 231 return (m != nullptr && m->is_continuation_enter_intrinsic()); 232 } 233 234 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) { 235 // we can do this because the entry frame is never inlined 236 Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame()) 237 ? map->stack_chunk()->interpreter_frame_method(f) 238 : ContinuationHelper::Frame::frame_method(f); 239 return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter; 240 } 241 242 // The parameter `sp` should be the actual sp and not the unextended sp because at 243 // least on PPC64 unextended_sp < sp is possible as interpreted frames are trimmed 244 // to the actual size of the expression stack before calls. The problem there is 245 // that even unextended_sp < entry_sp < sp is possible for an interpreted frame. 246 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) { 247 // entry_sp() returns the unextended_sp which is always greater or equal to the actual sp 248 return entry->entry_sp() > sp; 249 } 250 251 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) { 252 return is_sp_in_continuation(entry, f.sp()); 253 } 254 255 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) { 256 assert(thread != nullptr, ""); 257 ContinuationEntry* entry = thread->last_continuation(); 258 while (entry != nullptr && !is_sp_in_continuation(entry, sp)) { 259 entry = entry->parent(); 260 } 261 return entry; 262 } 263 264 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) { 265 assert(is_continuation_enterSpecial(f), ""); 266 ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp(); 267 assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry"); 268 return entry; 269 } 270 271 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) { 272 return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.sp()) != nullptr); 273 } 274 275 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) { 276 stackChunkOop chunk = cont.last_nonempty_chunk(); 277 map->set_stack_chunk(chunk); 278 return chunk != nullptr ? chunk->top_frame(map) : frame(); 279 } 280 281 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) { 282 ContinuationWrapper cont(continuation); 283 if (!cont.is_empty()) { 284 *frame = continuation_top_frame(cont, map); 285 return true; 286 } else { 287 return false; 288 } 289 } 290 291 frame Continuation::last_frame(oop continuation, RegisterMap *map) { 292 assert(map != nullptr, "a map must be given"); 293 return continuation_top_frame(ContinuationWrapper(continuation), map); 294 } 295 296 frame Continuation::top_frame(const frame& callee, RegisterMap* map) { 297 assert(map != nullptr, ""); 298 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp()); 299 assert(ce != nullptr, ""); 300 oop continuation = ce->cont_oop(map->thread()); 301 ContinuationWrapper cont(continuation); 302 return continuation_top_frame(cont, map); 303 } 304 305 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) { 306 assert(map != nullptr, "a map must be given"); 307 if (!ContinuationWrapper(continuation()).is_empty()) { 308 frame f = last_frame(continuation(), map); 309 for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) { 310 if (vf->is_java_frame()) { 311 return javaVFrame::cast(vf); 312 } 313 } 314 } 315 return nullptr; 316 } 317 318 frame Continuation::continuation_parent_frame(RegisterMap* map) { 319 assert(map->in_cont(), ""); 320 ContinuationWrapper cont(map); 321 assert(map->thread() != nullptr || !cont.is_mounted(), ""); 322 323 log_develop_trace(continuations)("continuation_parent_frame"); 324 if (map->update_map()) { 325 // we need to register the link address for the entry frame 326 if (cont.entry() != nullptr) { 327 cont.entry()->update_register_map(map); 328 } else { 329 map->clear(); 330 } 331 } 332 333 if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end 334 oop parent = jdk_internal_vm_Continuation::parent(cont.continuation()); 335 stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr; 336 if (chunk != nullptr) { 337 return chunk->top_frame(map); 338 } 339 340 map->set_stack_chunk(nullptr); 341 return frame(); 342 } 343 344 map->set_stack_chunk(nullptr); 345 346 #if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO) 347 frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC()); 348 #else 349 frame sender = frame(); 350 Unimplemented(); 351 #endif 352 353 return sender; 354 } 355 356 oop Continuation::continuation_scope(oop continuation) { 357 return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr; 358 } 359 360 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) { 361 if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) { 362 return false; 363 } 364 365 oop continuation; 366 if (map->in_cont()) { 367 continuation = map->cont(); 368 } else { 369 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp()); 370 if (ce == nullptr) { 371 return false; 372 } 373 continuation = ce->cont_oop(map->thread()); 374 } 375 if (continuation == nullptr) { 376 return false; 377 } 378 379 oop sc = continuation_scope(continuation); 380 assert(sc != nullptr, ""); 381 return sc == cont_scope; 382 } 383 384 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) { 385 ContinuationWrapper cont(map); 386 stackChunkOop chunk = cont.find_chunk_by_address(addr); 387 return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false; 388 } 389 390 bool Continuation::pin(JavaThread* current) { 391 ContinuationEntry* ce = current->last_continuation(); 392 if (ce == nullptr) { 393 return true; // no continuation mounted 394 } 395 return ce->pin(); 396 } 397 398 bool Continuation::unpin(JavaThread* current) { 399 ContinuationEntry* ce = current->last_continuation(); 400 if (ce == nullptr) { 401 return true; // no continuation mounted 402 } 403 return ce->unpin(); 404 } 405 406 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) { 407 assert (thread != nullptr, ""); 408 ContinuationEntry* ce = get_continuation_entry_for_sp(thread, callee.sp()); 409 assert(ce != nullptr, "callee.sp(): " INTPTR_FORMAT, p2i(callee.sp())); 410 411 log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT 412 " sender_sp: " INTPTR_FORMAT, 413 java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp)); 414 415 frame entry = ce->to_frame(); 416 if (callee.is_interpreted_frame()) { 417 entry.set_sp(sender_sp); // sp != unextended_sp 418 } 419 return entry; 420 } 421 422 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) { 423 ContinuationEntry* ce; 424 if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) { 425 return ce->entry_pc(); 426 } 427 return pc; 428 } 429 430 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) { 431 assert(thread != nullptr, ""); 432 bool fast = !thread->is_interp_only_mode(); 433 thread->set_cont_fastpath_thread_state(fast); 434 } 435 436 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) { 437 ContinuationEntry* entry = thread->last_continuation(); 438 439 if (entry == nullptr) { 440 return; 441 } 442 443 if (is_sp_in_continuation(entry, sp)) { 444 thread->push_cont_fastpath(sp); 445 return; 446 } 447 448 ContinuationEntry* prev; 449 do { 450 prev = entry; 451 entry = entry->parent(); 452 } while (entry != nullptr && !is_sp_in_continuation(entry, sp)); 453 454 if (entry == nullptr) { 455 return; 456 } 457 assert(is_sp_in_continuation(entry, sp), ""); 458 if (sp > prev->parent_cont_fastpath()) { 459 prev->set_parent_cont_fastpath(sp); 460 } 461 } 462 463 #ifndef PRODUCT 464 void Continuation::describe(FrameValues &values) { 465 JavaThread* thread = JavaThread::active(); 466 if (thread != nullptr) { 467 for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) { 468 intptr_t* bottom = ce->entry_sp(); 469 if (bottom != nullptr) { 470 values.describe(-1, bottom, "continuation entry"); 471 } 472 } 473 } 474 } 475 #endif 476 477 #ifdef ASSERT 478 void Continuation::debug_verify_continuation(oop contOop) { 479 if (!VerifyContinuations) { 480 return; 481 } 482 assert(contOop != nullptr, ""); 483 assert(oopDesc::is_oop(contOop), ""); 484 ContinuationWrapper cont(contOop); 485 486 assert(oopDesc::is_oop_or_null(cont.tail()), ""); 487 assert(cont.chunk_invariant(), ""); 488 489 bool nonempty_chunk = false; 490 size_t max_size = 0; 491 int num_chunks = 0; 492 int num_frames = 0; 493 int num_interpreted_frames = 0; 494 int num_oops = 0; 495 496 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 497 log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks); 498 chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames); 499 if (!chunk->is_empty()) { 500 nonempty_chunk = true; 501 } 502 num_chunks++; 503 } 504 505 const bool is_empty = cont.is_empty(); 506 assert(!nonempty_chunk || !is_empty, ""); 507 assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), ""); 508 } 509 510 void Continuation::print(oop continuation) { print_on(tty, continuation); } 511 512 void Continuation::print_on(outputStream* st, oop continuation) { 513 ContinuationWrapper cont(continuation); 514 515 st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d", 516 continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation)); 517 st->print_cr("CHUNKS:"); 518 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 519 st->print("* "); 520 chunk->print_on(true, st); 521 } 522 } 523 #endif // ASSERT 524 525 526 void continuations_init() { Continuations::init(); } 527 528 void Continuations::init() { 529 Continuation::init(); 530 } 531 532 bool Continuations::enabled() { 533 return VMContinuations; 534 } 535 536 #define CC (char*) /*cast a literal from (const char*)*/ 537 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) 538 539 static JNINativeMethod CONT_methods[] = { 540 {CC"pin", CC"()V", FN_PTR(CONT_pin)}, 541 {CC"unpin", CC"()V", FN_PTR(CONT_unpin)}, 542 {CC"isPinned0", CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)}, 543 }; 544 545 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) { 546 JavaThread* thread = JavaThread::current(); 547 ThreadToNativeFromVM trans(thread); 548 int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod)); 549 guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives"); 550 guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives"); 551 }