1 /* 2 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/barrierSetNMethod.hpp" 27 #include "runtime/arguments.hpp" 28 #include "runtime/continuation.hpp" 29 #include "runtime/continuationEntry.inline.hpp" 30 #include "runtime/continuationHelper.inline.hpp" 31 #include "runtime/continuationJavaClasses.inline.hpp" 32 #include "runtime/continuationWrapper.inline.hpp" 33 #include "runtime/interfaceSupport.inline.hpp" 34 #include "runtime/javaThread.inline.hpp" 35 #include "runtime/osThread.hpp" 36 #include "runtime/vframe.inline.hpp" 37 #include "runtime/vframe_hp.hpp" 38 39 // defined in continuationFreezeThaw.cpp 40 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope); 41 42 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) { 43 if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) { 44 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow"); 45 } 46 } 47 JVM_END 48 49 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) { 50 if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) { 51 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow"); 52 } 53 } 54 JVM_END 55 56 #ifndef PRODUCT 57 static jlong java_tid(JavaThread* thread) { 58 return java_lang_Thread::thread_id(thread->threadObj()); 59 } 60 #endif 61 62 const ContinuationEntry* Continuation::last_continuation(const JavaThread* thread, oop cont_scope) { 63 // guarantee (thread->has_last_Java_frame(), ""); 64 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) { 65 if (cont_scope == jdk_internal_vm_Continuation::scope(entry->cont_oop())) { 66 return entry; 67 } 68 } 69 return nullptr; 70 } 71 72 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) { 73 if (thread == nullptr || continuation == nullptr) { 74 return nullptr; 75 } 76 77 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) { 78 if (continuation == entry->cont_oop()) { 79 return entry; 80 } 81 } 82 return nullptr; 83 } 84 85 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) { 86 if (entry == nullptr) { 87 return false; 88 } 89 90 assert(thread->is_in_full_stack((address)entry), ""); 91 return true; 92 // return false if called when transitioning to Java on return from freeze 93 // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp(); 94 } 95 96 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) { 97 return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation)); 98 } 99 100 bool Continuation::is_continuation_scope_mounted(JavaThread* thread, oop cont_scope) { 101 return is_on_stack(thread, last_continuation(thread, cont_scope)); 102 } 103 104 // When walking the virtual stack, this method returns true 105 // iff the frame is a thawed continuation frame whose 106 // caller is still frozen on the h-stack. 107 // The continuation object can be extracted from the thread. 108 bool Continuation::is_cont_barrier_frame(const frame& f) { 109 assert(f.is_interpreted_frame() || f.cb() != nullptr, ""); 110 if (!Continuations::enabled()) return false; 111 return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f) 112 : ContinuationHelper::CompiledFrame::return_pc(f)); 113 } 114 115 bool Continuation::is_return_barrier_entry(const address pc) { 116 if (!Continuations::enabled()) return false; 117 return pc == StubRoutines::cont_returnBarrier(); 118 } 119 120 bool Continuation::is_continuation_enterSpecial(const frame& f) { 121 if (f.cb() == nullptr || !f.cb()->is_compiled()) { 122 return false; 123 } 124 Method* m = f.cb()->as_compiled_method()->method(); 125 return (m != nullptr && m->is_continuation_enter_intrinsic()); 126 } 127 128 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) { 129 // we can do this because the entry frame is never inlined 130 Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame()) 131 ? map->stack_chunk()->interpreter_frame_method(f) 132 : ContinuationHelper::Frame::frame_method(f); 133 return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter; 134 } 135 136 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) { 137 return entry->entry_sp() > sp; 138 } 139 140 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) { 141 return is_sp_in_continuation(entry, f.unextended_sp()); 142 } 143 144 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) { 145 assert(thread != nullptr, ""); 146 ContinuationEntry* entry = thread->last_continuation(); 147 while (entry != nullptr && !is_sp_in_continuation(entry, sp)) { 148 entry = entry->parent(); 149 } 150 return entry; 151 } 152 153 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) { 154 assert(is_continuation_enterSpecial(f), ""); 155 ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp(); 156 assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry"); 157 return entry; 158 } 159 160 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) { 161 return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.unextended_sp()) != nullptr); 162 } 163 164 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) { 165 stackChunkOop chunk = cont.last_nonempty_chunk(); 166 map->set_stack_chunk(chunk); 167 return chunk != nullptr ? chunk->top_frame(map) : frame(); 168 } 169 170 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) { 171 ContinuationWrapper cont(continuation); 172 if (!cont.is_empty()) { 173 *frame = continuation_top_frame(cont, map); 174 return true; 175 } else { 176 return false; 177 } 178 } 179 180 frame Continuation::last_frame(oop continuation, RegisterMap *map) { 181 assert(map != nullptr, "a map must be given"); 182 return continuation_top_frame(ContinuationWrapper(continuation), map); 183 } 184 185 frame Continuation::top_frame(const frame& callee, RegisterMap* map) { 186 assert(map != nullptr, ""); 187 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp()); 188 assert(ce != nullptr, ""); 189 oop continuation = ce->cont_oop(); 190 ContinuationWrapper cont(continuation); 191 return continuation_top_frame(cont, map); 192 } 193 194 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) { 195 assert(map != nullptr, "a map must be given"); 196 if (!ContinuationWrapper(continuation()).is_empty()) { 197 frame f = last_frame(continuation(), map); 198 for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) { 199 if (vf->is_java_frame()) { 200 return javaVFrame::cast(vf); 201 } 202 } 203 } 204 return nullptr; 205 } 206 207 frame Continuation::continuation_parent_frame(RegisterMap* map) { 208 assert(map->in_cont(), ""); 209 ContinuationWrapper cont(map); 210 assert(map->thread() != nullptr || !cont.is_mounted(), ""); 211 212 log_develop_trace(continuations)("continuation_parent_frame"); 213 if (map->update_map()) { 214 // we need to register the link address for the entry frame 215 if (cont.entry() != nullptr) { 216 cont.entry()->update_register_map(map); 217 } else { 218 map->clear(); 219 } 220 } 221 222 if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end 223 oop parent = jdk_internal_vm_Continuation::parent(cont.continuation()); 224 stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr; 225 if (chunk != nullptr) { 226 return chunk->top_frame(map); 227 } 228 229 map->set_stack_chunk(nullptr); 230 return frame(); 231 } 232 233 map->set_stack_chunk(nullptr); 234 235 #if (defined(X86) || defined(AARCH64)) && !defined(ZERO) 236 frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC()); 237 #else 238 frame sender = frame(); 239 Unimplemented(); 240 #endif 241 242 return sender; 243 } 244 245 oop Continuation::continuation_scope(oop continuation) { 246 return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr; 247 } 248 249 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) { 250 if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) { 251 return false; 252 } 253 254 oop continuation; 255 if (map->in_cont()) { 256 continuation = map->cont(); 257 } else { 258 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp()); 259 if (ce == nullptr) { 260 return false; 261 } 262 continuation = ce->cont_oop(); 263 } 264 if (continuation == nullptr) { 265 return false; 266 } 267 268 oop sc = continuation_scope(continuation); 269 assert(sc != nullptr, ""); 270 return sc == cont_scope; 271 } 272 273 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) { 274 ContinuationWrapper cont(map); 275 stackChunkOop chunk = cont.find_chunk_by_address(addr); 276 return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false; 277 } 278 279 bool Continuation::pin(JavaThread* current) { 280 ContinuationEntry* ce = current->last_continuation(); 281 if (ce == nullptr) { 282 return true; // no continuation mounted 283 } 284 return ce->pin(); 285 } 286 287 bool Continuation::unpin(JavaThread* current) { 288 ContinuationEntry* ce = current->last_continuation(); 289 if (ce == nullptr) { 290 return true; // no continuation mounted 291 } 292 return ce->unpin(); 293 } 294 295 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) { 296 assert (thread != nullptr, ""); 297 ContinuationEntry* ce = get_continuation_entry_for_sp(thread, 298 callee.is_interpreted_frame() ? callee.interpreter_frame_last_sp() : callee.unextended_sp()); 299 assert(ce != nullptr, "callee.unextended_sp(): " INTPTR_FORMAT, p2i(callee.unextended_sp())); 300 301 log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT 302 " sender_sp: " INTPTR_FORMAT, 303 java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp)); 304 305 frame entry = ce->to_frame(); 306 if (callee.is_interpreted_frame()) { 307 entry.set_sp(sender_sp); // sp != unextended_sp 308 } 309 return entry; 310 } 311 312 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) { 313 ContinuationEntry* ce; 314 if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) { 315 return ce->entry_pc(); 316 } 317 return pc; 318 } 319 320 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) { 321 assert(thread != nullptr, ""); 322 bool fast = !thread->is_interp_only_mode(); 323 thread->set_cont_fastpath_thread_state(fast); 324 } 325 326 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) { 327 ContinuationEntry* entry = thread->last_continuation(); 328 329 if (entry == nullptr) { 330 return; 331 } 332 333 if (is_sp_in_continuation(entry, sp)) { 334 thread->push_cont_fastpath(sp); 335 return; 336 } 337 338 ContinuationEntry* prev; 339 do { 340 prev = entry; 341 entry = entry->parent(); 342 } while (entry != nullptr && !is_sp_in_continuation(entry, sp)); 343 344 if (entry == nullptr) { 345 return; 346 } 347 assert(is_sp_in_continuation(entry, sp), ""); 348 if (sp > prev->parent_cont_fastpath()) { 349 prev->set_parent_cont_fastpath(sp); 350 } 351 } 352 353 #ifndef PRODUCT 354 void Continuation::describe(FrameValues &values) { 355 JavaThread* thread = JavaThread::active(); 356 if (thread != nullptr) { 357 for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) { 358 intptr_t* bottom = ce->entry_sp(); 359 if (bottom != nullptr) { 360 values.describe(-1, bottom, "continuation entry"); 361 } 362 } 363 } 364 } 365 #endif 366 367 #ifdef ASSERT 368 void Continuation::debug_verify_continuation(oop contOop) { 369 if (!VerifyContinuations) { 370 return; 371 } 372 assert(contOop != nullptr, ""); 373 assert(oopDesc::is_oop(contOop), ""); 374 ContinuationWrapper cont(contOop); 375 376 assert(oopDesc::is_oop_or_null(cont.tail()), ""); 377 assert(cont.chunk_invariant(), ""); 378 379 bool nonempty_chunk = false; 380 size_t max_size = 0; 381 int num_chunks = 0; 382 int num_frames = 0; 383 int num_interpreted_frames = 0; 384 int num_oops = 0; 385 386 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 387 log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks); 388 chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames); 389 if (!chunk->is_empty()) { 390 nonempty_chunk = true; 391 } 392 num_chunks++; 393 } 394 395 const bool is_empty = cont.is_empty(); 396 assert(!nonempty_chunk || !is_empty, ""); 397 assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), ""); 398 } 399 400 void Continuation::print(oop continuation) { print_on(tty, continuation); } 401 402 void Continuation::print_on(outputStream* st, oop continuation) { 403 ContinuationWrapper cont(continuation); 404 405 st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d", 406 continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation)); 407 st->print_cr("CHUNKS:"); 408 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) { 409 st->print("* "); 410 chunk->print_on(true, st); 411 } 412 } 413 #endif // ASSERT 414 415 416 void continuations_init() { Continuations::init(); } 417 418 void Continuations::init() { 419 Continuation::init(); 420 } 421 422 // While virtual threads are in Preview, there are some VM mechanisms we disable if continuations aren't used 423 // See NMethodSweeper::do_stack_scanning and nmethod::is_not_on_continuation_stack 424 bool Continuations::enabled() { 425 // return VMContinuations && Arguments::enable_preview(); 426 return VMContinuations; 427 } 428 429 // We initialize the _gc_epoch to 2, because previous_completed_gc_marking_cycle 430 // subtracts the value by 2, and the type is unsigned. We don't want underflow. 431 // 432 // Odd values mean that marking is in progress, and even values mean that no 433 // marking is currently active. 434 uint64_t Continuations::_gc_epoch = 2; 435 436 uint64_t Continuations::gc_epoch() { 437 return _gc_epoch; 438 } 439 440 bool Continuations::is_gc_marking_cycle_active() { 441 // Odd means that marking is active 442 return (_gc_epoch % 2) == 1; 443 } 444 445 uint64_t Continuations::previous_completed_gc_marking_cycle() { 446 if (is_gc_marking_cycle_active()) { 447 return _gc_epoch - 2; 448 } else { 449 return _gc_epoch - 1; 450 } 451 } 452 453 void Continuations::on_gc_marking_cycle_start() { 454 assert(!is_gc_marking_cycle_active(), "Previous marking cycle never ended"); 455 ++_gc_epoch; 456 } 457 458 void Continuations::on_gc_marking_cycle_finish() { 459 assert(is_gc_marking_cycle_active(), "Marking cycle started before last one finished"); 460 ++_gc_epoch; 461 } 462 463 void Continuations::arm_all_nmethods() { 464 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 465 if (bs_nm != NULL) { 466 bs_nm->arm_all_nmethods(); 467 } 468 } 469 470 #define CC (char*) /*cast a literal from (const char*)*/ 471 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) 472 473 static JNINativeMethod CONT_methods[] = { 474 {CC"pin", CC"()V", FN_PTR(CONT_pin)}, 475 {CC"unpin", CC"()V", FN_PTR(CONT_unpin)}, 476 {CC"isPinned0", CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)}, 477 }; 478 479 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) { 480 Thread* thread = Thread::current(); 481 assert(thread->is_Java_thread(), ""); 482 ThreadToNativeFromVM trans((JavaThread*)thread); 483 int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod)); 484 guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives"); 485 guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives"); 486 }