1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmSymbols.hpp"
26 #include "gc/shared/barrierSetNMethod.hpp"
27 #include "oops/method.inline.hpp"
28 #include "oops/oop.inline.hpp"
29 #include "prims/jvmtiThreadState.inline.hpp"
30 #include "runtime/continuation.hpp"
31 #include "runtime/continuationEntry.inline.hpp"
32 #include "runtime/continuationHelper.inline.hpp"
33 #include "runtime/continuationJavaClasses.inline.hpp"
34 #include "runtime/continuationWrapper.inline.hpp"
35 #include "runtime/interfaceSupport.inline.hpp"
36 #include "runtime/javaThread.inline.hpp"
37 #include "runtime/jniHandles.inline.hpp"
38 #include "runtime/osThread.hpp"
39 #include "runtime/vframe.inline.hpp"
40 #include "runtime/vframe_hp.hpp"
41
42 // defined in continuationFreezeThaw.cpp
43 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope);
44
45 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) {
46 if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) {
47 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow");
48 }
49 }
50 JVM_END
51
52 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) {
53 if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) {
54 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow");
55 }
56 }
57 JVM_END
58
59 #if INCLUDE_JVMTI
60 class JvmtiUnmountBeginMark : public StackObj {
61 Handle _vthread;
62 JavaThread* _current;
63 freeze_result _result;
64 bool _failed;
65
66 public:
67 JvmtiUnmountBeginMark(JavaThread* t) :
68 _vthread(t, t->vthread()), _current(t), _result(freeze_pinned_native), _failed(false) {
69 assert(!_current->is_in_VTMS_transition(), "must be");
70
71 if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
72 JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount((jthread)_vthread.raw_value(), true);
73
74 // Don't preempt if there is a pending popframe or earlyret operation. This can
75 // be installed in start_VTMS_transition() so we need to check it here.
76 if (JvmtiExport::can_pop_frame() || JvmtiExport::can_force_early_return()) {
77 JvmtiThreadState* state = _current->jvmti_thread_state();
78 if (_current->has_pending_popframe() || (state != nullptr && state->is_earlyret_pending())) {
79 _failed = true;
80 }
81 }
82
83 // Don't preempt in case there is an async exception installed since
84 // we would incorrectly throw it during the unmount logic in the carrier.
85 if (_current->has_async_exception_condition()) {
86 _failed = true;
87 }
88 } else {
89 _current->set_is_in_VTMS_transition(true);
90 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), true);
91 }
92 }
93 ~JvmtiUnmountBeginMark() {
94 assert(!_current->is_suspended(), "must be");
95
96 assert(_current->is_in_VTMS_transition(), "must be");
97 assert(java_lang_Thread::is_in_VTMS_transition(_vthread()), "must be");
98
99 // Read it again since for late binding agents the flag could have
100 // been set while blocked in the allocation path during freeze.
101 bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events();
102
103 if (_result != freeze_ok) {
104 // Undo transition
105 if (jvmti_present) {
106 JvmtiVTMSTransitionDisabler::VTMS_vthread_mount((jthread)_vthread.raw_value(), false);
107 } else {
108 _current->set_is_in_VTMS_transition(false);
109 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), false);
110 }
111 }
112 }
113 void set_result(freeze_result res) { _result = res; }
114 bool failed() { return _failed; }
115 };
116
117 static bool is_vthread_safe_to_preempt_for_jvmti(JavaThread* current) {
118 if (current->is_in_VTMS_transition()) {
119 // We are at the end of a mount transition.
120 return false;
121 }
122 return true;
123 }
124 #endif // INCLUDE_JVMTI
125
126 static bool is_vthread_safe_to_preempt(JavaThread* current, oop vthread) {
127 assert(java_lang_VirtualThread::is_instance(vthread), "");
128 if (java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) { // inside transition
129 return false;
130 }
131 return JVMTI_ONLY(is_vthread_safe_to_preempt_for_jvmti(current)) NOT_JVMTI(true);
132 }
133
134 typedef freeze_result (*FreezeContFnT)(JavaThread*, intptr_t*);
135
136 static void verify_preempt_preconditions(JavaThread* current, oop continuation) {
137 assert(current == JavaThread::current(), "no support for external preemption");
138 assert(current->has_last_Java_frame(), "");
139 assert(!current->preempting(), "");
140 assert(current->last_continuation() != nullptr, "");
141 assert(current->last_continuation()->cont_oop(current) == continuation, "");
142 assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), "");
143 assert(!current->has_pending_exception(), "");
144 }
145
146 freeze_result Continuation::try_preempt(JavaThread* current, oop continuation) {
147 verify_preempt_preconditions(current, continuation);
148
149 if (LockingMode == LM_LEGACY) {
150 return freeze_unsupported;
151 }
152
153 if (!is_vthread_safe_to_preempt(current, current->vthread())) {
154 return freeze_pinned_native;
155 }
156
157 JVMTI_ONLY(JvmtiUnmountBeginMark jubm(current);)
158 JVMTI_ONLY(if (jubm.failed()) return freeze_pinned_native;)
159 freeze_result res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(current, current->last_Java_sp());
160 log_trace(continuations, preempt)("try_preempt: %d", res);
161 JVMTI_ONLY(jubm.set_result(res);)
162
163 if (current->has_pending_exception()) {
164 assert(res == freeze_exception, "expecting an exception result from freeze");
165 // We don't want to throw exceptions, especially when returning
166 // from monitorenter since the compiler does not expect one. We
167 // just ignore the exception and pin the vthread to the carrier.
168 current->clear_pending_exception();
169 }
170 return res;
171 }
172
173 #ifndef PRODUCT
174 static jlong java_tid(JavaThread* thread) {
175 return java_lang_Thread::thread_id(thread->threadObj());
176 }
177 #endif
178
179 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) {
180 if (thread == nullptr || continuation == nullptr) {
181 return nullptr;
182 }
183
184 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) {
185 if (continuation == entry->cont_oop(thread)) {
186 return entry;
187 }
188 }
189 return nullptr;
190 }
191
192 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) {
193 if (entry == nullptr) {
194 return false;
195 }
196
197 assert(thread->is_in_full_stack((address)entry), "");
198 return true;
199 // return false if called when transitioning to Java on return from freeze
200 // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp();
201 }
202
203 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) {
204 return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation));
205 }
206
207 // When walking the virtual stack, this method returns true
208 // iff the frame is a thawed continuation frame whose
209 // caller is still frozen on the h-stack.
210 // The continuation object can be extracted from the thread.
211 bool Continuation::is_cont_barrier_frame(const frame& f) {
212 assert(f.is_interpreted_frame() || f.cb() != nullptr, "");
213 if (!Continuations::enabled()) return false;
214 return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f)
215 : ContinuationHelper::CompiledFrame::return_pc(f));
216 }
217
218 bool Continuation::is_return_barrier_entry(const address pc) {
219 if (!Continuations::enabled()) return false;
220 return pc == StubRoutines::cont_returnBarrier();
221 }
222
223 bool Continuation::is_continuation_enterSpecial(const frame& f) {
224 if (f.cb() == nullptr || !f.cb()->is_nmethod()) {
225 return false;
226 }
227 Method* m = f.cb()->as_nmethod()->method();
228 return (m != nullptr && m->is_continuation_enter_intrinsic());
229 }
230
231 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) {
232 // we can do this because the entry frame is never inlined
233 Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame())
234 ? map->stack_chunk()->interpreter_frame_method(f)
235 : ContinuationHelper::Frame::frame_method(f);
236 return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter;
237 }
238
239 // The parameter `sp` should be the actual sp and not the unextended sp because at
240 // least on PPC64 unextended_sp < sp is possible as interpreted frames are trimmed
241 // to the actual size of the expression stack before calls. The problem there is
242 // that even unextended_sp < entry_sp < sp is possible for an interpreted frame.
243 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) {
244 // entry_sp() returns the unextended_sp which is always greater or equal to the actual sp
245 return entry->entry_sp() > sp;
246 }
247
248 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) {
249 return is_sp_in_continuation(entry, f.sp());
250 }
251
252 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) {
253 assert(thread != nullptr, "");
254 ContinuationEntry* entry = thread->last_continuation();
255 while (entry != nullptr && !is_sp_in_continuation(entry, sp)) {
256 entry = entry->parent();
257 }
258 return entry;
259 }
260
261 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) {
262 assert(is_continuation_enterSpecial(f), "");
263 ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp();
264 assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry");
265 return entry;
266 }
267
268 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) {
269 return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.sp()) != nullptr);
270 }
271
272 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) {
273 stackChunkOop chunk = cont.last_nonempty_chunk();
274 map->set_stack_chunk(chunk);
275 return chunk != nullptr ? chunk->top_frame(map) : frame();
276 }
277
278 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) {
279 ContinuationWrapper cont(continuation);
280 if (!cont.is_empty()) {
281 *frame = continuation_top_frame(cont, map);
282 return true;
283 } else {
284 return false;
285 }
286 }
287
288 frame Continuation::last_frame(oop continuation, RegisterMap *map) {
289 assert(map != nullptr, "a map must be given");
290 return continuation_top_frame(ContinuationWrapper(continuation), map);
291 }
292
293 frame Continuation::top_frame(const frame& callee, RegisterMap* map) {
294 assert(map != nullptr, "");
295 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp());
296 assert(ce != nullptr, "");
297 oop continuation = ce->cont_oop(map->thread());
298 ContinuationWrapper cont(continuation);
299 return continuation_top_frame(cont, map);
300 }
301
302 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) {
303 assert(map != nullptr, "a map must be given");
304 if (!ContinuationWrapper(continuation()).is_empty()) {
305 frame f = last_frame(continuation(), map);
306 for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) {
307 if (vf->is_java_frame()) {
308 return javaVFrame::cast(vf);
309 }
310 }
311 }
312 return nullptr;
313 }
314
315 frame Continuation::continuation_parent_frame(RegisterMap* map) {
316 assert(map->in_cont(), "");
317 ContinuationWrapper cont(map);
318 assert(map->thread() != nullptr || !cont.is_mounted(), "");
319
320 log_develop_trace(continuations)("continuation_parent_frame");
321 if (map->update_map()) {
322 // we need to register the link address for the entry frame
323 if (cont.entry() != nullptr) {
324 cont.entry()->update_register_map(map);
325 } else {
326 map->clear();
327 }
328 }
329
330 if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end
331 oop parent = jdk_internal_vm_Continuation::parent(cont.continuation());
332 stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr;
333 if (chunk != nullptr) {
334 return chunk->top_frame(map);
335 }
336
337 map->set_stack_chunk(nullptr);
338 return frame();
339 }
340
341 map->set_stack_chunk(nullptr);
342
343 #if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO)
344 frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC());
345 #else
346 frame sender = frame();
347 Unimplemented();
348 #endif
349
350 return sender;
351 }
352
353 oop Continuation::continuation_scope(oop continuation) {
354 return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr;
355 }
356
357 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) {
358 if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) {
359 return false;
360 }
361
362 oop continuation;
363 if (map->in_cont()) {
364 continuation = map->cont();
365 } else {
366 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp());
367 if (ce == nullptr) {
368 return false;
369 }
370 continuation = ce->cont_oop(map->thread());
371 }
372 if (continuation == nullptr) {
373 return false;
374 }
375
376 oop sc = continuation_scope(continuation);
377 assert(sc != nullptr, "");
378 return sc == cont_scope;
379 }
380
381 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) {
382 ContinuationWrapper cont(map);
383 stackChunkOop chunk = cont.find_chunk_by_address(addr);
384 return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false;
385 }
386
387 bool Continuation::pin(JavaThread* current) {
388 ContinuationEntry* ce = current->last_continuation();
389 if (ce == nullptr) {
390 return true; // no continuation mounted
391 }
392 return ce->pin();
393 }
394
395 bool Continuation::unpin(JavaThread* current) {
396 ContinuationEntry* ce = current->last_continuation();
397 if (ce == nullptr) {
398 return true; // no continuation mounted
399 }
400 return ce->unpin();
401 }
402
403 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) {
404 assert (thread != nullptr, "");
405 ContinuationEntry* ce = get_continuation_entry_for_sp(thread, callee.sp());
406 assert(ce != nullptr, "callee.sp(): " INTPTR_FORMAT, p2i(callee.sp()));
407
408 log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT
409 " sender_sp: " INTPTR_FORMAT,
410 java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp));
411
412 frame entry = ce->to_frame();
413 if (callee.is_interpreted_frame()) {
414 entry.set_sp(sender_sp); // sp != unextended_sp
415 }
416 return entry;
417 }
418
419 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) {
420 ContinuationEntry* ce;
421 if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) {
422 return ce->entry_pc();
423 }
424 return pc;
425 }
426
427 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) {
428 assert(thread != nullptr, "");
429 bool fast = !thread->is_interp_only_mode();
430 thread->set_cont_fastpath_thread_state(fast);
431 }
432
433 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) {
434 ContinuationEntry* entry = thread->last_continuation();
435
436 if (entry == nullptr) {
437 return;
438 }
439
440 if (is_sp_in_continuation(entry, sp)) {
441 thread->push_cont_fastpath(sp);
442 return;
443 }
444
445 ContinuationEntry* prev;
446 do {
447 prev = entry;
448 entry = entry->parent();
449 } while (entry != nullptr && !is_sp_in_continuation(entry, sp));
450
451 if (entry == nullptr) {
452 return;
453 }
454 assert(is_sp_in_continuation(entry, sp), "");
455 if (sp > prev->parent_cont_fastpath()) {
456 prev->set_parent_cont_fastpath(sp);
457 }
458 }
459
460 #ifndef PRODUCT
461 void Continuation::describe(FrameValues &values) {
462 JavaThread* thread = JavaThread::active();
463 if (thread != nullptr) {
464 for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) {
465 intptr_t* bottom = ce->entry_sp();
466 if (bottom != nullptr) {
467 values.describe(-1, bottom, "continuation entry");
468 }
469 }
470 }
471 }
472 #endif
473
474 #ifdef ASSERT
475 void Continuation::debug_verify_continuation(oop contOop) {
476 if (!VerifyContinuations) {
477 return;
478 }
479 assert(contOop != nullptr, "");
480 assert(oopDesc::is_oop(contOop), "");
481 ContinuationWrapper cont(contOop);
482
483 assert(oopDesc::is_oop_or_null(cont.tail()), "");
484 assert(cont.chunk_invariant(), "");
485
486 bool nonempty_chunk = false;
487 size_t max_size = 0;
488 int num_chunks = 0;
489 int num_frames = 0;
490 int num_interpreted_frames = 0;
491 int num_oops = 0;
492
493 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
494 log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks);
495 chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames);
496 if (!chunk->is_empty()) {
497 nonempty_chunk = true;
498 }
499 num_chunks++;
500 }
501
502 const bool is_empty = cont.is_empty();
503 assert(!nonempty_chunk || !is_empty, "");
504 assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), "");
505 }
506
507 void Continuation::print(oop continuation) { print_on(tty, continuation); }
508
509 void Continuation::print_on(outputStream* st, oop continuation) {
510 ContinuationWrapper cont(continuation);
511
512 st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d",
513 continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation));
514 st->print_cr("CHUNKS:");
515 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
516 st->print("* ");
517 chunk->print_on(true, st);
518 }
519 }
520 #endif // ASSERT
521
522
523 void continuations_init() { Continuations::init(); }
524
525 void Continuations::init() {
526 Continuation::init();
527 }
528
529 bool Continuations::enabled() {
530 return VMContinuations;
531 }
532
533 #define CC (char*) /*cast a literal from (const char*)*/
534 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
535
536 static JNINativeMethod CONT_methods[] = {
537 {CC"pin", CC"()V", FN_PTR(CONT_pin)},
538 {CC"unpin", CC"()V", FN_PTR(CONT_unpin)},
539 {CC"isPinned0", CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)},
540 };
541
542 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) {
543 JavaThread* thread = JavaThread::current();
544 ThreadToNativeFromVM trans(thread);
545 int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod));
546 guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives");
547 guarantee(!env->ExceptionCheck(), "register jdk.internal.vm.Continuation natives");
548 }