1 /*
2 * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "gc/shared/barrierSetNMethod.hpp"
28 #include "oops/method.inline.hpp"
29 #include "oops/oop.inline.hpp"
30 #include "prims/jvmtiThreadState.inline.hpp"
31 #include "runtime/continuation.hpp"
32 #include "runtime/continuationEntry.inline.hpp"
33 #include "runtime/continuationHelper.inline.hpp"
34 #include "runtime/continuationJavaClasses.inline.hpp"
35 #include "runtime/continuationWrapper.inline.hpp"
36 #include "runtime/interfaceSupport.inline.hpp"
37 #include "runtime/javaThread.inline.hpp"
38 #include "runtime/jniHandles.inline.hpp"
39 #include "runtime/osThread.hpp"
40 #include "runtime/vframe.inline.hpp"
41 #include "runtime/vframe_hp.hpp"
42
43 // defined in continuationFreezeThaw.cpp
44 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope);
45
46 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) {
47 if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) {
48 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow");
49 }
50 }
51 JVM_END
52
53 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) {
54 if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) {
55 THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow");
56 }
57 }
58 JVM_END
59
60 #if INCLUDE_JVMTI
61 class JvmtiUnmountBeginMark : public StackObj {
62 Handle _vthread;
63 JavaThread* _target;
64 int _preempt_result;
65
66 public:
67 JvmtiUnmountBeginMark(JavaThread* t) :
68 _vthread(t, t->vthread()), _target(t), _preempt_result(freeze_pinned_native) {
69 assert(!_target->is_in_any_VTMS_transition(), "must be");
70
71 if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
72 JvmtiVTMSTransitionDisabler::start_VTMS_transition((jthread)_vthread.raw_value(), /* is_mount */ false);
73 } else {
74 _target->set_is_in_VTMS_transition(true);
75 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), true);
76 }
77 }
78 ~JvmtiUnmountBeginMark() {
79 assert(!_target->is_suspended(), "must be");
80
81 assert(_target->is_in_VTMS_transition(), "must be");
82 assert(java_lang_Thread::is_in_VTMS_transition(_vthread()), "must be");
83
84 // Read it again since for late binding agents the flag could have
85 // been set while blocked in the allocation path during freeze.
86 bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events();
87
88 if (_preempt_result != freeze_ok) {
89 // Undo transition
90 if (jvmti_present) {
91 JvmtiVTMSTransitionDisabler::finish_VTMS_transition((jthread)_vthread.raw_value(), false);
92 } else {
93 _target->set_is_in_VTMS_transition(false);
94 java_lang_Thread::set_is_in_VTMS_transition(_vthread(), false);
95 }
96 } else {
97 if (jvmti_present) {
98 _target->rebind_to_jvmti_thread_state_of(_target->threadObj());
99 if (JvmtiExport::should_post_vthread_mount()) {
100 _target->set_pending_jvmti_unmount_event(true);
101 }
102 }
103 }
104 }
105 void set_preempt_result(int res) { _preempt_result = res; }
106 };
107
108 static bool is_safe_vthread_to_preempt_for_jvmti(JavaThread* target, oop vthread) {
109 assert(!target->has_pending_popframe(), "should be true; no support for vthreads yet");
110 JvmtiThreadState* state = target->jvmti_thread_state();
111 assert(state == nullptr || !state->is_earlyret_pending(), "should be true; no support for vthreads yet");
112
113 if (target->is_in_any_VTMS_transition()) {
114 // We caught target at the end of a mount transition (is_in_VTMS_transition()) or at the
115 // beginning or end of a temporary switch to carrier thread (is_in_tmp_VTMS_transition()).
116 return false;
117 }
118 return true;
119 }
120 #endif
121
122 static bool is_safe_vthread_to_preempt(JavaThread* target, oop vthread) {
123 if (!java_lang_VirtualThread::is_instance(vthread) || // inside transition
124 java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) { // inside transition
125 return false;
126 }
127 return JVMTI_ONLY(is_safe_vthread_to_preempt_for_jvmti(target, vthread)) NOT_JVMTI(true);
128 }
129
130 typedef int (*FreezeContFnT)(JavaThread*, intptr_t*, int);
131
132 int Continuation::try_preempt(JavaThread* target, oop continuation, int preempt_kind) {
133 assert(target == JavaThread::current(), "no support for external preemption");
134 assert((preempt_kind == freeze_on_monitorenter && target->is_on_monitorenter()) ||
135 (preempt_kind == freeze_on_wait && target->current_waiting_monitor() != nullptr), "");
136 assert(target->has_last_Java_frame(), "");
137 assert(!target->preempting(), "");
138 assert(target->last_continuation() != nullptr, "");
139 assert(target->last_continuation()->cont_oop(target) == continuation, "");
140 assert(!is_continuation_preempted(continuation), "");
141 assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), "");
142 assert(!target->has_pending_exception(), "");
143 assert(!target->is_suspended() || target->is_disable_suspend(), "");
144
145 if (!VM_Version::supports_cont_preemption()) {
146 return unsupported;
147 }
148
149 if (is_continuation_done(continuation)) {
150 return freeze_not_mounted;
151 }
152
153 // Continuation is mounted and it's not done so check if it's safe to preempt.
154 if (!is_safe_vthread_to_preempt(target, target->vthread())) {
155 return freeze_pinned_native;
156 }
157
158 JVMTI_ONLY(JvmtiUnmountBeginMark jubm(target);)
159 target->set_preempting(true);
160 int res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(target, target->last_Java_sp(), preempt_kind);
161 log_trace(continuations, preempt)("try_preempt: %d", res);
162 JVMTI_ONLY(jubm.set_preempt_result(res);)
163 if (res != freeze_ok) {
164 target->set_preempting(false);
165 }
166 return res;
167 }
168
169 bool Continuation::is_continuation_preempted(oop cont) {
170 return jdk_internal_vm_Continuation::is_preempted(cont);
171 }
172
173 bool Continuation::is_continuation_done(oop cont) {
174 return jdk_internal_vm_Continuation::done(cont);
175 }
176
177 #ifndef PRODUCT
178 static jlong java_tid(JavaThread* thread) {
179 return java_lang_Thread::thread_id(thread->threadObj());
180 }
181 #endif
182
183 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) {
184 if (thread == nullptr || continuation == nullptr) {
185 return nullptr;
186 }
187
188 for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) {
189 if (continuation == entry->cont_oop(thread)) {
190 return entry;
191 }
192 }
193 return nullptr;
194 }
195
196 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) {
197 if (entry == nullptr) {
198 return false;
199 }
200
201 assert(thread->is_in_full_stack((address)entry), "");
202 return true;
203 // return false if called when transitioning to Java on return from freeze
204 // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp();
205 }
206
207 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) {
208 return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation));
209 }
210
211 // When walking the virtual stack, this method returns true
212 // iff the frame is a thawed continuation frame whose
213 // caller is still frozen on the h-stack.
214 // The continuation object can be extracted from the thread.
215 bool Continuation::is_cont_barrier_frame(const frame& f) {
216 assert(f.is_interpreted_frame() || f.cb() != nullptr, "");
217 if (!Continuations::enabled()) return false;
218 return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f)
219 : ContinuationHelper::CompiledFrame::return_pc(f));
220 }
221
222 bool Continuation::is_return_barrier_entry(const address pc) {
223 if (!Continuations::enabled()) return false;
224 return pc == StubRoutines::cont_returnBarrier();
225 }
226
227 bool Continuation::is_continuation_enterSpecial(const frame& f) {
228 if (f.cb() == nullptr || !f.cb()->is_nmethod()) {
229 return false;
230 }
231 Method* m = f.cb()->as_nmethod()->method();
232 return (m != nullptr && m->is_continuation_enter_intrinsic());
233 }
234
235 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) {
236 // we can do this because the entry frame is never inlined
237 Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame())
238 ? map->stack_chunk()->interpreter_frame_method(f)
239 : ContinuationHelper::Frame::frame_method(f);
240 return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter;
241 }
242
243 // The parameter `sp` should be the actual sp and not the unextended sp because at
244 // least on PPC64 unextended_sp < sp is possible as interpreted frames are trimmed
245 // to the actual size of the expression stack before calls. The problem there is
246 // that even unextended_sp < entry_sp < sp is possible for an interpreted frame.
247 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) {
248 // entry_sp() returns the unextended_sp which is always greater or equal to the actual sp
249 return entry->entry_sp() > sp;
250 }
251
252 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) {
253 return is_sp_in_continuation(entry, f.sp());
254 }
255
256 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) {
257 assert(thread != nullptr, "");
258 ContinuationEntry* entry = thread->last_continuation();
259 while (entry != nullptr && !is_sp_in_continuation(entry, sp)) {
260 entry = entry->parent();
261 }
262 return entry;
263 }
264
265 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) {
266 assert(is_continuation_enterSpecial(f), "");
267 ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp();
268 assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry");
269 return entry;
270 }
271
272 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) {
273 return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.sp()) != nullptr);
274 }
275
276 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) {
277 stackChunkOop chunk = cont.last_nonempty_chunk();
278 map->set_stack_chunk(chunk);
279 return chunk != nullptr ? chunk->top_frame(map) : frame();
280 }
281
282 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) {
283 ContinuationWrapper cont(continuation);
284 if (!cont.is_empty()) {
285 *frame = continuation_top_frame(cont, map);
286 return true;
287 } else {
288 return false;
289 }
290 }
291
292 frame Continuation::last_frame(oop continuation, RegisterMap *map) {
293 assert(map != nullptr, "a map must be given");
294 return continuation_top_frame(ContinuationWrapper(continuation), map);
295 }
296
297 frame Continuation::top_frame(const frame& callee, RegisterMap* map) {
298 assert(map != nullptr, "");
299 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp());
300 assert(ce != nullptr, "");
301 oop continuation = ce->cont_oop(map->thread());
302 ContinuationWrapper cont(continuation);
303 return continuation_top_frame(cont, map);
304 }
305
306 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) {
307 assert(map != nullptr, "a map must be given");
308 if (!ContinuationWrapper(continuation()).is_empty()) {
309 frame f = last_frame(continuation(), map);
310 for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) {
311 if (vf->is_java_frame()) {
312 return javaVFrame::cast(vf);
313 }
314 }
315 }
316 return nullptr;
317 }
318
319 frame Continuation::continuation_parent_frame(RegisterMap* map) {
320 assert(map->in_cont(), "");
321 ContinuationWrapper cont(map);
322 assert(map->thread() != nullptr || !cont.is_mounted(), "");
323
324 log_develop_trace(continuations)("continuation_parent_frame");
325 if (map->update_map()) {
326 // we need to register the link address for the entry frame
327 if (cont.entry() != nullptr) {
328 cont.entry()->update_register_map(map);
329 } else {
330 map->clear();
331 }
332 }
333
334 if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end
335 oop parent = jdk_internal_vm_Continuation::parent(cont.continuation());
336 stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr;
337 if (chunk != nullptr) {
338 return chunk->top_frame(map);
339 }
340
341 map->set_stack_chunk(nullptr);
342 return frame();
343 }
344
345 map->set_stack_chunk(nullptr);
346
347 #if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO)
348 frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC());
349 #else
350 frame sender = frame();
351 Unimplemented();
352 #endif
353
354 return sender;
355 }
356
357 oop Continuation::continuation_scope(oop continuation) {
358 return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr;
359 }
360
361 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) {
362 if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) {
363 return false;
364 }
365
366 oop continuation;
367 if (map->in_cont()) {
368 continuation = map->cont();
369 } else {
370 ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp());
371 if (ce == nullptr) {
372 return false;
373 }
374 continuation = ce->cont_oop(map->thread());
375 }
376 if (continuation == nullptr) {
377 return false;
378 }
379
380 oop sc = continuation_scope(continuation);
381 assert(sc != nullptr, "");
382 return sc == cont_scope;
383 }
384
385 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) {
386 ContinuationWrapper cont(map);
387 stackChunkOop chunk = cont.find_chunk_by_address(addr);
388 return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false;
389 }
390
391 bool Continuation::pin(JavaThread* current) {
392 ContinuationEntry* ce = current->last_continuation();
393 if (ce == nullptr) {
394 return true; // no continuation mounted
395 }
396 return ce->pin();
397 }
398
399 bool Continuation::unpin(JavaThread* current) {
400 ContinuationEntry* ce = current->last_continuation();
401 if (ce == nullptr) {
402 return true; // no continuation mounted
403 }
404 return ce->unpin();
405 }
406
407 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) {
408 assert (thread != nullptr, "");
409 ContinuationEntry* ce = get_continuation_entry_for_sp(thread, callee.sp());
410 assert(ce != nullptr, "callee.sp(): " INTPTR_FORMAT, p2i(callee.sp()));
411
412 log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT
413 " sender_sp: " INTPTR_FORMAT,
414 java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp));
415
416 frame entry = ce->to_frame();
417 if (callee.is_interpreted_frame()) {
418 entry.set_sp(sender_sp); // sp != unextended_sp
419 }
420 return entry;
421 }
422
423 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) {
424 ContinuationEntry* ce;
425 if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) {
426 return ce->entry_pc();
427 }
428 return pc;
429 }
430
431 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) {
432 assert(thread != nullptr, "");
433 bool fast = !thread->is_interp_only_mode();
434 thread->set_cont_fastpath_thread_state(fast);
435 }
436
437 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) {
438 ContinuationEntry* entry = thread->last_continuation();
439
440 if (entry == nullptr) {
441 return;
442 }
443
444 if (is_sp_in_continuation(entry, sp)) {
445 thread->push_cont_fastpath(sp);
446 return;
447 }
448
449 ContinuationEntry* prev;
450 do {
451 prev = entry;
452 entry = entry->parent();
453 } while (entry != nullptr && !is_sp_in_continuation(entry, sp));
454
455 if (entry == nullptr) {
456 return;
457 }
458 assert(is_sp_in_continuation(entry, sp), "");
459 if (sp > prev->parent_cont_fastpath()) {
460 prev->set_parent_cont_fastpath(sp);
461 }
462 }
463
464 #ifndef PRODUCT
465 void Continuation::describe(FrameValues &values) {
466 JavaThread* thread = JavaThread::active();
467 if (thread != nullptr) {
468 for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) {
469 intptr_t* bottom = ce->entry_sp();
470 if (bottom != nullptr) {
471 values.describe(-1, bottom, "continuation entry");
472 }
473 }
474 }
475 }
476 #endif
477
478 #ifdef ASSERT
479 void Continuation::debug_verify_continuation(oop contOop) {
480 if (!VerifyContinuations) {
481 return;
482 }
483 assert(contOop != nullptr, "");
484 assert(oopDesc::is_oop(contOop), "");
485 ContinuationWrapper cont(contOop);
486
487 assert(oopDesc::is_oop_or_null(cont.tail()), "");
488 assert(cont.chunk_invariant(), "");
489
490 bool nonempty_chunk = false;
491 size_t max_size = 0;
492 int num_chunks = 0;
493 int num_frames = 0;
494 int num_interpreted_frames = 0;
495 int num_oops = 0;
496
497 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
498 log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks);
499 chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames);
500 if (!chunk->is_empty()) {
501 nonempty_chunk = true;
502 }
503 num_chunks++;
504 }
505
506 const bool is_empty = cont.is_empty();
507 assert(!nonempty_chunk || !is_empty, "");
508 assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), "");
509 }
510
511 void Continuation::print(oop continuation) { print_on(tty, continuation); }
512
513 void Continuation::print_on(outputStream* st, oop continuation) {
514 ContinuationWrapper cont(continuation);
515
516 st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d",
517 continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation));
518 st->print_cr("CHUNKS:");
519 for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
520 st->print("* ");
521 chunk->print_on(true, st);
522 }
523 }
524 #endif // ASSERT
525
526
527 void continuations_init() { Continuations::init(); }
528
529 void Continuations::init() {
530 Continuation::init();
531 }
532
533 bool Continuations::enabled() {
534 return VMContinuations;
535 }
536
537 #define CC (char*) /*cast a literal from (const char*)*/
538 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
539
540 static JNINativeMethod CONT_methods[] = {
541 {CC"pin", CC"()V", FN_PTR(CONT_pin)},
542 {CC"unpin", CC"()V", FN_PTR(CONT_unpin)},
543 {CC"isPinned0", CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)},
544 };
545
546 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) {
547 JavaThread* thread = JavaThread::current();
548 ThreadToNativeFromVM trans(thread);
549 int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod));
550 guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives");
551 guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives");
552 }