1 /*
  2  * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "classfile/vmSymbols.hpp"
 27 #include "gc/shared/barrierSetNMethod.hpp"
 28 #include "oops/method.inline.hpp"
 29 #include "oops/oop.inline.hpp"
 30 #include "prims/jvmtiThreadState.inline.hpp"
 31 #include "runtime/continuation.hpp"
 32 #include "runtime/continuationEntry.inline.hpp"
 33 #include "runtime/continuationHelper.inline.hpp"
 34 #include "runtime/continuationJavaClasses.inline.hpp"
 35 #include "runtime/continuationWrapper.inline.hpp"
 36 #include "runtime/interfaceSupport.inline.hpp"
 37 #include "runtime/javaThread.inline.hpp"
 38 #include "runtime/jniHandles.inline.hpp"
 39 #include "runtime/osThread.hpp"
 40 #include "runtime/vframe.inline.hpp"
 41 #include "runtime/vframe_hp.hpp"
 42 
 43 // defined in continuationFreezeThaw.cpp
 44 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope);
 45 
 46 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) {
 47   if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) {
 48      THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow");
 49   }
 50 }
 51 JVM_END
 52 
 53 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) {
 54   if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) {
 55      THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow");
 56   }
 57 }
 58 JVM_END
 59 
 60 #if INCLUDE_JVMTI
 61 class JvmtiUnmountBeginMark : public StackObj {
 62   Handle _vthread;
 63   JavaThread* _target;
 64   int _preempt_result;
 65   bool _failed;
 66 
 67  public:
 68   JvmtiUnmountBeginMark(JavaThread* t) :
 69     _vthread(t, t->vthread()), _target(t), _preempt_result(freeze_pinned_native), _failed(false) {
 70     assert(!_target->is_in_any_VTMS_transition(), "must be");
 71 
 72     if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
 73       JvmtiVTMSTransitionDisabler::start_VTMS_transition((jthread)_vthread.raw_value(), /* is_mount */ false);
 74 
 75       // Don't preempt if there is a pending popframe or earlyret operation. This can
 76       // be installed in start_VTMS_transition() so we need to check it here.
 77       if (JvmtiExport::can_pop_frame() || JvmtiExport::can_force_early_return()) {
 78         JvmtiThreadState* state = _target->jvmti_thread_state();
 79         if (_target->has_pending_popframe() || (state != nullptr && state->is_earlyret_pending())) {
 80           _failed = true;
 81         }
 82       }
 83 
 84       // Don't preempt in case there is an async exception installed since
 85       // we would incorrectly throw it during the unmount logic in the carrier.
 86       if (_target->has_async_exception_condition()) {
 87         _failed = true;
 88       }
 89     } else {
 90       _target->set_is_in_VTMS_transition(true);
 91       java_lang_Thread::set_is_in_VTMS_transition(_vthread(), true);
 92     }
 93   }
 94   ~JvmtiUnmountBeginMark() {
 95     assert(!_target->is_suspended(), "must be");
 96 
 97     assert(_target->is_in_VTMS_transition(), "must be");
 98     assert(java_lang_Thread::is_in_VTMS_transition(_vthread()), "must be");
 99 
100     // Read it again since for late binding agents the flag could have
101     // been set while blocked in the allocation path during freeze.
102     bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events();
103 
104     if (_preempt_result != freeze_ok) {
105       // Undo transition
106       if (jvmti_present) {
107         JvmtiVTMSTransitionDisabler::finish_VTMS_transition((jthread)_vthread.raw_value(), false);
108       } else {
109         _target->set_is_in_VTMS_transition(false);
110         java_lang_Thread::set_is_in_VTMS_transition(_vthread(), false);
111       }
112     } else {
113       if (jvmti_present) {
114         _target->rebind_to_jvmti_thread_state_of(_target->threadObj());
115         if (JvmtiExport::should_post_vthread_unmount()) {
116           // We are inside the VTMS transition already so we will post the event
117           // once we finish it in JvmtiVTMSTransitionDisabler::VTMS_unmount_end().
118           _target->set_pending_jvmti_unmount_event(true);
119         }
120       }
121     }
122   }
123   void set_preempt_result(int res) { _preempt_result = res; }
124   bool failed() { return _failed; }
125 };
126 
127 static bool is_vthread_safe_to_preempt_for_jvmti(JavaThread* target, oop vthread) {
128   if (target->is_in_any_VTMS_transition()) {
129     // We caught target at the end of a mount transition (is_in_VTMS_transition()) or at the
130     // beginning or end of a temporary switch to carrier thread (is_in_tmp_VTMS_transition()).
131     return false;
132   }
133   return true;
134 }
135 #endif // INCLUDE_JVMTI
136 
137 static bool is_vthread_safe_to_preempt(JavaThread* target, oop vthread) {
138   if (!java_lang_VirtualThread::is_instance(vthread) ||                               // inside tmp transition
139       java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) {  // inside transition
140     return false;
141   }
142   return JVMTI_ONLY(is_vthread_safe_to_preempt_for_jvmti(target, vthread)) NOT_JVMTI(true);
143 }
144 
145 typedef int (*FreezeContFnT)(JavaThread*, intptr_t*);
146 
147 static void verify_preempt_preconditions(JavaThread* target, oop continuation) {
148   assert(target == JavaThread::current(), "no support for external preemption");
149   assert(target->has_last_Java_frame(), "");
150   assert(!target->preempting(), "");
151   assert(target->last_continuation() != nullptr, "");
152   assert(target->last_continuation()->cont_oop(target) == continuation, "");
153   assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), "");
154   assert(!target->has_pending_exception(), "");
155 }
156 
157 int Continuation::try_preempt(JavaThread* target, oop continuation) {
158   verify_preempt_preconditions(target, continuation);
159 
160   if (LockingMode == LM_LEGACY) {
161     return freeze_unsupported;
162   }
163 
164   if (!is_vthread_safe_to_preempt(target, target->vthread())) {
165     return freeze_pinned_native;
166   }
167 
168   JVMTI_ONLY(JvmtiUnmountBeginMark jubm(target);)
169   JVMTI_ONLY(if (jubm.failed()) return freeze_pinned_native;)
170   int res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(target, target->last_Java_sp());
171   log_trace(continuations, preempt)("try_preempt: %d", res);
172   JVMTI_ONLY(jubm.set_preempt_result(res);)
173   return res;
174 }
175 
176 #ifndef PRODUCT
177 static jlong java_tid(JavaThread* thread) {
178   return java_lang_Thread::thread_id(thread->threadObj());
179 }
180 #endif
181 
182 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) {
183   if (thread == nullptr || continuation == nullptr) {
184     return nullptr;
185   }
186 
187   for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) {
188     if (continuation == entry->cont_oop(thread)) {
189       return entry;
190     }
191   }
192   return nullptr;
193 }
194 
195 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) {
196   if (entry == nullptr) {
197     return false;
198   }
199 
200   assert(thread->is_in_full_stack((address)entry), "");
201   return true;
202   // return false if called when transitioning to Java on return from freeze
203   // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp();
204 }
205 
206 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) {
207   return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation));
208 }
209 
210 // When walking the virtual stack, this method returns true
211 // iff the frame is a thawed continuation frame whose
212 // caller is still frozen on the h-stack.
213 // The continuation object can be extracted from the thread.
214 bool Continuation::is_cont_barrier_frame(const frame& f) {
215   assert(f.is_interpreted_frame() || f.cb() != nullptr, "");
216   if (!Continuations::enabled()) return false;
217   return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f)
218                                                           : ContinuationHelper::CompiledFrame::return_pc(f));
219 }
220 
221 bool Continuation::is_return_barrier_entry(const address pc) {
222   if (!Continuations::enabled()) return false;
223   return pc == StubRoutines::cont_returnBarrier();
224 }
225 
226 bool Continuation::is_continuation_enterSpecial(const frame& f) {
227   if (f.cb() == nullptr || !f.cb()->is_nmethod()) {
228     return false;
229   }
230   Method* m = f.cb()->as_nmethod()->method();
231   return (m != nullptr && m->is_continuation_enter_intrinsic());
232 }
233 
234 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) {
235   // we can do this because the entry frame is never inlined
236   Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame())
237                   ? map->stack_chunk()->interpreter_frame_method(f)
238                   : ContinuationHelper::Frame::frame_method(f);
239   return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter;
240 }
241 
242 // The parameter `sp` should be the actual sp and not the unextended sp because at
243 // least on PPC64 unextended_sp < sp is possible as interpreted frames are trimmed
244 // to the actual size of the expression stack before calls. The problem there is
245 // that even unextended_sp < entry_sp < sp is possible for an interpreted frame.
246 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) {
247   // entry_sp() returns the unextended_sp which is always greater or equal to the actual sp
248   return entry->entry_sp() > sp;
249 }
250 
251 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) {
252   return is_sp_in_continuation(entry, f.sp());
253 }
254 
255 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) {
256   assert(thread != nullptr, "");
257   ContinuationEntry* entry = thread->last_continuation();
258   while (entry != nullptr && !is_sp_in_continuation(entry, sp)) {
259     entry = entry->parent();
260   }
261   return entry;
262 }
263 
264 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) {
265   assert(is_continuation_enterSpecial(f), "");
266   ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp();
267   assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry");
268   return entry;
269 }
270 
271 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) {
272   return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.sp()) != nullptr);
273 }
274 
275 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) {
276   stackChunkOop chunk = cont.last_nonempty_chunk();
277   map->set_stack_chunk(chunk);
278   return chunk != nullptr ? chunk->top_frame(map) : frame();
279 }
280 
281 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) {
282   ContinuationWrapper cont(continuation);
283   if (!cont.is_empty()) {
284     *frame = continuation_top_frame(cont, map);
285     return true;
286   } else {
287     return false;
288   }
289 }
290 
291 frame Continuation::last_frame(oop continuation, RegisterMap *map) {
292   assert(map != nullptr, "a map must be given");
293   return continuation_top_frame(ContinuationWrapper(continuation), map);
294 }
295 
296 frame Continuation::top_frame(const frame& callee, RegisterMap* map) {
297   assert(map != nullptr, "");
298   ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp());
299   assert(ce != nullptr, "");
300   oop continuation = ce->cont_oop(map->thread());
301   ContinuationWrapper cont(continuation);
302   return continuation_top_frame(cont, map);
303 }
304 
305 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) {
306   assert(map != nullptr, "a map must be given");
307   if (!ContinuationWrapper(continuation()).is_empty()) {
308     frame f = last_frame(continuation(), map);
309     for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) {
310       if (vf->is_java_frame()) {
311         return javaVFrame::cast(vf);
312       }
313     }
314   }
315   return nullptr;
316 }
317 
318 frame Continuation::continuation_parent_frame(RegisterMap* map) {
319   assert(map->in_cont(), "");
320   ContinuationWrapper cont(map);
321   assert(map->thread() != nullptr || !cont.is_mounted(), "");
322 
323   log_develop_trace(continuations)("continuation_parent_frame");
324   if (map->update_map()) {
325     // we need to register the link address for the entry frame
326     if (cont.entry() != nullptr) {
327       cont.entry()->update_register_map(map);
328     } else {
329       map->clear();
330     }
331   }
332 
333   if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end
334     oop parent = jdk_internal_vm_Continuation::parent(cont.continuation());
335     stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr;
336     if (chunk != nullptr) {
337       return chunk->top_frame(map);
338     }
339 
340     map->set_stack_chunk(nullptr);
341     return frame();
342   }
343 
344   map->set_stack_chunk(nullptr);
345 
346 #if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO)
347   frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC());
348 #else
349   frame sender = frame();
350   Unimplemented();
351 #endif
352 
353   return sender;
354 }
355 
356 oop Continuation::continuation_scope(oop continuation) {
357   return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr;
358 }
359 
360 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) {
361   if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) {
362     return false;
363   }
364 
365   oop continuation;
366   if (map->in_cont()) {
367     continuation = map->cont();
368   } else {
369     ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp());
370     if (ce == nullptr) {
371       return false;
372     }
373     continuation = ce->cont_oop(map->thread());
374   }
375   if (continuation == nullptr) {
376     return false;
377   }
378 
379   oop sc = continuation_scope(continuation);
380   assert(sc != nullptr, "");
381   return sc == cont_scope;
382 }
383 
384 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) {
385   ContinuationWrapper cont(map);
386   stackChunkOop chunk = cont.find_chunk_by_address(addr);
387   return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false;
388 }
389 
390 bool Continuation::pin(JavaThread* current) {
391   ContinuationEntry* ce = current->last_continuation();
392   if (ce == nullptr) {
393     return true; // no continuation mounted
394   }
395   return ce->pin();
396 }
397 
398 bool Continuation::unpin(JavaThread* current) {
399   ContinuationEntry* ce = current->last_continuation();
400   if (ce == nullptr) {
401     return true; // no continuation mounted
402   }
403   return ce->unpin();
404 }
405 
406 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) {
407   assert (thread != nullptr, "");
408   ContinuationEntry* ce = get_continuation_entry_for_sp(thread, callee.sp());
409   assert(ce != nullptr, "callee.sp(): " INTPTR_FORMAT, p2i(callee.sp()));
410 
411   log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT
412     " sender_sp: " INTPTR_FORMAT,
413     java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp));
414 
415   frame entry = ce->to_frame();
416   if (callee.is_interpreted_frame()) {
417     entry.set_sp(sender_sp); // sp != unextended_sp
418   }
419   return entry;
420 }
421 
422 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) {
423   ContinuationEntry* ce;
424   if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) {
425     return ce->entry_pc();
426   }
427   return pc;
428 }
429 
430 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) {
431   assert(thread != nullptr, "");
432   bool fast = !thread->is_interp_only_mode();
433   thread->set_cont_fastpath_thread_state(fast);
434 }
435 
436 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) {
437   ContinuationEntry* entry = thread->last_continuation();
438 
439   if (entry == nullptr) {
440     return;
441   }
442 
443   if (is_sp_in_continuation(entry, sp)) {
444     thread->push_cont_fastpath(sp);
445     return;
446   }
447 
448   ContinuationEntry* prev;
449   do {
450     prev = entry;
451     entry = entry->parent();
452   } while (entry != nullptr && !is_sp_in_continuation(entry, sp));
453 
454   if (entry == nullptr) {
455     return;
456   }
457   assert(is_sp_in_continuation(entry, sp), "");
458   if (sp > prev->parent_cont_fastpath()) {
459     prev->set_parent_cont_fastpath(sp);
460   }
461 }
462 
463 #ifndef PRODUCT
464 void Continuation::describe(FrameValues &values) {
465   JavaThread* thread = JavaThread::active();
466   if (thread != nullptr) {
467     for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) {
468       intptr_t* bottom = ce->entry_sp();
469       if (bottom != nullptr) {
470         values.describe(-1, bottom, "continuation entry");
471       }
472     }
473   }
474 }
475 #endif
476 
477 #ifdef ASSERT
478 void Continuation::debug_verify_continuation(oop contOop) {
479   if (!VerifyContinuations) {
480     return;
481   }
482   assert(contOop != nullptr, "");
483   assert(oopDesc::is_oop(contOop), "");
484   ContinuationWrapper cont(contOop);
485 
486   assert(oopDesc::is_oop_or_null(cont.tail()), "");
487   assert(cont.chunk_invariant(), "");
488 
489   bool nonempty_chunk = false;
490   size_t max_size = 0;
491   int num_chunks = 0;
492   int num_frames = 0;
493   int num_interpreted_frames = 0;
494   int num_oops = 0;
495 
496   for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
497     log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks);
498     chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames);
499     if (!chunk->is_empty()) {
500       nonempty_chunk = true;
501     }
502     num_chunks++;
503   }
504 
505   const bool is_empty = cont.is_empty();
506   assert(!nonempty_chunk || !is_empty, "");
507   assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), "");
508 }
509 
510 void Continuation::print(oop continuation) { print_on(tty, continuation); }
511 
512 void Continuation::print_on(outputStream* st, oop continuation) {
513   ContinuationWrapper cont(continuation);
514 
515   st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d",
516     continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation));
517   st->print_cr("CHUNKS:");
518   for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
519     st->print("* ");
520     chunk->print_on(true, st);
521   }
522 }
523 #endif // ASSERT
524 
525 
526 void continuations_init() { Continuations::init(); }
527 
528 void Continuations::init() {
529   Continuation::init();
530 }
531 
532 bool Continuations::enabled() {
533   return VMContinuations;
534 }
535 
536 #define CC (char*)  /*cast a literal from (const char*)*/
537 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
538 
539 static JNINativeMethod CONT_methods[] = {
540     {CC"pin",              CC"()V",                                    FN_PTR(CONT_pin)},
541     {CC"unpin",            CC"()V",                                    FN_PTR(CONT_unpin)},
542     {CC"isPinned0",        CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)},
543 };
544 
545 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) {
546     JavaThread* thread = JavaThread::current();
547     ThreadToNativeFromVM trans(thread);
548     int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod));
549     guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives");
550     guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives");
551 }