1 /*
  2  * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/barrierSetNMethod.hpp"
 27 #include "runtime/arguments.hpp"
 28 #include "runtime/continuation.hpp"
 29 #include "runtime/continuationEntry.inline.hpp"
 30 #include "runtime/continuationHelper.inline.hpp"
 31 #include "runtime/continuationJavaClasses.inline.hpp"
 32 #include "runtime/continuationWrapper.inline.hpp"
 33 #include "runtime/interfaceSupport.inline.hpp"
 34 #include "runtime/javaThread.inline.hpp"
 35 #include "runtime/osThread.hpp"
 36 #include "runtime/vframe.inline.hpp"
 37 #include "runtime/vframe_hp.hpp"
 38 
 39 // defined in continuationFreezeThaw.cpp
 40 extern "C" jint JNICALL CONT_isPinned0(JNIEnv* env, jobject cont_scope);
 41 
 42 JVM_ENTRY(void, CONT_pin(JNIEnv* env, jclass cls)) {
 43   if (!Continuation::pin(JavaThread::thread_from_jni_environment(env))) {
 44      THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin overflow");
 45   }
 46 }
 47 JVM_END
 48 
 49 JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) {
 50   if (!Continuation::unpin(JavaThread::thread_from_jni_environment(env))) {
 51      THROW_MSG(vmSymbols::java_lang_IllegalStateException(), "pin underflow");
 52   }
 53 }
 54 JVM_END
 55 
 56 #ifndef PRODUCT
 57 static jlong java_tid(JavaThread* thread) {
 58   return java_lang_Thread::thread_id(thread->threadObj());
 59 }
 60 #endif
 61 
 62 const ContinuationEntry* Continuation::last_continuation(const JavaThread* thread, oop cont_scope) {
 63   // guarantee (thread->has_last_Java_frame(), "");
 64   for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) {
 65     if (cont_scope == jdk_internal_vm_Continuation::scope(entry->cont_oop())) {
 66       return entry;
 67     }
 68   }
 69   return nullptr;
 70 }
 71 
 72 ContinuationEntry* Continuation::get_continuation_entry_for_continuation(JavaThread* thread, oop continuation) {
 73   if (thread == nullptr || continuation == nullptr) {
 74     return nullptr;
 75   }
 76 
 77   for (ContinuationEntry* entry = thread->last_continuation(); entry != nullptr; entry = entry->parent()) {
 78     if (continuation == entry->cont_oop()) {
 79       return entry;
 80     }
 81   }
 82   return nullptr;
 83 }
 84 
 85 static bool is_on_stack(JavaThread* thread, const ContinuationEntry* entry) {
 86   if (entry == nullptr) {
 87     return false;
 88   }
 89 
 90   assert(thread->is_in_full_stack((address)entry), "");
 91   return true;
 92   // return false if called when transitioning to Java on return from freeze
 93   // return !thread->has_last_Java_frame() || thread->last_Java_sp() < cont->entry_sp();
 94 }
 95 
 96 bool Continuation::is_continuation_mounted(JavaThread* thread, oop continuation) {
 97   return is_on_stack(thread, get_continuation_entry_for_continuation(thread, continuation));
 98 }
 99 
100 bool Continuation::is_continuation_scope_mounted(JavaThread* thread, oop cont_scope) {
101   return is_on_stack(thread, last_continuation(thread, cont_scope));
102 }
103 
104 // When walking the virtual stack, this method returns true
105 // iff the frame is a thawed continuation frame whose
106 // caller is still frozen on the h-stack.
107 // The continuation object can be extracted from the thread.
108 bool Continuation::is_cont_barrier_frame(const frame& f) {
109   assert(f.is_interpreted_frame() || f.cb() != nullptr, "");
110   if (!Continuations::enabled()) return false;
111   return is_return_barrier_entry(f.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::return_pc(f)
112                                                           : ContinuationHelper::CompiledFrame::return_pc(f));
113 }
114 
115 bool Continuation::is_return_barrier_entry(const address pc) {
116   if (!Continuations::enabled()) return false;
117   return pc == StubRoutines::cont_returnBarrier();
118 }
119 
120 bool Continuation::is_continuation_enterSpecial(const frame& f) {
121   if (f.cb() == nullptr || !f.cb()->is_compiled()) {
122     return false;
123   }
124   Method* m = f.cb()->as_compiled_method()->method();
125   return (m != nullptr && m->is_continuation_enter_intrinsic());
126 }
127 
128 bool Continuation::is_continuation_entry_frame(const frame& f, const RegisterMap *map) {
129   // we can do this because the entry frame is never inlined
130   Method* m = (map != nullptr && map->in_cont() && f.is_interpreted_frame())
131                   ? map->stack_chunk()->interpreter_frame_method(f)
132                   : ContinuationHelper::Frame::frame_method(f);
133   return m != nullptr && m->intrinsic_id() == vmIntrinsics::_Continuation_enter;
134 }
135 
136 static inline bool is_sp_in_continuation(const ContinuationEntry* entry, intptr_t* const sp) {
137   return entry->entry_sp() > sp;
138 }
139 
140 bool Continuation::is_frame_in_continuation(const ContinuationEntry* entry, const frame& f) {
141   return is_sp_in_continuation(entry, f.unextended_sp());
142 }
143 
144 ContinuationEntry* Continuation::get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp) {
145   assert(thread != nullptr, "");
146   ContinuationEntry* entry = thread->last_continuation();
147   while (entry != nullptr && !is_sp_in_continuation(entry, sp)) {
148     entry = entry->parent();
149   }
150   return entry;
151 }
152 
153 ContinuationEntry* Continuation::get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f) {
154   assert(is_continuation_enterSpecial(f), "");
155   ContinuationEntry* entry = (ContinuationEntry*)f.unextended_sp();
156   assert(entry == get_continuation_entry_for_sp(thread, f.sp()-2), "mismatched entry");
157   return entry;
158 }
159 
160 bool Continuation::is_frame_in_continuation(JavaThread* thread, const frame& f) {
161   return f.is_heap_frame() || (get_continuation_entry_for_sp(thread, f.unextended_sp()) != nullptr);
162 }
163 
164 static frame continuation_top_frame(const ContinuationWrapper& cont, RegisterMap* map) {
165   stackChunkOop chunk = cont.last_nonempty_chunk();
166   map->set_stack_chunk(chunk);
167   return chunk != nullptr ? chunk->top_frame(map) : frame();
168 }
169 
170 bool Continuation::has_last_Java_frame(oop continuation, frame* frame, RegisterMap* map) {
171   ContinuationWrapper cont(continuation);
172   if (!cont.is_empty()) {
173     *frame = continuation_top_frame(cont, map);
174     return true;
175   } else {
176     return false;
177   }
178 }
179 
180 frame Continuation::last_frame(oop continuation, RegisterMap *map) {
181   assert(map != nullptr, "a map must be given");
182   return continuation_top_frame(ContinuationWrapper(continuation), map);
183 }
184 
185 frame Continuation::top_frame(const frame& callee, RegisterMap* map) {
186   assert(map != nullptr, "");
187   ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), callee.sp());
188   assert(ce != nullptr, "");
189   oop continuation = ce->cont_oop();
190   ContinuationWrapper cont(continuation);
191   return continuation_top_frame(cont, map);
192 }
193 
194 javaVFrame* Continuation::last_java_vframe(Handle continuation, RegisterMap *map) {
195   assert(map != nullptr, "a map must be given");
196   if (!ContinuationWrapper(continuation()).is_empty()) {
197     frame f = last_frame(continuation(), map);
198     for (vframe* vf = vframe::new_vframe(&f, map, nullptr); vf; vf = vf->sender()) {
199       if (vf->is_java_frame()) {
200         return javaVFrame::cast(vf);
201       }
202     }
203   }
204   return nullptr;
205 }
206 
207 frame Continuation::continuation_parent_frame(RegisterMap* map) {
208   assert(map->in_cont(), "");
209   ContinuationWrapper cont(map);
210   assert(map->thread() != nullptr || !cont.is_mounted(), "");
211 
212   log_develop_trace(continuations)("continuation_parent_frame");
213   if (map->update_map()) {
214     // we need to register the link address for the entry frame
215     if (cont.entry() != nullptr) {
216       cont.entry()->update_register_map(map);
217     } else {
218       map->clear();
219     }
220   }
221 
222   if (!cont.is_mounted()) { // When we're walking an unmounted continuation and reached the end
223     oop parent = jdk_internal_vm_Continuation::parent(cont.continuation());
224     stackChunkOop chunk = parent != nullptr ? ContinuationWrapper(parent).last_nonempty_chunk() : nullptr;
225     if (chunk != nullptr) {
226       return chunk->top_frame(map);
227     }
228 
229     map->set_stack_chunk(nullptr);
230     return frame();
231   }
232 
233   map->set_stack_chunk(nullptr);
234 
235 #if (defined(X86) || defined(AARCH64)) && !defined(ZERO)
236   frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC());
237 #else
238   frame sender = frame();
239   Unimplemented();
240 #endif
241 
242   return sender;
243 }
244 
245 oop Continuation::continuation_scope(oop continuation) {
246   return continuation != nullptr ? jdk_internal_vm_Continuation::scope(continuation) : nullptr;
247 }
248 
249 bool Continuation::is_scope_bottom(oop cont_scope, const frame& f, const RegisterMap* map) {
250   if (cont_scope == nullptr || !is_continuation_entry_frame(f, map)) {
251     return false;
252   }
253 
254   oop continuation;
255   if (map->in_cont()) {
256     continuation = map->cont();
257   } else {
258     ContinuationEntry* ce = get_continuation_entry_for_sp(map->thread(), f.sp());
259     if (ce == nullptr) {
260       return false;
261     }
262     continuation = ce->cont_oop();
263   }
264   if (continuation == nullptr) {
265     return false;
266   }
267 
268   oop sc = continuation_scope(continuation);
269   assert(sc != nullptr, "");
270   return sc == cont_scope;
271 }
272 
273 bool Continuation::is_in_usable_stack(address addr, const RegisterMap* map) {
274   ContinuationWrapper cont(map);
275   stackChunkOop chunk = cont.find_chunk_by_address(addr);
276   return chunk != nullptr ? chunk->is_usable_in_chunk(addr) : false;
277 }
278 
279 bool Continuation::pin(JavaThread* current) {
280   ContinuationEntry* ce = current->last_continuation();
281   if (ce == nullptr) {
282     return true; // no continuation mounted
283   }
284   return ce->pin();
285 }
286 
287 bool Continuation::unpin(JavaThread* current) {
288   ContinuationEntry* ce = current->last_continuation();
289   if (ce == nullptr) {
290     return true; // no continuation mounted
291   }
292   return ce->unpin();
293 }
294 
295 frame Continuation::continuation_bottom_sender(JavaThread* thread, const frame& callee, intptr_t* sender_sp) {
296   assert (thread != nullptr, "");
297   ContinuationEntry* ce = get_continuation_entry_for_sp(thread,
298         callee.is_interpreted_frame() ? callee.interpreter_frame_last_sp() : callee.unextended_sp());
299   assert(ce != nullptr, "callee.unextended_sp(): " INTPTR_FORMAT, p2i(callee.unextended_sp()));
300 
301   log_develop_debug(continuations)("continuation_bottom_sender: [" JLONG_FORMAT "] [%d] callee: " INTPTR_FORMAT
302     " sender_sp: " INTPTR_FORMAT,
303     java_tid(thread), thread->osthread()->thread_id(), p2i(callee.sp()), p2i(sender_sp));
304 
305   frame entry = ce->to_frame();
306   if (callee.is_interpreted_frame()) {
307     entry.set_sp(sender_sp); // sp != unextended_sp
308   }
309   return entry;
310 }
311 
312 address Continuation::get_top_return_pc_post_barrier(JavaThread* thread, address pc) {
313   ContinuationEntry* ce;
314   if (thread != nullptr && is_return_barrier_entry(pc) && (ce = thread->last_continuation()) != nullptr) {
315     return ce->entry_pc();
316   }
317   return pc;
318 }
319 
320 void Continuation::set_cont_fastpath_thread_state(JavaThread* thread) {
321   assert(thread != nullptr, "");
322   bool fast = !thread->is_interp_only_mode();
323   thread->set_cont_fastpath_thread_state(fast);
324 }
325 
326 void Continuation::notify_deopt(JavaThread* thread, intptr_t* sp) {
327   ContinuationEntry* entry = thread->last_continuation();
328 
329   if (entry == nullptr) {
330     return;
331   }
332 
333   if (is_sp_in_continuation(entry, sp)) {
334     thread->push_cont_fastpath(sp);
335     return;
336   }
337 
338   ContinuationEntry* prev;
339   do {
340     prev = entry;
341     entry = entry->parent();
342   } while (entry != nullptr && !is_sp_in_continuation(entry, sp));
343 
344   if (entry == nullptr) {
345     return;
346   }
347   assert(is_sp_in_continuation(entry, sp), "");
348   if (sp > prev->parent_cont_fastpath()) {
349     prev->set_parent_cont_fastpath(sp);
350   }
351 }
352 
353 #ifndef PRODUCT
354 void Continuation::describe(FrameValues &values) {
355   JavaThread* thread = JavaThread::active();
356   if (thread != nullptr) {
357     for (ContinuationEntry* ce = thread->last_continuation(); ce != nullptr; ce = ce->parent()) {
358       intptr_t* bottom = ce->entry_sp();
359       if (bottom != nullptr) {
360         values.describe(-1, bottom, "continuation entry");
361       }
362     }
363   }
364 }
365 #endif
366 
367 #ifdef ASSERT
368 void Continuation::debug_verify_continuation(oop contOop) {
369   if (!VerifyContinuations) {
370     return;
371   }
372   assert(contOop != nullptr, "");
373   assert(oopDesc::is_oop(contOop), "");
374   ContinuationWrapper cont(contOop);
375 
376   assert(oopDesc::is_oop_or_null(cont.tail()), "");
377   assert(cont.chunk_invariant(), "");
378 
379   bool nonempty_chunk = false;
380   size_t max_size = 0;
381   int num_chunks = 0;
382   int num_frames = 0;
383   int num_interpreted_frames = 0;
384   int num_oops = 0;
385 
386   for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
387     log_develop_trace(continuations)("debug_verify_continuation chunk %d", num_chunks);
388     chunk->verify(&max_size, &num_oops, &num_frames, &num_interpreted_frames);
389     if (!chunk->is_empty()) {
390       nonempty_chunk = true;
391     }
392     num_chunks++;
393   }
394 
395   const bool is_empty = cont.is_empty();
396   assert(!nonempty_chunk || !is_empty, "");
397   assert(is_empty == (!nonempty_chunk && cont.last_frame().is_empty()), "");
398 }
399 
400 void Continuation::print(oop continuation) { print_on(tty, continuation); }
401 
402 void Continuation::print_on(outputStream* st, oop continuation) {
403   ContinuationWrapper cont(continuation);
404 
405   st->print_cr("CONTINUATION: " PTR_FORMAT " done: %d",
406     continuation->identity_hash(), jdk_internal_vm_Continuation::done(continuation));
407   st->print_cr("CHUNKS:");
408   for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
409     st->print("* ");
410     chunk->print_on(true, st);
411   }
412 }
413 #endif // ASSERT
414 
415 
416 void continuations_init() { Continuations::init(); }
417 
418 void Continuations::init() {
419   Continuation::init();
420 }
421 
422 // While virtual threads are in Preview, there are some VM mechanisms we disable if continuations aren't used
423 // See NMethodSweeper::do_stack_scanning and nmethod::is_not_on_continuation_stack
424 bool Continuations::enabled() {
425   // return VMContinuations && Arguments::enable_preview();
426   return VMContinuations;
427 }
428 
429 // We initialize the _gc_epoch to 2, because previous_completed_gc_marking_cycle
430 // subtracts the value by 2, and the type is unsigned. We don't want underflow.
431 //
432 // Odd values mean that marking is in progress, and even values mean that no
433 // marking is currently active.
434 uint64_t Continuations::_gc_epoch = 2;
435 
436 uint64_t Continuations::gc_epoch() {
437   return _gc_epoch;
438 }
439 
440 bool Continuations::is_gc_marking_cycle_active() {
441   // Odd means that marking is active
442   return (_gc_epoch % 2) == 1;
443 }
444 
445 uint64_t Continuations::previous_completed_gc_marking_cycle() {
446   if (is_gc_marking_cycle_active()) {
447     return _gc_epoch - 2;
448   } else {
449     return _gc_epoch - 1;
450   }
451 }
452 
453 void Continuations::on_gc_marking_cycle_start() {
454   assert(!is_gc_marking_cycle_active(), "Previous marking cycle never ended");
455   ++_gc_epoch;
456 }
457 
458 void Continuations::on_gc_marking_cycle_finish() {
459   assert(is_gc_marking_cycle_active(), "Marking cycle started before last one finished");
460   ++_gc_epoch;
461 }
462 
463 void Continuations::arm_all_nmethods() {
464   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
465   if (bs_nm != NULL) {
466     bs_nm->arm_all_nmethods();
467   }
468 }
469 
470 #define CC (char*)  /*cast a literal from (const char*)*/
471 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
472 
473 static JNINativeMethod CONT_methods[] = {
474     {CC"pin",              CC"()V",                                    FN_PTR(CONT_pin)},
475     {CC"unpin",            CC"()V",                                    FN_PTR(CONT_unpin)},
476     {CC"isPinned0",        CC"(Ljdk/internal/vm/ContinuationScope;)I", FN_PTR(CONT_isPinned0)},
477 };
478 
479 void CONT_RegisterNativeMethods(JNIEnv *env, jclass cls) {
480     Thread* thread = Thread::current();
481     assert(thread->is_Java_thread(), "");
482     ThreadToNativeFromVM trans((JavaThread*)thread);
483     int status = env->RegisterNatives(cls, CONT_methods, sizeof(CONT_methods)/sizeof(JNINativeMethod));
484     guarantee(status == JNI_OK, "register jdk.internal.vm.Continuation natives");
485     guarantee(!env->ExceptionOccurred(), "register jdk.internal.vm.Continuation natives");
486 }