1 /*
   2  * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.inline.hpp"
  27 #include "jvmtifiles/jvmtiEnv.hpp"
  28 #include "memory/resourceArea.hpp"
  29 #include "oops/oopHandle.inline.hpp"
  30 #include "prims/jvmtiEnvBase.hpp"
  31 #include "prims/jvmtiEventController.inline.hpp"
  32 #include "prims/jvmtiImpl.hpp"
  33 #include "prims/jvmtiThreadState.inline.hpp"
  34 #include "runtime/handles.inline.hpp"
  35 #include "runtime/interfaceSupport.inline.hpp"
  36 #include "runtime/jniHandles.inline.hpp"
  37 #include "runtime/safepointVerifiers.hpp"
  38 #include "runtime/stackFrameStream.inline.hpp"
  39 #include "runtime/vframe.hpp"
  40 
  41 // marker for when the stack depth has been reset and is now unknown.
  42 // any negative number would work but small ones might obscure an
  43 // underrun error.
  44 static const int UNKNOWN_STACK_DEPTH = -99;
  45 
  46 ///////////////////////////////////////////////////////////////
  47 //
  48 // class JvmtiThreadState
  49 //
  50 // Instances of JvmtiThreadState hang off of each thread.
  51 // Thread local storage for JVMTI.
  52 //
  53 
  54 JvmtiThreadState *JvmtiThreadState::_head = nullptr;
  55 bool JvmtiThreadState::_seen_interp_only_mode = false;
  56 
  57 JvmtiThreadState::JvmtiThreadState(JavaThread* thread, oop thread_oop)
  58   : _thread_event_enable() {
  59   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
  60   _thread               = thread;
  61   _thread_saved         = nullptr;
  62   _exception_state      = ES_CLEARED;
  63   _debuggable           = true;
  64   _hide_single_stepping = false;
  65   _pending_interp_only_mode = false;
  66   _hide_level           = 0;
  67   _pending_step_for_popframe = false;
  68   _class_being_redefined = nullptr;
  69   _class_load_kind = jvmti_class_load_kind_load;
  70   _classes_being_redefined = nullptr;
  71   _head_env_thread_state = nullptr;
  72   _dynamic_code_event_collector = nullptr;
  73   _vm_object_alloc_event_collector = nullptr;
  74   _sampled_object_alloc_event_collector = nullptr;
  75   _the_class_for_redefinition_verification = nullptr;
  76   _scratch_class_for_redefinition_verification = nullptr;
  77   _cur_stack_depth = UNKNOWN_STACK_DEPTH;
  78   _saved_interp_only_mode = 0;
  79 
  80   // JVMTI ForceEarlyReturn support
  81   _pending_step_for_earlyret = false;
  82   _earlyret_state = earlyret_inactive;
  83   _earlyret_tos = ilgl;
  84   _earlyret_value.j = 0L;
  85   _earlyret_oop = nullptr;
  86   _jvmti_event_queue = nullptr;
  87   _is_virtual = false;
  88 
  89   _thread_oop_h = OopHandle(JvmtiExport::jvmti_oop_storage(), thread_oop);
  90 
  91   // add all the JvmtiEnvThreadState to the new JvmtiThreadState
  92   {
  93     JvmtiEnvIterator it;
  94     for (JvmtiEnvBase* env = it.first(); env != nullptr; env = it.next(env)) {
  95       if (env->is_valid()) {
  96         add_env(env);
  97       }
  98     }
  99   }
 100 
 101   // link us into the list
 102   {
 103     // The thread state list manipulation code must not have safepoints.
 104     // See periodic_clean_up().
 105     debug_only(NoSafepointVerifier nosafepoint;)
 106 
 107     _prev = nullptr;
 108     _next = _head;
 109     if (_head != nullptr) {
 110       _head->_prev = this;
 111     }
 112     _head = this;
 113   }
 114 
 115   if (thread_oop != nullptr) {
 116     java_lang_Thread::set_jvmti_thread_state(thread_oop, this);
 117     _is_virtual = java_lang_VirtualThread::is_instance(thread_oop);
 118   }
 119 
 120   if (thread != nullptr) {
 121     if (thread_oop == nullptr || thread->jvmti_vthread() == nullptr || thread->jvmti_vthread() == thread_oop) {
 122       // The JavaThread for carrier or mounted virtual thread case.
 123       // Set this only if thread_oop is current thread->jvmti_vthread().
 124       thread->set_jvmti_thread_state(this);
 125     }
 126     thread->set_interp_only_mode(0);
 127   }
 128 }
 129 
 130 
 131 JvmtiThreadState::~JvmtiThreadState()   {
 132   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
 133 
 134   if (_classes_being_redefined != nullptr) {
 135     delete _classes_being_redefined; // free the GrowableArray on C heap
 136   }
 137 
 138   // clear this as the state for the thread
 139   get_thread()->set_jvmti_thread_state(nullptr);
 140 
 141   // zap our env thread states
 142   {
 143     JvmtiEnvBase::entering_dying_thread_env_iteration();
 144     JvmtiEnvThreadStateIterator it(this);
 145     for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ) {
 146       JvmtiEnvThreadState* zap = ets;
 147       ets = it.next(ets);
 148       delete zap;
 149     }
 150     JvmtiEnvBase::leaving_dying_thread_env_iteration();
 151   }
 152 
 153   // remove us from the list
 154   {
 155     // The thread state list manipulation code must not have safepoints.
 156     // See periodic_clean_up().
 157     debug_only(NoSafepointVerifier nosafepoint;)
 158 
 159     if (_prev == nullptr) {
 160       assert(_head == this, "sanity check");
 161       _head = _next;
 162     } else {
 163       assert(_head != this, "sanity check");
 164       _prev->_next = _next;
 165     }
 166     if (_next != nullptr) {
 167       _next->_prev = _prev;
 168     }
 169     _next = nullptr;
 170     _prev = nullptr;
 171   }
 172   if (get_thread_oop() != nullptr) {
 173     java_lang_Thread::set_jvmti_thread_state(get_thread_oop(), nullptr);
 174   }
 175   _thread_oop_h.release(JvmtiExport::jvmti_oop_storage());
 176 }
 177 
 178 
 179 void
 180 JvmtiThreadState::periodic_clean_up() {
 181   assert(SafepointSynchronize::is_at_safepoint(), "at safepoint");
 182 
 183   // This iteration is initialized with "_head" instead of "JvmtiThreadState::first()"
 184   // because the latter requires the JvmtiThreadState_lock.
 185   // This iteration is safe at a safepoint as well, see the NoSafepointVerifier
 186   // asserts at all list manipulation sites.
 187   for (JvmtiThreadState *state = _head; state != nullptr; state = state->next()) {
 188     // For each environment thread state corresponding to an invalid environment
 189     // unlink it from the list and deallocate it.
 190     JvmtiEnvThreadStateIterator it(state);
 191     JvmtiEnvThreadState* previous_ets = nullptr;
 192     JvmtiEnvThreadState* ets = it.first();
 193     while (ets != nullptr) {
 194       if (ets->get_env()->is_valid()) {
 195         previous_ets = ets;
 196         ets = it.next(ets);
 197       } else {
 198         // This one isn't valid, remove it from the list and deallocate it
 199         JvmtiEnvThreadState* defunct_ets = ets;
 200         ets = ets->next();
 201         if (previous_ets == nullptr) {
 202           assert(state->head_env_thread_state() == defunct_ets, "sanity check");
 203           state->set_head_env_thread_state(ets);
 204         } else {
 205           previous_ets->set_next(ets);
 206         }
 207         delete defunct_ets;
 208       }
 209     }
 210   }
 211 }
 212 
 213 //
 214 // Virtual Threads Mount State transition (VTMS transition) mechanism
 215 //
 216 
 217 // VTMS transitions cannot be disabled while this counter is positive.
 218 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_count = 0;
 219 
 220 // VTMS transitions for one virtual thread are disabled while it is positive
 221 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_disable_for_one_count = 0;
 222 
 223 // VTMS transitions for all virtual threads are disabled while it is positive
 224 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_disable_for_all_count = 0;
 225 
 226 // There is an active suspender or resumer.
 227 volatile bool JvmtiVTMSTransitionDisabler::_SR_mode = false;
 228 
 229 // Notifications from VirtualThread about VTMS events are enabled.
 230 bool JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events = false;
 231 
 232 // The JvmtiVTMSTransitionDisabler sync protocol is enabled if this count > 0.
 233 volatile int JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_count = 0;
 234 
 235 // JvmtiVTMSTraansitionDisabler sync protocol is enabled permanently after seeing a suspender.
 236 volatile bool JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_permanently = false;
 237 
 238 #ifdef ASSERT
 239 void
 240 JvmtiVTMSTransitionDisabler::print_info() {
 241   log_error(jvmti)("_VTMS_transition_count: %d\n", _VTMS_transition_count);
 242   log_error(jvmti)("_VTMS_transition_disable_for_one_count: %d\n", _VTMS_transition_disable_for_one_count);
 243   log_error(jvmti)("_VTMS_transition_disable_for_all_count: %d\n\n", _VTMS_transition_disable_for_all_count);
 244   int attempts = 10000;
 245   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *java_thread = jtiwh.next(); ) {
 246     ResourceMark rm;
 247     // Handshake with target.
 248     PrintStackTraceClosure pstc;
 249     Handshake::execute(&pstc, java_thread);
 250   }
 251 }
 252 #endif
 253 
 254 // disable VTMS transitions for one virtual thread
 255 // no-op if thread is non-null and not a virtual thread
 256 JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(jthread thread)
 257   : _is_SR(false), _thread(thread)
 258 {
 259   if (!Continuations::enabled()) {
 260     return; // JvmtiVTMSTransitionDisabler is no-op without virtual threads
 261   }
 262   if (Thread::current_or_null() == nullptr) {
 263     return;  // Detached thread, can be a call from Agent_OnLoad.
 264   }
 265   if (!sync_protocol_enabled_permanently()) {
 266     JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
 267   }
 268   if (_thread != nullptr) {
 269     VTMS_transition_disable_for_one(); // disable VTMS transitions for one virtual thread
 270   } else {
 271     VTMS_transition_disable_for_all(); // disable VTMS transitions for all virtual threads
 272   }
 273 }
 274 
 275 // disable VTMS transitions for all virtual threads
 276 JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(bool is_SR)
 277   : _is_SR(is_SR), _thread(nullptr)
 278 {
 279   if (!Continuations::enabled()) {
 280     return; // JvmtiVTMSTransitionDisabler is no-op without virtual threads
 281   }
 282   if (Thread::current_or_null() == nullptr) {
 283     return;  // Detached thread, can be a call from Agent_OnLoad.
 284   }
 285   if (!sync_protocol_enabled_permanently()) {
 286     JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
 287     if (is_SR) {
 288       Atomic::store(&_sync_protocol_enabled_permanently, true);
 289     }
 290   }
 291   VTMS_transition_disable_for_all();
 292 }
 293 
 294 JvmtiVTMSTransitionDisabler::~JvmtiVTMSTransitionDisabler() {
 295   if (!Continuations::enabled()) {
 296     return; // JvmtiVTMSTransitionDisabler is a no-op without virtual threads
 297   }
 298   if (Thread::current_or_null() == nullptr) {
 299     return;  // Detached thread, can be a call from Agent_OnLoad.
 300   }
 301   if (_thread != nullptr) {
 302     VTMS_transition_enable_for_one(); // enable VTMS transitions for one virtual thread
 303   } else {
 304     VTMS_transition_enable_for_all(); // enable VTMS transitions for all virtual threads
 305   }
 306   if (!sync_protocol_enabled_permanently()) {
 307     JvmtiVTMSTransitionDisabler::dec_sync_protocol_enabled_count();
 308   }
 309 }
 310 
 311 // disable VTMS transitions for one virtual thread
 312 void
 313 JvmtiVTMSTransitionDisabler::VTMS_transition_disable_for_one() {
 314   assert(_thread != nullptr, "sanity check");
 315   JavaThread* thread = JavaThread::current();
 316   HandleMark hm(thread);
 317   Handle vth = Handle(thread, JNIHandles::resolve_external_guard(_thread));
 318   if (!java_lang_VirtualThread::is_instance(vth())) {
 319     return; // no-op if _thread is not a virtual thread
 320   }
 321   MonitorLocker ml(JvmtiVTMSTransition_lock);
 322 
 323   while (_SR_mode) { // suspender or resumer is a JvmtiVTMSTransitionDisabler monopolist
 324     ml.wait(10); // wait while there is an active suspender or resumer
 325   }
 326   Atomic::inc(&_VTMS_transition_disable_for_one_count);
 327   java_lang_Thread::inc_VTMS_transition_disable_count(vth());
 328 
 329   while (java_lang_Thread::is_in_VTMS_transition(vth())) {
 330     ml.wait(10); // wait while the virtual thread is in transition
 331   }
 332 #ifdef ASSERT
 333   thread->set_is_VTMS_transition_disabler(true);
 334 #endif
 335 }
 336 
 337 // disable VTMS transitions for all virtual threads
 338 void
 339 JvmtiVTMSTransitionDisabler::VTMS_transition_disable_for_all() {
 340   JavaThread* thread = JavaThread::current();
 341   int attempts = 50000;
 342   {
 343     MonitorLocker ml(JvmtiVTMSTransition_lock);
 344 
 345     assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 346     assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
 347     while (_SR_mode) { // Suspender or resumer is a JvmtiVTMSTransitionDisabler monopolist.
 348       ml.wait(10);     // Wait while there is an active suspender or resumer.
 349     }
 350     if (_is_SR) {
 351       _SR_mode = true;
 352       while (_VTMS_transition_disable_for_all_count > 0 ||
 353              _VTMS_transition_disable_for_one_count > 0) {
 354         ml.wait(10);   // Wait while there is any active jvmtiVTMSTransitionDisabler.
 355       }
 356     }
 357     Atomic::inc(&_VTMS_transition_disable_for_all_count);
 358 
 359     // Block while some mount/unmount transitions are in progress.
 360     // Debug version fails and prints diagnostic information.
 361     while (_VTMS_transition_count > 0) {
 362       if (ml.wait(10)) {
 363         attempts--;
 364       }
 365       DEBUG_ONLY(if (attempts == 0) break;)
 366     }
 367     assert(!thread->is_VTMS_transition_disabler(), "VTMS_transition sanity check");
 368 #ifdef ASSERT
 369     if (attempts > 0) {
 370       thread->set_is_VTMS_transition_disabler(true);
 371     }
 372 #endif
 373   }
 374 #ifdef ASSERT
 375     if (attempts == 0) {
 376       print_info();
 377       fatal("stuck in JvmtiVTMSTransitionDisabler::VTMS_transition_disable");
 378     }
 379 #endif
 380 }
 381 
 382 // enable VTMS transitions for one virtual thread
 383 void
 384 JvmtiVTMSTransitionDisabler::VTMS_transition_enable_for_one() {
 385   JavaThread* thread = JavaThread::current();
 386   HandleMark hm(thread);
 387   Handle vth = Handle(thread, JNIHandles::resolve_external_guard(_thread));
 388   if (!java_lang_VirtualThread::is_instance(vth())) {
 389     return; // no-op if _thread is not a virtual thread
 390   }
 391   MonitorLocker ml(JvmtiVTMSTransition_lock);
 392   java_lang_Thread::dec_VTMS_transition_disable_count(vth());
 393   Atomic::dec(&_VTMS_transition_disable_for_one_count);
 394   if (_VTMS_transition_disable_for_one_count == 0) {
 395     ml.notify_all();
 396   }
 397 #ifdef ASSERT
 398   thread->set_is_VTMS_transition_disabler(false);
 399 #endif
 400 }
 401 
 402 // enable VTMS transitions for all virtual threads
 403 void
 404 JvmtiVTMSTransitionDisabler::VTMS_transition_enable_for_all() {
 405   JavaThread* current = JavaThread::current();
 406   {
 407     MonitorLocker ml(JvmtiVTMSTransition_lock);
 408     assert(_VTMS_transition_disable_for_all_count > 0, "VTMS_transition sanity check");
 409 
 410     if (_is_SR) {  // Disabler is suspender or resumer.
 411       _SR_mode = false;
 412     }
 413     Atomic::dec(&_VTMS_transition_disable_for_all_count);
 414     if (_VTMS_transition_disable_for_all_count == 0 || _is_SR) {
 415       ml.notify_all();
 416     }
 417 #ifdef ASSERT
 418     current->set_is_VTMS_transition_disabler(false);
 419 #endif
 420   }
 421 }
 422 
 423 void
 424 JvmtiVTMSTransitionDisabler::start_VTMS_transition(jthread vthread, bool is_mount) {
 425   JavaThread* thread = JavaThread::current();
 426   oop vt = JNIHandles::resolve_external_guard(vthread);
 427   assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
 428 
 429   // Avoid using MonitorLocker on performance critical path, use
 430   // two-level synchronization with lock-free operations on counters.
 431   Atomic::inc(&_VTMS_transition_count); // Try to enter VTMS transition section optmistically.
 432   java_lang_Thread::set_is_in_VTMS_transition(vt, true);
 433 
 434   if (!sync_protocol_enabled()) {
 435     thread->set_is_in_VTMS_transition(true);
 436     return;
 437   }
 438   HandleMark hm(thread);
 439   Handle vth = Handle(thread, vt);
 440   int attempts = 50000;
 441 
 442   // Do not allow suspends inside VTMS transitions.
 443   // Block while transitions are disabled or there are suspend requests.
 444   int64_t thread_id = java_lang_Thread::thread_id(vth());  // Cannot use oops while blocked.
 445 
 446   if (_VTMS_transition_disable_for_all_count > 0 ||
 447       java_lang_Thread::VTMS_transition_disable_count(vth()) > 0 ||
 448       thread->is_suspended() ||
 449       JvmtiVTSuspender::is_vthread_suspended(thread_id)
 450   ) {
 451     // Slow path: undo unsuccessful optimistic counter incrementation.
 452     // It can cause an extra waiting cycle for VTMS transition disablers.
 453     Atomic::dec(&_VTMS_transition_count);
 454     java_lang_Thread::set_is_in_VTMS_transition(vth(), false);
 455 
 456     while (true) {
 457       MonitorLocker ml(JvmtiVTMSTransition_lock);
 458 
 459       // Do not allow suspends inside VTMS transitions.
 460       // Block while transitions are disabled or there are suspend requests.
 461       if (_VTMS_transition_disable_for_all_count > 0 ||
 462           java_lang_Thread::VTMS_transition_disable_count(vth()) > 0 ||
 463           thread->is_suspended() ||
 464           JvmtiVTSuspender::is_vthread_suspended(thread_id)
 465       ) {
 466         // Block while transitions are disabled or there are suspend requests.
 467         if (ml.wait(10)) {
 468           attempts--;
 469         }
 470         DEBUG_ONLY(if (attempts == 0) break;)
 471         continue;  // ~ThreadBlockInVM has handshake-based suspend point.
 472       }
 473       Atomic::inc(&_VTMS_transition_count);
 474       java_lang_Thread::set_is_in_VTMS_transition(vth(), true);
 475       break;
 476     }
 477   }
 478 #ifdef ASSERT
 479   if (attempts == 0) {
 480     log_error(jvmti)("start_VTMS_transition: thread->is_suspended: %d is_vthread_suspended: %d\n\n",
 481                      thread->is_suspended(), JvmtiVTSuspender::is_vthread_suspended(thread_id));
 482     print_info();
 483     fatal("stuck in JvmtiVTMSTransitionDisabler::start_VTMS_transition");
 484   }
 485 #endif
 486   // Enter VTMS transition section.
 487   thread->set_is_in_VTMS_transition(true);
 488 }
 489 
 490 void
 491 JvmtiVTMSTransitionDisabler::finish_VTMS_transition(jthread vthread, bool is_mount) {
 492   JavaThread* thread = JavaThread::current();
 493 
 494   assert(thread->is_in_VTMS_transition(), "sanity check");
 495   thread->set_is_in_VTMS_transition(false);
 496   oop vt = JNIHandles::resolve_external_guard(vthread);
 497   java_lang_Thread::set_is_in_VTMS_transition(vt, false);
 498   Atomic::dec(&_VTMS_transition_count);
 499 
 500   if (!sync_protocol_enabled()) {
 501     return;
 502   }
 503   int64_t thread_id = java_lang_Thread::thread_id(vt);
 504 
 505   // Unblock waiting VTMS transition disablers.
 506   if (_VTMS_transition_disable_for_one_count > 0 ||
 507       _VTMS_transition_disable_for_all_count > 0) {
 508     MonitorLocker ml(JvmtiVTMSTransition_lock);
 509     ml.notify_all();
 510   }
 511   // In unmount case the carrier thread is attached after unmount transition.
 512   // Check and block it if there was external suspend request.
 513   int attempts = 10000;
 514   if (!is_mount && thread->is_carrier_thread_suspended()) {
 515     while (true) {
 516       MonitorLocker ml(JvmtiVTMSTransition_lock);
 517 
 518       // Block while there are suspend requests.
 519       if ((!is_mount && thread->is_carrier_thread_suspended()) ||
 520           (is_mount && JvmtiVTSuspender::is_vthread_suspended(thread_id))
 521       ) {
 522         // Block while there are suspend requests.
 523         if (ml.wait(10)) {
 524           attempts--;
 525         }
 526         DEBUG_ONLY(if (attempts == 0) break;)
 527         continue;
 528       }
 529       break;
 530     }
 531   }
 532 #ifdef ASSERT
 533   if (attempts == 0) {
 534     log_error(jvmti)("finish_VTMS_transition: thread->is_suspended: %d is_vthread_suspended: %d\n\n",
 535                      thread->is_suspended(), JvmtiVTSuspender::is_vthread_suspended(thread_id));
 536     print_info();
 537     fatal("stuck in JvmtiVTMSTransitionDisabler::finish_VTMS_transition");
 538   }
 539 #endif
 540 }
 541 
 542 // set VTMS transition bit value in JavaThread and java.lang.VirtualThread object
 543 void JvmtiVTMSTransitionDisabler::set_is_in_VTMS_transition(JavaThread* thread, jobject vthread, bool in_trans) {
 544   oop vt = JNIHandles::resolve_external_guard(vthread);
 545   java_lang_Thread::set_is_in_VTMS_transition(vt, in_trans);
 546   thread->set_is_in_VTMS_transition(in_trans);
 547 }
 548 
 549 void
 550 JvmtiVTMSTransitionDisabler::VTMS_vthread_start(jobject vthread) {
 551   VTMS_mount_end(vthread);
 552   JavaThread* thread = JavaThread::current();
 553 
 554   assert(!thread->is_in_VTMS_transition(), "sanity check");
 555   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 556 
 557   // If interp_only_mode has been enabled then we must eagerly create JvmtiThreadState
 558   // objects for globally enabled virtual thread filtered events. Otherwise,
 559   // it is an important optimization to create JvmtiThreadState objects lazily.
 560   // This optimization is disabled when watchpoint capabilities are present. It is to
 561   // work around a bug with virtual thread frames which can be not deoptimized in time.
 562   if (JvmtiThreadState::seen_interp_only_mode() ||
 563       JvmtiExport::should_post_field_access() ||
 564       JvmtiExport::should_post_field_modification()){
 565     JvmtiEventController::thread_started(thread);
 566   }
 567   if (JvmtiExport::should_post_vthread_start()) {
 568     JvmtiExport::post_vthread_start(vthread);
 569   }
 570   // post VirtualThreadMount event after VirtualThreadStart
 571   if (JvmtiExport::should_post_vthread_mount()) {
 572     JvmtiExport::post_vthread_mount(vthread);
 573   }
 574 }
 575 
 576 void
 577 JvmtiVTMSTransitionDisabler::VTMS_vthread_end(jobject vthread) {
 578   JavaThread* thread = JavaThread::current();
 579 
 580   assert(!thread->is_in_VTMS_transition(), "sanity check");
 581   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 582 
 583   // post VirtualThreadUnmount event before VirtualThreadEnd
 584   if (JvmtiExport::should_post_vthread_unmount()) {
 585     JvmtiExport::post_vthread_unmount(vthread);
 586   }
 587   if (JvmtiExport::should_post_vthread_end()) {
 588     JvmtiExport::post_vthread_end(vthread);
 589   }
 590   VTMS_unmount_begin(vthread, /* last_unmount */ true);
 591   if (thread->jvmti_thread_state() != nullptr) {
 592     JvmtiExport::cleanup_thread(thread);
 593     assert(thread->jvmti_thread_state() == nullptr, "should be null");
 594     assert(java_lang_Thread::jvmti_thread_state(JNIHandles::resolve(vthread)) == nullptr, "should be null");
 595   }
 596   thread->rebind_to_jvmti_thread_state_of(thread->threadObj());
 597 }
 598 
 599 void
 600 JvmtiVTMSTransitionDisabler::VTMS_vthread_mount(jobject vthread, bool hide) {
 601   if (hide) {
 602     VTMS_mount_begin(vthread);
 603   } else {
 604     VTMS_mount_end(vthread);
 605     if (JvmtiExport::should_post_vthread_mount()) {
 606       JvmtiExport::post_vthread_mount(vthread);
 607     }
 608   }
 609 }
 610 
 611 void
 612 JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount(jobject vthread, bool hide) {
 613   if (hide) {
 614     if (JvmtiExport::should_post_vthread_unmount()) {
 615       JvmtiExport::post_vthread_unmount(vthread);
 616     }
 617     VTMS_unmount_begin(vthread, /* last_unmount */ false);
 618   } else {
 619     VTMS_unmount_end(vthread);
 620   }
 621 }
 622 
 623 void
 624 JvmtiVTMSTransitionDisabler::VTMS_mount_begin(jobject vthread) {
 625   JavaThread* thread = JavaThread::current();
 626   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 627   assert(!thread->is_in_VTMS_transition(), "sanity check");
 628   start_VTMS_transition(vthread, /* is_mount */ true);
 629 }
 630 
 631 void
 632 JvmtiVTMSTransitionDisabler::VTMS_mount_end(jobject vthread) {
 633   JavaThread* thread = JavaThread::current();
 634   oop vt = JNIHandles::resolve(vthread);
 635 
 636   thread->rebind_to_jvmti_thread_state_of(vt);
 637 
 638   JvmtiThreadState* state = thread->jvmti_thread_state();
 639   if (state != nullptr && state->is_pending_interp_only_mode()) {
 640     MutexLocker mu(JvmtiThreadState_lock);
 641     state = thread->jvmti_thread_state();
 642     if (state != nullptr && state->is_pending_interp_only_mode()) {
 643       JvmtiEventController::enter_interp_only_mode();
 644     }
 645   }
 646   assert(thread->is_in_VTMS_transition(), "sanity check");
 647   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 648   finish_VTMS_transition(vthread, /* is_mount */ true);
 649 }
 650 
 651 void
 652 JvmtiVTMSTransitionDisabler::VTMS_unmount_begin(jobject vthread, bool last_unmount) {
 653   JavaThread* thread = JavaThread::current();
 654 
 655   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 656   assert(!thread->is_in_VTMS_transition(), "sanity check");
 657 
 658   start_VTMS_transition(vthread, /* is_mount */ false);
 659   if (!last_unmount) {
 660     thread->rebind_to_jvmti_thread_state_of(thread->threadObj());
 661   }
 662 }
 663 
 664 void
 665 JvmtiVTMSTransitionDisabler::VTMS_unmount_end(jobject vthread) {
 666   JavaThread* thread = JavaThread::current();
 667   assert(thread->is_in_VTMS_transition(), "sanity check");
 668   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 669   finish_VTMS_transition(vthread, /* is_mount */ false);
 670 }
 671 
 672 
 673 //
 674 // Virtual Threads Suspend/Resume management
 675 //
 676 
 677 JvmtiVTSuspender::SR_Mode
 678 JvmtiVTSuspender::_SR_mode = SR_none;
 679 
 680 VirtualThreadList*
 681 JvmtiVTSuspender::_suspended_list = new VirtualThreadList();
 682 
 683 VirtualThreadList*
 684 JvmtiVTSuspender::_not_suspended_list = new VirtualThreadList();
 685 
 686 void
 687 JvmtiVTSuspender::register_all_vthreads_suspend() {
 688   MonitorLocker ml(JvmtiVTMSTransition_lock);
 689 
 690   _SR_mode = SR_all;
 691   _suspended_list->invalidate();
 692   _not_suspended_list->invalidate();
 693 }
 694 
 695 void
 696 JvmtiVTSuspender::register_all_vthreads_resume() {
 697   MonitorLocker ml(JvmtiVTMSTransition_lock);
 698 
 699   _SR_mode = SR_none;
 700   _suspended_list->invalidate();
 701   _not_suspended_list->invalidate();
 702 }
 703 
 704 void
 705 JvmtiVTSuspender::register_vthread_suspend(oop vt) {
 706   int64_t id = java_lang_Thread::thread_id(vt);
 707   MonitorLocker ml(JvmtiVTMSTransition_lock);
 708 
 709   if (_SR_mode == SR_all) {
 710     assert(_not_suspended_list->contains(id),
 711            "register_vthread_suspend sanity check");
 712     _not_suspended_list->remove(id);
 713   } else {
 714     assert(!_suspended_list->contains(id),
 715            "register_vthread_suspend sanity check");
 716     _SR_mode = SR_ind;
 717     _suspended_list->append(id);
 718   }
 719 }
 720 
 721 void
 722 JvmtiVTSuspender::register_vthread_resume(oop vt) {
 723   int64_t id = java_lang_Thread::thread_id(vt);
 724   MonitorLocker ml(JvmtiVTMSTransition_lock);
 725 
 726   if (_SR_mode == SR_all) {
 727     assert(!_not_suspended_list->contains(id),
 728            "register_vthread_resume sanity check");
 729     _not_suspended_list->append(id);
 730   } else if (_SR_mode == SR_ind) {
 731     assert(_suspended_list->contains(id),
 732            "register_vthread_resume check");
 733     _suspended_list->remove(id);
 734     if (_suspended_list->length() == 0) {
 735       _SR_mode = SR_none;
 736     }
 737   } else {
 738     assert(false, "register_vthread_resume: no suspend mode enabled");
 739   }
 740 }
 741 
 742 bool
 743 JvmtiVTSuspender::is_vthread_suspended(int64_t thread_id) {
 744   bool suspend_is_needed =
 745    (_SR_mode == SR_all && !_not_suspended_list->contains(thread_id)) ||
 746    (_SR_mode == SR_ind && _suspended_list->contains(thread_id));
 747 
 748   return suspend_is_needed;
 749 }
 750 
 751 bool
 752 JvmtiVTSuspender::is_vthread_suspended(oop vt) {
 753   return is_vthread_suspended(java_lang_Thread::thread_id(vt));
 754 }
 755 
 756 void JvmtiThreadState::add_env(JvmtiEnvBase *env) {
 757   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
 758 
 759   JvmtiEnvThreadState *new_ets = new JvmtiEnvThreadState(this, env);
 760   // add this environment thread state to the end of the list (order is important)
 761   {
 762     // list deallocation (which occurs at a safepoint) cannot occur simultaneously
 763     debug_only(NoSafepointVerifier nosafepoint;)
 764 
 765     JvmtiEnvThreadStateIterator it(this);
 766     JvmtiEnvThreadState* previous_ets = nullptr;
 767     for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 768       previous_ets = ets;
 769     }
 770     if (previous_ets == nullptr) {
 771       set_head_env_thread_state(new_ets);
 772     } else {
 773       previous_ets->set_next(new_ets);
 774     }
 775   }
 776 }
 777 
 778 void JvmtiThreadState::enter_interp_only_mode() {
 779   assert(_thread != nullptr, "sanity check");
 780   _seen_interp_only_mode = true;
 781   _thread->increment_interp_only_mode();
 782   invalidate_cur_stack_depth();
 783 }
 784 
 785 void JvmtiThreadState::leave_interp_only_mode() {
 786   assert(is_interp_only_mode(), "leaving interp only when not in interp only mode");
 787   if (_thread == nullptr) {
 788     // Unmounted virtual thread updates the saved value.
 789     --_saved_interp_only_mode;
 790   } else {
 791     _thread->decrement_interp_only_mode();
 792   }
 793 }
 794 
 795 
 796 // Helper routine used in several places
 797 int JvmtiThreadState::count_frames() {
 798   JavaThread* thread = get_thread_or_saved();
 799   javaVFrame *jvf;
 800   ResourceMark rm;
 801   if (thread == nullptr) {
 802     oop thread_obj = get_thread_oop();
 803     jvf = JvmtiEnvBase::get_vthread_jvf(thread_obj);
 804   } else {
 805 #ifdef ASSERT
 806     Thread *current_thread = Thread::current();
 807 #endif
 808     assert(SafepointSynchronize::is_at_safepoint() ||
 809            thread->is_handshake_safe_for(current_thread),
 810            "call by myself / at safepoint / at handshake");
 811     if (!thread->has_last_Java_frame()) return 0;  // No Java frames.
 812     // TBD: This might need to be corrected for detached carrier threads.
 813     RegisterMap reg_map(thread,
 814                         RegisterMap::UpdateMap::skip,
 815                         RegisterMap::ProcessFrames::skip,
 816                         RegisterMap::WalkContinuation::include);
 817     jvf = thread->last_java_vframe(&reg_map);
 818     jvf = JvmtiEnvBase::check_and_skip_hidden_frames(thread, jvf);
 819   }
 820   return (int)JvmtiEnvBase::get_frame_count(jvf);
 821 }
 822 
 823 
 824 void JvmtiThreadState::invalidate_cur_stack_depth() {
 825   assert(SafepointSynchronize::is_at_safepoint() ||
 826          get_thread()->is_handshake_safe_for(Thread::current()),
 827          "bad synchronization with owner thread");
 828 
 829   _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 830 }
 831 
 832 void JvmtiThreadState::incr_cur_stack_depth() {
 833   guarantee(JavaThread::current() == get_thread(), "must be current thread");
 834 
 835   if (!is_interp_only_mode()) {
 836     _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 837   }
 838   if (_cur_stack_depth != UNKNOWN_STACK_DEPTH) {
 839     ++_cur_stack_depth;
 840   }
 841 }
 842 
 843 void JvmtiThreadState::decr_cur_stack_depth() {
 844   guarantee(JavaThread::current() == get_thread(), "must be current thread");
 845 
 846   if (!is_interp_only_mode()) {
 847     _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 848   }
 849   if (_cur_stack_depth != UNKNOWN_STACK_DEPTH) {
 850     --_cur_stack_depth;
 851     assert(_cur_stack_depth >= 0, "incr/decr_cur_stack_depth mismatch");
 852   }
 853 }
 854 
 855 int JvmtiThreadState::cur_stack_depth() {
 856   Thread *current = Thread::current();
 857   guarantee(get_thread()->is_handshake_safe_for(current),
 858             "must be current thread or direct handshake");
 859 
 860   if (!is_interp_only_mode() || _cur_stack_depth == UNKNOWN_STACK_DEPTH) {
 861     _cur_stack_depth = count_frames();
 862   } else {
 863 #ifdef ASSERT
 864     if (EnableJVMTIStackDepthAsserts) {
 865       // heavy weight assert
 866       jint num_frames = count_frames();
 867       assert(_cur_stack_depth == num_frames, "cur_stack_depth out of sync _cur_stack_depth: %d num_frames: %d", _cur_stack_depth, num_frames);
 868     }
 869 #endif
 870   }
 871   return _cur_stack_depth;
 872 }
 873 
 874 void JvmtiThreadState::process_pending_step_for_popframe() {
 875   // We are single stepping as the last part of the PopFrame() dance
 876   // so we have some house keeping to do.
 877 
 878   JavaThread *thr = get_thread();
 879   if (thr->popframe_condition() != JavaThread::popframe_inactive) {
 880     // If the popframe_condition field is not popframe_inactive, then
 881     // we missed all of the popframe_field cleanup points:
 882     //
 883     // - unpack_frames() was not called (nothing to deopt)
 884     // - remove_activation_preserving_args_entry() was not called
 885     //   (did not get suspended in a call_vm() family call and did
 886     //   not complete a call_vm() family call on the way here)
 887     thr->clear_popframe_condition();
 888   }
 889 
 890   // clearing the flag indicates we are done with the PopFrame() dance
 891   clr_pending_step_for_popframe();
 892 
 893   // If exception was thrown in this frame, need to reset jvmti thread state.
 894   // Single stepping may not get enabled correctly by the agent since
 895   // exception state is passed in MethodExit event which may be sent at some
 896   // time in the future. JDWP agent ignores MethodExit events if caused by
 897   // an exception.
 898   //
 899   if (is_exception_detected()) {
 900     clear_exception_state();
 901   }
 902   // If step is pending for popframe then it may not be
 903   // a repeat step. The new_bci and method_id is same as current_bci
 904   // and current method_id after pop and step for recursive calls.
 905   // Force the step by clearing the last location.
 906   JvmtiEnvThreadStateIterator it(this);
 907   for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 908     ets->clear_current_location();
 909   }
 910 }
 911 
 912 
 913 // Class:     JvmtiThreadState
 914 // Function:  update_for_pop_top_frame
 915 // Description:
 916 //   This function removes any frame pop notification request for
 917 //   the top frame and invalidates both the current stack depth and
 918 //   all cached frameIDs.
 919 //
 920 // Called by: PopFrame
 921 //
 922 void JvmtiThreadState::update_for_pop_top_frame() {
 923   if (is_interp_only_mode()) {
 924     // remove any frame pop notification request for the top frame
 925     // in any environment
 926     int popframe_number = cur_stack_depth();
 927     {
 928       JvmtiEnvThreadStateIterator it(this);
 929       for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 930         if (ets->is_frame_pop(popframe_number)) {
 931           ets->clear_frame_pop(popframe_number);
 932         }
 933       }
 934     }
 935     // force stack depth to be recalculated
 936     invalidate_cur_stack_depth();
 937   } else {
 938     assert(!is_enabled(JVMTI_EVENT_FRAME_POP), "Must have no framepops set");
 939   }
 940 }
 941 
 942 
 943 void JvmtiThreadState::process_pending_step_for_earlyret() {
 944   // We are single stepping as the last part of the ForceEarlyReturn
 945   // dance so we have some house keeping to do.
 946 
 947   if (is_earlyret_pending()) {
 948     // If the earlyret_state field is not earlyret_inactive, then
 949     // we missed all of the earlyret_field cleanup points:
 950     //
 951     // - remove_activation() was not called
 952     //   (did not get suspended in a call_vm() family call and did
 953     //   not complete a call_vm() family call on the way here)
 954     //
 955     // One legitimate way for us to miss all the cleanup points is
 956     // if we got here right after handling a compiled return. If that
 957     // is the case, then we consider our return from compiled code to
 958     // complete the ForceEarlyReturn request and we clear the condition.
 959     clr_earlyret_pending();
 960     set_earlyret_oop(nullptr);
 961     clr_earlyret_value();
 962   }
 963 
 964   // clearing the flag indicates we are done with
 965   // the ForceEarlyReturn() dance
 966   clr_pending_step_for_earlyret();
 967 
 968   // If exception was thrown in this frame, need to reset jvmti thread state.
 969   // Single stepping may not get enabled correctly by the agent since
 970   // exception state is passed in MethodExit event which may be sent at some
 971   // time in the future. JDWP agent ignores MethodExit events if caused by
 972   // an exception.
 973   //
 974   if (is_exception_detected()) {
 975     clear_exception_state();
 976   }
 977   // If step is pending for earlyret then it may not be a repeat step.
 978   // The new_bci and method_id is same as current_bci and current
 979   // method_id after earlyret and step for recursive calls.
 980   // Force the step by clearing the last location.
 981   JvmtiEnvThreadStateIterator it(this);
 982   for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 983     ets->clear_current_location();
 984   }
 985 }
 986 
 987 void JvmtiThreadState::oops_do(OopClosure* f, CodeBlobClosure* cf) {
 988   f->do_oop((oop*) &_earlyret_oop);
 989 
 990   // Keep nmethods from unloading on the event queue
 991   if (_jvmti_event_queue != nullptr) {
 992     _jvmti_event_queue->oops_do(f, cf);
 993   }
 994 }
 995 
 996 void JvmtiThreadState::nmethods_do(CodeBlobClosure* cf) {
 997   // Keep nmethods from unloading on the event queue
 998   if (_jvmti_event_queue != nullptr) {
 999     _jvmti_event_queue->nmethods_do(cf);
1000   }
1001 }
1002 
1003 // Thread local event queue.
1004 void JvmtiThreadState::enqueue_event(JvmtiDeferredEvent* event) {
1005   if (_jvmti_event_queue == nullptr) {
1006     _jvmti_event_queue = new JvmtiDeferredEventQueue();
1007   }
1008   // copy the event
1009   _jvmti_event_queue->enqueue(*event);
1010 }
1011 
1012 void JvmtiThreadState::post_events(JvmtiEnv* env) {
1013   if (_jvmti_event_queue != nullptr) {
1014     _jvmti_event_queue->post(env);  // deletes each queue node
1015     delete _jvmti_event_queue;
1016     _jvmti_event_queue = nullptr;
1017   }
1018 }
1019 
1020 void JvmtiThreadState::run_nmethod_entry_barriers() {
1021   if (_jvmti_event_queue != nullptr) {
1022     _jvmti_event_queue->run_nmethod_entry_barriers();
1023   }
1024 }
1025 
1026 oop JvmtiThreadState::get_thread_oop() {
1027   return _thread_oop_h.resolve();
1028 }
1029 
1030 void JvmtiThreadState::set_thread(JavaThread* thread) {
1031   _thread_saved = nullptr;  // Common case.
1032   if (!_is_virtual && thread == nullptr) {
1033     // Save JavaThread* if carrier thread is being detached.
1034     _thread_saved = _thread;
1035   }
1036   _thread = thread;
1037 }