1 /*
   2  * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.inline.hpp"
  27 #include "jvmtifiles/jvmtiEnv.hpp"
  28 #include "memory/resourceArea.hpp"
  29 #include "oops/oopHandle.inline.hpp"
  30 #include "prims/jvmtiEnvBase.hpp"
  31 #include "prims/jvmtiEventController.inline.hpp"
  32 #include "prims/jvmtiImpl.hpp"
  33 #include "prims/jvmtiThreadState.inline.hpp"
  34 #include "runtime/handles.inline.hpp"
  35 #include "runtime/interfaceSupport.inline.hpp"
  36 #include "runtime/jniHandles.inline.hpp"
  37 #include "runtime/safepointVerifiers.hpp"
  38 #include "runtime/stackFrameStream.inline.hpp"
  39 #include "runtime/vframe.hpp"
  40 
  41 // marker for when the stack depth has been reset and is now unknown.
  42 // any negative number would work but small ones might obscure an
  43 // underrun error.
  44 static const int UNKNOWN_STACK_DEPTH = -99;
  45 
  46 ///////////////////////////////////////////////////////////////
  47 //
  48 // class JvmtiThreadState
  49 //
  50 // Instances of JvmtiThreadState hang off of each thread.
  51 // Thread local storage for JVMTI.
  52 //
  53 
  54 JvmtiThreadState *JvmtiThreadState::_head = nullptr;
  55 bool JvmtiThreadState::_seen_interp_only_mode = false;
  56 
  57 JvmtiThreadState::JvmtiThreadState(JavaThread* thread, oop thread_oop)
  58   : _thread_event_enable() {
  59   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
  60   _thread               = thread;
  61   _thread_saved         = nullptr;
  62   _exception_state      = ES_CLEARED;
  63   _debuggable           = true;
  64   _hide_single_stepping = false;
  65   _pending_interp_only_mode = false;
  66   _hide_level           = 0;
  67   _pending_step_for_popframe = false;
  68   _class_being_redefined = nullptr;
  69   _class_load_kind = jvmti_class_load_kind_load;
  70   _classes_being_redefined = nullptr;
  71   _head_env_thread_state = nullptr;
  72   _dynamic_code_event_collector = nullptr;
  73   _vm_object_alloc_event_collector = nullptr;
  74   _sampled_object_alloc_event_collector = nullptr;
  75   _the_class_for_redefinition_verification = nullptr;
  76   _scratch_class_for_redefinition_verification = nullptr;
  77   _cur_stack_depth = UNKNOWN_STACK_DEPTH;
  78   _saved_interp_only_mode = 0;
  79 
  80   // JVMTI ForceEarlyReturn support
  81   _pending_step_for_earlyret = false;
  82   _earlyret_state = earlyret_inactive;
  83   _earlyret_tos = ilgl;
  84   _earlyret_value.j = 0L;
  85   _earlyret_oop = nullptr;
  86   _jvmti_event_queue = nullptr;
  87   _is_virtual = false;
  88 
  89   _thread_oop_h = OopHandle(JvmtiExport::jvmti_oop_storage(), thread_oop);
  90 
  91   // add all the JvmtiEnvThreadState to the new JvmtiThreadState
  92   {
  93     JvmtiEnvIterator it;
  94     for (JvmtiEnvBase* env = it.first(); env != nullptr; env = it.next(env)) {
  95       if (env->is_valid()) {
  96         add_env(env);
  97       }
  98     }
  99   }
 100 
 101   // link us into the list
 102   {
 103     // The thread state list manipulation code must not have safepoints.
 104     // See periodic_clean_up().
 105     debug_only(NoSafepointVerifier nosafepoint;)
 106 
 107     _prev = nullptr;
 108     _next = _head;
 109     if (_head != nullptr) {
 110       _head->_prev = this;
 111     }
 112     _head = this;
 113   }
 114 
 115   if (thread_oop != nullptr) {
 116     java_lang_Thread::set_jvmti_thread_state(thread_oop, this);
 117     _is_virtual = java_lang_VirtualThread::is_instance(thread_oop);
 118   }
 119 
 120   if (thread != nullptr) {
 121     if (thread_oop == nullptr || thread->jvmti_vthread() == nullptr || thread->jvmti_vthread() == thread_oop) {
 122       // The JavaThread for carrier or mounted virtual thread case.
 123       // Set this only if thread_oop is current thread->jvmti_vthread().
 124       thread->set_jvmti_thread_state(this);
 125     }
 126     thread->set_interp_only_mode(0);
 127   }
 128 }
 129 
 130 
 131 JvmtiThreadState::~JvmtiThreadState()   {
 132   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
 133 
 134   if (_classes_being_redefined != nullptr) {
 135     delete _classes_being_redefined; // free the GrowableArray on C heap
 136   }
 137 
 138   // clear this as the state for the thread
 139   get_thread()->set_jvmti_thread_state(nullptr);
 140 
 141   // zap our env thread states
 142   {
 143     JvmtiEnvBase::entering_dying_thread_env_iteration();
 144     JvmtiEnvThreadStateIterator it(this);
 145     for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ) {
 146       JvmtiEnvThreadState* zap = ets;
 147       ets = it.next(ets);
 148       delete zap;
 149     }
 150     JvmtiEnvBase::leaving_dying_thread_env_iteration();
 151   }
 152 
 153   // remove us from the list
 154   {
 155     // The thread state list manipulation code must not have safepoints.
 156     // See periodic_clean_up().
 157     debug_only(NoSafepointVerifier nosafepoint;)
 158 
 159     if (_prev == nullptr) {
 160       assert(_head == this, "sanity check");
 161       _head = _next;
 162     } else {
 163       assert(_head != this, "sanity check");
 164       _prev->_next = _next;
 165     }
 166     if (_next != nullptr) {
 167       _next->_prev = _prev;
 168     }
 169     _next = nullptr;
 170     _prev = nullptr;
 171   }
 172   if (get_thread_oop() != nullptr) {
 173     java_lang_Thread::set_jvmti_thread_state(get_thread_oop(), nullptr);
 174   }
 175   _thread_oop_h.release(JvmtiExport::jvmti_oop_storage());
 176 }
 177 
 178 
 179 void
 180 JvmtiThreadState::periodic_clean_up() {
 181   assert(SafepointSynchronize::is_at_safepoint(), "at safepoint");
 182 
 183   // This iteration is initialized with "_head" instead of "JvmtiThreadState::first()"
 184   // because the latter requires the JvmtiThreadState_lock.
 185   // This iteration is safe at a safepoint as well, see the NoSafepointVerifier
 186   // asserts at all list manipulation sites.
 187   for (JvmtiThreadState *state = _head; state != nullptr; state = state->next()) {
 188     // For each environment thread state corresponding to an invalid environment
 189     // unlink it from the list and deallocate it.
 190     JvmtiEnvThreadStateIterator it(state);
 191     JvmtiEnvThreadState* previous_ets = nullptr;
 192     JvmtiEnvThreadState* ets = it.first();
 193     while (ets != nullptr) {
 194       if (ets->get_env()->is_valid()) {
 195         previous_ets = ets;
 196         ets = it.next(ets);
 197       } else {
 198         // This one isn't valid, remove it from the list and deallocate it
 199         JvmtiEnvThreadState* defunct_ets = ets;
 200         ets = ets->next();
 201         if (previous_ets == nullptr) {
 202           assert(state->head_env_thread_state() == defunct_ets, "sanity check");
 203           state->set_head_env_thread_state(ets);
 204         } else {
 205           previous_ets->set_next(ets);
 206         }
 207         delete defunct_ets;
 208       }
 209     }
 210   }
 211 }
 212 
 213 //
 214 // Virtual Threads Mount State transition (VTMS transition) mechanism
 215 //
 216 
 217 // VTMS transitions cannot be disabled while this counter is positive.
 218 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_count = 0;
 219 
 220 // VTMS transitions for one virtual thread are disabled while it is positive
 221 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_disable_for_one_count = 0;
 222 
 223 // VTMS transitions for all virtual threads are disabled while it is positive
 224 volatile int JvmtiVTMSTransitionDisabler::_VTMS_transition_disable_for_all_count = 0;
 225 
 226 // There is an active suspender or resumer.
 227 volatile bool JvmtiVTMSTransitionDisabler::_SR_mode = false;
 228 
 229 // Notifications from VirtualThread about VTMS events are enabled.
 230 bool JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events = false;
 231 
 232 // The JvmtiVTMSTransitionDisabler sync protocol is enabled if this count > 0.
 233 volatile int JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_count = 0;
 234 
 235 // JvmtiVTMSTraansitionDisabler sync protocol is enabled permanently after seeing a suspender.
 236 volatile bool JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_permanently = false;
 237 
 238 #ifdef ASSERT
 239 void
 240 JvmtiVTMSTransitionDisabler::print_info() {
 241   log_error(jvmti)("_VTMS_transition_count: %d\n", _VTMS_transition_count);
 242   log_error(jvmti)("_VTMS_transition_disable_for_one_count: %d\n", _VTMS_transition_disable_for_one_count);
 243   log_error(jvmti)("_VTMS_transition_disable_for_all_count: %d\n\n", _VTMS_transition_disable_for_all_count);
 244   int attempts = 10000;
 245   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *java_thread = jtiwh.next(); ) {
 246     ResourceMark rm;
 247     // Handshake with target.
 248     PrintStackTraceClosure pstc;
 249     Handshake::execute(&pstc, java_thread);
 250   }
 251 }
 252 #endif
 253 
 254 // disable VTMS transitions for one virtual thread
 255 // no-op if thread is non-null and not a virtual thread
 256 JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(jthread thread)
 257   : _is_SR(false), _thread(thread)
 258 {
 259   if (!Continuations::enabled()) {
 260     return; // JvmtiVTMSTransitionDisabler is no-op without virtual threads
 261   }
 262   if (Thread::current_or_null() == nullptr) {
 263     return;  // Detached thread, can be a call from Agent_OnLoad.
 264   }
 265   if (!sync_protocol_enabled_permanently()) {
 266     JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
 267   }
 268   if (_thread != nullptr) {
 269     VTMS_transition_disable_for_one(); // disable VTMS transitions for one virtual thread
 270   } else {
 271     VTMS_transition_disable_for_all(); // disable VTMS transitions for all virtual threads
 272   }
 273 }
 274 
 275 // disable VTMS transitions for all virtual threads
 276 JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(bool is_SR)
 277   : _is_SR(is_SR), _thread(nullptr)
 278 {
 279   if (!Continuations::enabled()) {
 280     return; // JvmtiVTMSTransitionDisabler is no-op without virtual threads
 281   }
 282   if (Thread::current_or_null() == nullptr) {
 283     return;  // Detached thread, can be a call from Agent_OnLoad.
 284   }
 285   if (!sync_protocol_enabled_permanently()) {
 286     JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
 287     if (is_SR) {
 288       Atomic::store(&_sync_protocol_enabled_permanently, true);
 289     }
 290   }
 291   VTMS_transition_disable_for_all();
 292 }
 293 
 294 JvmtiVTMSTransitionDisabler::~JvmtiVTMSTransitionDisabler() {
 295   if (!Continuations::enabled()) {
 296     return; // JvmtiVTMSTransitionDisabler is a no-op without virtual threads
 297   }
 298   if (Thread::current_or_null() == nullptr) {
 299     return;  // Detached thread, can be a call from Agent_OnLoad.
 300   }
 301   if (_thread != nullptr) {
 302     VTMS_transition_enable_for_one(); // enable VTMS transitions for one virtual thread
 303   } else {
 304     VTMS_transition_enable_for_all(); // enable VTMS transitions for all virtual threads
 305   }
 306   if (!sync_protocol_enabled_permanently()) {
 307     JvmtiVTMSTransitionDisabler::dec_sync_protocol_enabled_count();
 308   }
 309 }
 310 
 311 // disable VTMS transitions for one virtual thread
 312 void
 313 JvmtiVTMSTransitionDisabler::VTMS_transition_disable_for_one() {
 314   assert(_thread != nullptr, "sanity check");
 315   JavaThread* thread = JavaThread::current();
 316   HandleMark hm(thread);
 317   Handle vth = Handle(thread, JNIHandles::resolve_external_guard(_thread));
 318   if (!java_lang_VirtualThread::is_instance(vth())) {
 319     return; // no-op if _thread is not a virtual thread
 320   }
 321   MonitorLocker ml(JvmtiVTMSTransition_lock);
 322 
 323   while (_SR_mode) { // suspender or resumer is a JvmtiVTMSTransitionDisabler monopolist
 324     ml.wait(10); // wait while there is an active suspender or resumer
 325   }
 326   Atomic::inc(&_VTMS_transition_disable_for_one_count);
 327   java_lang_Thread::inc_VTMS_transition_disable_count(vth());
 328 
 329   while (java_lang_Thread::is_in_VTMS_transition(vth())) {
 330     ml.wait(10); // wait while the virtual thread is in transition
 331   }
 332 #ifdef ASSERT
 333   thread->set_is_VTMS_transition_disabler(true);
 334 #endif
 335 }
 336 
 337 // disable VTMS transitions for all virtual threads
 338 void
 339 JvmtiVTMSTransitionDisabler::VTMS_transition_disable_for_all() {
 340   JavaThread* thread = JavaThread::current();
 341   int attempts = 50000;
 342   {
 343     MonitorLocker ml(JvmtiVTMSTransition_lock);
 344 
 345     assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 346     assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
 347     while (_SR_mode) { // Suspender or resumer is a JvmtiVTMSTransitionDisabler monopolist.
 348       ml.wait(10);     // Wait while there is an active suspender or resumer.
 349     }
 350     if (_is_SR) {
 351       _SR_mode = true;
 352       while (_VTMS_transition_disable_for_all_count > 0 ||
 353              _VTMS_transition_disable_for_one_count > 0) {
 354         ml.wait(10);   // Wait while there is any active jvmtiVTMSTransitionDisabler.
 355       }
 356     }
 357     Atomic::inc(&_VTMS_transition_disable_for_all_count);
 358 
 359     // Block while some mount/unmount transitions are in progress.
 360     // Debug version fails and prints diagnostic information.
 361     while (_VTMS_transition_count > 0) {
 362       if (ml.wait(10)) {
 363         attempts--;
 364       }
 365       DEBUG_ONLY(if (attempts == 0) break;)
 366     }
 367     assert(!thread->is_VTMS_transition_disabler(), "VTMS_transition sanity check");
 368 #ifdef ASSERT
 369     if (attempts > 0) {
 370       thread->set_is_VTMS_transition_disabler(true);
 371     }
 372 #endif
 373   }
 374 #ifdef ASSERT
 375     if (attempts == 0) {
 376       print_info();
 377       fatal("stuck in JvmtiVTMSTransitionDisabler::VTMS_transition_disable");
 378     }
 379 #endif
 380 }
 381 
 382 // enable VTMS transitions for one virtual thread
 383 void
 384 JvmtiVTMSTransitionDisabler::VTMS_transition_enable_for_one() {
 385   JavaThread* thread = JavaThread::current();
 386   HandleMark hm(thread);
 387   Handle vth = Handle(thread, JNIHandles::resolve_external_guard(_thread));
 388   if (!java_lang_VirtualThread::is_instance(vth())) {
 389     return; // no-op if _thread is not a virtual thread
 390   }
 391   MonitorLocker ml(JvmtiVTMSTransition_lock);
 392   java_lang_Thread::dec_VTMS_transition_disable_count(vth());
 393   Atomic::dec(&_VTMS_transition_disable_for_one_count);
 394   if (_VTMS_transition_disable_for_one_count == 0) {
 395     ml.notify_all();
 396   }
 397 #ifdef ASSERT
 398   thread->set_is_VTMS_transition_disabler(false);
 399 #endif
 400 }
 401 
 402 // enable VTMS transitions for all virtual threads
 403 void
 404 JvmtiVTMSTransitionDisabler::VTMS_transition_enable_for_all() {
 405   JavaThread* current = JavaThread::current();
 406   {
 407     MonitorLocker ml(JvmtiVTMSTransition_lock);
 408     assert(_VTMS_transition_disable_for_all_count > 0, "VTMS_transition sanity check");
 409 
 410     if (_is_SR) {  // Disabler is suspender or resumer.
 411       _SR_mode = false;
 412     }
 413     Atomic::dec(&_VTMS_transition_disable_for_all_count);
 414     if (_VTMS_transition_disable_for_all_count == 0 || _is_SR) {
 415       ml.notify_all();
 416     }
 417 #ifdef ASSERT
 418     current->set_is_VTMS_transition_disabler(false);
 419 #endif
 420   }
 421 }
 422 
 423 void
 424 JvmtiVTMSTransitionDisabler::start_VTMS_transition(jthread vthread, bool is_mount) {
 425   JavaThread* thread = JavaThread::current();
 426   oop vt = JNIHandles::resolve_external_guard(vthread);
 427   assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
 428 
 429   // Avoid using MonitorLocker on performance critical path, use
 430   // two-level synchronization with lock-free operations on counters.
 431   Atomic::inc(&_VTMS_transition_count); // Try to enter VTMS transition section optmistically.
 432   java_lang_Thread::set_is_in_VTMS_transition(vt, true);
 433 
 434   if (!sync_protocol_enabled()) {
 435     thread->set_is_in_VTMS_transition(true);
 436     return;
 437   }
 438   HandleMark hm(thread);
 439   Handle vth = Handle(thread, vt);
 440   int attempts = 50000;
 441 
 442   // Do not allow suspends inside VTMS transitions.
 443   // Block while transitions are disabled or there are suspend requests.
 444   int64_t thread_id = java_lang_Thread::thread_id(vth());  // Cannot use oops while blocked.
 445 
 446   if (_VTMS_transition_disable_for_all_count > 0 ||
 447       java_lang_Thread::VTMS_transition_disable_count(vth()) > 0 ||
 448       thread->is_suspended() ||
 449       JvmtiVTSuspender::is_vthread_suspended(thread_id)
 450   ) {
 451     // Slow path: undo unsuccessful optimistic counter incrementation.
 452     // It can cause an extra waiting cycle for VTMS transition disablers.
 453     Atomic::dec(&_VTMS_transition_count);
 454     java_lang_Thread::set_is_in_VTMS_transition(vth(), false);
 455 
 456     while (true) {
 457       MonitorLocker ml(JvmtiVTMSTransition_lock);
 458 
 459       // Do not allow suspends inside VTMS transitions.
 460       // Block while transitions are disabled or there are suspend requests.
 461       if (_VTMS_transition_disable_for_all_count > 0 ||
 462           java_lang_Thread::VTMS_transition_disable_count(vth()) > 0 ||
 463           thread->is_suspended() ||
 464           JvmtiVTSuspender::is_vthread_suspended(thread_id)
 465       ) {
 466         // Block while transitions are disabled or there are suspend requests.
 467         if (ml.wait(10)) {
 468           attempts--;
 469         }
 470         DEBUG_ONLY(if (attempts == 0) break;)
 471         continue;  // ~ThreadBlockInVM has handshake-based suspend point.
 472       }
 473       Atomic::inc(&_VTMS_transition_count);
 474       java_lang_Thread::set_is_in_VTMS_transition(vth(), true);
 475       break;
 476     }
 477   }
 478 #ifdef ASSERT
 479   if (attempts == 0) {
 480     log_error(jvmti)("start_VTMS_transition: thread->is_suspended: %d is_vthread_suspended: %d\n\n",
 481                      thread->is_suspended(), JvmtiVTSuspender::is_vthread_suspended(thread_id));
 482     print_info();
 483     fatal("stuck in JvmtiVTMSTransitionDisabler::start_VTMS_transition");
 484   }
 485 #endif
 486   // Enter VTMS transition section.
 487   thread->set_is_in_VTMS_transition(true);
 488 }
 489 
 490 void
 491 JvmtiVTMSTransitionDisabler::finish_VTMS_transition(jthread vthread, bool is_mount) {
 492   JavaThread* thread = JavaThread::current();
 493 
 494   assert(thread->is_in_VTMS_transition(), "sanity check");
 495   thread->set_is_in_VTMS_transition(false);
 496   oop vt = JNIHandles::resolve_external_guard(vthread);
 497   java_lang_Thread::set_is_in_VTMS_transition(vt, false);
 498   Atomic::dec(&_VTMS_transition_count);
 499 
 500   if (!sync_protocol_enabled()) {
 501     return;
 502   }
 503   int64_t thread_id = java_lang_Thread::thread_id(vt);
 504 
 505   // Unblock waiting VTMS transition disablers.
 506   if (_VTMS_transition_disable_for_one_count > 0 ||
 507       _VTMS_transition_disable_for_all_count > 0) {
 508     MonitorLocker ml(JvmtiVTMSTransition_lock);
 509     ml.notify_all();
 510   }
 511   // In unmount case the carrier thread is attached after unmount transition.
 512   // Check and block it if there was external suspend request.
 513   int attempts = 10000;
 514   if (!is_mount && thread->is_carrier_thread_suspended()) {
 515     while (true) {
 516       MonitorLocker ml(JvmtiVTMSTransition_lock);
 517 
 518       // Block while there are suspend requests.
 519       if ((!is_mount && thread->is_carrier_thread_suspended()) ||
 520           (is_mount && JvmtiVTSuspender::is_vthread_suspended(thread_id))
 521       ) {
 522         // Block while there are suspend requests.
 523         if (ml.wait(10)) {
 524           attempts--;
 525         }
 526         DEBUG_ONLY(if (attempts == 0) break;)
 527         continue;
 528       }
 529       break;
 530     }
 531   }
 532 #ifdef ASSERT
 533   if (attempts == 0) {
 534     log_error(jvmti)("finish_VTMS_transition: thread->is_suspended: %d is_vthread_suspended: %d\n\n",
 535                      thread->is_suspended(), JvmtiVTSuspender::is_vthread_suspended(thread_id));
 536     print_info();
 537     fatal("stuck in JvmtiVTMSTransitionDisabler::finish_VTMS_transition");
 538   }
 539 #endif
 540 }
 541 
 542 // set VTMS transition bit value in JavaThread and java.lang.VirtualThread object
 543 void JvmtiVTMSTransitionDisabler::set_is_in_VTMS_transition(JavaThread* thread, jobject vthread, bool in_trans) {
 544   oop vt = JNIHandles::resolve_external_guard(vthread);
 545   java_lang_Thread::set_is_in_VTMS_transition(vt, in_trans);
 546   thread->set_is_in_VTMS_transition(in_trans);
 547 }
 548 
 549 void
 550 JvmtiVTMSTransitionDisabler::VTMS_vthread_start(jobject vthread) {
 551   VTMS_mount_end(vthread);
 552   JavaThread* thread = JavaThread::current();
 553 
 554   assert(!thread->is_in_VTMS_transition(), "sanity check");
 555   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 556 
 557   // If interp_only_mode has been enabled then we must eagerly create JvmtiThreadState
 558   // objects for globally enabled virtual thread filtered events. Otherwise,
 559   // it is an important optimization to create JvmtiThreadState objects lazily.
 560   // This optimization is disabled when watchpoint capabilities are present. It is to
 561   // work around a bug with virtual thread frames which can be not deoptimized in time.
 562   if (JvmtiThreadState::seen_interp_only_mode() ||
 563       JvmtiExport::should_post_field_access() ||
 564       JvmtiExport::should_post_field_modification()){
 565     JvmtiEventController::thread_started(thread);
 566   }
 567   if (JvmtiExport::should_post_vthread_start()) {
 568     JvmtiExport::post_vthread_start(vthread);
 569   }
 570   // post VirtualThreadMount event after VirtualThreadStart
 571   if (JvmtiExport::should_post_vthread_mount()) {
 572     JvmtiExport::post_vthread_mount(vthread);
 573   }
 574 }
 575 
 576 void
 577 JvmtiVTMSTransitionDisabler::VTMS_vthread_end(jobject vthread) {
 578   JavaThread* thread = JavaThread::current();
 579 
 580   assert(!thread->is_in_VTMS_transition(), "sanity check");
 581   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 582 
 583   // post VirtualThreadUnmount event before VirtualThreadEnd
 584   if (JvmtiExport::should_post_vthread_unmount()) {
 585     JvmtiExport::post_vthread_unmount(vthread);
 586   }
 587   if (JvmtiExport::should_post_vthread_end()) {
 588     JvmtiExport::post_vthread_end(vthread);
 589   }
 590   VTMS_unmount_begin(vthread, /* last_unmount */ true);
 591   if (thread->jvmti_thread_state() != nullptr) {
 592     JvmtiExport::cleanup_thread(thread);
 593     assert(thread->jvmti_thread_state() == nullptr, "should be null");
 594     assert(java_lang_Thread::jvmti_thread_state(JNIHandles::resolve(vthread)) == nullptr, "should be null");
 595   }
 596   thread->rebind_to_jvmti_thread_state_of(thread->threadObj());
 597 }
 598 
 599 void
 600 JvmtiVTMSTransitionDisabler::VTMS_vthread_mount(jobject vthread, bool hide) {
 601   if (hide) {
 602     VTMS_mount_begin(vthread);
 603   } else {
 604     VTMS_mount_end(vthread);
 605     if (JvmtiExport::should_post_vthread_mount()) {
 606       JvmtiExport::post_vthread_mount(vthread);
 607     }
 608   }
 609 }
 610 
 611 void
 612 JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount(jobject vthread, bool hide) {
 613   if (hide) {
 614     if (JvmtiExport::should_post_vthread_unmount()) {
 615       JvmtiExport::post_vthread_unmount(vthread);
 616     }
 617     VTMS_unmount_begin(vthread, /* last_unmount */ false);
 618   } else {
 619     VTMS_unmount_end(vthread);
 620   }
 621 }
 622 
 623 void
 624 JvmtiVTMSTransitionDisabler::VTMS_mount_begin(jobject vthread) {
 625   JavaThread* thread = JavaThread::current();
 626   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 627   assert(!thread->is_in_VTMS_transition(), "sanity check");
 628   start_VTMS_transition(vthread, /* is_mount */ true);
 629 }
 630 
 631 void
 632 JvmtiVTMSTransitionDisabler::VTMS_mount_end(jobject vthread) {
 633   JavaThread* thread = JavaThread::current();
 634   oop vt = JNIHandles::resolve(vthread);
 635 
 636   thread->rebind_to_jvmti_thread_state_of(vt);
 637 
 638   JvmtiThreadState* state = thread->jvmti_thread_state();
 639   if (state != nullptr && state->is_pending_interp_only_mode()) {
 640     MutexLocker mu(JvmtiThreadState_lock);
 641     state = thread->jvmti_thread_state();
 642     if (state != nullptr && state->is_pending_interp_only_mode()) {
 643       JvmtiEventController::enter_interp_only_mode();
 644     }
 645   }
 646   assert(thread->is_in_VTMS_transition(), "sanity check");
 647   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 648   finish_VTMS_transition(vthread, /* is_mount */ true);
 649 }
 650 
 651 void
 652 JvmtiVTMSTransitionDisabler::VTMS_unmount_begin(jobject vthread, bool last_unmount) {
 653   JavaThread* thread = JavaThread::current();
 654 
 655   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 656   assert(!thread->is_in_VTMS_transition(), "sanity check");
 657 
 658   start_VTMS_transition(vthread, /* is_mount */ false);
 659   if (!last_unmount) {
 660     thread->rebind_to_jvmti_thread_state_of(thread->threadObj());
 661   }
 662 }
 663 
 664 void
 665 JvmtiVTMSTransitionDisabler::VTMS_unmount_end(jobject vthread) {
 666   JavaThread* thread = JavaThread::current();
 667   assert(thread->is_in_VTMS_transition(), "sanity check");
 668   assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
 669   finish_VTMS_transition(vthread, /* is_mount */ false);
 670 
 671   if (JvmtiExport::should_post_vthread_unmount() && thread->jvmti_unmount_event_pending()) {
 672     assert(java_lang_VirtualThread::is_preempted(JNIHandles::resolve(vthread)), "should be marked preempted");
 673     JvmtiExport::post_vthread_unmount(vthread);
 674     thread->set_jvmti_unmount_event_pending(false);
 675   }
 676 }
 677 
 678 
 679 //
 680 // Virtual Threads Suspend/Resume management
 681 //
 682 
 683 JvmtiVTSuspender::SR_Mode
 684 JvmtiVTSuspender::_SR_mode = SR_none;
 685 
 686 VirtualThreadList*
 687 JvmtiVTSuspender::_suspended_list = new VirtualThreadList();
 688 
 689 VirtualThreadList*
 690 JvmtiVTSuspender::_not_suspended_list = new VirtualThreadList();
 691 
 692 void
 693 JvmtiVTSuspender::register_all_vthreads_suspend() {
 694   MonitorLocker ml(JvmtiVTMSTransition_lock);
 695 
 696   _SR_mode = SR_all;
 697   _suspended_list->invalidate();
 698   _not_suspended_list->invalidate();
 699 }
 700 
 701 void
 702 JvmtiVTSuspender::register_all_vthreads_resume() {
 703   MonitorLocker ml(JvmtiVTMSTransition_lock);
 704 
 705   _SR_mode = SR_none;
 706   _suspended_list->invalidate();
 707   _not_suspended_list->invalidate();
 708 }
 709 
 710 void
 711 JvmtiVTSuspender::register_vthread_suspend(oop vt) {
 712   int64_t id = java_lang_Thread::thread_id(vt);
 713   MonitorLocker ml(JvmtiVTMSTransition_lock);
 714 
 715   if (_SR_mode == SR_all) {
 716     assert(_not_suspended_list->contains(id),
 717            "register_vthread_suspend sanity check");
 718     _not_suspended_list->remove(id);
 719   } else {
 720     assert(!_suspended_list->contains(id),
 721            "register_vthread_suspend sanity check");
 722     _SR_mode = SR_ind;
 723     _suspended_list->append(id);
 724   }
 725 }
 726 
 727 void
 728 JvmtiVTSuspender::register_vthread_resume(oop vt) {
 729   int64_t id = java_lang_Thread::thread_id(vt);
 730   MonitorLocker ml(JvmtiVTMSTransition_lock);
 731 
 732   if (_SR_mode == SR_all) {
 733     assert(!_not_suspended_list->contains(id),
 734            "register_vthread_resume sanity check");
 735     _not_suspended_list->append(id);
 736   } else if (_SR_mode == SR_ind) {
 737     assert(_suspended_list->contains(id),
 738            "register_vthread_resume check");
 739     _suspended_list->remove(id);
 740     if (_suspended_list->length() == 0) {
 741       _SR_mode = SR_none;
 742     }
 743   } else {
 744     assert(false, "register_vthread_resume: no suspend mode enabled");
 745   }
 746 }
 747 
 748 bool
 749 JvmtiVTSuspender::is_vthread_suspended(int64_t thread_id) {
 750   bool suspend_is_needed =
 751    (_SR_mode == SR_all && !_not_suspended_list->contains(thread_id)) ||
 752    (_SR_mode == SR_ind && _suspended_list->contains(thread_id));
 753 
 754   return suspend_is_needed;
 755 }
 756 
 757 bool
 758 JvmtiVTSuspender::is_vthread_suspended(oop vt) {
 759   return is_vthread_suspended(java_lang_Thread::thread_id(vt));
 760 }
 761 
 762 void JvmtiThreadState::add_env(JvmtiEnvBase *env) {
 763   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
 764 
 765   JvmtiEnvThreadState *new_ets = new JvmtiEnvThreadState(this, env);
 766   // add this environment thread state to the end of the list (order is important)
 767   {
 768     // list deallocation (which occurs at a safepoint) cannot occur simultaneously
 769     debug_only(NoSafepointVerifier nosafepoint;)
 770 
 771     JvmtiEnvThreadStateIterator it(this);
 772     JvmtiEnvThreadState* previous_ets = nullptr;
 773     for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 774       previous_ets = ets;
 775     }
 776     if (previous_ets == nullptr) {
 777       set_head_env_thread_state(new_ets);
 778     } else {
 779       previous_ets->set_next(new_ets);
 780     }
 781   }
 782 }
 783 
 784 void JvmtiThreadState::enter_interp_only_mode() {
 785   assert(_thread != nullptr, "sanity check");
 786   _seen_interp_only_mode = true;
 787   _thread->increment_interp_only_mode();
 788   invalidate_cur_stack_depth();
 789 }
 790 
 791 void JvmtiThreadState::leave_interp_only_mode() {
 792   assert(is_interp_only_mode(), "leaving interp only when not in interp only mode");
 793   if (_thread == nullptr) {
 794     // Unmounted virtual thread updates the saved value.
 795     --_saved_interp_only_mode;
 796   } else {
 797     _thread->decrement_interp_only_mode();
 798   }
 799 }
 800 
 801 
 802 // Helper routine used in several places
 803 int JvmtiThreadState::count_frames() {
 804   JavaThread* thread = get_thread_or_saved();
 805   javaVFrame *jvf;
 806   ResourceMark rm;
 807   if (thread == nullptr) {
 808     oop thread_obj = get_thread_oop();
 809     jvf = JvmtiEnvBase::get_vthread_jvf(thread_obj);
 810   } else {
 811 #ifdef ASSERT
 812     Thread *current_thread = Thread::current();
 813 #endif
 814     assert(SafepointSynchronize::is_at_safepoint() ||
 815            thread->is_handshake_safe_for(current_thread),
 816            "call by myself / at safepoint / at handshake");
 817     if (!thread->has_last_Java_frame()) return 0;  // No Java frames.
 818     // TBD: This might need to be corrected for detached carrier threads.
 819     RegisterMap reg_map(thread,
 820                         RegisterMap::UpdateMap::skip,
 821                         RegisterMap::ProcessFrames::skip,
 822                         RegisterMap::WalkContinuation::include);
 823     jvf = thread->last_java_vframe(&reg_map);
 824     jvf = JvmtiEnvBase::check_and_skip_hidden_frames(thread, jvf);
 825   }
 826   return (int)JvmtiEnvBase::get_frame_count(jvf);
 827 }
 828 
 829 
 830 void JvmtiThreadState::invalidate_cur_stack_depth() {
 831   assert(SafepointSynchronize::is_at_safepoint() ||
 832          get_thread()->is_handshake_safe_for(Thread::current()),
 833          "bad synchronization with owner thread");
 834 
 835   _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 836 }
 837 
 838 void JvmtiThreadState::incr_cur_stack_depth() {
 839   guarantee(JavaThread::current() == get_thread(), "must be current thread");
 840 
 841   if (!is_interp_only_mode()) {
 842     _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 843   }
 844   if (_cur_stack_depth != UNKNOWN_STACK_DEPTH) {
 845     ++_cur_stack_depth;
 846   }
 847 }
 848 
 849 void JvmtiThreadState::decr_cur_stack_depth() {
 850   guarantee(JavaThread::current() == get_thread(), "must be current thread");
 851 
 852   if (!is_interp_only_mode()) {
 853     _cur_stack_depth = UNKNOWN_STACK_DEPTH;
 854   }
 855   if (_cur_stack_depth != UNKNOWN_STACK_DEPTH) {
 856     --_cur_stack_depth;
 857     assert(_cur_stack_depth >= 0, "incr/decr_cur_stack_depth mismatch");
 858   }
 859 }
 860 
 861 int JvmtiThreadState::cur_stack_depth() {
 862   Thread *current = Thread::current();
 863   guarantee(get_thread()->is_handshake_safe_for(current),
 864             "must be current thread or direct handshake");
 865 
 866   if (!is_interp_only_mode() || _cur_stack_depth == UNKNOWN_STACK_DEPTH) {
 867     _cur_stack_depth = count_frames();
 868   } else {
 869 #ifdef ASSERT
 870     if (EnableJVMTIStackDepthAsserts) {
 871       // heavy weight assert
 872       jint num_frames = count_frames();
 873       assert(_cur_stack_depth == num_frames, "cur_stack_depth out of sync _cur_stack_depth: %d num_frames: %d", _cur_stack_depth, num_frames);
 874     }
 875 #endif
 876   }
 877   return _cur_stack_depth;
 878 }
 879 
 880 void JvmtiThreadState::process_pending_step_for_popframe() {
 881   // We are single stepping as the last part of the PopFrame() dance
 882   // so we have some house keeping to do.
 883 
 884   JavaThread *thr = get_thread();
 885   if (thr->popframe_condition() != JavaThread::popframe_inactive) {
 886     // If the popframe_condition field is not popframe_inactive, then
 887     // we missed all of the popframe_field cleanup points:
 888     //
 889     // - unpack_frames() was not called (nothing to deopt)
 890     // - remove_activation_preserving_args_entry() was not called
 891     //   (did not get suspended in a call_vm() family call and did
 892     //   not complete a call_vm() family call on the way here)
 893     thr->clear_popframe_condition();
 894   }
 895 
 896   // clearing the flag indicates we are done with the PopFrame() dance
 897   clr_pending_step_for_popframe();
 898 
 899   // If exception was thrown in this frame, need to reset jvmti thread state.
 900   // Single stepping may not get enabled correctly by the agent since
 901   // exception state is passed in MethodExit event which may be sent at some
 902   // time in the future. JDWP agent ignores MethodExit events if caused by
 903   // an exception.
 904   //
 905   if (is_exception_detected()) {
 906     clear_exception_state();
 907   }
 908   // If step is pending for popframe then it may not be
 909   // a repeat step. The new_bci and method_id is same as current_bci
 910   // and current method_id after pop and step for recursive calls.
 911   // Force the step by clearing the last location.
 912   JvmtiEnvThreadStateIterator it(this);
 913   for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 914     ets->clear_current_location();
 915   }
 916 }
 917 
 918 
 919 // Class:     JvmtiThreadState
 920 // Function:  update_for_pop_top_frame
 921 // Description:
 922 //   This function removes any frame pop notification request for
 923 //   the top frame and invalidates both the current stack depth and
 924 //   all cached frameIDs.
 925 //
 926 // Called by: PopFrame
 927 //
 928 void JvmtiThreadState::update_for_pop_top_frame() {
 929   if (is_interp_only_mode()) {
 930     // remove any frame pop notification request for the top frame
 931     // in any environment
 932     int popframe_number = cur_stack_depth();
 933     {
 934       JvmtiEnvThreadStateIterator it(this);
 935       for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 936         if (ets->is_frame_pop(popframe_number)) {
 937           ets->clear_frame_pop(popframe_number);
 938         }
 939       }
 940     }
 941     // force stack depth to be recalculated
 942     invalidate_cur_stack_depth();
 943   } else {
 944     assert(!is_enabled(JVMTI_EVENT_FRAME_POP), "Must have no framepops set");
 945   }
 946 }
 947 
 948 
 949 void JvmtiThreadState::process_pending_step_for_earlyret() {
 950   // We are single stepping as the last part of the ForceEarlyReturn
 951   // dance so we have some house keeping to do.
 952 
 953   if (is_earlyret_pending()) {
 954     // If the earlyret_state field is not earlyret_inactive, then
 955     // we missed all of the earlyret_field cleanup points:
 956     //
 957     // - remove_activation() was not called
 958     //   (did not get suspended in a call_vm() family call and did
 959     //   not complete a call_vm() family call on the way here)
 960     //
 961     // One legitimate way for us to miss all the cleanup points is
 962     // if we got here right after handling a compiled return. If that
 963     // is the case, then we consider our return from compiled code to
 964     // complete the ForceEarlyReturn request and we clear the condition.
 965     clr_earlyret_pending();
 966     set_earlyret_oop(nullptr);
 967     clr_earlyret_value();
 968   }
 969 
 970   // clearing the flag indicates we are done with
 971   // the ForceEarlyReturn() dance
 972   clr_pending_step_for_earlyret();
 973 
 974   // If exception was thrown in this frame, need to reset jvmti thread state.
 975   // Single stepping may not get enabled correctly by the agent since
 976   // exception state is passed in MethodExit event which may be sent at some
 977   // time in the future. JDWP agent ignores MethodExit events if caused by
 978   // an exception.
 979   //
 980   if (is_exception_detected()) {
 981     clear_exception_state();
 982   }
 983   // If step is pending for earlyret then it may not be a repeat step.
 984   // The new_bci and method_id is same as current_bci and current
 985   // method_id after earlyret and step for recursive calls.
 986   // Force the step by clearing the last location.
 987   JvmtiEnvThreadStateIterator it(this);
 988   for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
 989     ets->clear_current_location();
 990   }
 991 }
 992 
 993 void JvmtiThreadState::oops_do(OopClosure* f, CodeBlobClosure* cf) {
 994   f->do_oop((oop*) &_earlyret_oop);
 995 
 996   // Keep nmethods from unloading on the event queue
 997   if (_jvmti_event_queue != nullptr) {
 998     _jvmti_event_queue->oops_do(f, cf);
 999   }
1000 }
1001 
1002 void JvmtiThreadState::nmethods_do(CodeBlobClosure* cf) {
1003   // Keep nmethods from unloading on the event queue
1004   if (_jvmti_event_queue != nullptr) {
1005     _jvmti_event_queue->nmethods_do(cf);
1006   }
1007 }
1008 
1009 // Thread local event queue.
1010 void JvmtiThreadState::enqueue_event(JvmtiDeferredEvent* event) {
1011   if (_jvmti_event_queue == nullptr) {
1012     _jvmti_event_queue = new JvmtiDeferredEventQueue();
1013   }
1014   // copy the event
1015   _jvmti_event_queue->enqueue(*event);
1016 }
1017 
1018 void JvmtiThreadState::post_events(JvmtiEnv* env) {
1019   if (_jvmti_event_queue != nullptr) {
1020     _jvmti_event_queue->post(env);  // deletes each queue node
1021     delete _jvmti_event_queue;
1022     _jvmti_event_queue = nullptr;
1023   }
1024 }
1025 
1026 void JvmtiThreadState::run_nmethod_entry_barriers() {
1027   if (_jvmti_event_queue != nullptr) {
1028     _jvmti_event_queue->run_nmethod_entry_barriers();
1029   }
1030 }
1031 
1032 oop JvmtiThreadState::get_thread_oop() {
1033   return _thread_oop_h.resolve();
1034 }
1035 
1036 void JvmtiThreadState::set_thread(JavaThread* thread) {
1037   _thread_saved = nullptr;  // Common case.
1038   if (!_is_virtual && thread == nullptr) {
1039     // Save JavaThread* if carrier thread is being detached.
1040     _thread_saved = _thread;
1041   }
1042   _thread = thread;
1043 }