1 /* 2 * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP 26 #define SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP 27 28 #include "oops/stackChunkOop.hpp" 29 30 #include "gc/shared/collectedHeap.hpp" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetStackChunk.hpp" 33 #include "gc/shared/gc_globals.hpp" 34 #include "memory/memRegion.hpp" 35 #include "memory/universe.hpp" 36 #include "oops/access.inline.hpp" 37 #include "oops/instanceStackChunkKlass.inline.hpp" 38 #include "runtime/continuationJavaClasses.inline.hpp" 39 #include "runtime/frame.inline.hpp" 40 #include "runtime/globals.hpp" 41 #include "runtime/handles.inline.hpp" 42 #include "runtime/registerMap.hpp" 43 #include "runtime/smallRegisterMap.inline.hpp" 44 #include "utilities/macros.hpp" 45 #include CPU_HEADER_INLINE(stackChunkOop) 46 47 DEF_HANDLE_CONSTR(stackChunk, is_stackChunk_noinline) 48 49 inline stackChunkOop stackChunkOopDesc::cast(oop obj) { 50 assert(obj == nullptr || obj->is_stackChunk(), "Wrong type"); 51 return stackChunkOop(obj); 52 } 53 54 inline stackChunkOop stackChunkOopDesc::parent() const { return stackChunkOopDesc::cast(jdk_internal_vm_StackChunk::parent(as_oop())); } 55 inline void stackChunkOopDesc::set_parent(stackChunkOop value) { jdk_internal_vm_StackChunk::set_parent(this, value); } 56 template<typename P> 57 inline void stackChunkOopDesc::set_parent_raw(oop value) { jdk_internal_vm_StackChunk::set_parent_raw<P>(this, value); } 58 template<DecoratorSet decorators> 59 inline void stackChunkOopDesc::set_parent_access(oop value) { jdk_internal_vm_StackChunk::set_parent_access<decorators>(this, value); } 60 61 inline int stackChunkOopDesc::stack_size() const { return jdk_internal_vm_StackChunk::size(as_oop()); } 62 63 inline int stackChunkOopDesc::sp() const { return jdk_internal_vm_StackChunk::sp(as_oop()); } 64 inline void stackChunkOopDesc::set_sp(int value) { jdk_internal_vm_StackChunk::set_sp(this, value); } 65 66 inline address stackChunkOopDesc::pc() const { return jdk_internal_vm_StackChunk::pc(as_oop()); } 67 inline void stackChunkOopDesc::set_pc(address value) { jdk_internal_vm_StackChunk::set_pc(this, value); } 68 69 inline int stackChunkOopDesc::argsize() const { return jdk_internal_vm_StackChunk::argsize(as_oop()); } 70 inline void stackChunkOopDesc::set_argsize(int value) { jdk_internal_vm_StackChunk::set_argsize(as_oop(), value); } 71 72 inline uint8_t stackChunkOopDesc::flags() const { return jdk_internal_vm_StackChunk::flags(as_oop()); } 73 inline void stackChunkOopDesc::set_flags(uint8_t value) { jdk_internal_vm_StackChunk::set_flags(this, value); } 74 75 inline uint8_t stackChunkOopDesc::flags_acquire() const { return jdk_internal_vm_StackChunk::flags_acquire(as_oop()); } 76 77 inline void stackChunkOopDesc::release_set_flags(uint8_t value) { 78 jdk_internal_vm_StackChunk::release_set_flags(this, value); 79 } 80 81 inline bool stackChunkOopDesc::try_set_flags(uint8_t prev_flags, uint8_t new_flags) { 82 return jdk_internal_vm_StackChunk::try_set_flags(this, prev_flags, new_flags); 83 } 84 85 inline int stackChunkOopDesc::max_thawing_size() const { return jdk_internal_vm_StackChunk::maxThawingSize(as_oop()); } 86 inline void stackChunkOopDesc::set_max_thawing_size(int value) { 87 assert(value >= 0, "size must be >= 0"); 88 jdk_internal_vm_StackChunk::set_maxThawingSize(this, (jint)value); 89 } 90 91 inline oop stackChunkOopDesc::cont() const { 92 if (UseZGC && !ZGenerational) { 93 assert(!UseCompressedOops, "Non-generational ZGC does not support compressed oops"); 94 // The state of the cont oop is used by XCollectedHeap::requires_barriers, 95 // to determine the age of the stackChunkOopDesc. For that to work, it is 96 // only the GC that is allowed to perform a load barrier on the oop. 97 // This function is used by non-GC code and therfore create a stack-local 98 // copy on the oop and perform the load barrier on that copy instead. 99 oop obj = jdk_internal_vm_StackChunk::cont_raw<oop>(as_oop()); 100 obj = (oop)NativeAccess<>::oop_load(&obj); 101 return obj; 102 } 103 return jdk_internal_vm_StackChunk::cont(as_oop()); 104 } 105 inline void stackChunkOopDesc::set_cont(oop value) { jdk_internal_vm_StackChunk::set_cont(this, value); } 106 template<typename P> 107 inline void stackChunkOopDesc::set_cont_raw(oop value) { jdk_internal_vm_StackChunk::set_cont_raw<P>(this, value); } 108 template<DecoratorSet decorators> 109 inline void stackChunkOopDesc::set_cont_access(oop value) { jdk_internal_vm_StackChunk::set_cont_access<decorators>(this, value); } 110 111 inline int stackChunkOopDesc::bottom() const { return stack_size() - argsize() - frame::metadata_words_at_top; } 112 113 inline HeapWord* stackChunkOopDesc::start_of_stack() const { 114 return (HeapWord*)(cast_from_oop<intptr_t>(as_oop()) + InstanceStackChunkKlass::offset_of_stack()); 115 } 116 117 inline intptr_t* stackChunkOopDesc::start_address() const { return (intptr_t*)start_of_stack(); } 118 inline intptr_t* stackChunkOopDesc::end_address() const { return start_address() + stack_size(); } 119 inline intptr_t* stackChunkOopDesc::bottom_address() const { return start_address() + bottom(); } 120 inline intptr_t* stackChunkOopDesc::sp_address() const { return start_address() + sp(); } 121 122 inline int stackChunkOopDesc::to_offset(intptr_t* p) const { 123 assert(is_in_chunk(p) 124 || (p >= start_address() && (p - start_address()) <= stack_size() + frame::metadata_words), 125 "p: " PTR_FORMAT " start: " PTR_FORMAT " end: " PTR_FORMAT, p2i(p), p2i(start_address()), p2i(bottom_address())); 126 return (int)(p - start_address()); 127 } 128 129 inline intptr_t* stackChunkOopDesc::from_offset(int offset) const { 130 assert(offset <= stack_size(), ""); 131 return start_address() + offset; 132 } 133 134 inline bool stackChunkOopDesc::is_empty() const { 135 assert(sp() <= stack_size(), ""); 136 assert((sp() == stack_size()) == (sp() >= stack_size() - argsize() - frame::metadata_words_at_top), 137 "sp: %d size: %d argsize: %d", sp(), stack_size(), argsize()); 138 return sp() == stack_size(); 139 } 140 141 inline bool stackChunkOopDesc::is_in_chunk(void* p) const { 142 HeapWord* start = (HeapWord*)start_address(); 143 HeapWord* end = start + stack_size(); 144 return (HeapWord*)p >= start && (HeapWord*)p < end; 145 } 146 147 bool stackChunkOopDesc::is_usable_in_chunk(void* p) const { 148 HeapWord* start = (HeapWord*)start_address() + sp() - frame::metadata_words_at_bottom; 149 HeapWord* end = start + stack_size(); 150 return (HeapWord*)p >= start && (HeapWord*)p < end; 151 } 152 153 inline bool stackChunkOopDesc::is_flag(uint8_t flag) const { 154 return (flags() & flag) != 0; 155 } 156 inline bool stackChunkOopDesc::is_flag_acquire(uint8_t flag) const { 157 return (flags_acquire() & flag) != 0; 158 } 159 inline void stackChunkOopDesc::set_flag(uint8_t flag, bool value) { 160 uint32_t flags = this->flags(); 161 set_flags((uint8_t)(value ? flags |= flag : flags &= ~flag)); 162 } 163 inline void stackChunkOopDesc::clear_flags() { 164 set_flags(0); 165 } 166 167 inline bool stackChunkOopDesc::has_mixed_frames() const { return is_flag(FLAG_HAS_INTERPRETED_FRAMES); } 168 inline void stackChunkOopDesc::set_has_mixed_frames(bool value) { 169 assert((flags() & ~FLAG_HAS_INTERPRETED_FRAMES) == 0, "other flags should not be set"); 170 set_flag(FLAG_HAS_INTERPRETED_FRAMES, value); 171 } 172 173 inline bool stackChunkOopDesc::is_gc_mode() const { return is_flag(FLAG_GC_MODE); } 174 inline bool stackChunkOopDesc::is_gc_mode_acquire() const { return is_flag_acquire(FLAG_GC_MODE); } 175 inline void stackChunkOopDesc::set_gc_mode(bool value) { set_flag(FLAG_GC_MODE, value); } 176 177 inline bool stackChunkOopDesc::has_bitmap() const { return is_flag(FLAG_HAS_BITMAP); } 178 inline void stackChunkOopDesc::set_has_bitmap(bool value) { set_flag(FLAG_HAS_BITMAP, value); } 179 180 inline bool stackChunkOopDesc::has_thaw_slowpath_condition() const { return flags() != 0; } 181 182 inline bool stackChunkOopDesc::requires_barriers() { 183 return Universe::heap()->requires_barriers(this); 184 } 185 186 template <stackChunkOopDesc::BarrierType barrier, ChunkFrames frame_kind, typename RegisterMapT> 187 void stackChunkOopDesc::do_barriers(const StackChunkFrameStream<frame_kind>& f, const RegisterMapT* map) { 188 if (frame_kind == ChunkFrames::Mixed) { 189 // we could freeze deopted frames in slow mode. 190 f.handle_deopted(); 191 } 192 do_barriers0<barrier>(f, map); 193 } 194 195 template <class StackChunkFrameClosureType> 196 inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { 197 has_mixed_frames() ? iterate_stack<ChunkFrames::Mixed>(closure) 198 : iterate_stack<ChunkFrames::CompiledOnly>(closure); 199 } 200 201 template <ChunkFrames frame_kind, class StackChunkFrameClosureType> 202 inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { 203 const SmallRegisterMap* map = SmallRegisterMap::instance; 204 assert(!map->in_cont(), ""); 205 206 StackChunkFrameStream<frame_kind> f(this); 207 bool should_continue = true; 208 209 if (f.is_stub()) { 210 RegisterMap full_map(nullptr, 211 RegisterMap::UpdateMap::include, 212 RegisterMap::ProcessFrames::skip, 213 RegisterMap::WalkContinuation::include); 214 full_map.set_include_argument_oops(false); 215 216 f.next(&full_map); 217 218 assert(!f.is_done(), ""); 219 assert(f.is_compiled(), ""); 220 221 should_continue = closure->do_frame(f, &full_map); 222 f.next(map); 223 f.handle_deopted(); // the stub caller might be deoptimized (as it's not at a call) 224 } 225 assert(!f.is_stub(), ""); 226 227 for(; should_continue && !f.is_done(); f.next(map)) { 228 if (frame_kind == ChunkFrames::Mixed) { 229 // in slow mode we might freeze deoptimized frames 230 f.handle_deopted(); 231 } 232 should_continue = closure->do_frame(f, map); 233 } 234 } 235 236 inline frame stackChunkOopDesc::relativize(frame fr) const { relativize_frame(fr); return fr; } 237 inline frame stackChunkOopDesc::derelativize(frame fr) const { derelativize_frame(fr); return fr; } 238 239 inline void* stackChunkOopDesc::gc_data() const { 240 int stack_sz = stack_size(); 241 assert(stack_sz != 0, "stack should not be empty"); 242 243 // The gc data is located after the stack. 244 return start_of_stack() + stack_sz; 245 } 246 247 inline BitMapView stackChunkOopDesc::bitmap() const { 248 HeapWord* bitmap_addr = static_cast<HeapWord*>(gc_data()); 249 int stack_sz = stack_size(); 250 size_t bitmap_size_in_bits = InstanceStackChunkKlass::bitmap_size_in_bits(stack_sz); 251 252 BitMapView bitmap((BitMap::bm_word_t*)bitmap_addr, bitmap_size_in_bits); 253 254 DEBUG_ONLY(bitmap.verify_range(bit_index_for(start_address()), bit_index_for(end_address()));) 255 256 return bitmap; 257 } 258 259 inline BitMap::idx_t stackChunkOopDesc::bit_index_for(address p) const { 260 return UseCompressedOops ? bit_index_for((narrowOop*)p) : bit_index_for((oop*)p); 261 } 262 263 template <typename OopT> 264 inline BitMap::idx_t stackChunkOopDesc::bit_index_for(OopT* p) const { 265 assert(is_aligned(p, alignof(OopT)), "should be aligned: " PTR_FORMAT, p2i(p)); 266 assert(p >= (OopT*)start_address(), "Address not in chunk"); 267 return p - (OopT*)start_address(); 268 } 269 270 inline intptr_t* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const { 271 return UseCompressedOops ? (intptr_t*)address_for_bit<narrowOop>(index) : (intptr_t*)address_for_bit<oop>(index); 272 } 273 274 template <typename OopT> 275 inline OopT* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const { 276 return (OopT*)start_address() + index; 277 } 278 279 inline MemRegion stackChunkOopDesc::range() { 280 return MemRegion((HeapWord*)this, size()); 281 } 282 283 inline int stackChunkOopDesc::relativize_usp_offset(const frame& fr, const int usp_offset_in_bytes) const { 284 assert(fr.is_compiled_frame() || fr.cb()->is_safepoint_stub(), ""); 285 assert(is_in_chunk(fr.unextended_sp()), ""); 286 287 intptr_t* base = fr.real_fp(); // equal to the caller's sp 288 intptr_t* loc = (intptr_t*)((address)fr.unextended_sp() + usp_offset_in_bytes); 289 assert(base > loc, ""); 290 return (int)(base - loc); 291 } 292 293 inline address stackChunkOopDesc::usp_offset_to_location(const frame& fr, const int usp_offset_in_bytes) const { 294 assert(fr.is_compiled_frame(), ""); 295 return (address)derelativize_address(fr.offset_unextended_sp()) + usp_offset_in_bytes; 296 } 297 298 inline address stackChunkOopDesc::reg_to_location(const frame& fr, const RegisterMap* map, VMReg reg) const { 299 assert(fr.is_compiled_frame(), ""); 300 assert(map != nullptr, ""); 301 assert(map->stack_chunk() == as_oop(), ""); 302 303 // the offsets are saved in the map after going through relativize_usp_offset, so they are sp - loc, in words 304 intptr_t offset = (intptr_t)map->location(reg, nullptr); // see usp_offset_to_index for the chunk case 305 intptr_t* base = derelativize_address(fr.offset_sp()); 306 return (address)(base - offset); 307 } 308 309 inline Method* stackChunkOopDesc::interpreter_frame_method(const frame& fr) { 310 return derelativize(fr).interpreter_frame_method(); 311 } 312 313 inline address stackChunkOopDesc::interpreter_frame_bcp(const frame& fr) { 314 return derelativize(fr).interpreter_frame_bcp(); 315 } 316 317 inline intptr_t* stackChunkOopDesc::interpreter_frame_expression_stack_at(const frame& fr, int index) const { 318 frame heap_frame = derelativize(fr); 319 assert(heap_frame.is_heap_frame(), "must be"); 320 return heap_frame.interpreter_frame_expression_stack_at(index); 321 } 322 323 inline intptr_t* stackChunkOopDesc::interpreter_frame_local_at(const frame& fr, int index) const { 324 frame heap_frame = derelativize(fr); 325 assert(heap_frame.is_heap_frame(), "must be"); 326 return heap_frame.interpreter_frame_local_at(index); 327 } 328 329 inline void stackChunkOopDesc::copy_from_stack_to_chunk(intptr_t* from, intptr_t* to, int size) { 330 log_develop_trace(continuations)("Copying from v: " PTR_FORMAT " - " PTR_FORMAT " (%d words, %d bytes)", 331 p2i(from), p2i(from + size), size, size << LogBytesPerWord); 332 log_develop_trace(continuations)("Copying to h: " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)", 333 p2i(to), to - start_address(), relative_base() - to, p2i(to + size), to + size - start_address(), 334 relative_base() - (to + size), size, size << LogBytesPerWord); 335 336 assert(to >= start_address(), "Chunk underflow"); 337 assert(to + size <= end_address(), "Chunk overflow"); 338 339 #if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO) 340 // Suppress compilation warning-as-error on unimplemented architectures 341 // that stub out arch-specific methods. Some compilers are smart enough 342 // to figure out the argument is always null and then warn about it. 343 if (to != nullptr) 344 #endif 345 memcpy(to, from, size << LogBytesPerWord); 346 } 347 348 inline void stackChunkOopDesc::copy_from_chunk_to_stack(intptr_t* from, intptr_t* to, int size) { 349 log_develop_trace(continuations)("Copying from h: " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)", 350 p2i(from), from - start_address(), relative_base() - from, p2i(from + size), from + size - start_address(), 351 relative_base() - (from + size), size, size << LogBytesPerWord); 352 log_develop_trace(continuations)("Copying to v: " PTR_FORMAT " - " PTR_FORMAT " (%d words, %d bytes)", p2i(to), 353 p2i(to + size), size, size << LogBytesPerWord); 354 355 assert(from >= start_address(), ""); 356 assert(from + size <= end_address(), ""); 357 358 #if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO) 359 // Suppress compilation warning-as-error on unimplemented architectures 360 // that stub out arch-specific methods. Some compilers are smart enough 361 // to figure out the argument is always null and then warn about it. 362 if (to != nullptr) 363 #endif 364 memcpy(to, from, size << LogBytesPerWord); 365 } 366 367 template <typename OopT> 368 inline oop stackChunkOopDesc::load_oop(OopT* addr) { 369 return BarrierSet::barrier_set()->barrier_set_stack_chunk()->load_oop(this, addr); 370 } 371 372 inline intptr_t* stackChunkOopDesc::relative_base() const { 373 // we relativize with respect to end rather than start because GC might compact the chunk 374 return end_address() + frame::metadata_words; 375 } 376 377 inline intptr_t* stackChunkOopDesc::derelativize_address(int offset) const { 378 intptr_t* base = relative_base(); 379 intptr_t* p = base - offset; 380 assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT, 381 p2i(start_address()), p2i(p), p2i(base)); 382 return p; 383 } 384 385 inline int stackChunkOopDesc::relativize_address(intptr_t* p) const { 386 intptr_t* base = relative_base(); 387 intptr_t offset = base - p; 388 assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT, 389 p2i(start_address()), p2i(p), p2i(base)); 390 assert(0 <= offset && offset <= std::numeric_limits<int>::max(), "offset: " PTR_FORMAT, offset); 391 return (int)offset; 392 } 393 394 inline void stackChunkOopDesc::relativize_frame(frame& fr) const { 395 fr.set_offset_sp(relativize_address(fr.sp())); 396 fr.set_offset_unextended_sp(relativize_address(fr.unextended_sp())); 397 relativize_frame_pd(fr); 398 } 399 400 inline void stackChunkOopDesc::derelativize_frame(frame& fr) const { 401 fr.set_sp(derelativize_address(fr.offset_sp())); 402 fr.set_unextended_sp(derelativize_address(fr.offset_unextended_sp())); 403 derelativize_frame_pd(fr); 404 fr.set_frame_index(-1); // for the sake of assertions in frame 405 } 406 407 #endif // SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP