1 /*
  2  * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP
 26 #define SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP
 27 
 28 #include "oops/stackChunkOop.hpp"
 29 
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "gc/shared/barrierSet.hpp"
 32 #include "gc/shared/barrierSetStackChunk.hpp"
 33 #include "gc/shared/gc_globals.hpp"
 34 #include "memory/memRegion.hpp"
 35 #include "memory/universe.hpp"
 36 #include "oops/access.inline.hpp"
 37 #include "oops/instanceStackChunkKlass.inline.hpp"
 38 #include "runtime/continuationJavaClasses.inline.hpp"
 39 #include "runtime/frame.inline.hpp"
 40 #include "runtime/globals.hpp"
 41 #include "runtime/handles.inline.hpp"
 42 #include "runtime/registerMap.hpp"
 43 #include "runtime/smallRegisterMap.inline.hpp"
 44 #include "utilities/macros.hpp"
 45 #include CPU_HEADER_INLINE(stackChunkOop)
 46 
 47 DEF_HANDLE_CONSTR(stackChunk, is_stackChunk_noinline)
 48 
 49 inline stackChunkOop stackChunkOopDesc::cast(oop obj) {
 50   assert(obj == nullptr || obj->is_stackChunk(), "Wrong type");
 51   return stackChunkOop(obj);
 52 }
 53 
 54 inline stackChunkOop stackChunkOopDesc::parent() const         { return stackChunkOopDesc::cast(jdk_internal_vm_StackChunk::parent(as_oop())); }
 55 inline void stackChunkOopDesc::set_parent(stackChunkOop value) { jdk_internal_vm_StackChunk::set_parent(this, value); }
 56 template<typename P>
 57 inline void stackChunkOopDesc::set_parent_raw(oop value)       { jdk_internal_vm_StackChunk::set_parent_raw<P>(this, value); }
 58 template<DecoratorSet decorators>
 59 inline void stackChunkOopDesc::set_parent_access(oop value)    { jdk_internal_vm_StackChunk::set_parent_access<decorators>(this, value); }
 60 
 61 inline int stackChunkOopDesc::stack_size() const        { return jdk_internal_vm_StackChunk::size(as_oop()); }
 62 
 63 inline int stackChunkOopDesc::sp() const                { return jdk_internal_vm_StackChunk::sp(as_oop()); }
 64 inline void stackChunkOopDesc::set_sp(int value)        { jdk_internal_vm_StackChunk::set_sp(this, value); }
 65 
 66 inline address stackChunkOopDesc::pc() const            { return jdk_internal_vm_StackChunk::pc(as_oop()); }
 67 inline void stackChunkOopDesc::set_pc(address value)    { jdk_internal_vm_StackChunk::set_pc(this, value); }
 68 
 69 inline int stackChunkOopDesc::argsize() const           { return jdk_internal_vm_StackChunk::argsize(as_oop()); }
 70 inline void stackChunkOopDesc::set_argsize(int value)   { jdk_internal_vm_StackChunk::set_argsize(as_oop(), value); }
 71 
 72 inline uint8_t stackChunkOopDesc::flags() const         { return jdk_internal_vm_StackChunk::flags(as_oop()); }
 73 inline void stackChunkOopDesc::set_flags(uint8_t value) { jdk_internal_vm_StackChunk::set_flags(this, value); }
 74 
 75 inline uint8_t stackChunkOopDesc::flags_acquire() const { return jdk_internal_vm_StackChunk::flags_acquire(as_oop()); }
 76 
 77 inline void stackChunkOopDesc::release_set_flags(uint8_t value) {
 78   jdk_internal_vm_StackChunk::release_set_flags(this, value);
 79 }
 80 
 81 inline bool stackChunkOopDesc::try_set_flags(uint8_t prev_flags, uint8_t new_flags) {
 82   return jdk_internal_vm_StackChunk::try_set_flags(this, prev_flags, new_flags);
 83 }
 84 
 85 inline int stackChunkOopDesc::max_thawing_size() const          { return jdk_internal_vm_StackChunk::maxThawingSize(as_oop()); }
 86 inline void stackChunkOopDesc::set_max_thawing_size(int value)  {
 87   assert(value >= 0, "size must be >= 0");
 88   jdk_internal_vm_StackChunk::set_maxThawingSize(this, (jint)value);
 89 }
 90 
 91 inline oop stackChunkOopDesc::cont() const                {
 92   if (UseZGC && !ZGenerational) {
 93     assert(!UseCompressedOops, "Non-generational ZGC does not support compressed oops");
 94     // The state of the cont oop is used by XCollectedHeap::requires_barriers,
 95     // to determine the age of the stackChunkOopDesc. For that to work, it is
 96     // only the GC that is allowed to perform a load barrier on the oop.
 97     // This function is used by non-GC code and therfore create a stack-local
 98     // copy on the oop and perform the load barrier on that copy instead.
 99     oop obj = jdk_internal_vm_StackChunk::cont_raw<oop>(as_oop());
100     obj = (oop)NativeAccess<>::oop_load(&obj);
101     return obj;
102   }
103   return jdk_internal_vm_StackChunk::cont(as_oop());
104 }
105 inline void stackChunkOopDesc::set_cont(oop value)        { jdk_internal_vm_StackChunk::set_cont(this, value); }
106 template<typename P>
107 inline void stackChunkOopDesc::set_cont_raw(oop value)    { jdk_internal_vm_StackChunk::set_cont_raw<P>(this, value); }
108 template<DecoratorSet decorators>
109 inline void stackChunkOopDesc::set_cont_access(oop value) { jdk_internal_vm_StackChunk::set_cont_access<decorators>(this, value); }
110 
111 inline int stackChunkOopDesc::bottom() const { return stack_size() - argsize() - frame::metadata_words_at_top; }
112 
113 inline HeapWord* stackChunkOopDesc::start_of_stack() const {
114    return (HeapWord*)(cast_from_oop<intptr_t>(as_oop()) + InstanceStackChunkKlass::offset_of_stack());
115 }
116 
117 inline intptr_t* stackChunkOopDesc::start_address() const { return (intptr_t*)start_of_stack(); }
118 inline intptr_t* stackChunkOopDesc::end_address() const { return start_address() + stack_size(); }
119 inline intptr_t* stackChunkOopDesc::bottom_address() const { return start_address() + bottom(); }
120 inline intptr_t* stackChunkOopDesc::sp_address()  const { return start_address() + sp(); }
121 
122 inline int stackChunkOopDesc::to_offset(intptr_t* p) const {
123   assert(is_in_chunk(p)
124     || (p >= start_address() && (p - start_address()) <= stack_size() + frame::metadata_words),
125     "p: " PTR_FORMAT " start: " PTR_FORMAT " end: " PTR_FORMAT, p2i(p), p2i(start_address()), p2i(bottom_address()));
126   return p - start_address();
127 }
128 
129 inline intptr_t* stackChunkOopDesc::from_offset(int offset) const {
130   assert(offset <= stack_size(), "");
131   return start_address() + offset;
132 }
133 
134 inline bool stackChunkOopDesc::is_empty() const {
135   assert(sp() <= stack_size(), "");
136   assert((sp() == stack_size()) == (sp() >= stack_size() - argsize() - frame::metadata_words_at_top),
137     "sp: %d size: %d argsize: %d", sp(), stack_size(), argsize());
138   return sp() == stack_size();
139 }
140 
141 inline bool stackChunkOopDesc::is_in_chunk(void* p) const {
142   HeapWord* start = (HeapWord*)start_address();
143   HeapWord* end = start + stack_size();
144   return (HeapWord*)p >= start && (HeapWord*)p < end;
145 }
146 
147 bool stackChunkOopDesc::is_usable_in_chunk(void* p) const {
148   HeapWord* start = (HeapWord*)start_address() + sp() - frame::metadata_words_at_bottom;
149   HeapWord* end = start + stack_size();
150   return (HeapWord*)p >= start && (HeapWord*)p < end;
151 }
152 
153 inline bool stackChunkOopDesc::is_flag(uint8_t flag) const {
154   return (flags() & flag) != 0;
155 }
156 inline bool stackChunkOopDesc::is_flag_acquire(uint8_t flag) const {
157   return (flags_acquire() & flag) != 0;
158 }
159 inline void stackChunkOopDesc::set_flag(uint8_t flag, bool value) {
160   uint32_t flags = this->flags();
161   set_flags((uint8_t)(value ? flags |= flag : flags &= ~flag));
162 }
163 inline void stackChunkOopDesc::clear_flags() {
164   set_flags(0);
165 }
166 
167 inline bool stackChunkOopDesc::has_mixed_frames() const { return is_flag(FLAG_HAS_INTERPRETED_FRAMES); }
168 inline void stackChunkOopDesc::set_has_mixed_frames(bool value) {
169   assert((flags() & ~FLAG_HAS_INTERPRETED_FRAMES) == 0, "other flags should not be set");
170   set_flag(FLAG_HAS_INTERPRETED_FRAMES, value);
171 }
172 
173 inline bool stackChunkOopDesc::is_gc_mode() const                  { return is_flag(FLAG_GC_MODE); }
174 inline bool stackChunkOopDesc::is_gc_mode_acquire() const          { return is_flag_acquire(FLAG_GC_MODE); }
175 inline void stackChunkOopDesc::set_gc_mode(bool value)             { set_flag(FLAG_GC_MODE, value); }
176 
177 inline bool stackChunkOopDesc::has_bitmap() const                  { return is_flag(FLAG_HAS_BITMAP); }
178 inline void stackChunkOopDesc::set_has_bitmap(bool value)          { set_flag(FLAG_HAS_BITMAP, value); }
179 
180 inline bool stackChunkOopDesc::has_thaw_slowpath_condition() const { return flags() != 0; }
181 
182 inline bool stackChunkOopDesc::requires_barriers() {
183   return Universe::heap()->requires_barriers(this);
184 }
185 
186 template <stackChunkOopDesc::BarrierType barrier, ChunkFrames frame_kind, typename RegisterMapT>
187 void stackChunkOopDesc::do_barriers(const StackChunkFrameStream<frame_kind>& f, const RegisterMapT* map) {
188   if (frame_kind == ChunkFrames::Mixed) {
189     // we could freeze deopted frames in slow mode.
190     f.handle_deopted();
191   }
192   do_barriers0<barrier>(f, map);
193 }
194 
195 template <class StackChunkFrameClosureType>
196 inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) {
197   has_mixed_frames() ? iterate_stack<ChunkFrames::Mixed>(closure)
198                      : iterate_stack<ChunkFrames::CompiledOnly>(closure);
199 }
200 
201 template <ChunkFrames frame_kind, class StackChunkFrameClosureType>
202 inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) {
203   const SmallRegisterMap* map = SmallRegisterMap::instance;
204   assert(!map->in_cont(), "");
205 
206   StackChunkFrameStream<frame_kind> f(this);
207   bool should_continue = true;
208 
209   if (f.is_stub()) {
210     RegisterMap full_map(nullptr,
211                          RegisterMap::UpdateMap::include,
212                          RegisterMap::ProcessFrames::skip,
213                          RegisterMap::WalkContinuation::include);
214     full_map.set_include_argument_oops(false);
215 
216     f.next(&full_map);
217 
218     assert(!f.is_done(), "");
219     assert(f.is_compiled(), "");
220 
221     should_continue = closure->do_frame(f, &full_map);
222     f.next(map);
223     f.handle_deopted(); // the stub caller might be deoptimized (as it's not at a call)
224   }
225   assert(!f.is_stub(), "");
226 
227   for(; should_continue && !f.is_done(); f.next(map)) {
228     if (frame_kind == ChunkFrames::Mixed) {
229       // in slow mode we might freeze deoptimized frames
230       f.handle_deopted();
231     }
232     should_continue = closure->do_frame(f, map);
233   }
234 }
235 
236 inline frame stackChunkOopDesc::relativize(frame fr)   const { relativize_frame(fr);   return fr; }
237 inline frame stackChunkOopDesc::derelativize(frame fr) const { derelativize_frame(fr); return fr; }
238 
239 inline void* stackChunkOopDesc::gc_data() const {
240   int stack_sz = stack_size();
241   assert(stack_sz != 0, "stack should not be empty");
242 
243   // The gc data is located after the stack.
244   return start_of_stack() + stack_sz;
245 }
246 
247 inline BitMapView stackChunkOopDesc::bitmap() const {
248   HeapWord* bitmap_addr = static_cast<HeapWord*>(gc_data());
249   int stack_sz = stack_size();
250   size_t bitmap_size_in_bits = InstanceStackChunkKlass::bitmap_size_in_bits(stack_sz);
251 
252   BitMapView bitmap((BitMap::bm_word_t*)bitmap_addr, bitmap_size_in_bits);
253 
254   DEBUG_ONLY(bitmap.verify_range(bit_index_for(start_address()), bit_index_for(end_address()));)
255 
256   return bitmap;
257 }
258 
259 inline BitMap::idx_t stackChunkOopDesc::bit_index_for(intptr_t* p) const {
260   return UseCompressedOops ? bit_index_for((narrowOop*)p) : bit_index_for((oop*)p);
261 }
262 
263 template <typename OopT>
264 inline BitMap::idx_t stackChunkOopDesc::bit_index_for(OopT* p) const {
265   assert(p >= (OopT*)start_address(), "Address not in chunk");
266   return p - (OopT*)start_address();
267 }
268 
269 inline intptr_t* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const {
270   return UseCompressedOops ? (intptr_t*)address_for_bit<narrowOop>(index) : (intptr_t*)address_for_bit<oop>(index);
271 }
272 
273 template <typename OopT>
274 inline OopT* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const {
275   return (OopT*)start_address() + index;
276 }
277 
278 inline MemRegion stackChunkOopDesc::range() {
279   return MemRegion((HeapWord*)this, size());
280 }
281 
282 inline int stackChunkOopDesc::relativize_usp_offset(const frame& fr, const int usp_offset_in_bytes) const {
283   assert(fr.is_compiled_frame() || fr.cb()->is_safepoint_stub(), "");
284   assert(is_in_chunk(fr.unextended_sp()), "");
285 
286   intptr_t* base = fr.real_fp(); // equal to the caller's sp
287   intptr_t* loc = (intptr_t*)((address)fr.unextended_sp() + usp_offset_in_bytes);
288   assert(base > loc, "");
289   return (int)(base - loc);
290 }
291 
292 inline address stackChunkOopDesc::usp_offset_to_location(const frame& fr, const int usp_offset_in_bytes) const {
293   assert(fr.is_compiled_frame(), "");
294   return (address)derelativize_address(fr.offset_unextended_sp()) + usp_offset_in_bytes;
295 }
296 
297 inline address stackChunkOopDesc::reg_to_location(const frame& fr, const RegisterMap* map, VMReg reg) const {
298   assert(fr.is_compiled_frame(), "");
299   assert(map != nullptr, "");
300   assert(map->stack_chunk() == as_oop(), "");
301 
302   // the offsets are saved in the map after going through relativize_usp_offset, so they are sp - loc, in words
303   intptr_t offset = (intptr_t)map->location(reg, nullptr); // see usp_offset_to_index for the chunk case
304   intptr_t* base = derelativize_address(fr.offset_sp());
305   return (address)(base - offset);
306 }
307 
308 inline Method* stackChunkOopDesc::interpreter_frame_method(const frame& fr) {
309   return derelativize(fr).interpreter_frame_method();
310 }
311 
312 inline address stackChunkOopDesc::interpreter_frame_bcp(const frame& fr) {
313   return derelativize(fr).interpreter_frame_bcp();
314 }
315 
316 inline intptr_t* stackChunkOopDesc::interpreter_frame_expression_stack_at(const frame& fr, int index) const {
317   frame heap_frame = derelativize(fr);
318   assert(heap_frame.is_heap_frame(), "must be");
319   return heap_frame.interpreter_frame_expression_stack_at(index);
320 }
321 
322 inline intptr_t* stackChunkOopDesc::interpreter_frame_local_at(const frame& fr, int index) const {
323   frame heap_frame = derelativize(fr);
324   assert(heap_frame.is_heap_frame(), "must be");
325   return heap_frame.interpreter_frame_local_at(index);
326 }
327 
328 inline void stackChunkOopDesc::copy_from_stack_to_chunk(intptr_t* from, intptr_t* to, int size) {
329   log_develop_trace(continuations)("Copying from v: " PTR_FORMAT " - " PTR_FORMAT " (%d words, %d bytes)",
330     p2i(from), p2i(from + size), size, size << LogBytesPerWord);
331   log_develop_trace(continuations)("Copying to h: " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)",
332     p2i(to), to - start_address(), relative_base() - to, p2i(to + size), to + size - start_address(),
333     relative_base() - (to + size), size, size << LogBytesPerWord);
334 
335   assert(to >= start_address(), "Chunk underflow");
336   assert(to + size <= end_address(), "Chunk overflow");
337 
338 #if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO)
339   // Suppress compilation warning-as-error on unimplemented architectures
340   // that stub out arch-specific methods. Some compilers are smart enough
341   // to figure out the argument is always null and then warn about it.
342   if (to != nullptr)
343 #endif
344   memcpy(to, from, size << LogBytesPerWord);
345 }
346 
347 inline void stackChunkOopDesc::copy_from_chunk_to_stack(intptr_t* from, intptr_t* to, int size) {
348   log_develop_trace(continuations)("Copying from h: " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " PTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)",
349     p2i(from), from - start_address(), relative_base() - from, p2i(from + size), from + size - start_address(),
350     relative_base() - (from + size), size, size << LogBytesPerWord);
351   log_develop_trace(continuations)("Copying to v: " PTR_FORMAT " - " PTR_FORMAT " (%d words, %d bytes)", p2i(to),
352     p2i(to + size), size, size << LogBytesPerWord);
353 
354   assert(from >= start_address(), "");
355   assert(from + size <= end_address(), "");
356 
357 #if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO)
358   // Suppress compilation warning-as-error on unimplemented architectures
359   // that stub out arch-specific methods. Some compilers are smart enough
360   // to figure out the argument is always null and then warn about it.
361   if (to != nullptr)
362 #endif
363   memcpy(to, from, size << LogBytesPerWord);
364 }
365 
366 template <typename OopT>
367 inline oop stackChunkOopDesc::load_oop(OopT* addr) {
368   return BarrierSet::barrier_set()->barrier_set_stack_chunk()->load_oop(this, addr);
369 }
370 
371 inline intptr_t* stackChunkOopDesc::relative_base() const {
372   // we relativize with respect to end rather than start because GC might compact the chunk
373   return end_address() + frame::metadata_words;
374 }
375 
376 inline intptr_t* stackChunkOopDesc::derelativize_address(int offset) const {
377   intptr_t* base = relative_base();
378   intptr_t* p = base - offset;
379   assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT,
380          p2i(start_address()), p2i(p), p2i(base));
381   return p;
382 }
383 
384 inline int stackChunkOopDesc::relativize_address(intptr_t* p) const {
385   intptr_t* base = relative_base();
386   intptr_t offset = base - p;
387   assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT,
388          p2i(start_address()), p2i(p), p2i(base));
389   assert(0 <= offset && offset <= std::numeric_limits<int>::max(), "offset: " PTR_FORMAT, offset);
390   return offset;
391 }
392 
393 inline void stackChunkOopDesc::relativize_frame(frame& fr) const {
394   fr.set_offset_sp(relativize_address(fr.sp()));
395   fr.set_offset_unextended_sp(relativize_address(fr.unextended_sp()));
396   relativize_frame_pd(fr);
397 }
398 
399 inline void stackChunkOopDesc::derelativize_frame(frame& fr) const {
400   fr.set_sp(derelativize_address(fr.offset_sp()));
401   fr.set_unextended_sp(derelativize_address(fr.offset_unextended_sp()));
402   derelativize_frame_pd(fr);
403   fr.set_frame_index(-1); // for the sake of assertions in frame
404 }
405 
406 #endif // SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP