1 /*
2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "interpreter/bytecodeStream.hpp"
27 #include "interpreter/oopMapCache.hpp"
28 #include "logging/log.hpp"
29 #include "logging/logStream.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "oops/generateOopMap.hpp"
33 #include "oops/oop.inline.hpp"
34 #include "runtime/atomic.hpp"
35 #include "runtime/handles.inline.hpp"
36 #include "runtime/safepoint.hpp"
37 #include "runtime/signature.hpp"
38
39 class OopMapCacheEntry: private InterpreterOopMap {
40 friend class InterpreterOopMap;
41 friend class OopMapForCacheEntry;
42 friend class OopMapCache;
43 friend class VerifyClosure;
44
45 private:
46 OopMapCacheEntry* _next;
47
48 protected:
49 // Initialization
50 void fill(const methodHandle& method, int bci);
51 // fills the bit mask for native calls
52 void fill_for_native(const methodHandle& method);
53 void set_mask(CellTypeState* vars, CellTypeState* stack, int stack_top);
54
55 // Deallocate bit masks and initialize fields
56 void flush();
57
58 static void deallocate(OopMapCacheEntry* const entry);
59
60 private:
61 void allocate_bit_mask(); // allocates the bit mask on C heap f necessary
62 void deallocate_bit_mask(); // allocates the bit mask on C heap f necessary
63 bool verify_mask(CellTypeState *vars, CellTypeState *stack, int max_locals, int stack_top);
64
65 public:
66 OopMapCacheEntry() : InterpreterOopMap() {
67 _next = nullptr;
68 #ifdef ASSERT
69 _resource_allocate_bit_mask = false;
70 #endif
71 }
72 };
73
74
75 // Implementation of OopMapForCacheEntry
76 // (subclass of GenerateOopMap, initializes an OopMapCacheEntry for a given method and bci)
77
78 class OopMapForCacheEntry: public GenerateOopMap {
79 OopMapCacheEntry *_entry;
80 int _bci;
81 int _stack_top;
82
83 virtual bool report_results() const { return false; }
84 virtual bool possible_gc_point (BytecodeStream *bcs);
85 virtual void fill_stackmap_prolog (int nof_gc_points);
86 virtual void fill_stackmap_epilog ();
87 virtual void fill_stackmap_for_opcodes (BytecodeStream *bcs,
88 CellTypeState* vars,
89 CellTypeState* stack,
90 int stack_top);
91 virtual void fill_init_vars (GrowableArray<intptr_t> *init_vars);
92
93 public:
94 OopMapForCacheEntry(const methodHandle& method, int bci, OopMapCacheEntry *entry);
95
96 // Computes stack map for (method,bci) and initialize entry
97 bool compute_map(Thread* current);
98 int size();
99 };
100
101
102 OopMapForCacheEntry::OopMapForCacheEntry(const methodHandle& method, int bci, OopMapCacheEntry* entry) : GenerateOopMap(method) {
103 _bci = bci;
104 _entry = entry;
105 _stack_top = -1;
106 }
107
108
109 bool OopMapForCacheEntry::compute_map(Thread* current) {
110 assert(!method()->is_native(), "cannot compute oop map for native methods");
111 // First check if it is a method where the stackmap is always empty
112 if (method()->code_size() == 0 || method()->max_locals() + method()->max_stack() == 0) {
113 _entry->set_mask_size(0);
114 } else {
115 ResourceMark rm;
116 if (!GenerateOopMap::compute_map(current)) {
117 fatal("Unrecoverable verification or out-of-memory error");
118 return false;
119 }
120 result_for_basicblock(_bci);
121 }
122 return true;
123 }
124
125
126 bool OopMapForCacheEntry::possible_gc_point(BytecodeStream *bcs) {
127 return false; // We are not reporting any result. We call result_for_basicblock directly
128 }
129
130
131 void OopMapForCacheEntry::fill_stackmap_prolog(int nof_gc_points) {
132 // Do nothing
133 }
134
135
136 void OopMapForCacheEntry::fill_stackmap_epilog() {
137 // Do nothing
138 }
139
140
141 void OopMapForCacheEntry::fill_init_vars(GrowableArray<intptr_t> *init_vars) {
142 // Do nothing
143 }
144
145
146 void OopMapForCacheEntry::fill_stackmap_for_opcodes(BytecodeStream *bcs,
147 CellTypeState* vars,
148 CellTypeState* stack,
149 int stack_top) {
150 // Only interested in one specific bci
151 if (bcs->bci() == _bci) {
152 _entry->set_mask(vars, stack, stack_top);
153 _stack_top = stack_top;
154 }
155 }
156
157
158 int OopMapForCacheEntry::size() {
159 assert(_stack_top != -1, "compute_map must be called first");
160 return ((method()->is_static()) ? 0 : 1) + method()->max_locals() + _stack_top;
161 }
162
163
164 // Implementation of InterpreterOopMap and OopMapCacheEntry
165
166 class VerifyClosure : public OffsetClosure {
167 private:
168 OopMapCacheEntry* _entry;
169 bool _failed;
170
171 public:
172 VerifyClosure(OopMapCacheEntry* entry) { _entry = entry; _failed = false; }
173 void offset_do(int offset) { if (!_entry->is_oop(offset)) _failed = true; }
174 bool failed() const { return _failed; }
175 };
176
177 InterpreterOopMap::InterpreterOopMap() {
178 initialize();
179 #ifdef ASSERT
180 _resource_allocate_bit_mask = true;
181 #endif
182 }
183
184 InterpreterOopMap::~InterpreterOopMap() {
185 // The expectation is that the bit mask was allocated
186 // last in this resource area. That would make the free of the
187 // bit_mask effective (see how FREE_RESOURCE_ARRAY does a free).
188 // If it was not allocated last, there is not a correctness problem
189 // but the space for the bit_mask is not freed.
190 assert(_resource_allocate_bit_mask, "Trying to free C heap space");
191 if (mask_size() > small_mask_limit) {
192 FREE_RESOURCE_ARRAY(uintptr_t, _bit_mask[0], mask_word_size());
193 }
194 }
195
196 bool InterpreterOopMap::is_empty() const {
197 bool result = _method == nullptr;
198 assert(_method != nullptr || (_bci == 0 &&
199 (_mask_size == 0 || _mask_size == USHRT_MAX) &&
200 _bit_mask[0] == 0), "Should be completely empty");
201 return result;
202 }
203
204 void InterpreterOopMap::initialize() {
205 _method = nullptr;
206 _mask_size = USHRT_MAX; // This value should cause a failure quickly
207 _bci = 0;
208 _expression_stack_size = 0;
209 _num_oops = 0;
210 for (int i = 0; i < N; i++) _bit_mask[i] = 0;
211 }
212
213 void InterpreterOopMap::iterate_oop(OffsetClosure* oop_closure) const {
214 int n = number_of_entries();
215 int word_index = 0;
216 uintptr_t value = 0;
217 uintptr_t mask = 0;
218 // iterate over entries
219 for (int i = 0; i < n; i++, mask <<= bits_per_entry) {
220 // get current word
221 if (mask == 0) {
222 value = bit_mask()[word_index++];
223 mask = 1;
224 }
225 // test for oop
226 if ((value & (mask << oop_bit_number)) != 0) oop_closure->offset_do(i);
227 }
228 }
229
230 void InterpreterOopMap::print() const {
231 int n = number_of_entries();
232 tty->print("oop map for ");
233 method()->print_value();
234 tty->print(" @ %d = [%d] { ", bci(), n);
235 for (int i = 0; i < n; i++) {
236 if (is_dead(i)) tty->print("%d+ ", i);
237 else
238 if (is_oop(i)) tty->print("%d ", i);
239 }
240 tty->print_cr("}");
241 }
242
243 class MaskFillerForNative: public NativeSignatureIterator {
244 private:
245 uintptr_t * _mask; // the bit mask to be filled
246 int _size; // the mask size in bits
247
248 void set_one(int i) {
249 i *= InterpreterOopMap::bits_per_entry;
250 assert(0 <= i && i < _size, "offset out of bounds");
251 _mask[i / BitsPerWord] |= (((uintptr_t) 1 << InterpreterOopMap::oop_bit_number) << (i % BitsPerWord));
252 }
253
254 public:
255 void pass_byte() { /* ignore */ }
256 void pass_short() { /* ignore */ }
257 void pass_int() { /* ignore */ }
258 void pass_long() { /* ignore */ }
259 void pass_float() { /* ignore */ }
260 void pass_double() { /* ignore */ }
261 void pass_object() { set_one(offset()); }
262
263 MaskFillerForNative(const methodHandle& method, uintptr_t* mask, int size) : NativeSignatureIterator(method) {
264 _mask = mask;
265 _size = size;
266 // initialize with 0
267 int i = (size + BitsPerWord - 1) / BitsPerWord;
268 while (i-- > 0) _mask[i] = 0;
269 }
270
271 void generate() {
272 iterate();
273 }
274 };
275
276 bool OopMapCacheEntry::verify_mask(CellTypeState* vars, CellTypeState* stack, int max_locals, int stack_top) {
277 // Check mask includes map
278 VerifyClosure blk(this);
279 iterate_oop(&blk);
280 if (blk.failed()) return false;
281
282 // Check if map is generated correctly
283 // (Use ?: operator to make sure all 'true' & 'false' are represented exactly the same so we can use == afterwards)
284 const bool log = log_is_enabled(Trace, interpreter, oopmap);
285 LogStream st(Log(interpreter, oopmap)::trace());
286
287 if (log) st.print("Locals (%d): ", max_locals);
288 for(int i = 0; i < max_locals; i++) {
289 bool v1 = is_oop(i) ? true : false;
290 bool v2 = vars[i].is_reference();
291 assert(v1 == v2, "locals oop mask generation error");
292 if (log) st.print("%d", v1 ? 1 : 0);
293 }
294 if (log) st.cr();
295
296 if (log) st.print("Stack (%d): ", stack_top);
297 for(int j = 0; j < stack_top; j++) {
298 bool v1 = is_oop(max_locals + j) ? true : false;
299 bool v2 = stack[j].is_reference();
300 assert(v1 == v2, "stack oop mask generation error");
301 if (log) st.print("%d", v1 ? 1 : 0);
302 }
303 if (log) st.cr();
304 return true;
305 }
306
307 void OopMapCacheEntry::allocate_bit_mask() {
308 if (mask_size() > small_mask_limit) {
309 assert(_bit_mask[0] == 0, "bit mask should be new or just flushed");
310 _bit_mask[0] = (intptr_t)
311 NEW_C_HEAP_ARRAY(uintptr_t, mask_word_size(), mtClass);
312 }
313 }
314
315 void OopMapCacheEntry::deallocate_bit_mask() {
316 if (mask_size() > small_mask_limit && _bit_mask[0] != 0) {
317 assert(!Thread::current()->resource_area()->contains((void*)_bit_mask[0]),
318 "This bit mask should not be in the resource area");
319 FREE_C_HEAP_ARRAY(uintptr_t, _bit_mask[0]);
320 debug_only(_bit_mask[0] = 0;)
321 }
322 }
323
324
325 void OopMapCacheEntry::fill_for_native(const methodHandle& mh) {
326 assert(mh->is_native(), "method must be native method");
327 set_mask_size(mh->size_of_parameters() * bits_per_entry);
328 allocate_bit_mask();
329 // fill mask for parameters
330 MaskFillerForNative mf(mh, bit_mask(), mask_size());
331 mf.generate();
332 }
333
334
335 void OopMapCacheEntry::fill(const methodHandle& method, int bci) {
336 // Flush entry to deallocate an existing entry
337 flush();
338 set_method(method());
339 set_bci(checked_cast<unsigned short>(bci)); // bci is always u2
340 if (method->is_native()) {
341 // Native method activations have oops only among the parameters and one
342 // extra oop following the parameters (the mirror for static native methods).
343 fill_for_native(method);
344 } else {
345 OopMapForCacheEntry gen(method, bci, this);
346 if (!gen.compute_map(Thread::current())) {
347 fatal("Unrecoverable verification or out-of-memory error");
348 }
349 }
350 }
351
352
353 void OopMapCacheEntry::set_mask(CellTypeState *vars, CellTypeState *stack, int stack_top) {
354 // compute bit mask size
355 int max_locals = method()->max_locals();
356 int n_entries = max_locals + stack_top;
357 set_mask_size(n_entries * bits_per_entry);
358 allocate_bit_mask();
359 set_expression_stack_size(stack_top);
360
361 // compute bits
362 int word_index = 0;
363 uintptr_t value = 0;
364 uintptr_t mask = 1;
365
366 _num_oops = 0;
367 CellTypeState* cell = vars;
368 for (int entry_index = 0; entry_index < n_entries; entry_index++, mask <<= bits_per_entry, cell++) {
369 // store last word
370 if (mask == 0) {
371 bit_mask()[word_index++] = value;
372 value = 0;
373 mask = 1;
374 }
375
376 // switch to stack when done with locals
377 if (entry_index == max_locals) {
378 cell = stack;
379 }
380
381 // set oop bit
382 if (cell->is_reference()) {
383 value |= (mask << oop_bit_number );
384 _num_oops++;
385 }
386
387 // set dead bit
388 if (!cell->is_live()) {
389 value |= (mask << dead_bit_number);
390 assert(!cell->is_reference(), "dead value marked as oop");
391 }
392 }
393
394 // make sure last word is stored
395 bit_mask()[word_index] = value;
396
397 // verify bit mask
398 assert(verify_mask(vars, stack, max_locals, stack_top), "mask could not be verified");
399 }
400
401 void OopMapCacheEntry::flush() {
402 deallocate_bit_mask();
403 initialize();
404 }
405
406 void OopMapCacheEntry::deallocate(OopMapCacheEntry* const entry) {
407 entry->flush();
408 FREE_C_HEAP_OBJ(entry);
409 }
410
411 // Implementation of OopMapCache
412
413 void InterpreterOopMap::resource_copy(OopMapCacheEntry* from) {
414 assert(_resource_allocate_bit_mask,
415 "Should not resource allocate the _bit_mask");
416 assert(from->has_valid_mask(),
417 "Cannot copy entry with an invalid mask");
418
419 set_method(from->method());
420 set_bci(from->bci());
421 set_mask_size(from->mask_size());
422 set_expression_stack_size(from->expression_stack_size());
423 _num_oops = from->num_oops();
424
425 // Is the bit mask contained in the entry?
426 if (from->mask_size() <= small_mask_limit) {
427 memcpy((void *)_bit_mask, (void *)from->_bit_mask,
428 mask_word_size() * BytesPerWord);
429 } else {
430 // The expectation is that this InterpreterOopMap is a recently created
431 // and empty. It is used to get a copy of a cached entry.
432 // If the bit mask has a value, it should be in the
433 // resource area.
434 assert(_bit_mask[0] == 0 ||
435 Thread::current()->resource_area()->contains((void*)_bit_mask[0]),
436 "The bit mask should have been allocated from a resource area");
437 // Allocate the bit_mask from a Resource area for performance. Allocating
438 // from the C heap as is done for OopMapCache has a significant
439 // performance impact.
440 _bit_mask[0] = (uintptr_t) NEW_RESOURCE_ARRAY(uintptr_t, mask_word_size());
441 assert(_bit_mask[0] != 0, "bit mask was not allocated");
442 memcpy((void*) _bit_mask[0], (void*) from->_bit_mask[0],
443 mask_word_size() * BytesPerWord);
444 }
445 }
446
447 inline unsigned int OopMapCache::hash_value_for(const methodHandle& method, int bci) const {
448 // We use method->code_size() rather than method->identity_hash() below since
449 // the mark may not be present if a pointer to the method is already reversed.
450 return ((unsigned int) bci)
451 ^ ((unsigned int) method->max_locals() << 2)
452 ^ ((unsigned int) method->code_size() << 4)
453 ^ ((unsigned int) method->size_of_parameters() << 6);
454 }
455
456 OopMapCacheEntry* volatile OopMapCache::_old_entries = nullptr;
457
458 OopMapCache::OopMapCache() {
459 _array = NEW_C_HEAP_ARRAY(OopMapCacheEntry*, _size, mtClass);
460 for(int i = 0; i < _size; i++) _array[i] = nullptr;
461 }
462
463
464 OopMapCache::~OopMapCache() {
465 assert(_array != nullptr, "sanity check");
466 // Deallocate oop maps that are allocated out-of-line
467 flush();
468 // Deallocate array
469 FREE_C_HEAP_ARRAY(OopMapCacheEntry*, _array);
470 }
471
472 OopMapCacheEntry* OopMapCache::entry_at(int i) const {
473 return Atomic::load_acquire(&(_array[i % _size]));
474 }
475
476 bool OopMapCache::put_at(int i, OopMapCacheEntry* entry, OopMapCacheEntry* old) {
477 return Atomic::cmpxchg(&_array[i % _size], old, entry) == old;
478 }
479
480 void OopMapCache::flush() {
481 for (int i = 0; i < _size; i++) {
482 OopMapCacheEntry* entry = _array[i];
483 if (entry != nullptr) {
484 _array[i] = nullptr; // no barrier, only called in OopMapCache destructor
485 OopMapCacheEntry::deallocate(entry);
486 }
487 }
488 }
489
490 void OopMapCache::flush_obsolete_entries() {
491 assert(SafepointSynchronize::is_at_safepoint(), "called by RedefineClasses in a safepoint");
492 for (int i = 0; i < _size; i++) {
493 OopMapCacheEntry* entry = _array[i];
494 if (entry != nullptr && !entry->is_empty() && entry->method()->is_old()) {
495 // Cache entry is occupied by an old redefined method and we don't want
496 // to pin it down so flush the entry.
497 if (log_is_enabled(Debug, redefine, class, oopmap)) {
498 ResourceMark rm;
499 log_debug(redefine, class, interpreter, oopmap)
500 ("flush: %s(%s): cached entry @%d",
501 entry->method()->name()->as_C_string(), entry->method()->signature()->as_C_string(), i);
502 }
503 _array[i] = nullptr;
504 OopMapCacheEntry::deallocate(entry);
505 }
506 }
507 }
508
509 // Called by GC for thread root scan during a safepoint only. The other interpreted frame oopmaps
510 // are generated locally and not cached.
511 void OopMapCache::lookup(const methodHandle& method,
512 int bci,
513 InterpreterOopMap* entry_for) {
514 assert(SafepointSynchronize::is_at_safepoint(), "called by GC in a safepoint");
515 int probe = hash_value_for(method, bci);
516 int i;
517 OopMapCacheEntry* entry = nullptr;
518
519 if (log_is_enabled(Debug, interpreter, oopmap)) {
520 static int count = 0;
521 ResourceMark rm;
522 log_debug(interpreter, oopmap)
523 ("%d - Computing oopmap at bci %d for %s at hash %d", ++count, bci,
524 method()->name_and_sig_as_C_string(), probe);
525 }
526
527 // Search hashtable for match
528 for(i = 0; i < _probe_depth; i++) {
529 entry = entry_at(probe + i);
530 if (entry != nullptr && !entry->is_empty() && entry->match(method, bci)) {
531 entry_for->resource_copy(entry);
532 assert(!entry_for->is_empty(), "A non-empty oop map should be returned");
533 log_debug(interpreter, oopmap)("- found at hash %d", probe + i);
534 return;
535 }
536 }
537
538 // Entry is not in hashtable.
539 // Compute entry
540
541 OopMapCacheEntry* tmp = NEW_C_HEAP_OBJ(OopMapCacheEntry, mtClass);
542 tmp->initialize();
543 tmp->fill(method, bci);
544 entry_for->resource_copy(tmp);
545
546 if (method->should_not_be_cached()) {
547 // It is either not safe or not a good idea to cache this Method*
548 // at this time. We give the caller of lookup() a copy of the
549 // interesting info via parameter entry_for, but we don't add it to
550 // the cache. See the gory details in Method*.cpp.
551 OopMapCacheEntry::deallocate(tmp);
552 return;
553 }
554
555 // First search for an empty slot
556 for(i = 0; i < _probe_depth; i++) {
557 entry = entry_at(probe + i);
558 if (entry == nullptr) {
559 if (put_at(probe + i, tmp, nullptr)) {
560 assert(!entry_for->is_empty(), "A non-empty oop map should be returned");
561 return;
562 }
563 }
564 }
565
566 log_debug(interpreter, oopmap)("*** collision in oopmap cache - flushing item ***");
567
568 // No empty slot (uncommon case). Use (some approximation of a) LRU algorithm
569 // where the first entry in the collision array is replaced with the new one.
570 OopMapCacheEntry* old = entry_at(probe + 0);
571 if (put_at(probe + 0, tmp, old)) {
572 enqueue_for_cleanup(old);
573 } else {
574 OopMapCacheEntry::deallocate(tmp);
575 }
576
577 assert(!entry_for->is_empty(), "A non-empty oop map should be returned");
578 return;
579 }
580
581 void OopMapCache::enqueue_for_cleanup(OopMapCacheEntry* entry) {
582 bool success = false;
583 OopMapCacheEntry* head;
584 do {
585 head = _old_entries;
586 entry->_next = head;
587 success = Atomic::cmpxchg(&_old_entries, head, entry) == head;
588 } while (!success);
589
590 if (log_is_enabled(Debug, interpreter, oopmap)) {
591 ResourceMark rm;
592 log_debug(interpreter, oopmap)("enqueue %s at bci %d for cleanup",
593 entry->method()->name_and_sig_as_C_string(), entry->bci());
594 }
595 }
596
597 // This is called after GC threads are done and nothing is accessing the old_entries
598 // list, so no synchronization needed.
599 void OopMapCache::cleanup_old_entries() {
600 OopMapCacheEntry* entry = _old_entries;
601 _old_entries = nullptr;
602 while (entry != nullptr) {
603 if (log_is_enabled(Debug, interpreter, oopmap)) {
604 ResourceMark rm;
605 log_debug(interpreter, oopmap)("cleanup entry %s at bci %d",
606 entry->method()->name_and_sig_as_C_string(), entry->bci());
607 }
608 OopMapCacheEntry* next = entry->_next;
609 OopMapCacheEntry::deallocate(entry);
610 entry = next;
611 }
612 }
613
614 void OopMapCache::compute_one_oop_map(const methodHandle& method, int bci, InterpreterOopMap* entry) {
615 // Due to the invariants above it's tricky to allocate a temporary OopMapCacheEntry on the stack
616 OopMapCacheEntry* tmp = NEW_C_HEAP_OBJ(OopMapCacheEntry, mtClass);
617 tmp->initialize();
618 tmp->fill(method, bci);
619 if (tmp->has_valid_mask()) {
620 entry->resource_copy(tmp);
621 }
622 OopMapCacheEntry::deallocate(tmp);
623 }