1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "runtime/continuation.hpp"
33 #include "runtime/icache.hpp"
34 #include "runtime/safepointVerifiers.hpp"
35
36 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm) :
37 _nm(nm), _oops(nullptr), _oops_count(0), _patchable_jumps(nullptr), _patchable_jumps_count(0), _unregistered(false), _lock(), _ic_lock() {
38 init_from(nm);
39 }
40
41 ShenandoahNMethod::~ShenandoahNMethod() {
42 if (_oops != nullptr) {
43 FREE_C_HEAP_ARRAY(_oops);
44 }
45 if (_patchable_jumps != nullptr) {
46 FREE_C_HEAP_ARRAY(_patchable_jumps);
47 }
48 }
49
50 void ShenandoahNMethod::update() {
51 init_from(nm());
52 }
53
54 void ShenandoahNMethod::init_from(nmethod* nm) {
55 ResourceMark rm;
56 bool non_immediate_oops = false;
57 GrowableArray<oop*> oops;
58 GrowableArray<ShenandoahPatchableJump> jumps;
59
60 parse(nm, oops, non_immediate_oops, jumps);
61
62 int new_oops_count = oops.length();
63 if (_oops_count != new_oops_count) {
64 if (_oops != nullptr) {
65 FREE_C_HEAP_ARRAY(_oops);
66 _oops = nullptr;
67 }
68 if (new_oops_count > 0) {
69 _oops = NEW_C_HEAP_ARRAY(oop*, new_oops_count, mtGC);
70 }
71 }
72 _oops_count = new_oops_count;
73 for (int c = 0; c < _oops_count; c++) {
74 _oops[c] = oops.at(c);
75 }
76 assert_same_oops();
77
78 int new_jumps_count = jumps.length();
79 if (_patchable_jumps_count != new_jumps_count) {
80 if (_patchable_jumps != nullptr) {
81 FREE_C_HEAP_ARRAY(_patchable_jumps);
82 _patchable_jumps = nullptr;
83 }
84 if (new_jumps_count > 0) {
85 _patchable_jumps = NEW_C_HEAP_ARRAY(ShenandoahPatchableJump, new_jumps_count, mtGC);
86 }
87 }
88 _patchable_jumps_count = new_jumps_count;
89 for (int c = 0; c < _patchable_jumps_count; c++) {
90 _patchable_jumps[c] = jumps.at(c);
91 }
92
93 _has_non_immed_oops = non_immediate_oops;
94 }
95
96 void ShenandoahNMethod::parse(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops, GrowableArray<ShenandoahPatchableJump>& jumps) {
97 has_non_immed_oops = false;
98 address code_begin = nm->code_begin();
99 RelocIterator iter(nm);
100 while (iter.next()) {
101 switch (iter.type()) {
102 case relocInfo::oop_type: {
103 oop_Relocation* r = iter.oop_reloc();
104 if (!r->oop_is_immediate()) {
105 // Non-immediate oop found
106 has_non_immed_oops = true;
107 break;
108 }
109
110 oop value = r->oop_value();
111 if (value != nullptr) {
112 oop* addr = r->oop_addr();
113 shenandoah_assert_correct(addr, value);
114 shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
115 shenandoah_assert_not_forwarded(addr, value);
116 // Non-null immediate oop found. null oops can safely be
117 // ignored since the method will be re-registered if they
118 // are later patched to be non-null.
119 oops.push(addr);
120 }
121 break;
122 }
123 case relocInfo::patchable_barrier_type: {
124 patchable_barrier_Relocation* r = iter.patchable_barrier_reloc();
125
126 ShenandoahPatchableJump b;
127 b._rel_pc = checked_cast<int32_t>(pointer_delta(r->addr(), code_begin, 1));
128 b._rel_target_pc = r->target_offset();
129 b._gc_state = decode_reloc_gc_state(r->metadata());
130 b._jump_when_state = decode_reloc_jump_when_state(r->metadata());
131 jumps.push(b);
132 break;
133 }
134 default:
135 // We do not care about other relocations.
136 break;
137 }
138 }
139 }
140
141 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
142 return new ShenandoahNMethod(nm);
143 }
144
145 void ShenandoahNMethod::handle_oops(nmethod* nm, ICacheInvalidationContext* icic) {
146 ShenandoahNMethod* data = gc_data(nm);
147 assert(data != nullptr, "Sanity");
148 assert(data->lock()->owned_by_self(), "Must hold the lock");
149
150 ShenandoahHeap* const heap = ShenandoahHeap::heap();
151 if ((heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) ||
152 heap->is_concurrent_strong_root_in_progress()) {
153 heal_nmethod_metadata(data, icic);
154 } else if (heap->is_concurrent_mark_in_progress()) {
155 ShenandoahKeepAliveClosure cl;
156 data->oops_do(&cl, /* fix_relocations = */ false, /* icic = */ nullptr);
157 } else {
158 // There is possibility that GC is cancelled when it arrives final mark.
159 // In this case, concurrent root phase is skipped and degenerated GC should be
160 // followed, where nmethods are disarmed.
161 }
162 }
163
164 void ShenandoahNMethod::handle_jumps(nmethod* nm, ICacheInvalidationContext* icic) {
165 ShenandoahNMethod* data = gc_data(nm);
166 assert(data != nullptr, "Sanity");
167 assert(data->lock()->owned_by_self(), "Must hold the lock");
168
169 char gc_state = ShenandoahHeap::heap()->gc_state();
170 address code_begin = nm->code_begin();
171
172 bool changed = false;
173 for (int c = 0; c < data->_patchable_jumps_count; c++) {
174 ShenandoahPatchableJump& b = data->_patchable_jumps[c];
175 changed |= patch_jump(code_begin + b._rel_pc,
176 code_begin + b._rel_target_pc,
177 ((gc_state & b._gc_state) != 0) == b._jump_when_state);
178 }
179 if (changed) {
180 icic->set_has_modified_code();
181 }
182 }
183
184 // Use precise instruction rewrite code, and only when it recognizes the current insns.
185 //
186 // This patching code is non-atomic, but it runs in two safe contexts:
187 // a) For new nmethods that are not yet executing and not yet live. This covers the paths
188 // for newly compiled methods, nmethods that were just relocated, the nmethods that
189 // were AOT-loaded.
190 // b) For existing methods in the nmethod entry barrier context. The nmethod entry barriers
191 // are armed along with stack watermark machinery activation. Together they guarantee
192 // the nmethod updates are not interleaved with execution, and nmethod would be patched
193 // before allowing to proceed.
194 //
195 // The icache flushing is also handled on both paths.
196 //
197 bool ShenandoahNMethod::patch_jump(address pc, address target_pc, bool should_jump) {
198 bool patched = true;
199 if (should_jump && ShenandoahBarrierSetAssembler::is_patchable_nop(pc)) {
200 ShenandoahBarrierSetAssembler::insert_patchable_jump(pc, target_pc);
201 } else if (!should_jump && ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc)) {
202 ShenandoahBarrierSetAssembler::insert_patchable_nop(pc);
203 } else {
204 patched = false;
205 }
206
207 // Failing to change the jump is catastrophic for correctness,
208 // so prefer to crash hard even in product.
209 if (should_jump) {
210 guarantee(ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc),
211 "Should be jump to the same address");
212 assert(ShenandoahBarrierSetAssembler::parse_jump_address(pc) == target_pc,
213 "Cross-checking, jump should be to the same address");
214 } else {
215 guarantee(ShenandoahBarrierSetAssembler::is_patchable_nop(pc),
216 "Should be patchable nop");
217 }
218 return patched;
219 }
220
221 #ifdef ASSERT
222 void ShenandoahNMethod::assert_correct() {
223 ShenandoahHeap* heap = ShenandoahHeap::heap();
224 for (int c = 0; c < _oops_count; c++) {
225 oop *loc = _oops[c];
226 assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
227 oop o = RawAccess<>::oop_load(loc);
228 shenandoah_assert_correct_except(loc, o, o == nullptr || heap->is_full_gc_move_in_progress());
229 }
230
231 oop* const begin = _nm->oops_begin();
232 oop* const end = _nm->oops_end();
233 for (oop* p = begin; p < end; p++) {
234 if (*p != Universe::non_oop_word()) {
235 oop o = RawAccess<>::oop_load(p);
236 shenandoah_assert_correct_except(p, o, o == nullptr || heap->is_full_gc_move_in_progress());
237 }
238 }
239 }
240
241 class ShenandoahNMethodOopDetector : public OopClosure {
242 private:
243 ResourceMark rm; // For growable array allocation below.
244 GrowableArray<oop*> _oops;
245
246 public:
247 ShenandoahNMethodOopDetector() : _oops(10) {};
248
249 void do_oop(oop* o) {
250 _oops.append(o);
251 }
252 void do_oop(narrowOop* o) {
253 fatal("NMethods should not have compressed oops embedded.");
254 }
255
256 GrowableArray<oop*>* oops() {
257 return &_oops;
258 }
259 };
260
261 void ShenandoahNMethod::assert_same_oops() {
262 ShenandoahNMethodOopDetector detector;
263 nm()->oops_do(&detector);
264
265 GrowableArray<oop*>* oops = detector.oops();
266
267 int count = _oops_count;
268 for (int index = 0; index < _oops_count; index ++) {
269 assert(oops->contains(_oops[index]), "Must contain this oop");
270 }
271
272 for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) {
273 if (*p == Universe::non_oop_word()) continue;
274 count++;
275 assert(oops->contains(p), "Must contain this oop");
276 }
277
278 if (oops->length() < count) {
279 stringStream debug_stream;
280 debug_stream.print_cr("detected locs: %d", oops->length());
281 for (int i = 0; i < oops->length(); i++) {
282 debug_stream.print_cr("-> " PTR_FORMAT, p2i(oops->at(i)));
283 }
284 debug_stream.print_cr("recorded oops: %d", _oops_count);
285 for (int i = 0; i < _oops_count; i++) {
286 debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i]));
287 }
288 GrowableArray<oop*> check;
289 GrowableArray<ShenandoahPatchableJump> jumps;
290 bool non_immed;
291 parse(nm(), check, non_immed, jumps);
292 debug_stream.print_cr("check oops: %d", check.length());
293 for (int i = 0; i < check.length(); i++) {
294 debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i)));
295 }
296 fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s",
297 oops->length(), _oops_count, count, p2i(nm()->oops_begin()), p2i(nm()->oops_end()), debug_stream.freeze());
298 }
299 }
300 #endif
301
302 ShenandoahNMethodTable::ShenandoahNMethodTable() :
303 _heap(ShenandoahHeap::heap()),
304 _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()),
305 _index(0),
306 _itr_cnt(0) {
307 _list = new ShenandoahNMethodList(minSize);
308 }
309
310 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
311 assert(_list != nullptr, "Sanity");
312 _list->release();
313 }
314
315 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
316 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
317 assert(_index >= 0 && _index <= _list->size(), "Sanity");
318
319 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
320
321 if (data != nullptr) {
322 // Re-registering the existing nmethod. This is the C1 oop patching path.
323 // We expect no patchable jumps here, as only oops can change in C1 case.
324 assert(contain(nm), "Must have been registered");
325 assert(nm == data->nm(), "Must be same nmethod");
326 assert(nm->is_compiled_by_c1(), "Must be compiled by C1");
327 assert(!data->has_patchable_jumps(), "Must not have patchable jumps");
328 // Prevent updating a nmethod while concurrent iteration is in progress.
329 wait_until_concurrent_iteration_done();
330 ShenandoahNMethodLocker data_locker(data->lock());
331 data->update();
332 } else {
333 // New nmethod, not yet executing. We can safely append it to the list,
334 // because concurrent iteration will not touch it. Ditto we do jump
335 // fixups right here, without relying on nmethod entry barrier to be armed
336 // for new nmethods.
337 data = ShenandoahNMethod::for_nmethod(nm);
338 assert(data != nullptr, "Sanity");
339 ShenandoahNMethod::attach_gc_data(nm, data);
340 ShenandoahLocker locker(&_lock);
341 log_register_nmethod(nm);
342 append(data);
343 ShenandoahNMethodLocker data_locker(data->lock());
344 {
345 ICacheInvalidationContext icic;
346 ShenandoahNMethod::handle_jumps(nm, &icic);
347 }
348 ShenandoahNMethod::disarm_nmethod(nm);
349 }
350
351 assert(!data->has_patchable_jumps() || _bs_nm->supports_entry_barrier(nm),
352 "NMethods with patchable jumps require entry barrier support");
353 }
354
355 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
356 assert_locked_or_safepoint(CodeCache_lock);
357
358 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
359 assert(data != nullptr, "Sanity");
360 log_unregister_nmethod(nm);
361 ShenandoahLocker locker(&_lock);
362 assert(contain(nm), "Must have been registered");
363
364 int idx = index_of(nm);
365 assert(idx >= 0 && idx < _index, "Invalid index");
366 ShenandoahNMethod::attach_gc_data(nm, nullptr);
367 remove(idx);
368 }
369
370 bool ShenandoahNMethodTable::contain(nmethod* nm) const {
371 return index_of(nm) != -1;
372 }
373
374 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
375 assert(index >= 0 && index < _index, "Out of bound");
376 return _list->at(index);
377 }
378
379 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
380 for (int index = 0; index < length(); index ++) {
381 if (at(index)->nm() == nm) {
382 return index;
383 }
384 }
385 return -1;
386 }
387
388 void ShenandoahNMethodTable::remove(int idx) {
389 shenandoah_assert_locked_or_safepoint(CodeCache_lock);
390 assert(_index >= 0 && _index <= _list->size(), "Sanity");
391
392 assert(idx >= 0 && idx < _index, "Out of bound");
393 ShenandoahNMethod* snm = _list->at(idx);
394 ShenandoahNMethod* tmp = _list->at(_index - 1);
395 _list->set(idx, tmp);
396 _index --;
397
398 delete snm;
399 }
400
401 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
402 assert(CodeCache_lock->owned_by_self(), "Lock must be held");
403 while (iteration_in_progress()) {
404 CodeCache_lock->wait_without_safepoint_check();
405 }
406 }
407
408 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
409 if (is_full()) {
410 int new_size = 2 * _list->size();
411 // Rebuild table and replace current one
412 rebuild(new_size);
413 }
414
415 _list->set(_index++, snm);
416 assert(_index >= 0 && _index <= _list->size(), "Sanity");
417 }
418
419 void ShenandoahNMethodTable::rebuild(int size) {
420 ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size);
421 new_list->transfer(_list, _index);
422
423 // Release old list
424 _list->release();
425 _list = new_list;
426 }
427
428 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
429 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
430 _itr_cnt++;
431 return new ShenandoahNMethodTableSnapshot(this);
432 }
433
434 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
435 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
436 assert(iteration_in_progress(), "Why we here?");
437 assert(snapshot != nullptr, "No snapshot");
438 _itr_cnt--;
439
440 delete snapshot;
441 }
442
443 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) {
444 LogTarget(Debug, gc, nmethod) log;
445 if (!log.is_enabled()) {
446 return;
447 }
448
449 ResourceMark rm;
450 log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)",
451 nm->method()->method_holder()->external_name(),
452 nm->method()->name()->as_C_string(),
453 p2i(nm),
454 nm->compiler_name());
455 }
456
457 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) {
458 LogTarget(Debug, gc, nmethod) log;
459 if (!log.is_enabled()) {
460 return;
461 }
462
463 ResourceMark rm;
464 log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]",
465 nm->method()->method_holder()->external_name(),
466 nm->method()->name()->as_C_string(),
467 p2i(nm));
468 }
469
470 #ifdef ASSERT
471 void ShenandoahNMethodTable::assert_nmethods_correct() {
472 assert_locked_or_safepoint(CodeCache_lock);
473
474 for (int index = 0; index < length(); index ++) {
475 ShenandoahNMethod* m = _list->at(index);
476 // Concurrent unloading may have dead nmethods to be cleaned by sweeper
477 if (m->is_unregistered()) continue;
478 m->assert_correct();
479 }
480 }
481 #endif
482
483
484 ShenandoahNMethodList::ShenandoahNMethodList(int size) :
485 _size(size), _ref_count(1) {
486 _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
487 }
488
489 ShenandoahNMethodList::~ShenandoahNMethodList() {
490 assert(_list != nullptr, "Sanity");
491 assert(_ref_count == 0, "Must be");
492 FREE_C_HEAP_ARRAY(_list);
493 }
494
495 void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
496 assert(limit <= size(), "Sanity");
497 ShenandoahNMethod** old_list = list->list();
498 for (int index = 0; index < limit; index++) {
499 _list[index] = old_list[index];
500 }
501 }
502
503 ShenandoahNMethodList* ShenandoahNMethodList::acquire() {
504 assert_locked_or_safepoint(CodeCache_lock);
505 _ref_count++;
506 return this;
507 }
508
509 void ShenandoahNMethodList::release() {
510 assert_locked_or_safepoint(CodeCache_lock);
511 _ref_count--;
512 if (_ref_count == 0) {
513 delete this;
514 }
515 }
516
517 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
518 _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) {
519 }
520
521 ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() {
522 _list->release();
523 }
524
525 void ShenandoahNMethodTableSnapshot::parallel_nmethods_do(NMethodClosure *f) {
526 size_t stride = 256; // educated guess
527
528 ShenandoahNMethod** const list = _list->list();
529
530 size_t max = (size_t)_limit;
531 while (_claimed.load_relaxed() < max) {
532 size_t cur = _claimed.fetch_then_add(stride, memory_order_relaxed);
533 size_t start = cur;
534 size_t end = MIN2(cur + stride, max);
535 if (start >= max) break;
536
537 for (size_t idx = start; idx < end; idx++) {
538 ShenandoahNMethod* nmr = list[idx];
539 assert(nmr != nullptr, "Sanity");
540 if (nmr->is_unregistered()) {
541 continue;
542 }
543
544 nmr->assert_correct();
545 f->do_nmethod(nmr->nm());
546 }
547 }
548 }
549
550 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
551 size_t stride = 256; // educated guess
552
553 ShenandoahNMethod** list = _list->list();
554 size_t max = (size_t)_limit;
555 while (_claimed.load_relaxed() < max) {
556 size_t cur = _claimed.fetch_then_add(stride, memory_order_relaxed);
557 size_t start = cur;
558 size_t end = MIN2(cur + stride, max);
559 if (start >= max) break;
560
561 for (size_t idx = start; idx < end; idx++) {
562 ShenandoahNMethod* data = list[idx];
563 assert(data != nullptr, "Should not be null");
564 if (!data->is_unregistered()) {
565 cl->do_nmethod(data->nm());
566 }
567 }
568 }
569 }
570
571 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) :
572 _table(table),
573 _table_snapshot(nullptr),
574 _started_workers(0),
575 _finished_workers(0) {}
576
577 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) {
578 // Cannot safepoint when iteration is running, because this can cause deadlocks
579 // with other threads waiting on iteration to be over.
580 NoSafepointVerifier nsv;
581
582 MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag);
583
584 if (_finished_workers > 0) {
585 // Some threads have already finished. We are now in rampdown: we are now
586 // waiting for all currently recorded workers to finish. No new workers
587 // should start.
588 return;
589 }
590
591 // Record a new worker and initialize the snapshot if it is a first visitor.
592 if (_started_workers++ == 0) {
593 _table_snapshot = _table->snapshot_for_iteration();
594 }
595
596 // All set, relinquish the lock and go concurrent.
597 {
598 MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
599 _table_snapshot->concurrent_nmethods_do(cl);
600 }
601
602 // Record completion. Last worker shuts down the iterator and notifies any waiters.
603 uint count = ++_finished_workers;
604 if (count == _started_workers) {
605 _table->finish_iteration(_table_snapshot);
606 CodeCache_lock->notify_all();
607 }
608 }