1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "runtime/continuation.hpp"
33 #include "runtime/safepointVerifiers.hpp"
34
35 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm) :
36 _nm(nm), _oops(nullptr), _oops_count(0), _barriers(nullptr), _barriers_count(0), _unregistered(false), _lock(), _ic_lock() {
37 init_from(nm);
38 }
39
40 ShenandoahNMethod::~ShenandoahNMethod() {
41 if (_oops != nullptr) {
42 FREE_C_HEAP_ARRAY(_oops);
43 }
44 if (_barriers != nullptr) {
45 FREE_C_HEAP_ARRAY(_barriers);
46 }
47 }
48
49 void ShenandoahNMethod::update() {
50 init_from(nm());
51 }
52
53 void ShenandoahNMethod::init_from(nmethod* nm) {
54 ResourceMark rm;
55 bool non_immediate_oops = false;
56 GrowableArray<oop*> oops;
57 GrowableArray<ShenandoahNMethodBarrier> barriers;
58
59 parse(nm, oops, non_immediate_oops, barriers);
60
61 int new_oops_count = oops.length();
62 if (_oops_count != new_oops_count) {
63 if (_oops != nullptr) {
64 FREE_C_HEAP_ARRAY(_oops);
65 _oops = nullptr;
66 }
67 if (new_oops_count > 0) {
68 _oops = NEW_C_HEAP_ARRAY(oop*, new_oops_count, mtGC);
69 }
70 }
71 _oops_count = new_oops_count;
72 for (int c = 0; c < _oops_count; c++) {
73 _oops[c] = oops.at(c);
74 }
75 assert_same_oops();
76
77 int new_barriers_count = barriers.length();
78 if (_barriers_count != new_barriers_count) {
79 if (_barriers != nullptr) {
80 FREE_C_HEAP_ARRAY(_barriers);
81 _barriers = nullptr;
82 }
83 if (new_barriers_count > 0) {
84 _barriers = NEW_C_HEAP_ARRAY(ShenandoahNMethodBarrier, new_barriers_count, mtGC);
85 }
86 }
87 _barriers_count = new_barriers_count;
88 for (int c = 0; c < _barriers_count; c++) {
89 _barriers[c] = barriers.at(c);
90 }
91
92 _has_non_immed_oops = non_immediate_oops;
93 }
94
95 void ShenandoahNMethod::parse(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops, GrowableArray<ShenandoahNMethodBarrier>& barriers) {
96 has_non_immed_oops = false;
97 address code_begin = nm->code_begin();
98 RelocIterator iter(nm);
99 while (iter.next()) {
100 switch (iter.type()) {
101 case relocInfo::oop_type: {
102 oop_Relocation* r = iter.oop_reloc();
103 if (!r->oop_is_immediate()) {
104 // Non-immediate oop found
105 has_non_immed_oops = true;
106 break;
107 }
108
109 oop value = r->oop_value();
110 if (value != nullptr) {
111 oop* addr = r->oop_addr();
112 shenandoah_assert_correct(addr, value);
113 shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc());
114 shenandoah_assert_not_forwarded(addr, value);
115 // Non-null immediate oop found. null oops can safely be
116 // ignored since the method will be re-registered if they
117 // are later patched to be non-null.
118 oops.push(addr);
119 }
120 break;
121 }
122 case relocInfo::patchable_barrier_type: {
123 patchable_barrier_Relocation* r = iter.patchable_barrier_reloc();
124
125 ShenandoahNMethodBarrier b;
126 b._rel_pc = checked_cast<int32_t>(pointer_delta(r->addr(), code_begin, 1));
127 b._rel_target_pc = r->target_offset();
128 b._gc_state = decode_reloc_gc_state(r->metadata());
129 b._jump_when_state = decode_reloc_jump_when_state(r->metadata());
130 barriers.push(b);
131 break;
132 }
133 default:
134 // We do not care about other relocations.
135 break;
136 }
137 }
138 }
139
140 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
141 return new ShenandoahNMethod(nm);
142 }
143
144 bool ShenandoahNMethod::handle_oops(nmethod* nm) {
145 ShenandoahNMethod* data = gc_data(nm);
146 assert(data != nullptr, "Sanity");
147 assert(data->lock()->owned_by_self(), "Must hold the lock");
148
149 ShenandoahHeap* const heap = ShenandoahHeap::heap();
150 if ((heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) ||
151 heap->is_concurrent_strong_root_in_progress()) {
152 heal_nmethod_metadata(data);
153 // Assume healing changed the code.
154 return true;
155 } else if (heap->is_concurrent_mark_in_progress()) {
156 ShenandoahKeepAliveClosure cl;
157 data->oops_do(&cl);
158 } else {
159 // There is possibility that GC is cancelled when it arrives final mark.
160 // In this case, concurrent root phase is skipped and degenerated GC should be
161 // followed, where nmethods are disarmed.
162 }
163
164 // No code modifications happened
165 return false;
166 }
167
168 bool ShenandoahNMethod::handle_barriers(nmethod* nm) {
169 ShenandoahNMethod* data = gc_data(nm);
170 assert(data != nullptr, "Sanity");
171 assert(data->lock()->owned_by_self(), "Must hold the lock");
172
173 char gc_state = ShenandoahHeap::heap()->gc_state();
174 address code_begin = nm->code_begin();
175
176 bool changed = false;
177 for (int c = 0; c < data->_barriers_count; c++) {
178 ShenandoahNMethodBarrier& b = data->_barriers[c];
179 changed |= patch_barrier(code_begin + b._rel_pc,
180 code_begin + b._rel_target_pc,
181 ((gc_state & b._gc_state) != 0) == b._jump_when_state);
182 }
183 return changed;
184 }
185
186 bool ShenandoahNMethod::patch_barrier(address pc, address target_pc, bool should_jump) {
187 // Use precise instruction rewrite code, and only when it recognizes the current insns.
188 // This patching code is non-atomic, but it runs in two safe contexts:
189 // a) For new nmethods that are not yet executing;
190 // b) For existing methods in the nmethod entry barrier context. The nmethod entry barriers
191 // are armed along with stack watermark machinery activation, which together guarantee
192 // the nmethod updates are not interleaved with execution.
193 // The icache flushing is also handled on both paths.
194 bool patched = true;
195 if (should_jump && ShenandoahBarrierSetAssembler::is_patchable_nop(pc)) {
196 ShenandoahBarrierSetAssembler::insert_patchable_jump(pc, target_pc);
197 } else if (!should_jump && ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc)) {
198 ShenandoahBarrierSetAssembler::insert_patchable_nop(pc);
199 } else {
200 patched = false;
201 }
202
203 // Failing to change the barrier is catastrophic for correctness,
204 // so prefer to crash hard even in product.
205 if (should_jump) {
206 guarantee(ShenandoahBarrierSetAssembler::is_patchable_jump(pc, target_pc),
207 "Should be jump to the same address");
208 assert(ShenandoahBarrierSetAssembler::parse_jump_address(pc) == target_pc,
209 "Cross-checking, jump should be to the same address");
210 } else {
211 guarantee(ShenandoahBarrierSetAssembler::is_patchable_nop(pc),
212 "Should be patchable nop");
213 }
214 return patched;
215 }
216
217 #ifdef ASSERT
218 void ShenandoahNMethod::assert_correct() {
219 ShenandoahHeap* heap = ShenandoahHeap::heap();
220 for (int c = 0; c < _oops_count; c++) {
221 oop *loc = _oops[c];
222 assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
223 oop o = RawAccess<>::oop_load(loc);
224 shenandoah_assert_correct_except(loc, o, o == nullptr || heap->is_full_gc_move_in_progress());
225 }
226
227 oop* const begin = _nm->oops_begin();
228 oop* const end = _nm->oops_end();
229 for (oop* p = begin; p < end; p++) {
230 if (*p != Universe::non_oop_word()) {
231 oop o = RawAccess<>::oop_load(p);
232 shenandoah_assert_correct_except(p, o, o == nullptr || heap->is_full_gc_move_in_progress());
233 }
234 }
235 }
236
237 class ShenandoahNMethodOopDetector : public OopClosure {
238 private:
239 ResourceMark rm; // For growable array allocation below.
240 GrowableArray<oop*> _oops;
241
242 public:
243 ShenandoahNMethodOopDetector() : _oops(10) {};
244
245 void do_oop(oop* o) {
246 _oops.append(o);
247 }
248 void do_oop(narrowOop* o) {
249 fatal("NMethods should not have compressed oops embedded.");
250 }
251
252 GrowableArray<oop*>* oops() {
253 return &_oops;
254 }
255 };
256
257 void ShenandoahNMethod::assert_same_oops() {
258 ShenandoahNMethodOopDetector detector;
259 nm()->oops_do(&detector);
260
261 GrowableArray<oop*>* oops = detector.oops();
262
263 int count = _oops_count;
264 for (int index = 0; index < _oops_count; index ++) {
265 assert(oops->contains(_oops[index]), "Must contain this oop");
266 }
267
268 for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) {
269 if (*p == Universe::non_oop_word()) continue;
270 count++;
271 assert(oops->contains(p), "Must contain this oop");
272 }
273
274 if (oops->length() < count) {
275 stringStream debug_stream;
276 debug_stream.print_cr("detected locs: %d", oops->length());
277 for (int i = 0; i < oops->length(); i++) {
278 debug_stream.print_cr("-> " PTR_FORMAT, p2i(oops->at(i)));
279 }
280 debug_stream.print_cr("recorded oops: %d", _oops_count);
281 for (int i = 0; i < _oops_count; i++) {
282 debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i]));
283 }
284 GrowableArray<oop*> check;
285 GrowableArray<ShenandoahNMethodBarrier> barriers;
286 bool non_immed;
287 parse(nm(), check, non_immed, barriers);
288 debug_stream.print_cr("check oops: %d", check.length());
289 for (int i = 0; i < check.length(); i++) {
290 debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i)));
291 }
292 fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s",
293 oops->length(), _oops_count, count, p2i(nm()->oops_begin()), p2i(nm()->oops_end()), debug_stream.freeze());
294 }
295 }
296 #endif
297
298 ShenandoahNMethodTable::ShenandoahNMethodTable() :
299 _heap(ShenandoahHeap::heap()),
300 _index(0),
301 _itr_cnt(0) {
302 _list = new ShenandoahNMethodList(minSize);
303 }
304
305 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
306 assert(_list != nullptr, "Sanity");
307 _list->release();
308 }
309
310 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
311 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
312 assert(_index >= 0 && _index <= _list->size(), "Sanity");
313
314 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
315
316 if (data != nullptr) {
317 // Re-registering the existing nmethod. This is the C1 oop patching path.
318 // We expect no barriers here, as only oops can change in C1 case.
319 assert(contain(nm), "Must have been registered");
320 assert(nm == data->nm(), "Must be same nmethod");
321 assert(nm->is_compiled_by_c1(), "Must be compiled by C1");
322 assert(!data->has_barriers(), "Must not have barriers");
323 // Prevent updating a nmethod while concurrent iteration is in progress.
324 wait_until_concurrent_iteration_done();
325 ShenandoahNMethodLocker data_locker(data->lock());
326 data->update();
327 } else {
328 // New nmethod, not yet executing. We can safely append it to the list,
329 // because concurrent iteration will not touch it. Ditto we do barrier
330 // fixups right here, without relying on nmethod entry barrier to be armed
331 // for new nmethods.
332 data = ShenandoahNMethod::for_nmethod(nm);
333 assert(data != nullptr, "Sanity");
334 ShenandoahNMethod::attach_gc_data(nm, data);
335 ShenandoahLocker locker(&_lock);
336 log_register_nmethod(nm);
337 append(data);
338 ShenandoahNMethodLocker data_locker(data->lock());
339 if (ShenandoahNMethod::handle_barriers(nm)) {
340 ICache::invalidate_range(nm->code_begin(), nm->code_size());
341 }
342 ShenandoahNMethod::disarm_nmethod(nm);
343 }
344 }
345
346 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) {
347 assert_locked_or_safepoint(CodeCache_lock);
348
349 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
350 assert(data != nullptr, "Sanity");
351 log_unregister_nmethod(nm);
352 ShenandoahLocker locker(&_lock);
353 assert(contain(nm), "Must have been registered");
354
355 int idx = index_of(nm);
356 assert(idx >= 0 && idx < _index, "Invalid index");
357 ShenandoahNMethod::attach_gc_data(nm, nullptr);
358 remove(idx);
359 }
360
361 bool ShenandoahNMethodTable::contain(nmethod* nm) const {
362 return index_of(nm) != -1;
363 }
364
365 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
366 assert(index >= 0 && index < _index, "Out of bound");
367 return _list->at(index);
368 }
369
370 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
371 for (int index = 0; index < length(); index ++) {
372 if (at(index)->nm() == nm) {
373 return index;
374 }
375 }
376 return -1;
377 }
378
379 void ShenandoahNMethodTable::remove(int idx) {
380 shenandoah_assert_locked_or_safepoint(CodeCache_lock);
381 assert(_index >= 0 && _index <= _list->size(), "Sanity");
382
383 assert(idx >= 0 && idx < _index, "Out of bound");
384 ShenandoahNMethod* snm = _list->at(idx);
385 ShenandoahNMethod* tmp = _list->at(_index - 1);
386 _list->set(idx, tmp);
387 _index --;
388
389 delete snm;
390 }
391
392 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
393 assert(CodeCache_lock->owned_by_self(), "Lock must be held");
394 while (iteration_in_progress()) {
395 CodeCache_lock->wait_without_safepoint_check();
396 }
397 }
398
399 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
400 if (is_full()) {
401 int new_size = 2 * _list->size();
402 // Rebuild table and replace current one
403 rebuild(new_size);
404 }
405
406 _list->set(_index++, snm);
407 assert(_index >= 0 && _index <= _list->size(), "Sanity");
408 }
409
410 void ShenandoahNMethodTable::rebuild(int size) {
411 ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size);
412 new_list->transfer(_list, _index);
413
414 // Release old list
415 _list->release();
416 _list = new_list;
417 }
418
419 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
420 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
421 _itr_cnt++;
422 return new ShenandoahNMethodTableSnapshot(this);
423 }
424
425 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
426 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
427 assert(iteration_in_progress(), "Why we here?");
428 assert(snapshot != nullptr, "No snapshot");
429 _itr_cnt--;
430
431 delete snapshot;
432 }
433
434 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) {
435 LogTarget(Debug, gc, nmethod) log;
436 if (!log.is_enabled()) {
437 return;
438 }
439
440 ResourceMark rm;
441 log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)",
442 nm->method()->method_holder()->external_name(),
443 nm->method()->name()->as_C_string(),
444 p2i(nm),
445 nm->compiler_name());
446 }
447
448 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) {
449 LogTarget(Debug, gc, nmethod) log;
450 if (!log.is_enabled()) {
451 return;
452 }
453
454 ResourceMark rm;
455 log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]",
456 nm->method()->method_holder()->external_name(),
457 nm->method()->name()->as_C_string(),
458 p2i(nm));
459 }
460
461 #ifdef ASSERT
462 void ShenandoahNMethodTable::assert_nmethods_correct() {
463 assert_locked_or_safepoint(CodeCache_lock);
464
465 for (int index = 0; index < length(); index ++) {
466 ShenandoahNMethod* m = _list->at(index);
467 // Concurrent unloading may have dead nmethods to be cleaned by sweeper
468 if (m->is_unregistered()) continue;
469 m->assert_correct();
470 }
471 }
472 #endif
473
474
475 ShenandoahNMethodList::ShenandoahNMethodList(int size) :
476 _size(size), _ref_count(1) {
477 _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
478 }
479
480 ShenandoahNMethodList::~ShenandoahNMethodList() {
481 assert(_list != nullptr, "Sanity");
482 assert(_ref_count == 0, "Must be");
483 FREE_C_HEAP_ARRAY(_list);
484 }
485
486 void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
487 assert(limit <= size(), "Sanity");
488 ShenandoahNMethod** old_list = list->list();
489 for (int index = 0; index < limit; index++) {
490 _list[index] = old_list[index];
491 }
492 }
493
494 ShenandoahNMethodList* ShenandoahNMethodList::acquire() {
495 assert_locked_or_safepoint(CodeCache_lock);
496 _ref_count++;
497 return this;
498 }
499
500 void ShenandoahNMethodList::release() {
501 assert_locked_or_safepoint(CodeCache_lock);
502 _ref_count--;
503 if (_ref_count == 0) {
504 delete this;
505 }
506 }
507
508 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
509 _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) {
510 }
511
512 ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() {
513 _list->release();
514 }
515
516 void ShenandoahNMethodTableSnapshot::parallel_nmethods_do(NMethodClosure *f) {
517 size_t stride = 256; // educated guess
518
519 ShenandoahNMethod** const list = _list->list();
520
521 size_t max = (size_t)_limit;
522 while (_claimed.load_relaxed() < max) {
523 size_t cur = _claimed.fetch_then_add(stride, memory_order_relaxed);
524 size_t start = cur;
525 size_t end = MIN2(cur + stride, max);
526 if (start >= max) break;
527
528 for (size_t idx = start; idx < end; idx++) {
529 ShenandoahNMethod* nmr = list[idx];
530 assert(nmr != nullptr, "Sanity");
531 if (nmr->is_unregistered()) {
532 continue;
533 }
534
535 nmr->assert_correct();
536 f->do_nmethod(nmr->nm());
537 }
538 }
539 }
540
541 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
542 size_t stride = 256; // educated guess
543
544 ShenandoahNMethod** list = _list->list();
545 size_t max = (size_t)_limit;
546 while (_claimed.load_relaxed() < max) {
547 size_t cur = _claimed.fetch_then_add(stride, memory_order_relaxed);
548 size_t start = cur;
549 size_t end = MIN2(cur + stride, max);
550 if (start >= max) break;
551
552 for (size_t idx = start; idx < end; idx++) {
553 ShenandoahNMethod* data = list[idx];
554 assert(data != nullptr, "Should not be null");
555 if (!data->is_unregistered()) {
556 cl->do_nmethod(data->nm());
557 }
558 }
559 }
560 }
561
562 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) :
563 _table(table),
564 _table_snapshot(nullptr),
565 _started_workers(0),
566 _finished_workers(0) {}
567
568 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) {
569 // Cannot safepoint when iteration is running, because this can cause deadlocks
570 // with other threads waiting on iteration to be over.
571 NoSafepointVerifier nsv;
572
573 MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag);
574
575 if (_finished_workers > 0) {
576 // Some threads have already finished. We are now in rampdown: we are now
577 // waiting for all currently recorded workers to finish. No new workers
578 // should start.
579 return;
580 }
581
582 // Record a new worker and initialize the snapshot if it is a first visitor.
583 if (_started_workers++ == 0) {
584 _table_snapshot = _table->snapshot_for_iteration();
585 }
586
587 // All set, relinquish the lock and go concurrent.
588 {
589 MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
590 _table_snapshot->concurrent_nmethods_do(cl);
591 }
592
593 // Record completion. Last worker shuts down the iterator and notifies any waiters.
594 uint count = ++_finished_workers;
595 if (count == _started_workers) {
596 _table->finish_iteration(_table_snapshot);
597 CodeCache_lock->notify_all();
598 }
599 }