1 /*
2 * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "classfile/classLoaderData.hpp"
25 #include "classfile/classLoaderDataGraph.hpp"
26 #include "classfile/javaClasses.inline.hpp"
27 #include "code/nmethod.hpp"
28 #include "gc/shared/continuationGCSupport.inline.hpp"
29 #include "gc/shared/gc_globals.hpp"
30 #include "gc/shared/suspendibleThreadSet.hpp"
31 #include "gc/shared/workerThread.hpp"
32 #include "gc/z/zAbort.inline.hpp"
33 #include "gc/z/zAddress.inline.hpp"
34 #include "gc/z/zBarrier.inline.hpp"
35 #include "gc/z/zBarrierSetNMethod.hpp"
36 #include "gc/z/zGeneration.inline.hpp"
37 #include "gc/z/zGenerationId.hpp"
38 #include "gc/z/zHeap.inline.hpp"
39 #include "gc/z/zLock.inline.hpp"
40 #include "gc/z/zMark.inline.hpp"
41 #include "gc/z/zMarkCache.inline.hpp"
42 #include "gc/z/zMarkContext.inline.hpp"
43 #include "gc/z/zMarkStack.inline.hpp"
44 #include "gc/z/zMarkTerminate.inline.hpp"
45 #include "gc/z/zNMethod.hpp"
46 #include "gc/z/zPage.hpp"
47 #include "gc/z/zPageTable.inline.hpp"
48 #include "gc/z/zRootsIterator.hpp"
49 #include "gc/z/zStackWatermark.hpp"
50 #include "gc/z/zStat.hpp"
51 #include "gc/z/zTask.hpp"
52 #include "gc/z/zThreadLocalAllocBuffer.hpp"
53 #include "gc/z/zUncoloredRoot.inline.hpp"
54 #include "gc/z/zUtils.inline.hpp"
55 #include "gc/z/zWorkers.hpp"
56 #include "logging/log.hpp"
57 #include "memory/iterator.inline.hpp"
58 #include "oops/objArrayOop.inline.hpp"
59 #include "oops/oop.inline.hpp"
60 #include "runtime/continuation.hpp"
61 #include "runtime/handshake.hpp"
62 #include "runtime/icache.hpp"
63 #include "runtime/javaThread.hpp"
64 #include "runtime/prefetch.inline.hpp"
65 #include "runtime/safepointMechanism.hpp"
66 #include "runtime/stackWatermark.hpp"
67 #include "runtime/stackWatermarkSet.inline.hpp"
68 #include "runtime/threads.hpp"
69 #include "runtime/vmThread.hpp"
70 #include "utilities/align.hpp"
71 #include "utilities/globalDefinitions.hpp"
72 #include "utilities/powerOfTwo.hpp"
73 #include "utilities/ticks.hpp"
74
75 static const ZStatSubPhase ZSubPhaseConcurrentMarkRootUncoloredYoung("Concurrent Mark Root Uncolored", ZGenerationId::young);
76 static const ZStatSubPhase ZSubPhaseConcurrentMarkRootColoredYoung("Concurrent Mark Root Colored", ZGenerationId::young);
77 static const ZStatSubPhase ZSubPhaseConcurrentMarkRootUncoloredOld("Concurrent Mark Root Uncolored", ZGenerationId::old);
78 static const ZStatSubPhase ZSubPhaseConcurrentMarkRootColoredOld("Concurrent Mark Root Colored", ZGenerationId::old);
79
80 ZMark::ZMark(ZGeneration* generation, ZPageTable* page_table)
81 : _generation(generation),
82 _page_table(page_table),
83 _marking_smr(),
84 _stripes(),
85 _terminate(),
86 _work_nproactiveflush(0),
87 _work_nterminateflush(0),
88 _nproactiveflush(0),
89 _nterminateflush(0),
90 _ntrycomplete(0),
91 _ncontinue(0),
92 _nworkers(0) {}
93
94 size_t ZMark::calculate_nstripes(uint nworkers) const {
95 // Calculate the number of stripes from the number of workers we use,
96 // where the number of stripes must be a power of two and we want to
97 // have at least one worker per stripe.
98 const size_t nstripes = round_down_power_of_2(nworkers);
99 return MIN2(nstripes, ZMarkStripesMax);
100 }
101
102 void ZMark::start() {
103 // Verification
104 if (ZVerifyMarking) {
105 verify_all_stacks_empty();
106 }
107
108 // Reset flush/continue counters
109 _nproactiveflush = 0;
110 _nterminateflush = 0;
111 _ntrycomplete = 0;
112 _ncontinue = 0;
113
114 // Set number of workers to use
115 _nworkers = workers()->active_workers();
116
117 // Set number of mark stripes to use, based on number
118 // of workers we will use in the concurrent mark phase.
119 const size_t nstripes = calculate_nstripes(_nworkers);
120 _stripes.set_nstripes(nstripes);
121
122 // Update statistics
123 _generation->stat_mark()->at_mark_start(nstripes);
124
125 // Print worker/stripe distribution
126 LogTarget(Debug, gc, marking) log;
127 if (log.is_enabled()) {
128 log.print("Mark Worker/Stripe Distribution");
129 for (uint worker_id = 0; worker_id < _nworkers; worker_id++) {
130 const ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, worker_id);
131 const size_t stripe_id = _stripes.stripe_id(stripe);
132 log.print(" Worker %u(%u) -> Stripe %zu(%zu)",
133 worker_id, _nworkers, stripe_id, nstripes);
134 }
135 }
136 }
137
138 ZWorkers* ZMark::workers() const {
139 return _generation->workers();
140 }
141
142 void ZMark::prepare_work() {
143 // Set number of workers to use
144 _nworkers = workers()->active_workers();
145
146 // Set number of mark stripes to use, based on number
147 // of workers we will use in the concurrent mark phase.
148 const size_t nstripes = calculate_nstripes(_nworkers);
149 _stripes.set_nstripes(nstripes);
150
151 // Set number of active workers
152 _terminate.reset(_nworkers);
153
154 // Reset flush counters
155 _work_nproactiveflush.store_relaxed(0u);
156 _work_nterminateflush.store_relaxed(0u);
157 }
158
159 void ZMark::finish_work() {
160 // Accumulate proactive/terminate flush counters
161 _nproactiveflush += _work_nproactiveflush.load_relaxed();
162 _nterminateflush += _work_nterminateflush.load_relaxed();
163 }
164
165 void ZMark::follow_work_complete() {
166 follow_work(false /* partial */);
167 }
168
169 bool ZMark::follow_work_partial() {
170 return follow_work(true /* partial */);
171 }
172
173 bool ZMark::is_array(zaddress addr) const {
174 return to_oop(addr)->is_objArray();
175 }
176
177 static uintptr_t encode_partial_array_offset(zpointer* addr) {
178 return untype(ZAddress::offset(to_zaddress((uintptr_t)addr))) >> ZMarkPartialArrayMinSizeShift;
179 }
180
181 static zpointer* decode_partial_array_offset(uintptr_t offset) {
182 return (zpointer*)ZOffset::address(to_zoffset(offset << ZMarkPartialArrayMinSizeShift));
183 }
184
185 void ZMark::push_partial_array(zpointer* addr, size_t length, bool finalizable) {
186 assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned");
187 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(Thread::current(), _generation->id());
188 ZMarkStripe* const stripe = _stripes.stripe_for_addr((uintptr_t)addr);
189 const uintptr_t offset = encode_partial_array_offset(addr);
190 const ZMarkStackEntry entry(offset, length, finalizable);
191
192 log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (%zu), stripe: %zu",
193 p2i(addr), length, _stripes.stripe_id(stripe));
194
195 stacks->push(&_stripes, stripe, &_terminate, entry, false /* publish */);
196 }
197
198 static void mark_barrier_on_oop_array(volatile zpointer* p, size_t length, bool finalizable, bool young) {
199 for (volatile const zpointer* const end = p + length; p < end; p++) {
200 if (young) {
201 ZBarrier::mark_barrier_on_young_oop_field(p);
202 } else {
203 ZBarrier::mark_barrier_on_old_oop_field(p, finalizable);
204 }
205 }
206 }
207
208 void ZMark::follow_array_elements_small(zpointer* addr, size_t length, bool finalizable) {
209 assert(length <= ZMarkPartialArrayMinLength, "Too large, should be split");
210
211 log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (%zu)", p2i(addr), length);
212
213 mark_barrier_on_oop_array(addr, length, finalizable, _generation->is_young());
214 }
215
216 void ZMark::follow_array_elements_large(zpointer* addr, size_t length, bool finalizable) {
217 assert(length <= (size_t)arrayOopDesc::max_array_length(T_OBJECT), "Too large");
218 assert(length > ZMarkPartialArrayMinLength, "Too small, should not be split");
219
220 zpointer* const start = addr;
221 zpointer* const end = start + length;
222
223 // Calculate the aligned middle start/end/size, where the middle start
224 // should always be greater than the start (hence the +1 below) to make
225 // sure we always do some follow work, not just split the array into pieces.
226 zpointer* const middle_start = align_up(start + 1, ZMarkPartialArrayMinSize);
227 const size_t middle_length = align_down(end - middle_start, ZMarkPartialArrayMinLength);
228 zpointer* const middle_end = middle_start + middle_length;
229
230 log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (%zu), "
231 "middle: " PTR_FORMAT "-" PTR_FORMAT " (%zu)",
232 p2i(start), p2i(end), length, p2i(middle_start), p2i(middle_end), middle_length);
233
234 // Push unaligned trailing part
235 if (end > middle_end) {
236 zpointer* const trailing_addr = middle_end;
237 const size_t trailing_length = end - middle_end;
238 push_partial_array(trailing_addr, trailing_length, finalizable);
239 }
240
241 // Push aligned middle part(s)
242 zpointer* partial_addr = middle_end;
243 while (partial_addr > middle_start) {
244 const size_t parts = 2;
245 const size_t partial_length = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinLength);
246 partial_addr -= partial_length;
247 push_partial_array(partial_addr, partial_length, finalizable);
248 }
249
250 // Follow leading part
251 assert(start < middle_start, "Miscalculated middle start");
252 zpointer* const leading_addr = start;
253 const size_t leading_length = middle_start - start;
254 follow_array_elements_small(leading_addr, leading_length, finalizable);
255 }
256
257 void ZMark::follow_array_elements(zpointer* addr, size_t length, bool finalizable) {
258 if (length <= ZMarkPartialArrayMinLength) {
259 follow_array_elements_small(addr, length, finalizable);
260 } else {
261 follow_array_elements_large(addr, length, finalizable);
262 }
263 }
264
265 void ZMark::follow_partial_array(ZMarkStackEntry entry, bool finalizable) {
266 zpointer* const addr = decode_partial_array_offset(entry.partial_array_offset());
267 const size_t length = entry.partial_array_length();
268
269 follow_array_elements(addr, length, finalizable);
270 }
271
272 template <bool finalizable, ZGenerationIdOptional generation>
273 class ZMarkBarrierFollowOopClosure : public OopIterateClosure {
274 private:
275 static int claim_value() {
276 return finalizable ? ClassLoaderData::_claim_finalizable
277 : ClassLoaderData::_claim_strong;
278 }
279
280 static ReferenceDiscoverer* discoverer() {
281 if (!finalizable) {
282 return ZGeneration::old()->reference_discoverer();
283 } else {
284 return nullptr;
285 }
286 }
287
288 static bool visit_metadata() {
289 // Only visit metadata if we're marking through the old generation
290 return ZGeneration::old()->is_phase_mark();
291 }
292
293 const bool _visit_metadata;
294
295 public:
296 ZMarkBarrierFollowOopClosure()
297 : OopIterateClosure(discoverer()),
298 _visit_metadata(visit_metadata()) {}
299
300 virtual void do_oop(oop* p) {
301 switch (generation) {
302 case ZGenerationIdOptional::young:
303 ZBarrier::mark_barrier_on_young_oop_field((volatile zpointer*)p);
304 break;
305 case ZGenerationIdOptional::old:
306 ZBarrier::mark_barrier_on_old_oop_field((volatile zpointer*)p, finalizable);
307 break;
308 case ZGenerationIdOptional::none:
309 ZBarrier::mark_barrier_on_oop_field((volatile zpointer*)p, finalizable);
310 break;
311 }
312 }
313
314 virtual void do_oop(narrowOop* p) {
315 ShouldNotReachHere();
316 }
317
318 virtual bool do_metadata() final {
319 // Only help out with metadata visiting
320 return _visit_metadata;
321 }
322
323 virtual void do_nmethod(nmethod* nm) {
324 assert(do_metadata(), "Don't call otherwise");
325 assert(!finalizable, "Can't handle finalizable marking of nmethods");
326 nm->run_nmethod_entry_barrier();
327 }
328
329 virtual void do_method(Method* m) {
330 // Mark interpreted frames for class redefinition
331 m->record_gc_epoch();
332 }
333
334 virtual void do_klass(Klass* klass) {
335 ClassLoaderData* cld = klass->class_loader_data();
336 ZMarkBarrierFollowOopClosure<finalizable, ZGenerationIdOptional::none> cl;
337 cld->oops_do(&cl, claim_value());
338 }
339
340 virtual void do_cld(ClassLoaderData* cld) {
341 ZMarkBarrierFollowOopClosure<finalizable, ZGenerationIdOptional::none> cl;
342 cld->oops_do(&cl, claim_value());
343 }
344 };
345
346 void ZMark::follow_array_object(objArrayOop obj, bool finalizable) {
347 if (_generation->is_old()) {
348 if (finalizable) {
349 ZMarkBarrierFollowOopClosure<true /* finalizable */, ZGenerationIdOptional::old> cl;
350 cl.do_klass(obj->klass());
351 } else {
352 ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::old> cl;
353 cl.do_klass(obj->klass());
354 }
355 } else {
356 ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::none> cl;
357 if (cl.do_metadata()) {
358 cl.do_klass(obj->klass());
359 }
360 }
361
362 // Should be convertible to colorless oop
363 check_is_valid_zaddress(obj);
364
365 zpointer* const addr = (zpointer*)obj->base();
366 const size_t length = (size_t)obj->length();
367
368 follow_array_elements(addr, length, finalizable);
369 }
370
371 void ZMark::follow_object(oop obj, bool finalizable) {
372 if (_generation->is_old()) {
373 assert(ZHeap::heap()->is_old(to_zaddress(obj)), "Should only follow objects from old gen");
374 if (obj->is_stackChunk()) {
375 // No support for tracing through stack chunks as finalizably reachable
376 ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::old> cl;
377 ZIterator::oop_iterate(obj, &cl);
378 } else if (finalizable) {
379 ZMarkBarrierFollowOopClosure<true /* finalizable */, ZGenerationIdOptional::old> cl;
380 ZIterator::oop_iterate(obj, &cl);
381 } else {
382 ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::old> cl;
383 ZIterator::oop_iterate(obj, &cl);
384 }
385 } else {
386 // Young gen must help out with old marking
387 ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::young> cl;
388 ZIterator::oop_iterate(obj, &cl);
389 }
390 }
391
392 void ZMark::mark_and_follow(ZMarkContext* context, ZMarkStackEntry entry) {
393 // Decode flags
394 const bool finalizable = entry.finalizable();
395 const bool partial_array = entry.partial_array();
396
397 if (partial_array) {
398 follow_partial_array(entry, finalizable);
399 return;
400 }
401
402 // Decode object address and additional flags
403 const zaddress addr = ZOffset::address(to_zoffset(entry.object_address()));
404 const bool mark = entry.mark();
405 bool inc_live = entry.inc_live();
406 const bool follow = entry.follow();
407
408 ZPage* const page = _page_table->get(addr);
409 assert(page->is_relocatable(), "Invalid page state");
410
411 // Mark
412 if (mark && !page->mark_object(addr, finalizable, inc_live)) {
413 // Already marked
414 return;
415 }
416
417 // Increment live
418 if (inc_live) {
419 // Update live objects/bytes for page. We use the aligned object
420 // size since that is the actual number of bytes used on the page
421 // and alignment paddings can never be reclaimed.
422 const oop obj = to_oop(addr);
423 const size_t size = ZUtils::object_size(addr);
424 const size_t aligned_size = align_up(size, page->object_alignment());
425 context->cache()->inc_live(page, aligned_size);
426 // Track objects that will expand by one HeapWord during relocation due to compact
427 // identity hashcode: their FROM size is one word smaller than their TO size.
428 if (UseCompactObjectHeaders && obj->mark().is_hashed_not_expanded()
429 && obj->klass()->expand_for_hash(obj, obj->mark())) {
430 page->inc_will_expand(1);
431 }
432 }
433
434 // Follow
435 if (follow) {
436 if (is_array(addr)) {
437 follow_array_object(objArrayOop(to_oop(addr)), finalizable);
438 } else {
439 follow_object(to_oop(addr), finalizable);
440 }
441 }
442 }
443
444 // This function returns true if we need to stop working to resize threads or
445 // abort marking
446 bool ZMark::rebalance_work(ZMarkContext* context) {
447 const size_t assumed_nstripes = context->nstripes();
448 const size_t nstripes = _stripes.nstripes();
449
450 if (assumed_nstripes != nstripes) {
451 // The number of stripes has changed; reflect that change locally
452 context->set_nstripes(nstripes);
453 } else if (nstripes < calculate_nstripes(_nworkers) && _stripes.is_crowded()) {
454 // We are running on a reduced number of threads to minimize the amount of work
455 // hidden in local stacks when the stripes are less well balanced. When this situation
456 // starts getting crowded, we bump the number of stripes again.
457 const size_t new_nstripes = nstripes << 1;
458 if (_stripes.try_set_nstripes(nstripes, new_nstripes)) {
459 context->set_nstripes(new_nstripes);
460 }
461 }
462
463 ZMarkStripe* stripe = _stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id());
464 if (context->stripe() != stripe) {
465 // Need to switch stripe
466 context->set_stripe(stripe);
467 flush(Thread::current());
468 } else if (!_terminate.saturated()) {
469 // Work imbalance detected; striped marking is likely going to be in the way
470 flush(Thread::current());
471 }
472
473 SuspendibleThreadSet::yield();
474
475 return ZAbort::should_abort() || _generation->should_worker_resize();
476 }
477
478 bool ZMark::drain(ZMarkContext* context) {
479 ZMarkThreadLocalStacks* const stacks = context->stacks();
480 ZMarkStackEntry entry;
481 size_t processed = 0;
482
483 context->set_stripe(_stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id()));
484 context->set_nstripes(_stripes.nstripes());
485
486 // Drain stripe stacks
487 while (stacks->pop(&_marking_smr, &_stripes, context->stripe(), &entry)) {
488 mark_and_follow(context, entry);
489
490 if ((processed++ & 31) == 0 && rebalance_work(context)) {
491 return false;
492 }
493 }
494
495 return true;
496 }
497
498 bool ZMark::try_steal_local(ZMarkContext* context) {
499 ZMarkStripe* const stripe = context->stripe();
500 ZMarkThreadLocalStacks* const stacks = context->stacks();
501
502 // Try to steal a local stack from another stripe
503 for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
504 victim_stripe != stripe;
505 victim_stripe = _stripes.stripe_next(victim_stripe)) {
506 ZMarkStack* const stack = stacks->steal(&_stripes, victim_stripe);
507 if (stack != nullptr) {
508 // Success, install the stolen stack
509 stacks->install(&_stripes, stripe, stack);
510 return true;
511 }
512 }
513
514 // Nothing to steal
515 return false;
516 }
517
518 bool ZMark::try_steal_global(ZMarkContext* context) {
519 ZMarkStripe* const stripe = context->stripe();
520 ZMarkThreadLocalStacks* const stacks = context->stacks();
521
522 // Try to steal a stack from another stripe
523 for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
524 victim_stripe != stripe;
525 victim_stripe = _stripes.stripe_next(victim_stripe)) {
526 ZMarkStack* const stack = victim_stripe->steal_stack(&_marking_smr);
527 if (stack != nullptr) {
528 // Success, install the stolen stack
529 stacks->install(&_stripes, stripe, stack);
530 return true;
531 }
532 }
533
534 // Nothing to steal
535 return false;
536 }
537
538 bool ZMark::try_steal(ZMarkContext* context) {
539 return try_steal_local(context) || try_steal_global(context);
540 }
541
542 class ZMarkFlushStacksHandshakeClosure : public HandshakeClosure {
543 private:
544 ZMark* const _mark;
545 bool _flushed;
546
547 public:
548 ZMarkFlushStacksHandshakeClosure(ZMark* mark)
549 : HandshakeClosure("ZMarkFlushStacks"),
550 _mark(mark),
551 _flushed(false) {}
552
553 void do_thread(Thread* thread) {
554 if (_mark->flush(thread)) {
555 _flushed = true;
556 if (SafepointSynchronize::is_at_safepoint()) {
557 log_debug(gc, marking)("Thread broke mark termination %s", thread->name());
558 }
559 }
560 }
561
562 bool flushed() const {
563 return _flushed;
564 }
565 };
566
567 class VM_ZMarkFlushOperation : public VM_Operation {
568 private:
569 ThreadClosure* _cl;
570
571 public:
572 VM_ZMarkFlushOperation(ThreadClosure* cl)
573 : _cl(cl) {}
574
575 virtual bool evaluate_at_safepoint() const {
576 return false;
577 }
578
579 virtual void doit() {
580 // Flush VM thread
581 Thread* const thread = Thread::current();
582 _cl->do_thread(thread);
583 }
584
585 virtual VMOp_Type type() const {
586 return VMOp_ZMarkFlushOperation;
587 }
588
589 virtual bool is_gc_operation() const {
590 return true;
591 }
592 };
593
594 bool ZMark::flush() {
595 ZMarkFlushStacksHandshakeClosure cl(this);
596 VM_ZMarkFlushOperation vm_cl(&cl);
597 Handshake::execute(&cl);
598 VMThread::execute(&vm_cl);
599
600 // Returns true if more work is available
601 return cl.flushed() || !_stripes.is_empty();
602 }
603
604 bool ZMark::try_terminate_flush() {
605 _work_nterminateflush.add_then_fetch(1u);
606 _terminate.set_resurrected(false);
607
608 if (ZVerifyMarking) {
609 verify_worker_stacks_empty();
610 }
611
612 return flush() || _terminate.resurrected();
613 }
614
615 bool ZMark::try_proactive_flush() {
616 // Only do proactive flushes from worker 0
617 if (WorkerThread::worker_id() != 0) {
618 return false;
619 }
620
621 if (_work_nproactiveflush.load_relaxed() == ZMarkProactiveFlushMax) {
622 // Limit reached or we're trying to terminate
623 return false;
624 }
625
626 _work_nproactiveflush.add_then_fetch(1u);
627
628 SuspendibleThreadSetLeaver sts_leaver;
629 return flush();
630 }
631
632 bool ZMark::try_terminate(ZMarkContext* context) {
633 return _terminate.try_terminate(&_stripes, context->nstripes());
634 }
635
636 void ZMark::leave() {
637 _terminate.leave();
638 }
639
640 // Returning true means marking finished successfully after marking as far as it could.
641 // Returning false means that marking finished unsuccessfully due to abort or resizing.
642 bool ZMark::follow_work(bool partial) {
643 ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id());
644 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(Thread::current(), _generation->id());
645 ZMarkContext context(ZMarkStripesMax, stripe, stacks);
646
647 for (;;) {
648 if (!drain(&context)) {
649 leave();
650 return false;
651 }
652
653 if (try_steal(&context)) {
654 // Stole work
655 continue;
656 }
657
658 if (partial) {
659 return true;
660 }
661
662 if (try_proactive_flush()) {
663 // Work available
664 continue;
665 }
666
667 if (try_terminate(&context)) {
668 // Terminate
669 return true;
670 }
671 }
672 }
673
674 class ZMarkOopClosure : public OopClosure {
675 public:
676 virtual void do_oop(oop* p) {
677 ZBarrier::mark_barrier_on_oop_field((zpointer*)p, false /* finalizable */);
678 }
679
680 virtual void do_oop(narrowOop* p) {
681 ShouldNotReachHere();
682 }
683 };
684
685 class ZMarkYoungOopClosure : public OopClosure {
686 public:
687 virtual void do_oop(oop* p) {
688 ZBarrier::mark_young_good_barrier_on_oop_field((zpointer*)p);
689 }
690
691 virtual void do_oop(narrowOop* p) {
692 ShouldNotReachHere();
693 }
694 };
695
696 class ZMarkThreadClosure : public ThreadClosure {
697 private:
698 static ZUncoloredRoot::RootFunction root_function() {
699 return ZUncoloredRoot::mark;
700 }
701
702 public:
703 ZMarkThreadClosure() {
704 ZThreadLocalAllocBuffer::reset_statistics();
705 }
706 ~ZMarkThreadClosure() {
707 ZThreadLocalAllocBuffer::publish_statistics();
708 }
709
710 virtual void do_thread(Thread* thread) {
711 JavaThread* const jt = JavaThread::cast(thread);
712
713 StackWatermarkSet::finish_processing(jt, (void*)root_function(), StackWatermarkKind::gc);
714 ZThreadLocalAllocBuffer::update_stats(jt);
715 }
716 };
717
718 class ZMarkNMethodClosure : public NMethodClosure {
719 private:
720 ZBarrierSetNMethod* const _bs_nm;
721
722 public:
723 ZMarkNMethodClosure()
724 : _bs_nm(static_cast<ZBarrierSetNMethod*>(BarrierSet::barrier_set()->barrier_set_nmethod())) {}
725
726 virtual void do_nmethod(nmethod* nm) {
727 ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
728 if (_bs_nm->is_armed(nm)) {
729 {
730 ICacheInvalidationContext icic;
731 // Heal barriers
732 ZNMethod::nmethod_patch_barriers(nm, &icic);
733
734 // Heal oops
735 ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm));
736 ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
737 }
738
739 // CodeCache unloading support
740 nm->mark_as_maybe_on_stack();
741
742 log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old", p2i(nm));
743
744 // Disarm
745 _bs_nm->disarm(nm);
746 }
747 }
748 };
749
750 class ZMarkYoungNMethodClosure : public NMethodClosure {
751 private:
752 ZBarrierSetNMethod* const _bs_nm;
753
754 public:
755 ZMarkYoungNMethodClosure()
756 : _bs_nm(static_cast<ZBarrierSetNMethod*>(BarrierSet::barrier_set()->barrier_set_nmethod())) {}
757
758 virtual void do_nmethod(nmethod* nm) {
759 ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
760 if (nm->is_unloading()) {
761 return;
762 }
763
764 if (_bs_nm->is_armed(nm)) {
765 const uintptr_t prev_color = ZNMethod::color(nm);
766
767 // Disarm only the young marking, not any potential old marking cycle
768
769 const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
770 const uintptr_t old_marked = prev_color & old_marked_mask;
771
772 const zpointer new_disarm_value_ptr = ZAddress::color(zaddress::null, ZPointerLoadGoodMask | ZPointerMarkedYoung | old_marked | ZPointerRemembered);
773
774 // Check if disarming for young mark, completely disarms the nmethod entry barrier
775 const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr);
776
777 {
778 ICacheInvalidationContext icic;
779 if (complete_disarm) {
780 // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
781 ZNMethod::nmethod_patch_barriers(nm, &icic);
782 }
783
784 // Heal oops
785 ZUncoloredRootMarkYoungOopClosure cl(prev_color);
786 ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
787 }
788
789 _bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr));
790
791 if (complete_disarm) {
792 log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by young (complete) [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, untype(new_disarm_value_ptr));
793 assert(!_bs_nm->is_armed(nm), "Must not be considered armed anymore");
794 } else {
795 log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by young (incomplete) [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, untype(new_disarm_value_ptr));
796 assert(_bs_nm->is_armed(nm), "Must be considered armed");
797 }
798 }
799 }
800 };
801
802 typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_strong> ZMarkOldCLDClosure;
803
804 class ZMarkOldRootsTask : public ZTask {
805 private:
806 ZRootsIteratorStrongColored _roots_colored;
807 ZRootsIteratorStrongUncolored _roots_uncolored;
808
809 ZMarkOopClosure _cl_colored;
810 ZMarkOldCLDClosure _cld_cl;
811
812 ZMarkThreadClosure _thread_cl;
813 ZMarkNMethodClosure _nm_cl;
814
815 public:
816 ZMarkOldRootsTask()
817 : ZTask("ZMarkOldRootsTask"),
818 _roots_colored(ZGenerationIdOptional::old),
819 _roots_uncolored(ZGenerationIdOptional::old),
820 _cl_colored(),
821 _cld_cl(&_cl_colored),
822 _thread_cl(),
823 _nm_cl() {}
824
825 virtual void work() {
826 {
827 ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootColoredOld);
828 _roots_colored.apply(&_cl_colored,
829 &_cld_cl);
830 }
831
832 {
833 ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootUncoloredOld);
834 _roots_uncolored.apply(&_thread_cl,
835 &_nm_cl);
836 }
837
838 // Flush and free worker stacks. Needed here since
839 // the set of workers executing during root scanning
840 // can be different from the set of workers executing
841 // during mark.
842 ZHeap::heap()->mark_flush(Thread::current());
843 }
844 };
845
846 class ZMarkYoungCLDClosure : public ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> {
847 public:
848 virtual void do_cld(ClassLoaderData* cld) {
849 if (!cld->is_alive()) {
850 // Skip marking through concurrently unloading CLDs
851 return;
852 }
853 ClaimingCLDToOopClosure<ClassLoaderData::_claim_none>::do_cld(cld);
854 }
855
856 ZMarkYoungCLDClosure(OopClosure* cl)
857 : ClaimingCLDToOopClosure<ClassLoaderData::_claim_none>(cl) {}
858 };
859
860 class ZMarkYoungRootsTask : public ZTask {
861 private:
862 ZRootsIteratorAllColored _roots_colored;
863 ZRootsIteratorAllUncolored _roots_uncolored;
864
865 ZMarkYoungOopClosure _cl_colored;
866 ZMarkYoungCLDClosure _cld_cl;
867
868 ZMarkThreadClosure _thread_cl;
869 ZMarkYoungNMethodClosure _nm_cl;
870
871 public:
872 ZMarkYoungRootsTask()
873 : ZTask("ZMarkYoungRootsTask"),
874 _roots_colored(ZGenerationIdOptional::young),
875 _roots_uncolored(ZGenerationIdOptional::young),
876 _cl_colored(),
877 _cld_cl(&_cl_colored),
878 _thread_cl(),
879 _nm_cl() {}
880
881 virtual void work() {
882 {
883 ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootColoredYoung);
884 _roots_colored.apply(&_cl_colored,
885 &_cld_cl);
886 }
887
888 {
889 ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootUncoloredYoung);
890 _roots_uncolored.apply(&_thread_cl,
891 &_nm_cl);
892 }
893
894 // Flush and free worker stacks. Needed here since
895 // the set of workers executing during root scanning
896 // can be different from the set of workers executing
897 // during mark.
898 ZHeap::heap()->mark_flush(Thread::current());
899 }
900 };
901
902 class ZMarkTask : public ZRestartableTask {
903 private:
904 ZMark* const _mark;
905
906 public:
907 ZMarkTask(ZMark* mark)
908 : ZRestartableTask("ZMarkTask"),
909 _mark(mark) {
910 _mark->prepare_work();
911 }
912
913 ~ZMarkTask() {
914 _mark->finish_work();
915 }
916
917 virtual void work() {
918 SuspendibleThreadSetJoiner sts_joiner;
919 _mark->follow_work_complete();
920 // We might have found pointers into the other generation, and then we want to
921 // publish such marking stacks to prevent that generation from getting a mark continue.
922 // We also flush in case of a resize where a new worker thread continues the marking
923 // work, causing a mark continue for the collected generation.
924 ZHeap::heap()->mark_flush(Thread::current());
925 }
926
927 virtual void resize_workers(uint nworkers) {
928 _mark->resize_workers(nworkers);
929 }
930 };
931
932 void ZMark::resize_workers(uint nworkers) {
933 _nworkers = nworkers;
934 const size_t nstripes = calculate_nstripes(nworkers);
935 _stripes.set_nstripes(nstripes);
936 _terminate.reset(nworkers);
937 }
938
939 void ZMark::mark_young_roots() {
940 SuspendibleThreadSetJoiner sts_joiner;
941 ZMarkYoungRootsTask task;
942 workers()->run(&task);
943 }
944
945 void ZMark::mark_old_roots() {
946 SuspendibleThreadSetJoiner sts_joiner;
947 ZMarkOldRootsTask task;
948 workers()->run(&task);
949 }
950
951 void ZMark::mark_follow() {
952 for (;;) {
953 ZMarkTask task(this);
954 workers()->run(&task);
955 if (ZAbort::should_abort() || !try_terminate_flush()) {
956 break;
957 }
958 }
959 }
960
961 bool ZMark::try_end() {
962 if (_terminate.resurrected()) {
963 // An oop was resurrected after concurrent termination.
964 return false;
965 }
966
967 // Try end marking
968 ZMarkFlushStacksHandshakeClosure cl(this);
969 Threads::non_java_threads_do(&cl);
970
971 // Check if non-java threads have any pending marking
972 if (cl.flushed() || !_stripes.is_empty()) {
973 return false;
974 }
975
976 // Mark completed
977 return true;
978 }
979
980 bool ZMark::end() {
981 // Try end marking
982 if (!try_end()) {
983 // Mark not completed
984 _ncontinue++;
985 return false;
986 }
987
988 // Verification
989 if (ZVerifyMarking) {
990 verify_all_stacks_empty();
991 }
992
993 // Update statistics
994 _generation->stat_mark()->at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
995
996 // Mark completed
997 return true;
998 }
999
1000 void ZMark::free() {
1001 // Free any unused mark stack space
1002 _marking_smr.free();
1003 }
1004
1005 bool ZMark::flush(Thread* thread) {
1006 if (thread->is_Java_thread()) {
1007 ZThreadLocalData::store_barrier_buffer(thread)->flush();
1008 }
1009 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(thread, _generation->id());
1010 return stacks->flush(&_stripes, &_terminate);
1011 }
1012
1013 class ZVerifyMarkStacksEmptyClosure : public ThreadClosure {
1014 private:
1015 const ZMarkStripeSet* const _stripes;
1016 const ZGenerationId _generation_id;
1017
1018 public:
1019 ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes, ZGenerationId id)
1020 : _stripes(stripes),
1021 _generation_id(id) {}
1022
1023 void do_thread(Thread* thread) {
1024 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(thread, _generation_id);
1025 guarantee(stacks->is_empty(_stripes), "Should be empty");
1026 }
1027 };
1028
1029 void ZMark::verify_all_stacks_empty() const {
1030 // Verify thread stacks
1031 ZVerifyMarkStacksEmptyClosure cl(&_stripes, _generation->id());
1032 Threads::threads_do(&cl);
1033
1034 // Verify stripe stacks
1035 guarantee(_stripes.is_empty(), "Should be empty");
1036 }
1037
1038 void ZMark::verify_worker_stacks_empty() const {
1039 // Verify thread stacks
1040 ZVerifyMarkStacksEmptyClosure cl(&_stripes, _generation->id());
1041 workers()->threads_do(&cl);
1042 }