54 shenandoah_assert_not_forwarded(nullptr, obj);
55 shenandoah_assert_marked(nullptr, obj);
56 shenandoah_assert_not_in_cset_except(nullptr, obj, ShenandoahHeap::heap()->cancelled_gc());
57
58 // Are we in weak subgraph scan?
59 bool weak = task->is_weak();
60 cl->set_weak(weak);
61
62 if (task->is_not_chunked()) {
63 Klass* klass = obj->klass();
64 if (klass->is_instance_klass()) {
65 // Case 1: Normal oop, process as usual.
66 if (STRING_DEDUP && (klass == vmClasses::String_klass())) {
67 dedup_string(obj, req);
68 }
69 if (klass->is_stack_chunk_instance_klass()) {
70 // Loom doesn't support mixing of weak marking and strong marking of stack chunks.
71 cl->set_weak(false);
72 }
73 obj->oop_iterate(cl);
74 } else if (klass->is_objArray_klass()) {
75 // Case 2: Object array instance and no chunk is set. Must be the first
76 // time we visit it, start the chunked processing.
77 do_chunked_array_start<T>(q, cl, obj, klass, weak);
78 } else {
79 // Case 3: Primitive array. Do nothing, no oops there. We use the same
80 // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using:
81 // We skip iterating over the klass pointer since we know that
82 // Universe::TypeArrayKlass never moves.
83 assert(klass->is_typeArray_klass(), "should be type array");
84 }
85 // Count liveness the last: push the outstanding work to the queues first
86 // Avoid double-counting objects that are visited twice due to upgrade
87 // from final- to strong mark.
88 if (task->count_liveness()) {
89 count_liveness<GENERATION>(live_data, obj, klass, worker_id);
90 }
91 } else {
92 // Case 4: Array chunk, has sensible chunk id. Process it.
93 do_chunked_array<T>(q, cl, obj, task->chunk(), task->pow(), weak);
94 }
95 }
96
97 inline void ShenandoahMark::dedup_string(oop obj, StringDedup::Requests* const req) {
98 assert(req != nullptr, "Should be available if dedup is enabled");
99
100 // Skip if already requested or dedup is forbidden.
101 // The overwhelming majority of Strings would be filtered here.
102 // These bits are also sticky, so older Strings would be filtered here too.
103 if (java_lang_String::deduplication_requested_or_forbidden(obj)) {
104 return;
105 }
106
107 // Accept deduplication request.
108 if (!java_lang_String::test_and_set_deduplication_requested(obj)) {
109 req->add(obj);
110 }
111 }
112
139 } else {
140 // still good, remember in locals
141 live_data[region_idx] = (ShenandoahLiveData) new_val;
142 }
143 } else {
144 shenandoah_assert_in_correct_region(nullptr, obj);
145 size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
146
147 assert(region->is_affiliated(), "Do not count live data within FREE Humongous Start Region %zu", region_idx);
148 for (size_t i = region_idx; i < region_idx + num_regions; i++) {
149 ShenandoahHeapRegion* chain_reg = heap->get_region(i);
150 assert(chain_reg->is_humongous(), "Expecting a humongous region");
151 assert(chain_reg->is_affiliated(), "Do not count live data within FREE Humongous Continuation Region %zu", i);
152 chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize);
153 }
154 }
155 }
156
157 template <class T>
158 void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj, Klass* klass, bool weak) {
159 assert(obj->is_objArray(), "expect object array");
160 objArrayOop array = objArrayOop(obj);
161 int len = array->length();
162
163 // Mark objArray klass metadata
164 if (Devirtualizer::do_metadata(cl)) {
165 Devirtualizer::do_klass(cl, klass);
166 }
167
168 if (len <= (int) ObjArrayMarkingStride*2) {
169 // A few slices only, process directly
170 array->oop_iterate_elements_range(cl, 0, len);
171 } else {
172 int bits = log2i_graceful(len);
173 // Compensate for non-power-of-two arrays, cover the array in excess:
174 if (len != (1 << bits)) bits++;
175
176 // Only allow full chunks on the queue. This frees do_chunked_array() from checking from/to
177 // boundaries against array->length(), touching the array header on every chunk.
178 //
179 // To do this, we cut the prefix in full-sized chunks, and submit them on the queue.
180 // If the array is not divided in chunk sizes, then there would be an irregular tail,
206 if (left_chunk_end < len) {
207 bool pushed = q->push(ShenandoahMarkTask(array, true, weak, left_chunk, pow));
208 assert(pushed, "overflow queue should always succeed pushing");
209 chunk = right_chunk;
210 last_idx = left_chunk_end;
211 } else {
212 chunk = left_chunk;
213 }
214 }
215
216 // Process the irregular tail, if present
217 int from = last_idx;
218 if (from < len) {
219 array->oop_iterate_elements_range(cl, from, len);
220 }
221 }
222 }
223
224 template <class T>
225 void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow, bool weak) {
226 assert(obj->is_objArray(), "expect object array");
227 objArrayOop array = objArrayOop(obj);
228
229 // Split out tasks, as suggested in ShenandoahMarkTask docs. Avoid pushing tasks that
230 // are known to start beyond the array.
231 while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) {
232 pow--;
233 chunk *= 2;
234 bool pushed = q->push(ShenandoahMarkTask(array, true, weak, chunk - 1, pow));
235 assert(pushed, "overflow queue should always succeed pushing");
236 }
237
238 int chunk_size = 1 << pow;
239
240 int from = (chunk - 1) * chunk_size;
241 int to = chunk * chunk_size;
242
243 #ifdef ASSERT
244 int len = array->length();
245 assert (0 <= from && from < len, "from is sane: %d/%d", from, len);
246 assert (0 < to && to <= len, "to is sane: %d/%d", to, len);
247 #endif
|
54 shenandoah_assert_not_forwarded(nullptr, obj);
55 shenandoah_assert_marked(nullptr, obj);
56 shenandoah_assert_not_in_cset_except(nullptr, obj, ShenandoahHeap::heap()->cancelled_gc());
57
58 // Are we in weak subgraph scan?
59 bool weak = task->is_weak();
60 cl->set_weak(weak);
61
62 if (task->is_not_chunked()) {
63 Klass* klass = obj->klass();
64 if (klass->is_instance_klass()) {
65 // Case 1: Normal oop, process as usual.
66 if (STRING_DEDUP && (klass == vmClasses::String_klass())) {
67 dedup_string(obj, req);
68 }
69 if (klass->is_stack_chunk_instance_klass()) {
70 // Loom doesn't support mixing of weak marking and strong marking of stack chunks.
71 cl->set_weak(false);
72 }
73 obj->oop_iterate(cl);
74 } else if (klass->is_refArray_klass()) {
75 // Case 2: Object array instance and no chunk is set. Must be the first
76 // time we visit it, start the chunked processing.
77 do_chunked_array_start<T>(q, cl, obj, klass, weak);
78 } else if (klass->is_flatArray_klass()) {
79 // Case 3: Flat array instance, all elements are embedded.
80 obj->oop_iterate(cl);
81 } else {
82 // Case 4: Primitive array. Do nothing, no oops there. We use the same
83 // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using:
84 // We skip iterating over the klass pointer since we know that
85 // Universe::TypeArrayKlass never moves.
86 assert(klass->is_typeArray_klass(), "should be type array");
87 }
88 // Count liveness the last: push the outstanding work to the queues first
89 // Avoid double-counting objects that are visited twice due to upgrade
90 // from final- to strong mark.
91 if (task->count_liveness()) {
92 count_liveness<GENERATION>(live_data, obj, klass, worker_id);
93 }
94 } else {
95 // Case 5: Array chunk, has sensible chunk id. Process it.
96 do_chunked_array<T>(q, cl, obj, task->chunk(), task->pow(), weak);
97 }
98 }
99
100 inline void ShenandoahMark::dedup_string(oop obj, StringDedup::Requests* const req) {
101 assert(req != nullptr, "Should be available if dedup is enabled");
102
103 // Skip if already requested or dedup is forbidden.
104 // The overwhelming majority of Strings would be filtered here.
105 // These bits are also sticky, so older Strings would be filtered here too.
106 if (java_lang_String::deduplication_requested_or_forbidden(obj)) {
107 return;
108 }
109
110 // Accept deduplication request.
111 if (!java_lang_String::test_and_set_deduplication_requested(obj)) {
112 req->add(obj);
113 }
114 }
115
142 } else {
143 // still good, remember in locals
144 live_data[region_idx] = (ShenandoahLiveData) new_val;
145 }
146 } else {
147 shenandoah_assert_in_correct_region(nullptr, obj);
148 size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
149
150 assert(region->is_affiliated(), "Do not count live data within FREE Humongous Start Region %zu", region_idx);
151 for (size_t i = region_idx; i < region_idx + num_regions; i++) {
152 ShenandoahHeapRegion* chain_reg = heap->get_region(i);
153 assert(chain_reg->is_humongous(), "Expecting a humongous region");
154 assert(chain_reg->is_affiliated(), "Do not count live data within FREE Humongous Continuation Region %zu", i);
155 chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize);
156 }
157 }
158 }
159
160 template <class T>
161 void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj, Klass* klass, bool weak) {
162 assert(obj->is_refArray(), "expect ref array");
163 refArrayOop array = refArrayOop(obj);
164 int len = array->length();
165
166 // Mark objArray klass metadata
167 if (Devirtualizer::do_metadata(cl)) {
168 Devirtualizer::do_klass(cl, klass);
169 }
170
171 if (len <= (int) ObjArrayMarkingStride*2) {
172 // A few slices only, process directly
173 array->oop_iterate_elements_range(cl, 0, len);
174 } else {
175 int bits = log2i_graceful(len);
176 // Compensate for non-power-of-two arrays, cover the array in excess:
177 if (len != (1 << bits)) bits++;
178
179 // Only allow full chunks on the queue. This frees do_chunked_array() from checking from/to
180 // boundaries against array->length(), touching the array header on every chunk.
181 //
182 // To do this, we cut the prefix in full-sized chunks, and submit them on the queue.
183 // If the array is not divided in chunk sizes, then there would be an irregular tail,
209 if (left_chunk_end < len) {
210 bool pushed = q->push(ShenandoahMarkTask(array, true, weak, left_chunk, pow));
211 assert(pushed, "overflow queue should always succeed pushing");
212 chunk = right_chunk;
213 last_idx = left_chunk_end;
214 } else {
215 chunk = left_chunk;
216 }
217 }
218
219 // Process the irregular tail, if present
220 int from = last_idx;
221 if (from < len) {
222 array->oop_iterate_elements_range(cl, from, len);
223 }
224 }
225 }
226
227 template <class T>
228 void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow, bool weak) {
229 assert(obj->is_refArray(), "expect ref array");
230 refArrayOop array = refArrayOop(obj);
231
232 // Split out tasks, as suggested in ShenandoahMarkTask docs. Avoid pushing tasks that
233 // are known to start beyond the array.
234 while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) {
235 pow--;
236 chunk *= 2;
237 bool pushed = q->push(ShenandoahMarkTask(array, true, weak, chunk - 1, pow));
238 assert(pushed, "overflow queue should always succeed pushing");
239 }
240
241 int chunk_size = 1 << pow;
242
243 int from = (chunk - 1) * chunk_size;
244 int to = chunk * chunk_size;
245
246 #ifdef ASSERT
247 int len = array->length();
248 assert (0 <= from && from < len, "from is sane: %d/%d", from, len);
249 assert (0 < to && to <= len, "to is sane: %d/%d", to, len);
250 #endif
|