1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classFileParser.hpp"
27 #include "classfile/fieldLayoutBuilder.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "classfile/vmSymbols.hpp"
30 #include "jvm.h"
31 #include "memory/resourceArea.hpp"
32 #include "oops/array.hpp"
33 #include "oops/fieldStreams.inline.hpp"
34 #include "oops/instanceMirrorKlass.hpp"
35 #include "oops/instanceKlass.inline.hpp"
36 #include "oops/klass.inline.hpp"
37 #include "oops/inlineKlass.inline.hpp"
38 #include "runtime/fieldDescriptor.inline.hpp"
39 #include "utilities/powerOfTwo.hpp"
40
41 static LayoutKind field_layout_selection(FieldInfo field_info, Array<InlineLayoutInfo>* inline_layout_info_array) {
42
43 if (field_info.field_flags().is_injected()) {
44 // don't flatten injected fields
45 return LayoutKind::REFERENCE;
46 }
47
48 if (inline_layout_info_array == nullptr || inline_layout_info_array->adr_at(field_info.index())->klass() == nullptr) {
49 // field's type is not a known value class, using a reference
50 return LayoutKind::REFERENCE;
51 }
52
53 InlineLayoutInfo* inline_field_info = inline_layout_info_array->adr_at(field_info.index());
54 InlineKlass* vk = inline_field_info->klass();
55
56 if (field_info.field_flags().is_null_free_inline_type()) {
57 assert(vk->is_implicitly_constructible(), "null-free fields must be implicitly constructible");
58 if (vk->must_be_atomic() || field_info.access_flags().is_volatile() || AlwaysAtomicAccesses) {
59 return vk->has_atomic_layout() ? LayoutKind::ATOMIC_FLAT : LayoutKind::REFERENCE;
60 } else {
61 return vk->has_non_atomic_layout() ? LayoutKind::NON_ATOMIC_FLAT : LayoutKind::REFERENCE;
62 }
63 } else {
64 if (NullableFieldFlattening && vk->has_nullable_layout()) {
65 return LayoutKind::NULLABLE_ATOMIC_FLAT;
66 } else {
67 return LayoutKind::REFERENCE;
68 }
69 }
70 }
71
72 static void get_size_and_alignment(InlineKlass* vk, LayoutKind kind, int* size, int* alignment) {
73 switch(kind) {
74 case LayoutKind::NON_ATOMIC_FLAT:
75 *size = vk->non_atomic_size_in_bytes();
76 *alignment = vk->non_atomic_alignment();
77 break;
78 case LayoutKind::ATOMIC_FLAT:
79 *size = vk->atomic_size_in_bytes();
80 *alignment = *size;
81 break;
82 case LayoutKind::NULLABLE_ATOMIC_FLAT:
83 *size = vk->nullable_size_in_bytes();
84 *alignment = *size;
85 break;
86 default:
87 ShouldNotReachHere();
88 }
89 }
90
91 LayoutRawBlock::LayoutRawBlock(Kind kind, int size) :
92 _next_block(nullptr),
93 _prev_block(nullptr),
94 _inline_klass(nullptr),
95 _block_kind(kind),
96 _offset(-1),
97 _alignment(1),
98 _size(size),
99 _field_index(-1) {
100 assert(kind == EMPTY || kind == RESERVED || kind == PADDING || kind == INHERITED || kind == NULL_MARKER,
101 "Otherwise, should use the constructor with a field index argument");
102 assert(size > 0, "Sanity check");
103 }
104
105
106 LayoutRawBlock::LayoutRawBlock(int index, Kind kind, int size, int alignment) :
107 _next_block(nullptr),
108 _prev_block(nullptr),
109 _inline_klass(nullptr),
110 _block_kind(kind),
111 _offset(-1),
112 _alignment(alignment),
113 _size(size),
114 _field_index(index) {
115 assert(kind == REGULAR || kind == FLAT || kind == INHERITED,
116 "Other kind do not have a field index");
117 assert(size > 0, "Sanity check");
118 assert(alignment > 0, "Sanity check");
119 }
120
121 bool LayoutRawBlock::fit(int size, int alignment) {
122 int adjustment = 0;
123 if ((_offset % alignment) != 0) {
124 adjustment = alignment - (_offset % alignment);
125 }
126 return _size >= size + adjustment;
127 }
128
129 FieldGroup::FieldGroup(int contended_group) :
130 _next(nullptr),
131 _small_primitive_fields(nullptr),
132 _big_primitive_fields(nullptr),
133 _oop_fields(nullptr),
134 _contended_group(contended_group), // -1 means no contended group, 0 means default contended group
135 _oop_count(0) {}
136
137 void FieldGroup::add_primitive_field(int idx, BasicType type) {
138 int size = type2aelembytes(type);
139 LayoutRawBlock* block = new LayoutRawBlock(idx, LayoutRawBlock::REGULAR, size, size /* alignment == size for primitive types */);
140 if (size >= oopSize) {
141 add_to_big_primitive_list(block);
142 } else {
143 add_to_small_primitive_list(block);
144 }
145 }
146
147 void FieldGroup::add_oop_field(int idx) {
148 int size = type2aelembytes(T_OBJECT);
149 LayoutRawBlock* block = new LayoutRawBlock(idx, LayoutRawBlock::REGULAR, size, size /* alignment == size for oops */);
150 if (_oop_fields == nullptr) {
151 _oop_fields = new GrowableArray<LayoutRawBlock*>(INITIAL_LIST_SIZE);
152 }
153 _oop_fields->append(block);
154 _oop_count++;
155 }
156
157 void FieldGroup::add_flat_field(int idx, InlineKlass* vk, LayoutKind lk, int size, int alignment) {
158 LayoutRawBlock* block = new LayoutRawBlock(idx, LayoutRawBlock::FLAT, size, alignment);
159 block->set_inline_klass(vk);
160 block->set_layout_kind(lk);
161 if (block->size() >= oopSize) {
162 add_to_big_primitive_list(block);
163 } else {
164 add_to_small_primitive_list(block);
165 }
166 }
167
168 void FieldGroup::sort_by_size() {
169 if (_small_primitive_fields != nullptr) {
170 _small_primitive_fields->sort(LayoutRawBlock::compare_size_inverted);
171 }
172 if (_big_primitive_fields != nullptr) {
173 _big_primitive_fields->sort(LayoutRawBlock::compare_size_inverted);
174 }
175 }
176
177 void FieldGroup::add_to_small_primitive_list(LayoutRawBlock* block) {
178 if (_small_primitive_fields == nullptr) {
179 _small_primitive_fields = new GrowableArray<LayoutRawBlock*>(INITIAL_LIST_SIZE);
180 }
181 _small_primitive_fields->append(block);
182 }
183
184 void FieldGroup::add_to_big_primitive_list(LayoutRawBlock* block) {
185 if (_big_primitive_fields == nullptr) {
186 _big_primitive_fields = new GrowableArray<LayoutRawBlock*>(INITIAL_LIST_SIZE);
187 }
188 _big_primitive_fields->append(block);
189 }
190
191 FieldLayout::FieldLayout(GrowableArray<FieldInfo>* field_info, Array<InlineLayoutInfo>* inline_layout_info_array, ConstantPool* cp) :
192 _field_info(field_info),
193 _inline_layout_info_array(inline_layout_info_array),
194 _cp(cp),
195 _blocks(nullptr),
196 _start(_blocks),
197 _last(_blocks),
198 _super_first_field_offset(-1),
199 _super_alignment(-1),
200 _super_min_align_required(-1),
201 _default_value_offset(-1),
202 _null_reset_value_offset(-1),
203 _super_has_fields(false),
204 _has_inherited_fields(false) {}
205
206 void FieldLayout::initialize_static_layout() {
207 _blocks = new LayoutRawBlock(LayoutRawBlock::EMPTY, INT_MAX);
208 _blocks->set_offset(0);
209 _last = _blocks;
210 _start = _blocks;
211 // Note: at this stage, InstanceMirrorKlass::offset_of_static_fields() could be zero, because
212 // during bootstrapping, the size of the java.lang.Class is still not known when layout
213 // of static field is computed. Field offsets are fixed later when the size is known
214 // (see java_lang_Class::fixup_mirror())
215 if (InstanceMirrorKlass::offset_of_static_fields() > 0) {
216 insert(first_empty_block(), new LayoutRawBlock(LayoutRawBlock::RESERVED, InstanceMirrorKlass::offset_of_static_fields()));
217 _blocks->set_offset(0);
218 }
219 }
220
221 void FieldLayout::initialize_instance_layout(const InstanceKlass* super_klass) {
222 if (super_klass == nullptr) {
223 _blocks = new LayoutRawBlock(LayoutRawBlock::EMPTY, INT_MAX);
224 _blocks->set_offset(0);
225 _last = _blocks;
226 _start = _blocks;
227 insert(first_empty_block(), new LayoutRawBlock(LayoutRawBlock::RESERVED, instanceOopDesc::base_offset_in_bytes()));
228 } else {
229 _super_has_fields = reconstruct_layout(super_klass);
230 fill_holes(super_klass);
231 if ((!super_klass->has_contended_annotations()) || !_super_has_fields) {
232 _start = _blocks; // start allocating fields from the first empty block
233 } else {
234 _start = _last; // append fields at the end of the reconstructed layout
235 }
236 }
237 }
238
239 LayoutRawBlock* FieldLayout::first_field_block() {
240 LayoutRawBlock* block = _blocks;
241 while (block != nullptr
242 && block->block_kind() != LayoutRawBlock::INHERITED
243 && block->block_kind() != LayoutRawBlock::REGULAR
244 && block->block_kind() != LayoutRawBlock::FLAT
245 && block->block_kind() != LayoutRawBlock::NULL_MARKER) {
246 block = block->next_block();
247 }
248 return block;
249 }
250
251 // Insert a set of fields into a layout.
252 // For each field, search for an empty slot able to fit the field
253 // (satisfying both size and alignment requirements), if none is found,
254 // add the field at the end of the layout.
255 // Fields cannot be inserted before the block specified in the "start" argument
256 void FieldLayout::add(GrowableArray<LayoutRawBlock*>* list, LayoutRawBlock* start) {
257 if (list == nullptr) return;
258 if (start == nullptr) start = this->_start;
259 bool last_search_success = false;
260 int last_size = 0;
261 int last_alignment = 0;
262 for (int i = 0; i < list->length(); i ++) {
263 LayoutRawBlock* b = list->at(i);
264 LayoutRawBlock* cursor = nullptr;
265 LayoutRawBlock* candidate = nullptr;
266 // if start is the last block, just append the field
267 if (start == last_block()) {
268 candidate = last_block();
269 }
270 // Before iterating over the layout to find an empty slot fitting the field's requirements,
271 // check if the previous field had the same requirements and if the search for a fitting slot
272 // was successful. If the requirements were the same but the search failed, a new search will
273 // fail the same way, so just append the field at the of the layout.
274 else if (b->size() == last_size && b->alignment() == last_alignment && !last_search_success) {
275 candidate = last_block();
276 } else {
277 // Iterate over the layout to find an empty slot fitting the field's requirements
278 last_size = b->size();
279 last_alignment = b->alignment();
280 cursor = last_block()->prev_block();
281 assert(cursor != nullptr, "Sanity check");
282 last_search_success = true;
283
284 while (cursor != start) {
285 if (cursor->block_kind() == LayoutRawBlock::EMPTY && cursor->fit(b->size(), b->alignment())) {
286 if (candidate == nullptr || cursor->size() < candidate->size()) {
287 candidate = cursor;
288 }
289 }
290 cursor = cursor->prev_block();
291 }
292 if (candidate == nullptr) {
293 candidate = last_block();
294 last_search_success = false;
295 }
296 assert(candidate != nullptr, "Candidate must not be null");
297 assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
298 assert(candidate->fit(b->size(), b->alignment()), "Candidate must be able to store the block");
299 }
300 insert_field_block(candidate, b);
301 }
302 }
303
304 // Used for classes with hard coded field offsets, insert a field at the specified offset */
305 void FieldLayout::add_field_at_offset(LayoutRawBlock* block, int offset, LayoutRawBlock* start) {
306 assert(block != nullptr, "Sanity check");
307 block->set_offset(offset);
308 if (start == nullptr) {
309 start = this->_start;
310 }
311 LayoutRawBlock* slot = start;
312 while (slot != nullptr) {
313 if ((slot->offset() <= block->offset() && (slot->offset() + slot->size()) > block->offset()) ||
314 slot == _last){
315 assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Matching slot must be an empty slot");
316 assert(slot->size() >= block->offset() - slot->offset() + block->size() ,"Matching slot must be big enough");
317 if (slot->offset() < block->offset()) {
318 int adjustment = block->offset() - slot->offset();
319 LayoutRawBlock* adj = new LayoutRawBlock(LayoutRawBlock::EMPTY, adjustment);
320 insert(slot, adj);
321 }
322 insert(slot, block);
323 if (slot->size() == 0) {
324 remove(slot);
325 }
326 if (block->block_kind() == LayoutRawBlock::REGULAR || block->block_kind() == LayoutRawBlock::FLAT) {
327 _field_info->adr_at(block->field_index())->set_offset(block->offset());
328 }
329 return;
330 }
331 slot = slot->next_block();
332 }
333 fatal("Should have found a matching slot above, corrupted layout or invalid offset");
334 }
335
336 // The allocation logic uses a best fit strategy: the set of fields is allocated
337 // in the first empty slot big enough to contain the whole set ((including padding
338 // to fit alignment constraints).
339 void FieldLayout::add_contiguously(GrowableArray<LayoutRawBlock*>* list, LayoutRawBlock* start) {
340 if (list == nullptr) return;
341 if (start == nullptr) {
342 start = _start;
343 }
344 // This code assumes that if the first block is well aligned, the following
345 // blocks would naturally be well aligned (no need for adjustment)
346 int size = 0;
347 for (int i = 0; i < list->length(); i++) {
348 size += list->at(i)->size();
349 }
350
351 LayoutRawBlock* candidate = nullptr;
352 if (start == last_block()) {
353 candidate = last_block();
354 } else {
355 LayoutRawBlock* first = list->at(0);
356 candidate = last_block()->prev_block();
357 while (candidate->block_kind() != LayoutRawBlock::EMPTY || !candidate->fit(size, first->alignment())) {
358 if (candidate == start) {
359 candidate = last_block();
360 break;
361 }
362 candidate = candidate->prev_block();
363 }
364 assert(candidate != nullptr, "Candidate must not be null");
365 assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
366 assert(candidate->fit(size, first->alignment()), "Candidate must be able to store the whole contiguous block");
367 }
368
369 for (int i = 0; i < list->length(); i++) {
370 LayoutRawBlock* b = list->at(i);
371 insert_field_block(candidate, b);
372 assert((candidate->offset() % b->alignment() == 0), "Contiguous blocks must be naturally well aligned");
373 }
374 }
375
376 LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawBlock* block) {
377 assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
378 if (slot->offset() % block->alignment() != 0) {
379 int adjustment = block->alignment() - (slot->offset() % block->alignment());
380 LayoutRawBlock* adj = new LayoutRawBlock(LayoutRawBlock::EMPTY, adjustment);
381 insert(slot, adj);
382 }
383 assert(block->size() >= block->size(), "Enough space must remain after adjustment");
384 insert(slot, block);
385 if (slot->size() == 0) {
386 remove(slot);
387 }
388 // NULL_MARKER blocks are not real fields, so they don't have an entry in the FieldInfo array
389 if (block->block_kind() != LayoutRawBlock::NULL_MARKER) {
390 _field_info->adr_at(block->field_index())->set_offset(block->offset());
391 if (_field_info->adr_at(block->field_index())->name(_cp) == vmSymbols::default_value_name()) {
392 _default_value_offset = block->offset();
393 }
394 if (_field_info->adr_at(block->field_index())->name(_cp) == vmSymbols::null_reset_value_name()) {
395 _null_reset_value_offset = block->offset();
396 }
397 }
398 if (block->block_kind() == LayoutRawBlock::FLAT && block->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) {
399 int nm_offset = block->inline_klass()->null_marker_offset() - block->inline_klass()->first_field_offset() + block->offset();
400 _field_info->adr_at(block->field_index())->set_null_marker_offset(nm_offset);
401 _inline_layout_info_array->adr_at(block->field_index())->set_null_marker_offset(nm_offset);
402 }
403
404 return block;
405 }
406
407 bool FieldLayout::reconstruct_layout(const InstanceKlass* ik) {
408 bool has_instance_fields = false;
409 if (ik->is_abstract() && !ik->is_identity_class()) {
410 _super_alignment = type2aelembytes(BasicType::T_LONG);
411 }
412 GrowableArray<LayoutRawBlock*>* all_fields = new GrowableArray<LayoutRawBlock*>(32);
413 while (ik != nullptr) {
414 for (AllFieldStream fs(ik->fieldinfo_stream(), ik->constants()); !fs.done(); fs.next()) {
415 BasicType type = Signature::basic_type(fs.signature());
416 // distinction between static and non-static fields is missing
417 if (fs.access_flags().is_static()) continue;
418 has_instance_fields = true;
419 _has_inherited_fields = true;
420 if (_super_first_field_offset == -1 || fs.offset() < _super_first_field_offset) _super_first_field_offset = fs.offset();
421 LayoutRawBlock* block;
422 if (fs.is_flat()) {
423 InlineLayoutInfo layout_info = ik->inline_layout_info(fs.index());
424 InlineKlass* vk = layout_info.klass();
425 block = new LayoutRawBlock(fs.index(), LayoutRawBlock::INHERITED,
426 vk->layout_size_in_bytes(layout_info.kind()),
427 vk->layout_alignment(layout_info.kind()));
428 assert(_super_alignment == -1 || _super_alignment >= vk->payload_alignment(), "Invalid value alignment");
429 _super_min_align_required = _super_min_align_required > vk->payload_alignment() ? _super_min_align_required : vk->payload_alignment();
430 } else {
431 int size = type2aelembytes(type);
432 // INHERITED blocks are marked as non-reference because oop_maps are handled by their holder class
433 block = new LayoutRawBlock(fs.index(), LayoutRawBlock::INHERITED, size, size);
434 // For primitive types, the alignment is equal to the size
435 assert(_super_alignment == -1 || _super_alignment >= size, "Invalid value alignment");
436 _super_min_align_required = _super_min_align_required > size ? _super_min_align_required : size;
437 }
438 block->set_offset(fs.offset());
439 all_fields->append(block);
440 }
441 ik = ik->super() == nullptr ? nullptr : InstanceKlass::cast(ik->super());
442 }
443 all_fields->sort(LayoutRawBlock::compare_offset);
444 _blocks = new LayoutRawBlock(LayoutRawBlock::RESERVED, instanceOopDesc::base_offset_in_bytes());
445 _blocks->set_offset(0);
446 _last = _blocks;
447 for(int i = 0; i < all_fields->length(); i++) {
448 LayoutRawBlock* b = all_fields->at(i);
449 _last->set_next_block(b);
450 b->set_prev_block(_last);
451 _last = b;
452 }
453 _start = _blocks;
454 return has_instance_fields;
455 }
456
457 // Called during the reconstruction of a layout, after fields from super
458 // classes have been inserted. It fills unused slots between inserted fields
459 // with EMPTY blocks, so the regular field insertion methods would work.
460 // This method handles classes with @Contended annotations differently
461 // by inserting PADDING blocks instead of EMPTY block to prevent subclasses'
462 // fields to interfere with contended fields/classes.
463 void FieldLayout::fill_holes(const InstanceKlass* super_klass) {
464 assert(_blocks != nullptr, "Sanity check");
465 assert(_blocks->offset() == 0, "first block must be at offset zero");
466 LayoutRawBlock::Kind filling_type = super_klass->has_contended_annotations() ? LayoutRawBlock::PADDING: LayoutRawBlock::EMPTY;
467 LayoutRawBlock* b = _blocks;
468 while (b->next_block() != nullptr) {
469 if (b->next_block()->offset() > (b->offset() + b->size())) {
470 int size = b->next_block()->offset() - (b->offset() + b->size());
471 // FIXME it would be better if initial empty block where tagged as PADDING for value classes
472 LayoutRawBlock* empty = new LayoutRawBlock(filling_type, size);
473 empty->set_offset(b->offset() + b->size());
474 empty->set_next_block(b->next_block());
475 b->next_block()->set_prev_block(empty);
476 b->set_next_block(empty);
477 empty->set_prev_block(b);
478 }
479 b = b->next_block();
480 }
481 assert(b->next_block() == nullptr, "Invariant at this point");
482 assert(b->block_kind() != LayoutRawBlock::EMPTY, "Sanity check");
483 // If the super class has @Contended annotation, a padding block is
484 // inserted at the end to ensure that fields from the subclasses won't share
485 // the cache line of the last field of the contended class
486 if (super_klass->has_contended_annotations() && ContendedPaddingWidth > 0) {
487 LayoutRawBlock* p = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
488 p->set_offset(b->offset() + b->size());
489 b->set_next_block(p);
490 p->set_prev_block(b);
491 b = p;
492 }
493
494 LayoutRawBlock* last = new LayoutRawBlock(LayoutRawBlock::EMPTY, INT_MAX);
495 last->set_offset(b->offset() + b->size());
496 assert(last->offset() > 0, "Sanity check");
497 b->set_next_block(last);
498 last->set_prev_block(b);
499 _last = last;
500 }
501
502 LayoutRawBlock* FieldLayout::insert(LayoutRawBlock* slot, LayoutRawBlock* block) {
503 assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
504 assert(slot->offset() % block->alignment() == 0, "Incompatible alignment");
505 block->set_offset(slot->offset());
506 slot->set_offset(slot->offset() + block->size());
507 assert((slot->size() - block->size()) < slot->size(), "underflow checking");
508 assert(slot->size() - block->size() >= 0, "no negative size allowed");
509 slot->set_size(slot->size() - block->size());
510 block->set_prev_block(slot->prev_block());
511 block->set_next_block(slot);
512 slot->set_prev_block(block);
513 if (block->prev_block() != nullptr) {
514 block->prev_block()->set_next_block(block);
515 }
516 if (_blocks == slot) {
517 _blocks = block;
518 }
519 if (_start == slot) {
520 _start = block;
521 }
522 return block;
523 }
524
525 void FieldLayout::remove(LayoutRawBlock* block) {
526 assert(block != nullptr, "Sanity check");
527 assert(block != _last, "Sanity check");
528 if (_blocks == block) {
529 _blocks = block->next_block();
530 if (_blocks != nullptr) {
531 _blocks->set_prev_block(nullptr);
532 }
533 } else {
534 assert(block->prev_block() != nullptr, "_prev should be set for non-head blocks");
535 block->prev_block()->set_next_block(block->next_block());
536 block->next_block()->set_prev_block(block->prev_block());
537 }
538 if (block == _start) {
539 _start = block->prev_block();
540 }
541 }
542
543 void FieldLayout::shift_fields(int shift) {
544 LayoutRawBlock* b = first_field_block();
545 LayoutRawBlock* previous = b->prev_block();
546 if (previous->block_kind() == LayoutRawBlock::EMPTY) {
547 previous->set_size(previous->size() + shift);
548 } else {
549 LayoutRawBlock* nb = new LayoutRawBlock(LayoutRawBlock::PADDING, shift);
550 nb->set_offset(b->offset());
551 previous->set_next_block(nb);
552 nb->set_prev_block(previous);
553 b->set_prev_block(nb);
554 nb->set_next_block(b);
555 }
556 while (b != nullptr) {
557 b->set_offset(b->offset() + shift);
558 if (b->block_kind() == LayoutRawBlock::REGULAR || b->block_kind() == LayoutRawBlock::FLAT) {
559 _field_info->adr_at(b->field_index())->set_offset(b->offset());
560 if (b->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) {
561 int new_nm_offset = _field_info->adr_at(b->field_index())->null_marker_offset() + shift;
562 _field_info->adr_at(b->field_index())->set_null_marker_offset(new_nm_offset);
563 _inline_layout_info_array->adr_at(b->field_index())->set_null_marker_offset(new_nm_offset);
564
565 }
566 }
567 assert(b->block_kind() == LayoutRawBlock::EMPTY || b->offset() % b->alignment() == 0, "Must still be correctly aligned");
568 b = b->next_block();
569 }
570 }
571
572 LayoutRawBlock* FieldLayout::find_null_marker() {
573 LayoutRawBlock* b = _blocks;
574 while (b != nullptr) {
575 if (b->block_kind() == LayoutRawBlock::NULL_MARKER) {
576 return b;
577 }
578 b = b->next_block();
579 }
580 ShouldNotReachHere();
581 }
582
583 void FieldLayout::remove_null_marker() {
584 LayoutRawBlock* b = first_field_block();
585 while (b != nullptr) {
586 if (b->block_kind() == LayoutRawBlock::NULL_MARKER) {
587 if (b->next_block()->block_kind() == LayoutRawBlock::EMPTY) {
588 LayoutRawBlock* n = b->next_block();
589 remove(b);
590 n->set_offset(b->offset());
591 n->set_size(n->size() + b->size());
592 } else {
593 b->set_block_kind(LayoutRawBlock::EMPTY);
594 }
595 return;
596 }
597 b = b->next_block();
598 }
599 ShouldNotReachHere(); // if we reach this point, the null marker was not found!
600 }
601
602 static const char* layout_kind_to_string(LayoutKind lk) {
603 switch(lk) {
604 case LayoutKind::REFERENCE:
605 return "REFERENCE";
606 case LayoutKind::NON_ATOMIC_FLAT:
607 return "NON_ATOMIC_FLAT";
608 case LayoutKind::ATOMIC_FLAT:
609 return "ATOMIC_FLAT";
610 case LayoutKind::NULLABLE_ATOMIC_FLAT:
611 return "NULLABLE_ATOMIC_FLAT";
612 case LayoutKind::UNKNOWN:
613 return "UNKNOWN";
614 default:
615 ShouldNotReachHere();
616 }
617 }
618
619 void FieldLayout::print(outputStream* output, bool is_static, const InstanceKlass* super, Array<InlineLayoutInfo>* inline_fields) {
620 ResourceMark rm;
621 LayoutRawBlock* b = _blocks;
622 while(b != _last) {
623 switch(b->block_kind()) {
624 case LayoutRawBlock::REGULAR: {
625 FieldInfo* fi = _field_info->adr_at(b->field_index());
626 output->print_cr(" @%d %s %d/%d \"%s\" %s",
627 b->offset(),
628 "REGULAR",
629 b->size(),
630 b->alignment(),
631 fi->name(_cp)->as_C_string(),
632 fi->signature(_cp)->as_C_string());
633 break;
634 }
635 case LayoutRawBlock::FLAT: {
636 FieldInfo* fi = _field_info->adr_at(b->field_index());
637 InlineKlass* ik = inline_fields->adr_at(fi->index())->klass();
638 assert(ik != nullptr, "");
639 output->print_cr(" @%d %s %d/%d \"%s\" %s %s@%p %s",
640 b->offset(),
641 "FLAT",
642 b->size(),
643 b->alignment(),
644 fi->name(_cp)->as_C_string(),
645 fi->signature(_cp)->as_C_string(),
646 ik->name()->as_C_string(),
647 ik->class_loader_data(), layout_kind_to_string(b->layout_kind()));
648 break;
649 }
650 case LayoutRawBlock::RESERVED: {
651 output->print_cr(" @%d %s %d/-",
652 b->offset(),
653 "RESERVED",
654 b->size());
655 break;
656 }
657 case LayoutRawBlock::INHERITED: {
658 assert(!is_static, "Static fields are not inherited in layouts");
659 assert(super != nullptr, "super klass must be provided to retrieve inherited fields info");
660 bool found = false;
661 const InstanceKlass* ik = super;
662 while (!found && ik != nullptr) {
663 for (AllFieldStream fs(ik->fieldinfo_stream(), ik->constants()); !fs.done(); fs.next()) {
664 if (fs.offset() == b->offset() && fs.access_flags().is_static() == is_static) {
665 output->print_cr(" @%d %s %d/%d \"%s\" %s",
666 b->offset(),
667 "INHERITED",
668 b->size(),
669 b->size(), // so far, alignment constraint == size, will change with Valhalla => FIXME
670 fs.name()->as_C_string(),
671 fs.signature()->as_C_string());
672 found = true;
673 break;
674 }
675 }
676 ik = ik->java_super();
677 }
678 break;
679 }
680 case LayoutRawBlock::EMPTY:
681 output->print_cr(" @%d %s %d/1",
682 b->offset(),
683 "EMPTY",
684 b->size());
685 break;
686 case LayoutRawBlock::PADDING:
687 output->print_cr(" @%d %s %d/1",
688 b->offset(),
689 "PADDING",
690 b->size());
691 break;
692 case LayoutRawBlock::NULL_MARKER:
693 {
694 output->print_cr(" @%d %s %d/1 ",
695 b->offset(),
696 "NULL_MARKER",
697 b->size());
698 break;
699 }
700 default:
701 fatal("Unknown block type");
702 }
703 b = b->next_block();
704 }
705 }
706
707 FieldLayoutBuilder::FieldLayoutBuilder(const Symbol* classname, ClassLoaderData* loader_data, const InstanceKlass* super_klass, ConstantPool* constant_pool,
708 GrowableArray<FieldInfo>* field_info, bool is_contended, bool is_inline_type,bool is_abstract_value,
709 bool must_be_atomic, FieldLayoutInfo* info, Array<InlineLayoutInfo>* inline_layout_info_array) :
710 _classname(classname),
711 _loader_data(loader_data),
712 _super_klass(super_klass),
713 _constant_pool(constant_pool),
714 _field_info(field_info),
715 _info(info),
716 _inline_layout_info_array(inline_layout_info_array),
717 _root_group(nullptr),
718 _contended_groups(GrowableArray<FieldGroup*>(8)),
719 _static_fields(nullptr),
720 _layout(nullptr),
721 _static_layout(nullptr),
722 _nonstatic_oopmap_count(0),
723 _payload_alignment(-1),
724 _first_field_offset(-1),
725 _null_marker_offset(-1),
726 _payload_size_in_bytes(-1),
727 _non_atomic_layout_size_in_bytes(-1),
728 _non_atomic_layout_alignment(-1),
729 _atomic_layout_size_in_bytes(-1),
730 _nullable_layout_size_in_bytes(-1),
731 _fields_size_sum(0),
732 _declared_non_static_fields_count(0),
733 _has_non_naturally_atomic_fields(false),
734 _is_naturally_atomic(false),
735 _must_be_atomic(must_be_atomic),
736 _has_nonstatic_fields(false),
737 _has_inline_type_fields(false),
738 _is_contended(is_contended),
739 _is_inline_type(is_inline_type),
740 _is_abstract_value(is_abstract_value),
741 _has_flattening_information(is_inline_type),
742 _is_empty_inline_class(false) {}
743
744 FieldGroup* FieldLayoutBuilder::get_or_create_contended_group(int g) {
745 assert(g > 0, "must only be called for named contended groups");
746 FieldGroup* fg = nullptr;
747 for (int i = 0; i < _contended_groups.length(); i++) {
748 fg = _contended_groups.at(i);
749 if (fg->contended_group() == g) return fg;
750 }
751 fg = new FieldGroup(g);
752 _contended_groups.append(fg);
753 return fg;
754 }
755
756 void FieldLayoutBuilder::prologue() {
757 _layout = new FieldLayout(_field_info, _inline_layout_info_array, _constant_pool);
758 const InstanceKlass* super_klass = _super_klass;
759 _layout->initialize_instance_layout(super_klass);
760 _nonstatic_oopmap_count = super_klass == nullptr ? 0 : super_klass->nonstatic_oop_map_count();
761 if (super_klass != nullptr) {
762 _has_nonstatic_fields = super_klass->has_nonstatic_fields();
763 }
764 _static_layout = new FieldLayout(_field_info, _inline_layout_info_array, _constant_pool);
765 _static_layout->initialize_static_layout();
766 _static_fields = new FieldGroup();
767 _root_group = new FieldGroup();
768 }
769
770 // Field sorting for regular (non-inline) classes:
771 // - fields are sorted in static and non-static fields
772 // - non-static fields are also sorted according to their contention group
773 // (support of the @Contended annotation)
774 // - @Contended annotation is ignored for static fields
775 // - field flattening decisions are taken in this method
776 void FieldLayoutBuilder::regular_field_sorting() {
777 int idx = 0;
778 for (GrowableArrayIterator<FieldInfo> it = _field_info->begin(); it != _field_info->end(); ++it, ++idx) {
779 FieldGroup* group = nullptr;
780 FieldInfo fieldinfo = *it;
781 if (fieldinfo.access_flags().is_static()) {
782 group = _static_fields;
783 } else {
784 _has_nonstatic_fields = true;
785 if (fieldinfo.field_flags().is_contended()) {
786 int g = fieldinfo.contended_group();
787 if (g == 0) {
788 group = new FieldGroup(true);
789 _contended_groups.append(group);
790 } else {
791 group = get_or_create_contended_group(g);
792 }
793 } else {
794 group = _root_group;
795 }
796 }
797 assert(group != nullptr, "invariant");
798 BasicType type = Signature::basic_type(fieldinfo.signature(_constant_pool));
799 switch(type) {
800 case T_BYTE:
801 case T_CHAR:
802 case T_DOUBLE:
803 case T_FLOAT:
804 case T_INT:
805 case T_LONG:
806 case T_SHORT:
807 case T_BOOLEAN:
808 group->add_primitive_field(idx, type);
809 break;
810 case T_OBJECT:
811 case T_ARRAY:
812 {
813 LayoutKind lk = field_layout_selection(fieldinfo, _inline_layout_info_array);
814 if (fieldinfo.field_flags().is_null_free_inline_type() || lk != LayoutKind::REFERENCE
815 || (!fieldinfo.field_flags().is_injected()
816 && _inline_layout_info_array != nullptr && _inline_layout_info_array->adr_at(fieldinfo.index())->klass() != nullptr
817 && !_inline_layout_info_array->adr_at(fieldinfo.index())->klass()->is_identity_class())) {
818 _has_inline_type_fields = true;
819 _has_flattening_information = true;
820 }
821 if (lk == LayoutKind::REFERENCE) {
822 if (group != _static_fields) _nonstatic_oopmap_count++;
823 group->add_oop_field(idx);
824 } else {
825 _has_flattening_information = true;
826 InlineKlass* vk = _inline_layout_info_array->adr_at(fieldinfo.index())->klass();
827 int size, alignment;
828 get_size_and_alignment(vk, lk, &size, &alignment);
829 group->add_flat_field(idx, vk, lk, size, alignment);
830 _inline_layout_info_array->adr_at(fieldinfo.index())->set_kind(lk);
831 _nonstatic_oopmap_count += vk->nonstatic_oop_map_count();
832 _field_info->adr_at(idx)->field_flags_addr()->update_flat(true);
833 _field_info->adr_at(idx)->set_layout_kind(lk);
834 // no need to update _must_be_atomic if vk->must_be_atomic() is true because current class is not an inline class
835 }
836 break;
837 }
838 default:
839 fatal("Something wrong?");
840 }
841 }
842 _root_group->sort_by_size();
843 _static_fields->sort_by_size();
844 if (!_contended_groups.is_empty()) {
845 for (int i = 0; i < _contended_groups.length(); i++) {
846 _contended_groups.at(i)->sort_by_size();
847 }
848 }
849 }
850
851 /* Field sorting for inline classes:
852 * - because inline classes are immutable, the @Contended annotation is ignored
853 * when computing their layout (with only read operation, there's no false
854 * sharing issue)
855 * - this method also records the alignment of the field with the most
856 * constraining alignment, this value is then used as the alignment
857 * constraint when flattening this inline type into another container
858 * - field flattening decisions are taken in this method (those decisions are
859 * currently only based in the size of the fields to be flattened, the size
860 * of the resulting instance is not considered)
861 */
862 void FieldLayoutBuilder::inline_class_field_sorting() {
863 assert(_is_inline_type || _is_abstract_value, "Should only be used for inline classes");
864 int alignment = -1;
865 int idx = 0;
866 for (GrowableArrayIterator<FieldInfo> it = _field_info->begin(); it != _field_info->end(); ++it, ++idx) {
867 FieldGroup* group = nullptr;
868 FieldInfo fieldinfo = *it;
869 int field_alignment = 1;
870 if (fieldinfo.access_flags().is_static()) {
871 group = _static_fields;
872 } else {
873 _has_nonstatic_fields = true;
874 _declared_non_static_fields_count++;
875 group = _root_group;
876 }
877 assert(group != nullptr, "invariant");
878 BasicType type = Signature::basic_type(fieldinfo.signature(_constant_pool));
879 switch(type) {
880 case T_BYTE:
881 case T_CHAR:
882 case T_DOUBLE:
883 case T_FLOAT:
884 case T_INT:
885 case T_LONG:
886 case T_SHORT:
887 case T_BOOLEAN:
888 if (group != _static_fields) {
889 field_alignment = type2aelembytes(type); // alignment == size for primitive types
890 }
891 group->add_primitive_field(fieldinfo.index(), type);
892 break;
893 case T_OBJECT:
894 case T_ARRAY:
895 {
896 LayoutKind lk = field_layout_selection(fieldinfo, _inline_layout_info_array);
897 if (fieldinfo.field_flags().is_null_free_inline_type() || lk != LayoutKind::REFERENCE
898 || (!fieldinfo.field_flags().is_injected()
899 && _inline_layout_info_array != nullptr && _inline_layout_info_array->adr_at(fieldinfo.index())->klass() != nullptr
900 && !_inline_layout_info_array->adr_at(fieldinfo.index())->klass()->is_identity_class())) {
901 _has_inline_type_fields = true;
902 _has_flattening_information = true;
903 }
904 if (lk == LayoutKind::REFERENCE) {
905 if (group != _static_fields) {
906 _nonstatic_oopmap_count++;
907 field_alignment = type2aelembytes(type); // alignment == size for oops
908 }
909 group->add_oop_field(idx);
910 } else {
911 _has_flattening_information = true;
912 InlineKlass* vk = _inline_layout_info_array->adr_at(fieldinfo.index())->klass();
913 if (!vk->is_naturally_atomic()) _has_non_naturally_atomic_fields = true;
914 int size, alignment;
915 get_size_and_alignment(vk, lk, &size, &alignment);
916 group->add_flat_field(idx, vk, lk, size, alignment);
917 _inline_layout_info_array->adr_at(fieldinfo.index())->set_kind(lk);
918 _nonstatic_oopmap_count += vk->nonstatic_oop_map_count();
919 field_alignment = alignment;
920 _field_info->adr_at(idx)->field_flags_addr()->update_flat(true);
921 _field_info->adr_at(idx)->set_layout_kind(lk);
922 // default is atomic, but class file parsing could have set _must_be_atomic to false (@LooselyConsistentValue + checks)
923 // Presence of a must_be_atomic field must revert the _must_be_atomic flag of the holder to true
924 if (vk->must_be_atomic()) {
925 _must_be_atomic = true;
926 }
927 }
928 break;
929 }
930 default:
931 fatal("Unexpected BasicType");
932 }
933 if (!fieldinfo.access_flags().is_static() && field_alignment > alignment) alignment = field_alignment;
934 }
935 _payload_alignment = alignment;
936 assert(_has_nonstatic_fields || _is_abstract_value, "Concrete value types do not support zero instance size yet");
937 }
938
939 void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
940 if (ContendedPaddingWidth > 0) {
941 LayoutRawBlock* padding = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
942 _layout->insert(slot, padding);
943 }
944 }
945
946 /* Computation of regular classes layout is an evolution of the previous default layout
947 * (FieldAllocationStyle 1):
948 * - primitive fields (both primitive types and flat inline types) are allocated
949 * first, from the biggest to the smallest
950 * - then oop fields are allocated (to increase chances to have contiguous oops and
951 * a simpler oopmap).
952 */
953 void FieldLayoutBuilder::compute_regular_layout() {
954 bool need_tail_padding = false;
955 prologue();
956 regular_field_sorting();
957 if (_is_contended) {
958 _layout->set_start(_layout->last_block());
959 // insertion is currently easy because the current strategy doesn't try to fill holes
960 // in super classes layouts => the _start block is by consequence the _last_block
961 insert_contended_padding(_layout->start());
962 need_tail_padding = true;
963 }
964 _layout->add(_root_group->big_primitive_fields());
965 _layout->add(_root_group->small_primitive_fields());
966 _layout->add(_root_group->oop_fields());
967
968 if (!_contended_groups.is_empty()) {
969 for (int i = 0; i < _contended_groups.length(); i++) {
970 FieldGroup* cg = _contended_groups.at(i);
971 LayoutRawBlock* start = _layout->last_block();
972 insert_contended_padding(start);
973 _layout->add(cg->big_primitive_fields());
974 _layout->add(cg->small_primitive_fields(), start);
975 _layout->add(cg->oop_fields(), start);
976 need_tail_padding = true;
977 }
978 }
979
980 if (need_tail_padding) {
981 insert_contended_padding(_layout->last_block());
982 }
983
984 // Warning: IntanceMirrorKlass expects static oops to be allocated first
985 _static_layout->add_contiguously(_static_fields->oop_fields());
986 _static_layout->add(_static_fields->big_primitive_fields());
987 _static_layout->add(_static_fields->small_primitive_fields());
988
989 epilogue();
990 }
991
992 /* Computation of inline classes has a slightly different strategy than for
993 * regular classes. Regular classes have their oop fields allocated at the end
994 * of the layout to increase GC performances. Unfortunately, this strategy
995 * increases the number of empty slots inside an instance. Because the purpose
996 * of inline classes is to be embedded into other containers, it is critical
997 * to keep their size as small as possible. For this reason, the allocation
998 * strategy is:
999 * - big primitive fields (primitive types and flat inline type smaller
1000 * than an oop) are allocated first (from the biggest to the smallest)
1001 * - then oop fields
1002 * - then small primitive fields (from the biggest to the smallest)
1003 */
1004 void FieldLayoutBuilder::compute_inline_class_layout() {
1005
1006 // Test if the concrete inline class is an empty class (no instance fields)
1007 // and insert a dummy field if needed
1008 if (!_is_abstract_value) {
1009 bool declares_non_static_fields = false;
1010 for (GrowableArrayIterator<FieldInfo> it = _field_info->begin(); it != _field_info->end(); ++it) {
1011 FieldInfo fieldinfo = *it;
1012 if (!fieldinfo.access_flags().is_static()) {
1013 declares_non_static_fields = true;
1014 break;
1015 }
1016 }
1017 if (!declares_non_static_fields) {
1018 bool has_inherited_fields = false;
1019 const InstanceKlass* super = _super_klass;
1020 while(super != nullptr) {
1021 if (super->has_nonstatic_fields()) {
1022 has_inherited_fields = true;
1023 break;
1024 }
1025 super = super->super() == nullptr ? nullptr : InstanceKlass::cast(super->super());
1026 }
1027
1028 if (!has_inherited_fields) {
1029 // Inject ".empty" dummy field
1030 _is_empty_inline_class = true;
1031 FieldInfo::FieldFlags fflags(0);
1032 fflags.update_injected(true);
1033 AccessFlags aflags;
1034 FieldInfo fi(aflags,
1035 (u2)vmSymbols::as_int(VM_SYMBOL_ENUM_NAME(empty_marker_name)),
1036 (u2)vmSymbols::as_int(VM_SYMBOL_ENUM_NAME(byte_signature)),
1037 0,
1038 fflags);
1039 int idx = _field_info->append(fi);
1040 _field_info->adr_at(idx)->set_index(idx);
1041 }
1042 }
1043 }
1044
1045 prologue();
1046 inline_class_field_sorting();
1047
1048 assert(_layout->start()->block_kind() == LayoutRawBlock::RESERVED, "Unexpected");
1049
1050 if (_layout->super_has_fields() && !_is_abstract_value) { // non-static field layout
1051 if (!_has_nonstatic_fields) {
1052 assert(_is_abstract_value, "Concrete value types have at least one field");
1053 // Nothing to do
1054 } else {
1055 // decide which alignment to use, then set first allowed field offset
1056
1057 assert(_layout->super_alignment() >= _payload_alignment, "Incompatible alignment");
1058 assert(_layout->super_alignment() % _payload_alignment == 0, "Incompatible alignment");
1059
1060 if (_payload_alignment < _layout->super_alignment()) {
1061 int new_alignment = _payload_alignment > _layout->super_min_align_required() ? _payload_alignment : _layout->super_min_align_required();
1062 assert(new_alignment % _payload_alignment == 0, "Must be");
1063 assert(new_alignment % _layout->super_min_align_required() == 0, "Must be");
1064 _payload_alignment = new_alignment;
1065 }
1066 if (_layout->first_empty_block()->offset() < _layout->first_field_block()->offset()) {
1067 LayoutRawBlock* first_empty = _layout->start()->next_block();
1068 if (first_empty->offset() % _payload_alignment != 0) {
1069 int size = _payload_alignment - (first_empty->offset() % _payload_alignment);
1070 LayoutRawBlock* padding = new LayoutRawBlock(LayoutRawBlock::PADDING, size);
1071 _layout->insert(first_empty, padding);
1072 _layout->set_start(padding);
1073 } else {
1074 _layout->set_start( _layout->start());
1075 }
1076 } else {
1077 _layout->set_start(_layout->first_field_block());
1078 }
1079 }
1080 } else {
1081 if (_is_abstract_value && _has_nonstatic_fields) {
1082 _payload_alignment = type2aelembytes(BasicType::T_LONG);
1083 }
1084 assert(_layout->start()->next_block()->block_kind() == LayoutRawBlock::EMPTY || !UseCompressedClassPointers, "Unexpected");
1085 LayoutRawBlock* first_empty = _layout->start()->next_block();
1086 if (first_empty->offset() % _payload_alignment != 0) {
1087 LayoutRawBlock* padding = new LayoutRawBlock(LayoutRawBlock::PADDING, _payload_alignment - (first_empty->offset() % _payload_alignment));
1088 _layout->insert(first_empty, padding);
1089 if (first_empty->size() == 0) {
1090 _layout->remove(first_empty);
1091 }
1092 _layout->set_start(padding);
1093 }
1094 }
1095
1096 _layout->add(_root_group->big_primitive_fields());
1097 _layout->add(_root_group->oop_fields());
1098 _layout->add(_root_group->small_primitive_fields());
1099
1100 LayoutRawBlock* first_field = _layout->first_field_block();
1101 if (first_field != nullptr) {
1102 _first_field_offset = _layout->first_field_block()->offset();
1103 _payload_size_in_bytes = _layout->last_block()->offset() - _layout->first_field_block()->offset();
1104 } else {
1105 assert(_is_abstract_value, "Concrete inline types must have at least one field");
1106 _first_field_offset = _layout->blocks()->size();
1107 _payload_size_in_bytes = 0;
1108 }
1109
1110 // Determining if the value class is naturally atomic:
1111 if ((!_layout->super_has_fields() && _declared_non_static_fields_count <= 1 && !_has_non_naturally_atomic_fields)
1112 || (_layout->super_has_fields() && _super_klass->is_naturally_atomic() && _declared_non_static_fields_count == 0)) {
1113 _is_naturally_atomic = true;
1114 }
1115
1116 // At this point, the characteristics of the raw layout (used in standalone instances) are known.
1117 // From this, additional layouts will be computed: atomic and nullable layouts
1118 // Once those additional layouts are computed, the raw layout might need some adjustments
1119
1120 if (!_is_abstract_value) { // Flat layouts are only for concrete value classes
1121 // Validation of the non atomic layout
1122 if ((InlineFieldMaxFlatSize < 0 || _payload_size_in_bytes * BitsPerByte <= InlineFieldMaxFlatSize)
1123 && (!_must_be_atomic || _is_naturally_atomic)) {
1124 _non_atomic_layout_size_in_bytes = _payload_size_in_bytes;
1125 _non_atomic_layout_alignment = _payload_alignment;
1126 }
1127
1128 // Next step is to compute the characteristics for a layout enabling atomic updates
1129 if (AtomicFieldFlattening) {
1130 int atomic_size = _payload_size_in_bytes == 0 ? 0 : round_up_power_of_2(_payload_size_in_bytes);
1131 if ( atomic_size <= (int)MAX_ATOMIC_OP_SIZE
1132 && (InlineFieldMaxFlatSize < 0 || atomic_size * BitsPerByte <= InlineFieldMaxFlatSize)) {
1133 _atomic_layout_size_in_bytes = atomic_size;
1134 }
1135 }
1136
1137 // Next step is the nullable layout: the layout must include a null marker and must also be atomic
1138 if (NullableFieldFlattening) {
1139 // Looking if there's an empty slot inside the layout that could be used to store a null marker
1140 // FIXME: could it be possible to re-use the .empty field as a null marker for empty values?
1141 LayoutRawBlock* b = _layout->first_field_block();
1142 assert(b != nullptr, "A concrete value class must have at least one (possible dummy) field");
1143 int null_marker_offset = -1;
1144 if (_is_empty_inline_class) {
1145 // Reusing the dummy field as a field marker
1146 assert(_field_info->adr_at(b->field_index())->name(_constant_pool) == vmSymbols::empty_marker_name(), "b must be the dummy field");
1147 null_marker_offset = b->offset();
1148 } else {
1149 while (b != _layout->last_block()) {
1150 if (b->block_kind() == LayoutRawBlock::EMPTY) {
1151 break;
1152 }
1153 b = b->next_block();
1154 }
1155 if (b != _layout->last_block()) {
1156 // found an empty slot, register its offset from the beginning of the payload
1157 null_marker_offset = b->offset();
1158 LayoutRawBlock* marker = new LayoutRawBlock(LayoutRawBlock::NULL_MARKER, 1);
1159 _layout->add_field_at_offset(marker, b->offset());
1160 }
1161 if (null_marker_offset == -1) { // no empty slot available to store the null marker, need to inject one
1162 int last_offset = _layout->last_block()->offset();
1163 LayoutRawBlock* marker = new LayoutRawBlock(LayoutRawBlock::NULL_MARKER, 1);
1164 _layout->insert_field_block(_layout->last_block(), marker);
1165 assert(marker->offset() == last_offset, "Null marker should have been inserted at the end");
1166 null_marker_offset = marker->offset();
1167 }
1168 }
1169
1170 // Now that the null marker is there, the size of the nullable layout must computed (remember, must be atomic too)
1171 int new_raw_size = _layout->last_block()->offset() - _layout->first_field_block()->offset();
1172 int nullable_size = round_up_power_of_2(new_raw_size);
1173 if (nullable_size <= (int)MAX_ATOMIC_OP_SIZE
1174 && (InlineFieldMaxFlatSize < 0 || nullable_size * BitsPerByte <= InlineFieldMaxFlatSize)) {
1175 _nullable_layout_size_in_bytes = nullable_size;
1176 _null_marker_offset = null_marker_offset;
1177 } else {
1178 // If the nullable layout is rejected, the NULL_MARKER block should be removed
1179 // from the layout, otherwise it will appear anyway if the layout is printer
1180 _layout->remove_null_marker();
1181 _null_marker_offset = -1;
1182 }
1183 }
1184 // If the inline class has an atomic or nullable (which is also atomic) layout,
1185 // we want the raw layout to have the same alignment as those atomic layouts so access codes
1186 // could remain simple (single instruction without intermediate copy). This might required
1187 // to shift all fields in the raw layout, but this operation is possible only if the class
1188 // doesn't have inherited fields (offsets of inherited fields cannot be changed). If a
1189 // field shift is needed but not possible, all atomic layouts are disabled and only reference
1190 // and loosely consistent are supported.
1191 int required_alignment = _payload_alignment;
1192 if (has_atomic_layout() && _payload_alignment < atomic_layout_size_in_bytes()) {
1193 required_alignment = atomic_layout_size_in_bytes();
1194 }
1195 if (has_nullable_layout() && _payload_alignment < nullable_layout_size_in_bytes()) {
1196 required_alignment = nullable_layout_size_in_bytes();
1197 }
1198 int shift = first_field->offset() % required_alignment;
1199 if (shift != 0) {
1200 if (required_alignment > _payload_alignment && !_layout->has_inherited_fields()) {
1201 assert(_layout->first_field_block() != nullptr, "A concrete value class must have at least one (possible dummy) field");
1202 _layout->shift_fields(shift);
1203 _first_field_offset = _layout->first_field_block()->offset();
1204 if (has_nullable_layout()) {
1205 assert(!_is_empty_inline_class, "Should not get here with empty values");
1206 _null_marker_offset = _layout->find_null_marker()->offset();
1207 }
1208 _payload_alignment = required_alignment;
1209 } else {
1210 _atomic_layout_size_in_bytes = -1;
1211 if (has_nullable_layout() && !_is_empty_inline_class) { // empty values don't have a dedicated NULL_MARKER block
1212 _layout->remove_null_marker();
1213 }
1214 _nullable_layout_size_in_bytes = -1;
1215 _null_marker_offset = -1;
1216 }
1217 } else {
1218 _payload_alignment = required_alignment;
1219 }
1220
1221 // If the inline class has a nullable layout, the layout used in heap allocated standalone
1222 // instances must also be the nullable layout, in order to be able to set the null marker to
1223 // non-null before copying the payload to other containers.
1224 if (has_nullable_layout() && payload_layout_size_in_bytes() < nullable_layout_size_in_bytes()) {
1225 _payload_size_in_bytes = nullable_layout_size_in_bytes();
1226 }
1227 }
1228 // Warning:: InstanceMirrorKlass expects static oops to be allocated first
1229 _static_layout->add_contiguously(_static_fields->oop_fields());
1230 _static_layout->add(_static_fields->big_primitive_fields());
1231 _static_layout->add(_static_fields->small_primitive_fields());
1232
1233 epilogue();
1234 }
1235
1236 void FieldLayoutBuilder::add_flat_field_oopmap(OopMapBlocksBuilder* nonstatic_oop_maps,
1237 InlineKlass* vklass, int offset) {
1238 int diff = offset - vklass->first_field_offset();
1239 const OopMapBlock* map = vklass->start_of_nonstatic_oop_maps();
1240 const OopMapBlock* last_map = map + vklass->nonstatic_oop_map_count();
1241 while (map < last_map) {
1242 nonstatic_oop_maps->add(map->offset() + diff, map->count());
1243 map++;
1244 }
1245 }
1246
1247 void FieldLayoutBuilder::register_embedded_oops_from_list(OopMapBlocksBuilder* nonstatic_oop_maps, GrowableArray<LayoutRawBlock*>* list) {
1248 if (list == nullptr) return;
1249 for (int i = 0; i < list->length(); i++) {
1250 LayoutRawBlock* f = list->at(i);
1251 if (f->block_kind() == LayoutRawBlock::FLAT) {
1252 InlineKlass* vk = f->inline_klass();
1253 assert(vk != nullptr, "Should have been initialized");
1254 if (vk->contains_oops()) {
1255 add_flat_field_oopmap(nonstatic_oop_maps, vk, f->offset());
1256 }
1257 }
1258 }
1259 }
1260
1261 void FieldLayoutBuilder::register_embedded_oops(OopMapBlocksBuilder* nonstatic_oop_maps, FieldGroup* group) {
1262 if (group->oop_fields() != nullptr) {
1263 for (int i = 0; i < group->oop_fields()->length(); i++) {
1264 LayoutRawBlock* b = group->oop_fields()->at(i);
1265 nonstatic_oop_maps->add(b->offset(), 1);
1266 }
1267 }
1268 register_embedded_oops_from_list(nonstatic_oop_maps, group->big_primitive_fields());
1269 register_embedded_oops_from_list(nonstatic_oop_maps, group->small_primitive_fields());
1270 }
1271
1272 void FieldLayoutBuilder::epilogue() {
1273 // Computing oopmaps
1274 OopMapBlocksBuilder* nonstatic_oop_maps =
1275 new OopMapBlocksBuilder(_nonstatic_oopmap_count);
1276 int super_oop_map_count = (_super_klass == nullptr) ? 0 :_super_klass->nonstatic_oop_map_count();
1277 if (super_oop_map_count > 0) {
1278 nonstatic_oop_maps->initialize_inherited_blocks(_super_klass->start_of_nonstatic_oop_maps(),
1279 _super_klass->nonstatic_oop_map_count());
1280 }
1281 register_embedded_oops(nonstatic_oop_maps, _root_group);
1282 if (!_contended_groups.is_empty()) {
1283 for (int i = 0; i < _contended_groups.length(); i++) {
1284 FieldGroup* cg = _contended_groups.at(i);
1285 if (cg->oop_count() > 0) {
1286 assert(cg->oop_fields() != nullptr && cg->oop_fields()->at(0) != nullptr, "oop_count > 0 but no oop fields found");
1287 register_embedded_oops(nonstatic_oop_maps, cg);
1288 }
1289 }
1290 }
1291 nonstatic_oop_maps->compact();
1292
1293 int instance_end = align_up(_layout->last_block()->offset(), wordSize);
1294 int static_fields_end = align_up(_static_layout->last_block()->offset(), wordSize);
1295 int static_fields_size = (static_fields_end -
1296 InstanceMirrorKlass::offset_of_static_fields()) / wordSize;
1297 int nonstatic_field_end = align_up(_layout->last_block()->offset(), heapOopSize);
1298
1299 // Pass back information needed for InstanceKlass creation
1300
1301 _info->oop_map_blocks = nonstatic_oop_maps;
1302 _info->_instance_size = align_object_size(instance_end / wordSize);
1303 _info->_static_field_size = static_fields_size;
1304 _info->_nonstatic_field_size = (nonstatic_field_end - instanceOopDesc::base_offset_in_bytes()) / heapOopSize;
1305 _info->_has_nonstatic_fields = _has_nonstatic_fields;
1306 _info->_has_inline_fields = _has_inline_type_fields;
1307 _info->_is_naturally_atomic = _is_naturally_atomic;
1308 if (_is_inline_type) {
1309 _info->_must_be_atomic = _must_be_atomic;
1310 _info->_payload_alignment = _payload_alignment;
1311 _info->_first_field_offset = _first_field_offset;
1312 _info->_payload_size_in_bytes = _payload_size_in_bytes;
1313 _info->_non_atomic_size_in_bytes = _non_atomic_layout_size_in_bytes;
1314 _info->_non_atomic_alignment = _non_atomic_layout_alignment;
1315 _info->_atomic_layout_size_in_bytes = _atomic_layout_size_in_bytes;
1316 _info->_nullable_layout_size_in_bytes = _nullable_layout_size_in_bytes;
1317 _info->_null_marker_offset = _null_marker_offset;
1318 _info->_default_value_offset = _static_layout->default_value_offset();
1319 _info->_null_reset_value_offset = _static_layout->null_reset_value_offset();
1320 _info->_is_empty_inline_klass = _is_empty_inline_class;
1321 }
1322
1323 // This may be too restrictive, since if all the fields fit in 64
1324 // bits we could make the decision to align instances of this class
1325 // to 64-bit boundaries, and load and store them as single words.
1326 // And on machines which supported larger atomics we could similarly
1327 // allow larger values to be atomic, if properly aligned.
1328
1329 #ifdef ASSERT
1330 // Tests verifying integrity of field layouts are using the output of -XX:+PrintFieldLayout
1331 // which prints the details of LayoutRawBlocks used to compute the layout.
1332 // The code below checks that offsets in the _field_info meta-data match offsets
1333 // in the LayoutRawBlocks
1334 LayoutRawBlock* b = _layout->blocks();
1335 while(b != _layout->last_block()) {
1336 if (b->block_kind() == LayoutRawBlock::REGULAR || b->block_kind() == LayoutRawBlock::FLAT) {
1337 if (_field_info->adr_at(b->field_index())->offset() != (u4)b->offset()) {
1338 tty->print_cr("Offset from field info = %d, offset from block = %d", (int)_field_info->adr_at(b->field_index())->offset(), b->offset());
1339 }
1340 assert(_field_info->adr_at(b->field_index())->offset() == (u4)b->offset()," Must match");
1341 }
1342 b = b->next_block();
1343 }
1344 b = _static_layout->blocks();
1345 while(b != _static_layout->last_block()) {
1346 if (b->block_kind() == LayoutRawBlock::REGULAR || b->block_kind() == LayoutRawBlock::FLAT) {
1347 assert(_field_info->adr_at(b->field_index())->offset() == (u4)b->offset()," Must match");
1348 }
1349 b = b->next_block();
1350 }
1351 #endif // ASSERT
1352
1353 static bool first_layout_print = true;
1354
1355
1356 if (PrintFieldLayout || (PrintInlineLayout && _has_flattening_information)) {
1357 ResourceMark rm;
1358 stringStream st;
1359 if (first_layout_print) {
1360 st.print_cr("Field layout log format: @offset size/alignment [name] [signature] [comment]");
1361 st.print_cr("Heap oop size = %d", heapOopSize);
1362 first_layout_print = false;
1363 }
1364 if (_super_klass != nullptr) {
1365 st.print_cr("Layout of class %s@%p extends %s@%p", _classname->as_C_string(),
1366 _loader_data, _super_klass->name()->as_C_string(), _super_klass->class_loader_data());
1367 } else {
1368 st.print_cr("Layout of class %s@%p", _classname->as_C_string(), _loader_data);
1369 }
1370 st.print_cr("Instance fields:");
1371 _layout->print(&st, false, _super_klass, _inline_layout_info_array);
1372 st.print_cr("Static fields:");
1373 _static_layout->print(&st, true, nullptr, _inline_layout_info_array);
1374 st.print_cr("Instance size = %d bytes", _info->_instance_size * wordSize);
1375 if (_is_inline_type) {
1376 st.print_cr("First field offset = %d", _first_field_offset);
1377 st.print_cr("Payload layout: %d/%d", _payload_size_in_bytes, _payload_alignment);
1378 if (has_non_atomic_flat_layout()) {
1379 st.print_cr("Non atomic flat layout: %d/%d", _non_atomic_layout_size_in_bytes, _non_atomic_layout_alignment);
1380 } else {
1381 st.print_cr("Non atomic flat layout: -/-");
1382 }
1383 if (has_atomic_layout()) {
1384 st.print_cr("Atomic flat layout: %d/%d", _atomic_layout_size_in_bytes, _atomic_layout_size_in_bytes);
1385 } else {
1386 st.print_cr("Atomic flat layout: -/-");
1387 }
1388 if (has_nullable_layout()) {
1389 st.print_cr("Nullable flat layout: %d/%d", _nullable_layout_size_in_bytes, _nullable_layout_size_in_bytes);
1390 } else {
1391 st.print_cr("Nullable flat layout: -/-");
1392 }
1393 if (_null_marker_offset != -1) {
1394 st.print_cr("Null marker offset = %d", _null_marker_offset);
1395 }
1396 }
1397 st.print_cr("---");
1398 // Print output all together.
1399 tty->print_raw(st.as_string());
1400 }
1401 }
1402
1403 void FieldLayoutBuilder::build_layout() {
1404 if (_is_inline_type || _is_abstract_value) {
1405 compute_inline_class_layout();
1406 } else {
1407 compute_regular_layout();
1408 }
1409 }