1 /*
2 * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "cds/archiveUtils.hpp"
26 #include "cds/cdsConfig.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/collectedHeap.inline.hpp"
31 #include "gc/shared/gcLocker.inline.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "logging/log.hpp"
34 #include "memory/metadataFactory.hpp"
35 #include "memory/metaspaceClosure.hpp"
36 #include "oops/access.hpp"
37 #include "oops/arrayKlass.hpp"
38 #include "oops/compressedOops.inline.hpp"
39 #include "oops/fieldStreams.inline.hpp"
40 #include "oops/flatArrayKlass.hpp"
41 #include "oops/inlineKlass.inline.hpp"
42 #include "oops/instanceKlass.inline.hpp"
43 #include "oops/method.hpp"
44 #include "oops/objArrayKlass.hpp"
45 #include "oops/oop.inline.hpp"
46 #include "oops/refArrayKlass.hpp"
47 #include "runtime/fieldDescriptor.inline.hpp"
48 #include "runtime/handles.inline.hpp"
49 #include "runtime/interfaceSupport.inline.hpp"
50 #include "runtime/registerMap.hpp"
51 #include "runtime/safepointVerifiers.hpp"
52 #include "runtime/sharedRuntime.hpp"
53 #include "runtime/signature.hpp"
54 #include "runtime/thread.inline.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/stringUtils.hpp"
57
58 InlineKlass::Members::Members()
59 : _extended_sig(nullptr),
60 _return_regs(nullptr),
61 _pack_handler(nullptr),
62 _pack_handler_jobject(nullptr),
63 _unpack_handler(nullptr),
64 _null_reset_value_offset(0),
65 _payload_offset(-1),
66 _payload_size_in_bytes(-1),
67 _payload_alignment(-1),
68 _null_free_non_atomic_size_in_bytes(-1),
69 _null_free_non_atomic_alignment(-1),
70 _null_free_atomic_size_in_bytes(-1),
71 _nullable_atomic_size_in_bytes(-1),
72 _nullable_non_atomic_size_in_bytes(-1),
73 _null_marker_offset(-1) {
74 }
75
76 InlineKlass::InlineKlass() {
77 assert(CDSConfig::is_dumping_archive() || UseSharedSpaces, "only for CDS");
78 }
79
80 // Constructor
81 InlineKlass::InlineKlass(const ClassFileParser& parser)
82 : InstanceKlass(parser, InlineKlass::Kind, markWord::inline_type_prototype()) {
83 assert(is_inline_klass(), "sanity");
84 assert(prototype_header().is_inline_type(), "sanity");
85
86 // Set up the offset to the members of this klass
87 _adr_inline_klass_members = calculate_members_address();
88
89 // Placement install the members
90 new (_adr_inline_klass_members) Members();
91
92 // Sanity check construction of the members
93 assert(pack_handler() == nullptr, "pack handler not null");
94 }
95
96 address InlineKlass::calculate_members_address() const {
97 // The members are placed after all other contents inherited from the InstanceKlass
98 return end_of_instance_klass();
99 }
100
101 oop InlineKlass::null_reset_value() {
102 assert(is_initialized() || is_being_initialized() || is_in_error_state(), "null reset value is set at the beginning of initialization");
103 oop val = java_mirror()->obj_field_acquire(null_reset_value_offset());
104 assert(val != nullptr, "Sanity check");
105 return val;
106 }
107
108 void InlineKlass::set_null_reset_value(oop val) {
109 assert(val != nullptr, "Sanity check");
110 assert(oopDesc::is_oop(val), "Sanity check");
111 assert(val->is_inline_type(), "Sanity check");
112 assert(val->klass() == this, "sanity check");
113 java_mirror()->obj_field_put(null_reset_value_offset(), val);
114 }
115
116 instanceOop InlineKlass::allocate_instance(TRAPS) {
117 instanceOop oop = InstanceKlass::allocate_instance(CHECK_NULL);
118 assert(oop->mark().is_inline_type(), "Expected inline type");
119 return oop;
120 }
121
122 int InlineKlass::nonstatic_oop_count() {
123 int oops = 0;
124 int map_count = nonstatic_oop_map_count();
125 OopMapBlock* block = start_of_nonstatic_oop_maps();
126 OopMapBlock* end = block + map_count;
127 while (block != end) {
128 oops += block->count();
129 block++;
130 }
131 return oops;
132 }
133
134 int InlineKlass::layout_size_in_bytes(LayoutKind kind) const {
135 switch(kind) {
136 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT:
137 assert(has_null_free_non_atomic_layout(), "Layout not available");
138 return null_free_non_atomic_size_in_bytes();
139 break;
140 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
141 assert(has_null_free_atomic_layout(), "Layout not available");
142 return null_free_atomic_size_in_bytes();
143 break;
144 case LayoutKind::NULLABLE_ATOMIC_FLAT:
145 assert(has_nullable_atomic_layout(), "Layout not available");
146 return nullable_atomic_size_in_bytes();
147 break;
148 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
149 assert(has_nullable_non_atomic_layout(), "Layout not available");
150 return nullable_non_atomic_size_in_bytes();
151 break;
152 case LayoutKind::BUFFERED:
153 return payload_size_in_bytes();
154 break;
155 default:
156 ShouldNotReachHere();
157 }
158 }
159
160 int InlineKlass::layout_alignment(LayoutKind kind) const {
161 switch(kind) {
162 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT:
163 assert(has_null_free_non_atomic_layout(), "Layout not available");
164 return null_free_non_atomic_alignment();
165 break;
166 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
167 assert(has_null_free_atomic_layout(), "Layout not available");
168 return null_free_atomic_size_in_bytes();
169 break;
170 case LayoutKind::NULLABLE_ATOMIC_FLAT:
171 assert(has_nullable_atomic_layout(), "Layout not available");
172 return nullable_atomic_size_in_bytes();
173 break;
174 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
175 assert(has_nullable_non_atomic_layout(), "Layout not available");
176 return null_free_non_atomic_alignment();
177 break;
178 case LayoutKind::BUFFERED:
179 return payload_alignment();
180 break;
181 default:
182 ShouldNotReachHere();
183 }
184 }
185
186 bool InlineKlass::is_layout_supported(LayoutKind lk) {
187 switch(lk) {
188 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT:
189 return has_null_free_non_atomic_layout();
190 break;
191 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
192 return has_null_free_atomic_layout();
193 break;
194 case LayoutKind::NULLABLE_ATOMIC_FLAT:
195 return has_nullable_atomic_layout();
196 break;
197 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
198 return has_nullable_non_atomic_layout();
199 break;
200 case LayoutKind::BUFFERED:
201 return true;
202 break;
203 default:
204 ShouldNotReachHere();
205 }
206 }
207
208 void InlineKlass::copy_payload_to_addr(void* src, void* dst, LayoutKind lk) {
209 assert(is_layout_supported(lk), "Unsupported layout");
210 assert(lk != LayoutKind::REFERENCE && lk != LayoutKind::UNKNOWN, "Sanity check");
211 switch(lk) {
212 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
213 case LayoutKind::NULLABLE_ATOMIC_FLAT: {
214 if (is_payload_marked_as_null((address)src)) {
215 // copy null_reset value to dest
216 HeapAccess<>::value_copy(payload_addr(null_reset_value()), dst, this, lk);
217 } else {
218 // Copy has to be performed, even if this is an empty value, because of the null marker
219 HeapAccess<>::value_copy(src, dst, this, lk);
220 }
221 }
222 break;
223 case LayoutKind::BUFFERED:
224 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
225 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
226 if (is_empty_inline_type()) return; // nothing to do
227 HeapAccess<>::value_copy(src, dst, this, lk);
228 }
229 break;
230 default:
231 ShouldNotReachHere();
232 }
233 }
234
235 oop InlineKlass::read_payload_from_addr(const oop src, size_t offset, LayoutKind lk, TRAPS) {
236 assert(src != nullptr, "Must be");
237 assert(is_layout_supported(lk), "Unsupported layout");
238 switch(lk) {
239 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
240 case LayoutKind::NULLABLE_ATOMIC_FLAT: {
241 if (is_payload_marked_as_null(cast_from_oop<address>(src) + offset)) {
242 return nullptr;
243 }
244 } // Fallthrough
245 case LayoutKind::BUFFERED:
246 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
247 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
248 Handle obj_h(THREAD, src);
249 ZERO_ONLY(ThreadInVMfromJava tivmj(THREAD);) // Zero enters here from C++ intepreter
250 oop res = allocate_instance(CHECK_NULL);
251 copy_payload_to_addr((void*)(cast_from_oop<address>(obj_h()) + offset), payload_addr(res), lk);
252
253 // After copying, re-check if the payload is now marked as null. Another
254 // thread could have marked the src object as null after the initial check
255 // but before the copy operation, causing the null-marker to be marked in
256 // the destination. In this case, discard the allocated object and
257 // return nullptr.
258 if (LayoutKindHelper::is_nullable_flat(lk)) {
259 if (is_payload_marked_as_null(payload_addr(res))) {
260 return nullptr;
261 }
262 }
263
264 return res;
265 }
266 break;
267 default:
268 ShouldNotReachHere();
269 }
270 }
271
272 void InlineKlass::write_value_to_addr(oop src, void* dst, LayoutKind lk, TRAPS) {
273 void* src_addr = nullptr;
274 if (src == nullptr) {
275 if (!LayoutKindHelper::is_nullable_flat(lk)) {
276 THROW_MSG(vmSymbols::java_lang_NullPointerException(), "Value is null");
277 }
278 // Writing null to a nullable flat field/element is usually done by writing
279 // the whole pre-allocated null_reset_value at the payload address to ensure
280 // that the null marker and all potential oops are reset to "zeros".
281 // However, the null_reset_value is allocated during class initialization.
282 // If the current value of the field is null, it is possible that the class
283 // of the field has not been initialized yet and thus the null_reset_value
284 // might not be available yet.
285 // Writing null over an already null value should not trigger class initialization.
286 // The solution is to detect null being written over null cases and return immediately
287 // (writing null over null is a no-op from a field modification point of view)
288 if (is_payload_marked_as_null((address)dst)) return;
289 src_addr = payload_addr(null_reset_value());
290 } else {
291 src_addr = payload_addr(src);
292 if (LayoutKindHelper::is_nullable_flat(lk)) {
293 mark_payload_as_non_null((address)src_addr);
294 }
295 }
296 copy_payload_to_addr(src_addr, dst, lk);
297 }
298
299 // Arrays of...
300
301 bool InlineKlass::maybe_flat_in_array() {
302 if (!UseArrayFlattening) {
303 return false;
304 }
305 // Too many embedded oops
306 if ((FlatArrayElementMaxOops >= 0) && (nonstatic_oop_count() > FlatArrayElementMaxOops)) {
307 return false;
308 }
309 // No flat layout?
310 if (!has_nullable_atomic_layout() && !has_null_free_atomic_layout() && !has_null_free_non_atomic_layout()) {
311 return false;
312 }
313 return true;
314 }
315
316 bool InlineKlass::is_always_flat_in_array() {
317 if (!UseArrayFlattening) {
318 return false;
319 }
320 // Too many embedded oops
321 if ((FlatArrayElementMaxOops >= 0) && (nonstatic_oop_count() > FlatArrayElementMaxOops)) {
322 return false;
323 }
324
325 // An instance is always flat in an array if we have all layouts. Note that this could change in the future when the
326 // flattening policies are updated or if new APIs are added that allow the creation of reference arrays directly.
327 return has_nullable_atomic_layout() && has_null_free_atomic_layout() && has_null_free_non_atomic_layout();
328 }
329
330 // Inline type arguments are not passed by reference, instead each
331 // field of the inline type is passed as an argument. This helper
332 // function collects the flat field (recursively)
333 // in a list. Included with the field's type is
334 // the offset of each field in the inline type: i2c and c2i adapters
335 // need that to load or store fields. Finally, the list of fields is
336 // sorted in order of increasing offsets: the adapters and the
337 // compiled code need to agree upon the order of fields.
338 //
339 // The list of basic types that is returned starts with a T_METADATA
340 // and ends with an extra T_VOID. T_METADATA/T_VOID pairs are used as
341 // delimiters. Every entry between the two is a field of the inline
342 // type. If there's an embedded inline type in the list, it also starts
343 // with a T_METADATA and ends with a T_VOID. This is so we can
344 // generate a unique fingerprint for the method's adapters and we can
345 // generate the list of basic types from the interpreter point of view
346 // (inline types passed as reference: iterate on the list until a
347 // T_METADATA, drop everything until and including the closing
348 // T_VOID) or the compiler point of view (each field of the inline
349 // types is an argument: drop all T_METADATA/T_VOID from the list).
350 //
351 // Value classes could also have fields in abstract super value classes.
352 // Use a HierarchicalFieldStream to get them as well.
353 int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, int base_off, int null_marker_offset) {
354 int count = 0;
355 SigEntry::add_entry(sig, T_METADATA, name(), base_off);
356 for (TopDownHierarchicalNonStaticFieldStreamBase fs(this); !fs.done(); fs.next()) {
357 assert(!fs.access_flags().is_static(), "TopDownHierarchicalNonStaticFieldStreamBase should not let static fields pass.");
358 int offset = base_off + fs.offset() - (base_off > 0 ? payload_offset() : 0);
359 InstanceKlass* field_holder = fs.field_descriptor().field_holder();
360 // TODO 8284443 Use different heuristic to decide what should be scalarized in the calling convention
361 if (fs.is_flat()) {
362 // Resolve klass of flat field and recursively collect fields
363 int field_null_marker_offset = -1;
364 if (!fs.is_null_free_inline_type()) {
365 field_null_marker_offset = base_off + fs.null_marker_offset() - (base_off > 0 ? payload_offset() : 0);
366 }
367 Klass* vk = field_holder->get_inline_type_field_klass(fs.index());
368 count += InlineKlass::cast(vk)->collect_fields(sig, offset, field_null_marker_offset);
369 } else {
370 BasicType bt = Signature::basic_type(fs.signature());
371 SigEntry::add_entry(sig, bt, fs.name(), offset);
372 count += type2size[bt];
373 }
374 }
375 int offset = base_off + size_helper()*HeapWordSize - (base_off > 0 ? payload_offset() : 0);
376 // Null markers are no real fields, add them manually at the end (C2 relies on this) of the flat fields
377 if (null_marker_offset != -1) {
378 SigEntry::add_null_marker(sig, name(), null_marker_offset);
379 count++;
380 }
381 SigEntry::add_entry(sig, T_VOID, name(), offset);
382 assert(sig->at(0)._bt == T_METADATA && sig->at(sig->length()-1)._bt == T_VOID, "broken structure");
383 return count;
384 }
385
386 void InlineKlass::initialize_calling_convention(TRAPS) {
387 // Because the pack and unpack handler addresses need to be loadable from generated code,
388 // they are stored at a fixed offset in the klass metadata. Since inline type klasses do
389 // not have a vtable, the vtable offset is used to store these addresses.
390 if (InlineTypeReturnedAsFields || InlineTypePassFieldsAsArgs) {
391 ResourceMark rm;
392 GrowableArray<SigEntry> sig_vk;
393 int nb_fields = collect_fields(&sig_vk);
394 if (*PrintInlineKlassFields != '\0') {
395 const char* class_name_str = _name->as_C_string();
396 if (StringUtils::class_list_match(PrintInlineKlassFields, class_name_str)) {
397 ttyLocker ttyl;
398 tty->print_cr("Fields of InlineKlass: %s", class_name_str);
399 for (const SigEntry& entry : sig_vk) {
400 tty->print(" %s: %s+%d", entry._name->as_C_string(), type2name(entry._bt), entry._offset);
401 if (entry._null_marker) {
402 tty->print(" (null marker)");
403 }
404 tty->print_cr("");
405 }
406 }
407 }
408 Array<SigEntry>* extended_sig = MetadataFactory::new_array<SigEntry>(class_loader_data(), sig_vk.length(), CHECK);
409 set_extended_sig(extended_sig);
410 for (int i = 0; i < sig_vk.length(); i++) {
411 extended_sig->at_put(i, sig_vk.at(i));
412 }
413 if (can_be_returned_as_fields(/* init= */ true)) {
414 nb_fields++;
415 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nb_fields);
416 sig_bt[0] = T_METADATA;
417 SigEntry::fill_sig_bt(&sig_vk, sig_bt+1);
418 VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, nb_fields);
419 int total = SharedRuntime::java_return_convention(sig_bt, regs, nb_fields);
420
421 if (total > 0) {
422 Array<VMRegPair>* return_regs = MetadataFactory::new_array<VMRegPair>(class_loader_data(), nb_fields, CHECK);
423 set_return_regs(return_regs);
424 for (int i = 0; i < nb_fields; i++) {
425 return_regs->at_put(i, regs[i]);
426 }
427
428 BufferedInlineTypeBlob* buffered_blob = SharedRuntime::generate_buffered_inline_type_adapter(this);
429 if (buffered_blob == nullptr) {
430 THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "Out of space in CodeCache for adapters");
431 }
432 set_pack_handler(buffered_blob->pack_fields());
433 set_pack_handler_jobject(buffered_blob->pack_fields_jobject());
434 set_unpack_handler(buffered_blob->unpack_fields());
435 assert(CodeCache::find_blob(pack_handler()) == buffered_blob, "lost track of blob");
436 assert(can_be_returned_as_fields(), "sanity");
437 }
438 }
439 if (!can_be_returned_as_fields() && !can_be_passed_as_fields()) {
440 MetadataFactory::free_array<SigEntry>(class_loader_data(), extended_sig);
441 assert(return_regs() == nullptr, "sanity");
442 }
443 }
444 }
445
446 void InlineKlass::deallocate_contents(ClassLoaderData* loader_data) {
447 if (extended_sig() != nullptr) {
448 MetadataFactory::free_array<SigEntry>(loader_data, members()._extended_sig);
449 set_extended_sig(nullptr);
450 }
451 if (return_regs() != nullptr) {
452 MetadataFactory::free_array<VMRegPair>(loader_data, members()._return_regs);
453 set_return_regs(nullptr);
454 }
455 cleanup_blobs();
456 InstanceKlass::deallocate_contents(loader_data);
457 }
458
459 void InlineKlass::cleanup(InlineKlass* ik) {
460 ik->cleanup_blobs();
461 }
462
463 void InlineKlass::cleanup_blobs() {
464 if (pack_handler() != nullptr) {
465 CodeBlob* buffered_blob = CodeCache::find_blob(pack_handler());
466 assert(buffered_blob->is_buffered_inline_type_blob(), "bad blob type");
467 BufferBlob::free((BufferBlob*)buffered_blob);
468 set_pack_handler(nullptr);
469 set_pack_handler_jobject(nullptr);
470 set_unpack_handler(nullptr);
471 }
472 }
473
474 // Can this inline type be passed as multiple values?
475 bool InlineKlass::can_be_passed_as_fields() const {
476 return InlineTypePassFieldsAsArgs;
477 }
478
479 // Can this inline type be returned as multiple values?
480 bool InlineKlass::can_be_returned_as_fields(bool init) const {
481 return InlineTypeReturnedAsFields && (init || return_regs() != nullptr);
482 }
483
484 // Create handles for all oop fields returned in registers that are going to be live across a safepoint
485 void InlineKlass::save_oop_fields(const RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
486 Thread* thread = Thread::current();
487 const Array<SigEntry>* sig_vk = extended_sig();
488 const Array<VMRegPair>* regs = return_regs();
489 int j = 1;
490
491 for (int i = 0; i < sig_vk->length(); i++) {
492 BasicType bt = sig_vk->at(i)._bt;
493 if (bt == T_OBJECT || bt == T_ARRAY) {
494 VMRegPair pair = regs->at(j);
495 address loc = reg_map.location(pair.first(), nullptr);
496 oop o = *(oop*)loc;
497 assert(oopDesc::is_oop_or_null(o), "Bad oop value: " PTR_FORMAT, p2i(o));
498 handles.push(Handle(thread, o));
499 }
500 if (bt == T_METADATA) {
501 continue;
502 }
503 if (bt == T_VOID &&
504 sig_vk->at(i-1)._bt != T_LONG &&
505 sig_vk->at(i-1)._bt != T_DOUBLE) {
506 continue;
507 }
508 j++;
509 }
510 assert(j == regs->length(), "missed a field?");
511 }
512
513 // Update oop fields in registers from handles after a safepoint
514 void InlineKlass::restore_oop_results(RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
515 assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
516 const Array<SigEntry>* sig_vk = extended_sig();
517 const Array<VMRegPair>* regs = return_regs();
518 assert(regs != nullptr, "inconsistent");
519
520 int j = 1;
521 int k = 0;
522 for (int i = 0; i < sig_vk->length(); i++) {
523 BasicType bt = sig_vk->at(i)._bt;
524 if (bt == T_OBJECT || bt == T_ARRAY) {
525 VMRegPair pair = regs->at(j);
526 address loc = reg_map.location(pair.first(), nullptr);
527 *(oop*)loc = handles.at(k++)();
528 }
529 if (bt == T_METADATA) {
530 continue;
531 }
532 if (bt == T_VOID &&
533 sig_vk->at(i-1)._bt != T_LONG &&
534 sig_vk->at(i-1)._bt != T_DOUBLE) {
535 continue;
536 }
537 j++;
538 }
539 assert(k == handles.length(), "missed a handle?");
540 assert(j == regs->length(), "missed a field?");
541 }
542
543 // Fields are in registers. Create an instance of the inline type and
544 // initialize it with the values of the fields.
545 oop InlineKlass::realloc_result(const RegisterMap& reg_map, const GrowableArray<Handle>& handles, TRAPS) {
546 oop new_vt = allocate_instance(CHECK_NULL);
547 const Array<SigEntry>* sig_vk = extended_sig();
548 const Array<VMRegPair>* regs = return_regs();
549
550 int j = 1;
551 int k = 0;
552 for (int i = 0; i < sig_vk->length(); i++) {
553 BasicType bt = sig_vk->at(i)._bt;
554 if (bt == T_METADATA) {
555 continue;
556 }
557 if (bt == T_VOID) {
558 if (sig_vk->at(i-1)._bt == T_LONG ||
559 sig_vk->at(i-1)._bt == T_DOUBLE) {
560 j++;
561 }
562 continue;
563 }
564 int off = sig_vk->at(i)._offset;
565 assert(off > 0, "offset in object should be positive");
566 VMRegPair pair = regs->at(j);
567 address loc = reg_map.location(pair.first(), nullptr);
568 switch(bt) {
569 case T_BOOLEAN: {
570 new_vt->bool_field_put(off, *(jboolean*)loc);
571 break;
572 }
573 case T_CHAR: {
574 new_vt->char_field_put(off, *(jchar*)loc);
575 break;
576 }
577 case T_BYTE: {
578 new_vt->byte_field_put(off, *(jbyte*)loc);
579 break;
580 }
581 case T_SHORT: {
582 new_vt->short_field_put(off, *(jshort*)loc);
583 break;
584 }
585 case T_INT: {
586 new_vt->int_field_put(off, *(jint*)loc);
587 break;
588 }
589 case T_LONG: {
590 #ifdef _LP64
591 new_vt->double_field_put(off, *(jdouble*)loc);
592 #else
593 Unimplemented();
594 #endif
595 break;
596 }
597 case T_OBJECT:
598 case T_ARRAY: {
599 Handle handle = handles.at(k++);
600 new_vt->obj_field_put(off, handle());
601 break;
602 }
603 case T_FLOAT: {
604 new_vt->float_field_put(off, *(jfloat*)loc);
605 break;
606 }
607 case T_DOUBLE: {
608 new_vt->double_field_put(off, *(jdouble*)loc);
609 break;
610 }
611 default:
612 ShouldNotReachHere();
613 }
614 *(intptr_t*)loc = 0xDEAD;
615 j++;
616 }
617 assert(j == regs->length(), "missed a field?");
618 assert(k == handles.length(), "missed an oop?");
619 return new_vt;
620 }
621
622 // Check if we return an inline type in scalarized form, i.e. check if either
623 // - The return value is a tagged InlineKlass pointer, or
624 // - The return value is an inline type oop that is also returned in scalarized form
625 InlineKlass* InlineKlass::returned_inline_klass(const RegisterMap& map, bool* return_oop, Method* method) {
626 BasicType bt = T_METADATA;
627 VMRegPair pair;
628 int nb = SharedRuntime::java_return_convention(&bt, &pair, 1);
629 assert(nb == 1, "broken");
630
631 address loc = map.location(pair.first(), nullptr);
632 intptr_t ptr = *(intptr_t*)loc;
633 if (is_set_nth_bit(ptr, 0)) {
634 // Return value is tagged, must be an InlineKlass pointer
635 clear_nth_bit(ptr, 0);
636 assert(Metaspace::contains((void*)ptr), "should be klass");
637 InlineKlass* vk = (InlineKlass*)ptr;
638 assert(vk->can_be_returned_as_fields(), "must be able to return as fields");
639 if (return_oop != nullptr) {
640 // Not returning an oop
641 *return_oop = false;
642 }
643 return vk;
644 }
645 // Return value is not tagged, must be a valid oop
646 oop o = cast_to_oop(ptr);
647 assert(oopDesc::is_oop_or_null(o), "Bad oop return: " PTR_FORMAT, ptr);
648 if (return_oop != nullptr && o != nullptr && o->is_inline_type()) {
649 // Check if inline type is also returned in scalarized form
650 InlineKlass* vk_val = InlineKlass::cast(o->klass());
651 InlineKlass* vk_sig = method->returns_inline_type();
652 if (vk_val->can_be_returned_as_fields() && vk_sig != nullptr) {
653 assert(vk_val == vk_sig, "Unexpected return value");
654 return vk_val;
655 }
656 }
657 return nullptr;
658 }
659
660 // CDS support
661 #if INCLUDE_CDS
662
663 void InlineKlass::remove_unshareable_info() {
664 InstanceKlass::remove_unshareable_info();
665
666 // update it to point to the "buffered" copy of this class.
667 _adr_inline_klass_members = calculate_members_address();
668 ArchivePtrMarker::mark_pointer(&_adr_inline_klass_members);
669
670 set_extended_sig(nullptr);
671 set_return_regs(nullptr);
672 set_pack_handler(nullptr);
673 set_pack_handler_jobject(nullptr);
674 set_unpack_handler(nullptr);
675
676 assert(pack_handler() == nullptr, "pack handler not null");
677 }
678
679 #endif // CDS
680
681 // Verification
682
683 void InlineKlass::verify_on(outputStream* st) {
684 InstanceKlass::verify_on(st);
685 guarantee(prototype_header().is_inline_type(), "Prototype header is not inline type");
686 }
687
688 void InlineKlass::oop_verify_on(oop obj, outputStream* st) {
689 InstanceKlass::oop_verify_on(obj, st);
690 guarantee(obj->mark().is_inline_type(), "Header is not inline type");
691 }