1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP
26 #define SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP
27
28 #include "oops/valuePayload.hpp"
29
30 #include "cppstdlib/type_traits.hpp"
31 #include "oops/flatArrayKlass.inline.hpp"
32 #include "oops/flatArrayOop.inline.hpp"
33 #include "oops/inlineKlass.inline.hpp"
34 #include "oops/layoutKind.hpp"
35 #include "oops/oopHandle.inline.hpp"
36 #include "oops/oopsHierarchy.hpp"
37 #include "oops/resolvedFieldEntry.hpp"
38 #include "runtime/fieldDescriptor.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/javaThread.inline.hpp"
41 #include "utilities/debug.hpp"
42 #include "utilities/ostream.hpp"
43 #include "utilities/vmError.hpp"
44
45 template <typename OopOrHandle>
46 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl()
47 : _container(nullptr),
48 _offset(BAD_OFFSET),
49 _klass(nullptr),
50 _layout_kind(LayoutKind::UNKNOWN),
51 _uses_absolute_addr(false) {}
52
53 template <typename OopOrHandle>
54 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(OopOrHandle container,
55 ptrdiff_t offset,
56 InlineKlass* klass,
57 LayoutKind layout_kind)
58 : _container(container),
59 _offset(offset),
60 _klass(klass),
61 _layout_kind(layout_kind),
62 _uses_absolute_addr(false) {}
63
64 template <typename OopOrHandle>
65 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(address absolute_addr,
66 InlineKlass* klass,
67 LayoutKind layout_kind)
68 : _absolute_addr(absolute_addr),
69 _klass(klass),
70 _layout_kind(layout_kind),
71 _uses_absolute_addr(true) {}
72
73 template <typename OopOrHandle>
74 inline ValuePayload::StorageImpl<OopOrHandle>::~StorageImpl() {
75 #ifdef CHECK_UNHANDLED_OOPS
76 if (!_uses_absolute_addr) {
77 _container.~OopOrHandle();
78 }
79 #else // CHECK_UNHANDLED_OOPS
80 static_assert(std::is_trivially_destructible_v<OopOrHandle>);
81 #endif // CHECK_UNHANDLED_OOPS
82 }
83
84 template <typename OopOrHandle>
85 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(const StorageImpl& other)
86 : _klass(other._klass),
87 _layout_kind(other._layout_kind),
88 _uses_absolute_addr(other._uses_absolute_addr) {
89 if (_uses_absolute_addr) {
90 _absolute_addr = other._absolute_addr;
91 } else {
92 _container = other._container;
93 _offset = other._offset;
94 }
95 }
96
97 template <typename OopOrHandle>
98 inline ValuePayload::StorageImpl<OopOrHandle>&
99 ValuePayload::StorageImpl<OopOrHandle>::operator=(const StorageImpl& other) {
100 if (&other != this) {
101 _klass = other._klass;
102 _layout_kind = other._layout_kind;
103 _uses_absolute_addr = other._uses_absolute_addr;
104 if (_uses_absolute_addr) {
105 _absolute_addr = other._absolute_addr;
106 } else {
107 _container = other._container;
108 _offset = other._offset;
109 }
110 }
111 return *this;
112 }
113
114 template <typename OopOrHandle>
115 inline OopOrHandle& ValuePayload::StorageImpl<OopOrHandle>::container() {
116 precond(!_uses_absolute_addr);
117 return _container;
118 }
119
120 template <typename OopOrHandle>
121 inline OopOrHandle ValuePayload::StorageImpl<OopOrHandle>::container() const {
122 precond(!_uses_absolute_addr);
123 return _container;
124 }
125
126 template <typename OopOrHandle>
127 inline ptrdiff_t& ValuePayload::StorageImpl<OopOrHandle>::offset() {
128 precond(!_uses_absolute_addr);
129 return _offset;
130 }
131
132 template <typename OopOrHandle>
133 inline ptrdiff_t ValuePayload::StorageImpl<OopOrHandle>::offset() const {
134 precond(!_uses_absolute_addr);
135 return _offset;
136 }
137
138 template <typename OopOrHandle>
139 inline address& ValuePayload::StorageImpl<OopOrHandle>::absolute_addr() {
140 precond(_uses_absolute_addr);
141 return _absolute_addr;
142 }
143
144 template <typename OopOrHandle>
145 inline address ValuePayload::StorageImpl<OopOrHandle>::absolute_addr() const {
146 precond(_uses_absolute_addr);
147 return _absolute_addr;
148
149 }
150
151 template <typename OopOrHandle>
152 inline InlineKlass* ValuePayload::StorageImpl<OopOrHandle>::klass() const {
153 return _klass;
154 }
155
156 template <typename OopOrHandle>
157 inline LayoutKind ValuePayload::StorageImpl<OopOrHandle>::layout_kind() const {
158 return _layout_kind;
159 }
160
161 template <typename OopOrHandle>
162 inline bool ValuePayload::StorageImpl<OopOrHandle>::uses_absolute_addr() const {
163 return _uses_absolute_addr;
164 }
165
166 inline ValuePayload::ValuePayload(oop container,
167 ptrdiff_t offset,
168 InlineKlass* klass,
169 LayoutKind layout_kind)
170 : _storage{container, offset, klass, layout_kind} {
171 assert_post_construction_invariants();
172 }
173
174 inline ValuePayload::ValuePayload(address absolute_addr,
175 InlineKlass* klass,
176 LayoutKind layout_kind)
177 : _storage{absolute_addr, klass, layout_kind} {
178 assert_post_construction_invariants();
179 }
180
181 inline void ValuePayload::set_offset(ptrdiff_t offset) {
182 _storage.offset() = offset;
183 }
184
185 inline void ValuePayload::copy(const ValuePayload& src,
186 const ValuePayload& dst,
187 LayoutKind copy_layout_kind) {
188 assert_pre_copy_invariants(src, dst, copy_layout_kind);
189
190 InlineKlass* const klass = src.klass();
191
192 switch (copy_layout_kind) {
193 case LayoutKind::NULLABLE_ATOMIC_FLAT:
194 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT: {
195 if (src.is_payload_null()) {
196 HeapAccess<>::value_store_null(dst);
197 } else {
198 HeapAccess<>::value_copy(src, dst);
199 }
200 } break;
201 case LayoutKind::BUFFERED:
202 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
203 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
204 if (!klass->is_empty_inline_type()) {
205 HeapAccess<>::value_copy(src, dst);
206 }
207 } break;
208 default:
209 ShouldNotReachHere();
210 }
211 }
212
213 inline void ValuePayload::mark_as_non_null() {
214 precond(has_null_marker());
215 klass()->mark_payload_as_non_null(addr());
216 }
217
218 inline void ValuePayload::mark_as_null() {
219 precond(has_null_marker());
220 klass()->mark_payload_as_null(addr());
221 }
222
223 inline bool ValuePayload::uses_absolute_addr() const {
224 return _storage.uses_absolute_addr();
225 }
226
227 inline oop& ValuePayload::container() {
228 return _storage.container();
229 }
230
231 inline oop ValuePayload::container() const {
232 return _storage.container();
233 }
234
235 #ifdef ASSERT
236
237 inline void ValuePayload::print_on(outputStream* st) const {
238 if (uses_absolute_addr()) {
239 st->print_cr("--- absolute_addr ---");
240 StreamIndentor si(st);
241 st->print_cr("_absolute_addr: " PTR_FORMAT, p2i(_storage.absolute_addr()));
242 } else {
243 {
244 oop container = _storage.container();
245 st->print_cr("--- container ---");
246 StreamIndentor si(st);
247 st->print_cr("_container: " PTR_FORMAT, p2i(container));
248 if (container != nullptr) {
249 container->print_on(st);
250 st->cr();
251 }
252 }
253 {
254 st->print_cr("--- offset ---");
255 StreamIndentor si(st);
256 st->print_cr("_offset: %zd", _storage.offset());
257 }
258 }
259 {
260 InlineKlass* const klass = _storage.klass();
261 st->print_cr("--- klass ---");
262 StreamIndentor si(st);
263 st->print_cr("_klass: " PTR_FORMAT, p2i(klass));
264 if (klass != nullptr) {
265 klass->print_on(st);
266 st->cr();
267 }
268 }
269 {
270 const LayoutKind layout_kind = _storage.layout_kind();
271 st->print_cr("--- layout_kind ---");
272 StreamIndentor si(st);
273 st->print_cr("_layout_kind: %u", (uint32_t)layout_kind);
274 LayoutKindHelper::print_on(layout_kind, st);
275 st->cr();
276 }
277 }
278
279 inline void ValuePayload::assert_is_flat_field(const InstanceKlass* klass, int offset) const {
280 OnVMError on_assertion_failure_field_descriptor([&](outputStream* st) {
281 st->print_cr("=== assert_is_flat_field(" PTR_FORMAT ", %d) failure ===", p2i(klass), offset);
282 StreamIndentor si(st);
283 if (klass != nullptr) {
284 klass->print_on(st);
285 st->cr();
286 }
287 });
288
289 fieldDescriptor field_descriptor;
290 postcond(klass->find_flat_field_containing_offset(offset, &field_descriptor));
291
292 const InlineLayoutInfo inline_layout_info = field_descriptor.field_holder()->inline_layout_info(field_descriptor.index());
293
294 OnVMError on_assertion_failure_inline_layout_info([&](outputStream* st) {
295 st->print_cr("=== assert_is_flat_field(" PTR_FORMAT ", %d) failure ===", p2i(klass), offset);
296 StreamIndentor si(st);
297 st->print("field_descriptor: ");
298 field_descriptor.print_on(st);
299 st->cr();
300 st->print("inline_layout_info: ");
301 inline_layout_info.print_on(st);
302 st->cr();
303 });
304
305 if (inline_layout_info.klass() == this->klass()) {
306 // Found the field in klass
307 postcond(offset == field_descriptor.offset());
308 postcond(inline_layout_info.kind() == layout_kind());
309 postcond(field_descriptor.layout_kind() == layout_kind());
310 postcond(field_descriptor.is_flat());
311 } else {
312 // Nested flat field
313 postcond(offset >= field_descriptor.offset());
314 const InlineKlass* const field_klass = inline_layout_info.klass();
315 const int payload_offset = field_klass->payload_offset();
316 assert_is_flat_field(field_klass, offset - field_descriptor.offset() + payload_offset);
317 }
318 }
319
320 inline void ValuePayload::assert_post_construction_invariants() const {
321 OnVMError on_assertion_failure([&](outputStream* st) {
322 st->print_cr("=== assert_post_construction_invariants failure ===");
323 StreamIndentor si(st);
324 print_on(st);
325 st->cr();
326 });
327
328 postcond(layout_kind() != LayoutKind::REFERENCE);
329 postcond(layout_kind() != LayoutKind::UNKNOWN);
330 postcond(klass()->is_layout_supported(layout_kind()));
331
332 if (!uses_absolute_addr()) {
333 postcond(container() != nullptr);
334 const Klass* const container_klass = container()->klass();
335 if (container_klass == klass()) {
336 postcond(layout_kind() == LayoutKind::BUFFERED);
337 } else {
338 postcond(layout_kind() != LayoutKind::BUFFERED);
339 if (container_klass->is_mirror_instance_klass()) {
340 fatal("java.lang.Class has no flat fields. Static fields are not flattened");
341 } else if (container_klass->is_instance_klass()) {
342 assert_is_flat_field(InstanceKlass::cast(container_klass), checked_cast<int>(offset()));
343 } else {
344 const FlatArrayKlass* const container_flat_array_klass = FlatArrayKlass::cast(container_klass);
345 if (container_flat_array_klass->element_klass() == klass()) {
346 postcond(container_flat_array_klass->layout_kind() == layout_kind());
347 } else {
348 // Accessing nested flat field
349 const InlineKlass* const element_klass = container_flat_array_klass->element_klass();
350 const int element_offset =
351 (checked_cast<int>(this->offset()) -
352 checked_cast<int>(flatArrayOopDesc::base_offset_in_bytes())) %
353 container_flat_array_klass->element_byte_size();
354 const int payload_offset = element_klass->payload_offset();
355 assert_is_flat_field(element_klass, element_offset + payload_offset);
356 }
357 }
358 }
359 }
360 }
361
362 inline void ValuePayload::assert_pre_copy_invariants(const ValuePayload& src,
363 const ValuePayload& dst,
364 LayoutKind copy_layout_kind) {
365 OnVMError on_assertion_failuire([&](outputStream* st) {
366 st->print_cr("=== assert_post_construction_invariants failure ===");
367 StreamIndentor si(st);
368 {
369 st->print_cr("--- src payload ---");
370 StreamIndentor si(st);
371 src.print_on(st);
372 st->cr();
373 }
374 {
375 st->print_cr("--- dst payload ---");
376 StreamIndentor si(st);
377 dst.print_on(st);
378 st->cr();
379 }
380 {
381 st->print_cr("--- copy layout kind ---");
382 StreamIndentor si(st);
383 LayoutKindHelper::print_on(copy_layout_kind, st);
384 st->cr();
385 }
386 });
387
388 const InlineKlass* const src_klass = src.klass();
389 const InlineKlass* const dst_klass = dst.klass();
390
391 precond(src_klass == dst_klass);
392
393 const bool src_is_buffered = src.layout_kind() == LayoutKind::BUFFERED;
394 const bool dst_is_buffered = dst.layout_kind() == LayoutKind::BUFFERED;
395 const bool src_and_dst_same_layout_kind = src.layout_kind() == dst.layout_kind();
396 const bool src_has_copy_layout = src.layout_kind() == copy_layout_kind;
397 const bool dst_has_copy_layout = dst.layout_kind() == copy_layout_kind;
398
399 precond(src_is_buffered || dst_is_buffered || src_and_dst_same_layout_kind);
400 precond(src_has_copy_layout || dst_has_copy_layout);
401
402 if (src_is_buffered) {
403 oop container = src.uses_absolute_addr()
404 ? cast_to_oop(src.addr() - src_klass->payload_offset())
405 : src.container();
406
407 precond(container != src_klass->null_reset_value());
408 }
409
410 const int src_layout_size_in_bytes = src_klass->layout_size_in_bytes(src.layout_kind());
411 const int dst_layout_size_in_bytes = dst_klass->layout_size_in_bytes(dst.layout_kind());
412 const int copy_layout_size_in_bytes =
413 src_has_copy_layout
414 ? src_layout_size_in_bytes
415 : dst_layout_size_in_bytes;
416
417 precond(copy_layout_size_in_bytes <= src_layout_size_in_bytes);
418 precond(copy_layout_size_in_bytes <= dst_layout_size_in_bytes);
419 precond(LayoutKindHelper::get_copy_layout(src.layout_kind(),
420 dst.layout_kind()) == copy_layout_kind);
421 }
422
423 #endif // ASSERT
424
425 inline InlineKlass* ValuePayload::klass() const {
426 return _storage.klass();
427 }
428
429 inline ptrdiff_t ValuePayload::offset() const {
430 precond(_storage.offset() != BAD_OFFSET);
431 return _storage.offset();
432 }
433
434 inline LayoutKind ValuePayload::layout_kind() const {
435 return _storage.layout_kind();
436 }
437
438 inline address ValuePayload::addr() const {
439 return uses_absolute_addr()
440 ? _storage.absolute_addr()
441 : (cast_from_oop<address>(container()) + offset());
442 }
443
444 inline bool ValuePayload::has_null_marker() const {
445 return klass()->layout_has_null_marker(layout_kind());
446 }
447
448 inline bool ValuePayload::is_payload_null() const {
449 return has_null_marker() && klass()->is_payload_marked_as_null(addr());
450 }
451
452 inline ValuePayload ValuePayload::construct_from_parts(address absolute_addr,
453 InlineKlass* klass,
454 LayoutKind layout_kind) {
455 return ValuePayload(absolute_addr, klass, layout_kind);
456 }
457
458 inline BufferedValuePayload::BufferedValuePayload(inlineOop container,
459 ptrdiff_t offset,
460 InlineKlass* klass,
461 LayoutKind layout_kind)
462 : ValuePayload(container, offset, klass, layout_kind) {}
463
464 inline BufferedValuePayload::BufferedValuePayload(inlineOop buffer)
465 : BufferedValuePayload(buffer, InlineKlass::cast(buffer->klass())) {}
466
467 inline BufferedValuePayload::BufferedValuePayload(inlineOop buffer,
468 InlineKlass* klass)
469 : ValuePayload(buffer, klass->payload_offset(), klass, LayoutKind::BUFFERED) {}
470
471 inline inlineOop BufferedValuePayload::container() const {
472 return inlineOop(ValuePayload::container());
473 }
474
475 inline void BufferedValuePayload::copy_to(const BufferedValuePayload& dst) {
476 copy(*this, dst, LayoutKind::BUFFERED);
477 }
478
479 inline FlatValuePayload::FlatValuePayload(oop container,
480 ptrdiff_t offset,
481 InlineKlass* klass,
482 LayoutKind layout_kind)
483 : ValuePayload(container, offset, klass, layout_kind) {}
484
485 inline inlineOop FlatValuePayload::allocate_instance(TRAPS) {
486 // Preserve the container oop across the instance allocation.
487 oop& container = this->container();
488 ::Handle container_handle(THREAD, container);
489 inlineOop res = klass()->allocate_instance(THREAD);
490 container = container_handle();
491 return res;
492 }
493
494 inline bool FlatValuePayload::copy_to(BufferedValuePayload& dst) {
495 // Copy from FLAT to BUFFERED, null marker fix may be required.
496
497 // Copy the payload to the buffered object.
498 copy(*this, dst, layout_kind());
499
500 if (!has_null_marker() && dst.has_null_marker()) {
501 // We must fix the null marker if the src does not have a null marker but
502 // the buffered object does.
503 dst.mark_as_non_null();
504
505 // The buffered object was just marked non null.
506 return true;
507 }
508
509 if (dst.is_payload_null()) {
510 // A null payload is not a valid payload for a buffered value.
511 return false;
512 }
513
514 return true;
515 }
516
517 inline void FlatValuePayload::copy_from(BufferedValuePayload& src) {
518 // Copy from BUFFERED to FLAT, null marker fix may be required.
519
520 if (has_null_marker()) {
521 // The FLAT payload has a null mark. So make sure that buffered is marked as
522 // non null. It is the callers responsibility to ensure that this is a
523 // valid non null value.
524 src.mark_as_non_null();
525 }
526 copy(src, *this, layout_kind());
527 }
528
529 inline void FlatValuePayload::copy_to(const FlatValuePayload& dst) {
530 copy(*this, dst, layout_kind());
531 }
532
533 inline inlineOop FlatValuePayload::read(TRAPS) {
534 switch (layout_kind()) {
535 case LayoutKind::NULLABLE_ATOMIC_FLAT:
536 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT: {
537 if (is_payload_null()) {
538 return nullptr;
539 }
540 } // Fallthrough
541 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
542 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
543 inlineOop res = allocate_instance(CHECK_NULL);
544 BufferedValuePayload dst(res, klass());
545 if (!copy_to(dst)) {
546 // copy_to may fail if the payload has been updated with a null value
547 // between our is_payload_null() check above and the copy.
548 // In this case we have copied a null value into the buffer the payload.
549 return nullptr;
550 }
551 // Must ensure the content of the buffered value is visible
552 // before publishing the buffered value oop
553 OrderAccess::storestore();
554 return res;
555 } break;
556 default:
557 ShouldNotReachHere();
558 }
559 }
560
561 inline void FlatValuePayload::write_without_nullability_check(inlineOop obj) {
562 if (obj == nullptr) {
563 assert(has_null_marker(), "Payload must support null values");
564 HeapAccess<>::value_store_null(*this);
565 } else {
566 // Copy the obj payload
567 BufferedValuePayload obj_payload(obj);
568 copy_from(obj_payload);
569 }
570 }
571
572 inline void FlatValuePayload::write(inlineOop obj, TRAPS) {
573 if (obj == nullptr && !has_null_marker()) {
574 // This payload does not have a null marker and cannot represent a null
575 // value.
576 THROW_MSG(vmSymbols::java_lang_NullPointerException(), "Value is null");
577 }
578 write_without_nullability_check(obj);
579 }
580
581 inline FlatValuePayload FlatValuePayload::construct_from_parts(oop container,
582 ptrdiff_t offset,
583 InlineKlass* klass,
584 LayoutKind layout_kind) {
585 return FlatValuePayload(container, offset, klass, layout_kind);
586 }
587
588 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
589 ptrdiff_t offset,
590 InlineKlass* klass,
591 LayoutKind layout_kind)
592 : FlatValuePayload(container, offset, klass, layout_kind) {}
593
594 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
595 ptrdiff_t offset,
596 InlineLayoutInfo* inline_layout_info)
597 : FlatValuePayload(container, offset, inline_layout_info->klass(), inline_layout_info->kind()) {}
598
599 #ifdef ASSERT
600
601 inline void FlatFieldPayload::assert_post_construction_invariants(instanceOop container,
602 ResolvedFieldEntry* resolved_field_entry,
603 InstanceKlass* klass) const {
604 OnVMError on_assertion_failure([&](outputStream* st) {
605 st->print_cr("=== assert_post_construction_invariants failure ===");
606 StreamIndentor si(st);
607 print_on(st);
608 st->cr();
609 });
610
611 postcond(container->klass()->is_subclass_of(klass));
612 postcond(klass == resolved_field_entry->field_holder());
613 postcond(resolved_field_entry->is_flat());
614 }
615
616 inline void FlatFieldPayload::assert_post_construction_invariants(instanceOop container,
617 fieldDescriptor* field_descriptor,
618 InstanceKlass* klass) const {
619 OnVMError on_assertion_failure([&](outputStream* st) {
620 st->print_cr("=== assert_post_construction_invariants failure ===");
621 StreamIndentor si(st);
622 print_on(st);
623 st->cr();
624 });
625
626 postcond(container->klass()->is_subclass_of(klass));
627 postcond(klass == field_descriptor->field_holder());
628 postcond(field_descriptor->is_flat());
629 }
630
631 #endif // ASSERT
632
633 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
634 fieldDescriptor* field_descriptor)
635 : FlatFieldPayload(container, field_descriptor,
636 InstanceKlass::cast(container->klass())) {}
637
638 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
639 fieldDescriptor* field_descriptor,
640 InstanceKlass* klass)
641 : FlatFieldPayload(container,
642 field_descriptor->offset(),
643 klass->inline_layout_info_adr(field_descriptor->index())) {
644 assert_post_construction_invariants(container, field_descriptor, klass);
645 }
646
647 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
648 ResolvedFieldEntry* resolved_field_entry)
649 : FlatFieldPayload(container,
650 resolved_field_entry,
651 resolved_field_entry->field_holder()) {}
652
653 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
654 ResolvedFieldEntry* resolved_field_entry,
655 InstanceKlass* klass)
656 : FlatFieldPayload(container,
657 resolved_field_entry->field_offset(),
658 klass->inline_layout_info_adr(resolved_field_entry->field_index())) {
659 assert_post_construction_invariants(container, resolved_field_entry, klass);
660 }
661
662 inline instanceOop FlatFieldPayload::container() const {
663 return instanceOop(ValuePayload::container());
664 }
665
666 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container,
667 ptrdiff_t offset,
668 InlineKlass* klass,
669 LayoutKind layout_kind,
670 jint layout_helper,
671 int element_size)
672 : FlatValuePayload(container, offset, klass, layout_kind),
673 _storage{layout_helper, element_size} {}
674
675 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container)
676 : FlatArrayPayload(container, container->klass()) {}
677
678 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container, FlatArrayKlass* klass)
679 : FlatArrayPayload(container,
680 BAD_OFFSET,
681 klass->element_klass(),
682 klass->layout_kind(),
683 klass->layout_helper(),
684 klass->element_byte_size()) {
685 postcond(container->klass() == klass);
686 }
687
688 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container, int index)
689 : FlatArrayPayload(container, index, container->klass()) {}
690
691 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container,
692 int index,
693 FlatArrayKlass* klass)
694 : FlatArrayPayload(container,
695 (ptrdiff_t)container->value_offset(index, klass->layout_helper()),
696 klass->element_klass(),
697 klass->layout_kind(),
698 klass->layout_helper(),
699 klass->element_byte_size()) {
700 postcond(container->klass() == klass);
701 }
702
703 inline flatArrayOop FlatArrayPayload::container() const {
704 return flatArrayOop(ValuePayload::container());
705 }
706
707 inline void FlatArrayPayload::set_index(int index) {
708 set_offset((ptrdiff_t)container()->value_offset(index, _storage._layout_helper));
709 }
710
711 inline void FlatArrayPayload::advance_index(int delta) {
712 set_offset(this->offset() + delta * _storage._element_size);
713 }
714
715 inline void FlatArrayPayload::next_element() {
716 advance_index(1);
717 }
718
719 inline void FlatArrayPayload::previous_element() {
720 advance_index(-1);
721 }
722
723 inline void FlatArrayPayload::set_offset(ptrdiff_t offset) {
724 #if defined(ASSERT) && defined(_LP64)
725 // For ease of use as iterators we allow the offset to point one element size
726 // beyond the first and last element. If there are no elements only the base
727 // offset is allowed. However we treat these as terminal states, and set the
728 // offset to a BAD_OFFSET in debug builds.
729
730 const ptrdiff_t element_size = _storage._element_size;
731 const ptrdiff_t length = container()->length();
732 const ptrdiff_t base_offset = (ptrdiff_t)flatArrayOopDesc::base_offset_in_bytes();
733
734 const ptrdiff_t min_offset = base_offset - (length == 0 ? 0 : element_size);
735 const ptrdiff_t max_offset = base_offset + length * element_size;
736 assert(min_offset <= offset && offset <= max_offset,
737 "Offset out-ouf-bounds: %zd <= %zd <= %zd", min_offset, offset, max_offset);
738
739 if (offset == min_offset || offset == max_offset) {
740 // Terminal state of iteration, set a bad value.
741 ValuePayload::set_offset(BAD_OFFSET);
742 } else {
743 ValuePayload::set_offset(offset);
744 }
745 #else // ASSERT
746 ValuePayload::set_offset(offset);
747 #endif // ASSERT
748 }
749
750 inline ValuePayload::Handle::Handle(const ValuePayload& payload, JavaThread* thread)
751 : _storage{::Handle(thread, payload.container()),
752 payload.offset(),
753 payload.klass(),
754 payload.layout_kind()} {}
755
756 inline oop ValuePayload::Handle::container() const {
757 return _storage.container()();
758 }
759
760 inline InlineKlass* ValuePayload::Handle::klass() const {
761 return _storage.klass();
762 }
763
764 inline ptrdiff_t ValuePayload::Handle::offset() const {
765 return _storage.offset();
766 }
767
768 inline LayoutKind ValuePayload::Handle::layout_kind() const {
769 return _storage.layout_kind();
770 }
771
772 inline ValuePayload::OopHandle::OopHandle(const ValuePayload& payload, OopStorage* storage)
773 : _storage{::OopHandle(storage, payload.container()),
774 payload.offset(),
775 payload.klass(),
776 payload.layout_kind()} {}
777
778 inline oop ValuePayload::OopHandle::container() const {
779 return _storage.container().resolve();
780 }
781
782 inline void ValuePayload::OopHandle::release(OopStorage* storage) {
783 return _storage.container().release(storage);
784 }
785
786 inline InlineKlass* ValuePayload::OopHandle::klass() const {
787 return _storage.klass();
788 }
789
790 inline ptrdiff_t ValuePayload::OopHandle::offset() const {
791 return _storage.offset();
792 }
793
794 inline LayoutKind ValuePayload::OopHandle::layout_kind() const {
795 return _storage.layout_kind();
796 }
797
798 inline BufferedValuePayload::Handle::Handle(const BufferedValuePayload& payload, JavaThread* thread)
799 : ValuePayload::Handle(payload, thread) {}
800
801 inline BufferedValuePayload BufferedValuePayload::Handle::operator()() const {
802 return BufferedValuePayload(container(), offset(), klass(), layout_kind());
803 }
804
805 inline inlineOop BufferedValuePayload::Handle::container() const {
806 return inlineOop(ValuePayload::Handle::container());
807 }
808
809 inline BufferedValuePayload::Handle BufferedValuePayload::make_handle(JavaThread* thread) const {
810 return Handle(*this, thread);
811 }
812
813 inline BufferedValuePayload::OopHandle::OopHandle(const BufferedValuePayload& payload,
814 OopStorage* storage)
815 : ValuePayload::OopHandle(payload, storage) {}
816
817 inline BufferedValuePayload BufferedValuePayload::OopHandle::operator()() const {
818 return BufferedValuePayload(container(), offset(), klass(), layout_kind());
819 }
820
821 inline inlineOop BufferedValuePayload::OopHandle::container() const {
822 return inlineOop(ValuePayload::OopHandle::container());
823 }
824
825 inline BufferedValuePayload::OopHandle BufferedValuePayload::make_oop_handle(OopStorage* storage) const {
826 return OopHandle(*this, storage);
827 }
828
829 inline FlatValuePayload::Handle::Handle(const FlatValuePayload& payload, JavaThread* thread)
830 : ValuePayload::Handle(payload, thread) {}
831
832 inline FlatValuePayload FlatValuePayload::Handle::operator()() const {
833 return FlatValuePayload(container(), offset(), klass(), layout_kind());
834 }
835
836 inline FlatValuePayload::Handle FlatValuePayload::make_handle(JavaThread* thread) const {
837 return Handle(*this, thread);
838 }
839
840 inline FlatValuePayload::OopHandle::OopHandle(const FlatValuePayload& payload, OopStorage* storage)
841 : ValuePayload::OopHandle(payload, storage) {}
842
843 inline FlatValuePayload FlatValuePayload::OopHandle::operator()() const {
844 return FlatValuePayload(container(), offset(), klass(), layout_kind());
845 }
846
847 inline FlatValuePayload::OopHandle FlatValuePayload::make_oop_handle(OopStorage* storage) const {
848 return OopHandle(*this, storage);
849 }
850
851 inline FlatFieldPayload::Handle::Handle(const FlatFieldPayload& payload, JavaThread* thread)
852 : FlatValuePayload::Handle(payload, thread) {}
853
854 inline FlatFieldPayload FlatFieldPayload::Handle::operator()() const {
855 return FlatFieldPayload(container(), offset(), klass(), layout_kind());
856 }
857
858 inline instanceOop FlatFieldPayload::Handle::container() const {
859 return instanceOop(ValuePayload::Handle::container());
860 }
861
862 inline FlatFieldPayload::Handle FlatFieldPayload::make_handle(JavaThread* thread) const {
863 return Handle(*this, thread);
864 }
865
866 inline FlatFieldPayload::OopHandle::OopHandle(const FlatFieldPayload& payload, OopStorage* storage)
867 : FlatValuePayload::OopHandle(payload, storage) {}
868
869 inline FlatFieldPayload FlatFieldPayload::OopHandle::operator()() const {
870 return FlatFieldPayload(container(), offset(), klass(), layout_kind());
871 }
872
873 inline instanceOop FlatFieldPayload::OopHandle::container() const {
874 return instanceOop(ValuePayload::OopHandle::container());
875 }
876
877 inline FlatFieldPayload::OopHandle FlatFieldPayload::make_oop_handle(OopStorage* storage) const {
878 return OopHandle(*this, storage);
879 }
880
881 inline FlatArrayPayload::Handle::Handle(const FlatArrayPayload& payload, JavaThread* thread)
882 : FlatValuePayload::Handle(payload, thread), _storage(payload._storage) {}
883
884 inline FlatArrayPayload FlatArrayPayload::Handle::operator()() const {
885 return FlatArrayPayload(container(),
886 offset(),
887 klass(),
888 layout_kind(),
889 _storage._layout_helper,
890 _storage._element_size);
891 }
892
893 inline flatArrayOop FlatArrayPayload::Handle::container() const {
894 return flatArrayOop(ValuePayload::Handle::container());
895 }
896
897 inline FlatArrayPayload::Handle FlatArrayPayload::make_handle(JavaThread* thread) const {
898 return Handle(*this, thread);
899 }
900
901 inline FlatArrayPayload::OopHandle::OopHandle(const FlatArrayPayload& payload, OopStorage* storage)
902 : FlatValuePayload::OopHandle(payload, storage),
903 _storage(payload._storage) {}
904
905 inline FlatArrayPayload FlatArrayPayload::OopHandle::operator()() const {
906 return FlatArrayPayload(container(),
907 offset(),
908 klass(),
909 layout_kind(),
910 _storage._layout_helper,
911 _storage._element_size);
912 }
913
914 inline flatArrayOop FlatArrayPayload::OopHandle::container() const {
915 return flatArrayOop(ValuePayload::OopHandle::container());
916 }
917
918 inline FlatArrayPayload::OopHandle FlatArrayPayload::make_oop_handle(OopStorage* storage) const {
919 return OopHandle(*this, storage);
920 }
921
922 #endif // SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP