1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP
26 #define SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP
27
28 #include "oops/valuePayload.hpp"
29
30 #include "cppstdlib/type_traits.hpp"
31 #include "oops/flatArrayKlass.inline.hpp"
32 #include "oops/flatArrayOop.inline.hpp"
33 #include "oops/inlineKlass.inline.hpp"
34 #include "oops/layoutKind.hpp"
35 #include "oops/oopHandle.inline.hpp"
36 #include "oops/oopsHierarchy.hpp"
37 #include "oops/resolvedFieldEntry.hpp"
38 #include "runtime/fieldDescriptor.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/javaThread.inline.hpp"
41 #include "utilities/debug.hpp"
42 #include "utilities/ostream.hpp"
43 #include "utilities/vmError.hpp"
44
45 template <typename OopOrHandle>
46 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl()
47 : _container(nullptr),
48 _offset(BAD_OFFSET),
49 _klass(nullptr),
50 _layout_kind(LayoutKind::UNKNOWN),
51 _uses_absolute_addr(false) {}
52
53 template <typename OopOrHandle>
54 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(OopOrHandle container,
55 ptrdiff_t offset,
56 InlineKlass* klass,
57 LayoutKind layout_kind)
58 : _container(container),
59 _offset(offset),
60 _klass(klass),
61 _layout_kind(layout_kind),
62 _uses_absolute_addr(false) {}
63
64 template <typename OopOrHandle>
65 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(address absolute_addr,
66 InlineKlass* klass,
67 LayoutKind layout_kind)
68 : _absolute_addr(absolute_addr),
69 _klass(klass),
70 _layout_kind(layout_kind),
71 _uses_absolute_addr(true) {}
72
73 template <typename OopOrHandle>
74 inline ValuePayload::StorageImpl<OopOrHandle>::~StorageImpl() {
75 #ifdef CHECK_UNHANDLED_OOPS
76 if (!_uses_absolute_addr) {
77 _container.~OopOrHandle();
78 }
79 #else // CHECK_UNHANDLED_OOPS
80 static_assert(std::is_trivially_destructible_v<OopOrHandle>);
81 #endif // CHECK_UNHANDLED_OOPS
82 }
83
84 template <typename OopOrHandle>
85 inline ValuePayload::StorageImpl<OopOrHandle>::StorageImpl(const StorageImpl& other)
86 : _klass(other._klass),
87 _layout_kind(other._layout_kind),
88 _uses_absolute_addr(other._uses_absolute_addr) {
89 if (_uses_absolute_addr) {
90 _absolute_addr = other._absolute_addr;
91 } else {
92 _container = other._container;
93 _offset = other._offset;
94 }
95 }
96
97 template <typename OopOrHandle>
98 inline ValuePayload::StorageImpl<OopOrHandle>&
99 ValuePayload::StorageImpl<OopOrHandle>::operator=(const StorageImpl& other) {
100 if (&other != this) {
101 _klass = other._klass;
102 _layout_kind = other._layout_kind;
103 _uses_absolute_addr = other._uses_absolute_addr;
104 if (_uses_absolute_addr) {
105 _absolute_addr = other._absolute_addr;
106 } else {
107 _container = other._container;
108 _offset = other._offset;
109 }
110 }
111 return *this;
112 }
113
114 template <typename OopOrHandle>
115 inline OopOrHandle& ValuePayload::StorageImpl<OopOrHandle>::container() {
116 precond(!_uses_absolute_addr);
117 return _container;
118 }
119
120 template <typename OopOrHandle>
121 inline OopOrHandle ValuePayload::StorageImpl<OopOrHandle>::container() const {
122 precond(!_uses_absolute_addr);
123 return _container;
124 }
125
126 template <typename OopOrHandle>
127 inline ptrdiff_t& ValuePayload::StorageImpl<OopOrHandle>::offset() {
128 precond(!_uses_absolute_addr);
129 return _offset;
130 }
131
132 template <typename OopOrHandle>
133 inline ptrdiff_t ValuePayload::StorageImpl<OopOrHandle>::offset() const {
134 precond(!_uses_absolute_addr);
135 return _offset;
136 }
137
138 template <typename OopOrHandle>
139 inline address& ValuePayload::StorageImpl<OopOrHandle>::absolute_addr() {
140 precond(_uses_absolute_addr);
141 return _absolute_addr;
142 }
143
144 template <typename OopOrHandle>
145 inline address ValuePayload::StorageImpl<OopOrHandle>::absolute_addr() const {
146 precond(_uses_absolute_addr);
147 return _absolute_addr;
148
149 }
150
151 template <typename OopOrHandle>
152 inline InlineKlass* ValuePayload::StorageImpl<OopOrHandle>::klass() const {
153 return _klass;
154 }
155
156 template <typename OopOrHandle>
157 inline LayoutKind ValuePayload::StorageImpl<OopOrHandle>::layout_kind() const {
158 return _layout_kind;
159 }
160
161 template <typename OopOrHandle>
162 inline bool ValuePayload::StorageImpl<OopOrHandle>::uses_absolute_addr() const {
163 return _uses_absolute_addr;
164 }
165
166 inline ValuePayload::ValuePayload(oop container,
167 ptrdiff_t offset,
168 InlineKlass* klass,
169 LayoutKind layout_kind)
170 : _storage{container, offset, klass, layout_kind} {
171 assert_post_construction_invariants();
172 }
173
174 inline ValuePayload::ValuePayload(address absolute_addr,
175 InlineKlass* klass,
176 LayoutKind layout_kind)
177 : _storage{absolute_addr, klass, layout_kind} {
178 assert_post_construction_invariants();
179 }
180
181 inline void ValuePayload::set_offset(ptrdiff_t offset) {
182 _storage.offset() = offset;
183 }
184
185 inline void ValuePayload::copy(const ValuePayload& src,
186 const ValuePayload& dst,
187 LayoutKind copy_layout_kind) {
188 assert_pre_copy_invariants(src, dst, copy_layout_kind);
189
190 InlineKlass* const klass = src.klass();
191
192 switch (copy_layout_kind) {
193 case LayoutKind::NULLABLE_ATOMIC_FLAT:
194 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT: {
195 if (src.is_payload_null()) {
196 HeapAccess<>::value_store_null(dst);
197 } else {
198 HeapAccess<>::value_copy(src, dst);
199 }
200 } break;
201 case LayoutKind::BUFFERED:
202 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
203 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
204 if (!klass->is_empty_inline_type()) {
205 HeapAccess<>::value_copy(src, dst);
206 }
207 } break;
208 default:
209 ShouldNotReachHere();
210 }
211 }
212
213 inline void ValuePayload::mark_as_non_null() {
214 precond(has_null_marker());
215 klass()->mark_payload_as_non_null(addr());
216 }
217
218 inline void ValuePayload::mark_as_null() {
219 precond(has_null_marker());
220 klass()->mark_payload_as_null(addr());
221 }
222
223 inline bool ValuePayload::uses_absolute_addr() const {
224 return _storage.uses_absolute_addr();
225 }
226
227 inline oop& ValuePayload::container() {
228 return _storage.container();
229 }
230
231 inline oop ValuePayload::container() const {
232 return _storage.container();
233 }
234
235 #ifdef ASSERT
236
237 inline void ValuePayload::print_on(outputStream* st) const {
238 if (uses_absolute_addr()) {
239 st->print_cr("--- absolute_addr ---");
240 StreamIndentor si(st);
241 st->print_cr("_absolute_addr: " PTR_FORMAT, p2i(_storage.absolute_addr()));
242 } else {
243 {
244 oop container = _storage.container();
245 st->print_cr("--- container ---");
246 StreamIndentor si(st);
247 st->print_cr("_container: " PTR_FORMAT, p2i(container));
248 if (container != nullptr) {
249 container->print_on(st);
250 st->cr();
251 }
252 }
253 {
254 st->print_cr("--- offset ---");
255 StreamIndentor si(st);
256 st->print_cr("_offset: %zd", _storage.offset());
257 }
258 }
259 {
260 InlineKlass* const klass = _storage.klass();
261 st->print_cr("--- klass ---");
262 StreamIndentor si(st);
263 st->print_cr("_klass: " PTR_FORMAT, p2i(klass));
264 if (klass != nullptr) {
265 klass->print_on(st);
266 st->cr();
267 }
268 }
269 {
270 const LayoutKind layout_kind = _storage.layout_kind();
271 st->print_cr("--- layout_kind ---");
272 StreamIndentor si(st);
273 st->print_cr("_layout_kind: %u", (uint32_t)layout_kind);
274 LayoutKindHelper::print_on(layout_kind, st);
275 st->cr();
276 }
277 }
278
279 inline void ValuePayload::assert_is_flat_field(const InstanceKlass* klass, int offset) const {
280 OnVMError on_assertion_failure_field_descriptor([&](outputStream* st) {
281 st->print_cr("=== assert_is_flat_field(" PTR_FORMAT ", %d) failure ===", p2i(klass), offset);
282 StreamIndentor si(st);
283 if (klass != nullptr) {
284 klass->print_on(st);
285 st->cr();
286 }
287 });
288
289 fieldDescriptor field_descriptor;
290 postcond(klass->find_flat_field_containing_offset(offset, &field_descriptor));
291
292 const InlineLayoutInfo inline_layout_info = field_descriptor.field_holder()->inline_layout_info(field_descriptor.index());
293
294 OnVMError on_assertion_failure_inline_layout_info([&](outputStream* st) {
295 st->print_cr("=== assert_is_flat_field(" PTR_FORMAT ", %d) failure ===", p2i(klass), offset);
296 StreamIndentor si(st);
297 st->print("field_descriptor: ");
298 field_descriptor.print_on(st);
299 st->cr();
300 st->print("inline_layout_info: ");
301 inline_layout_info.print_on(st);
302 st->cr();
303 });
304
305 if (inline_layout_info.klass() == this->klass()) {
306 // Found the field in klass
307 postcond(offset == field_descriptor.offset());
308 postcond(inline_layout_info.kind() == layout_kind());
309 postcond(field_descriptor.layout_kind() == layout_kind());
310 postcond(field_descriptor.is_flat());
311 } else {
312 // Nested flat field
313 postcond(offset >= field_descriptor.offset());
314 const InlineKlass* const field_klass = inline_layout_info.klass();
315 const int payload_offset = field_klass->payload_offset();
316 assert_is_flat_field(field_klass, offset - field_descriptor.offset() + payload_offset);
317 }
318 }
319
320 inline void ValuePayload::assert_post_construction_invariants() const {
321 OnVMError on_assertion_failure([&](outputStream* st) {
322 st->print_cr("=== assert_post_construction_invariants failure ===");
323 StreamIndentor si(st);
324 print_on(st);
325 st->cr();
326 });
327
328 postcond(layout_kind() != LayoutKind::REFERENCE);
329 postcond(layout_kind() != LayoutKind::UNKNOWN);
330 postcond(klass()->is_layout_supported(layout_kind()));
331
332 if (!uses_absolute_addr()) {
333 postcond(container() != nullptr);
334 const Klass* const container_klass = container()->klass();
335 if (container_klass == klass()) {
336 postcond(layout_kind() == LayoutKind::BUFFERED);
337 } else {
338 postcond(layout_kind() != LayoutKind::BUFFERED);
339 if (container_klass->is_mirror_instance_klass()) {
340 fatal("java.lang.Class has no flat fields. Static fields are not flattened");
341 } else if (container_klass->is_instance_klass()) {
342 assert_is_flat_field(InstanceKlass::cast(container_klass), checked_cast<int>(offset()));
343 } else {
344 const FlatArrayKlass* const container_flat_array_klass = FlatArrayKlass::cast(container_klass);
345 if (container_flat_array_klass->element_klass() == klass()) {
346 postcond(container_flat_array_klass->layout_kind() == layout_kind());
347 } else {
348 // Accessing nested flat field
349 const InlineKlass* const element_klass = container_flat_array_klass->element_klass();
350 const int element_offset =
351 (checked_cast<int>(this->offset()) -
352 checked_cast<int>(flatArrayOopDesc::base_offset_in_bytes())) %
353 container_flat_array_klass->element_byte_size();
354 const int payload_offset = element_klass->payload_offset();
355 assert_is_flat_field(element_klass, element_offset + payload_offset);
356 }
357 }
358 }
359 }
360 }
361
362 inline void ValuePayload::assert_pre_copy_invariants(const ValuePayload& src,
363 const ValuePayload& dst,
364 LayoutKind copy_layout_kind) {
365 OnVMError on_assertion_failuire([&](outputStream* st) {
366 st->print_cr("=== assert_post_construction_invariants failure ===");
367 StreamIndentor si(st);
368 {
369 st->print_cr("--- src payload ---");
370 StreamIndentor si(st);
371 src.print_on(st);
372 st->cr();
373 }
374 {
375 st->print_cr("--- dst payload ---");
376 StreamIndentor si(st);
377 dst.print_on(st);
378 st->cr();
379 }
380 {
381 st->print_cr("--- copy layout kind ---");
382 StreamIndentor si(st);
383 LayoutKindHelper::print_on(copy_layout_kind, st);
384 st->cr();
385 }
386 });
387
388 const InlineKlass* const src_klass = src.klass();
389 const InlineKlass* const dst_klass = dst.klass();
390
391 precond(src_klass == dst_klass);
392
393 const bool src_is_buffered = src.layout_kind() == LayoutKind::BUFFERED;
394 const bool dst_is_buffered = dst.layout_kind() == LayoutKind::BUFFERED;
395 const bool src_and_dst_same_layout_kind = src.layout_kind() == dst.layout_kind();
396 const bool src_has_copy_layout = src.layout_kind() == copy_layout_kind;
397 const bool dst_has_copy_layout = dst.layout_kind() == copy_layout_kind;
398
399 precond(src_is_buffered || dst_is_buffered || src_and_dst_same_layout_kind);
400 precond(src_has_copy_layout || dst_has_copy_layout);
401
402 if (src_is_buffered) {
403 oop container = src.uses_absolute_addr()
404 ? cast_to_oop(src.addr() - src_klass->payload_offset())
405 : src.container();
406
407 precond(!src_klass->supports_nullable_layouts() || container != src_klass->null_reset_value());
408 }
409
410 const int src_layout_size_in_bytes = src_klass->layout_size_in_bytes(src.layout_kind());
411 const int dst_layout_size_in_bytes = dst_klass->layout_size_in_bytes(dst.layout_kind());
412 const int copy_layout_size_in_bytes =
413 src_has_copy_layout
414 ? src_layout_size_in_bytes
415 : dst_layout_size_in_bytes;
416
417 precond(copy_layout_size_in_bytes <= src_layout_size_in_bytes);
418 precond(copy_layout_size_in_bytes <= dst_layout_size_in_bytes);
419 precond(LayoutKindHelper::get_copy_layout(src.layout_kind(),
420 dst.layout_kind()) == copy_layout_kind);
421 }
422
423 #endif // ASSERT
424
425 inline InlineKlass* ValuePayload::klass() const {
426 return _storage.klass();
427 }
428
429 inline ptrdiff_t ValuePayload::offset() const {
430 precond(_storage.offset() != BAD_OFFSET);
431 return _storage.offset();
432 }
433
434 inline LayoutKind ValuePayload::layout_kind() const {
435 return _storage.layout_kind();
436 }
437
438 inline address ValuePayload::addr() const {
439 return uses_absolute_addr()
440 ? _storage.absolute_addr()
441 : (cast_from_oop<address>(container()) + offset());
442 }
443
444 inline bool ValuePayload::has_null_marker() const {
445 return klass()->layout_has_null_marker(layout_kind());
446 }
447
448 inline bool ValuePayload::is_payload_null() const {
449 return has_null_marker() && klass()->is_payload_marked_as_null(addr());
450 }
451
452 inline ValuePayload ValuePayload::construct_from_parts(address absolute_addr,
453 InlineKlass* klass,
454 LayoutKind layout_kind) {
455 return ValuePayload(absolute_addr, klass, layout_kind);
456 }
457
458 inline BufferedValuePayload::BufferedValuePayload(inlineOop container,
459 ptrdiff_t offset,
460 InlineKlass* klass,
461 LayoutKind layout_kind)
462 : ValuePayload(container, offset, klass, layout_kind) {}
463
464 inline BufferedValuePayload::BufferedValuePayload(inlineOop buffer)
465 : BufferedValuePayload(buffer, InlineKlass::cast(buffer->klass())) {}
466
467 inline BufferedValuePayload::BufferedValuePayload(inlineOop buffer,
468 InlineKlass* klass)
469 : ValuePayload(buffer, klass->payload_offset(), klass, LayoutKind::BUFFERED) {}
470
471 inline inlineOop BufferedValuePayload::container() const {
472 return inlineOop(ValuePayload::container());
473 }
474
475 inline void BufferedValuePayload::copy_to(const BufferedValuePayload& dst) {
476 copy(*this, dst, LayoutKind::BUFFERED);
477 }
478
479 inline FlatValuePayload::FlatValuePayload(oop container,
480 ptrdiff_t offset,
481 InlineKlass* klass,
482 LayoutKind layout_kind)
483 : ValuePayload(container, offset, klass, layout_kind) {}
484
485 inline inlineOop FlatValuePayload::allocate_instance(TRAPS) {
486 // Preserve the container oop across the instance allocation.
487 oop& container = this->container();
488 ::Handle container_handle(THREAD, container);
489 inlineOop res = klass()->allocate_instance(THREAD);
490 container = container_handle();
491 return res;
492 }
493
494 inline bool FlatValuePayload::copy_to(BufferedValuePayload& dst) {
495 // Copy from FLAT to BUFFERED, null marker fix may be required.
496
497 // Copy the payload to the buffered object.
498 copy(*this, dst, layout_kind());
499
500 if (!has_null_marker() && dst.has_null_marker()) {
501 // We must fix the null marker if the src does not have a null marker but
502 // the buffered object does.
503 dst.mark_as_non_null();
504
505 // The buffered object was just marked non null.
506 return true;
507 }
508
509 if (dst.is_payload_null()) {
510 // A null payload is not a valid payload for a buffered value.
511 return false;
512 }
513
514 return true;
515 }
516
517 inline void FlatValuePayload::copy_from(BufferedValuePayload& src) {
518 // Copy from BUFFERED to FLAT, null marker fix may be required.
519
520 if (has_null_marker()) {
521 // The FLAT payload has a null mark. So make sure that buffered is marked as
522 // non null. It is the callers responsibility to ensure that this is a
523 // valid non null value.
524 src.mark_as_non_null();
525 }
526 copy(src, *this, layout_kind());
527 }
528
529 inline void FlatValuePayload::copy_to(const FlatValuePayload& dst) {
530 copy(*this, dst, layout_kind());
531 }
532
533 inline inlineOop FlatValuePayload::read(TRAPS) {
534 switch (layout_kind()) {
535 case LayoutKind::NULLABLE_ATOMIC_FLAT:
536 case LayoutKind::NULLABLE_NON_ATOMIC_FLAT: {
537 if (is_payload_null()) {
538 return nullptr;
539 }
540 } // Fallthrough
541 case LayoutKind::NULL_FREE_ATOMIC_FLAT:
542 case LayoutKind::NULL_FREE_NON_ATOMIC_FLAT: {
543 inlineOop res = allocate_instance(CHECK_NULL);
544 BufferedValuePayload dst(res, klass());
545 if (!copy_to(dst)) {
546 // copy_to may fail if the payload has been updated with a null value
547 // between our is_payload_null() check above and the copy.
548 // In this case we have copied a null value into the buffer the payload.
549 return nullptr;
550 }
551 // Must ensure the content of the buffered value is visible
552 // before publishing the buffered value oop
553 OrderAccess::storestore();
554 return res;
555 } break;
556 default:
557 ShouldNotReachHere();
558 }
559 }
560
561 inline void FlatValuePayload::write_without_nullability_check(inlineOop obj) {
562 if (obj == nullptr) {
563 assert(has_null_marker(), "Payload must support null values");
564 HeapAccess<>::value_store_null(*this);
565 } else {
566 // Copy the obj payload
567 BufferedValuePayload obj_payload(obj);
568 copy_from(obj_payload);
569 }
570 }
571
572 inline void FlatValuePayload::write(inlineOop obj, TRAPS) {
573 if (obj == nullptr && !has_null_marker()) {
574 // This payload does not have a null marker and cannot represent a null
575 // value.
576 THROW_MSG(vmSymbols::java_lang_NullPointerException(), "Value is null");
577 }
578 write_without_nullability_check(obj);
579 }
580
581 inline FlatValuePayload FlatValuePayload::construct_from_parts(oop container,
582 ptrdiff_t offset,
583 InlineKlass* klass,
584 LayoutKind layout_kind) {
585 return FlatValuePayload(container, offset, klass, layout_kind);
586 }
587
588 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
589 ptrdiff_t offset,
590 InlineKlass* klass,
591 LayoutKind layout_kind)
592 : FlatValuePayload(container, offset, klass, layout_kind) {}
593
594 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
595 ptrdiff_t offset,
596 InlineLayoutInfo* inline_layout_info)
597 : FlatValuePayload(container, offset, inline_layout_info->klass(), inline_layout_info->kind()) {}
598
599 #ifdef ASSERT
600
601 inline void FlatFieldPayload::assert_post_construction_invariants(instanceOop container,
602 ResolvedFieldEntry* resolved_field_entry) const {
603 OnVMError on_assertion_failure([&](outputStream* st) {
604 st->print_cr("=== assert_post_construction_invariants failure ===");
605 StreamIndentor si(st);
606 print_on(st);
607 st->cr();
608 });
609
610 postcond(container->klass()->is_subclass_of(resolved_field_entry->field_holder()));
611 postcond(resolved_field_entry->is_flat());
612 }
613
614 inline void FlatFieldPayload::assert_post_construction_invariants(instanceOop container,
615 fieldDescriptor* field_descriptor) const {
616 OnVMError on_assertion_failure([&](outputStream* st) {
617 st->print_cr("=== assert_post_construction_invariants failure ===");
618 StreamIndentor si(st);
619 print_on(st);
620 st->cr();
621 });
622
623 postcond(container->klass()->is_subclass_of(field_descriptor->field_holder()));
624 postcond(field_descriptor->is_flat());
625 }
626
627 #endif // ASSERT
628
629 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
630 fieldDescriptor* field_descriptor)
631 : FlatFieldPayload(container,
632 field_descriptor->offset(),
633 field_descriptor->field_holder()->inline_layout_info_adr(field_descriptor->index())) {
634 assert_post_construction_invariants(container, field_descriptor);
635 }
636
637 inline FlatFieldPayload::FlatFieldPayload(instanceOop container,
638 ResolvedFieldEntry* resolved_field_entry)
639 : FlatFieldPayload(container,
640 resolved_field_entry->field_offset(),
641 resolved_field_entry->field_holder()->inline_layout_info_adr(resolved_field_entry->field_index())) {
642 assert_post_construction_invariants(container, resolved_field_entry);
643 }
644
645 inline instanceOop FlatFieldPayload::container() const {
646 return instanceOop(ValuePayload::container());
647 }
648
649 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container,
650 ptrdiff_t offset,
651 InlineKlass* klass,
652 LayoutKind layout_kind,
653 jint layout_helper,
654 int element_size)
655 : FlatValuePayload(container, offset, klass, layout_kind),
656 _storage{layout_helper, element_size} {}
657
658 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container)
659 : FlatArrayPayload(container, container->klass()) {}
660
661 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container, FlatArrayKlass* klass)
662 : FlatArrayPayload(container,
663 BAD_OFFSET,
664 klass->element_klass(),
665 klass->layout_kind(),
666 klass->layout_helper(),
667 klass->element_byte_size()) {
668 postcond(container->klass() == klass);
669 }
670
671 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container, int index)
672 : FlatArrayPayload(container, index, container->klass()) {}
673
674 inline FlatArrayPayload::FlatArrayPayload(flatArrayOop container,
675 int index,
676 FlatArrayKlass* klass)
677 : FlatArrayPayload(container,
678 (ptrdiff_t)container->value_offset(index, klass->layout_helper()),
679 klass->element_klass(),
680 klass->layout_kind(),
681 klass->layout_helper(),
682 klass->element_byte_size()) {
683 postcond(container->klass() == klass);
684 }
685
686 inline flatArrayOop FlatArrayPayload::container() const {
687 return flatArrayOop(ValuePayload::container());
688 }
689
690 inline void FlatArrayPayload::set_index(int index) {
691 set_offset((ptrdiff_t)container()->value_offset(index, _storage._layout_helper));
692 }
693
694 inline void FlatArrayPayload::advance_index(int delta) {
695 set_offset(this->offset() + delta * _storage._element_size);
696 }
697
698 inline void FlatArrayPayload::next_element() {
699 advance_index(1);
700 }
701
702 inline void FlatArrayPayload::previous_element() {
703 advance_index(-1);
704 }
705
706 inline void FlatArrayPayload::set_offset(ptrdiff_t offset) {
707 #if defined(ASSERT) && defined(_LP64)
708 // For ease of use as iterators we allow the offset to point one element size
709 // beyond the first and last element. If there are no elements only the base
710 // offset is allowed. However we treat these as terminal states, and set the
711 // offset to a BAD_OFFSET in debug builds.
712
713 const ptrdiff_t element_size = _storage._element_size;
714 const ptrdiff_t length = container()->length();
715 const ptrdiff_t base_offset = (ptrdiff_t)flatArrayOopDesc::base_offset_in_bytes();
716
717 const ptrdiff_t min_offset = base_offset - (length == 0 ? 0 : element_size);
718 const ptrdiff_t max_offset = base_offset + length * element_size;
719 assert(min_offset <= offset && offset <= max_offset,
720 "Offset out-ouf-bounds: %zd <= %zd <= %zd", min_offset, offset, max_offset);
721
722 if (offset == min_offset || offset == max_offset) {
723 // Terminal state of iteration, set a bad value.
724 ValuePayload::set_offset(BAD_OFFSET);
725 } else {
726 ValuePayload::set_offset(offset);
727 }
728 #else // ASSERT
729 ValuePayload::set_offset(offset);
730 #endif // ASSERT
731 }
732
733 inline ValuePayload::Handle::Handle(const ValuePayload& payload, JavaThread* thread)
734 : _storage{::Handle(thread, payload.container()),
735 payload.offset(),
736 payload.klass(),
737 payload.layout_kind()} {}
738
739 inline oop ValuePayload::Handle::container() const {
740 return _storage.container()();
741 }
742
743 inline InlineKlass* ValuePayload::Handle::klass() const {
744 return _storage.klass();
745 }
746
747 inline ptrdiff_t ValuePayload::Handle::offset() const {
748 return _storage.offset();
749 }
750
751 inline LayoutKind ValuePayload::Handle::layout_kind() const {
752 return _storage.layout_kind();
753 }
754
755 inline ValuePayload::OopHandle::OopHandle(const ValuePayload& payload, OopStorage* storage)
756 : _storage{::OopHandle(storage, payload.container()),
757 payload.offset(),
758 payload.klass(),
759 payload.layout_kind()} {}
760
761 inline oop ValuePayload::OopHandle::container() const {
762 return _storage.container().resolve();
763 }
764
765 inline void ValuePayload::OopHandle::release(OopStorage* storage) {
766 return _storage.container().release(storage);
767 }
768
769 inline InlineKlass* ValuePayload::OopHandle::klass() const {
770 return _storage.klass();
771 }
772
773 inline ptrdiff_t ValuePayload::OopHandle::offset() const {
774 return _storage.offset();
775 }
776
777 inline LayoutKind ValuePayload::OopHandle::layout_kind() const {
778 return _storage.layout_kind();
779 }
780
781 inline BufferedValuePayload::Handle::Handle(const BufferedValuePayload& payload, JavaThread* thread)
782 : ValuePayload::Handle(payload, thread) {}
783
784 inline BufferedValuePayload BufferedValuePayload::Handle::operator()() const {
785 return BufferedValuePayload(container(), offset(), klass(), layout_kind());
786 }
787
788 inline inlineOop BufferedValuePayload::Handle::container() const {
789 return inlineOop(ValuePayload::Handle::container());
790 }
791
792 inline BufferedValuePayload::Handle BufferedValuePayload::make_handle(JavaThread* thread) const {
793 return Handle(*this, thread);
794 }
795
796 inline BufferedValuePayload::OopHandle::OopHandle(const BufferedValuePayload& payload,
797 OopStorage* storage)
798 : ValuePayload::OopHandle(payload, storage) {}
799
800 inline BufferedValuePayload BufferedValuePayload::OopHandle::operator()() const {
801 return BufferedValuePayload(container(), offset(), klass(), layout_kind());
802 }
803
804 inline inlineOop BufferedValuePayload::OopHandle::container() const {
805 return inlineOop(ValuePayload::OopHandle::container());
806 }
807
808 inline BufferedValuePayload::OopHandle BufferedValuePayload::make_oop_handle(OopStorage* storage) const {
809 return OopHandle(*this, storage);
810 }
811
812 inline FlatValuePayload::Handle::Handle(const FlatValuePayload& payload, JavaThread* thread)
813 : ValuePayload::Handle(payload, thread) {}
814
815 inline FlatValuePayload FlatValuePayload::Handle::operator()() const {
816 return FlatValuePayload(container(), offset(), klass(), layout_kind());
817 }
818
819 inline FlatValuePayload::Handle FlatValuePayload::make_handle(JavaThread* thread) const {
820 return Handle(*this, thread);
821 }
822
823 inline FlatValuePayload::OopHandle::OopHandle(const FlatValuePayload& payload, OopStorage* storage)
824 : ValuePayload::OopHandle(payload, storage) {}
825
826 inline FlatValuePayload FlatValuePayload::OopHandle::operator()() const {
827 return FlatValuePayload(container(), offset(), klass(), layout_kind());
828 }
829
830 inline FlatValuePayload::OopHandle FlatValuePayload::make_oop_handle(OopStorage* storage) const {
831 return OopHandle(*this, storage);
832 }
833
834 inline FlatFieldPayload::Handle::Handle(const FlatFieldPayload& payload, JavaThread* thread)
835 : FlatValuePayload::Handle(payload, thread) {}
836
837 inline FlatFieldPayload FlatFieldPayload::Handle::operator()() const {
838 return FlatFieldPayload(container(), offset(), klass(), layout_kind());
839 }
840
841 inline instanceOop FlatFieldPayload::Handle::container() const {
842 return instanceOop(ValuePayload::Handle::container());
843 }
844
845 inline FlatFieldPayload::Handle FlatFieldPayload::make_handle(JavaThread* thread) const {
846 return Handle(*this, thread);
847 }
848
849 inline FlatFieldPayload::OopHandle::OopHandle(const FlatFieldPayload& payload, OopStorage* storage)
850 : FlatValuePayload::OopHandle(payload, storage) {}
851
852 inline FlatFieldPayload FlatFieldPayload::OopHandle::operator()() const {
853 return FlatFieldPayload(container(), offset(), klass(), layout_kind());
854 }
855
856 inline instanceOop FlatFieldPayload::OopHandle::container() const {
857 return instanceOop(ValuePayload::OopHandle::container());
858 }
859
860 inline FlatFieldPayload::OopHandle FlatFieldPayload::make_oop_handle(OopStorage* storage) const {
861 return OopHandle(*this, storage);
862 }
863
864 inline FlatArrayPayload::Handle::Handle(const FlatArrayPayload& payload, JavaThread* thread)
865 : FlatValuePayload::Handle(payload, thread), _storage(payload._storage) {}
866
867 inline FlatArrayPayload FlatArrayPayload::Handle::operator()() const {
868 return FlatArrayPayload(container(),
869 offset(),
870 klass(),
871 layout_kind(),
872 _storage._layout_helper,
873 _storage._element_size);
874 }
875
876 inline flatArrayOop FlatArrayPayload::Handle::container() const {
877 return flatArrayOop(ValuePayload::Handle::container());
878 }
879
880 inline FlatArrayPayload::Handle FlatArrayPayload::make_handle(JavaThread* thread) const {
881 return Handle(*this, thread);
882 }
883
884 inline FlatArrayPayload::OopHandle::OopHandle(const FlatArrayPayload& payload, OopStorage* storage)
885 : FlatValuePayload::OopHandle(payload, storage),
886 _storage(payload._storage) {}
887
888 inline FlatArrayPayload FlatArrayPayload::OopHandle::operator()() const {
889 return FlatArrayPayload(container(),
890 offset(),
891 klass(),
892 layout_kind(),
893 _storage._layout_helper,
894 _storage._element_size);
895 }
896
897 inline flatArrayOop FlatArrayPayload::OopHandle::container() const {
898 return flatArrayOop(ValuePayload::OopHandle::container());
899 }
900
901 inline FlatArrayPayload::OopHandle FlatArrayPayload::make_oop_handle(OopStorage* storage) const {
902 return OopHandle(*this, storage);
903 }
904
905 #endif // SHARE_VM_OOPS_VALUEPAYLOAD_INLINE_HPP