1 /*
2 * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_METHODDATA_HPP
26 #define SHARE_OOPS_METHODDATA_HPP
27
28 #include "interpreter/bytecodes.hpp"
29 #include "oops/metadata.hpp"
30 #include "oops/method.hpp"
31 #include "oops/oop.hpp"
32 #include "runtime/atomic.hpp"
33 #include "runtime/deoptimization.hpp"
34 #include "runtime/mutex.hpp"
35 #include "utilities/align.hpp"
36 #include "utilities/copy.hpp"
37
38 class BytecodeStream;
39
40 // The MethodData object collects counts and other profile information
41 // during zeroth-tier (interpreter) and third-tier (C1 with full profiling)
42 // execution.
43 //
44 // The profile is used later by compilation heuristics. Some heuristics
45 // enable use of aggressive (or "heroic") optimizations. An aggressive
46 // optimization often has a down-side, a corner case that it handles
47 // poorly, but which is thought to be rare. The profile provides
48 // evidence of this rarity for a given method or even BCI. It allows
49 // the compiler to back out of the optimization at places where it
50 // has historically been a poor choice. Other heuristics try to use
51 // specific information gathered about types observed at a given site.
52 //
53 // All data in the profile is approximate. It is expected to be accurate
54 // on the whole, but the system expects occasional inaccuraces, due to
55 // counter overflow, multiprocessor races during data collection, space
56 // limitations, missing MDO blocks, etc. Bad or missing data will degrade
57 // optimization quality but will not affect correctness. Also, each MDO
58 // is marked with its birth-date ("creation_mileage") which can be used
59 // to assess the quality ("maturity") of its data.
60 //
61 // Short (<32-bit) counters are designed to overflow to a known "saturated"
62 // state. Also, certain recorded per-BCI events are given one-bit counters
63 // which overflow to a saturated state which applied to all counters at
64 // that BCI. In other words, there is a small lattice which approximates
65 // the ideal of an infinite-precision counter for each event at each BCI,
66 // and the lattice quickly "bottoms out" in a state where all counters
67 // are taken to be indefinitely large.
68 //
69 // The reader will find many data races in profile gathering code, starting
70 // with invocation counter incrementation. None of these races harm correct
71 // execution of the compiled code.
72
73 // forward decl
74 class ProfileData;
75
76 // DataLayout
77 //
78 // Overlay for generic profiling data.
79 class DataLayout {
80 friend class VMStructs;
81 friend class JVMCIVMStructs;
82
83 private:
84 // Every data layout begins with a header. This header
85 // contains a tag, which is used to indicate the size/layout
86 // of the data, 8 bits of flags, which can be used in any way,
87 // 32 bits of trap history (none/one reason/many reasons),
88 // and a bci, which is used to tie this piece of data to a
89 // specific bci in the bytecodes.
90 union {
91 u8 _bits;
92 struct {
93 u1 _tag;
94 u1 _flags;
95 u2 _bci;
96 u4 _traps;
97 } _struct;
98 } _header;
99
100 // The data layout has an arbitrary number of cells, each sized
101 // to accommodate a pointer or an integer.
102 intptr_t _cells[1];
103
104 // Some types of data layouts need a length field.
105 static bool needs_array_len(u1 tag);
106
107 public:
108 enum {
109 counter_increment = 1
110 };
111
112 enum {
113 cell_size = sizeof(intptr_t)
114 };
115
116 // Tag values
117 enum {
118 no_tag,
119 bit_data_tag,
120 counter_data_tag,
121 jump_data_tag,
122 receiver_type_data_tag,
123 virtual_call_data_tag,
124 ret_data_tag,
125 branch_data_tag,
126 multi_branch_data_tag,
127 arg_info_data_tag,
128 call_type_data_tag,
129 virtual_call_type_data_tag,
130 parameters_type_data_tag,
131 speculative_trap_data_tag,
132 array_load_store_data_tag,
133 acmp_data_tag
134 };
135
136 enum {
137 // The trap state breaks down as [recompile:1 | reason:31].
138 // This further breakdown is defined in deoptimization.cpp.
139 // See Deoptimization::trap_state_reason for an assert that
140 // trap_bits is big enough to hold reasons < Reason_RECORDED_LIMIT.
141 //
142 // The trap_state is collected only if ProfileTraps is true.
143 trap_bits = 1+31, // 31: enough to distinguish [0..Reason_RECORDED_LIMIT].
144 trap_mask = -1,
145 first_flag = 0
146 };
147
148 // Size computation
149 static int header_size_in_bytes() {
150 return header_size_in_cells() * cell_size;
151 }
152 static int header_size_in_cells() {
153 return LP64_ONLY(1) NOT_LP64(2);
154 }
155
156 static int compute_size_in_bytes(int cell_count) {
157 return header_size_in_bytes() + cell_count * cell_size;
158 }
159
160 // Initialization
161 void initialize(u1 tag, u2 bci, int cell_count);
162
163 // Accessors
164 u1 tag() {
165 return _header._struct._tag;
166 }
167
168 // Return 32 bits of trap state.
169 // The state tells if traps with zero, one, or many reasons have occurred.
170 // It also tells whether zero or many recompilations have occurred.
171 // The associated trap histogram in the MDO itself tells whether
172 // traps are common or not. If a BCI shows that a trap X has
173 // occurred, and the MDO shows N occurrences of X, we make the
174 // simplifying assumption that all N occurrences can be blamed
175 // on that BCI.
176 uint trap_state() const {
177 return _header._struct._traps;
178 }
179
180 void set_trap_state(uint new_state) {
181 assert(ProfileTraps, "used only under +ProfileTraps");
182 uint old_flags = _header._struct._traps;
183 _header._struct._traps = new_state | old_flags;
184 }
185
186 u1 flags() const {
187 return _header._struct._flags;
188 }
189
190 u2 bci() const {
191 return _header._struct._bci;
192 }
193
194 void set_header(u8 value) {
195 _header._bits = value;
196 }
197 u8 header() {
198 return _header._bits;
199 }
200 void set_cell_at(int index, intptr_t value) {
201 _cells[index] = value;
202 }
203 void release_set_cell_at(int index, intptr_t value);
204 intptr_t cell_at(int index) const {
205 return _cells[index];
206 }
207
208 void set_flag_at(u1 flag_number) {
209 _header._struct._flags |= (0x1 << flag_number);
210 }
211 bool flag_at(u1 flag_number) const {
212 return (_header._struct._flags & (0x1 << flag_number)) != 0;
213 }
214
215 // Low-level support for code generation.
216 static ByteSize header_offset() {
217 return byte_offset_of(DataLayout, _header);
218 }
219 static ByteSize tag_offset() {
220 return byte_offset_of(DataLayout, _header._struct._tag);
221 }
222 static ByteSize flags_offset() {
223 return byte_offset_of(DataLayout, _header._struct._flags);
224 }
225 static ByteSize bci_offset() {
226 return byte_offset_of(DataLayout, _header._struct._bci);
227 }
228 static ByteSize cell_offset(int index) {
229 return byte_offset_of(DataLayout, _cells) + in_ByteSize(index * cell_size);
230 }
231 // Return a value which, when or-ed as a byte into _flags, sets the flag.
232 static u1 flag_number_to_constant(u1 flag_number) {
233 DataLayout temp; temp.set_header(0);
234 temp.set_flag_at(flag_number);
235 return temp._header._struct._flags;
236 }
237 // Return a value which, when or-ed as a word into _header, sets the flag.
238 static u8 flag_mask_to_header_mask(uint byte_constant) {
239 DataLayout temp; temp.set_header(0);
240 temp._header._struct._flags = byte_constant;
241 return temp._header._bits;
242 }
243
244 ProfileData* data_in();
245
246 int size_in_bytes() {
247 int cells = cell_count();
248 assert(cells >= 0, "invalid number of cells");
249 return DataLayout::compute_size_in_bytes(cells);
250 }
251 int cell_count();
252
253 // GC support
254 void clean_weak_klass_links(bool always_clean);
255 };
256
257
258 // ProfileData class hierarchy
259 class ProfileData;
260 class BitData;
261 class CounterData;
262 class ReceiverTypeData;
263 class VirtualCallData;
264 class VirtualCallTypeData;
265 class RetData;
266 class CallTypeData;
267 class JumpData;
268 class BranchData;
269 class ACmpData;
270 class ArrayData;
271 class MultiBranchData;
272 class ArgInfoData;
273 class ParametersTypeData;
274 class SpeculativeTrapData;
275 class ArrayLoadStoreData;
276
277 // ProfileData
278 //
279 // A ProfileData object is created to refer to a section of profiling
280 // data in a structured way.
281 class ProfileData : public ResourceObj {
282 friend class TypeEntries;
283 friend class SingleTypeEntry;
284 friend class TypeStackSlotEntries;
285 private:
286 enum {
287 tab_width_one = 16,
288 tab_width_two = 36
289 };
290
291 // This is a pointer to a section of profiling data.
292 DataLayout* _data;
293
294 char* print_data_on_helper(const MethodData* md) const;
295
296 protected:
297 DataLayout* data() { return _data; }
298 const DataLayout* data() const { return _data; }
299
300 enum {
301 cell_size = DataLayout::cell_size
302 };
303
304 public:
305 // How many cells are in this?
306 virtual int cell_count() const {
307 ShouldNotReachHere();
308 return -1;
309 }
310
311 // Return the size of this data.
312 int size_in_bytes() {
313 return DataLayout::compute_size_in_bytes(cell_count());
314 }
315
316 protected:
317 // Low-level accessors for underlying data
318 void set_intptr_at(int index, intptr_t value) {
319 assert(0 <= index && index < cell_count(), "oob");
320 data()->set_cell_at(index, value);
321 }
322 void release_set_intptr_at(int index, intptr_t value);
323 intptr_t intptr_at(int index) const {
324 assert(0 <= index && index < cell_count(), "oob");
325 return data()->cell_at(index);
326 }
327 void set_uint_at(int index, uint value) {
328 set_intptr_at(index, (intptr_t) value);
329 }
330 void release_set_uint_at(int index, uint value);
331 uint uint_at(int index) const {
332 return (uint)intptr_at(index);
333 }
334 void set_int_at(int index, int value) {
335 set_intptr_at(index, (intptr_t) value);
336 }
337 void release_set_int_at(int index, int value);
338 int int_at(int index) const {
339 return (int)intptr_at(index);
340 }
341 int int_at_unchecked(int index) const {
342 return (int)data()->cell_at(index);
343 }
344 void set_oop_at(int index, oop value) {
345 set_intptr_at(index, cast_from_oop<intptr_t>(value));
346 }
347 oop oop_at(int index) const {
348 return cast_to_oop(intptr_at(index));
349 }
350
351 void set_flag_at(int flag_number) {
352 data()->set_flag_at(flag_number);
353 }
354 bool flag_at(int flag_number) const {
355 return data()->flag_at(flag_number);
356 }
357
358 // two convenient imports for use by subclasses:
359 static ByteSize cell_offset(int index) {
360 return DataLayout::cell_offset(index);
361 }
362 static int flag_number_to_constant(int flag_number) {
363 return DataLayout::flag_number_to_constant(flag_number);
364 }
365
366 ProfileData(DataLayout* data) {
367 _data = data;
368 }
369
370 public:
371 // Constructor for invalid ProfileData.
372 ProfileData();
373
374 u2 bci() const {
375 return data()->bci();
376 }
377
378 address dp() {
379 return (address)_data;
380 }
381
382 int trap_state() const {
383 return data()->trap_state();
384 }
385 void set_trap_state(int new_state) {
386 data()->set_trap_state(new_state);
387 }
388
389 // Type checking
390 virtual bool is_BitData() const { return false; }
391 virtual bool is_CounterData() const { return false; }
392 virtual bool is_JumpData() const { return false; }
393 virtual bool is_ReceiverTypeData()const { return false; }
394 virtual bool is_VirtualCallData() const { return false; }
395 virtual bool is_RetData() const { return false; }
396 virtual bool is_BranchData() const { return false; }
397 virtual bool is_ArrayData() const { return false; }
398 virtual bool is_MultiBranchData() const { return false; }
399 virtual bool is_ArgInfoData() const { return false; }
400 virtual bool is_CallTypeData() const { return false; }
401 virtual bool is_VirtualCallTypeData()const { return false; }
402 virtual bool is_ParametersTypeData() const { return false; }
403 virtual bool is_SpeculativeTrapData()const { return false; }
404 virtual bool is_ArrayLoadStoreData() const { return false; }
405 virtual bool is_ACmpData() const { return false; }
406
407
408 BitData* as_BitData() const {
409 assert(is_BitData(), "wrong type");
410 return is_BitData() ? (BitData*) this : nullptr;
411 }
412 CounterData* as_CounterData() const {
413 assert(is_CounterData(), "wrong type");
414 return is_CounterData() ? (CounterData*) this : nullptr;
415 }
416 JumpData* as_JumpData() const {
417 assert(is_JumpData(), "wrong type");
418 return is_JumpData() ? (JumpData*) this : nullptr;
419 }
420 ReceiverTypeData* as_ReceiverTypeData() const {
421 assert(is_ReceiverTypeData(), "wrong type");
422 return is_ReceiverTypeData() ? (ReceiverTypeData*)this : nullptr;
423 }
424 VirtualCallData* as_VirtualCallData() const {
425 assert(is_VirtualCallData(), "wrong type");
426 return is_VirtualCallData() ? (VirtualCallData*)this : nullptr;
427 }
428 RetData* as_RetData() const {
429 assert(is_RetData(), "wrong type");
430 return is_RetData() ? (RetData*) this : nullptr;
431 }
432 BranchData* as_BranchData() const {
433 assert(is_BranchData(), "wrong type");
434 return is_BranchData() ? (BranchData*) this : nullptr;
435 }
436 ArrayData* as_ArrayData() const {
437 assert(is_ArrayData(), "wrong type");
438 return is_ArrayData() ? (ArrayData*) this : nullptr;
439 }
440 MultiBranchData* as_MultiBranchData() const {
441 assert(is_MultiBranchData(), "wrong type");
442 return is_MultiBranchData() ? (MultiBranchData*)this : nullptr;
443 }
444 ArgInfoData* as_ArgInfoData() const {
445 assert(is_ArgInfoData(), "wrong type");
446 return is_ArgInfoData() ? (ArgInfoData*)this : nullptr;
447 }
448 CallTypeData* as_CallTypeData() const {
449 assert(is_CallTypeData(), "wrong type");
450 return is_CallTypeData() ? (CallTypeData*)this : nullptr;
451 }
452 VirtualCallTypeData* as_VirtualCallTypeData() const {
453 assert(is_VirtualCallTypeData(), "wrong type");
454 return is_VirtualCallTypeData() ? (VirtualCallTypeData*)this : nullptr;
455 }
456 ParametersTypeData* as_ParametersTypeData() const {
457 assert(is_ParametersTypeData(), "wrong type");
458 return is_ParametersTypeData() ? (ParametersTypeData*)this : nullptr;
459 }
460 SpeculativeTrapData* as_SpeculativeTrapData() const {
461 assert(is_SpeculativeTrapData(), "wrong type");
462 return is_SpeculativeTrapData() ? (SpeculativeTrapData*)this : nullptr;
463 }
464 ArrayLoadStoreData* as_ArrayLoadStoreData() const {
465 assert(is_ArrayLoadStoreData(), "wrong type");
466 return is_ArrayLoadStoreData() ? (ArrayLoadStoreData*)this : NULL;
467 }
468 ACmpData* as_ACmpData() const {
469 assert(is_ACmpData(), "wrong type");
470 return is_ACmpData() ? (ACmpData*)this : NULL;
471 }
472
473
474 // Subclass specific initialization
475 virtual void post_initialize(BytecodeStream* stream, MethodData* mdo) {}
476
477 // GC support
478 virtual void clean_weak_klass_links(bool always_clean) {}
479
480 // CI translation: ProfileData can represent both MethodDataOop data
481 // as well as CIMethodData data. This function is provided for translating
482 // an oop in a ProfileData to the ci equivalent. Generally speaking,
483 // most ProfileData don't require any translation, so we provide the null
484 // translation here, and the required translators are in the ci subclasses.
485 virtual void translate_from(const ProfileData* data) {}
486
487 virtual void print_data_on(outputStream* st, const char* extra = nullptr) const {
488 ShouldNotReachHere();
489 }
490
491 void print_data_on(outputStream* st, const MethodData* md) const;
492
493 void print_shared(outputStream* st, const char* name, const char* extra) const;
494 void tab(outputStream* st, bool first = false) const;
495 };
496
497 // BitData
498 //
499 // A BitData holds a flag or two in its header.
500 class BitData : public ProfileData {
501 friend class VMStructs;
502 friend class JVMCIVMStructs;
503 protected:
504 enum {
505 // null_seen:
506 // saw a null operand (cast/aastore/instanceof)
507 null_seen_flag = DataLayout::first_flag + 0
508 #if INCLUDE_JVMCI
509 // bytecode threw any exception
510 , exception_seen_flag = null_seen_flag + 1
511 #endif
512 };
513 enum { bit_cell_count = 0 }; // no additional data fields needed.
514 public:
515 BitData(DataLayout* layout) : ProfileData(layout) {
516 }
517
518 virtual bool is_BitData() const { return true; }
519
520 static int static_cell_count() {
521 return bit_cell_count;
522 }
523
524 virtual int cell_count() const {
525 return static_cell_count();
526 }
527
528 // Accessor
529
530 // The null_seen flag bit is specially known to the interpreter.
531 // Consulting it allows the compiler to avoid setting up null_check traps.
532 bool null_seen() { return flag_at(null_seen_flag); }
533 void set_null_seen() { set_flag_at(null_seen_flag); }
534
535 #if INCLUDE_JVMCI
536 // true if an exception was thrown at the specific BCI
537 bool exception_seen() { return flag_at(exception_seen_flag); }
538 void set_exception_seen() { set_flag_at(exception_seen_flag); }
539 #endif
540
541 // Code generation support
542 static int null_seen_byte_constant() {
543 return flag_number_to_constant(null_seen_flag);
544 }
545
546 static ByteSize bit_data_size() {
547 return cell_offset(bit_cell_count);
548 }
549
550 void print_data_on(outputStream* st, const char* extra = nullptr) const;
551 };
552
553 // CounterData
554 //
555 // A CounterData corresponds to a simple counter.
556 class CounterData : public BitData {
557 friend class VMStructs;
558 friend class JVMCIVMStructs;
559 protected:
560 enum {
561 count_off,
562 counter_cell_count
563 };
564 public:
565 CounterData(DataLayout* layout) : BitData(layout) {}
566
567 virtual bool is_CounterData() const { return true; }
568
569 static int static_cell_count() {
570 return counter_cell_count;
571 }
572
573 virtual int cell_count() const {
574 return static_cell_count();
575 }
576
577 // Direct accessor
578 int count() const {
579 intptr_t raw_data = intptr_at(count_off);
580 if (raw_data > max_jint) {
581 raw_data = max_jint;
582 } else if (raw_data < min_jint) {
583 raw_data = min_jint;
584 }
585 return int(raw_data);
586 }
587
588 // Code generation support
589 static ByteSize count_offset() {
590 return cell_offset(count_off);
591 }
592 static ByteSize counter_data_size() {
593 return cell_offset(counter_cell_count);
594 }
595
596 void set_count(int count) {
597 set_int_at(count_off, count);
598 }
599
600 void print_data_on(outputStream* st, const char* extra = nullptr) const;
601 };
602
603 // JumpData
604 //
605 // A JumpData is used to access profiling information for a direct
606 // branch. It is a counter, used for counting the number of branches,
607 // plus a data displacement, used for realigning the data pointer to
608 // the corresponding target bci.
609 class JumpData : public ProfileData {
610 friend class VMStructs;
611 friend class JVMCIVMStructs;
612 protected:
613 enum {
614 taken_off_set,
615 displacement_off_set,
616 jump_cell_count
617 };
618
619 void set_displacement(int displacement) {
620 set_int_at(displacement_off_set, displacement);
621 }
622
623 public:
624 JumpData(DataLayout* layout) : ProfileData(layout) {
625 assert(layout->tag() == DataLayout::jump_data_tag ||
626 layout->tag() == DataLayout::branch_data_tag ||
627 layout->tag() == DataLayout::acmp_data_tag, "wrong type");
628 }
629
630 virtual bool is_JumpData() const { return true; }
631
632 static int static_cell_count() {
633 return jump_cell_count;
634 }
635
636 virtual int cell_count() const {
637 return static_cell_count();
638 }
639
640 // Direct accessor
641 uint taken() const {
642 return uint_at(taken_off_set);
643 }
644
645 void set_taken(uint cnt) {
646 set_uint_at(taken_off_set, cnt);
647 }
648
649 // Saturating counter
650 uint inc_taken() {
651 uint cnt = taken() + 1;
652 // Did we wrap? Will compiler screw us??
653 if (cnt == 0) cnt--;
654 set_uint_at(taken_off_set, cnt);
655 return cnt;
656 }
657
658 int displacement() const {
659 return int_at(displacement_off_set);
660 }
661
662 // Code generation support
663 static ByteSize taken_offset() {
664 return cell_offset(taken_off_set);
665 }
666
667 static ByteSize displacement_offset() {
668 return cell_offset(displacement_off_set);
669 }
670
671 // Specific initialization.
672 void post_initialize(BytecodeStream* stream, MethodData* mdo);
673
674 void print_data_on(outputStream* st, const char* extra = nullptr) const;
675 };
676
677 // Entries in a ProfileData object to record types: it can either be
678 // none (no profile), unknown (conflicting profile data) or a klass if
679 // a single one is seen. Whether a null reference was seen is also
680 // recorded. No counter is associated with the type and a single type
681 // is tracked (unlike VirtualCallData).
682 class TypeEntries {
683
684 public:
685
686 // A single cell is used to record information for a type:
687 // - the cell is initialized to 0
688 // - when a type is discovered it is stored in the cell
689 // - bit zero of the cell is used to record whether a null reference
690 // was encountered or not
691 // - bit 1 is set to record a conflict in the type information
692
693 enum {
694 null_seen = 1,
695 type_mask = ~null_seen,
696 type_unknown = 2,
697 status_bits = null_seen | type_unknown,
698 type_klass_mask = ~status_bits
699 };
700
701 // what to initialize a cell to
702 static intptr_t type_none() {
703 return 0;
704 }
705
706 // null seen = bit 0 set?
707 static bool was_null_seen(intptr_t v) {
708 return (v & null_seen) != 0;
709 }
710
711 // conflicting type information = bit 1 set?
712 static bool is_type_unknown(intptr_t v) {
713 return (v & type_unknown) != 0;
714 }
715
716 // not type information yet = all bits cleared, ignoring bit 0?
717 static bool is_type_none(intptr_t v) {
718 return (v & type_mask) == 0;
719 }
720
721 // recorded type: cell without bit 0 and 1
722 static intptr_t klass_part(intptr_t v) {
723 intptr_t r = v & type_klass_mask;
724 return r;
725 }
726
727 // type recorded
728 static Klass* valid_klass(intptr_t k) {
729 if (!is_type_none(k) &&
730 !is_type_unknown(k)) {
731 Klass* res = (Klass*)klass_part(k);
732 assert(res != nullptr, "invalid");
733 return res;
734 } else {
735 return nullptr;
736 }
737 }
738
739 static intptr_t with_status(intptr_t k, intptr_t in) {
740 return k | (in & status_bits);
741 }
742
743 static intptr_t with_status(Klass* k, intptr_t in) {
744 return with_status((intptr_t)k, in);
745 }
746
747 static void print_klass(outputStream* st, intptr_t k);
748
749 protected:
750 // ProfileData object these entries are part of
751 ProfileData* _pd;
752 // offset within the ProfileData object where the entries start
753 const int _base_off;
754
755 TypeEntries(int base_off)
756 : _pd(nullptr), _base_off(base_off) {}
757
758 void set_intptr_at(int index, intptr_t value) {
759 _pd->set_intptr_at(index, value);
760 }
761
762 intptr_t intptr_at(int index) const {
763 return _pd->intptr_at(index);
764 }
765
766 public:
767 void set_profile_data(ProfileData* pd) {
768 _pd = pd;
769 }
770 };
771
772 // Type entries used for arguments passed at a call and parameters on
773 // method entry. 2 cells per entry: one for the type encoded as in
774 // TypeEntries and one initialized with the stack slot where the
775 // profiled object is to be found so that the interpreter can locate
776 // it quickly.
777 class TypeStackSlotEntries : public TypeEntries {
778
779 private:
780 enum {
781 stack_slot_entry,
782 type_entry,
783 per_arg_cell_count
784 };
785
786 // offset of cell for stack slot for entry i within ProfileData object
787 int stack_slot_offset(int i) const {
788 return _base_off + stack_slot_local_offset(i);
789 }
790
791 const int _number_of_entries;
792
793 // offset of cell for type for entry i within ProfileData object
794 int type_offset_in_cells(int i) const {
795 return _base_off + type_local_offset(i);
796 }
797
798 public:
799
800 TypeStackSlotEntries(int base_off, int nb_entries)
801 : TypeEntries(base_off), _number_of_entries(nb_entries) {}
802
803 static int compute_cell_count(Symbol* signature, bool include_receiver, int max);
804
805 void post_initialize(Symbol* signature, bool has_receiver, bool include_receiver);
806
807 int number_of_entries() const { return _number_of_entries; }
808
809 // offset of cell for stack slot for entry i within this block of cells for a TypeStackSlotEntries
810 static int stack_slot_local_offset(int i) {
811 return i * per_arg_cell_count + stack_slot_entry;
812 }
813
814 // offset of cell for type for entry i within this block of cells for a TypeStackSlotEntries
815 static int type_local_offset(int i) {
816 return i * per_arg_cell_count + type_entry;
817 }
818
819 // stack slot for entry i
820 uint stack_slot(int i) const {
821 assert(i >= 0 && i < _number_of_entries, "oob");
822 return _pd->uint_at(stack_slot_offset(i));
823 }
824
825 // set stack slot for entry i
826 void set_stack_slot(int i, uint num) {
827 assert(i >= 0 && i < _number_of_entries, "oob");
828 _pd->set_uint_at(stack_slot_offset(i), num);
829 }
830
831 // type for entry i
832 intptr_t type(int i) const {
833 assert(i >= 0 && i < _number_of_entries, "oob");
834 return _pd->intptr_at(type_offset_in_cells(i));
835 }
836
837 // set type for entry i
838 void set_type(int i, intptr_t k) {
839 assert(i >= 0 && i < _number_of_entries, "oob");
840 _pd->set_intptr_at(type_offset_in_cells(i), k);
841 }
842
843 static ByteSize per_arg_size() {
844 return in_ByteSize(per_arg_cell_count * DataLayout::cell_size);
845 }
846
847 static int per_arg_count() {
848 return per_arg_cell_count;
849 }
850
851 ByteSize type_offset(int i) const {
852 return DataLayout::cell_offset(type_offset_in_cells(i));
853 }
854
855 // GC support
856 void clean_weak_klass_links(bool always_clean);
857
858 void print_data_on(outputStream* st) const;
859 };
860
861 // Type entry used for return from a call. A single cell to record the
862 // type.
863 class SingleTypeEntry : public TypeEntries {
864
865 private:
866 enum {
867 cell_count = 1
868 };
869
870 public:
871 SingleTypeEntry(int base_off)
872 : TypeEntries(base_off) {}
873
874 void post_initialize() {
875 set_type(type_none());
876 }
877
878 intptr_t type() const {
879 return _pd->intptr_at(_base_off);
880 }
881
882 void set_type(intptr_t k) {
883 _pd->set_intptr_at(_base_off, k);
884 }
885
886 static int static_cell_count() {
887 return cell_count;
888 }
889
890 static ByteSize size() {
891 return in_ByteSize(cell_count * DataLayout::cell_size);
892 }
893
894 ByteSize type_offset() {
895 return DataLayout::cell_offset(_base_off);
896 }
897
898 // GC support
899 void clean_weak_klass_links(bool always_clean);
900
901 void print_data_on(outputStream* st) const;
902 };
903
904 // Entries to collect type information at a call: contains arguments
905 // (TypeStackSlotEntries), a return type (SingleTypeEntry) and a
906 // number of cells. Because the number of cells for the return type is
907 // smaller than the number of cells for the type of an arguments, the
908 // number of cells is used to tell how many arguments are profiled and
909 // whether a return value is profiled. See has_arguments() and
910 // has_return().
911 class TypeEntriesAtCall {
912 private:
913 static int stack_slot_local_offset(int i) {
914 return header_cell_count() + TypeStackSlotEntries::stack_slot_local_offset(i);
915 }
916
917 static int argument_type_local_offset(int i) {
918 return header_cell_count() + TypeStackSlotEntries::type_local_offset(i);
919 }
920
921 public:
922
923 static int header_cell_count() {
924 return 1;
925 }
926
927 static int cell_count_local_offset() {
928 return 0;
929 }
930
931 static int compute_cell_count(BytecodeStream* stream);
932
933 static void initialize(DataLayout* dl, int base, int cell_count) {
934 int off = base + cell_count_local_offset();
935 dl->set_cell_at(off, cell_count - base - header_cell_count());
936 }
937
938 static bool arguments_profiling_enabled();
939 static bool return_profiling_enabled();
940
941 // Code generation support
942 static ByteSize cell_count_offset() {
943 return in_ByteSize(cell_count_local_offset() * DataLayout::cell_size);
944 }
945
946 static ByteSize args_data_offset() {
947 return in_ByteSize(header_cell_count() * DataLayout::cell_size);
948 }
949
950 static ByteSize stack_slot_offset(int i) {
951 return in_ByteSize(stack_slot_local_offset(i) * DataLayout::cell_size);
952 }
953
954 static ByteSize argument_type_offset(int i) {
955 return in_ByteSize(argument_type_local_offset(i) * DataLayout::cell_size);
956 }
957
958 static ByteSize return_only_size() {
959 return SingleTypeEntry::size() + in_ByteSize(header_cell_count() * DataLayout::cell_size);
960 }
961
962 };
963
964 // CallTypeData
965 //
966 // A CallTypeData is used to access profiling information about a non
967 // virtual call for which we collect type information about arguments
968 // and return value.
969 class CallTypeData : public CounterData {
970 private:
971 // entries for arguments if any
972 TypeStackSlotEntries _args;
973 // entry for return type if any
974 SingleTypeEntry _ret;
975
976 int cell_count_global_offset() const {
977 return CounterData::static_cell_count() + TypeEntriesAtCall::cell_count_local_offset();
978 }
979
980 // number of cells not counting the header
981 int cell_count_no_header() const {
982 return uint_at(cell_count_global_offset());
983 }
984
985 void check_number_of_arguments(int total) {
986 assert(number_of_arguments() == total, "should be set in DataLayout::initialize");
987 }
988
989 public:
990 CallTypeData(DataLayout* layout) :
991 CounterData(layout),
992 _args(CounterData::static_cell_count()+TypeEntriesAtCall::header_cell_count(), number_of_arguments()),
993 _ret(cell_count() - SingleTypeEntry::static_cell_count())
994 {
995 assert(layout->tag() == DataLayout::call_type_data_tag, "wrong type");
996 // Some compilers (VC++) don't want this passed in member initialization list
997 _args.set_profile_data(this);
998 _ret.set_profile_data(this);
999 }
1000
1001 const TypeStackSlotEntries* args() const {
1002 assert(has_arguments(), "no profiling of arguments");
1003 return &_args;
1004 }
1005
1006 const SingleTypeEntry* ret() const {
1007 assert(has_return(), "no profiling of return value");
1008 return &_ret;
1009 }
1010
1011 virtual bool is_CallTypeData() const { return true; }
1012
1013 static int static_cell_count() {
1014 return -1;
1015 }
1016
1017 static int compute_cell_count(BytecodeStream* stream) {
1018 return CounterData::static_cell_count() + TypeEntriesAtCall::compute_cell_count(stream);
1019 }
1020
1021 static void initialize(DataLayout* dl, int cell_count) {
1022 TypeEntriesAtCall::initialize(dl, CounterData::static_cell_count(), cell_count);
1023 }
1024
1025 virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1026
1027 virtual int cell_count() const {
1028 return CounterData::static_cell_count() +
1029 TypeEntriesAtCall::header_cell_count() +
1030 int_at_unchecked(cell_count_global_offset());
1031 }
1032
1033 int number_of_arguments() const {
1034 return cell_count_no_header() / TypeStackSlotEntries::per_arg_count();
1035 }
1036
1037 void set_argument_type(int i, Klass* k) {
1038 assert(has_arguments(), "no arguments!");
1039 intptr_t current = _args.type(i);
1040 _args.set_type(i, TypeEntries::with_status(k, current));
1041 }
1042
1043 void set_return_type(Klass* k) {
1044 assert(has_return(), "no return!");
1045 intptr_t current = _ret.type();
1046 _ret.set_type(TypeEntries::with_status(k, current));
1047 }
1048
1049 // An entry for a return value takes less space than an entry for an
1050 // argument so if the number of cells exceeds the number of cells
1051 // needed for an argument, this object contains type information for
1052 // at least one argument.
1053 bool has_arguments() const {
1054 bool res = cell_count_no_header() >= TypeStackSlotEntries::per_arg_count();
1055 assert (!res || TypeEntriesAtCall::arguments_profiling_enabled(), "no profiling of arguments");
1056 return res;
1057 }
1058
1059 // An entry for a return value takes less space than an entry for an
1060 // argument, so if the remainder of the number of cells divided by
1061 // the number of cells for an argument is not null, a return value
1062 // is profiled in this object.
1063 bool has_return() const {
1064 bool res = (cell_count_no_header() % TypeStackSlotEntries::per_arg_count()) != 0;
1065 assert (!res || TypeEntriesAtCall::return_profiling_enabled(), "no profiling of return values");
1066 return res;
1067 }
1068
1069 // Code generation support
1070 static ByteSize args_data_offset() {
1071 return cell_offset(CounterData::static_cell_count()) + TypeEntriesAtCall::args_data_offset();
1072 }
1073
1074 ByteSize argument_type_offset(int i) {
1075 return _args.type_offset(i);
1076 }
1077
1078 ByteSize return_type_offset() {
1079 return _ret.type_offset();
1080 }
1081
1082 // GC support
1083 virtual void clean_weak_klass_links(bool always_clean) {
1084 if (has_arguments()) {
1085 _args.clean_weak_klass_links(always_clean);
1086 }
1087 if (has_return()) {
1088 _ret.clean_weak_klass_links(always_clean);
1089 }
1090 }
1091
1092 virtual void print_data_on(outputStream* st, const char* extra = nullptr) const;
1093 };
1094
1095 // ReceiverTypeData
1096 //
1097 // A ReceiverTypeData is used to access profiling information about a
1098 // dynamic type check. It consists of a counter which counts the total times
1099 // that the check is reached, and a series of (Klass*, count) pairs
1100 // which are used to store a type profile for the receiver of the check.
1101 class ReceiverTypeData : public CounterData {
1102 friend class VMStructs;
1103 friend class JVMCIVMStructs;
1104 protected:
1105 enum {
1106 #if INCLUDE_JVMCI
1107 // Description of the different counters
1108 // ReceiverTypeData for instanceof/checkcast/aastore:
1109 // count is decremented for failed type checks
1110 // JVMCI only: nonprofiled_count is incremented on type overflow
1111 // VirtualCallData for invokevirtual/invokeinterface:
1112 // count is incremented on type overflow
1113 // JVMCI only: nonprofiled_count is incremented on method overflow
1114
1115 // JVMCI is interested in knowing the percentage of type checks involving a type not explicitly in the profile
1116 nonprofiled_count_off_set = counter_cell_count,
1117 receiver0_offset,
1118 #else
1119 receiver0_offset = counter_cell_count,
1120 #endif
1121 count0_offset,
1122 receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset
1123 };
1124
1125 public:
1126 ReceiverTypeData(DataLayout* layout) : CounterData(layout) {
1127 assert(layout->tag() == DataLayout::receiver_type_data_tag ||
1128 layout->tag() == DataLayout::virtual_call_data_tag ||
1129 layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1130 }
1131
1132 virtual bool is_ReceiverTypeData() const { return true; }
1133
1134 static int static_cell_count() {
1135 return counter_cell_count + (uint) TypeProfileWidth * receiver_type_row_cell_count JVMCI_ONLY(+ 1);
1136 }
1137
1138 virtual int cell_count() const {
1139 return static_cell_count();
1140 }
1141
1142 // Direct accessors
1143 static uint row_limit() {
1144 return TypeProfileWidth;
1145 }
1146 static int receiver_cell_index(uint row) {
1147 return receiver0_offset + row * receiver_type_row_cell_count;
1148 }
1149 static int receiver_count_cell_index(uint row) {
1150 return count0_offset + row * receiver_type_row_cell_count;
1151 }
1152
1153 Klass* receiver(uint row) const {
1154 assert(row < row_limit(), "oob");
1155
1156 Klass* recv = (Klass*)intptr_at(receiver_cell_index(row));
1157 assert(recv == nullptr || recv->is_klass(), "wrong type");
1158 return recv;
1159 }
1160
1161 void set_receiver(uint row, Klass* k) {
1162 assert((uint)row < row_limit(), "oob");
1163 set_intptr_at(receiver_cell_index(row), (uintptr_t)k);
1164 }
1165
1166 uint receiver_count(uint row) const {
1167 assert(row < row_limit(), "oob");
1168 return uint_at(receiver_count_cell_index(row));
1169 }
1170
1171 void set_receiver_count(uint row, uint count) {
1172 assert(row < row_limit(), "oob");
1173 set_uint_at(receiver_count_cell_index(row), count);
1174 }
1175
1176 void clear_row(uint row) {
1177 assert(row < row_limit(), "oob");
1178 // Clear total count - indicator of polymorphic call site.
1179 // The site may look like as monomorphic after that but
1180 // it allow to have more accurate profiling information because
1181 // there was execution phase change since klasses were unloaded.
1182 // If the site is still polymorphic then MDO will be updated
1183 // to reflect it. But it could be the case that the site becomes
1184 // only bimorphic. Then keeping total count not 0 will be wrong.
1185 // Even if we use monomorphic (when it is not) for compilation
1186 // we will only have trap, deoptimization and recompile again
1187 // with updated MDO after executing method in Interpreter.
1188 // An additional receiver will be recorded in the cleaned row
1189 // during next call execution.
1190 //
1191 // Note: our profiling logic works with empty rows in any slot.
1192 // We do sorting a profiling info (ciCallProfile) for compilation.
1193 //
1194 set_count(0);
1195 set_receiver(row, nullptr);
1196 set_receiver_count(row, 0);
1197 #if INCLUDE_JVMCI
1198 if (!this->is_VirtualCallData()) {
1199 // if this is a ReceiverTypeData for JVMCI, the nonprofiled_count
1200 // must also be reset (see "Description of the different counters" above)
1201 set_nonprofiled_count(0);
1202 }
1203 #endif
1204 }
1205
1206 // Code generation support
1207 static ByteSize receiver_offset(uint row) {
1208 return cell_offset(receiver_cell_index(row));
1209 }
1210 static ByteSize receiver_count_offset(uint row) {
1211 return cell_offset(receiver_count_cell_index(row));
1212 }
1213 #if INCLUDE_JVMCI
1214 static ByteSize nonprofiled_receiver_count_offset() {
1215 return cell_offset(nonprofiled_count_off_set);
1216 }
1217 uint nonprofiled_count() const {
1218 return uint_at(nonprofiled_count_off_set);
1219 }
1220 void set_nonprofiled_count(uint count) {
1221 set_uint_at(nonprofiled_count_off_set, count);
1222 }
1223 #endif // INCLUDE_JVMCI
1224 static ByteSize receiver_type_data_size() {
1225 return cell_offset(static_cell_count());
1226 }
1227
1228 // GC support
1229 virtual void clean_weak_klass_links(bool always_clean);
1230
1231 void print_receiver_data_on(outputStream* st) const;
1232 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1233 };
1234
1235 // VirtualCallData
1236 //
1237 // A VirtualCallData is used to access profiling information about a
1238 // virtual call. For now, it has nothing more than a ReceiverTypeData.
1239 class VirtualCallData : public ReceiverTypeData {
1240 public:
1241 VirtualCallData(DataLayout* layout) : ReceiverTypeData(layout) {
1242 assert(layout->tag() == DataLayout::virtual_call_data_tag ||
1243 layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1244 }
1245
1246 virtual bool is_VirtualCallData() const { return true; }
1247
1248 static int static_cell_count() {
1249 // At this point we could add more profile state, e.g., for arguments.
1250 // But for now it's the same size as the base record type.
1251 return ReceiverTypeData::static_cell_count();
1252 }
1253
1254 virtual int cell_count() const {
1255 return static_cell_count();
1256 }
1257
1258 // Direct accessors
1259 static ByteSize virtual_call_data_size() {
1260 return cell_offset(static_cell_count());
1261 }
1262
1263 void print_method_data_on(outputStream* st) const NOT_JVMCI_RETURN;
1264 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1265 };
1266
1267 // VirtualCallTypeData
1268 //
1269 // A VirtualCallTypeData is used to access profiling information about
1270 // a virtual call for which we collect type information about
1271 // arguments and return value.
1272 class VirtualCallTypeData : public VirtualCallData {
1273 private:
1274 // entries for arguments if any
1275 TypeStackSlotEntries _args;
1276 // entry for return type if any
1277 SingleTypeEntry _ret;
1278
1279 int cell_count_global_offset() const {
1280 return VirtualCallData::static_cell_count() + TypeEntriesAtCall::cell_count_local_offset();
1281 }
1282
1283 // number of cells not counting the header
1284 int cell_count_no_header() const {
1285 return uint_at(cell_count_global_offset());
1286 }
1287
1288 void check_number_of_arguments(int total) {
1289 assert(number_of_arguments() == total, "should be set in DataLayout::initialize");
1290 }
1291
1292 public:
1293 VirtualCallTypeData(DataLayout* layout) :
1294 VirtualCallData(layout),
1295 _args(VirtualCallData::static_cell_count()+TypeEntriesAtCall::header_cell_count(), number_of_arguments()),
1296 _ret(cell_count() - SingleTypeEntry::static_cell_count())
1297 {
1298 assert(layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1299 // Some compilers (VC++) don't want this passed in member initialization list
1300 _args.set_profile_data(this);
1301 _ret.set_profile_data(this);
1302 }
1303
1304 const TypeStackSlotEntries* args() const {
1305 assert(has_arguments(), "no profiling of arguments");
1306 return &_args;
1307 }
1308
1309 const SingleTypeEntry* ret() const {
1310 assert(has_return(), "no profiling of return value");
1311 return &_ret;
1312 }
1313
1314 virtual bool is_VirtualCallTypeData() const { return true; }
1315
1316 static int static_cell_count() {
1317 return -1;
1318 }
1319
1320 static int compute_cell_count(BytecodeStream* stream) {
1321 return VirtualCallData::static_cell_count() + TypeEntriesAtCall::compute_cell_count(stream);
1322 }
1323
1324 static void initialize(DataLayout* dl, int cell_count) {
1325 TypeEntriesAtCall::initialize(dl, VirtualCallData::static_cell_count(), cell_count);
1326 }
1327
1328 virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1329
1330 virtual int cell_count() const {
1331 return VirtualCallData::static_cell_count() +
1332 TypeEntriesAtCall::header_cell_count() +
1333 int_at_unchecked(cell_count_global_offset());
1334 }
1335
1336 int number_of_arguments() const {
1337 return cell_count_no_header() / TypeStackSlotEntries::per_arg_count();
1338 }
1339
1340 void set_argument_type(int i, Klass* k) {
1341 assert(has_arguments(), "no arguments!");
1342 intptr_t current = _args.type(i);
1343 _args.set_type(i, TypeEntries::with_status(k, current));
1344 }
1345
1346 void set_return_type(Klass* k) {
1347 assert(has_return(), "no return!");
1348 intptr_t current = _ret.type();
1349 _ret.set_type(TypeEntries::with_status(k, current));
1350 }
1351
1352 // An entry for a return value takes less space than an entry for an
1353 // argument, so if the remainder of the number of cells divided by
1354 // the number of cells for an argument is not null, a return value
1355 // is profiled in this object.
1356 bool has_return() const {
1357 bool res = (cell_count_no_header() % TypeStackSlotEntries::per_arg_count()) != 0;
1358 assert (!res || TypeEntriesAtCall::return_profiling_enabled(), "no profiling of return values");
1359 return res;
1360 }
1361
1362 // An entry for a return value takes less space than an entry for an
1363 // argument so if the number of cells exceeds the number of cells
1364 // needed for an argument, this object contains type information for
1365 // at least one argument.
1366 bool has_arguments() const {
1367 bool res = cell_count_no_header() >= TypeStackSlotEntries::per_arg_count();
1368 assert (!res || TypeEntriesAtCall::arguments_profiling_enabled(), "no profiling of arguments");
1369 return res;
1370 }
1371
1372 // Code generation support
1373 static ByteSize args_data_offset() {
1374 return cell_offset(VirtualCallData::static_cell_count()) + TypeEntriesAtCall::args_data_offset();
1375 }
1376
1377 ByteSize argument_type_offset(int i) {
1378 return _args.type_offset(i);
1379 }
1380
1381 ByteSize return_type_offset() {
1382 return _ret.type_offset();
1383 }
1384
1385 // GC support
1386 virtual void clean_weak_klass_links(bool always_clean) {
1387 ReceiverTypeData::clean_weak_klass_links(always_clean);
1388 if (has_arguments()) {
1389 _args.clean_weak_klass_links(always_clean);
1390 }
1391 if (has_return()) {
1392 _ret.clean_weak_klass_links(always_clean);
1393 }
1394 }
1395
1396 virtual void print_data_on(outputStream* st, const char* extra = nullptr) const;
1397 };
1398
1399 // RetData
1400 //
1401 // A RetData is used to access profiling information for a ret bytecode.
1402 // It is composed of a count of the number of times that the ret has
1403 // been executed, followed by a series of triples of the form
1404 // (bci, count, di) which count the number of times that some bci was the
1405 // target of the ret and cache a corresponding data displacement.
1406 class RetData : public CounterData {
1407 protected:
1408 enum {
1409 bci0_offset = counter_cell_count,
1410 count0_offset,
1411 displacement0_offset,
1412 ret_row_cell_count = (displacement0_offset + 1) - bci0_offset
1413 };
1414
1415 void set_bci(uint row, int bci) {
1416 assert((uint)row < row_limit(), "oob");
1417 set_int_at(bci0_offset + row * ret_row_cell_count, bci);
1418 }
1419 void release_set_bci(uint row, int bci);
1420 void set_bci_count(uint row, uint count) {
1421 assert((uint)row < row_limit(), "oob");
1422 set_uint_at(count0_offset + row * ret_row_cell_count, count);
1423 }
1424 void set_bci_displacement(uint row, int disp) {
1425 set_int_at(displacement0_offset + row * ret_row_cell_count, disp);
1426 }
1427
1428 public:
1429 RetData(DataLayout* layout) : CounterData(layout) {
1430 assert(layout->tag() == DataLayout::ret_data_tag, "wrong type");
1431 }
1432
1433 virtual bool is_RetData() const { return true; }
1434
1435 enum {
1436 no_bci = -1 // value of bci when bci1/2 are not in use.
1437 };
1438
1439 static int static_cell_count() {
1440 return counter_cell_count + (uint) BciProfileWidth * ret_row_cell_count;
1441 }
1442
1443 virtual int cell_count() const {
1444 return static_cell_count();
1445 }
1446
1447 static uint row_limit() {
1448 return BciProfileWidth;
1449 }
1450 static int bci_cell_index(uint row) {
1451 return bci0_offset + row * ret_row_cell_count;
1452 }
1453 static int bci_count_cell_index(uint row) {
1454 return count0_offset + row * ret_row_cell_count;
1455 }
1456 static int bci_displacement_cell_index(uint row) {
1457 return displacement0_offset + row * ret_row_cell_count;
1458 }
1459
1460 // Direct accessors
1461 int bci(uint row) const {
1462 return int_at(bci_cell_index(row));
1463 }
1464 uint bci_count(uint row) const {
1465 return uint_at(bci_count_cell_index(row));
1466 }
1467 int bci_displacement(uint row) const {
1468 return int_at(bci_displacement_cell_index(row));
1469 }
1470
1471 // Interpreter Runtime support
1472 address fixup_ret(int return_bci, MethodData* mdo);
1473
1474 // Code generation support
1475 static ByteSize bci_offset(uint row) {
1476 return cell_offset(bci_cell_index(row));
1477 }
1478 static ByteSize bci_count_offset(uint row) {
1479 return cell_offset(bci_count_cell_index(row));
1480 }
1481 static ByteSize bci_displacement_offset(uint row) {
1482 return cell_offset(bci_displacement_cell_index(row));
1483 }
1484
1485 // Specific initialization.
1486 void post_initialize(BytecodeStream* stream, MethodData* mdo);
1487
1488 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1489 };
1490
1491 // BranchData
1492 //
1493 // A BranchData is used to access profiling data for a two-way branch.
1494 // It consists of taken and not_taken counts as well as a data displacement
1495 // for the taken case.
1496 class BranchData : public JumpData {
1497 friend class VMStructs;
1498 friend class JVMCIVMStructs;
1499 protected:
1500 enum {
1501 not_taken_off_set = jump_cell_count,
1502 branch_cell_count
1503 };
1504
1505 void set_displacement(int displacement) {
1506 set_int_at(displacement_off_set, displacement);
1507 }
1508
1509 public:
1510 BranchData(DataLayout* layout) : JumpData(layout) {
1511 assert(layout->tag() == DataLayout::branch_data_tag || layout->tag() == DataLayout::acmp_data_tag, "wrong type");
1512 }
1513
1514 virtual bool is_BranchData() const { return true; }
1515
1516 static int static_cell_count() {
1517 return branch_cell_count;
1518 }
1519
1520 virtual int cell_count() const {
1521 return static_cell_count();
1522 }
1523
1524 // Direct accessor
1525 uint not_taken() const {
1526 return uint_at(not_taken_off_set);
1527 }
1528
1529 void set_not_taken(uint cnt) {
1530 set_uint_at(not_taken_off_set, cnt);
1531 }
1532
1533 uint inc_not_taken() {
1534 uint cnt = not_taken() + 1;
1535 // Did we wrap? Will compiler screw us??
1536 if (cnt == 0) cnt--;
1537 set_uint_at(not_taken_off_set, cnt);
1538 return cnt;
1539 }
1540
1541 // Code generation support
1542 static ByteSize not_taken_offset() {
1543 return cell_offset(not_taken_off_set);
1544 }
1545 static ByteSize branch_data_size() {
1546 return cell_offset(branch_cell_count);
1547 }
1548
1549 // Specific initialization.
1550 void post_initialize(BytecodeStream* stream, MethodData* mdo);
1551
1552 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1553 };
1554
1555 // ArrayData
1556 //
1557 // A ArrayData is a base class for accessing profiling data which does
1558 // not have a statically known size. It consists of an array length
1559 // and an array start.
1560 class ArrayData : public ProfileData {
1561 friend class VMStructs;
1562 friend class JVMCIVMStructs;
1563 protected:
1564 friend class DataLayout;
1565
1566 enum {
1567 array_len_off_set,
1568 array_start_off_set
1569 };
1570
1571 uint array_uint_at(int index) const {
1572 int aindex = index + array_start_off_set;
1573 return uint_at(aindex);
1574 }
1575 int array_int_at(int index) const {
1576 int aindex = index + array_start_off_set;
1577 return int_at(aindex);
1578 }
1579 oop array_oop_at(int index) const {
1580 int aindex = index + array_start_off_set;
1581 return oop_at(aindex);
1582 }
1583 void array_set_int_at(int index, int value) {
1584 int aindex = index + array_start_off_set;
1585 set_int_at(aindex, value);
1586 }
1587
1588 // Code generation support for subclasses.
1589 static ByteSize array_element_offset(int index) {
1590 return cell_offset(array_start_off_set + index);
1591 }
1592
1593 public:
1594 ArrayData(DataLayout* layout) : ProfileData(layout) {}
1595
1596 virtual bool is_ArrayData() const { return true; }
1597
1598 static int static_cell_count() {
1599 return -1;
1600 }
1601
1602 int array_len() const {
1603 return int_at_unchecked(array_len_off_set);
1604 }
1605
1606 virtual int cell_count() const {
1607 return array_len() + 1;
1608 }
1609
1610 // Code generation support
1611 static ByteSize array_len_offset() {
1612 return cell_offset(array_len_off_set);
1613 }
1614 static ByteSize array_start_offset() {
1615 return cell_offset(array_start_off_set);
1616 }
1617 };
1618
1619 // MultiBranchData
1620 //
1621 // A MultiBranchData is used to access profiling information for
1622 // a multi-way branch (*switch bytecodes). It consists of a series
1623 // of (count, displacement) pairs, which count the number of times each
1624 // case was taken and specify the data displacement for each branch target.
1625 class MultiBranchData : public ArrayData {
1626 friend class VMStructs;
1627 friend class JVMCIVMStructs;
1628 protected:
1629 enum {
1630 default_count_off_set,
1631 default_disaplacement_off_set,
1632 case_array_start
1633 };
1634 enum {
1635 relative_count_off_set,
1636 relative_displacement_off_set,
1637 per_case_cell_count
1638 };
1639
1640 void set_default_displacement(int displacement) {
1641 array_set_int_at(default_disaplacement_off_set, displacement);
1642 }
1643 void set_displacement_at(int index, int displacement) {
1644 array_set_int_at(case_array_start +
1645 index * per_case_cell_count +
1646 relative_displacement_off_set,
1647 displacement);
1648 }
1649
1650 public:
1651 MultiBranchData(DataLayout* layout) : ArrayData(layout) {
1652 assert(layout->tag() == DataLayout::multi_branch_data_tag, "wrong type");
1653 }
1654
1655 virtual bool is_MultiBranchData() const { return true; }
1656
1657 static int compute_cell_count(BytecodeStream* stream);
1658
1659 int number_of_cases() const {
1660 int alen = array_len() - 2; // get rid of default case here.
1661 assert(alen % per_case_cell_count == 0, "must be even");
1662 return (alen / per_case_cell_count);
1663 }
1664
1665 uint default_count() const {
1666 return array_uint_at(default_count_off_set);
1667 }
1668 int default_displacement() const {
1669 return array_int_at(default_disaplacement_off_set);
1670 }
1671
1672 uint count_at(int index) const {
1673 return array_uint_at(case_array_start +
1674 index * per_case_cell_count +
1675 relative_count_off_set);
1676 }
1677 int displacement_at(int index) const {
1678 return array_int_at(case_array_start +
1679 index * per_case_cell_count +
1680 relative_displacement_off_set);
1681 }
1682
1683 // Code generation support
1684 static ByteSize default_count_offset() {
1685 return array_element_offset(default_count_off_set);
1686 }
1687 static ByteSize default_displacement_offset() {
1688 return array_element_offset(default_disaplacement_off_set);
1689 }
1690 static ByteSize case_count_offset(int index) {
1691 return case_array_offset() +
1692 (per_case_size() * index) +
1693 relative_count_offset();
1694 }
1695 static ByteSize case_array_offset() {
1696 return array_element_offset(case_array_start);
1697 }
1698 static ByteSize per_case_size() {
1699 return in_ByteSize(per_case_cell_count) * cell_size;
1700 }
1701 static ByteSize relative_count_offset() {
1702 return in_ByteSize(relative_count_off_set) * cell_size;
1703 }
1704 static ByteSize relative_displacement_offset() {
1705 return in_ByteSize(relative_displacement_off_set) * cell_size;
1706 }
1707
1708 // Specific initialization.
1709 void post_initialize(BytecodeStream* stream, MethodData* mdo);
1710
1711 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1712 };
1713
1714 class ArgInfoData : public ArrayData {
1715
1716 public:
1717 ArgInfoData(DataLayout* layout) : ArrayData(layout) {
1718 assert(layout->tag() == DataLayout::arg_info_data_tag, "wrong type");
1719 }
1720
1721 virtual bool is_ArgInfoData() const { return true; }
1722
1723
1724 int number_of_args() const {
1725 return array_len();
1726 }
1727
1728 uint arg_modified(int arg) const {
1729 return array_uint_at(arg);
1730 }
1731
1732 void set_arg_modified(int arg, uint val) {
1733 array_set_int_at(arg, val);
1734 }
1735
1736 void print_data_on(outputStream* st, const char* extra = nullptr) const;
1737 };
1738
1739 // ParametersTypeData
1740 //
1741 // A ParametersTypeData is used to access profiling information about
1742 // types of parameters to a method
1743 class ParametersTypeData : public ArrayData {
1744
1745 private:
1746 TypeStackSlotEntries _parameters;
1747
1748 static int stack_slot_local_offset(int i) {
1749 assert_profiling_enabled();
1750 return array_start_off_set + TypeStackSlotEntries::stack_slot_local_offset(i);
1751 }
1752
1753 static int type_local_offset(int i) {
1754 assert_profiling_enabled();
1755 return array_start_off_set + TypeStackSlotEntries::type_local_offset(i);
1756 }
1757
1758 static bool profiling_enabled();
1759 static void assert_profiling_enabled() {
1760 assert(profiling_enabled(), "method parameters profiling should be on");
1761 }
1762
1763 public:
1764 ParametersTypeData(DataLayout* layout) : ArrayData(layout), _parameters(1, number_of_parameters()) {
1765 assert(layout->tag() == DataLayout::parameters_type_data_tag, "wrong type");
1766 // Some compilers (VC++) don't want this passed in member initialization list
1767 _parameters.set_profile_data(this);
1768 }
1769
1770 static int compute_cell_count(Method* m);
1771
1772 virtual bool is_ParametersTypeData() const { return true; }
1773
1774 virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1775
1776 int number_of_parameters() const {
1777 return array_len() / TypeStackSlotEntries::per_arg_count();
1778 }
1779
1780 const TypeStackSlotEntries* parameters() const { return &_parameters; }
1781
1782 uint stack_slot(int i) const {
1783 return _parameters.stack_slot(i);
1784 }
1785
1786 void set_type(int i, Klass* k) {
1787 intptr_t current = _parameters.type(i);
1788 _parameters.set_type(i, TypeEntries::with_status((intptr_t)k, current));
1789 }
1790
1791 virtual void clean_weak_klass_links(bool always_clean) {
1792 _parameters.clean_weak_klass_links(always_clean);
1793 }
1794
1795 virtual void print_data_on(outputStream* st, const char* extra = nullptr) const;
1796
1797 static ByteSize stack_slot_offset(int i) {
1798 return cell_offset(stack_slot_local_offset(i));
1799 }
1800
1801 static ByteSize type_offset(int i) {
1802 return cell_offset(type_local_offset(i));
1803 }
1804 };
1805
1806 // SpeculativeTrapData
1807 //
1808 // A SpeculativeTrapData is used to record traps due to type
1809 // speculation. It records the root of the compilation: that type
1810 // speculation is wrong in the context of one compilation (for
1811 // method1) doesn't mean it's wrong in the context of another one (for
1812 // method2). Type speculation could have more/different data in the
1813 // context of the compilation of method2 and it's worthwhile to try an
1814 // optimization that failed for compilation of method1 in the context
1815 // of compilation of method2.
1816 // Space for SpeculativeTrapData entries is allocated from the extra
1817 // data space in the MDO. If we run out of space, the trap data for
1818 // the ProfileData at that bci is updated.
1819 class SpeculativeTrapData : public ProfileData {
1820 protected:
1821 enum {
1822 speculative_trap_method,
1823 #ifndef _LP64
1824 // The size of the area for traps is a multiple of the header
1825 // size, 2 cells on 32 bits. Packed at the end of this area are
1826 // argument info entries (with tag
1827 // DataLayout::arg_info_data_tag). The logic in
1828 // MethodData::bci_to_extra_data() that guarantees traps don't
1829 // overflow over argument info entries assumes the size of a
1830 // SpeculativeTrapData is twice the header size. On 32 bits, a
1831 // SpeculativeTrapData must be 4 cells.
1832 padding,
1833 #endif
1834 speculative_trap_cell_count
1835 };
1836 public:
1837 SpeculativeTrapData(DataLayout* layout) : ProfileData(layout) {
1838 assert(layout->tag() == DataLayout::speculative_trap_data_tag, "wrong type");
1839 }
1840
1841 virtual bool is_SpeculativeTrapData() const { return true; }
1842
1843 static int static_cell_count() {
1844 return speculative_trap_cell_count;
1845 }
1846
1847 virtual int cell_count() const {
1848 return static_cell_count();
1849 }
1850
1851 // Direct accessor
1852 Method* method() const {
1853 return (Method*)intptr_at(speculative_trap_method);
1854 }
1855
1856 void set_method(Method* m) {
1857 assert(!m->is_old(), "cannot add old methods");
1858 set_intptr_at(speculative_trap_method, (intptr_t)m);
1859 }
1860
1861 static ByteSize method_offset() {
1862 return cell_offset(speculative_trap_method);
1863 }
1864
1865 virtual void print_data_on(outputStream* st, const char* extra = nullptr) const;
1866 };
1867
1868 class ArrayLoadStoreData : public ProfileData {
1869 private:
1870 enum {
1871 flat_array_flag = DataLayout::first_flag,
1872 null_free_array_flag = flat_array_flag + 1,
1873 };
1874
1875 SingleTypeEntry _array;
1876 SingleTypeEntry _element;
1877
1878 public:
1879 ArrayLoadStoreData(DataLayout* layout) :
1880 ProfileData(layout),
1881 _array(0),
1882 _element(SingleTypeEntry::static_cell_count()) {
1883 assert(layout->tag() == DataLayout::array_load_store_data_tag, "wrong type");
1884 _array.set_profile_data(this);
1885 _element.set_profile_data(this);
1886 }
1887
1888 const SingleTypeEntry* array() const {
1889 return &_array;
1890 }
1891
1892 const SingleTypeEntry* element() const {
1893 return &_element;
1894 }
1895
1896 virtual bool is_ArrayLoadStoreData() const { return true; }
1897
1898 static int static_cell_count() {
1899 return SingleTypeEntry::static_cell_count() * 2;
1900 }
1901
1902 virtual int cell_count() const {
1903 return static_cell_count();
1904 }
1905
1906 void set_flat_array() { set_flag_at(flat_array_flag); }
1907 bool flat_array() const { return flag_at(flat_array_flag); }
1908
1909 void set_null_free_array() { set_flag_at(null_free_array_flag); }
1910 bool null_free_array() const { return flag_at(null_free_array_flag); }
1911
1912 // Code generation support
1913 static int flat_array_byte_constant() {
1914 return flag_number_to_constant(flat_array_flag);
1915 }
1916
1917 static int null_free_array_byte_constant() {
1918 return flag_number_to_constant(null_free_array_flag);
1919 }
1920
1921 static ByteSize array_offset() {
1922 return cell_offset(0);
1923 }
1924
1925 static ByteSize element_offset() {
1926 return cell_offset(SingleTypeEntry::static_cell_count());
1927 }
1928
1929 virtual void clean_weak_klass_links(bool always_clean) {
1930 _array.clean_weak_klass_links(always_clean);
1931 _element.clean_weak_klass_links(always_clean);
1932 }
1933
1934 static ByteSize array_load_store_data_size() {
1935 return cell_offset(static_cell_count());
1936 }
1937
1938 virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
1939 };
1940
1941 class ACmpData : public BranchData {
1942 private:
1943 enum {
1944 left_inline_type_flag = DataLayout::first_flag,
1945 right_inline_type_flag
1946 };
1947
1948 SingleTypeEntry _left;
1949 SingleTypeEntry _right;
1950
1951 public:
1952 ACmpData(DataLayout* layout) :
1953 BranchData(layout),
1954 _left(BranchData::static_cell_count()),
1955 _right(BranchData::static_cell_count() + SingleTypeEntry::static_cell_count()) {
1956 assert(layout->tag() == DataLayout::acmp_data_tag, "wrong type");
1957 _left.set_profile_data(this);
1958 _right.set_profile_data(this);
1959 }
1960
1961 const SingleTypeEntry* left() const {
1962 return &_left;
1963 }
1964
1965 const SingleTypeEntry* right() const {
1966 return &_right;
1967 }
1968
1969 virtual bool is_ACmpData() const { return true; }
1970
1971 static int static_cell_count() {
1972 return BranchData::static_cell_count() + SingleTypeEntry::static_cell_count() * 2;
1973 }
1974
1975 virtual int cell_count() const {
1976 return static_cell_count();
1977 }
1978
1979 void set_left_inline_type() { set_flag_at(left_inline_type_flag); }
1980 bool left_inline_type() const { return flag_at(left_inline_type_flag); }
1981
1982 void set_right_inline_type() { set_flag_at(right_inline_type_flag); }
1983 bool right_inline_type() const { return flag_at(right_inline_type_flag); }
1984
1985 // Code generation support
1986 static int left_inline_type_byte_constant() {
1987 return flag_number_to_constant(left_inline_type_flag);
1988 }
1989
1990 static int right_inline_type_byte_constant() {
1991 return flag_number_to_constant(right_inline_type_flag);
1992 }
1993
1994 static ByteSize left_offset() {
1995 return cell_offset(BranchData::static_cell_count());
1996 }
1997
1998 static ByteSize right_offset() {
1999 return cell_offset(BranchData::static_cell_count() + SingleTypeEntry::static_cell_count());
2000 }
2001
2002 virtual void clean_weak_klass_links(bool always_clean) {
2003 _left.clean_weak_klass_links(always_clean);
2004 _right.clean_weak_klass_links(always_clean);
2005 }
2006
2007 static ByteSize acmp_data_size() {
2008 return cell_offset(static_cell_count());
2009 }
2010
2011 virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
2012 };
2013
2014 // MethodData*
2015 //
2016 // A MethodData* holds information which has been collected about
2017 // a method. Its layout looks like this:
2018 //
2019 // -----------------------------
2020 // | header |
2021 // | klass |
2022 // -----------------------------
2023 // | method |
2024 // | size of the MethodData* |
2025 // -----------------------------
2026 // | Data entries... |
2027 // | (variable size) |
2028 // | |
2029 // . .
2030 // . .
2031 // . .
2032 // | |
2033 // -----------------------------
2034 //
2035 // The data entry area is a heterogeneous array of DataLayouts. Each
2036 // DataLayout in the array corresponds to a specific bytecode in the
2037 // method. The entries in the array are sorted by the corresponding
2038 // bytecode. Access to the data is via resource-allocated ProfileData,
2039 // which point to the underlying blocks of DataLayout structures.
2040 //
2041 // During interpretation, if profiling in enabled, the interpreter
2042 // maintains a method data pointer (mdp), which points at the entry
2043 // in the array corresponding to the current bci. In the course of
2044 // interpretation, when a bytecode is encountered that has profile data
2045 // associated with it, the entry pointed to by mdp is updated, then the
2046 // mdp is adjusted to point to the next appropriate DataLayout. If mdp
2047 // is null to begin with, the interpreter assumes that the current method
2048 // is not (yet) being profiled.
2049 //
2050 // In MethodData* parlance, "dp" is a "data pointer", the actual address
2051 // of a DataLayout element. A "di" is a "data index", the offset in bytes
2052 // from the base of the data entry array. A "displacement" is the byte offset
2053 // in certain ProfileData objects that indicate the amount the mdp must be
2054 // adjusted in the event of a change in control flow.
2055 //
2056
2057 class CleanExtraDataClosure : public StackObj {
2058 public:
2059 virtual bool is_live(Method* m) = 0;
2060 };
2061
2062
2063 #if INCLUDE_JVMCI
2064 // Encapsulates an encoded speculation reason. These are linked together in
2065 // a list that is atomically appended to during deoptimization. Entries are
2066 // never removed from the list.
2067 // @see jdk.vm.ci.hotspot.HotSpotSpeculationLog.HotSpotSpeculationEncoding
2068 class FailedSpeculation: public CHeapObj<mtCompiler> {
2069 private:
2070 // The length of HotSpotSpeculationEncoding.toByteArray(). The data itself
2071 // is an array embedded at the end of this object.
2072 int _data_len;
2073
2074 // Next entry in a linked list.
2075 FailedSpeculation* _next;
2076
2077 FailedSpeculation(address data, int data_len);
2078
2079 FailedSpeculation** next_adr() { return &_next; }
2080
2081 // Placement new operator for inlining the speculation data into
2082 // the FailedSpeculation object.
2083 void* operator new(size_t size, size_t fs_size) throw();
2084
2085 public:
2086 char* data() { return (char*)(((address) this) + sizeof(FailedSpeculation)); }
2087 int data_len() const { return _data_len; }
2088 FailedSpeculation* next() const { return _next; }
2089
2090 // Atomically appends a speculation from nm to the list whose head is at (*failed_speculations_address).
2091 // Returns false if the FailedSpeculation object could not be allocated.
2092 static bool add_failed_speculation(nmethod* nm, FailedSpeculation** failed_speculations_address, address speculation, int speculation_len);
2093
2094 // Frees all entries in the linked list whose head is at (*failed_speculations_address).
2095 static void free_failed_speculations(FailedSpeculation** failed_speculations_address);
2096 };
2097 #endif
2098
2099 class ciMethodData;
2100
2101 class MethodData : public Metadata {
2102 friend class VMStructs;
2103 friend class JVMCIVMStructs;
2104 private:
2105 friend class ProfileData;
2106 friend class TypeEntriesAtCall;
2107 friend class ciMethodData;
2108
2109 // If you add a new field that points to any metaspace object, you
2110 // must add this field to MethodData::metaspace_pointers_do().
2111
2112 // Back pointer to the Method*
2113 Method* _method;
2114
2115 // Size of this oop in bytes
2116 int _size;
2117
2118 // Cached hint for bci_to_dp and bci_to_data
2119 int _hint_di;
2120
2121 Mutex _extra_data_lock;
2122
2123 MethodData(const methodHandle& method);
2124 public:
2125 static MethodData* allocate(ClassLoaderData* loader_data, const methodHandle& method, TRAPS);
2126
2127 virtual bool is_methodData() const { return true; }
2128 void initialize();
2129
2130 // Whole-method sticky bits and flags
2131 enum {
2132 _trap_hist_limit = Deoptimization::Reason_TRAP_HISTORY_LENGTH,
2133 _trap_hist_mask = max_jubyte,
2134 _extra_data_count = 4 // extra DataLayout headers, for trap history
2135 }; // Public flag values
2136
2137 // Compiler-related counters.
2138 class CompilerCounters {
2139 friend class VMStructs;
2140 friend class JVMCIVMStructs;
2141
2142 uint _nof_decompiles; // count of all nmethod removals
2143 uint _nof_overflow_recompiles; // recompile count, excluding recomp. bits
2144 uint _nof_overflow_traps; // trap count, excluding _trap_hist
2145 union {
2146 intptr_t _align;
2147 // JVMCI separates trap history for OSR compilations from normal compilations
2148 u1 _array[JVMCI_ONLY(2 *) MethodData::_trap_hist_limit];
2149 } _trap_hist;
2150
2151 public:
2152 CompilerCounters() : _nof_decompiles(0), _nof_overflow_recompiles(0), _nof_overflow_traps(0) {
2153 #ifndef ZERO
2154 // Some Zero platforms do not have expected alignment, and do not use
2155 // this code. static_assert would still fire and fail for them.
2156 static_assert(sizeof(_trap_hist) % HeapWordSize == 0, "align");
2157 #endif
2158 uint size_in_words = sizeof(_trap_hist) / HeapWordSize;
2159 Copy::zero_to_words((HeapWord*) &_trap_hist, size_in_words);
2160 }
2161
2162 // Return (uint)-1 for overflow.
2163 uint trap_count(int reason) const {
2164 assert((uint)reason < ARRAY_SIZE(_trap_hist._array), "oob");
2165 return (int)((_trap_hist._array[reason]+1) & _trap_hist_mask) - 1;
2166 }
2167
2168 uint inc_trap_count(int reason) {
2169 // Count another trap, anywhere in this method.
2170 assert(reason >= 0, "must be single trap");
2171 assert((uint)reason < ARRAY_SIZE(_trap_hist._array), "oob");
2172 uint cnt1 = 1 + _trap_hist._array[reason];
2173 if ((cnt1 & _trap_hist_mask) != 0) { // if no counter overflow...
2174 _trap_hist._array[reason] = cnt1;
2175 return cnt1;
2176 } else {
2177 return _trap_hist_mask + (++_nof_overflow_traps);
2178 }
2179 }
2180
2181 uint overflow_trap_count() const {
2182 return _nof_overflow_traps;
2183 }
2184 uint overflow_recompile_count() const {
2185 return _nof_overflow_recompiles;
2186 }
2187 uint inc_overflow_recompile_count() {
2188 return ++_nof_overflow_recompiles;
2189 }
2190 uint decompile_count() const {
2191 return _nof_decompiles;
2192 }
2193 uint inc_decompile_count() {
2194 return ++_nof_decompiles;
2195 }
2196
2197 // Support for code generation
2198 static ByteSize trap_history_offset() {
2199 return byte_offset_of(CompilerCounters, _trap_hist._array);
2200 }
2201 };
2202
2203 private:
2204 CompilerCounters _compiler_counters;
2205
2206 // Support for interprocedural escape analysis, from Thomas Kotzmann.
2207 intx _eflags; // flags on escape information
2208 intx _arg_local; // bit set of non-escaping arguments
2209 intx _arg_stack; // bit set of stack-allocatable arguments
2210 intx _arg_returned; // bit set of returned arguments
2211
2212 int _creation_mileage; // method mileage at MDO creation
2213
2214 // How many invocations has this MDO seen?
2215 // These counters are used to determine the exact age of MDO.
2216 // We need those because in tiered a method can be concurrently
2217 // executed at different levels.
2218 InvocationCounter _invocation_counter;
2219 // Same for backedges.
2220 InvocationCounter _backedge_counter;
2221 // Counter values at the time profiling started.
2222 int _invocation_counter_start;
2223 int _backedge_counter_start;
2224 uint _tenure_traps;
2225 int _invoke_mask; // per-method Tier0InvokeNotifyFreqLog
2226 int _backedge_mask; // per-method Tier0BackedgeNotifyFreqLog
2227
2228 #if INCLUDE_RTM_OPT
2229 // State of RTM code generation during compilation of the method
2230 int _rtm_state;
2231 #endif
2232
2233 // Number of loops and blocks is computed when compiling the first
2234 // time with C1. It is used to determine if method is trivial.
2235 short _num_loops;
2236 short _num_blocks;
2237 // Does this method contain anything worth profiling?
2238 enum WouldProfile {unknown, no_profile, profile};
2239 WouldProfile _would_profile;
2240
2241 #if INCLUDE_JVMCI
2242 // Support for HotSpotMethodData.setCompiledIRSize(int)
2243 int _jvmci_ir_size;
2244 FailedSpeculation* _failed_speculations;
2245 #endif
2246
2247 // Size of _data array in bytes. (Excludes header and extra_data fields.)
2248 int _data_size;
2249
2250 // data index for the area dedicated to parameters. -1 if no
2251 // parameter profiling.
2252 enum { no_parameters = -2, parameters_uninitialized = -1 };
2253 int _parameters_type_data_di;
2254
2255 // Beginning of the data entries
2256 intptr_t _data[1];
2257
2258 // Helper for size computation
2259 static int compute_data_size(BytecodeStream* stream);
2260 static int bytecode_cell_count(Bytecodes::Code code);
2261 static bool is_speculative_trap_bytecode(Bytecodes::Code code);
2262 enum { no_profile_data = -1, variable_cell_count = -2 };
2263
2264 // Helper for initialization
2265 DataLayout* data_layout_at(int data_index) const {
2266 assert(data_index % sizeof(intptr_t) == 0, "unaligned");
2267 return (DataLayout*) (((address)_data) + data_index);
2268 }
2269
2270 // Initialize an individual data segment. Returns the size of
2271 // the segment in bytes.
2272 int initialize_data(BytecodeStream* stream, int data_index);
2273
2274 // Helper for data_at
2275 DataLayout* limit_data_position() const {
2276 return data_layout_at(_data_size);
2277 }
2278 bool out_of_bounds(int data_index) const {
2279 return data_index >= data_size();
2280 }
2281
2282 // Give each of the data entries a chance to perform specific
2283 // data initialization.
2284 void post_initialize(BytecodeStream* stream);
2285
2286 // hint accessors
2287 int hint_di() const { return _hint_di; }
2288 void set_hint_di(int di) {
2289 assert(!out_of_bounds(di), "hint_di out of bounds");
2290 _hint_di = di;
2291 }
2292
2293 DataLayout* data_layout_before(int bci) {
2294 // avoid SEGV on this edge case
2295 if (data_size() == 0)
2296 return nullptr;
2297 DataLayout* layout = data_layout_at(hint_di());
2298 if (layout->bci() <= bci)
2299 return layout;
2300 return data_layout_at(first_di());
2301 }
2302
2303 // What is the index of the first data entry?
2304 int first_di() const { return 0; }
2305
2306 ProfileData* bci_to_extra_data_helper(int bci, Method* m, DataLayout*& dp, bool concurrent);
2307 // Find or create an extra ProfileData:
2308 ProfileData* bci_to_extra_data(int bci, Method* m, bool create_if_missing);
2309
2310 // return the argument info cell
2311 ArgInfoData *arg_info();
2312
2313 enum {
2314 no_type_profile = 0,
2315 type_profile_jsr292 = 1,
2316 type_profile_all = 2
2317 };
2318
2319 static bool profile_jsr292(const methodHandle& m, int bci);
2320 static bool profile_unsafe(const methodHandle& m, int bci);
2321 static bool profile_memory_access(const methodHandle& m, int bci);
2322 static int profile_arguments_flag();
2323 static bool profile_all_arguments();
2324 static bool profile_arguments_for_invoke(const methodHandle& m, int bci);
2325 static int profile_return_flag();
2326 static bool profile_all_return();
2327 static bool profile_return_for_invoke(const methodHandle& m, int bci);
2328 static int profile_parameters_flag();
2329 static bool profile_parameters_jsr292_only();
2330 static bool profile_all_parameters();
2331
2332 void clean_extra_data_helper(DataLayout* dp, int shift, bool reset = false);
2333 void verify_extra_data_clean(CleanExtraDataClosure* cl);
2334
2335 public:
2336 void clean_extra_data(CleanExtraDataClosure* cl);
2337
2338 static int header_size() {
2339 return sizeof(MethodData)/wordSize;
2340 }
2341
2342 // Compute the size of a MethodData* before it is created.
2343 static int compute_allocation_size_in_bytes(const methodHandle& method);
2344 static int compute_allocation_size_in_words(const methodHandle& method);
2345 static int compute_extra_data_count(int data_size, int empty_bc_count, bool needs_speculative_traps);
2346
2347 // Determine if a given bytecode can have profile information.
2348 static bool bytecode_has_profile(Bytecodes::Code code) {
2349 return bytecode_cell_count(code) != no_profile_data;
2350 }
2351
2352 // reset into original state
2353 void init();
2354
2355 // My size
2356 int size_in_bytes() const { return _size; }
2357 int size() const { return align_metadata_size(align_up(_size, BytesPerWord)/BytesPerWord); }
2358
2359 int creation_mileage() const { return _creation_mileage; }
2360 void set_creation_mileage(int x) { _creation_mileage = x; }
2361
2362 int invocation_count() {
2363 if (invocation_counter()->carry()) {
2364 return InvocationCounter::count_limit;
2365 }
2366 return invocation_counter()->count();
2367 }
2368 int backedge_count() {
2369 if (backedge_counter()->carry()) {
2370 return InvocationCounter::count_limit;
2371 }
2372 return backedge_counter()->count();
2373 }
2374
2375 int invocation_count_start() {
2376 if (invocation_counter()->carry()) {
2377 return 0;
2378 }
2379 return _invocation_counter_start;
2380 }
2381
2382 int backedge_count_start() {
2383 if (backedge_counter()->carry()) {
2384 return 0;
2385 }
2386 return _backedge_counter_start;
2387 }
2388
2389 int invocation_count_delta() { return invocation_count() - invocation_count_start(); }
2390 int backedge_count_delta() { return backedge_count() - backedge_count_start(); }
2391
2392 void reset_start_counters() {
2393 _invocation_counter_start = invocation_count();
2394 _backedge_counter_start = backedge_count();
2395 }
2396
2397 InvocationCounter* invocation_counter() { return &_invocation_counter; }
2398 InvocationCounter* backedge_counter() { return &_backedge_counter; }
2399
2400 #if INCLUDE_JVMCI
2401 FailedSpeculation** get_failed_speculations_address() {
2402 return &_failed_speculations;
2403 }
2404 #endif
2405
2406 #if INCLUDE_RTM_OPT
2407 int rtm_state() const {
2408 return _rtm_state;
2409 }
2410 void set_rtm_state(RTMState rstate) {
2411 _rtm_state = (int)rstate;
2412 }
2413 void atomic_set_rtm_state(RTMState rstate) {
2414 Atomic::store(&_rtm_state, (int)rstate);
2415 }
2416
2417 static ByteSize rtm_state_offset() {
2418 return byte_offset_of(MethodData, _rtm_state);
2419 }
2420 #endif
2421
2422 void set_would_profile(bool p) { _would_profile = p ? profile : no_profile; }
2423 bool would_profile() const { return _would_profile != no_profile; }
2424
2425 int num_loops() const { return _num_loops; }
2426 void set_num_loops(int n) { _num_loops = n; }
2427 int num_blocks() const { return _num_blocks; }
2428 void set_num_blocks(int n) { _num_blocks = n; }
2429
2430 bool is_mature() const; // consult mileage and ProfileMaturityPercentage
2431 static int mileage_of(Method* m);
2432
2433 // Support for interprocedural escape analysis, from Thomas Kotzmann.
2434 enum EscapeFlag {
2435 estimated = 1 << 0,
2436 return_local = 1 << 1,
2437 return_allocated = 1 << 2,
2438 allocated_escapes = 1 << 3,
2439 unknown_modified = 1 << 4
2440 };
2441
2442 intx eflags() { return _eflags; }
2443 intx arg_local() { return _arg_local; }
2444 intx arg_stack() { return _arg_stack; }
2445 intx arg_returned() { return _arg_returned; }
2446 uint arg_modified(int a) { ArgInfoData *aid = arg_info();
2447 assert(aid != nullptr, "arg_info must be not null");
2448 assert(a >= 0 && a < aid->number_of_args(), "valid argument number");
2449 return aid->arg_modified(a); }
2450
2451 void set_eflags(intx v) { _eflags = v; }
2452 void set_arg_local(intx v) { _arg_local = v; }
2453 void set_arg_stack(intx v) { _arg_stack = v; }
2454 void set_arg_returned(intx v) { _arg_returned = v; }
2455 void set_arg_modified(int a, uint v) { ArgInfoData *aid = arg_info();
2456 assert(aid != nullptr, "arg_info must be not null");
2457 assert(a >= 0 && a < aid->number_of_args(), "valid argument number");
2458 aid->set_arg_modified(a, v); }
2459
2460 void clear_escape_info() { _eflags = _arg_local = _arg_stack = _arg_returned = 0; }
2461
2462 // Location and size of data area
2463 address data_base() const {
2464 return (address) _data;
2465 }
2466 int data_size() const {
2467 return _data_size;
2468 }
2469
2470 int parameters_size_in_bytes() const {
2471 ParametersTypeData* param = parameters_type_data();
2472 return param == nullptr ? 0 : param->size_in_bytes();
2473 }
2474
2475 // Accessors
2476 Method* method() const { return _method; }
2477
2478 // Get the data at an arbitrary (sort of) data index.
2479 ProfileData* data_at(int data_index) const;
2480
2481 // Walk through the data in order.
2482 ProfileData* first_data() const { return data_at(first_di()); }
2483 ProfileData* next_data(ProfileData* current) const;
2484 DataLayout* next_data_layout(DataLayout* current) const;
2485 bool is_valid(ProfileData* current) const { return current != nullptr; }
2486 bool is_valid(DataLayout* current) const { return current != nullptr; }
2487
2488 // Convert a dp (data pointer) to a di (data index).
2489 int dp_to_di(address dp) const {
2490 return dp - ((address)_data);
2491 }
2492
2493 // bci to di/dp conversion.
2494 address bci_to_dp(int bci);
2495 int bci_to_di(int bci) {
2496 return dp_to_di(bci_to_dp(bci));
2497 }
2498
2499 // Get the data at an arbitrary bci, or null if there is none.
2500 ProfileData* bci_to_data(int bci);
2501
2502 // Same, but try to create an extra_data record if one is needed:
2503 ProfileData* allocate_bci_to_data(int bci, Method* m) {
2504 ProfileData* data = nullptr;
2505 // If m not null, try to allocate a SpeculativeTrapData entry
2506 if (m == nullptr) {
2507 data = bci_to_data(bci);
2508 }
2509 if (data != nullptr) {
2510 return data;
2511 }
2512 data = bci_to_extra_data(bci, m, true);
2513 if (data != nullptr) {
2514 return data;
2515 }
2516 // If SpeculativeTrapData allocation fails try to allocate a
2517 // regular entry
2518 data = bci_to_data(bci);
2519 if (data != nullptr) {
2520 return data;
2521 }
2522 return bci_to_extra_data(bci, nullptr, true);
2523 }
2524
2525 // Add a handful of extra data records, for trap tracking.
2526 DataLayout* extra_data_base() const { return limit_data_position(); }
2527 DataLayout* extra_data_limit() const { return (DataLayout*)((address)this + size_in_bytes()); }
2528 DataLayout* args_data_limit() const { return (DataLayout*)((address)this + size_in_bytes() -
2529 parameters_size_in_bytes()); }
2530 int extra_data_size() const { return (address)extra_data_limit() - (address)extra_data_base(); }
2531 static DataLayout* next_extra(DataLayout* dp);
2532
2533 // Return (uint)-1 for overflow.
2534 uint trap_count(int reason) const {
2535 return _compiler_counters.trap_count(reason);
2536 }
2537 // For loops:
2538 static uint trap_reason_limit() { return _trap_hist_limit; }
2539 static uint trap_count_limit() { return _trap_hist_mask; }
2540 uint inc_trap_count(int reason) {
2541 return _compiler_counters.inc_trap_count(reason);
2542 }
2543
2544 uint overflow_trap_count() const {
2545 return _compiler_counters.overflow_trap_count();
2546 }
2547 uint overflow_recompile_count() const {
2548 return _compiler_counters.overflow_recompile_count();
2549 }
2550 uint inc_overflow_recompile_count() {
2551 return _compiler_counters.inc_overflow_recompile_count();
2552 }
2553 uint decompile_count() const {
2554 return _compiler_counters.decompile_count();
2555 }
2556 uint inc_decompile_count() {
2557 uint dec_count = _compiler_counters.inc_decompile_count();
2558 if (dec_count > (uint)PerMethodRecompilationCutoff) {
2559 method()->set_not_compilable("decompile_count > PerMethodRecompilationCutoff", CompLevel_full_optimization);
2560 }
2561 return dec_count;
2562 }
2563 uint tenure_traps() const {
2564 return _tenure_traps;
2565 }
2566 void inc_tenure_traps() {
2567 _tenure_traps += 1;
2568 }
2569
2570 // Return pointer to area dedicated to parameters in MDO
2571 ParametersTypeData* parameters_type_data() const {
2572 assert(_parameters_type_data_di != parameters_uninitialized, "called too early");
2573 return _parameters_type_data_di != no_parameters ? data_layout_at(_parameters_type_data_di)->data_in()->as_ParametersTypeData() : nullptr;
2574 }
2575
2576 int parameters_type_data_di() const {
2577 assert(_parameters_type_data_di != parameters_uninitialized && _parameters_type_data_di != no_parameters, "no args type data");
2578 return _parameters_type_data_di;
2579 }
2580
2581 // Support for code generation
2582 static ByteSize data_offset() {
2583 return byte_offset_of(MethodData, _data[0]);
2584 }
2585
2586 static ByteSize trap_history_offset() {
2587 return byte_offset_of(MethodData, _compiler_counters) + CompilerCounters::trap_history_offset();
2588 }
2589
2590 static ByteSize invocation_counter_offset() {
2591 return byte_offset_of(MethodData, _invocation_counter);
2592 }
2593
2594 static ByteSize backedge_counter_offset() {
2595 return byte_offset_of(MethodData, _backedge_counter);
2596 }
2597
2598 static ByteSize invoke_mask_offset() {
2599 return byte_offset_of(MethodData, _invoke_mask);
2600 }
2601
2602 static ByteSize backedge_mask_offset() {
2603 return byte_offset_of(MethodData, _backedge_mask);
2604 }
2605
2606 static ByteSize parameters_type_data_di_offset() {
2607 return byte_offset_of(MethodData, _parameters_type_data_di);
2608 }
2609
2610 virtual void metaspace_pointers_do(MetaspaceClosure* iter);
2611 virtual MetaspaceObj::Type type() const { return MethodDataType; }
2612
2613 // Deallocation support
2614 void deallocate_contents(ClassLoaderData* loader_data);
2615 void release_C_heap_structures();
2616
2617 // GC support
2618 void set_size(int object_size_in_bytes) { _size = object_size_in_bytes; }
2619
2620 // Printing
2621 void print_on (outputStream* st) const;
2622 void print_value_on(outputStream* st) const;
2623
2624 // printing support for method data
2625 void print_data_on(outputStream* st) const;
2626
2627 const char* internal_name() const { return "{method data}"; }
2628
2629 // verification
2630 void verify_on(outputStream* st);
2631 void verify_data_on(outputStream* st);
2632
2633 static bool profile_parameters_for_method(const methodHandle& m);
2634 static bool profile_arguments();
2635 static bool profile_arguments_jsr292_only();
2636 static bool profile_return();
2637 static bool profile_parameters();
2638 static bool profile_return_jsr292_only();
2639
2640 void clean_method_data(bool always_clean);
2641 void clean_weak_method_links();
2642 Mutex* extra_data_lock() { return &_extra_data_lock; }
2643 };
2644
2645 #endif // SHARE_OOPS_METHODDATA_HPP