1 /*
   2  * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_OOPS_METHODDATA_HPP
  26 #define SHARE_OOPS_METHODDATA_HPP
  27 
  28 #include "interpreter/bytecodes.hpp"
  29 #include "oops/metadata.hpp"
  30 #include "oops/method.hpp"
  31 #include "oops/oop.hpp"
  32 #include "runtime/atomic.hpp"
  33 #include "runtime/deoptimization.hpp"
  34 #include "runtime/mutex.hpp"
  35 #include "utilities/align.hpp"
  36 #include "utilities/copy.hpp"
  37 
  38 class BytecodeStream;
  39 
  40 // The MethodData object collects counts and other profile information
  41 // during zeroth-tier (interpretive) and first-tier execution.
  42 // The profile is used later by compilation heuristics.  Some heuristics
  43 // enable use of aggressive (or "heroic") optimizations.  An aggressive
  44 // optimization often has a down-side, a corner case that it handles
  45 // poorly, but which is thought to be rare.  The profile provides
  46 // evidence of this rarity for a given method or even BCI.  It allows
  47 // the compiler to back out of the optimization at places where it
  48 // has historically been a poor choice.  Other heuristics try to use
  49 // specific information gathered about types observed at a given site.
  50 //
  51 // All data in the profile is approximate.  It is expected to be accurate
  52 // on the whole, but the system expects occasional inaccuraces, due to
  53 // counter overflow, multiprocessor races during data collection, space
  54 // limitations, missing MDO blocks, etc.  Bad or missing data will degrade
  55 // optimization quality but will not affect correctness.  Also, each MDO
  56 // is marked with its birth-date ("creation_mileage") which can be used
  57 // to assess the quality ("maturity") of its data.
  58 //
  59 // Short (<32-bit) counters are designed to overflow to a known "saturated"
  60 // state.  Also, certain recorded per-BCI events are given one-bit counters
  61 // which overflow to a saturated state which applied to all counters at
  62 // that BCI.  In other words, there is a small lattice which approximates
  63 // the ideal of an infinite-precision counter for each event at each BCI,
  64 // and the lattice quickly "bottoms out" in a state where all counters
  65 // are taken to be indefinitely large.
  66 //
  67 // The reader will find many data races in profile gathering code, starting
  68 // with invocation counter incrementation.  None of these races harm correct
  69 // execution of the compiled code.
  70 
  71 // forward decl
  72 class ProfileData;
  73 
  74 // DataLayout
  75 //
  76 // Overlay for generic profiling data.
  77 class DataLayout {
  78   friend class VMStructs;
  79   friend class JVMCIVMStructs;
  80 
  81 private:
  82   // Every data layout begins with a header.  This header
  83   // contains a tag, which is used to indicate the size/layout
  84   // of the data, 8 bits of flags, which can be used in any way,
  85   // 32 bits of trap history (none/one reason/many reasons),
  86   // and a bci, which is used to tie this piece of data to a
  87   // specific bci in the bytecodes.
  88   union {
  89     u8 _bits;
  90     struct {
  91       u1 _tag;
  92       u1 _flags;
  93       u2 _bci;
  94       u4 _traps;
  95     } _struct;
  96   } _header;
  97 
  98   // The data layout has an arbitrary number of cells, each sized
  99   // to accomodate a pointer or an integer.
 100   intptr_t _cells[1];
 101 
 102   // Some types of data layouts need a length field.
 103   static bool needs_array_len(u1 tag);
 104 
 105 public:
 106   enum {
 107     counter_increment = 1
 108   };
 109 
 110   enum {
 111     cell_size = sizeof(intptr_t)
 112   };
 113 
 114   // Tag values
 115   enum {
 116     no_tag,
 117     bit_data_tag,
 118     counter_data_tag,
 119     jump_data_tag,
 120     receiver_type_data_tag,
 121     virtual_call_data_tag,
 122     ret_data_tag,
 123     branch_data_tag,
 124     multi_branch_data_tag,
 125     arg_info_data_tag,
 126     call_type_data_tag,
 127     virtual_call_type_data_tag,
 128     parameters_type_data_tag,
 129     speculative_trap_data_tag
 130   };
 131 
 132   enum {
 133     // The trap state breaks down as [recompile:1 | reason:31].
 134     // This further breakdown is defined in deoptimization.cpp.
 135     // See Deoptimization::trap_state_reason for an assert that
 136     // trap_bits is big enough to hold reasons < Reason_RECORDED_LIMIT.
 137     //
 138     // The trap_state is collected only if ProfileTraps is true.
 139     trap_bits = 1+31,  // 31: enough to distinguish [0..Reason_RECORDED_LIMIT].
 140     trap_mask = -1,
 141     first_flag = 0
 142   };
 143 
 144   // Size computation
 145   static int header_size_in_bytes() {
 146     return header_size_in_cells() * cell_size;
 147   }
 148   static int header_size_in_cells() {
 149     return LP64_ONLY(1) NOT_LP64(2);
 150   }
 151 
 152   static int compute_size_in_bytes(int cell_count) {
 153     return header_size_in_bytes() + cell_count * cell_size;
 154   }
 155 
 156   // Initialization
 157   void initialize(u1 tag, u2 bci, int cell_count);
 158 
 159   // Accessors
 160   u1 tag() {
 161     return _header._struct._tag;
 162   }
 163 
 164   // Return 32 bits of trap state.
 165   // The state tells if traps with zero, one, or many reasons have occurred.
 166   // It also tells whether zero or many recompilations have occurred.
 167   // The associated trap histogram in the MDO itself tells whether
 168   // traps are common or not.  If a BCI shows that a trap X has
 169   // occurred, and the MDO shows N occurrences of X, we make the
 170   // simplifying assumption that all N occurrences can be blamed
 171   // on that BCI.
 172   uint trap_state() const {
 173     return _header._struct._traps;
 174   }
 175 
 176   void set_trap_state(uint new_state) {
 177     assert(ProfileTraps, "used only under +ProfileTraps");
 178     uint old_flags = _header._struct._traps;
 179     _header._struct._traps = new_state | old_flags;
 180   }
 181 
 182   u1 flags() const {
 183     return _header._struct._flags;
 184   }
 185 
 186   u2 bci() const {
 187     return _header._struct._bci;
 188   }
 189 
 190   void set_header(u8 value) {
 191     _header._bits = value;
 192   }
 193   u8 header() {
 194     return _header._bits;
 195   }
 196   void set_cell_at(int index, intptr_t value) {
 197     _cells[index] = value;
 198   }
 199   void release_set_cell_at(int index, intptr_t value);
 200   intptr_t cell_at(int index) const {
 201     return _cells[index];
 202   }
 203 
 204   void set_flag_at(u1 flag_number) {
 205     _header._struct._flags |= (0x1 << flag_number);
 206   }
 207   bool flag_at(u1 flag_number) const {
 208     return (_header._struct._flags & (0x1 << flag_number)) != 0;
 209   }
 210 
 211   // Low-level support for code generation.
 212   static ByteSize header_offset() {
 213     return byte_offset_of(DataLayout, _header);
 214   }
 215   static ByteSize tag_offset() {
 216     return byte_offset_of(DataLayout, _header._struct._tag);
 217   }
 218   static ByteSize flags_offset() {
 219     return byte_offset_of(DataLayout, _header._struct._flags);
 220   }
 221   static ByteSize bci_offset() {
 222     return byte_offset_of(DataLayout, _header._struct._bci);
 223   }
 224   static ByteSize cell_offset(int index) {
 225     return byte_offset_of(DataLayout, _cells) + in_ByteSize(index * cell_size);
 226   }
 227   // Return a value which, when or-ed as a byte into _flags, sets the flag.
 228   static u1 flag_number_to_constant(u1 flag_number) {
 229     DataLayout temp; temp.set_header(0);
 230     temp.set_flag_at(flag_number);
 231     return temp._header._struct._flags;
 232   }
 233   // Return a value which, when or-ed as a word into _header, sets the flag.
 234   static u8 flag_mask_to_header_mask(uint byte_constant) {
 235     DataLayout temp; temp.set_header(0);
 236     temp._header._struct._flags = byte_constant;
 237     return temp._header._bits;
 238   }
 239 
 240   ProfileData* data_in();
 241 
 242   int size_in_bytes() {
 243     int cells = cell_count();
 244     assert(cells >= 0, "invalid number of cells");
 245     return DataLayout::compute_size_in_bytes(cells);
 246   }
 247   int cell_count();
 248 
 249   // GC support
 250   void clean_weak_klass_links(bool always_clean);
 251 };
 252 
 253 
 254 // ProfileData class hierarchy
 255 class ProfileData;
 256 class   BitData;
 257 class     CounterData;
 258 class       ReceiverTypeData;
 259 class         VirtualCallData;
 260 class           VirtualCallTypeData;
 261 class       RetData;
 262 class       CallTypeData;
 263 class   JumpData;
 264 class     BranchData;
 265 class   ArrayData;
 266 class     MultiBranchData;
 267 class     ArgInfoData;
 268 class     ParametersTypeData;
 269 class   SpeculativeTrapData;
 270 
 271 // ProfileData
 272 //
 273 // A ProfileData object is created to refer to a section of profiling
 274 // data in a structured way.
 275 class ProfileData : public ResourceObj {
 276   friend class TypeEntries;
 277   friend class ReturnTypeEntry;
 278   friend class TypeStackSlotEntries;
 279 private:
 280   enum {
 281     tab_width_one = 16,
 282     tab_width_two = 36
 283   };
 284 
 285   // This is a pointer to a section of profiling data.
 286   DataLayout* _data;
 287 
 288   char* print_data_on_helper(const MethodData* md) const;
 289 
 290 protected:
 291   DataLayout* data() { return _data; }
 292   const DataLayout* data() const { return _data; }
 293 
 294   enum {
 295     cell_size = DataLayout::cell_size
 296   };
 297 
 298 public:
 299   // How many cells are in this?
 300   virtual int cell_count() const {
 301     ShouldNotReachHere();
 302     return -1;
 303   }
 304 
 305   // Return the size of this data.
 306   int size_in_bytes() {
 307     return DataLayout::compute_size_in_bytes(cell_count());
 308   }
 309 
 310 protected:
 311   // Low-level accessors for underlying data
 312   void set_intptr_at(int index, intptr_t value) {
 313     assert(0 <= index && index < cell_count(), "oob");
 314     data()->set_cell_at(index, value);
 315   }
 316   void release_set_intptr_at(int index, intptr_t value);
 317   intptr_t intptr_at(int index) const {
 318     assert(0 <= index && index < cell_count(), "oob");
 319     return data()->cell_at(index);
 320   }
 321   void set_uint_at(int index, uint value) {
 322     set_intptr_at(index, (intptr_t) value);
 323   }
 324   void release_set_uint_at(int index, uint value);
 325   uint uint_at(int index) const {
 326     return (uint)intptr_at(index);
 327   }
 328   void set_int_at(int index, int value) {
 329     set_intptr_at(index, (intptr_t) value);
 330   }
 331   void release_set_int_at(int index, int value);
 332   int int_at(int index) const {
 333     return (int)intptr_at(index);
 334   }
 335   int int_at_unchecked(int index) const {
 336     return (int)data()->cell_at(index);
 337   }
 338   void set_oop_at(int index, oop value) {
 339     set_intptr_at(index, cast_from_oop<intptr_t>(value));
 340   }
 341   oop oop_at(int index) const {
 342     return cast_to_oop(intptr_at(index));
 343   }
 344 
 345   void set_flag_at(int flag_number) {
 346     data()->set_flag_at(flag_number);
 347   }
 348   bool flag_at(int flag_number) const {
 349     return data()->flag_at(flag_number);
 350   }
 351 
 352   // two convenient imports for use by subclasses:
 353   static ByteSize cell_offset(int index) {
 354     return DataLayout::cell_offset(index);
 355   }
 356   static int flag_number_to_constant(int flag_number) {
 357     return DataLayout::flag_number_to_constant(flag_number);
 358   }
 359 
 360   ProfileData(DataLayout* data) {
 361     _data = data;
 362   }
 363 
 364 public:
 365   // Constructor for invalid ProfileData.
 366   ProfileData();
 367 
 368   u2 bci() const {
 369     return data()->bci();
 370   }
 371 
 372   address dp() {
 373     return (address)_data;
 374   }
 375 
 376   int trap_state() const {
 377     return data()->trap_state();
 378   }
 379   void set_trap_state(int new_state) {
 380     data()->set_trap_state(new_state);
 381   }
 382 
 383   // Type checking
 384   virtual bool is_BitData()         const { return false; }
 385   virtual bool is_CounterData()     const { return false; }
 386   virtual bool is_JumpData()        const { return false; }
 387   virtual bool is_ReceiverTypeData()const { return false; }
 388   virtual bool is_VirtualCallData() const { return false; }
 389   virtual bool is_RetData()         const { return false; }
 390   virtual bool is_BranchData()      const { return false; }
 391   virtual bool is_ArrayData()       const { return false; }
 392   virtual bool is_MultiBranchData() const { return false; }
 393   virtual bool is_ArgInfoData()     const { return false; }
 394   virtual bool is_CallTypeData()    const { return false; }
 395   virtual bool is_VirtualCallTypeData()const { return false; }
 396   virtual bool is_ParametersTypeData() const { return false; }
 397   virtual bool is_SpeculativeTrapData()const { return false; }
 398 
 399 
 400   BitData* as_BitData() const {
 401     assert(is_BitData(), "wrong type");
 402     return is_BitData()         ? (BitData*)        this : NULL;
 403   }
 404   CounterData* as_CounterData() const {
 405     assert(is_CounterData(), "wrong type");
 406     return is_CounterData()     ? (CounterData*)    this : NULL;
 407   }
 408   JumpData* as_JumpData() const {
 409     assert(is_JumpData(), "wrong type");
 410     return is_JumpData()        ? (JumpData*)       this : NULL;
 411   }
 412   ReceiverTypeData* as_ReceiverTypeData() const {
 413     assert(is_ReceiverTypeData(), "wrong type");
 414     return is_ReceiverTypeData() ? (ReceiverTypeData*)this : NULL;
 415   }
 416   VirtualCallData* as_VirtualCallData() const {
 417     assert(is_VirtualCallData(), "wrong type");
 418     return is_VirtualCallData() ? (VirtualCallData*)this : NULL;
 419   }
 420   RetData* as_RetData() const {
 421     assert(is_RetData(), "wrong type");
 422     return is_RetData()         ? (RetData*)        this : NULL;
 423   }
 424   BranchData* as_BranchData() const {
 425     assert(is_BranchData(), "wrong type");
 426     return is_BranchData()      ? (BranchData*)     this : NULL;
 427   }
 428   ArrayData* as_ArrayData() const {
 429     assert(is_ArrayData(), "wrong type");
 430     return is_ArrayData()       ? (ArrayData*)      this : NULL;
 431   }
 432   MultiBranchData* as_MultiBranchData() const {
 433     assert(is_MultiBranchData(), "wrong type");
 434     return is_MultiBranchData() ? (MultiBranchData*)this : NULL;
 435   }
 436   ArgInfoData* as_ArgInfoData() const {
 437     assert(is_ArgInfoData(), "wrong type");
 438     return is_ArgInfoData() ? (ArgInfoData*)this : NULL;
 439   }
 440   CallTypeData* as_CallTypeData() const {
 441     assert(is_CallTypeData(), "wrong type");
 442     return is_CallTypeData() ? (CallTypeData*)this : NULL;
 443   }
 444   VirtualCallTypeData* as_VirtualCallTypeData() const {
 445     assert(is_VirtualCallTypeData(), "wrong type");
 446     return is_VirtualCallTypeData() ? (VirtualCallTypeData*)this : NULL;
 447   }
 448   ParametersTypeData* as_ParametersTypeData() const {
 449     assert(is_ParametersTypeData(), "wrong type");
 450     return is_ParametersTypeData() ? (ParametersTypeData*)this : NULL;
 451   }
 452   SpeculativeTrapData* as_SpeculativeTrapData() const {
 453     assert(is_SpeculativeTrapData(), "wrong type");
 454     return is_SpeculativeTrapData() ? (SpeculativeTrapData*)this : NULL;
 455   }
 456 
 457 
 458   // Subclass specific initialization
 459   virtual void post_initialize(BytecodeStream* stream, MethodData* mdo) {}
 460 
 461   // GC support
 462   virtual void clean_weak_klass_links(bool always_clean) {}
 463 
 464   // CI translation: ProfileData can represent both MethodDataOop data
 465   // as well as CIMethodData data. This function is provided for translating
 466   // an oop in a ProfileData to the ci equivalent. Generally speaking,
 467   // most ProfileData don't require any translation, so we provide the null
 468   // translation here, and the required translators are in the ci subclasses.
 469   virtual void translate_from(const ProfileData* data) {}
 470 
 471   virtual void print_data_on(outputStream* st, const char* extra = NULL) const {
 472     ShouldNotReachHere();
 473   }
 474 
 475   void print_data_on(outputStream* st, const MethodData* md) const;
 476 
 477   void print_shared(outputStream* st, const char* name, const char* extra) const;
 478   void tab(outputStream* st, bool first = false) const;
 479 };
 480 
 481 // BitData
 482 //
 483 // A BitData holds a flag or two in its header.
 484 class BitData : public ProfileData {
 485   friend class VMStructs;
 486   friend class JVMCIVMStructs;
 487 protected:
 488   enum {
 489     // null_seen:
 490     //  saw a null operand (cast/aastore/instanceof)
 491       null_seen_flag              = DataLayout::first_flag + 0
 492 #if INCLUDE_JVMCI
 493     // bytecode threw any exception
 494     , exception_seen_flag         = null_seen_flag + 1
 495 #endif
 496   };
 497   enum { bit_cell_count = 0 };  // no additional data fields needed.
 498 public:
 499   BitData(DataLayout* layout) : ProfileData(layout) {
 500   }
 501 
 502   virtual bool is_BitData() const { return true; }
 503 
 504   static int static_cell_count() {
 505     return bit_cell_count;
 506   }
 507 
 508   virtual int cell_count() const {
 509     return static_cell_count();
 510   }
 511 
 512   // Accessor
 513 
 514   // The null_seen flag bit is specially known to the interpreter.
 515   // Consulting it allows the compiler to avoid setting up null_check traps.
 516   bool null_seen()     { return flag_at(null_seen_flag); }
 517   void set_null_seen()    { set_flag_at(null_seen_flag); }
 518 
 519 #if INCLUDE_JVMCI
 520   // true if an exception was thrown at the specific BCI
 521   bool exception_seen() { return flag_at(exception_seen_flag); }
 522   void set_exception_seen() { set_flag_at(exception_seen_flag); }
 523 #endif
 524 
 525   // Code generation support
 526   static int null_seen_byte_constant() {
 527     return flag_number_to_constant(null_seen_flag);
 528   }
 529 
 530   static ByteSize bit_data_size() {
 531     return cell_offset(bit_cell_count);
 532   }
 533 
 534   void print_data_on(outputStream* st, const char* extra = NULL) const;
 535 };
 536 
 537 // CounterData
 538 //
 539 // A CounterData corresponds to a simple counter.
 540 class CounterData : public BitData {
 541   friend class VMStructs;
 542   friend class JVMCIVMStructs;
 543 protected:
 544   enum {
 545     count_off,
 546     counter_cell_count
 547   };
 548 public:
 549   CounterData(DataLayout* layout) : BitData(layout) {}
 550 
 551   virtual bool is_CounterData() const { return true; }
 552 
 553   static int static_cell_count() {
 554     return counter_cell_count;
 555   }
 556 
 557   virtual int cell_count() const {
 558     return static_cell_count();
 559   }
 560 
 561   // Direct accessor
 562   int count() const {
 563     intptr_t raw_data = intptr_at(count_off);
 564     if (raw_data > max_jint) {
 565       raw_data = max_jint;
 566     } else if (raw_data < min_jint) {
 567       raw_data = min_jint;
 568     }
 569     return int(raw_data);
 570   }
 571 
 572   // Code generation support
 573   static ByteSize count_offset() {
 574     return cell_offset(count_off);
 575   }
 576   static ByteSize counter_data_size() {
 577     return cell_offset(counter_cell_count);
 578   }
 579 
 580   void set_count(int count) {
 581     set_int_at(count_off, count);
 582   }
 583 
 584   void print_data_on(outputStream* st, const char* extra = NULL) const;
 585 };
 586 
 587 // JumpData
 588 //
 589 // A JumpData is used to access profiling information for a direct
 590 // branch.  It is a counter, used for counting the number of branches,
 591 // plus a data displacement, used for realigning the data pointer to
 592 // the corresponding target bci.
 593 class JumpData : public ProfileData {
 594   friend class VMStructs;
 595   friend class JVMCIVMStructs;
 596 protected:
 597   enum {
 598     taken_off_set,
 599     displacement_off_set,
 600     jump_cell_count
 601   };
 602 
 603   void set_displacement(int displacement) {
 604     set_int_at(displacement_off_set, displacement);
 605   }
 606 
 607 public:
 608   JumpData(DataLayout* layout) : ProfileData(layout) {
 609     assert(layout->tag() == DataLayout::jump_data_tag ||
 610       layout->tag() == DataLayout::branch_data_tag, "wrong type");
 611   }
 612 
 613   virtual bool is_JumpData() const { return true; }
 614 
 615   static int static_cell_count() {
 616     return jump_cell_count;
 617   }
 618 
 619   virtual int cell_count() const {
 620     return static_cell_count();
 621   }
 622 
 623   // Direct accessor
 624   uint taken() const {
 625     return uint_at(taken_off_set);
 626   }
 627 
 628   void set_taken(uint cnt) {
 629     set_uint_at(taken_off_set, cnt);
 630   }
 631 
 632   // Saturating counter
 633   uint inc_taken() {
 634     uint cnt = taken() + 1;
 635     // Did we wrap? Will compiler screw us??
 636     if (cnt == 0) cnt--;
 637     set_uint_at(taken_off_set, cnt);
 638     return cnt;
 639   }
 640 
 641   int displacement() const {
 642     return int_at(displacement_off_set);
 643   }
 644 
 645   // Code generation support
 646   static ByteSize taken_offset() {
 647     return cell_offset(taken_off_set);
 648   }
 649 
 650   static ByteSize displacement_offset() {
 651     return cell_offset(displacement_off_set);
 652   }
 653 
 654   // Specific initialization.
 655   void post_initialize(BytecodeStream* stream, MethodData* mdo);
 656 
 657   void print_data_on(outputStream* st, const char* extra = NULL) const;
 658 };
 659 
 660 // Entries in a ProfileData object to record types: it can either be
 661 // none (no profile), unknown (conflicting profile data) or a klass if
 662 // a single one is seen. Whether a null reference was seen is also
 663 // recorded. No counter is associated with the type and a single type
 664 // is tracked (unlike VirtualCallData).
 665 class TypeEntries {
 666 
 667 public:
 668 
 669   // A single cell is used to record information for a type:
 670   // - the cell is initialized to 0
 671   // - when a type is discovered it is stored in the cell
 672   // - bit zero of the cell is used to record whether a null reference
 673   // was encountered or not
 674   // - bit 1 is set to record a conflict in the type information
 675 
 676   enum {
 677     null_seen = 1,
 678     type_mask = ~null_seen,
 679     type_unknown = 2,
 680     status_bits = null_seen | type_unknown,
 681     type_klass_mask = ~status_bits
 682   };
 683 
 684   // what to initialize a cell to
 685   static intptr_t type_none() {
 686     return 0;
 687   }
 688 
 689   // null seen = bit 0 set?
 690   static bool was_null_seen(intptr_t v) {
 691     return (v & null_seen) != 0;
 692   }
 693 
 694   // conflicting type information = bit 1 set?
 695   static bool is_type_unknown(intptr_t v) {
 696     return (v & type_unknown) != 0;
 697   }
 698 
 699   // not type information yet = all bits cleared, ignoring bit 0?
 700   static bool is_type_none(intptr_t v) {
 701     return (v & type_mask) == 0;
 702   }
 703 
 704   // recorded type: cell without bit 0 and 1
 705   static intptr_t klass_part(intptr_t v) {
 706     intptr_t r = v & type_klass_mask;
 707     return r;
 708   }
 709 
 710   // type recorded
 711   static Klass* valid_klass(intptr_t k) {
 712     if (!is_type_none(k) &&
 713         !is_type_unknown(k)) {
 714       Klass* res = (Klass*)klass_part(k);
 715       assert(res != NULL, "invalid");
 716       return res;
 717     } else {
 718       return NULL;
 719     }
 720   }
 721 
 722   static intptr_t with_status(intptr_t k, intptr_t in) {
 723     return k | (in & status_bits);
 724   }
 725 
 726   static intptr_t with_status(Klass* k, intptr_t in) {
 727     return with_status((intptr_t)k, in);
 728   }
 729 
 730   static void print_klass(outputStream* st, intptr_t k);
 731 
 732 protected:
 733   // ProfileData object these entries are part of
 734   ProfileData* _pd;
 735   // offset within the ProfileData object where the entries start
 736   const int _base_off;
 737 
 738   TypeEntries(int base_off)
 739     : _pd(NULL), _base_off(base_off) {}
 740 
 741   void set_intptr_at(int index, intptr_t value) {
 742     _pd->set_intptr_at(index, value);
 743   }
 744 
 745   intptr_t intptr_at(int index) const {
 746     return _pd->intptr_at(index);
 747   }
 748 
 749 public:
 750   void set_profile_data(ProfileData* pd) {
 751     _pd = pd;
 752   }
 753 };
 754 
 755 // Type entries used for arguments passed at a call and parameters on
 756 // method entry. 2 cells per entry: one for the type encoded as in
 757 // TypeEntries and one initialized with the stack slot where the
 758 // profiled object is to be found so that the interpreter can locate
 759 // it quickly.
 760 class TypeStackSlotEntries : public TypeEntries {
 761 
 762 private:
 763   enum {
 764     stack_slot_entry,
 765     type_entry,
 766     per_arg_cell_count
 767   };
 768 
 769   // offset of cell for stack slot for entry i within ProfileData object
 770   int stack_slot_offset(int i) const {
 771     return _base_off + stack_slot_local_offset(i);
 772   }
 773 
 774   const int _number_of_entries;
 775 
 776   // offset of cell for type for entry i within ProfileData object
 777   int type_offset_in_cells(int i) const {
 778     return _base_off + type_local_offset(i);
 779   }
 780 
 781 public:
 782 
 783   TypeStackSlotEntries(int base_off, int nb_entries)
 784     : TypeEntries(base_off), _number_of_entries(nb_entries) {}
 785 
 786   static int compute_cell_count(Symbol* signature, bool include_receiver, int max);
 787 
 788   void post_initialize(Symbol* signature, bool has_receiver, bool include_receiver);
 789 
 790   int number_of_entries() const { return _number_of_entries; }
 791 
 792   // offset of cell for stack slot for entry i within this block of cells for a TypeStackSlotEntries
 793   static int stack_slot_local_offset(int i) {
 794     return i * per_arg_cell_count + stack_slot_entry;
 795   }
 796 
 797   // offset of cell for type for entry i within this block of cells for a TypeStackSlotEntries
 798   static int type_local_offset(int i) {
 799     return i * per_arg_cell_count + type_entry;
 800   }
 801 
 802   // stack slot for entry i
 803   uint stack_slot(int i) const {
 804     assert(i >= 0 && i < _number_of_entries, "oob");
 805     return _pd->uint_at(stack_slot_offset(i));
 806   }
 807 
 808   // set stack slot for entry i
 809   void set_stack_slot(int i, uint num) {
 810     assert(i >= 0 && i < _number_of_entries, "oob");
 811     _pd->set_uint_at(stack_slot_offset(i), num);
 812   }
 813 
 814   // type for entry i
 815   intptr_t type(int i) const {
 816     assert(i >= 0 && i < _number_of_entries, "oob");
 817     return _pd->intptr_at(type_offset_in_cells(i));
 818   }
 819 
 820   // set type for entry i
 821   void set_type(int i, intptr_t k) {
 822     assert(i >= 0 && i < _number_of_entries, "oob");
 823     _pd->set_intptr_at(type_offset_in_cells(i), k);
 824   }
 825 
 826   static ByteSize per_arg_size() {
 827     return in_ByteSize(per_arg_cell_count * DataLayout::cell_size);
 828   }
 829 
 830   static int per_arg_count() {
 831     return per_arg_cell_count;
 832   }
 833 
 834   ByteSize type_offset(int i) const {
 835     return DataLayout::cell_offset(type_offset_in_cells(i));
 836   }
 837 
 838   // GC support
 839   void clean_weak_klass_links(bool always_clean);
 840 
 841   void print_data_on(outputStream* st) const;
 842 };
 843 
 844 // Type entry used for return from a call. A single cell to record the
 845 // type.
 846 class ReturnTypeEntry : public TypeEntries {
 847 
 848 private:
 849   enum {
 850     cell_count = 1
 851   };
 852 
 853 public:
 854   ReturnTypeEntry(int base_off)
 855     : TypeEntries(base_off) {}
 856 
 857   void post_initialize() {
 858     set_type(type_none());
 859   }
 860 
 861   intptr_t type() const {
 862     return _pd->intptr_at(_base_off);
 863   }
 864 
 865   void set_type(intptr_t k) {
 866     _pd->set_intptr_at(_base_off, k);
 867   }
 868 
 869   static int static_cell_count() {
 870     return cell_count;
 871   }
 872 
 873   static ByteSize size() {
 874     return in_ByteSize(cell_count * DataLayout::cell_size);
 875   }
 876 
 877   ByteSize type_offset() {
 878     return DataLayout::cell_offset(_base_off);
 879   }
 880 
 881   // GC support
 882   void clean_weak_klass_links(bool always_clean);
 883 
 884   void print_data_on(outputStream* st) const;
 885 };
 886 
 887 // Entries to collect type information at a call: contains arguments
 888 // (TypeStackSlotEntries), a return type (ReturnTypeEntry) and a
 889 // number of cells. Because the number of cells for the return type is
 890 // smaller than the number of cells for the type of an arguments, the
 891 // number of cells is used to tell how many arguments are profiled and
 892 // whether a return value is profiled. See has_arguments() and
 893 // has_return().
 894 class TypeEntriesAtCall {
 895 private:
 896   static int stack_slot_local_offset(int i) {
 897     return header_cell_count() + TypeStackSlotEntries::stack_slot_local_offset(i);
 898   }
 899 
 900   static int argument_type_local_offset(int i) {
 901     return header_cell_count() + TypeStackSlotEntries::type_local_offset(i);
 902   }
 903 
 904 public:
 905 
 906   static int header_cell_count() {
 907     return 1;
 908   }
 909 
 910   static int cell_count_local_offset() {
 911     return 0;
 912   }
 913 
 914   static int compute_cell_count(BytecodeStream* stream);
 915 
 916   static void initialize(DataLayout* dl, int base, int cell_count) {
 917     int off = base + cell_count_local_offset();
 918     dl->set_cell_at(off, cell_count - base - header_cell_count());
 919   }
 920 
 921   static bool arguments_profiling_enabled();
 922   static bool return_profiling_enabled();
 923 
 924   // Code generation support
 925   static ByteSize cell_count_offset() {
 926     return in_ByteSize(cell_count_local_offset() * DataLayout::cell_size);
 927   }
 928 
 929   static ByteSize args_data_offset() {
 930     return in_ByteSize(header_cell_count() * DataLayout::cell_size);
 931   }
 932 
 933   static ByteSize stack_slot_offset(int i) {
 934     return in_ByteSize(stack_slot_local_offset(i) * DataLayout::cell_size);
 935   }
 936 
 937   static ByteSize argument_type_offset(int i) {
 938     return in_ByteSize(argument_type_local_offset(i) * DataLayout::cell_size);
 939   }
 940 
 941   static ByteSize return_only_size() {
 942     return ReturnTypeEntry::size() + in_ByteSize(header_cell_count() * DataLayout::cell_size);
 943   }
 944 
 945 };
 946 
 947 // CallTypeData
 948 //
 949 // A CallTypeData is used to access profiling information about a non
 950 // virtual call for which we collect type information about arguments
 951 // and return value.
 952 class CallTypeData : public CounterData {
 953 private:
 954   // entries for arguments if any
 955   TypeStackSlotEntries _args;
 956   // entry for return type if any
 957   ReturnTypeEntry _ret;
 958 
 959   int cell_count_global_offset() const {
 960     return CounterData::static_cell_count() + TypeEntriesAtCall::cell_count_local_offset();
 961   }
 962 
 963   // number of cells not counting the header
 964   int cell_count_no_header() const {
 965     return uint_at(cell_count_global_offset());
 966   }
 967 
 968   void check_number_of_arguments(int total) {
 969     assert(number_of_arguments() == total, "should be set in DataLayout::initialize");
 970   }
 971 
 972 public:
 973   CallTypeData(DataLayout* layout) :
 974     CounterData(layout),
 975     _args(CounterData::static_cell_count()+TypeEntriesAtCall::header_cell_count(), number_of_arguments()),
 976     _ret(cell_count() - ReturnTypeEntry::static_cell_count())
 977   {
 978     assert(layout->tag() == DataLayout::call_type_data_tag, "wrong type");
 979     // Some compilers (VC++) don't want this passed in member initialization list
 980     _args.set_profile_data(this);
 981     _ret.set_profile_data(this);
 982   }
 983 
 984   const TypeStackSlotEntries* args() const {
 985     assert(has_arguments(), "no profiling of arguments");
 986     return &_args;
 987   }
 988 
 989   const ReturnTypeEntry* ret() const {
 990     assert(has_return(), "no profiling of return value");
 991     return &_ret;
 992   }
 993 
 994   virtual bool is_CallTypeData() const { return true; }
 995 
 996   static int static_cell_count() {
 997     return -1;
 998   }
 999 
1000   static int compute_cell_count(BytecodeStream* stream) {
1001     return CounterData::static_cell_count() + TypeEntriesAtCall::compute_cell_count(stream);
1002   }
1003 
1004   static void initialize(DataLayout* dl, int cell_count) {
1005     TypeEntriesAtCall::initialize(dl, CounterData::static_cell_count(), cell_count);
1006   }
1007 
1008   virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1009 
1010   virtual int cell_count() const {
1011     return CounterData::static_cell_count() +
1012       TypeEntriesAtCall::header_cell_count() +
1013       int_at_unchecked(cell_count_global_offset());
1014   }
1015 
1016   int number_of_arguments() const {
1017     return cell_count_no_header() / TypeStackSlotEntries::per_arg_count();
1018   }
1019 
1020   void set_argument_type(int i, Klass* k) {
1021     assert(has_arguments(), "no arguments!");
1022     intptr_t current = _args.type(i);
1023     _args.set_type(i, TypeEntries::with_status(k, current));
1024   }
1025 
1026   void set_return_type(Klass* k) {
1027     assert(has_return(), "no return!");
1028     intptr_t current = _ret.type();
1029     _ret.set_type(TypeEntries::with_status(k, current));
1030   }
1031 
1032   // An entry for a return value takes less space than an entry for an
1033   // argument so if the number of cells exceeds the number of cells
1034   // needed for an argument, this object contains type information for
1035   // at least one argument.
1036   bool has_arguments() const {
1037     bool res = cell_count_no_header() >= TypeStackSlotEntries::per_arg_count();
1038     assert (!res || TypeEntriesAtCall::arguments_profiling_enabled(), "no profiling of arguments");
1039     return res;
1040   }
1041 
1042   // An entry for a return value takes less space than an entry for an
1043   // argument, so if the remainder of the number of cells divided by
1044   // the number of cells for an argument is not null, a return value
1045   // is profiled in this object.
1046   bool has_return() const {
1047     bool res = (cell_count_no_header() % TypeStackSlotEntries::per_arg_count()) != 0;
1048     assert (!res || TypeEntriesAtCall::return_profiling_enabled(), "no profiling of return values");
1049     return res;
1050   }
1051 
1052   // Code generation support
1053   static ByteSize args_data_offset() {
1054     return cell_offset(CounterData::static_cell_count()) + TypeEntriesAtCall::args_data_offset();
1055   }
1056 
1057   ByteSize argument_type_offset(int i) {
1058     return _args.type_offset(i);
1059   }
1060 
1061   ByteSize return_type_offset() {
1062     return _ret.type_offset();
1063   }
1064 
1065   // GC support
1066   virtual void clean_weak_klass_links(bool always_clean) {
1067     if (has_arguments()) {
1068       _args.clean_weak_klass_links(always_clean);
1069     }
1070     if (has_return()) {
1071       _ret.clean_weak_klass_links(always_clean);
1072     }
1073   }
1074 
1075   virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
1076 };
1077 
1078 // ReceiverTypeData
1079 //
1080 // A ReceiverTypeData is used to access profiling information about a
1081 // dynamic type check.  It consists of a counter which counts the total times
1082 // that the check is reached, and a series of (Klass*, count) pairs
1083 // which are used to store a type profile for the receiver of the check.
1084 class ReceiverTypeData : public CounterData {
1085   friend class VMStructs;
1086   friend class JVMCIVMStructs;
1087 protected:
1088   enum {
1089 #if INCLUDE_JVMCI
1090     // Description of the different counters
1091     // ReceiverTypeData for instanceof/checkcast/aastore:
1092     //   count is decremented for failed type checks
1093     //   JVMCI only: nonprofiled_count is incremented on type overflow
1094     // VirtualCallData for invokevirtual/invokeinterface:
1095     //   count is incremented on type overflow
1096     //   JVMCI only: nonprofiled_count is incremented on method overflow
1097 
1098     // JVMCI is interested in knowing the percentage of type checks involving a type not explicitly in the profile
1099     nonprofiled_count_off_set = counter_cell_count,
1100     receiver0_offset,
1101 #else
1102     receiver0_offset = counter_cell_count,
1103 #endif
1104     count0_offset,
1105     receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset
1106   };
1107 
1108 public:
1109   ReceiverTypeData(DataLayout* layout) : CounterData(layout) {
1110     assert(layout->tag() == DataLayout::receiver_type_data_tag ||
1111            layout->tag() == DataLayout::virtual_call_data_tag ||
1112            layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1113   }
1114 
1115   virtual bool is_ReceiverTypeData() const { return true; }
1116 
1117   static int static_cell_count() {
1118     return counter_cell_count + (uint) TypeProfileWidth * receiver_type_row_cell_count JVMCI_ONLY(+ 1);
1119   }
1120 
1121   virtual int cell_count() const {
1122     return static_cell_count();
1123   }
1124 
1125   // Direct accessors
1126   static uint row_limit() {
1127     return TypeProfileWidth;
1128   }
1129   static int receiver_cell_index(uint row) {
1130     return receiver0_offset + row * receiver_type_row_cell_count;
1131   }
1132   static int receiver_count_cell_index(uint row) {
1133     return count0_offset + row * receiver_type_row_cell_count;
1134   }
1135 
1136   Klass* receiver(uint row) const {
1137     assert(row < row_limit(), "oob");
1138 
1139     Klass* recv = (Klass*)intptr_at(receiver_cell_index(row));
1140     assert(recv == NULL || recv->is_klass(), "wrong type");
1141     return recv;
1142   }
1143 
1144   void set_receiver(uint row, Klass* k) {
1145     assert((uint)row < row_limit(), "oob");
1146     set_intptr_at(receiver_cell_index(row), (uintptr_t)k);
1147   }
1148 
1149   uint receiver_count(uint row) const {
1150     assert(row < row_limit(), "oob");
1151     return uint_at(receiver_count_cell_index(row));
1152   }
1153 
1154   void set_receiver_count(uint row, uint count) {
1155     assert(row < row_limit(), "oob");
1156     set_uint_at(receiver_count_cell_index(row), count);
1157   }
1158 
1159   void clear_row(uint row) {
1160     assert(row < row_limit(), "oob");
1161     // Clear total count - indicator of polymorphic call site.
1162     // The site may look like as monomorphic after that but
1163     // it allow to have more accurate profiling information because
1164     // there was execution phase change since klasses were unloaded.
1165     // If the site is still polymorphic then MDO will be updated
1166     // to reflect it. But it could be the case that the site becomes
1167     // only bimorphic. Then keeping total count not 0 will be wrong.
1168     // Even if we use monomorphic (when it is not) for compilation
1169     // we will only have trap, deoptimization and recompile again
1170     // with updated MDO after executing method in Interpreter.
1171     // An additional receiver will be recorded in the cleaned row
1172     // during next call execution.
1173     //
1174     // Note: our profiling logic works with empty rows in any slot.
1175     // We do sorting a profiling info (ciCallProfile) for compilation.
1176     //
1177     set_count(0);
1178     set_receiver(row, NULL);
1179     set_receiver_count(row, 0);
1180 #if INCLUDE_JVMCI
1181     if (!this->is_VirtualCallData()) {
1182       // if this is a ReceiverTypeData for JVMCI, the nonprofiled_count
1183       // must also be reset (see "Description of the different counters" above)
1184       set_nonprofiled_count(0);
1185     }
1186 #endif
1187   }
1188 
1189   // Code generation support
1190   static ByteSize receiver_offset(uint row) {
1191     return cell_offset(receiver_cell_index(row));
1192   }
1193   static ByteSize receiver_count_offset(uint row) {
1194     return cell_offset(receiver_count_cell_index(row));
1195   }
1196 #if INCLUDE_JVMCI
1197   static ByteSize nonprofiled_receiver_count_offset() {
1198     return cell_offset(nonprofiled_count_off_set);
1199   }
1200   uint nonprofiled_count() const {
1201     return uint_at(nonprofiled_count_off_set);
1202   }
1203   void set_nonprofiled_count(uint count) {
1204     set_uint_at(nonprofiled_count_off_set, count);
1205   }
1206 #endif // INCLUDE_JVMCI
1207   static ByteSize receiver_type_data_size() {
1208     return cell_offset(static_cell_count());
1209   }
1210 
1211   // GC support
1212   virtual void clean_weak_klass_links(bool always_clean);
1213 
1214   void print_receiver_data_on(outputStream* st) const;
1215   void print_data_on(outputStream* st, const char* extra = NULL) const;
1216 };
1217 
1218 // VirtualCallData
1219 //
1220 // A VirtualCallData is used to access profiling information about a
1221 // virtual call.  For now, it has nothing more than a ReceiverTypeData.
1222 class VirtualCallData : public ReceiverTypeData {
1223 public:
1224   VirtualCallData(DataLayout* layout) : ReceiverTypeData(layout) {
1225     assert(layout->tag() == DataLayout::virtual_call_data_tag ||
1226            layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1227   }
1228 
1229   virtual bool is_VirtualCallData() const { return true; }
1230 
1231   static int static_cell_count() {
1232     // At this point we could add more profile state, e.g., for arguments.
1233     // But for now it's the same size as the base record type.
1234     return ReceiverTypeData::static_cell_count();
1235   }
1236 
1237   virtual int cell_count() const {
1238     return static_cell_count();
1239   }
1240 
1241   // Direct accessors
1242   static ByteSize virtual_call_data_size() {
1243     return cell_offset(static_cell_count());
1244   }
1245 
1246   void print_method_data_on(outputStream* st) const NOT_JVMCI_RETURN;
1247   void print_data_on(outputStream* st, const char* extra = NULL) const;
1248 };
1249 
1250 // VirtualCallTypeData
1251 //
1252 // A VirtualCallTypeData is used to access profiling information about
1253 // a virtual call for which we collect type information about
1254 // arguments and return value.
1255 class VirtualCallTypeData : public VirtualCallData {
1256 private:
1257   // entries for arguments if any
1258   TypeStackSlotEntries _args;
1259   // entry for return type if any
1260   ReturnTypeEntry _ret;
1261 
1262   int cell_count_global_offset() const {
1263     return VirtualCallData::static_cell_count() + TypeEntriesAtCall::cell_count_local_offset();
1264   }
1265 
1266   // number of cells not counting the header
1267   int cell_count_no_header() const {
1268     return uint_at(cell_count_global_offset());
1269   }
1270 
1271   void check_number_of_arguments(int total) {
1272     assert(number_of_arguments() == total, "should be set in DataLayout::initialize");
1273   }
1274 
1275 public:
1276   VirtualCallTypeData(DataLayout* layout) :
1277     VirtualCallData(layout),
1278     _args(VirtualCallData::static_cell_count()+TypeEntriesAtCall::header_cell_count(), number_of_arguments()),
1279     _ret(cell_count() - ReturnTypeEntry::static_cell_count())
1280   {
1281     assert(layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type");
1282     // Some compilers (VC++) don't want this passed in member initialization list
1283     _args.set_profile_data(this);
1284     _ret.set_profile_data(this);
1285   }
1286 
1287   const TypeStackSlotEntries* args() const {
1288     assert(has_arguments(), "no profiling of arguments");
1289     return &_args;
1290   }
1291 
1292   const ReturnTypeEntry* ret() const {
1293     assert(has_return(), "no profiling of return value");
1294     return &_ret;
1295   }
1296 
1297   virtual bool is_VirtualCallTypeData() const { return true; }
1298 
1299   static int static_cell_count() {
1300     return -1;
1301   }
1302 
1303   static int compute_cell_count(BytecodeStream* stream) {
1304     return VirtualCallData::static_cell_count() + TypeEntriesAtCall::compute_cell_count(stream);
1305   }
1306 
1307   static void initialize(DataLayout* dl, int cell_count) {
1308     TypeEntriesAtCall::initialize(dl, VirtualCallData::static_cell_count(), cell_count);
1309   }
1310 
1311   virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1312 
1313   virtual int cell_count() const {
1314     return VirtualCallData::static_cell_count() +
1315       TypeEntriesAtCall::header_cell_count() +
1316       int_at_unchecked(cell_count_global_offset());
1317   }
1318 
1319   int number_of_arguments() const {
1320     return cell_count_no_header() / TypeStackSlotEntries::per_arg_count();
1321   }
1322 
1323   void set_argument_type(int i, Klass* k) {
1324     assert(has_arguments(), "no arguments!");
1325     intptr_t current = _args.type(i);
1326     _args.set_type(i, TypeEntries::with_status(k, current));
1327   }
1328 
1329   void set_return_type(Klass* k) {
1330     assert(has_return(), "no return!");
1331     intptr_t current = _ret.type();
1332     _ret.set_type(TypeEntries::with_status(k, current));
1333   }
1334 
1335   // An entry for a return value takes less space than an entry for an
1336   // argument, so if the remainder of the number of cells divided by
1337   // the number of cells for an argument is not null, a return value
1338   // is profiled in this object.
1339   bool has_return() const {
1340     bool res = (cell_count_no_header() % TypeStackSlotEntries::per_arg_count()) != 0;
1341     assert (!res || TypeEntriesAtCall::return_profiling_enabled(), "no profiling of return values");
1342     return res;
1343   }
1344 
1345   // An entry for a return value takes less space than an entry for an
1346   // argument so if the number of cells exceeds the number of cells
1347   // needed for an argument, this object contains type information for
1348   // at least one argument.
1349   bool has_arguments() const {
1350     bool res = cell_count_no_header() >= TypeStackSlotEntries::per_arg_count();
1351     assert (!res || TypeEntriesAtCall::arguments_profiling_enabled(), "no profiling of arguments");
1352     return res;
1353   }
1354 
1355   // Code generation support
1356   static ByteSize args_data_offset() {
1357     return cell_offset(VirtualCallData::static_cell_count()) + TypeEntriesAtCall::args_data_offset();
1358   }
1359 
1360   ByteSize argument_type_offset(int i) {
1361     return _args.type_offset(i);
1362   }
1363 
1364   ByteSize return_type_offset() {
1365     return _ret.type_offset();
1366   }
1367 
1368   // GC support
1369   virtual void clean_weak_klass_links(bool always_clean) {
1370     ReceiverTypeData::clean_weak_klass_links(always_clean);
1371     if (has_arguments()) {
1372       _args.clean_weak_klass_links(always_clean);
1373     }
1374     if (has_return()) {
1375       _ret.clean_weak_klass_links(always_clean);
1376     }
1377   }
1378 
1379   virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
1380 };
1381 
1382 // RetData
1383 //
1384 // A RetData is used to access profiling information for a ret bytecode.
1385 // It is composed of a count of the number of times that the ret has
1386 // been executed, followed by a series of triples of the form
1387 // (bci, count, di) which count the number of times that some bci was the
1388 // target of the ret and cache a corresponding data displacement.
1389 class RetData : public CounterData {
1390 protected:
1391   enum {
1392     bci0_offset = counter_cell_count,
1393     count0_offset,
1394     displacement0_offset,
1395     ret_row_cell_count = (displacement0_offset + 1) - bci0_offset
1396   };
1397 
1398   void set_bci(uint row, int bci) {
1399     assert((uint)row < row_limit(), "oob");
1400     set_int_at(bci0_offset + row * ret_row_cell_count, bci);
1401   }
1402   void release_set_bci(uint row, int bci);
1403   void set_bci_count(uint row, uint count) {
1404     assert((uint)row < row_limit(), "oob");
1405     set_uint_at(count0_offset + row * ret_row_cell_count, count);
1406   }
1407   void set_bci_displacement(uint row, int disp) {
1408     set_int_at(displacement0_offset + row * ret_row_cell_count, disp);
1409   }
1410 
1411 public:
1412   RetData(DataLayout* layout) : CounterData(layout) {
1413     assert(layout->tag() == DataLayout::ret_data_tag, "wrong type");
1414   }
1415 
1416   virtual bool is_RetData() const { return true; }
1417 
1418   enum {
1419     no_bci = -1 // value of bci when bci1/2 are not in use.
1420   };
1421 
1422   static int static_cell_count() {
1423     return counter_cell_count + (uint) BciProfileWidth * ret_row_cell_count;
1424   }
1425 
1426   virtual int cell_count() const {
1427     return static_cell_count();
1428   }
1429 
1430   static uint row_limit() {
1431     return BciProfileWidth;
1432   }
1433   static int bci_cell_index(uint row) {
1434     return bci0_offset + row * ret_row_cell_count;
1435   }
1436   static int bci_count_cell_index(uint row) {
1437     return count0_offset + row * ret_row_cell_count;
1438   }
1439   static int bci_displacement_cell_index(uint row) {
1440     return displacement0_offset + row * ret_row_cell_count;
1441   }
1442 
1443   // Direct accessors
1444   int bci(uint row) const {
1445     return int_at(bci_cell_index(row));
1446   }
1447   uint bci_count(uint row) const {
1448     return uint_at(bci_count_cell_index(row));
1449   }
1450   int bci_displacement(uint row) const {
1451     return int_at(bci_displacement_cell_index(row));
1452   }
1453 
1454   // Interpreter Runtime support
1455   address fixup_ret(int return_bci, MethodData* mdo);
1456 
1457   // Code generation support
1458   static ByteSize bci_offset(uint row) {
1459     return cell_offset(bci_cell_index(row));
1460   }
1461   static ByteSize bci_count_offset(uint row) {
1462     return cell_offset(bci_count_cell_index(row));
1463   }
1464   static ByteSize bci_displacement_offset(uint row) {
1465     return cell_offset(bci_displacement_cell_index(row));
1466   }
1467 
1468   // Specific initialization.
1469   void post_initialize(BytecodeStream* stream, MethodData* mdo);
1470 
1471   void print_data_on(outputStream* st, const char* extra = NULL) const;
1472 };
1473 
1474 // BranchData
1475 //
1476 // A BranchData is used to access profiling data for a two-way branch.
1477 // It consists of taken and not_taken counts as well as a data displacement
1478 // for the taken case.
1479 class BranchData : public JumpData {
1480   friend class VMStructs;
1481   friend class JVMCIVMStructs;
1482 protected:
1483   enum {
1484     not_taken_off_set = jump_cell_count,
1485     branch_cell_count
1486   };
1487 
1488   void set_displacement(int displacement) {
1489     set_int_at(displacement_off_set, displacement);
1490   }
1491 
1492 public:
1493   BranchData(DataLayout* layout) : JumpData(layout) {
1494     assert(layout->tag() == DataLayout::branch_data_tag, "wrong type");
1495   }
1496 
1497   virtual bool is_BranchData() const { return true; }
1498 
1499   static int static_cell_count() {
1500     return branch_cell_count;
1501   }
1502 
1503   virtual int cell_count() const {
1504     return static_cell_count();
1505   }
1506 
1507   // Direct accessor
1508   uint not_taken() const {
1509     return uint_at(not_taken_off_set);
1510   }
1511 
1512   void set_not_taken(uint cnt) {
1513     set_uint_at(not_taken_off_set, cnt);
1514   }
1515 
1516   uint inc_not_taken() {
1517     uint cnt = not_taken() + 1;
1518     // Did we wrap? Will compiler screw us??
1519     if (cnt == 0) cnt--;
1520     set_uint_at(not_taken_off_set, cnt);
1521     return cnt;
1522   }
1523 
1524   // Code generation support
1525   static ByteSize not_taken_offset() {
1526     return cell_offset(not_taken_off_set);
1527   }
1528   static ByteSize branch_data_size() {
1529     return cell_offset(branch_cell_count);
1530   }
1531 
1532   // Specific initialization.
1533   void post_initialize(BytecodeStream* stream, MethodData* mdo);
1534 
1535   void print_data_on(outputStream* st, const char* extra = NULL) const;
1536 };
1537 
1538 // ArrayData
1539 //
1540 // A ArrayData is a base class for accessing profiling data which does
1541 // not have a statically known size.  It consists of an array length
1542 // and an array start.
1543 class ArrayData : public ProfileData {
1544   friend class VMStructs;
1545   friend class JVMCIVMStructs;
1546 protected:
1547   friend class DataLayout;
1548 
1549   enum {
1550     array_len_off_set,
1551     array_start_off_set
1552   };
1553 
1554   uint array_uint_at(int index) const {
1555     int aindex = index + array_start_off_set;
1556     return uint_at(aindex);
1557   }
1558   int array_int_at(int index) const {
1559     int aindex = index + array_start_off_set;
1560     return int_at(aindex);
1561   }
1562   oop array_oop_at(int index) const {
1563     int aindex = index + array_start_off_set;
1564     return oop_at(aindex);
1565   }
1566   void array_set_int_at(int index, int value) {
1567     int aindex = index + array_start_off_set;
1568     set_int_at(aindex, value);
1569   }
1570 
1571   // Code generation support for subclasses.
1572   static ByteSize array_element_offset(int index) {
1573     return cell_offset(array_start_off_set + index);
1574   }
1575 
1576 public:
1577   ArrayData(DataLayout* layout) : ProfileData(layout) {}
1578 
1579   virtual bool is_ArrayData() const { return true; }
1580 
1581   static int static_cell_count() {
1582     return -1;
1583   }
1584 
1585   int array_len() const {
1586     return int_at_unchecked(array_len_off_set);
1587   }
1588 
1589   virtual int cell_count() const {
1590     return array_len() + 1;
1591   }
1592 
1593   // Code generation support
1594   static ByteSize array_len_offset() {
1595     return cell_offset(array_len_off_set);
1596   }
1597   static ByteSize array_start_offset() {
1598     return cell_offset(array_start_off_set);
1599   }
1600 };
1601 
1602 // MultiBranchData
1603 //
1604 // A MultiBranchData is used to access profiling information for
1605 // a multi-way branch (*switch bytecodes).  It consists of a series
1606 // of (count, displacement) pairs, which count the number of times each
1607 // case was taken and specify the data displacment for each branch target.
1608 class MultiBranchData : public ArrayData {
1609   friend class VMStructs;
1610   friend class JVMCIVMStructs;
1611 protected:
1612   enum {
1613     default_count_off_set,
1614     default_disaplacement_off_set,
1615     case_array_start
1616   };
1617   enum {
1618     relative_count_off_set,
1619     relative_displacement_off_set,
1620     per_case_cell_count
1621   };
1622 
1623   void set_default_displacement(int displacement) {
1624     array_set_int_at(default_disaplacement_off_set, displacement);
1625   }
1626   void set_displacement_at(int index, int displacement) {
1627     array_set_int_at(case_array_start +
1628                      index * per_case_cell_count +
1629                      relative_displacement_off_set,
1630                      displacement);
1631   }
1632 
1633 public:
1634   MultiBranchData(DataLayout* layout) : ArrayData(layout) {
1635     assert(layout->tag() == DataLayout::multi_branch_data_tag, "wrong type");
1636   }
1637 
1638   virtual bool is_MultiBranchData() const { return true; }
1639 
1640   static int compute_cell_count(BytecodeStream* stream);
1641 
1642   int number_of_cases() const {
1643     int alen = array_len() - 2; // get rid of default case here.
1644     assert(alen % per_case_cell_count == 0, "must be even");
1645     return (alen / per_case_cell_count);
1646   }
1647 
1648   uint default_count() const {
1649     return array_uint_at(default_count_off_set);
1650   }
1651   int default_displacement() const {
1652     return array_int_at(default_disaplacement_off_set);
1653   }
1654 
1655   uint count_at(int index) const {
1656     return array_uint_at(case_array_start +
1657                          index * per_case_cell_count +
1658                          relative_count_off_set);
1659   }
1660   int displacement_at(int index) const {
1661     return array_int_at(case_array_start +
1662                         index * per_case_cell_count +
1663                         relative_displacement_off_set);
1664   }
1665 
1666   // Code generation support
1667   static ByteSize default_count_offset() {
1668     return array_element_offset(default_count_off_set);
1669   }
1670   static ByteSize default_displacement_offset() {
1671     return array_element_offset(default_disaplacement_off_set);
1672   }
1673   static ByteSize case_count_offset(int index) {
1674     return case_array_offset() +
1675            (per_case_size() * index) +
1676            relative_count_offset();
1677   }
1678   static ByteSize case_array_offset() {
1679     return array_element_offset(case_array_start);
1680   }
1681   static ByteSize per_case_size() {
1682     return in_ByteSize(per_case_cell_count) * cell_size;
1683   }
1684   static ByteSize relative_count_offset() {
1685     return in_ByteSize(relative_count_off_set) * cell_size;
1686   }
1687   static ByteSize relative_displacement_offset() {
1688     return in_ByteSize(relative_displacement_off_set) * cell_size;
1689   }
1690 
1691   // Specific initialization.
1692   void post_initialize(BytecodeStream* stream, MethodData* mdo);
1693 
1694   void print_data_on(outputStream* st, const char* extra = NULL) const;
1695 };
1696 
1697 class ArgInfoData : public ArrayData {
1698 
1699 public:
1700   ArgInfoData(DataLayout* layout) : ArrayData(layout) {
1701     assert(layout->tag() == DataLayout::arg_info_data_tag, "wrong type");
1702   }
1703 
1704   virtual bool is_ArgInfoData() const { return true; }
1705 
1706 
1707   int number_of_args() const {
1708     return array_len();
1709   }
1710 
1711   uint arg_modified(int arg) const {
1712     return array_uint_at(arg);
1713   }
1714 
1715   void set_arg_modified(int arg, uint val) {
1716     array_set_int_at(arg, val);
1717   }
1718 
1719   void print_data_on(outputStream* st, const char* extra = NULL) const;
1720 };
1721 
1722 // ParametersTypeData
1723 //
1724 // A ParametersTypeData is used to access profiling information about
1725 // types of parameters to a method
1726 class ParametersTypeData : public ArrayData {
1727 
1728 private:
1729   TypeStackSlotEntries _parameters;
1730 
1731   static int stack_slot_local_offset(int i) {
1732     assert_profiling_enabled();
1733     return array_start_off_set + TypeStackSlotEntries::stack_slot_local_offset(i);
1734   }
1735 
1736   static int type_local_offset(int i) {
1737     assert_profiling_enabled();
1738     return array_start_off_set + TypeStackSlotEntries::type_local_offset(i);
1739   }
1740 
1741   static bool profiling_enabled();
1742   static void assert_profiling_enabled() {
1743     assert(profiling_enabled(), "method parameters profiling should be on");
1744   }
1745 
1746 public:
1747   ParametersTypeData(DataLayout* layout) : ArrayData(layout), _parameters(1, number_of_parameters()) {
1748     assert(layout->tag() == DataLayout::parameters_type_data_tag, "wrong type");
1749     // Some compilers (VC++) don't want this passed in member initialization list
1750     _parameters.set_profile_data(this);
1751   }
1752 
1753   static int compute_cell_count(Method* m);
1754 
1755   virtual bool is_ParametersTypeData() const { return true; }
1756 
1757   virtual void post_initialize(BytecodeStream* stream, MethodData* mdo);
1758 
1759   int number_of_parameters() const {
1760     return array_len() / TypeStackSlotEntries::per_arg_count();
1761   }
1762 
1763   const TypeStackSlotEntries* parameters() const { return &_parameters; }
1764 
1765   uint stack_slot(int i) const {
1766     return _parameters.stack_slot(i);
1767   }
1768 
1769   void set_type(int i, Klass* k) {
1770     intptr_t current = _parameters.type(i);
1771     _parameters.set_type(i, TypeEntries::with_status((intptr_t)k, current));
1772   }
1773 
1774   virtual void clean_weak_klass_links(bool always_clean) {
1775     _parameters.clean_weak_klass_links(always_clean);
1776   }
1777 
1778   virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
1779 
1780   static ByteSize stack_slot_offset(int i) {
1781     return cell_offset(stack_slot_local_offset(i));
1782   }
1783 
1784   static ByteSize type_offset(int i) {
1785     return cell_offset(type_local_offset(i));
1786   }
1787 };
1788 
1789 // SpeculativeTrapData
1790 //
1791 // A SpeculativeTrapData is used to record traps due to type
1792 // speculation. It records the root of the compilation: that type
1793 // speculation is wrong in the context of one compilation (for
1794 // method1) doesn't mean it's wrong in the context of another one (for
1795 // method2). Type speculation could have more/different data in the
1796 // context of the compilation of method2 and it's worthwhile to try an
1797 // optimization that failed for compilation of method1 in the context
1798 // of compilation of method2.
1799 // Space for SpeculativeTrapData entries is allocated from the extra
1800 // data space in the MDO. If we run out of space, the trap data for
1801 // the ProfileData at that bci is updated.
1802 class SpeculativeTrapData : public ProfileData {
1803 protected:
1804   enum {
1805     speculative_trap_method,
1806 #ifndef _LP64
1807     // The size of the area for traps is a multiple of the header
1808     // size, 2 cells on 32 bits. Packed at the end of this area are
1809     // argument info entries (with tag
1810     // DataLayout::arg_info_data_tag). The logic in
1811     // MethodData::bci_to_extra_data() that guarantees traps don't
1812     // overflow over argument info entries assumes the size of a
1813     // SpeculativeTrapData is twice the header size. On 32 bits, a
1814     // SpeculativeTrapData must be 4 cells.
1815     padding,
1816 #endif
1817     speculative_trap_cell_count
1818   };
1819 public:
1820   SpeculativeTrapData(DataLayout* layout) : ProfileData(layout) {
1821     assert(layout->tag() == DataLayout::speculative_trap_data_tag, "wrong type");
1822   }
1823 
1824   virtual bool is_SpeculativeTrapData() const { return true; }
1825 
1826   static int static_cell_count() {
1827     return speculative_trap_cell_count;
1828   }
1829 
1830   virtual int cell_count() const {
1831     return static_cell_count();
1832   }
1833 
1834   // Direct accessor
1835   Method* method() const {
1836     return (Method*)intptr_at(speculative_trap_method);
1837   }
1838 
1839   void set_method(Method* m) {
1840     assert(!m->is_old(), "cannot add old methods");
1841     set_intptr_at(speculative_trap_method, (intptr_t)m);
1842   }
1843 
1844   static ByteSize method_offset() {
1845     return cell_offset(speculative_trap_method);
1846   }
1847 
1848   virtual void print_data_on(outputStream* st, const char* extra = NULL) const;
1849 };
1850 
1851 // MethodData*
1852 //
1853 // A MethodData* holds information which has been collected about
1854 // a method.  Its layout looks like this:
1855 //
1856 // -----------------------------
1857 // | header                    |
1858 // | klass                     |
1859 // -----------------------------
1860 // | method                    |
1861 // | size of the MethodData* |
1862 // -----------------------------
1863 // | Data entries...           |
1864 // |   (variable size)         |
1865 // |                           |
1866 // .                           .
1867 // .                           .
1868 // .                           .
1869 // |                           |
1870 // -----------------------------
1871 //
1872 // The data entry area is a heterogeneous array of DataLayouts. Each
1873 // DataLayout in the array corresponds to a specific bytecode in the
1874 // method.  The entries in the array are sorted by the corresponding
1875 // bytecode.  Access to the data is via resource-allocated ProfileData,
1876 // which point to the underlying blocks of DataLayout structures.
1877 //
1878 // During interpretation, if profiling in enabled, the interpreter
1879 // maintains a method data pointer (mdp), which points at the entry
1880 // in the array corresponding to the current bci.  In the course of
1881 // intepretation, when a bytecode is encountered that has profile data
1882 // associated with it, the entry pointed to by mdp is updated, then the
1883 // mdp is adjusted to point to the next appropriate DataLayout.  If mdp
1884 // is NULL to begin with, the interpreter assumes that the current method
1885 // is not (yet) being profiled.
1886 //
1887 // In MethodData* parlance, "dp" is a "data pointer", the actual address
1888 // of a DataLayout element.  A "di" is a "data index", the offset in bytes
1889 // from the base of the data entry array.  A "displacement" is the byte offset
1890 // in certain ProfileData objects that indicate the amount the mdp must be
1891 // adjusted in the event of a change in control flow.
1892 //
1893 
1894 class CleanExtraDataClosure : public StackObj {
1895 public:
1896   virtual bool is_live(Method* m) = 0;
1897 };
1898 
1899 
1900 #if INCLUDE_JVMCI
1901 // Encapsulates an encoded speculation reason. These are linked together in
1902 // a list that is atomically appended to during deoptimization. Entries are
1903 // never removed from the list.
1904 // @see jdk.vm.ci.hotspot.HotSpotSpeculationLog.HotSpotSpeculationEncoding
1905 class FailedSpeculation: public CHeapObj<mtCompiler> {
1906  private:
1907   // The length of HotSpotSpeculationEncoding.toByteArray(). The data itself
1908   // is an array embedded at the end of this object.
1909   int   _data_len;
1910 
1911   // Next entry in a linked list.
1912   FailedSpeculation* _next;
1913 
1914   FailedSpeculation(address data, int data_len);
1915 
1916   FailedSpeculation** next_adr() { return &_next; }
1917 
1918   // Placement new operator for inlining the speculation data into
1919   // the FailedSpeculation object.
1920   void* operator new(size_t size, size_t fs_size) throw();
1921 
1922  public:
1923   char* data()         { return (char*)(((address) this) + sizeof(FailedSpeculation)); }
1924   int data_len() const { return _data_len; }
1925   FailedSpeculation* next() const { return _next; }
1926 
1927   // Atomically appends a speculation from nm to the list whose head is at (*failed_speculations_address).
1928   // Returns false if the FailedSpeculation object could not be allocated.
1929   static bool add_failed_speculation(nmethod* nm, FailedSpeculation** failed_speculations_address, address speculation, int speculation_len);
1930 
1931   // Frees all entries in the linked list whose head is at (*failed_speculations_address).
1932   static void free_failed_speculations(FailedSpeculation** failed_speculations_address);
1933 };
1934 #endif
1935 
1936 class ciMethodData;
1937 
1938 class MethodData : public Metadata {
1939   friend class VMStructs;
1940   friend class JVMCIVMStructs;
1941 private:
1942   friend class ProfileData;
1943   friend class TypeEntriesAtCall;
1944   friend class ciMethodData;
1945 
1946   // If you add a new field that points to any metaspace object, you
1947   // must add this field to MethodData::metaspace_pointers_do().
1948 
1949   // Back pointer to the Method*
1950   Method* _method;
1951 
1952   // Size of this oop in bytes
1953   int _size;
1954 
1955   // Cached hint for bci_to_dp and bci_to_data
1956   int _hint_di;
1957 
1958   Mutex _extra_data_lock;
1959 
1960   MethodData(const methodHandle& method);
1961 public:
1962   static MethodData* allocate(ClassLoaderData* loader_data, const methodHandle& method, TRAPS);
1963 
1964   virtual bool is_methodData() const { return true; }
1965   void initialize();
1966 
1967   // Whole-method sticky bits and flags
1968   enum {
1969     _trap_hist_limit    = Deoptimization::Reason_TRAP_HISTORY_LENGTH,
1970     _trap_hist_mask     = max_jubyte,
1971     _extra_data_count   = 4     // extra DataLayout headers, for trap history
1972   }; // Public flag values
1973 
1974   // Compiler-related counters.
1975   class CompilerCounters {
1976     friend class VMStructs;
1977     friend class JVMCIVMStructs;
1978 
1979     uint _nof_decompiles;             // count of all nmethod removals
1980     uint _nof_overflow_recompiles;    // recompile count, excluding recomp. bits
1981     uint _nof_overflow_traps;         // trap count, excluding _trap_hist
1982     union {
1983       intptr_t _align;
1984       // JVMCI separates trap history for OSR compilations from normal compilations
1985       u1 _array[JVMCI_ONLY(2 *) MethodData::_trap_hist_limit];
1986     } _trap_hist;
1987 
1988   public:
1989     CompilerCounters() : _nof_decompiles(0), _nof_overflow_recompiles(0), _nof_overflow_traps(0) {
1990 #ifndef ZERO
1991       // Some Zero platforms do not have expected alignment, and do not use
1992       // this code. static_assert would still fire and fail for them.
1993       static_assert(sizeof(_trap_hist) % HeapWordSize == 0, "align");
1994 #endif
1995       uint size_in_words = sizeof(_trap_hist) / HeapWordSize;
1996       Copy::zero_to_words((HeapWord*) &_trap_hist, size_in_words);
1997     }
1998 
1999     // Return (uint)-1 for overflow.
2000     uint trap_count(int reason) const {
2001       assert((uint)reason < ARRAY_SIZE(_trap_hist._array), "oob");
2002       return (int)((_trap_hist._array[reason]+1) & _trap_hist_mask) - 1;
2003     }
2004 
2005     uint inc_trap_count(int reason) {
2006       // Count another trap, anywhere in this method.
2007       assert(reason >= 0, "must be single trap");
2008       assert((uint)reason < ARRAY_SIZE(_trap_hist._array), "oob");
2009       uint cnt1 = 1 + _trap_hist._array[reason];
2010       if ((cnt1 & _trap_hist_mask) != 0) {  // if no counter overflow...
2011         _trap_hist._array[reason] = cnt1;
2012         return cnt1;
2013       } else {
2014         return _trap_hist_mask + (++_nof_overflow_traps);
2015       }
2016     }
2017 
2018     uint overflow_trap_count() const {
2019       return _nof_overflow_traps;
2020     }
2021     uint overflow_recompile_count() const {
2022       return _nof_overflow_recompiles;
2023     }
2024     uint inc_overflow_recompile_count() {
2025       return ++_nof_overflow_recompiles;
2026     }
2027     uint decompile_count() const {
2028       return _nof_decompiles;
2029     }
2030     uint inc_decompile_count() {
2031       return ++_nof_decompiles;
2032     }
2033 
2034     // Support for code generation
2035     static ByteSize trap_history_offset() {
2036       return byte_offset_of(CompilerCounters, _trap_hist._array);
2037     }
2038   };
2039 
2040 private:
2041   CompilerCounters _compiler_counters;
2042 
2043   // Support for interprocedural escape analysis, from Thomas Kotzmann.
2044   intx              _eflags;          // flags on escape information
2045   intx              _arg_local;       // bit set of non-escaping arguments
2046   intx              _arg_stack;       // bit set of stack-allocatable arguments
2047   intx              _arg_returned;    // bit set of returned arguments
2048 
2049   int               _creation_mileage; // method mileage at MDO creation
2050 
2051   // How many invocations has this MDO seen?
2052   // These counters are used to determine the exact age of MDO.
2053   // We need those because in tiered a method can be concurrently
2054   // executed at different levels.
2055   InvocationCounter _invocation_counter;
2056   // Same for backedges.
2057   InvocationCounter _backedge_counter;
2058   // Counter values at the time profiling started.
2059   int               _invocation_counter_start;
2060   int               _backedge_counter_start;
2061   uint              _tenure_traps;
2062   int               _invoke_mask;      // per-method Tier0InvokeNotifyFreqLog
2063   int               _backedge_mask;    // per-method Tier0BackedgeNotifyFreqLog
2064 
2065 #if INCLUDE_RTM_OPT
2066   // State of RTM code generation during compilation of the method
2067   int               _rtm_state;
2068 #endif
2069 
2070   // Number of loops and blocks is computed when compiling the first
2071   // time with C1. It is used to determine if method is trivial.
2072   short             _num_loops;
2073   short             _num_blocks;
2074   // Does this method contain anything worth profiling?
2075   enum WouldProfile {unknown, no_profile, profile};
2076   WouldProfile      _would_profile;
2077 
2078 #if INCLUDE_JVMCI
2079   // Support for HotSpotMethodData.setCompiledIRSize(int)
2080   int                _jvmci_ir_size;
2081   FailedSpeculation* _failed_speculations;
2082 #endif
2083 
2084   // Size of _data array in bytes.  (Excludes header and extra_data fields.)
2085   int _data_size;
2086 
2087   // data index for the area dedicated to parameters. -1 if no
2088   // parameter profiling.
2089   enum { no_parameters = -2, parameters_uninitialized = -1 };
2090   int _parameters_type_data_di;
2091 
2092   // Beginning of the data entries
2093   intptr_t _data[1];
2094 
2095   // Helper for size computation
2096   static int compute_data_size(BytecodeStream* stream);
2097   static int bytecode_cell_count(Bytecodes::Code code);
2098   static bool is_speculative_trap_bytecode(Bytecodes::Code code);
2099   enum { no_profile_data = -1, variable_cell_count = -2 };
2100 
2101   // Helper for initialization
2102   DataLayout* data_layout_at(int data_index) const {
2103     assert(data_index % sizeof(intptr_t) == 0, "unaligned");
2104     return (DataLayout*) (((address)_data) + data_index);
2105   }
2106 
2107   // Initialize an individual data segment.  Returns the size of
2108   // the segment in bytes.
2109   int initialize_data(BytecodeStream* stream, int data_index);
2110 
2111   // Helper for data_at
2112   DataLayout* limit_data_position() const {
2113     return data_layout_at(_data_size);
2114   }
2115   bool out_of_bounds(int data_index) const {
2116     return data_index >= data_size();
2117   }
2118 
2119   // Give each of the data entries a chance to perform specific
2120   // data initialization.
2121   void post_initialize(BytecodeStream* stream);
2122 
2123   // hint accessors
2124   int      hint_di() const  { return _hint_di; }
2125   void set_hint_di(int di)  {
2126     assert(!out_of_bounds(di), "hint_di out of bounds");
2127     _hint_di = di;
2128   }
2129 
2130   DataLayout* data_layout_before(int bci) {
2131     // avoid SEGV on this edge case
2132     if (data_size() == 0)
2133       return NULL;
2134     DataLayout* layout = data_layout_at(hint_di());
2135     if (layout->bci() <= bci)
2136       return layout;
2137     return data_layout_at(first_di());
2138   }
2139 
2140   // What is the index of the first data entry?
2141   int first_di() const { return 0; }
2142 
2143   ProfileData* bci_to_extra_data_helper(int bci, Method* m, DataLayout*& dp, bool concurrent);
2144   // Find or create an extra ProfileData:
2145   ProfileData* bci_to_extra_data(int bci, Method* m, bool create_if_missing);
2146 
2147   // return the argument info cell
2148   ArgInfoData *arg_info();
2149 
2150   enum {
2151     no_type_profile = 0,
2152     type_profile_jsr292 = 1,
2153     type_profile_all = 2
2154   };
2155 
2156   static bool profile_jsr292(const methodHandle& m, int bci);
2157   static bool profile_unsafe(const methodHandle& m, int bci);
2158   static bool profile_memory_access(const methodHandle& m, int bci);
2159   static int profile_arguments_flag();
2160   static bool profile_all_arguments();
2161   static bool profile_arguments_for_invoke(const methodHandle& m, int bci);
2162   static int profile_return_flag();
2163   static bool profile_all_return();
2164   static bool profile_return_for_invoke(const methodHandle& m, int bci);
2165   static int profile_parameters_flag();
2166   static bool profile_parameters_jsr292_only();
2167   static bool profile_all_parameters();
2168 
2169   void clean_extra_data_helper(DataLayout* dp, int shift, bool reset = false);
2170   void verify_extra_data_clean(CleanExtraDataClosure* cl);
2171 
2172 public:
2173   void clean_extra_data(CleanExtraDataClosure* cl);
2174 
2175   static int header_size() {
2176     return sizeof(MethodData)/wordSize;
2177   }
2178 
2179   // Compute the size of a MethodData* before it is created.
2180   static int compute_allocation_size_in_bytes(const methodHandle& method);
2181   static int compute_allocation_size_in_words(const methodHandle& method);
2182   static int compute_extra_data_count(int data_size, int empty_bc_count, bool needs_speculative_traps);
2183 
2184   // Determine if a given bytecode can have profile information.
2185   static bool bytecode_has_profile(Bytecodes::Code code) {
2186     return bytecode_cell_count(code) != no_profile_data;
2187   }
2188 
2189   // reset into original state
2190   void init();
2191 
2192   // My size
2193   int size_in_bytes() const { return _size; }
2194   int size() const    { return align_metadata_size(align_up(_size, BytesPerWord)/BytesPerWord); }
2195 
2196   int      creation_mileage() const { return _creation_mileage; }
2197   void set_creation_mileage(int x)  { _creation_mileage = x; }
2198 
2199   int invocation_count() {
2200     if (invocation_counter()->carry()) {
2201       return InvocationCounter::count_limit;
2202     }
2203     return invocation_counter()->count();
2204   }
2205   int backedge_count() {
2206     if (backedge_counter()->carry()) {
2207       return InvocationCounter::count_limit;
2208     }
2209     return backedge_counter()->count();
2210   }
2211 
2212   int invocation_count_start() {
2213     if (invocation_counter()->carry()) {
2214       return 0;
2215     }
2216     return _invocation_counter_start;
2217   }
2218 
2219   int backedge_count_start() {
2220     if (backedge_counter()->carry()) {
2221       return 0;
2222     }
2223     return _backedge_counter_start;
2224   }
2225 
2226   int invocation_count_delta() { return invocation_count() - invocation_count_start(); }
2227   int backedge_count_delta()   { return backedge_count()   - backedge_count_start();   }
2228 
2229   void reset_start_counters() {
2230     _invocation_counter_start = invocation_count();
2231     _backedge_counter_start = backedge_count();
2232   }
2233 
2234   InvocationCounter* invocation_counter()     { return &_invocation_counter; }
2235   InvocationCounter* backedge_counter()       { return &_backedge_counter;   }
2236 
2237 #if INCLUDE_JVMCI
2238   FailedSpeculation** get_failed_speculations_address() {
2239     return &_failed_speculations;
2240   }
2241 #endif
2242 
2243 #if INCLUDE_RTM_OPT
2244   int rtm_state() const {
2245     return _rtm_state;
2246   }
2247   void set_rtm_state(RTMState rstate) {
2248     _rtm_state = (int)rstate;
2249   }
2250   void atomic_set_rtm_state(RTMState rstate) {
2251     Atomic::store(&_rtm_state, (int)rstate);
2252   }
2253 
2254   static int rtm_state_offset_in_bytes() {
2255     return offset_of(MethodData, _rtm_state);
2256   }
2257 #endif
2258 
2259   void set_would_profile(bool p)              { _would_profile = p ? profile : no_profile; }
2260   bool would_profile() const                  { return _would_profile != no_profile; }
2261 
2262   int num_loops() const                       { return _num_loops;  }
2263   void set_num_loops(int n)                   { _num_loops = n;     }
2264   int num_blocks() const                      { return _num_blocks; }
2265   void set_num_blocks(int n)                  { _num_blocks = n;    }
2266 
2267   bool is_mature() const;  // consult mileage and ProfileMaturityPercentage
2268   static int mileage_of(Method* m);
2269 
2270   // Support for interprocedural escape analysis, from Thomas Kotzmann.
2271   enum EscapeFlag {
2272     estimated    = 1 << 0,
2273     return_local = 1 << 1,
2274     return_allocated = 1 << 2,
2275     allocated_escapes = 1 << 3,
2276     unknown_modified = 1 << 4
2277   };
2278 
2279   intx eflags()                                  { return _eflags; }
2280   intx arg_local()                               { return _arg_local; }
2281   intx arg_stack()                               { return _arg_stack; }
2282   intx arg_returned()                            { return _arg_returned; }
2283   uint arg_modified(int a)                       { ArgInfoData *aid = arg_info();
2284                                                    assert(aid != NULL, "arg_info must be not null");
2285                                                    assert(a >= 0 && a < aid->number_of_args(), "valid argument number");
2286                                                    return aid->arg_modified(a); }
2287 
2288   void set_eflags(intx v)                        { _eflags = v; }
2289   void set_arg_local(intx v)                     { _arg_local = v; }
2290   void set_arg_stack(intx v)                     { _arg_stack = v; }
2291   void set_arg_returned(intx v)                  { _arg_returned = v; }
2292   void set_arg_modified(int a, uint v)           { ArgInfoData *aid = arg_info();
2293                                                    assert(aid != NULL, "arg_info must be not null");
2294                                                    assert(a >= 0 && a < aid->number_of_args(), "valid argument number");
2295                                                    aid->set_arg_modified(a, v); }
2296 
2297   void clear_escape_info()                       { _eflags = _arg_local = _arg_stack = _arg_returned = 0; }
2298 
2299   // Location and size of data area
2300   address data_base() const {
2301     return (address) _data;
2302   }
2303   int data_size() const {
2304     return _data_size;
2305   }
2306 
2307   int parameters_size_in_bytes() const {
2308     ParametersTypeData* param = parameters_type_data();
2309     return param == NULL ? 0 : param->size_in_bytes();
2310   }
2311 
2312   // Accessors
2313   Method* method() const { return _method; }
2314 
2315   // Get the data at an arbitrary (sort of) data index.
2316   ProfileData* data_at(int data_index) const;
2317 
2318   // Walk through the data in order.
2319   ProfileData* first_data() const { return data_at(first_di()); }
2320   ProfileData* next_data(ProfileData* current) const;
2321   DataLayout*  next_data_layout(DataLayout* current) const;
2322   bool is_valid(ProfileData* current) const { return current != NULL; }
2323   bool is_valid(DataLayout*  current) const { return current != NULL; }
2324 
2325   // Convert a dp (data pointer) to a di (data index).
2326   int dp_to_di(address dp) const {
2327     return dp - ((address)_data);
2328   }
2329 
2330   // bci to di/dp conversion.
2331   address bci_to_dp(int bci);
2332   int bci_to_di(int bci) {
2333     return dp_to_di(bci_to_dp(bci));
2334   }
2335 
2336   // Get the data at an arbitrary bci, or NULL if there is none.
2337   ProfileData* bci_to_data(int bci);
2338 
2339   // Same, but try to create an extra_data record if one is needed:
2340   ProfileData* allocate_bci_to_data(int bci, Method* m) {
2341     ProfileData* data = NULL;
2342     // If m not NULL, try to allocate a SpeculativeTrapData entry
2343     if (m == NULL) {
2344       data = bci_to_data(bci);
2345     }
2346     if (data != NULL) {
2347       return data;
2348     }
2349     data = bci_to_extra_data(bci, m, true);
2350     if (data != NULL) {
2351       return data;
2352     }
2353     // If SpeculativeTrapData allocation fails try to allocate a
2354     // regular entry
2355     data = bci_to_data(bci);
2356     if (data != NULL) {
2357       return data;
2358     }
2359     return bci_to_extra_data(bci, NULL, true);
2360   }
2361 
2362   // Add a handful of extra data records, for trap tracking.
2363   DataLayout* extra_data_base() const  { return limit_data_position(); }
2364   DataLayout* extra_data_limit() const { return (DataLayout*)((address)this + size_in_bytes()); }
2365   DataLayout* args_data_limit() const  { return (DataLayout*)((address)this + size_in_bytes() -
2366                                                               parameters_size_in_bytes()); }
2367   int extra_data_size() const          { return (address)extra_data_limit() - (address)extra_data_base(); }
2368   static DataLayout* next_extra(DataLayout* dp);
2369 
2370   // Return (uint)-1 for overflow.
2371   uint trap_count(int reason) const {
2372     return _compiler_counters.trap_count(reason);
2373   }
2374   // For loops:
2375   static uint trap_reason_limit() { return _trap_hist_limit; }
2376   static uint trap_count_limit()  { return _trap_hist_mask; }
2377   uint inc_trap_count(int reason) {
2378     return _compiler_counters.inc_trap_count(reason);
2379   }
2380 
2381   uint overflow_trap_count() const {
2382     return _compiler_counters.overflow_trap_count();
2383   }
2384   uint overflow_recompile_count() const {
2385     return _compiler_counters.overflow_recompile_count();
2386   }
2387   uint inc_overflow_recompile_count() {
2388     return _compiler_counters.inc_overflow_recompile_count();
2389   }
2390   uint decompile_count() const {
2391     return _compiler_counters.decompile_count();
2392   }
2393   uint inc_decompile_count() {
2394     uint dec_count = _compiler_counters.inc_decompile_count();
2395     if (dec_count > (uint)PerMethodRecompilationCutoff) {
2396       method()->set_not_compilable("decompile_count > PerMethodRecompilationCutoff", CompLevel_full_optimization);
2397     }
2398     return dec_count;
2399   }
2400   uint tenure_traps() const {
2401     return _tenure_traps;
2402   }
2403   void inc_tenure_traps() {
2404     _tenure_traps += 1;
2405   }
2406 
2407   // Return pointer to area dedicated to parameters in MDO
2408   ParametersTypeData* parameters_type_data() const {
2409     assert(_parameters_type_data_di != parameters_uninitialized, "called too early");
2410     return _parameters_type_data_di != no_parameters ? data_layout_at(_parameters_type_data_di)->data_in()->as_ParametersTypeData() : NULL;
2411   }
2412 
2413   int parameters_type_data_di() const {
2414     assert(_parameters_type_data_di != parameters_uninitialized && _parameters_type_data_di != no_parameters, "no args type data");
2415     return _parameters_type_data_di;
2416   }
2417 
2418   // Support for code generation
2419   static ByteSize data_offset() {
2420     return byte_offset_of(MethodData, _data[0]);
2421   }
2422 
2423   static ByteSize trap_history_offset() {
2424     return byte_offset_of(MethodData, _compiler_counters) + CompilerCounters::trap_history_offset();
2425   }
2426 
2427   static ByteSize invocation_counter_offset() {
2428     return byte_offset_of(MethodData, _invocation_counter);
2429   }
2430 
2431   static ByteSize backedge_counter_offset() {
2432     return byte_offset_of(MethodData, _backedge_counter);
2433   }
2434 
2435   static ByteSize invoke_mask_offset() {
2436     return byte_offset_of(MethodData, _invoke_mask);
2437   }
2438 
2439   static ByteSize backedge_mask_offset() {
2440     return byte_offset_of(MethodData, _backedge_mask);
2441   }
2442 
2443   static ByteSize parameters_type_data_di_offset() {
2444     return byte_offset_of(MethodData, _parameters_type_data_di);
2445   }
2446 
2447   virtual void metaspace_pointers_do(MetaspaceClosure* iter);
2448   virtual MetaspaceObj::Type type() const { return MethodDataType; }
2449 
2450   // Deallocation support - no metaspace pointer fields to deallocate
2451   void deallocate_contents(ClassLoaderData* loader_data) {}
2452 
2453   // GC support
2454   void set_size(int object_size_in_bytes) { _size = object_size_in_bytes; }
2455 
2456   // Printing
2457   void print_on      (outputStream* st) const;
2458   void print_value_on(outputStream* st) const;
2459 
2460   // printing support for method data
2461   void print_data_on(outputStream* st) const;
2462 
2463   const char* internal_name() const { return "{method data}"; }
2464 
2465   // verification
2466   void verify_on(outputStream* st);
2467   void verify_data_on(outputStream* st);
2468 
2469   static bool profile_parameters_for_method(const methodHandle& m);
2470   static bool profile_arguments();
2471   static bool profile_arguments_jsr292_only();
2472   static bool profile_return();
2473   static bool profile_parameters();
2474   static bool profile_return_jsr292_only();
2475 
2476   void clean_method_data(bool always_clean);
2477   void clean_weak_method_links();
2478   Mutex* extra_data_lock() { return &_extra_data_lock; }
2479 };
2480 
2481 #endif // SHARE_OOPS_METHODDATA_HPP