< prev index next >

src/hotspot/cpu/x86/interp_masm_x86.cpp

Print this page

  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "compiler/compiler_globals.hpp"
  27 #include "interp_masm_x86.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "logging/log.hpp"
  31 #include "oops/arrayOop.hpp"

  32 #include "oops/markWord.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/method.hpp"

  35 #include "oops/resolvedFieldEntry.hpp"
  36 #include "oops/resolvedIndyEntry.hpp"
  37 #include "oops/resolvedMethodEntry.hpp"
  38 #include "prims/jvmtiExport.hpp"
  39 #include "prims/jvmtiThreadState.hpp"
  40 #include "runtime/basicLock.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/javaThread.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "utilities/powerOfTwo.hpp"
  46 
  47 // Implementation of InterpreterMacroAssembler
  48 
  49 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  50   assert(entry, "Entry must have been generated by now");
  51   jump(RuntimeAddress(entry));
  52 }
  53 
  54 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {

 136         Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i))-off_to_args);
 137         profile_obj_type(tmp, mdo_arg_addr);
 138 
 139         int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
 140         addptr(mdp, to_add);
 141         off_to_args += to_add;
 142       }
 143 
 144       if (MethodData::profile_return()) {
 145         movptr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())-off_to_args));
 146         subl(tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
 147       }
 148 
 149       bind(done);
 150 
 151       if (MethodData::profile_return()) {
 152         // We're right after the type profile for the last
 153         // argument. tmp is the number of cells left in the
 154         // CallTypeData/VirtualCallTypeData to reach its end. Non null
 155         // if there's a return to profile.
 156         assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
 157         shll(tmp, log2i_exact((int)DataLayout::cell_size));
 158         addptr(mdp, tmp);
 159       }
 160       movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp);
 161     } else {
 162       assert(MethodData::profile_return(), "either profile call args or call ret");
 163       update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
 164     }
 165 
 166     // mdp points right after the end of the
 167     // CallTypeData/VirtualCallTypeData, right after the cells for the
 168     // return value type if there's one
 169 
 170     bind(profile_continue);
 171   }
 172 }
 173 
 174 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
 175   assert_different_registers(mdp, ret, tmp, _bcp_register);
 176   if (ProfileInterpreter && MethodData::profile_return()) {

 181     if (MethodData::profile_return_jsr292_only()) {
 182       assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2");
 183 
 184       // If we don't profile all invoke bytecodes we must make sure
 185       // it's a bytecode we indeed profile. We can't go back to the
 186       // beginning of the ProfileData we intend to update to check its
 187       // type because we're right after it and we don't known its
 188       // length
 189       Label do_profile;
 190       cmpb(Address(_bcp_register, 0), Bytecodes::_invokedynamic);
 191       jcc(Assembler::equal, do_profile);
 192       cmpb(Address(_bcp_register, 0), Bytecodes::_invokehandle);
 193       jcc(Assembler::equal, do_profile);
 194       get_method(tmp);
 195       cmpw(Address(tmp, Method::intrinsic_id_offset()), static_cast<int>(vmIntrinsics::_compiledLambdaForm));
 196       jcc(Assembler::notEqual, profile_continue);
 197 
 198       bind(do_profile);
 199     }
 200 
 201     Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
 202     mov(tmp, ret);
 203     profile_obj_type(tmp, mdo_ret_addr);
 204 
 205     bind(profile_continue);
 206   }
 207 }
 208 
 209 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
 210   if (ProfileInterpreter && MethodData::profile_parameters()) {
 211     Label profile_continue;
 212 
 213     test_method_data_pointer(mdp, profile_continue);
 214 
 215     // Load the offset of the area within the MDO used for
 216     // parameters. If it's negative we're not profiling any parameters
 217     movl(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
 218     testl(tmp1, tmp1);
 219     jcc(Assembler::negative, profile_continue);
 220 
 221     // Compute a pointer to the area for parameters from the offset

 476                                                              Register cpool,
 477                                                              Register index) {
 478   assert_different_registers(cpool, index);
 479 
 480   movw(index, Address(cpool, index, Address::times_ptr, sizeof(ConstantPool)));
 481   Register resolved_klasses = cpool;
 482   movptr(resolved_klasses, Address(cpool, ConstantPool::resolved_klasses_offset()));
 483   movptr(klass, Address(resolved_klasses, index, Address::times_ptr, Array<Klass*>::base_offset_in_bytes()));
 484 }
 485 
 486 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
 487 // subtype of super_klass.
 488 //
 489 // Args:
 490 //      rax: superklass
 491 //      Rsub_klass: subklass
 492 //
 493 // Kills:
 494 //      rcx, rdi
 495 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 496                                                   Label& ok_is_subtype) {

 497   assert(Rsub_klass != rax, "rax holds superklass");
 498   LP64_ONLY(assert(Rsub_klass != r14, "r14 holds locals");)
 499   LP64_ONLY(assert(Rsub_klass != r13, "r13 holds bcp");)
 500   assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
 501   assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
 502 
 503   // Profile the not-null value's klass.
 504   profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi


 505 
 506   // Do the check.
 507   check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
 508 }
 509 
 510 
 511 #ifndef _LP64
 512 void InterpreterMacroAssembler::f2ieee() {
 513   if (IEEEPrecision) {
 514     fstp_s(Address(rsp, 0));
 515     fld_s(Address(rsp, 0));
 516   }
 517 }
 518 
 519 
 520 void InterpreterMacroAssembler::d2ieee() {
 521   if (IEEEPrecision) {
 522     fstp_d(Address(rsp, 0));
 523     fld_d(Address(rsp, 0));
 524   }

 936   // the stack, will call InterpreterRuntime::at_unwind.
 937   Label slow_path;
 938   Label fast_path;
 939   safepoint_poll(slow_path, rthread, true /* at_return */, false /* in_nmethod */);
 940   jmp(fast_path);
 941   bind(slow_path);
 942   push(state);
 943   set_last_Java_frame(rthread, noreg, rbp, (address)pc(), rscratch1);
 944   super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
 945   NOT_LP64(get_thread(rthread);) // call_VM clobbered it, restore
 946   reset_last_Java_frame(rthread, true);
 947   pop(state);
 948   bind(fast_path);
 949 
 950   // get the value of _do_not_unlock_if_synchronized into rdx
 951   const Address do_not_unlock_if_synchronized(rthread,
 952     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 953   movbool(rbx, do_not_unlock_if_synchronized);
 954   movbool(do_not_unlock_if_synchronized, false); // reset the flag
 955 
 956  // get method access flags
 957   movptr(rcx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
 958   movl(rcx, Address(rcx, Method::access_flags_offset()));
 959   testl(rcx, JVM_ACC_SYNCHRONIZED);
 960   jcc(Assembler::zero, unlocked);
 961 
 962   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 963   // is set.
 964   testbool(rbx);
 965   jcc(Assembler::notZero, no_unlock);
 966 
 967   // unlock monitor
 968   push(state); // save result
 969 
 970   // BasicObjectLock will be first in list, since this is a
 971   // synchronized method. However, need to check that the object has
 972   // not been unlocked by an explicit monitorexit bytecode.
 973   const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset *
 974                         wordSize - (int) sizeof(BasicObjectLock));
 975   // We use c_rarg1/rdx so that if we go slow path it will be the correct
 976   // register for unlock_object to pass to VM directly

1062     bind(loop);
1063     // check if current entry is used
1064     cmpptr(Address(rmon, BasicObjectLock::obj_offset()), NULL_WORD);
1065     jcc(Assembler::notEqual, exception);
1066 
1067     addptr(rmon, entry_size); // otherwise advance to next entry
1068     bind(entry);
1069     cmpptr(rmon, rbx); // check if bottom reached
1070     jcc(Assembler::notEqual, loop); // if not at bottom then check this entry
1071   }
1072 
1073   bind(no_unlock);
1074 
1075   // jvmti support
1076   if (notify_jvmdi) {
1077     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
1078   } else {
1079     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
1080   }
1081 
1082   // remove activation
1083   // get sender sp
1084   movptr(rbx,
1085          Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
1086   if (StackReservedPages > 0) {


1087     // testing if reserved zone needs to be re-enabled
1088     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
1089     Label no_reserved_zone_enabling;
1090 
1091     NOT_LP64(get_thread(rthread);)
1092 
1093     // check if already enabled - if so no re-enabling needed
1094     assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
1095     cmpl(Address(rthread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_enabled);
1096     jcc(Assembler::equal, no_reserved_zone_enabling);
1097 
1098     cmpptr(rbx, Address(rthread, JavaThread::reserved_stack_activation_offset()));
1099     jcc(Assembler::lessEqual, no_reserved_zone_enabling);
1100 
1101     call_VM_leaf(
1102       CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
1103     call_VM(noreg, CAST_FROM_FN_PTR(address,
1104                    InterpreterRuntime::throw_delayed_StackOverflowError));
1105     should_not_reach_here();
1106 
1107     bind(no_reserved_zone_enabling);
1108   }








































1109   leave();                           // remove frame anchor
1110   pop(ret_addr);                     // get return address
1111   mov(rsp, rbx);                     // set sp to sender sp
1112   pop_cont_fastpath();
1113 }
1114 
1115 void InterpreterMacroAssembler::get_method_counters(Register method,
1116                                                     Register mcs, Label& skip) {
1117   Label has_counters;
1118   movptr(mcs, Address(method, Method::method_counters_offset()));
1119   testptr(mcs, mcs);
1120   jcc(Assembler::notZero, has_counters);
1121   call_VM(noreg, CAST_FROM_FN_PTR(address,
1122           InterpreterRuntime::build_method_counters), method);
1123   movptr(mcs, Address(method,Method::method_counters_offset()));
1124   testptr(mcs, mcs);
1125   jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
1126   bind(has_counters);
1127 }
1128 










































































































1129 
1130 // Lock object
1131 //
1132 // Args:
1133 //      rdx, c_rarg1: BasicObjectLock to be used for locking
1134 //
1135 // Kills:
1136 //      rax, rbx
1137 void InterpreterMacroAssembler::lock_object(Register lock_reg) {
1138   assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1139          "The argument is only for looks. It must be c_rarg1");
1140 
1141   if (LockingMode == LM_MONITOR) {
1142     call_VM(noreg,
1143             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1144             lock_reg);
1145   } else {
1146     Label count_locking, done, slow_case;
1147 
1148     const Register swap_reg = rax; // Must use rax for cmpxchg instruction

1164       testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1165       jcc(Assembler::notZero, slow_case);
1166     }
1167 
1168     if (LockingMode == LM_LIGHTWEIGHT) {
1169 #ifdef _LP64
1170       const Register thread = r15_thread;
1171 #else
1172       const Register thread = lock_reg;
1173       get_thread(thread);
1174 #endif
1175       // Load object header, prepare for CAS from unlocked to locked.
1176       movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1177       lightweight_lock(obj_reg, swap_reg, thread, tmp_reg, slow_case);
1178     } else if (LockingMode == LM_LEGACY) {
1179       // Load immediate 1 into swap_reg %rax
1180       movl(swap_reg, 1);
1181 
1182       // Load (object->mark() | 1) into swap_reg %rax
1183       orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));




1184 
1185       // Save (object->mark() | 1) into BasicLock's displaced header
1186       movptr(Address(lock_reg, mark_offset), swap_reg);
1187 
1188       assert(lock_offset == 0,
1189              "displaced header must be first word in BasicObjectLock");
1190 
1191       lock();
1192       cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1193       jcc(Assembler::zero, count_locking);
1194 
1195       const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1196 
1197       // Fast check for recursive lock.
1198       //
1199       // Can apply the optimization only if this is a stack lock
1200       // allocated in this thread. For efficiency, we can focus on
1201       // recently allocated stack locks (instead of reading the stack
1202       // base and checking whether 'mark' points inside the current
1203       // thread stack):

1536     test_method_data_pointer(mdp, profile_continue);
1537 
1538     // We are taking a branch.  Increment the taken count.
1539     // We inline increment_mdp_data_at to return bumped_count in a register
1540     //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1541     Address data(mdp, in_bytes(JumpData::taken_offset()));
1542     movptr(bumped_count, data);
1543     assert(DataLayout::counter_increment == 1,
1544             "flow-free idiom only works with 1");
1545     addptr(bumped_count, DataLayout::counter_increment);
1546     sbbptr(bumped_count, 0);
1547     movptr(data, bumped_count); // Store back out
1548 
1549     // The method data pointer needs to be updated to reflect the new target.
1550     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1551     bind(profile_continue);
1552   }
1553 }
1554 
1555 
1556 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1557   if (ProfileInterpreter) {
1558     Label profile_continue;
1559 
1560     // If no method data exists, go to profile_continue.
1561     test_method_data_pointer(mdp, profile_continue);
1562 
1563     // We are taking a branch.  Increment the not taken count.
1564     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1565 
1566     // The method data pointer needs to be updated to correspond to
1567     // the next bytecode
1568     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1569     bind(profile_continue);
1570   }
1571 }
1572 
1573 void InterpreterMacroAssembler::profile_call(Register mdp) {
1574   if (ProfileInterpreter) {
1575     Label profile_continue;
1576 
1577     // If no method data exists, go to profile_continue.
1578     test_method_data_pointer(mdp, profile_continue);
1579 
1580     // We are making a call.  Increment the count.
1581     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1582 
1583     // The method data pointer needs to be updated to reflect the new target.
1584     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1585     bind(profile_continue);
1586   }
1587 }
1588 

1611                                                      Register reg2,
1612                                                      bool receiver_can_be_null) {
1613   if (ProfileInterpreter) {
1614     Label profile_continue;
1615 
1616     // If no method data exists, go to profile_continue.
1617     test_method_data_pointer(mdp, profile_continue);
1618 
1619     Label skip_receiver_profile;
1620     if (receiver_can_be_null) {
1621       Label not_null;
1622       testptr(receiver, receiver);
1623       jccb(Assembler::notZero, not_null);
1624       // We are making a call.  Increment the count for null receiver.
1625       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1626       jmp(skip_receiver_profile);
1627       bind(not_null);
1628     }
1629 
1630     // Record the receiver type.
1631     record_klass_in_profile(receiver, mdp, reg2, true);
1632     bind(skip_receiver_profile);
1633 
1634     // The method data pointer needs to be updated to reflect the new target.
1635     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1636     bind(profile_continue);
1637   }
1638 }
1639 
1640 // This routine creates a state machine for updating the multi-row
1641 // type profile at a virtual call site (or other type-sensitive bytecode).
1642 // The machine visits each row (of receiver/count) until the receiver type
1643 // is found, or until it runs out of rows.  At the same time, it remembers
1644 // the location of the first empty row.  (An empty row records null for its
1645 // receiver, and can be allocated for a newly-observed receiver type.)
1646 // Because there are two degrees of freedom in the state, a simple linear
1647 // search will not work; it must be a decision tree.  Hence this helper
1648 // function is recursive, to generate the required tree structured code.
1649 // It's the interpreter, so we are trading off code space for speed.
1650 // See below for example code.
1651 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1652                                         Register receiver, Register mdp,
1653                                         Register reg2, int start_row,
1654                                         Label& done, bool is_virtual_call) {
1655   if (TypeProfileWidth == 0) {
1656     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1657   } else {
1658     record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
1659                                   &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
1660   }
1661 }
1662 
1663 void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp, Register reg2, int start_row,
1664                                                               Label& done, int total_rows,
1665                                                               OffsetFunction item_offset_fn,
1666                                                               OffsetFunction item_count_offset_fn) {
1667   int last_row = total_rows - 1;
1668   assert(start_row <= last_row, "must be work left to do");
1669   // Test this row for both the item and for null.
1670   // Take any of three different outcomes:
1671   //   1. found item => increment count and goto done
1672   //   2. found null => keep looking for case 1, maybe allocate this cell
1673   //   3. found something else => keep looking for cases 1 and 2
1674   // Case 3 is handled by a recursive call.

1738 //     // inner copy of decision tree, rooted at row[1]
1739 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1740 //     if (row[1].rec != nullptr) {
1741 //       // degenerate decision tree, rooted at row[2]
1742 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1743 //       if (row[2].rec != nullptr) { count.incr(); goto done; } // overflow
1744 //       row[2].init(rec); goto done;
1745 //     } else {
1746 //       // remember row[1] is empty
1747 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1748 //       row[1].init(rec); goto done;
1749 //     }
1750 //   } else {
1751 //     // remember row[0] is empty
1752 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1753 //     if (row[2].rec == rec) { row[2].incr(); goto done; }
1754 //     row[0].init(rec); goto done;
1755 //   }
1756 //   done:
1757 
1758 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1759                                                         Register mdp, Register reg2,
1760                                                         bool is_virtual_call) {
1761   assert(ProfileInterpreter, "must be profiling");
1762   Label done;
1763 
1764   record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
1765 
1766   bind (done);
1767 }
1768 
1769 void InterpreterMacroAssembler::profile_ret(Register return_bci,
1770                                             Register mdp) {
1771   if (ProfileInterpreter) {
1772     Label profile_continue;
1773     uint row;
1774 
1775     // If no method data exists, go to profile_continue.
1776     test_method_data_pointer(mdp, profile_continue);
1777 
1778     // Update the total ret count.
1779     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1780 
1781     for (row = 0; row < RetData::row_limit(); row++) {
1782       Label next_test;
1783 
1784       // See if return_bci is equal to bci[n]:

1821     update_mdp_by_constant(mdp, mdp_delta);
1822 
1823     bind(profile_continue);
1824   }
1825 }
1826 
1827 
1828 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
1829   if (ProfileInterpreter) {
1830     Label profile_continue;
1831 
1832     // If no method data exists, go to profile_continue.
1833     test_method_data_pointer(mdp, profile_continue);
1834 
1835     // The method data pointer needs to be updated.
1836     int mdp_delta = in_bytes(BitData::bit_data_size());
1837     if (TypeProfileCasts) {
1838       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1839 
1840       // Record the object type.
1841       record_klass_in_profile(klass, mdp, reg2, false);
1842       NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");)
1843       NOT_LP64(restore_locals();)         // Restore EDI
1844     }
1845     update_mdp_by_constant(mdp, mdp_delta);
1846 
1847     bind(profile_continue);
1848   }
1849 }
1850 
1851 
1852 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1853   if (ProfileInterpreter) {
1854     Label profile_continue;
1855 
1856     // If no method data exists, go to profile_continue.
1857     test_method_data_pointer(mdp, profile_continue);
1858 
1859     // Update the default case count
1860     increment_mdp_data_at(mdp,
1861                           in_bytes(MultiBranchData::default_count_offset()));

1883     // case_array_offset_in_bytes()
1884     movl(reg2, in_bytes(MultiBranchData::per_case_size()));
1885     imulptr(index, reg2); // XXX l ?
1886     addptr(index, in_bytes(MultiBranchData::case_array_offset())); // XXX l ?
1887 
1888     // Update the case count
1889     increment_mdp_data_at(mdp,
1890                           index,
1891                           in_bytes(MultiBranchData::relative_count_offset()));
1892 
1893     // The method data pointer needs to be updated.
1894     update_mdp_by_offset(mdp,
1895                          index,
1896                          in_bytes(MultiBranchData::
1897                                   relative_displacement_offset()));
1898 
1899     bind(profile_continue);
1900   }
1901 }
1902 


















































































































1903 
1904 
1905 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1906   if (state == atos) {
1907     MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1908   }
1909 }
1910 
1911 void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) {
1912 #ifndef _LP64
1913   if ((state == ftos && UseSSE < 1) ||
1914       (state == dtos && UseSSE < 2)) {
1915     MacroAssembler::verify_FPU(stack_depth);
1916   }
1917 #endif
1918 }
1919 
1920 // Jump if ((*counter_addr += increment) & mask) == 0
1921 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, Address mask,
1922                                                         Register scratch, Label* where) {

  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "compiler/compiler_globals.hpp"
  27 #include "interp_masm_x86.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "logging/log.hpp"
  31 #include "oops/arrayOop.hpp"
  32 #include "oops/constMethodFlags.hpp"
  33 #include "oops/markWord.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/inlineKlass.hpp"
  37 #include "oops/resolvedFieldEntry.hpp"
  38 #include "oops/resolvedIndyEntry.hpp"
  39 #include "oops/resolvedMethodEntry.hpp"
  40 #include "prims/jvmtiExport.hpp"
  41 #include "prims/jvmtiThreadState.hpp"
  42 #include "runtime/basicLock.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/javaThread.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 
  49 // Implementation of InterpreterMacroAssembler
  50 
  51 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  52   assert(entry, "Entry must have been generated by now");
  53   jump(RuntimeAddress(entry));
  54 }
  55 
  56 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {

 138         Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i))-off_to_args);
 139         profile_obj_type(tmp, mdo_arg_addr);
 140 
 141         int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
 142         addptr(mdp, to_add);
 143         off_to_args += to_add;
 144       }
 145 
 146       if (MethodData::profile_return()) {
 147         movptr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())-off_to_args));
 148         subl(tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
 149       }
 150 
 151       bind(done);
 152 
 153       if (MethodData::profile_return()) {
 154         // We're right after the type profile for the last
 155         // argument. tmp is the number of cells left in the
 156         // CallTypeData/VirtualCallTypeData to reach its end. Non null
 157         // if there's a return to profile.
 158         assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
 159         shll(tmp, log2i_exact((int)DataLayout::cell_size));
 160         addptr(mdp, tmp);
 161       }
 162       movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp);
 163     } else {
 164       assert(MethodData::profile_return(), "either profile call args or call ret");
 165       update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
 166     }
 167 
 168     // mdp points right after the end of the
 169     // CallTypeData/VirtualCallTypeData, right after the cells for the
 170     // return value type if there's one
 171 
 172     bind(profile_continue);
 173   }
 174 }
 175 
 176 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
 177   assert_different_registers(mdp, ret, tmp, _bcp_register);
 178   if (ProfileInterpreter && MethodData::profile_return()) {

 183     if (MethodData::profile_return_jsr292_only()) {
 184       assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2");
 185 
 186       // If we don't profile all invoke bytecodes we must make sure
 187       // it's a bytecode we indeed profile. We can't go back to the
 188       // beginning of the ProfileData we intend to update to check its
 189       // type because we're right after it and we don't known its
 190       // length
 191       Label do_profile;
 192       cmpb(Address(_bcp_register, 0), Bytecodes::_invokedynamic);
 193       jcc(Assembler::equal, do_profile);
 194       cmpb(Address(_bcp_register, 0), Bytecodes::_invokehandle);
 195       jcc(Assembler::equal, do_profile);
 196       get_method(tmp);
 197       cmpw(Address(tmp, Method::intrinsic_id_offset()), static_cast<int>(vmIntrinsics::_compiledLambdaForm));
 198       jcc(Assembler::notEqual, profile_continue);
 199 
 200       bind(do_profile);
 201     }
 202 
 203     Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
 204     mov(tmp, ret);
 205     profile_obj_type(tmp, mdo_ret_addr);
 206 
 207     bind(profile_continue);
 208   }
 209 }
 210 
 211 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
 212   if (ProfileInterpreter && MethodData::profile_parameters()) {
 213     Label profile_continue;
 214 
 215     test_method_data_pointer(mdp, profile_continue);
 216 
 217     // Load the offset of the area within the MDO used for
 218     // parameters. If it's negative we're not profiling any parameters
 219     movl(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
 220     testl(tmp1, tmp1);
 221     jcc(Assembler::negative, profile_continue);
 222 
 223     // Compute a pointer to the area for parameters from the offset

 478                                                              Register cpool,
 479                                                              Register index) {
 480   assert_different_registers(cpool, index);
 481 
 482   movw(index, Address(cpool, index, Address::times_ptr, sizeof(ConstantPool)));
 483   Register resolved_klasses = cpool;
 484   movptr(resolved_klasses, Address(cpool, ConstantPool::resolved_klasses_offset()));
 485   movptr(klass, Address(resolved_klasses, index, Address::times_ptr, Array<Klass*>::base_offset_in_bytes()));
 486 }
 487 
 488 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
 489 // subtype of super_klass.
 490 //
 491 // Args:
 492 //      rax: superklass
 493 //      Rsub_klass: subklass
 494 //
 495 // Kills:
 496 //      rcx, rdi
 497 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 498                                                   Label& ok_is_subtype,
 499                                                   bool profile) {
 500   assert(Rsub_klass != rax, "rax holds superklass");
 501   LP64_ONLY(assert(Rsub_klass != r14, "r14 holds locals");)
 502   LP64_ONLY(assert(Rsub_klass != r13, "r13 holds bcp");)
 503   assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
 504   assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
 505 
 506   // Profile the not-null value's klass.
 507   if (profile) {
 508     profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi
 509   }
 510 
 511   // Do the check.
 512   check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
 513 }
 514 
 515 
 516 #ifndef _LP64
 517 void InterpreterMacroAssembler::f2ieee() {
 518   if (IEEEPrecision) {
 519     fstp_s(Address(rsp, 0));
 520     fld_s(Address(rsp, 0));
 521   }
 522 }
 523 
 524 
 525 void InterpreterMacroAssembler::d2ieee() {
 526   if (IEEEPrecision) {
 527     fstp_d(Address(rsp, 0));
 528     fld_d(Address(rsp, 0));
 529   }

 941   // the stack, will call InterpreterRuntime::at_unwind.
 942   Label slow_path;
 943   Label fast_path;
 944   safepoint_poll(slow_path, rthread, true /* at_return */, false /* in_nmethod */);
 945   jmp(fast_path);
 946   bind(slow_path);
 947   push(state);
 948   set_last_Java_frame(rthread, noreg, rbp, (address)pc(), rscratch1);
 949   super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
 950   NOT_LP64(get_thread(rthread);) // call_VM clobbered it, restore
 951   reset_last_Java_frame(rthread, true);
 952   pop(state);
 953   bind(fast_path);
 954 
 955   // get the value of _do_not_unlock_if_synchronized into rdx
 956   const Address do_not_unlock_if_synchronized(rthread,
 957     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 958   movbool(rbx, do_not_unlock_if_synchronized);
 959   movbool(do_not_unlock_if_synchronized, false); // reset the flag
 960 
 961   // get method access flags
 962   movptr(rcx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
 963   movl(rcx, Address(rcx, Method::access_flags_offset()));
 964   testl(rcx, JVM_ACC_SYNCHRONIZED);
 965   jcc(Assembler::zero, unlocked);
 966 
 967   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 968   // is set.
 969   testbool(rbx);
 970   jcc(Assembler::notZero, no_unlock);
 971 
 972   // unlock monitor
 973   push(state); // save result
 974 
 975   // BasicObjectLock will be first in list, since this is a
 976   // synchronized method. However, need to check that the object has
 977   // not been unlocked by an explicit monitorexit bytecode.
 978   const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset *
 979                         wordSize - (int) sizeof(BasicObjectLock));
 980   // We use c_rarg1/rdx so that if we go slow path it will be the correct
 981   // register for unlock_object to pass to VM directly

1067     bind(loop);
1068     // check if current entry is used
1069     cmpptr(Address(rmon, BasicObjectLock::obj_offset()), NULL_WORD);
1070     jcc(Assembler::notEqual, exception);
1071 
1072     addptr(rmon, entry_size); // otherwise advance to next entry
1073     bind(entry);
1074     cmpptr(rmon, rbx); // check if bottom reached
1075     jcc(Assembler::notEqual, loop); // if not at bottom then check this entry
1076   }
1077 
1078   bind(no_unlock);
1079 
1080   // jvmti support
1081   if (notify_jvmdi) {
1082     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
1083   } else {
1084     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
1085   }
1086 




1087   if (StackReservedPages > 0) {
1088     movptr(rbx,
1089                Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
1090     // testing if reserved zone needs to be re-enabled
1091     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
1092     Label no_reserved_zone_enabling;
1093 
1094     NOT_LP64(get_thread(rthread);)
1095 
1096     // check if already enabled - if so no re-enabling needed
1097     assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
1098     cmpl(Address(rthread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_enabled);
1099     jcc(Assembler::equal, no_reserved_zone_enabling);
1100 
1101     cmpptr(rbx, Address(rthread, JavaThread::reserved_stack_activation_offset()));
1102     jcc(Assembler::lessEqual, no_reserved_zone_enabling);
1103 
1104     call_VM_leaf(
1105       CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
1106     call_VM(noreg, CAST_FROM_FN_PTR(address,
1107                    InterpreterRuntime::throw_delayed_StackOverflowError));
1108     should_not_reach_here();
1109 
1110     bind(no_reserved_zone_enabling);
1111   }
1112 
1113   // remove activation
1114   // get sender sp
1115   movptr(rbx,
1116          Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
1117 
1118   if (state == atos && InlineTypeReturnedAsFields) {
1119     // Check if we are returning an non-null inline type and load its fields into registers
1120     Label skip;
1121     test_oop_is_not_inline_type(rax, rscratch1, skip);
1122 
1123 #ifndef _LP64
1124     super_call_VM_leaf(StubRoutines::load_inline_type_fields_in_regs());
1125 #else
1126     // Load fields from a buffered value with an inline class specific handler
1127     load_klass(rdi, rax, rscratch1);
1128     movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
1129     movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
1130     // Unpack handler can be null if inline type is not scalarizable in returns
1131     testptr(rdi, rdi);
1132     jcc(Assembler::zero, skip);
1133     call(rdi);
1134 #endif
1135 #ifdef ASSERT
1136     // TODO 8284443 Enable
1137     if (StressCallingConvention && false) {
1138       Label skip_stress;
1139       movptr(rscratch1, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
1140       movl(rscratch1, Address(rscratch1, Method::flags_offset()));
1141       testl(rcx, ConstMethodFlags::has_scalarized_return_flag());
1142       jcc(Assembler::zero, skip_stress);
1143       load_klass(rax, rax, rscratch1);
1144       orptr(rax, 1);
1145       bind(skip_stress);
1146     }
1147 #endif
1148     // call above kills the value in rbx. Reload it.
1149     movptr(rbx, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
1150     bind(skip);
1151   }
1152   leave();                           // remove frame anchor
1153   pop(ret_addr);                     // get return address
1154   mov(rsp, rbx);                     // set sp to sender sp
1155   pop_cont_fastpath();
1156 }
1157 
1158 void InterpreterMacroAssembler::get_method_counters(Register method,
1159                                                     Register mcs, Label& skip) {
1160   Label has_counters;
1161   movptr(mcs, Address(method, Method::method_counters_offset()));
1162   testptr(mcs, mcs);
1163   jcc(Assembler::notZero, has_counters);
1164   call_VM(noreg, CAST_FROM_FN_PTR(address,
1165           InterpreterRuntime::build_method_counters), method);
1166   movptr(mcs, Address(method,Method::method_counters_offset()));
1167   testptr(mcs, mcs);
1168   jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
1169   bind(has_counters);
1170 }
1171 
1172 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
1173                                                   Register t1, Register t2,
1174                                                   bool clear_fields, Label& alloc_failed) {
1175   MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
1176   {
1177     SkipIfEqual skip_if(this, &DTraceAllocProbes, 0, rscratch1);
1178     // Trigger dtrace event for fastpath
1179     push(atos);
1180     call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
1181     pop(atos);
1182   }
1183 }
1184 
1185 
1186 void InterpreterMacroAssembler::read_flat_field(Register holder_klass,
1187                                                 Register field_index, Register field_offset,
1188                                                 Register obj) {
1189   Label alloc_failed, empty_value, done;
1190   const Register src = field_offset;
1191   const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi);
1192   const Register dst_temp   = LP64_ONLY(rscratch2) NOT_LP64(rdi);
1193   assert_different_registers(obj, holder_klass, field_index, field_offset, dst_temp);
1194 
1195   // Grap the inline field klass
1196   push(holder_klass);
1197   const Register field_klass = holder_klass;
1198   get_inline_type_field_klass(holder_klass, field_index, field_klass);
1199 
1200   //check for empty value klass
1201   test_klass_is_empty_inline_type(field_klass, dst_temp, empty_value);
1202 
1203   // allocate buffer
1204   push(obj); // save holder
1205   allocate_instance(field_klass, obj, alloc_temp, dst_temp, false, alloc_failed);
1206 
1207   // Have an oop instance buffer, copy into it
1208   data_for_oop(obj, dst_temp, field_klass);
1209   pop(alloc_temp);             // restore holder
1210   lea(src, Address(alloc_temp, field_offset));
1211   // call_VM_leaf, clobbers a few regs, save restore new obj
1212   push(obj);
1213   access_value_copy(IS_DEST_UNINITIALIZED, src, dst_temp, field_klass);
1214   pop(obj);
1215   pop(holder_klass);
1216   jmp(done);
1217 
1218   bind(empty_value);
1219   get_empty_inline_type_oop(field_klass, dst_temp, obj);
1220   pop(holder_klass);
1221   jmp(done);
1222 
1223   bind(alloc_failed);
1224   pop(obj);
1225   pop(holder_klass);
1226   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
1227           obj, field_index, holder_klass);
1228 
1229   bind(done);
1230 }
1231 
1232 void InterpreterMacroAssembler::read_flat_element(Register array, Register index,
1233                                                   Register t1, Register t2,
1234                                                   Register obj) {
1235   assert_different_registers(array, index, t1, t2);
1236   Label alloc_failed, empty_value, done;
1237   const Register array_klass = t2;
1238   const Register elem_klass = t1;
1239   const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi);
1240   const Register dst_temp   = LP64_ONLY(rscratch2) NOT_LP64(rdi);
1241 
1242   // load in array->klass()->element_klass()
1243   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1244   load_klass(array_klass, array, tmp_load_klass);
1245   movptr(elem_klass, Address(array_klass, ArrayKlass::element_klass_offset()));
1246 
1247   //check for empty value klass
1248   test_klass_is_empty_inline_type(elem_klass, dst_temp, empty_value);
1249 
1250   // calc source into "array_klass" and free up some regs
1251   const Register src = array_klass;
1252   push(index); // preserve index reg in case alloc_failed
1253   data_for_value_array_index(array, array_klass, index, src);
1254 
1255   allocate_instance(elem_klass, obj, alloc_temp, dst_temp, false, alloc_failed);
1256   // Have an oop instance buffer, copy into it
1257   store_ptr(0, obj); // preserve obj (overwrite index, no longer needed)
1258   data_for_oop(obj, dst_temp, elem_klass);
1259   access_value_copy(IS_DEST_UNINITIALIZED, src, dst_temp, elem_klass);
1260   pop(obj);
1261   jmp(done);
1262 
1263   bind(empty_value);
1264   get_empty_inline_type_oop(elem_klass, dst_temp, obj);
1265   jmp(done);
1266 
1267   bind(alloc_failed);
1268   pop(index);
1269   if (array == c_rarg2) {
1270     mov(elem_klass, array);
1271     array = elem_klass;
1272   }
1273   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), array, index);
1274 
1275   bind(done);
1276 }
1277 
1278 
1279 // Lock object
1280 //
1281 // Args:
1282 //      rdx, c_rarg1: BasicObjectLock to be used for locking
1283 //
1284 // Kills:
1285 //      rax, rbx
1286 void InterpreterMacroAssembler::lock_object(Register lock_reg) {
1287   assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1288          "The argument is only for looks. It must be c_rarg1");
1289 
1290   if (LockingMode == LM_MONITOR) {
1291     call_VM(noreg,
1292             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1293             lock_reg);
1294   } else {
1295     Label count_locking, done, slow_case;
1296 
1297     const Register swap_reg = rax; // Must use rax for cmpxchg instruction

1313       testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1314       jcc(Assembler::notZero, slow_case);
1315     }
1316 
1317     if (LockingMode == LM_LIGHTWEIGHT) {
1318 #ifdef _LP64
1319       const Register thread = r15_thread;
1320 #else
1321       const Register thread = lock_reg;
1322       get_thread(thread);
1323 #endif
1324       // Load object header, prepare for CAS from unlocked to locked.
1325       movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1326       lightweight_lock(obj_reg, swap_reg, thread, tmp_reg, slow_case);
1327     } else if (LockingMode == LM_LEGACY) {
1328       // Load immediate 1 into swap_reg %rax
1329       movl(swap_reg, 1);
1330 
1331       // Load (object->mark() | 1) into swap_reg %rax
1332       orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1333       if (EnableValhalla) {
1334         // Mask inline_type bit such that we go to the slow path if object is an inline type
1335         andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
1336       }
1337 
1338       // Save (object->mark() | 1) into BasicLock's displaced header
1339       movptr(Address(lock_reg, mark_offset), swap_reg);
1340 
1341       assert(lock_offset == 0,
1342              "displaced header must be first word in BasicObjectLock");
1343 
1344       lock();
1345       cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1346       jcc(Assembler::zero, count_locking);
1347 
1348       const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1349 
1350       // Fast check for recursive lock.
1351       //
1352       // Can apply the optimization only if this is a stack lock
1353       // allocated in this thread. For efficiency, we can focus on
1354       // recently allocated stack locks (instead of reading the stack
1355       // base and checking whether 'mark' points inside the current
1356       // thread stack):

1689     test_method_data_pointer(mdp, profile_continue);
1690 
1691     // We are taking a branch.  Increment the taken count.
1692     // We inline increment_mdp_data_at to return bumped_count in a register
1693     //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1694     Address data(mdp, in_bytes(JumpData::taken_offset()));
1695     movptr(bumped_count, data);
1696     assert(DataLayout::counter_increment == 1,
1697             "flow-free idiom only works with 1");
1698     addptr(bumped_count, DataLayout::counter_increment);
1699     sbbptr(bumped_count, 0);
1700     movptr(data, bumped_count); // Store back out
1701 
1702     // The method data pointer needs to be updated to reflect the new target.
1703     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1704     bind(profile_continue);
1705   }
1706 }
1707 
1708 
1709 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1710   if (ProfileInterpreter) {
1711     Label profile_continue;
1712 
1713     // If no method data exists, go to profile_continue.
1714     test_method_data_pointer(mdp, profile_continue);
1715 
1716     // We are taking a branch.  Increment the not taken count.
1717     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1718 
1719     // The method data pointer needs to be updated to correspond to
1720     // the next bytecode
1721     update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()): in_bytes(BranchData::branch_data_size()));
1722     bind(profile_continue);
1723   }
1724 }
1725 
1726 void InterpreterMacroAssembler::profile_call(Register mdp) {
1727   if (ProfileInterpreter) {
1728     Label profile_continue;
1729 
1730     // If no method data exists, go to profile_continue.
1731     test_method_data_pointer(mdp, profile_continue);
1732 
1733     // We are making a call.  Increment the count.
1734     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1735 
1736     // The method data pointer needs to be updated to reflect the new target.
1737     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1738     bind(profile_continue);
1739   }
1740 }
1741 

1764                                                      Register reg2,
1765                                                      bool receiver_can_be_null) {
1766   if (ProfileInterpreter) {
1767     Label profile_continue;
1768 
1769     // If no method data exists, go to profile_continue.
1770     test_method_data_pointer(mdp, profile_continue);
1771 
1772     Label skip_receiver_profile;
1773     if (receiver_can_be_null) {
1774       Label not_null;
1775       testptr(receiver, receiver);
1776       jccb(Assembler::notZero, not_null);
1777       // We are making a call.  Increment the count for null receiver.
1778       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1779       jmp(skip_receiver_profile);
1780       bind(not_null);
1781     }
1782 
1783     // Record the receiver type.
1784     record_klass_in_profile(receiver, mdp, reg2);
1785     bind(skip_receiver_profile);
1786 
1787     // The method data pointer needs to be updated to reflect the new target.
1788     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1789     bind(profile_continue);
1790   }
1791 }
1792 
1793 // This routine creates a state machine for updating the multi-row
1794 // type profile at a virtual call site (or other type-sensitive bytecode).
1795 // The machine visits each row (of receiver/count) until the receiver type
1796 // is found, or until it runs out of rows.  At the same time, it remembers
1797 // the location of the first empty row.  (An empty row records null for its
1798 // receiver, and can be allocated for a newly-observed receiver type.)
1799 // Because there are two degrees of freedom in the state, a simple linear
1800 // search will not work; it must be a decision tree.  Hence this helper
1801 // function is recursive, to generate the required tree structured code.
1802 // It's the interpreter, so we are trading off code space for speed.
1803 // See below for example code.
1804 void InterpreterMacroAssembler::record_klass_in_profile_helper(Register receiver, Register mdp,
1805                                                                Register reg2, int start_row,
1806                                                                Label& done) {

1807   if (TypeProfileWidth == 0) {
1808     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1809   } else {
1810     record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
1811                                   &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
1812   }
1813 }
1814 
1815 void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp, Register reg2, int start_row,
1816                                                               Label& done, int total_rows,
1817                                                               OffsetFunction item_offset_fn,
1818                                                               OffsetFunction item_count_offset_fn) {
1819   int last_row = total_rows - 1;
1820   assert(start_row <= last_row, "must be work left to do");
1821   // Test this row for both the item and for null.
1822   // Take any of three different outcomes:
1823   //   1. found item => increment count and goto done
1824   //   2. found null => keep looking for case 1, maybe allocate this cell
1825   //   3. found something else => keep looking for cases 1 and 2
1826   // Case 3 is handled by a recursive call.

1890 //     // inner copy of decision tree, rooted at row[1]
1891 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1892 //     if (row[1].rec != nullptr) {
1893 //       // degenerate decision tree, rooted at row[2]
1894 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1895 //       if (row[2].rec != nullptr) { count.incr(); goto done; } // overflow
1896 //       row[2].init(rec); goto done;
1897 //     } else {
1898 //       // remember row[1] is empty
1899 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1900 //       row[1].init(rec); goto done;
1901 //     }
1902 //   } else {
1903 //     // remember row[0] is empty
1904 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1905 //     if (row[2].rec == rec) { row[2].incr(); goto done; }
1906 //     row[0].init(rec); goto done;
1907 //   }
1908 //   done:
1909 
1910 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, Register mdp, Register reg2) {


1911   assert(ProfileInterpreter, "must be profiling");
1912   Label done;
1913 
1914   record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
1915 
1916   bind (done);
1917 }
1918 
1919 void InterpreterMacroAssembler::profile_ret(Register return_bci,
1920                                             Register mdp) {
1921   if (ProfileInterpreter) {
1922     Label profile_continue;
1923     uint row;
1924 
1925     // If no method data exists, go to profile_continue.
1926     test_method_data_pointer(mdp, profile_continue);
1927 
1928     // Update the total ret count.
1929     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1930 
1931     for (row = 0; row < RetData::row_limit(); row++) {
1932       Label next_test;
1933 
1934       // See if return_bci is equal to bci[n]:

1971     update_mdp_by_constant(mdp, mdp_delta);
1972 
1973     bind(profile_continue);
1974   }
1975 }
1976 
1977 
1978 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
1979   if (ProfileInterpreter) {
1980     Label profile_continue;
1981 
1982     // If no method data exists, go to profile_continue.
1983     test_method_data_pointer(mdp, profile_continue);
1984 
1985     // The method data pointer needs to be updated.
1986     int mdp_delta = in_bytes(BitData::bit_data_size());
1987     if (TypeProfileCasts) {
1988       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1989 
1990       // Record the object type.
1991       record_klass_in_profile(klass, mdp, reg2);
1992       NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");)
1993       NOT_LP64(restore_locals();)         // Restore EDI
1994     }
1995     update_mdp_by_constant(mdp, mdp_delta);
1996 
1997     bind(profile_continue);
1998   }
1999 }
2000 
2001 
2002 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
2003   if (ProfileInterpreter) {
2004     Label profile_continue;
2005 
2006     // If no method data exists, go to profile_continue.
2007     test_method_data_pointer(mdp, profile_continue);
2008 
2009     // Update the default case count
2010     increment_mdp_data_at(mdp,
2011                           in_bytes(MultiBranchData::default_count_offset()));

2033     // case_array_offset_in_bytes()
2034     movl(reg2, in_bytes(MultiBranchData::per_case_size()));
2035     imulptr(index, reg2); // XXX l ?
2036     addptr(index, in_bytes(MultiBranchData::case_array_offset())); // XXX l ?
2037 
2038     // Update the case count
2039     increment_mdp_data_at(mdp,
2040                           index,
2041                           in_bytes(MultiBranchData::relative_count_offset()));
2042 
2043     // The method data pointer needs to be updated.
2044     update_mdp_by_offset(mdp,
2045                          index,
2046                          in_bytes(MultiBranchData::
2047                                   relative_displacement_offset()));
2048 
2049     bind(profile_continue);
2050   }
2051 }
2052 
2053 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
2054                                                                               Register array,
2055                                                                               Register tmp) {
2056   if (ProfileInterpreter) {
2057     Label profile_continue;
2058 
2059     // If no method data exists, go to profile_continue.
2060     test_method_data_pointer(mdp, profile_continue);
2061 
2062     mov(tmp, array);
2063     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
2064 
2065     Label not_flat;
2066     test_non_flat_array_oop(array, tmp, not_flat);
2067 
2068     set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
2069 
2070     bind(not_flat);
2071 
2072     Label not_null_free;
2073     test_non_null_free_array_oop(array, tmp, not_null_free);
2074 
2075     set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
2076 
2077     bind(not_null_free);
2078 
2079     bind(profile_continue);
2080   }
2081 }
2082 
2083 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
2084                                                                            Register array,
2085                                                                            Register tmp);
2086 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
2087                                                                             Register array,
2088                                                                             Register tmp);
2089 
2090 
2091 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
2092   if (ProfileInterpreter) {
2093     Label profile_continue;
2094 
2095     // If no method data exists, go to profile_continue.
2096     test_method_data_pointer(mdp, profile_continue);
2097 
2098     Label done, update;
2099     testptr(element, element);
2100     jccb(Assembler::notZero, update);
2101     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
2102     jmp(done);
2103 
2104     bind(update);
2105     load_klass(tmp, element, rscratch1);
2106 
2107     // Record the object type.
2108     record_klass_in_profile(tmp, mdp, tmp2);
2109 
2110     bind(done);
2111 
2112     // The method data pointer needs to be updated.
2113     update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
2114 
2115     bind(profile_continue);
2116   }
2117 }
2118 
2119 void InterpreterMacroAssembler::profile_element_type(Register mdp,
2120                                                      Register element,
2121                                                      Register tmp) {
2122   if (ProfileInterpreter) {
2123     Label profile_continue;
2124 
2125     // If no method data exists, go to profile_continue.
2126     test_method_data_pointer(mdp, profile_continue);
2127 
2128     mov(tmp, element);
2129     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
2130 
2131     // The method data pointer needs to be updated.
2132     update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
2133 
2134     bind(profile_continue);
2135   }
2136 }
2137 
2138 void InterpreterMacroAssembler::profile_acmp(Register mdp,
2139                                              Register left,
2140                                              Register right,
2141                                              Register tmp) {
2142   if (ProfileInterpreter) {
2143     Label profile_continue;
2144 
2145     // If no method data exists, go to profile_continue.
2146     test_method_data_pointer(mdp, profile_continue);
2147 
2148     mov(tmp, left);
2149     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
2150 
2151     Label left_not_inline_type;
2152     test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
2153     set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
2154     bind(left_not_inline_type);
2155 
2156     mov(tmp, right);
2157     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
2158 
2159     Label right_not_inline_type;
2160     test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
2161     set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
2162     bind(right_not_inline_type);
2163 
2164     bind(profile_continue);
2165   }
2166 }
2167 
2168 
2169 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
2170   if (state == atos) {
2171     MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
2172   }
2173 }
2174 
2175 void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) {
2176 #ifndef _LP64
2177   if ((state == ftos && UseSSE < 1) ||
2178       (state == dtos && UseSSE < 2)) {
2179     MacroAssembler::verify_FPU(stack_depth);
2180   }
2181 #endif
2182 }
2183 
2184 // Jump if ((*counter_addr += increment) & mask) == 0
2185 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, Address mask,
2186                                                         Register scratch, Label* where) {
< prev index next >