< prev index next >

src/cpu/x86/vm/sharedRuntime_x86_64.cpp

Print this page




1334     } else if (in_regs[i].first()->is_XMMRegister()) {
1335       if (in_sig_bt[i] == T_FLOAT) {
1336         int offset = slot * VMRegImpl::stack_slot_size;
1337         slot++;
1338         assert(slot <= stack_slots, "overflow");
1339         if (map != NULL) {
1340           __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1341         } else {
1342           __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1343         }
1344       }
1345     } else if (in_regs[i].first()->is_stack()) {
1346       if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1347         int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1348         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1349       }
1350     }
1351   }
1352 }
1353 































































































1354 
1355 // Check GC_locker::needs_gc and enter the runtime if it's true.  This
1356 // keeps a new JNI critical region from starting until a GC has been
1357 // forced.  Save down any oops in registers and describe them in an
1358 // OopMap.
1359 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1360                                                int stack_slots,
1361                                                int total_c_args,
1362                                                int total_in_args,
1363                                                int arg_save_area,
1364                                                OopMapSet* oop_maps,
1365                                                VMRegPair* in_regs,
1366                                                BasicType* in_sig_bt) {
1367   __ block_comment("check GC_locker::needs_gc");
1368   Label cont;
1369   __ cmp8(ExternalAddress((address)GC_locker::needs_gc_address()), false);
1370   __ jcc(Assembler::equal, cont);
1371 
1372   // Save down any incoming oops and call into the runtime to halt for a GC
1373 


2023     }
2024 
2025 #ifdef ASSERT
2026     {
2027       Label L;
2028       __ mov(rax, rsp);
2029       __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2030       __ cmpptr(rax, rsp);
2031       __ jcc(Assembler::equal, L);
2032       __ stop("improperly aligned stack");
2033       __ bind(L);
2034     }
2035 #endif /* ASSERT */
2036 
2037 
2038   // We use r14 as the oop handle for the receiver/klass
2039   // It is callee save so it survives the call to native
2040 
2041   const Register oop_handle_reg = r14;
2042 
2043   if (is_critical_native) {
2044     check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
2045                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2046   }
2047 
2048   //
2049   // We immediately shuffle the arguments so that any vm call we have to
2050   // make from here on out (sync slow path, jvmti, etc.) we will have
2051   // captured the oops from our caller and have a valid oopMap for
2052   // them.
2053 
2054   // -----------------
2055   // The Grand Shuffle
2056 
2057   // The Java calling convention is either equal (linux) or denser (win64) than the
2058   // c calling convention. However the because of the jni_env argument the c calling
2059   // convention always has at least one more (and two for static) arguments than Java.
2060   // Therefore if we move the args from java -> c backwards then we will never have
2061   // a register->register conflict and we don't have to build a dependency graph
2062   // and figure out how to break any cycles.
2063   //


2080   // All inbound args are referenced based on rbp and all outbound args via rsp.
2081 
2082 
2083 #ifdef ASSERT
2084   bool reg_destroyed[RegisterImpl::number_of_registers];
2085   bool freg_destroyed[XMMRegisterImpl::number_of_registers];
2086   for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2087     reg_destroyed[r] = false;
2088   }
2089   for ( int f = 0 ; f < XMMRegisterImpl::number_of_registers ; f++ ) {
2090     freg_destroyed[f] = false;
2091   }
2092 
2093 #endif /* ASSERT */
2094 
2095   // This may iterate in two different directions depending on the
2096   // kind of native it is.  The reason is that for regular JNI natives
2097   // the incoming and outgoing registers are offset upwards and for
2098   // critical natives they are offset down.
2099   GrowableArray<int> arg_order(2 * total_in_args);





2100   VMRegPair tmp_vmreg;
2101   tmp_vmreg.set1(rbx->as_VMReg());
2102 
2103   if (!is_critical_native) {
2104     for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2105       arg_order.push(i);
2106       arg_order.push(c_arg);
2107     }
2108   } else {
2109     // Compute a valid move order, using tmp_vmreg to break any cycles
2110     ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
2111   }
2112 
2113   int temploc = -1;
2114   for (int ai = 0; ai < arg_order.length(); ai += 2) {
2115     int i = arg_order.at(ai);
2116     int c_arg = arg_order.at(ai + 1);
2117     __ block_comment(err_msg("move %d -> %d", i, c_arg));
2118     if (c_arg == -1) {
2119       assert(is_critical_native, "should only be required for critical natives");


2127       // Read from the temporary location
2128       assert(temploc != -1, "must be valid");
2129       i = temploc;
2130       temploc = -1;
2131     }
2132 #ifdef ASSERT
2133     if (in_regs[i].first()->is_Register()) {
2134       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2135     } else if (in_regs[i].first()->is_XMMRegister()) {
2136       assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2137     }
2138     if (out_regs[c_arg].first()->is_Register()) {
2139       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2140     } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2141       freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2142     }
2143 #endif /* ASSERT */
2144     switch (in_sig_bt[i]) {
2145       case T_ARRAY:
2146         if (is_critical_native) {






2147           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
2148           c_arg++;
2149 #ifdef ASSERT
2150           if (out_regs[c_arg].first()->is_Register()) {
2151             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2152           } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2153             freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2154           }
2155 #endif
2156           break;
2157         }
2158       case T_OBJECT:
2159         assert(!is_critical_native, "no oop arguments");
2160         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2161                     ((i == 0) && (!is_static)),
2162                     &receiver_offset);
2163         break;
2164       case T_VOID:
2165         break;
2166 


2341   // Verify or restore cpu control state after JNI call
2342   __ restore_cpu_control_state_after_jni();
2343 
2344   // Unpack native results.
2345   switch (ret_type) {
2346   case T_BOOLEAN: __ c2bool(rax);            break;
2347   case T_CHAR   : __ movzwl(rax, rax);      break;
2348   case T_BYTE   : __ sign_extend_byte (rax); break;
2349   case T_SHORT  : __ sign_extend_short(rax); break;
2350   case T_INT    : /* nothing to do */        break;
2351   case T_DOUBLE :
2352   case T_FLOAT  :
2353     // Result is in xmm0 we'll save as needed
2354     break;
2355   case T_ARRAY:                 // Really a handle
2356   case T_OBJECT:                // Really a handle
2357       break; // can't de-handlize until after safepoint check
2358   case T_VOID: break;
2359   case T_LONG: break;
2360   default       : ShouldNotReachHere();













2361   }
2362 
2363   // Switch thread to "native transition" state before reading the synchronization state.
2364   // This additional state is necessary because reading and testing the synchronization
2365   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2366   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2367   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2368   //     Thread A is resumed to finish this native method, but doesn't block here since it
2369   //     didn't see any synchronization is progress, and escapes.
2370   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2371 
2372   if(os::is_MP()) {
2373     if (UseMembar) {
2374       // Force this write out before the read below
2375       __ membar(Assembler::Membar_mask_bits(
2376            Assembler::LoadLoad | Assembler::LoadStore |
2377            Assembler::StoreLoad | Assembler::StoreStore));
2378     } else {
2379       // Write serialization page so VM thread can do a pseudo remote membar.
2380       // We use the current thread pointer to calculate a thread specific




1334     } else if (in_regs[i].first()->is_XMMRegister()) {
1335       if (in_sig_bt[i] == T_FLOAT) {
1336         int offset = slot * VMRegImpl::stack_slot_size;
1337         slot++;
1338         assert(slot <= stack_slots, "overflow");
1339         if (map != NULL) {
1340           __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1341         } else {
1342           __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1343         }
1344       }
1345     } else if (in_regs[i].first()->is_stack()) {
1346       if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1347         int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1348         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1349       }
1350     }
1351   }
1352 }
1353 
1354 // Pin incoming array argument of java critical method
1355 static void pin_critical_native_array(MacroAssembler* masm,
1356                                       VMRegPair reg,
1357                                       int& pinned_slot) {
1358   __ block_comment("pin_critical_native_array {");
1359   Register tmp_reg = rax;
1360 
1361   Label is_null;
1362   VMRegPair tmp;
1363   VMRegPair in_reg = reg;
1364   bool on_stack = false;
1365 
1366   tmp.set_ptr(tmp_reg->as_VMReg());
1367   if (reg.first()->is_stack()) {
1368     // Load the arg up from the stack
1369     move_ptr(masm, reg, tmp);
1370    reg = tmp;
1371     on_stack = true;
1372   } else {
1373     __ movptr(rax, reg.first()->as_Register());
1374   }
1375  __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1376   __ jccb(Assembler::equal, is_null);
1377 
1378   __ push(c_rarg0);
1379   __ push(c_rarg1);
1380   __ push(c_rarg2);
1381   __ push(c_rarg3);
1382 #ifdef _WIN64
1383   // caller-saved registers on Windows
1384   __ push(r10);
1385   __ push(r11);
1386 #else
1387   __ push(c_rarg4);
1388   __ push(c_rarg5);
1389 #endif
1390 
1391   if (reg.first()->as_Register() != c_rarg1) {
1392     __ movptr(c_rarg1, reg.first()->as_Register());
1393   }
1394   __ movptr(c_rarg0, r15_thread);
1395   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::pin_object)));
1396 
1397 #ifdef _WIN64
1398   __ pop(r11);
1399   __ pop(r10);
1400 #else
1401  __ pop(c_rarg5);
1402   __ pop(c_rarg4);
1403 #endif
1404   __ pop(c_rarg3);
1405   __ pop(c_rarg2);
1406   __ pop(c_rarg1);
1407   __ pop(c_rarg0);
1408 
1409   if (on_stack) {
1410     __ movptr(Address(rbp, reg2offset_in(in_reg.first())), rax);
1411     __ bind(is_null);
1412   } else {
1413     __ movptr(reg.first()->as_Register(), rax);
1414 
1415     // save on stack for unpinning later
1416     __ bind(is_null);
1417     assert(reg.first()->is_Register(), "Must be a register");
1418     int offset = pinned_slot * VMRegImpl::stack_slot_size;
1419     pinned_slot += VMRegImpl::slots_per_word;
1420     __ movq(Address(rsp, offset), rax);
1421   }
1422   __ block_comment("} pin_critical_native_array");
1423 }
1424 
1425 // Unpin array argument of java critical method
1426 static void unpin_critical_native_array(MacroAssembler* masm,
1427                                         VMRegPair reg,
1428                                         int& pinned_slot) {
1429   __ block_comment("unpin_critical_native_array {");
1430   Label is_null;
1431 
1432   if (reg.first()->is_stack()) {
1433     __ movptr(c_rarg1, Address(rbp, reg2offset_in(reg.first())));
1434   } else {
1435     int offset = pinned_slot * VMRegImpl::stack_slot_size;
1436     pinned_slot += VMRegImpl::slots_per_word;
1437     __ movq(c_rarg1, Address(rsp, offset));
1438   }
1439   __ testptr(c_rarg1, c_rarg1);
1440   __ jccb(Assembler::equal, is_null);
1441 
1442   __ movptr(c_rarg0, r15_thread);
1443   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object)));
1444 
1445   __ bind(is_null);
1446   __ block_comment("} unpin_critical_native_array");
1447 }
1448 
1449 
1450 // Check GC_locker::needs_gc and enter the runtime if it's true.  This
1451 // keeps a new JNI critical region from starting until a GC has been
1452 // forced.  Save down any oops in registers and describe them in an
1453 // OopMap.
1454 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1455                                                int stack_slots,
1456                                                int total_c_args,
1457                                                int total_in_args,
1458                                                int arg_save_area,
1459                                                OopMapSet* oop_maps,
1460                                                VMRegPair* in_regs,
1461                                                BasicType* in_sig_bt) {
1462   __ block_comment("check GC_locker::needs_gc");
1463   Label cont;
1464   __ cmp8(ExternalAddress((address)GC_locker::needs_gc_address()), false);
1465   __ jcc(Assembler::equal, cont);
1466 
1467   // Save down any incoming oops and call into the runtime to halt for a GC
1468 


2118     }
2119 
2120 #ifdef ASSERT
2121     {
2122       Label L;
2123       __ mov(rax, rsp);
2124       __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2125       __ cmpptr(rax, rsp);
2126       __ jcc(Assembler::equal, L);
2127       __ stop("improperly aligned stack");
2128       __ bind(L);
2129     }
2130 #endif /* ASSERT */
2131 
2132 
2133   // We use r14 as the oop handle for the receiver/klass
2134   // It is callee save so it survives the call to native
2135 
2136   const Register oop_handle_reg = r14;
2137 
2138   if (is_critical_native && !Universe::heap()->supports_object_pinning()) {
2139     check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
2140                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2141   }
2142 
2143   //
2144   // We immediately shuffle the arguments so that any vm call we have to
2145   // make from here on out (sync slow path, jvmti, etc.) we will have
2146   // captured the oops from our caller and have a valid oopMap for
2147   // them.
2148 
2149   // -----------------
2150   // The Grand Shuffle
2151 
2152   // The Java calling convention is either equal (linux) or denser (win64) than the
2153   // c calling convention. However the because of the jni_env argument the c calling
2154   // convention always has at least one more (and two for static) arguments than Java.
2155   // Therefore if we move the args from java -> c backwards then we will never have
2156   // a register->register conflict and we don't have to build a dependency graph
2157   // and figure out how to break any cycles.
2158   //


2175   // All inbound args are referenced based on rbp and all outbound args via rsp.
2176 
2177 
2178 #ifdef ASSERT
2179   bool reg_destroyed[RegisterImpl::number_of_registers];
2180   bool freg_destroyed[XMMRegisterImpl::number_of_registers];
2181   for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2182     reg_destroyed[r] = false;
2183   }
2184   for ( int f = 0 ; f < XMMRegisterImpl::number_of_registers ; f++ ) {
2185     freg_destroyed[f] = false;
2186   }
2187 
2188 #endif /* ASSERT */
2189 
2190   // This may iterate in two different directions depending on the
2191   // kind of native it is.  The reason is that for regular JNI natives
2192   // the incoming and outgoing registers are offset upwards and for
2193   // critical natives they are offset down.
2194   GrowableArray<int> arg_order(2 * total_in_args);
2195   // Inbound arguments that need to be pinned for critical natives
2196   GrowableArray<int> pinned_args(total_in_args);
2197   // Current stack slot for storing register based array argument
2198   int pinned_slot = oop_handle_offset;
2199 
2200   VMRegPair tmp_vmreg;
2201   tmp_vmreg.set1(rbx->as_VMReg());
2202 
2203   if (!is_critical_native) {
2204     for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2205       arg_order.push(i);
2206       arg_order.push(c_arg);
2207     }
2208   } else {
2209     // Compute a valid move order, using tmp_vmreg to break any cycles
2210     ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
2211   }
2212 
2213   int temploc = -1;
2214   for (int ai = 0; ai < arg_order.length(); ai += 2) {
2215     int i = arg_order.at(ai);
2216     int c_arg = arg_order.at(ai + 1);
2217     __ block_comment(err_msg("move %d -> %d", i, c_arg));
2218     if (c_arg == -1) {
2219       assert(is_critical_native, "should only be required for critical natives");


2227       // Read from the temporary location
2228       assert(temploc != -1, "must be valid");
2229       i = temploc;
2230       temploc = -1;
2231     }
2232 #ifdef ASSERT
2233     if (in_regs[i].first()->is_Register()) {
2234       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2235     } else if (in_regs[i].first()->is_XMMRegister()) {
2236       assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2237     }
2238     if (out_regs[c_arg].first()->is_Register()) {
2239       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2240     } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2241       freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2242     }
2243 #endif /* ASSERT */
2244     switch (in_sig_bt[i]) {
2245       case T_ARRAY:
2246         if (is_critical_native) {
2247           // pin before unpack
2248           if (Universe::heap()->supports_object_pinning()) {
2249             assert(pinned_slot <= stack_slots, "overflow");
2250             pin_critical_native_array(masm, in_regs[i], pinned_slot);
2251             pinned_args.append(i);
2252           }
2253           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
2254           c_arg++;
2255 #ifdef ASSERT
2256           if (out_regs[c_arg].first()->is_Register()) {
2257             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2258           } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2259             freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2260           }
2261 #endif
2262           break;
2263         }
2264       case T_OBJECT:
2265         assert(!is_critical_native, "no oop arguments");
2266         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2267                     ((i == 0) && (!is_static)),
2268                     &receiver_offset);
2269         break;
2270       case T_VOID:
2271         break;
2272 


2447   // Verify or restore cpu control state after JNI call
2448   __ restore_cpu_control_state_after_jni();
2449 
2450   // Unpack native results.
2451   switch (ret_type) {
2452   case T_BOOLEAN: __ c2bool(rax);            break;
2453   case T_CHAR   : __ movzwl(rax, rax);      break;
2454   case T_BYTE   : __ sign_extend_byte (rax); break;
2455   case T_SHORT  : __ sign_extend_short(rax); break;
2456   case T_INT    : /* nothing to do */        break;
2457   case T_DOUBLE :
2458   case T_FLOAT  :
2459     // Result is in xmm0 we'll save as needed
2460     break;
2461   case T_ARRAY:                 // Really a handle
2462   case T_OBJECT:                // Really a handle
2463       break; // can't de-handlize until after safepoint check
2464   case T_VOID: break;
2465   case T_LONG: break;
2466   default       : ShouldNotReachHere();
2467   }
2468 
2469   // unpin pinned arguments
2470   pinned_slot = oop_handle_offset;
2471   if (pinned_args.length() > 0) {
2472     // save return value that may be overwritten otherwise.
2473     save_native_result(masm, ret_type, stack_slots);
2474     for (int index = 0; index < pinned_args.length(); index ++) {
2475       int i = pinned_args.at(index);
2476       assert(pinned_slot <= stack_slots, "overflow");
2477       unpin_critical_native_array(masm, in_regs[i], pinned_slot);
2478     }
2479     restore_native_result(masm, ret_type, stack_slots);
2480   }
2481 
2482   // Switch thread to "native transition" state before reading the synchronization state.
2483   // This additional state is necessary because reading and testing the synchronization
2484   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2485   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2486   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2487   //     Thread A is resumed to finish this native method, but doesn't block here since it
2488   //     didn't see any synchronization is progress, and escapes.
2489   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2490 
2491   if(os::is_MP()) {
2492     if (UseMembar) {
2493       // Force this write out before the read below
2494       __ membar(Assembler::Membar_mask_bits(
2495            Assembler::LoadLoad | Assembler::LoadStore |
2496            Assembler::StoreLoad | Assembler::StoreStore));
2497     } else {
2498       // Write serialization page so VM thread can do a pseudo remote membar.
2499       // We use the current thread pointer to calculate a thread specific


< prev index next >