1334 } else if (in_regs[i].first()->is_XMMRegister()) {
1335 if (in_sig_bt[i] == T_FLOAT) {
1336 int offset = slot * VMRegImpl::stack_slot_size;
1337 slot++;
1338 assert(slot <= stack_slots, "overflow");
1339 if (map != NULL) {
1340 __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1341 } else {
1342 __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1343 }
1344 }
1345 } else if (in_regs[i].first()->is_stack()) {
1346 if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1347 int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1348 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1349 }
1350 }
1351 }
1352 }
1353
1354
1355 // Check GC_locker::needs_gc and enter the runtime if it's true. This
1356 // keeps a new JNI critical region from starting until a GC has been
1357 // forced. Save down any oops in registers and describe them in an
1358 // OopMap.
1359 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1360 int stack_slots,
1361 int total_c_args,
1362 int total_in_args,
1363 int arg_save_area,
1364 OopMapSet* oop_maps,
1365 VMRegPair* in_regs,
1366 BasicType* in_sig_bt) {
1367 __ block_comment("check GC_locker::needs_gc");
1368 Label cont;
1369 __ cmp8(ExternalAddress((address)GC_locker::needs_gc_address()), false);
1370 __ jcc(Assembler::equal, cont);
1371
1372 // Save down any incoming oops and call into the runtime to halt for a GC
1373
2023 }
2024
2025 #ifdef ASSERT
2026 {
2027 Label L;
2028 __ mov(rax, rsp);
2029 __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2030 __ cmpptr(rax, rsp);
2031 __ jcc(Assembler::equal, L);
2032 __ stop("improperly aligned stack");
2033 __ bind(L);
2034 }
2035 #endif /* ASSERT */
2036
2037
2038 // We use r14 as the oop handle for the receiver/klass
2039 // It is callee save so it survives the call to native
2040
2041 const Register oop_handle_reg = r14;
2042
2043 if (is_critical_native) {
2044 check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
2045 oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2046 }
2047
2048 //
2049 // We immediately shuffle the arguments so that any vm call we have to
2050 // make from here on out (sync slow path, jvmti, etc.) we will have
2051 // captured the oops from our caller and have a valid oopMap for
2052 // them.
2053
2054 // -----------------
2055 // The Grand Shuffle
2056
2057 // The Java calling convention is either equal (linux) or denser (win64) than the
2058 // c calling convention. However the because of the jni_env argument the c calling
2059 // convention always has at least one more (and two for static) arguments than Java.
2060 // Therefore if we move the args from java -> c backwards then we will never have
2061 // a register->register conflict and we don't have to build a dependency graph
2062 // and figure out how to break any cycles.
2063 //
2080 // All inbound args are referenced based on rbp and all outbound args via rsp.
2081
2082
2083 #ifdef ASSERT
2084 bool reg_destroyed[RegisterImpl::number_of_registers];
2085 bool freg_destroyed[XMMRegisterImpl::number_of_registers];
2086 for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2087 reg_destroyed[r] = false;
2088 }
2089 for ( int f = 0 ; f < XMMRegisterImpl::number_of_registers ; f++ ) {
2090 freg_destroyed[f] = false;
2091 }
2092
2093 #endif /* ASSERT */
2094
2095 // This may iterate in two different directions depending on the
2096 // kind of native it is. The reason is that for regular JNI natives
2097 // the incoming and outgoing registers are offset upwards and for
2098 // critical natives they are offset down.
2099 GrowableArray<int> arg_order(2 * total_in_args);
2100 VMRegPair tmp_vmreg;
2101 tmp_vmreg.set2(rbx->as_VMReg());
2102
2103 if (!is_critical_native) {
2104 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2105 arg_order.push(i);
2106 arg_order.push(c_arg);
2107 }
2108 } else {
2109 // Compute a valid move order, using tmp_vmreg to break any cycles
2110 ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
2111 }
2112
2113 int temploc = -1;
2114 for (int ai = 0; ai < arg_order.length(); ai += 2) {
2115 int i = arg_order.at(ai);
2116 int c_arg = arg_order.at(ai + 1);
2117 __ block_comment(err_msg("move %d -> %d", i, c_arg));
2118 if (c_arg == -1) {
2119 assert(is_critical_native, "should only be required for critical natives");
2127 // Read from the temporary location
2128 assert(temploc != -1, "must be valid");
2129 i = temploc;
2130 temploc = -1;
2131 }
2132 #ifdef ASSERT
2133 if (in_regs[i].first()->is_Register()) {
2134 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2135 } else if (in_regs[i].first()->is_XMMRegister()) {
2136 assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2137 }
2138 if (out_regs[c_arg].first()->is_Register()) {
2139 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2140 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2141 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2142 }
2143 #endif /* ASSERT */
2144 switch (in_sig_bt[i]) {
2145 case T_ARRAY:
2146 if (is_critical_native) {
2147 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
2148 c_arg++;
2149 #ifdef ASSERT
2150 if (out_regs[c_arg].first()->is_Register()) {
2151 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2152 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2153 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2154 }
2155 #endif
2156 break;
2157 }
2158 case T_OBJECT:
2159 assert(!is_critical_native, "no oop arguments");
2160 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2161 ((i == 0) && (!is_static)),
2162 &receiver_offset);
2163 break;
2164 case T_VOID:
2165 break;
2166
2341 // Verify or restore cpu control state after JNI call
2342 __ restore_cpu_control_state_after_jni();
2343
2344 // Unpack native results.
2345 switch (ret_type) {
2346 case T_BOOLEAN: __ c2bool(rax); break;
2347 case T_CHAR : __ movzwl(rax, rax); break;
2348 case T_BYTE : __ sign_extend_byte (rax); break;
2349 case T_SHORT : __ sign_extend_short(rax); break;
2350 case T_INT : /* nothing to do */ break;
2351 case T_DOUBLE :
2352 case T_FLOAT :
2353 // Result is in xmm0 we'll save as needed
2354 break;
2355 case T_ARRAY: // Really a handle
2356 case T_OBJECT: // Really a handle
2357 break; // can't de-handlize until after safepoint check
2358 case T_VOID: break;
2359 case T_LONG: break;
2360 default : ShouldNotReachHere();
2361 }
2362
2363 // Switch thread to "native transition" state before reading the synchronization state.
2364 // This additional state is necessary because reading and testing the synchronization
2365 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2366 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2367 // VM thread changes sync state to synchronizing and suspends threads for GC.
2368 // Thread A is resumed to finish this native method, but doesn't block here since it
2369 // didn't see any synchronization is progress, and escapes.
2370 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2371
2372 if(os::is_MP()) {
2373 if (UseMembar) {
2374 // Force this write out before the read below
2375 __ membar(Assembler::Membar_mask_bits(
2376 Assembler::LoadLoad | Assembler::LoadStore |
2377 Assembler::StoreLoad | Assembler::StoreStore));
2378 } else {
2379 // Write serialization page so VM thread can do a pseudo remote membar.
2380 // We use the current thread pointer to calculate a thread specific
|
1334 } else if (in_regs[i].first()->is_XMMRegister()) {
1335 if (in_sig_bt[i] == T_FLOAT) {
1336 int offset = slot * VMRegImpl::stack_slot_size;
1337 slot++;
1338 assert(slot <= stack_slots, "overflow");
1339 if (map != NULL) {
1340 __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1341 } else {
1342 __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1343 }
1344 }
1345 } else if (in_regs[i].first()->is_stack()) {
1346 if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1347 int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1348 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1349 }
1350 }
1351 }
1352 }
1353
1354 // Pin incoming array argument of java critical method
1355 static void pin_critical_native_array(MacroAssembler* masm,
1356 VMRegPair reg,
1357 int& pinned_slot) {
1358 __ block_comment("pin_critical_native_array {");
1359 Register tmp_reg = rax;
1360
1361 Label is_null;
1362 VMRegPair tmp;
1363 VMRegPair in_reg = reg;
1364 bool on_stack = false;
1365
1366 tmp.set_ptr(tmp_reg->as_VMReg());
1367 if (reg.first()->is_stack()) {
1368 // Load the arg up from the stack
1369 move_ptr(masm, reg, tmp);
1370 reg = tmp;
1371 on_stack = true;
1372 } else {
1373 __ movptr(rax, reg.first()->as_Register());
1374 }
1375 __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1376 __ jccb(Assembler::equal, is_null);
1377
1378 __ push(c_rarg0);
1379 __ push(c_rarg1);
1380 __ push(c_rarg2);
1381 __ push(c_rarg3);
1382 #ifdef _WIN64
1383 // caller-saved registers on Windows
1384 __ push(r10);
1385 __ push(r11);
1386 #else
1387 __ push(c_rarg4);
1388 __ push(c_rarg5);
1389 #endif
1390
1391 if (reg.first()->as_Register() != c_rarg1) {
1392 __ movptr(c_rarg1, reg.first()->as_Register());
1393 }
1394 __ movptr(c_rarg0, r15_thread);
1395 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::pin_object)));
1396
1397 #ifdef _WIN64
1398 __ pop(r11);
1399 __ pop(r10);
1400 #else
1401 __ pop(c_rarg5);
1402 __ pop(c_rarg4);
1403 #endif
1404 __ pop(c_rarg3);
1405 __ pop(c_rarg2);
1406 __ pop(c_rarg1);
1407 __ pop(c_rarg0);
1408
1409 if (on_stack) {
1410 __ movptr(Address(rbp, reg2offset_in(in_reg.first())), rax);
1411 __ bind(is_null);
1412 } else {
1413 __ movptr(reg.first()->as_Register(), rax);
1414
1415 // save on stack for unpinning later
1416 __ bind(is_null);
1417 assert(reg.first()->is_Register(), "Must be a register");
1418 int offset = pinned_slot * VMRegImpl::stack_slot_size;
1419 pinned_slot += VMRegImpl::slots_per_word;
1420 __ movq(Address(rsp, offset), rax);
1421 }
1422 __ block_comment("} pin_critical_native_array");
1423 }
1424
1425 // Unpin array argument of java critical method
1426 static void unpin_critical_native_array(MacroAssembler* masm,
1427 VMRegPair reg,
1428 int& pinned_slot) {
1429 __ block_comment("unpin_critical_native_array {");
1430 Label is_null;
1431
1432 if (reg.first()->is_stack()) {
1433 __ movptr(c_rarg1, Address(rbp, reg2offset_in(reg.first())));
1434 } else {
1435 int offset = pinned_slot * VMRegImpl::stack_slot_size;
1436 pinned_slot += VMRegImpl::slots_per_word;
1437 __ movq(c_rarg1, Address(rsp, offset));
1438 }
1439 __ testptr(c_rarg1, c_rarg1);
1440 __ jccb(Assembler::equal, is_null);
1441
1442 __ movptr(c_rarg0, r15_thread);
1443 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object)));
1444
1445 __ bind(is_null);
1446 __ block_comment("} unpin_critical_native_array");
1447 }
1448
1449
1450 // Check GC_locker::needs_gc and enter the runtime if it's true. This
1451 // keeps a new JNI critical region from starting until a GC has been
1452 // forced. Save down any oops in registers and describe them in an
1453 // OopMap.
1454 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1455 int stack_slots,
1456 int total_c_args,
1457 int total_in_args,
1458 int arg_save_area,
1459 OopMapSet* oop_maps,
1460 VMRegPair* in_regs,
1461 BasicType* in_sig_bt) {
1462 __ block_comment("check GC_locker::needs_gc");
1463 Label cont;
1464 __ cmp8(ExternalAddress((address)GC_locker::needs_gc_address()), false);
1465 __ jcc(Assembler::equal, cont);
1466
1467 // Save down any incoming oops and call into the runtime to halt for a GC
1468
2118 }
2119
2120 #ifdef ASSERT
2121 {
2122 Label L;
2123 __ mov(rax, rsp);
2124 __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2125 __ cmpptr(rax, rsp);
2126 __ jcc(Assembler::equal, L);
2127 __ stop("improperly aligned stack");
2128 __ bind(L);
2129 }
2130 #endif /* ASSERT */
2131
2132
2133 // We use r14 as the oop handle for the receiver/klass
2134 // It is callee save so it survives the call to native
2135
2136 const Register oop_handle_reg = r14;
2137
2138 if (is_critical_native && !Universe::heap()->supports_object_pinning()) {
2139 check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
2140 oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2141 }
2142
2143 //
2144 // We immediately shuffle the arguments so that any vm call we have to
2145 // make from here on out (sync slow path, jvmti, etc.) we will have
2146 // captured the oops from our caller and have a valid oopMap for
2147 // them.
2148
2149 // -----------------
2150 // The Grand Shuffle
2151
2152 // The Java calling convention is either equal (linux) or denser (win64) than the
2153 // c calling convention. However the because of the jni_env argument the c calling
2154 // convention always has at least one more (and two for static) arguments than Java.
2155 // Therefore if we move the args from java -> c backwards then we will never have
2156 // a register->register conflict and we don't have to build a dependency graph
2157 // and figure out how to break any cycles.
2158 //
2175 // All inbound args are referenced based on rbp and all outbound args via rsp.
2176
2177
2178 #ifdef ASSERT
2179 bool reg_destroyed[RegisterImpl::number_of_registers];
2180 bool freg_destroyed[XMMRegisterImpl::number_of_registers];
2181 for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2182 reg_destroyed[r] = false;
2183 }
2184 for ( int f = 0 ; f < XMMRegisterImpl::number_of_registers ; f++ ) {
2185 freg_destroyed[f] = false;
2186 }
2187
2188 #endif /* ASSERT */
2189
2190 // This may iterate in two different directions depending on the
2191 // kind of native it is. The reason is that for regular JNI natives
2192 // the incoming and outgoing registers are offset upwards and for
2193 // critical natives they are offset down.
2194 GrowableArray<int> arg_order(2 * total_in_args);
2195 // Inbound arguments that need to be pinned for critical natives
2196 GrowableArray<int> pinned_args(total_in_args);
2197 // Current stack slot for storing register based array argument
2198 int pinned_slot = oop_handle_offset;
2199
2200 VMRegPair tmp_vmreg;
2201 tmp_vmreg.set2(rbx->as_VMReg());
2202
2203 if (!is_critical_native) {
2204 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2205 arg_order.push(i);
2206 arg_order.push(c_arg);
2207 }
2208 } else {
2209 // Compute a valid move order, using tmp_vmreg to break any cycles
2210 ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
2211 }
2212
2213 int temploc = -1;
2214 for (int ai = 0; ai < arg_order.length(); ai += 2) {
2215 int i = arg_order.at(ai);
2216 int c_arg = arg_order.at(ai + 1);
2217 __ block_comment(err_msg("move %d -> %d", i, c_arg));
2218 if (c_arg == -1) {
2219 assert(is_critical_native, "should only be required for critical natives");
2227 // Read from the temporary location
2228 assert(temploc != -1, "must be valid");
2229 i = temploc;
2230 temploc = -1;
2231 }
2232 #ifdef ASSERT
2233 if (in_regs[i].first()->is_Register()) {
2234 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2235 } else if (in_regs[i].first()->is_XMMRegister()) {
2236 assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2237 }
2238 if (out_regs[c_arg].first()->is_Register()) {
2239 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2240 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2241 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2242 }
2243 #endif /* ASSERT */
2244 switch (in_sig_bt[i]) {
2245 case T_ARRAY:
2246 if (is_critical_native) {
2247 // pin before unpack
2248 if (Universe::heap()->supports_object_pinning()) {
2249 assert(pinned_slot <= stack_slots, "overflow");
2250 pin_critical_native_array(masm, in_regs[i], pinned_slot);
2251 pinned_args.append(i);
2252 }
2253 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
2254 c_arg++;
2255 #ifdef ASSERT
2256 if (out_regs[c_arg].first()->is_Register()) {
2257 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2258 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2259 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2260 }
2261 #endif
2262 break;
2263 }
2264 case T_OBJECT:
2265 assert(!is_critical_native, "no oop arguments");
2266 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2267 ((i == 0) && (!is_static)),
2268 &receiver_offset);
2269 break;
2270 case T_VOID:
2271 break;
2272
2447 // Verify or restore cpu control state after JNI call
2448 __ restore_cpu_control_state_after_jni();
2449
2450 // Unpack native results.
2451 switch (ret_type) {
2452 case T_BOOLEAN: __ c2bool(rax); break;
2453 case T_CHAR : __ movzwl(rax, rax); break;
2454 case T_BYTE : __ sign_extend_byte (rax); break;
2455 case T_SHORT : __ sign_extend_short(rax); break;
2456 case T_INT : /* nothing to do */ break;
2457 case T_DOUBLE :
2458 case T_FLOAT :
2459 // Result is in xmm0 we'll save as needed
2460 break;
2461 case T_ARRAY: // Really a handle
2462 case T_OBJECT: // Really a handle
2463 break; // can't de-handlize until after safepoint check
2464 case T_VOID: break;
2465 case T_LONG: break;
2466 default : ShouldNotReachHere();
2467 }
2468
2469 // unpin pinned arguments
2470 pinned_slot = oop_handle_offset;
2471 if (pinned_args.length() > 0) {
2472 // save return value that may be overwritten otherwise.
2473 save_native_result(masm, ret_type, stack_slots);
2474 for (int index = 0; index < pinned_args.length(); index ++) {
2475 int i = pinned_args.at(index);
2476 assert(pinned_slot <= stack_slots, "overflow");
2477 unpin_critical_native_array(masm, in_regs[i], pinned_slot);
2478 }
2479 restore_native_result(masm, ret_type, stack_slots);
2480 }
2481
2482 // Switch thread to "native transition" state before reading the synchronization state.
2483 // This additional state is necessary because reading and testing the synchronization
2484 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2485 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2486 // VM thread changes sync state to synchronizing and suspends threads for GC.
2487 // Thread A is resumed to finish this native method, but doesn't block here since it
2488 // didn't see any synchronization is progress, and escapes.
2489 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2490
2491 if(os::is_MP()) {
2492 if (UseMembar) {
2493 // Force this write out before the read below
2494 __ membar(Assembler::Membar_mask_bits(
2495 Assembler::LoadLoad | Assembler::LoadStore |
2496 Assembler::StoreLoad | Assembler::StoreStore));
2497 } else {
2498 // Write serialization page so VM thread can do a pseudo remote membar.
2499 // We use the current thread pointer to calculate a thread specific
|