28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/output.hpp"
32 #include "opto/subnode.hpp"
33 #include "runtime/objectMonitorTable.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "runtime/synchronizer.hpp"
36 #include "utilities/globalDefinitions.hpp"
37
38 #ifdef PRODUCT
39 #define BLOCK_COMMENT(str) /* nothing */
40 #define STOP(error) stop(error)
41 #else
42 #define BLOCK_COMMENT(str) block_comment(str)
43 #define STOP(error) block_comment(error); stop(error)
44 #endif
45
46 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
47
48 void C2_MacroAssembler::fast_lock(Register obj, Register box,
49 Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
50 // Flag register, zero for success; non-zero for failure.
51 Register flag = t1;
52
53 assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag, t0);
54
55 mv(flag, 1);
56
57 // Handle inflated monitor.
58 Label inflated;
59 // Finish fast lock successfully. MUST branch to with flag == 0
60 Label locked;
61 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
62 Label slow_path;
63
64 if (UseObjectMonitorTable) {
65 // Clear cache in case fast locking succeeds or we need to take the slow-path.
66 sd(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
67 }
2338
2339 __ j(stub.continuation());
2340 #undef __
2341 }
2342
2343 // j.l.Float.float16ToFloat
2344 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
2345 auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
2346
2347 // On riscv, NaN needs a special process as fcvt does not work in that case.
2348 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
2349 // but we consider to get the slow path to process NaN and Inf at the same time,
2350 // as both of them are rare cases, and if we try to get the slow path to handle
2351 // only NaN case it would sacrifise the performance for normal cases,
2352 // i.e. non-NaN and non-Inf cases.
2353
2354 // check whether it's a NaN or +/- Inf.
2355 mv(t0, 0x7c00);
2356 andr(tmp, src, t0);
2357 // jump to stub processing NaN and Inf cases.
2358 beq(t0, tmp, stub->entry(), true);
2359
2360 // non-NaN or non-Inf cases, just use built-in instructions.
2361 fmv_h_x(dst, src);
2362 fcvt_s_h(dst, dst);
2363
2364 bind(stub->continuation());
2365 }
2366
2367 static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Register, FloatRegister, Register>& stub) {
2368 #define __ masm.
2369 Register dst = stub.data<0>();
2370 FloatRegister src = stub.data<1>();
2371 Register tmp = stub.data<2>();
2372 __ bind(stub.entry());
2373
2374 __ float_to_float16_NaN(dst, src, t0, tmp);
2375
2376 __ j(stub.continuation());
2377 #undef __
2378 }
2379
2380 // j.l.Float.floatToFloat16
2381 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
2382 auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 64, float_to_float16_slow_path);
2383
2384 // On riscv, NaN needs a special process as fcvt does not work in that case.
2385
2386 // check whether it's a NaN.
2387 // replace fclass with feq as performance optimization.
2388 feq_s(t0, src, src);
2389 // jump to stub processing NaN cases.
2390 beqz(t0, stub->entry(), true);
2391
2392 // non-NaN cases, just use built-in instructions.
2393 fcvt_h_s(ftmp, src);
2394 fmv_x_h(dst, ftmp);
2395
2396 bind(stub->continuation());
2397 }
2398
2399 static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
2400 #define __ masm.
2401 VectorRegister dst = stub.data<0>();
2402 VectorRegister src = stub.data<1>();
2403 uint vector_length = stub.data<2>();
2404 __ bind(stub.entry());
2405
2406 // following instructions mainly focus on NaN, as riscv does not handle
2407 // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
2408 //
2409 // construct NaN's in 32 bits from the NaN's in 16 bits,
2410 // we need the payloads of non-canonical NaNs to be preserved.
2431 // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
2432 // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
2433 // but we consider to get the slow path to process NaN and Inf at the same time,
2434 // as both of them are rare cases, and if we try to get the slow path to handle
2435 // only NaN case it would sacrifise the performance for normal cases,
2436 // i.e. non-NaN and non-Inf cases.
2437
2438 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
2439
2440 // check whether there is a NaN or +/- Inf.
2441 mv(t0, 0x7c00);
2442 vand_vx(v0, src, t0);
2443 // v0 will be used as mask in slow path.
2444 vmseq_vx(v0, v0, t0);
2445 vcpop_m(t0, v0);
2446
2447 // For non-NaN or non-Inf cases, just use built-in instructions.
2448 vfwcvt_f_f_v(dst, src);
2449
2450 // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
2451 bnez(t0, stub->entry(), true);
2452
2453 bind(stub->continuation());
2454 }
2455
2456 static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
2457 C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
2458 #define __ masm.
2459 VectorRegister dst = stub.data<0>();
2460 VectorRegister src = stub.data<1>();
2461 VectorRegister vtmp = stub.data<2>();
2462 assert_different_registers(dst, src, vtmp);
2463
2464 __ bind(stub.entry());
2465
2466 // Active elements (NaNs) are marked in v0 mask register.
2467 // mul is already set to mf2 in float_to_float16_v.
2468
2469 // Float (32 bits)
2470 // Bit: 31 30 to 23 22 to 0
2471 // +---+------------------+-----------------------------+
2524 assert_different_registers(dst, src, vtmp);
2525
2526 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
2527 (dst, src, vtmp, 56, float_to_float16_v_slow_path);
2528
2529 // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
2530
2531 vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
2532
2533 // check whether there is a NaN.
2534 // replace v_fclass with vmfne_vv as performance optimization.
2535 vmfne_vv(v0, src, src);
2536 vcpop_m(t0, v0);
2537
2538 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
2539
2540 // For non-NaN cases, just use built-in instructions.
2541 vfncvt_f_f_w(dst, src);
2542
2543 // jump to stub processing NaN cases.
2544 bnez(t0, stub->entry(), true);
2545
2546 bind(stub->continuation());
2547 }
2548
2549 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
2550 vsetvli_helper(bt, vlen);
2551
2552 // check if input is -0, +0, signaling NaN or quiet NaN
2553 vfclass_v(v0, dst);
2554 mv(t0, FClassBits::zero | FClassBits::nan);
2555 vand_vx(v0, v0, t0);
2556 vmseq_vi(v0, v0, 0);
2557
2558 // use floating-point 1.0 with a sign of input
2559 vfsgnj_vv(dst, one, dst, v0_t);
2560 }
2561
2562 // j.l.Math.round(float)
2563 // Returns the closest int to the argument, with ties rounding to positive infinity.
2564 // We need to handle 3 special cases defined by java api spec:
|
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/output.hpp"
32 #include "opto/subnode.hpp"
33 #include "runtime/objectMonitorTable.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "runtime/synchronizer.hpp"
36 #include "utilities/globalDefinitions.hpp"
37
38 #ifdef PRODUCT
39 #define BLOCK_COMMENT(str) /* nothing */
40 #define STOP(error) stop(error)
41 #else
42 #define BLOCK_COMMENT(str) block_comment(str)
43 #define STOP(error) block_comment(error); stop(error)
44 #endif
45
46 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
47
48 void C2_MacroAssembler::entry_barrier() {
49 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
50 // Dummy labels for just measuring the code size
51 Label dummy_slow_path;
52 Label dummy_continuation;
53 Label dummy_guard;
54 Label* slow_path = &dummy_slow_path;
55 Label* continuation = &dummy_continuation;
56 Label* guard = &dummy_guard;
57
58 if (!Compile::current()->output()->in_scratch_emit_size()) {
59 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
60 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
61 Compile::current()->output()->add_stub(stub);
62 slow_path = &stub->entry();
63 continuation = &stub->continuation();
64 guard = &stub->guard();
65 }
66
67 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
68 bs->nmethod_entry_barrier(this, slow_path, continuation, guard);
69 }
70
71 void C2_MacroAssembler::fast_lock(Register obj, Register box,
72 Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
73 // Flag register, zero for success; non-zero for failure.
74 Register flag = t1;
75
76 assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag, t0);
77
78 mv(flag, 1);
79
80 // Handle inflated monitor.
81 Label inflated;
82 // Finish fast lock successfully. MUST branch to with flag == 0
83 Label locked;
84 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
85 Label slow_path;
86
87 if (UseObjectMonitorTable) {
88 // Clear cache in case fast locking succeeds or we need to take the slow-path.
89 sd(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
90 }
2361
2362 __ j(stub.continuation());
2363 #undef __
2364 }
2365
2366 // j.l.Float.float16ToFloat
2367 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
2368 auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
2369
2370 // On riscv, NaN needs a special process as fcvt does not work in that case.
2371 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
2372 // but we consider to get the slow path to process NaN and Inf at the same time,
2373 // as both of them are rare cases, and if we try to get the slow path to handle
2374 // only NaN case it would sacrifise the performance for normal cases,
2375 // i.e. non-NaN and non-Inf cases.
2376
2377 // check whether it's a NaN or +/- Inf.
2378 mv(t0, 0x7c00);
2379 andr(tmp, src, t0);
2380 // jump to stub processing NaN and Inf cases.
2381 beq(t0, tmp, stub->entry(), /* is_far */ true);
2382
2383 // non-NaN or non-Inf cases, just use built-in instructions.
2384 fmv_h_x(dst, src);
2385 fcvt_s_h(dst, dst);
2386
2387 bind(stub->continuation());
2388 }
2389
2390 static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Register, FloatRegister, Register>& stub) {
2391 #define __ masm.
2392 Register dst = stub.data<0>();
2393 FloatRegister src = stub.data<1>();
2394 Register tmp = stub.data<2>();
2395 __ bind(stub.entry());
2396
2397 __ float_to_float16_NaN(dst, src, t0, tmp);
2398
2399 __ j(stub.continuation());
2400 #undef __
2401 }
2402
2403 // j.l.Float.floatToFloat16
2404 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
2405 auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 64, float_to_float16_slow_path);
2406
2407 // On riscv, NaN needs a special process as fcvt does not work in that case.
2408
2409 // check whether it's a NaN.
2410 // replace fclass with feq as performance optimization.
2411 feq_s(t0, src, src);
2412 // jump to stub processing NaN cases.
2413 beqz(t0, stub->entry(), /* is_far */ true);
2414
2415 // non-NaN cases, just use built-in instructions.
2416 fcvt_h_s(ftmp, src);
2417 fmv_x_h(dst, ftmp);
2418
2419 bind(stub->continuation());
2420 }
2421
2422 static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
2423 #define __ masm.
2424 VectorRegister dst = stub.data<0>();
2425 VectorRegister src = stub.data<1>();
2426 uint vector_length = stub.data<2>();
2427 __ bind(stub.entry());
2428
2429 // following instructions mainly focus on NaN, as riscv does not handle
2430 // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
2431 //
2432 // construct NaN's in 32 bits from the NaN's in 16 bits,
2433 // we need the payloads of non-canonical NaNs to be preserved.
2454 // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
2455 // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
2456 // but we consider to get the slow path to process NaN and Inf at the same time,
2457 // as both of them are rare cases, and if we try to get the slow path to handle
2458 // only NaN case it would sacrifise the performance for normal cases,
2459 // i.e. non-NaN and non-Inf cases.
2460
2461 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
2462
2463 // check whether there is a NaN or +/- Inf.
2464 mv(t0, 0x7c00);
2465 vand_vx(v0, src, t0);
2466 // v0 will be used as mask in slow path.
2467 vmseq_vx(v0, v0, t0);
2468 vcpop_m(t0, v0);
2469
2470 // For non-NaN or non-Inf cases, just use built-in instructions.
2471 vfwcvt_f_f_v(dst, src);
2472
2473 // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
2474 bnez(t0, stub->entry(), /* is_far */ true);
2475
2476 bind(stub->continuation());
2477 }
2478
2479 static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
2480 C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
2481 #define __ masm.
2482 VectorRegister dst = stub.data<0>();
2483 VectorRegister src = stub.data<1>();
2484 VectorRegister vtmp = stub.data<2>();
2485 assert_different_registers(dst, src, vtmp);
2486
2487 __ bind(stub.entry());
2488
2489 // Active elements (NaNs) are marked in v0 mask register.
2490 // mul is already set to mf2 in float_to_float16_v.
2491
2492 // Float (32 bits)
2493 // Bit: 31 30 to 23 22 to 0
2494 // +---+------------------+-----------------------------+
2547 assert_different_registers(dst, src, vtmp);
2548
2549 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
2550 (dst, src, vtmp, 56, float_to_float16_v_slow_path);
2551
2552 // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
2553
2554 vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
2555
2556 // check whether there is a NaN.
2557 // replace v_fclass with vmfne_vv as performance optimization.
2558 vmfne_vv(v0, src, src);
2559 vcpop_m(t0, v0);
2560
2561 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
2562
2563 // For non-NaN cases, just use built-in instructions.
2564 vfncvt_f_f_w(dst, src);
2565
2566 // jump to stub processing NaN cases.
2567 bnez(t0, stub->entry(), /* is_far */ true);
2568
2569 bind(stub->continuation());
2570 }
2571
2572 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
2573 vsetvli_helper(bt, vlen);
2574
2575 // check if input is -0, +0, signaling NaN or quiet NaN
2576 vfclass_v(v0, dst);
2577 mv(t0, FClassBits::zero | FClassBits::nan);
2578 vand_vx(v0, v0, t0);
2579 vmseq_vi(v0, v0, 0);
2580
2581 // use floating-point 1.0 with a sign of input
2582 vfsgnj_vv(dst, one, dst, v0_t);
2583 }
2584
2585 // j.l.Math.round(float)
2586 // Returns the closest int to the argument, with ties rounding to positive infinity.
2587 // We need to handle 3 special cases defined by java api spec:
|