2361 Label cont;
2362 Label object_has_monitor;
2363
2364 assert_different_registers(oop, box, tmp, disp_hdr, t0);
2365
2366 // Load markWord from object into displaced_header.
2367 __ ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
2368
2369 if (DiagnoseSyncOnValueBasedClasses != 0) {
2370 __ load_klass(flag, oop);
2371 __ lwu(flag, Address(flag, Klass::access_flags_offset()));
2372 __ andi(flag, flag, JVM_ACC_IS_VALUE_BASED_CLASS, tmp /* tmp */);
2373 __ bnez(flag, cont, true /* is_far */);
2374 }
2375
2376 // Check for existing monitor
2377 __ andi(t0, disp_hdr, markWord::monitor_value);
2378 __ bnez(t0, object_has_monitor);
2379
2380 if (!UseHeavyMonitors) {
2381 // Set tmp to be (markWord of object | UNLOCK_VALUE).
2382 __ ori(tmp, disp_hdr, markWord::unlocked_value);
2383
2384 // Initialize the box. (Must happen before we update the object mark!)
2385 __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2386
2387 // Compare object markWord with an unlocked value (tmp) and if
2388 // equal exchange the stack address of our box with object markWord.
2389 // On failure disp_hdr contains the possibly locked markWord.
2390 __ cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq,
2391 Assembler::rl, /*result*/disp_hdr);
2392 __ mv(flag, zr);
2393 __ beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas
2394
2395 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2396
2397 // If the compare-and-exchange succeeded, then we found an unlocked
2398 // object, will have now locked it will continue at label cont
2399 // We did not see an unlocked object so try the fast recursive case.
2400
2401 // Check if the owner is self by comparing the value in the
2402 // markWord of object (disp_hdr) with the stack pointer.
2403 __ sub(disp_hdr, disp_hdr, sp);
2404 __ mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
2405 // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont,
2406 // hence we can store 0 as the displaced header in the box, which indicates that it is a
2407 // recursive lock.
2408 __ andr(tmp/*==0?*/, disp_hdr, tmp);
2409 __ sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2410 __ mv(flag, tmp); // we can use the value of tmp as the result here
2411 } else {
2412 __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
2413 }
2414
2415 __ j(cont);
2416
2417 // Handle existing monitor.
2418 __ bind(object_has_monitor);
2419 // The object's monitor m is unlocked iff m->owner == NULL,
2420 // otherwise m->owner may contain a thread or a stack address.
2421 //
2422 // Try to CAS m->owner from NULL to current thread.
2423 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes() - markWord::monitor_value));
2424 __ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq,
2425 Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected)
2426
2427 // Store a non-null value into the box to avoid looking like a re-entrant
2428 // lock. The fast-path monitor unlock code checks for
2429 // markWord::monitor_value so use markWord::unused_mark which has the
2430 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
2431 __ mv(tmp, (address)markWord::unused_mark().value());
2432 __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2433
2434 __ beqz(flag, cont); // CAS success means locking succeeded
2435
2436 __ bne(flag, xthread, cont); // Check for recursive locking
2437
2438 // Recursive lock case
2439 __ mv(flag, zr);
2440 __ ld(tmp, Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value));
2441 __ add(tmp, tmp, 1u);
2442 __ sd(tmp, Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value));
2443
2444 __ bind(cont);
2445 %}
2446
2447 // using cr flag to indicate the fast_unlock result: 0 for success; others failed.
2448 enc_class riscv_enc_fast_unlock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{
2449 C2_MacroAssembler _masm(&cbuf);
2450 Register flag = t1;
2451 Register oop = as_Register($object$$reg);
2452 Register box = as_Register($box$$reg);
2453 Register disp_hdr = as_Register($tmp1$$reg);
2454 Register tmp = as_Register($tmp2$$reg);
2455 Label cont;
2456 Label object_has_monitor;
2457
2458 assert_different_registers(oop, box, tmp, disp_hdr, flag);
2459
2460 if (!UseHeavyMonitors) {
2461 // Find the lock address and load the displaced header from the stack.
2462 __ ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2463
2464 // If the displaced header is 0, we have a recursive unlock.
2465 __ mv(flag, disp_hdr);
2466 __ beqz(disp_hdr, cont);
2467 }
2468
2469 // Handle existing monitor.
2470 __ ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
2471 __ andi(t0, disp_hdr, markWord::monitor_value);
2472 __ bnez(t0, object_has_monitor);
2473
2474 if (!UseHeavyMonitors) {
2475 // Check if it is still a light weight lock, this is true if we
2476 // see the stack address of the basicLock in the markWord of the
2477 // object.
2478
2479 __ cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed,
2480 Assembler::rl, /*result*/tmp);
2481 __ xorr(flag, box, tmp); // box == tmp if cas succeeds
2482 } else {
2483 __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
2484 }
2485 __ j(cont);
2486
2487 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2488
2489 // Handle existing monitor.
2490 __ bind(object_has_monitor);
2491 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2492 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
2493 __ ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
2494
2495 Label notRecursive;
2496 __ beqz(disp_hdr, notRecursive); // Will be 0 if not recursive.
2497
2498 // Recursive lock
2499 __ addi(disp_hdr, disp_hdr, -1);
2500 __ sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
2501 __ mv(flag, zr);
2502 __ j(cont);
2503
2504 __ bind(notRecursive);
2505 __ ld(flag, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
2506 __ ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
2507 __ orr(flag, flag, disp_hdr); // Will be 0 if both are 0.
2508 __ bnez(flag, cont);
2509 // need a release store here
2510 __ la(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
2511 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
2512 __ sd(zr, Address(tmp)); // set unowned
10377 // for this guy.
10378 instruct tlsLoadP(javaThread_RegP dst)
10379 %{
10380 match(Set dst (ThreadLocal));
10381
10382 ins_cost(0);
10383
10384 format %{ " -- \t// $dst=Thread::current(), empty, #@tlsLoadP" %}
10385
10386 size(0);
10387
10388 ins_encode( /*empty*/ );
10389
10390 ins_pipe(pipe_class_empty);
10391 %}
10392
10393 // inlined locking and unlocking
10394 // using t1 as the 'flag' register to bridge the BoolNode producers and consumers
10395 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2)
10396 %{
10397 match(Set cr (FastLock object box));
10398 effect(TEMP tmp1, TEMP tmp2);
10399
10400 ins_cost(LOAD_COST * 2 + STORE_COST * 3 + ALU_COST * 6 + BRANCH_COST * 3);
10401 format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2, #@cmpFastLock" %}
10402
10403 ins_encode(riscv_enc_fast_lock(object, box, tmp1, tmp2));
10404
10405 ins_pipe(pipe_serial);
10406 %}
10407
10408 // using t1 as the 'flag' register to bridge the BoolNode producers and consumers
10409 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2)
10410 %{
10411 match(Set cr (FastUnlock object box));
10412 effect(TEMP tmp1, TEMP tmp2);
10413
10414 ins_cost(LOAD_COST * 2 + STORE_COST + ALU_COST * 2 + BRANCH_COST * 4);
10415 format %{ "fastunlock $object,$box\t! kills $tmp1, $tmp2, #@cmpFastUnlock" %}
10416
10417 ins_encode(riscv_enc_fast_unlock(object, box, tmp1, tmp2));
10418
10419 ins_pipe(pipe_serial);
10420 %}
10421
10422 // Tail Call; Jump from runtime stub to Java code.
10423 // Also known as an 'interprocedural jump'.
10424 // Target of jump will eventually return to caller.
10425 // TailJump below removes the return address.
10426 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
10427 %{
10428 match(TailCall jump_target method_oop);
10429
10430 ins_cost(BRANCH_COST);
10431
10432 format %{ "jalr $jump_target\t# $method_oop holds method oop, #@TailCalljmpInd." %}
|
2361 Label cont;
2362 Label object_has_monitor;
2363
2364 assert_different_registers(oop, box, tmp, disp_hdr, t0);
2365
2366 // Load markWord from object into displaced_header.
2367 __ ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
2368
2369 if (DiagnoseSyncOnValueBasedClasses != 0) {
2370 __ load_klass(flag, oop);
2371 __ lwu(flag, Address(flag, Klass::access_flags_offset()));
2372 __ andi(flag, flag, JVM_ACC_IS_VALUE_BASED_CLASS, tmp /* tmp */);
2373 __ bnez(flag, cont, true /* is_far */);
2374 }
2375
2376 // Check for existing monitor
2377 __ andi(t0, disp_hdr, markWord::monitor_value);
2378 __ bnez(t0, object_has_monitor);
2379
2380 if (!UseHeavyMonitors) {
2381 Label slow;
2382 __ fast_lock(oop, disp_hdr, box, tmp, t0, slow);
2383
2384 // Indicate success at cont.
2385 __ mv(flag, zr);
2386 __ j(cont);
2387 __ bind(slow);
2388 }
2389
2390 __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
2391 __ j(cont);
2392
2393 // Handle existing monitor.
2394 __ bind(object_has_monitor);
2395 // The object's monitor m is unlocked iff m->owner == NULL,
2396 // otherwise m->owner may contain a thread or a stack address.
2397 //
2398 // Try to CAS m->owner from NULL to current thread.
2399 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes() - markWord::monitor_value));
2400 __ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq,
2401 Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected)
2402
2403 __ beqz(flag, cont); // CAS success means locking succeeded
2404
2405 __ bne(flag, xthread, cont); // Check for recursive locking
2406
2407 // Recursive lock case
2408 __ mv(flag, zr);
2409 __ ld(tmp, Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value));
2410 __ add(tmp, tmp, 1u);
2411 __ sd(tmp, Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value));
2412
2413 __ bind(cont);
2414 %}
2415
2416 // using cr flag to indicate the fast_unlock result: 0 for success; others failed.
2417 enc_class riscv_enc_fast_unlock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{
2418 C2_MacroAssembler _masm(&cbuf);
2419 Register flag = t1;
2420 Register oop = as_Register($object$$reg);
2421 Register box = as_Register($box$$reg);
2422 Register disp_hdr = as_Register($tmp1$$reg);
2423 Register tmp = as_Register($tmp2$$reg);
2424 Label cont;
2425 Label object_has_monitor;
2426
2427 assert_different_registers(oop, box, tmp, disp_hdr, flag);
2428
2429 // Handle existing monitor.
2430 __ ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
2431
2432 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2433
2434 if (!UseHeavyMonitors) {
2435 Label slow;
2436 __ andi(t0, tmp, markWord::monitor_value);
2437 __ bnez(t0, object_has_monitor);
2438
2439 __ fast_unlock(oop, tmp, box, disp_hdr, slow);
2440
2441 // Indicate success at cont.
2442 __ mv(flag, zr);
2443 __ j(cont);
2444 __ bind(slow);
2445 }
2446
2447 __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
2448 __ j(cont);
2449
2450 // Handle existing monitor.
2451 __ bind(object_has_monitor);
2452 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2453 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
2454
2455 // If the owner is anonymous, we need to fix it -- in the slow-path.
2456 {
2457 Label L;
2458 __ ld(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
2459 __ mv(t0, (unsigned char)(intptr_t)ANONYMOUS_OWNER);
2460 __ bne(disp_hdr, t0, L);
2461 __ mv(flag, 1); // Indicate failure at cont -- dive into slow-path.
2462 __ j(cont);
2463 __ bind(L);
2464 }
2465
2466 __ ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
2467
2468 Label notRecursive;
2469 __ beqz(disp_hdr, notRecursive); // Will be 0 if not recursive.
2470
2471 // Recursive lock
2472 __ addi(disp_hdr, disp_hdr, -1);
2473 __ sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
2474 __ mv(flag, zr);
2475 __ j(cont);
2476
2477 __ bind(notRecursive);
2478 __ ld(flag, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
2479 __ ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
2480 __ orr(flag, flag, disp_hdr); // Will be 0 if both are 0.
2481 __ bnez(flag, cont);
2482 // need a release store here
2483 __ la(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
2484 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
2485 __ sd(zr, Address(tmp)); // set unowned
10350 // for this guy.
10351 instruct tlsLoadP(javaThread_RegP dst)
10352 %{
10353 match(Set dst (ThreadLocal));
10354
10355 ins_cost(0);
10356
10357 format %{ " -- \t// $dst=Thread::current(), empty, #@tlsLoadP" %}
10358
10359 size(0);
10360
10361 ins_encode( /*empty*/ );
10362
10363 ins_pipe(pipe_class_empty);
10364 %}
10365
10366 // inlined locking and unlocking
10367 // using t1 as the 'flag' register to bridge the BoolNode producers and consumers
10368 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2)
10369 %{
10370 match(Set cr (FastLock object));
10371 effect(TEMP box, TEMP tmp1, TEMP tmp2);
10372
10373 ins_cost(LOAD_COST * 2 + STORE_COST * 3 + ALU_COST * 6 + BRANCH_COST * 3);
10374 format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2, #@cmpFastLock" %}
10375
10376 ins_encode(riscv_enc_fast_lock(object, box, tmp1, tmp2));
10377
10378 ins_pipe(pipe_serial);
10379 %}
10380
10381 // using t1 as the 'flag' register to bridge the BoolNode producers and consumers
10382 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2)
10383 %{
10384 match(Set cr (FastUnlock object));
10385 effect(TEMP box, TEMP tmp1, TEMP tmp2);
10386
10387 ins_cost(LOAD_COST * 2 + STORE_COST + ALU_COST * 2 + BRANCH_COST * 4);
10388 format %{ "fastunlock $object,$box\t! kills $tmp1, $tmp2, #@cmpFastUnlock" %}
10389
10390 ins_encode(riscv_enc_fast_unlock(object, box, tmp1, tmp2));
10391
10392 ins_pipe(pipe_serial);
10393 %}
10394
10395 // Tail Call; Jump from runtime stub to Java code.
10396 // Also known as an 'interprocedural jump'.
10397 // Target of jump will eventually return to caller.
10398 // TailJump below removes the return address.
10399 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
10400 %{
10401 match(TailCall jump_target method_oop);
10402
10403 ins_cost(BRANCH_COST);
10404
10405 format %{ "jalr $jump_target\t# $method_oop holds method oop, #@TailCalljmpInd." %}
|