1 /* 2 * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "c1/c1_MacroAssembler.hpp" 28 #include "c1/c1_Runtime1.hpp" 29 #include "gc/shared/barrierSetAssembler.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/tlab_globals.hpp" 34 #include "interpreter/interpreter.hpp" 35 #include "oops/arrayOop.hpp" 36 #include "oops/markWord.hpp" 37 #include "runtime/basicLock.hpp" 38 #include "runtime/os.hpp" 39 #include "runtime/sharedRuntime.hpp" 40 #include "runtime/stubRoutines.hpp" 41 42 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result, 43 FloatRegister f0, FloatRegister f1, 44 Register result) 45 { 46 Label done; 47 if (is_float) { 48 fcmps(f0, f1); 49 } else { 50 fcmpd(f0, f1); 51 } 52 if (unordered_result < 0) { 53 // we want -1 for unordered or less than, 0 for equal and 1 for 54 // greater than. 55 cset(result, NE); // Not equal or unordered 56 cneg(result, result, LT); // Less than or unordered 57 } else { 58 // we want -1 for less than, 0 for equal and 1 for unordered or 59 // greater than. 60 cset(result, NE); // Not equal or unordered 61 cneg(result, result, LO); // Less than 62 } 63 } 64 65 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { 66 const int aligned_mask = BytesPerWord -1; 67 const int hdr_offset = oopDesc::mark_offset_in_bytes(); 68 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); 69 Label done; 70 int null_check_offset = -1; 71 72 verify_oop(obj); 73 74 // save object being locked into the BasicObjectLock 75 str(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); 76 77 null_check_offset = offset(); 78 79 if (DiagnoseSyncOnValueBasedClasses != 0) { 80 load_klass(hdr, obj); 81 ldrw(hdr, Address(hdr, Klass::access_flags_offset())); 82 tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS); 83 br(Assembler::NE, slow_case); 84 } 85 86 // Load object header 87 ldr(hdr, Address(obj, hdr_offset)); 88 // and mark it as unlocked 89 orr(hdr, hdr, markWord::unlocked_value); 90 91 if (EnableValhalla) { 92 // Mask always_locked bit such that we go to the slow path if object is an inline type 93 andr(hdr, hdr, ~markWord::inline_type_bit_in_place); 94 } 95 96 // save unlocked object header into the displaced header location on the stack 97 str(hdr, Address(disp_hdr, 0)); 98 // test if object header is still the same (i.e. unlocked), and if so, store the 99 // displaced header address in the object header - if it is not the same, get the 100 // object header instead 101 lea(rscratch2, Address(obj, hdr_offset)); 102 cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL); 103 // if the object header was the same, we're done 104 // if the object header was not the same, it is now in the hdr register 105 // => test if it is a stack pointer into the same stack (recursive locking), i.e.: 106 // 107 // 1) (hdr & aligned_mask) == 0 108 // 2) sp <= hdr 109 // 3) hdr <= sp + page_size 110 // 111 // these 3 tests can be done by evaluating the following expression: 112 // 113 // (hdr - sp) & (aligned_mask - page_size) 114 // 115 // assuming both the stack pointer and page_size have their least 116 // significant 2 bits cleared and page_size is a power of 2 117 mov(rscratch1, sp); 118 sub(hdr, hdr, rscratch1); 119 ands(hdr, hdr, aligned_mask - (int)os::vm_page_size()); 120 // for recursive locking, the result is zero => save it in the displaced header 121 // location (NULL in the displaced hdr location indicates recursive locking) 122 str(hdr, Address(disp_hdr, 0)); 123 // otherwise we don't care about the result and handle locking via runtime call 124 cbnz(hdr, slow_case); 125 // done 126 bind(done); 127 increment(Address(rthread, JavaThread::held_monitor_count_offset())); 128 return null_check_offset; 129 } 130 131 132 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { 133 const int aligned_mask = BytesPerWord -1; 134 const int hdr_offset = oopDesc::mark_offset_in_bytes(); 135 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); 136 Label done; 137 138 // load displaced header 139 ldr(hdr, Address(disp_hdr, 0)); 140 // if the loaded hdr is NULL we had recursive locking 141 // if we had recursive locking, we are done 142 cbz(hdr, done); 143 // load object 144 ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); 145 verify_oop(obj); 146 // test if object header is pointing to the displaced header, and if so, restore 147 // the displaced header in the object - if the object header is not pointing to 148 // the displaced header, get the object header instead 149 // if the object header was not pointing to the displaced header, 150 // we do unlocking via runtime call 151 if (hdr_offset) { 152 lea(rscratch1, Address(obj, hdr_offset)); 153 cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case); 154 } else { 155 cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case); 156 } 157 // done 158 bind(done); 159 decrement(Address(rthread, JavaThread::held_monitor_count_offset())); 160 } 161 162 163 // Defines obj, preserves var_size_in_bytes 164 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) { 165 if (UseTLAB) { 166 tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 167 } else { 168 b(slow_case); 169 } 170 } 171 172 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { 173 assert_different_registers(obj, klass, len); 174 if (EnableValhalla) { 175 // Need to copy markWord::prototype header for klass 176 assert_different_registers(obj, klass, len, t1, t2); 177 ldr(t1, Address(klass, Klass::prototype_header_offset())); 178 } else { 179 // This assumes that all prototype bits fit in an int32_t 180 mov(t1, (int32_t)(intptr_t)markWord::prototype().value()); 181 } 182 str(t1, Address(obj, oopDesc::mark_offset_in_bytes())); 183 184 if (UseCompressedClassPointers) { // Take care not to kill klass 185 encode_klass_not_null(t1, klass); 186 strw(t1, Address(obj, oopDesc::klass_offset_in_bytes())); 187 } else { 188 str(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 189 } 190 191 if (len->is_valid()) { 192 strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes())); 193 } else if (UseCompressedClassPointers) { 194 store_klass_gap(obj, zr); 195 } 196 } 197 198 // preserves obj, destroys len_in_bytes 199 // 200 // Scratch registers: t1 = r10, t2 = r11 201 // 202 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1, Register t2) { 203 assert(hdr_size_in_bytes >= 0, "header size must be positive or 0"); 204 assert(t1 == r10 && t2 == r11, "must be"); 205 206 Label done; 207 208 // len_in_bytes is positive and ptr sized 209 subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes); 210 br(Assembler::EQ, done); 211 212 // zero_words() takes ptr in r10 and count in words in r11 213 mov(rscratch1, len_in_bytes); 214 lea(t1, Address(obj, hdr_size_in_bytes)); 215 lsr(t2, rscratch1, LogBytesPerWord); 216 address tpc = zero_words(t1, t2); 217 218 bind(done); 219 if (tpc == nullptr) { 220 Compilation::current()->bailout("no space for trampoline stub"); 221 } 222 } 223 224 225 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) { 226 assert_different_registers(obj, t1, t2); // XXX really? 227 assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); 228 229 try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case); 230 231 initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2, UseTLAB); 232 } 233 234 // Scratch registers: t1 = r10, t2 = r11 235 void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, bool is_tlab_allocated) { 236 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, 237 "con_size_in_bytes is not multiple of alignment"); 238 const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize; 239 240 initialize_header(obj, klass, noreg, t1, t2); 241 242 if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) { 243 // clear rest of allocated space 244 const Register index = t2; 245 if (var_size_in_bytes != noreg) { 246 mov(index, var_size_in_bytes); 247 initialize_body(obj, index, hdr_size_in_bytes, t1, t2); 248 if (Compilation::current()->bailed_out()) { 249 return; 250 } 251 } else if (con_size_in_bytes > hdr_size_in_bytes) { 252 con_size_in_bytes -= hdr_size_in_bytes; 253 lea(t1, Address(obj, hdr_size_in_bytes)); 254 address tpc = zero_words(t1, con_size_in_bytes / BytesPerWord); 255 if (tpc == nullptr) { 256 Compilation::current()->bailout("no space for trampoline stub"); 257 return; 258 } 259 } 260 } 261 262 membar(StoreStore); 263 264 if (CURRENT_ENV->dtrace_alloc_probes()) { 265 assert(obj == r0, "must be"); 266 far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); 267 } 268 269 verify_oop(obj); 270 } 271 272 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) { 273 assert_different_registers(obj, len, t1, t2, klass); 274 275 // determine alignment mask 276 assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work"); 277 278 // check for negative or excessive length 279 mov(rscratch1, (int32_t)max_array_allocation_length); 280 cmp(len, rscratch1); 281 br(Assembler::HS, slow_case); 282 283 const Register arr_size = t2; // okay to be the same 284 // align object end 285 mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask); 286 add(arr_size, arr_size, len, ext::uxtw, f); 287 andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask); 288 289 try_allocate(obj, arr_size, 0, t1, t2, slow_case); 290 291 initialize_header(obj, klass, len, t1, t2); 292 293 // clear rest of allocated space 294 initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2); 295 if (Compilation::current()->bailed_out()) { 296 return; 297 } 298 299 membar(StoreStore); 300 301 if (CURRENT_ENV->dtrace_alloc_probes()) { 302 assert(obj == r0, "must be"); 303 far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); 304 } 305 306 verify_oop(obj); 307 } 308 309 310 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) { 311 verify_oop(receiver); 312 // explicit NULL check not needed since load from [klass_offset] causes a trap 313 // check against inline cache 314 assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check"); 315 316 cmp_klass(receiver, iCache, rscratch1); 317 } 318 319 void C1_MacroAssembler::build_frame_helper(int frame_size_in_bytes, int sp_offset_for_orig_pc, int sp_inc, bool reset_orig_pc, bool needs_stack_repair) { 320 MacroAssembler::build_frame(frame_size_in_bytes); 321 322 if (needs_stack_repair) { 323 save_stack_increment(sp_inc, frame_size_in_bytes); 324 } 325 if (reset_orig_pc) { 326 // Zero orig_pc to detect deoptimization during buffering in the entry points 327 str(zr, Address(sp, sp_offset_for_orig_pc)); 328 } 329 } 330 331 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes, int sp_offset_for_orig_pc, bool needs_stack_repair, bool has_scalarized_args, Label* verified_inline_entry_label) { 332 // Make sure there is enough stack space for this method's activation. 333 // Note that we do this before creating a frame. 334 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect"); 335 generate_stack_overflow_check(bang_size_in_bytes); 336 337 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, 0, has_scalarized_args, needs_stack_repair); 338 339 // Insert nmethod entry barrier into frame. 340 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 341 bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */); 342 343 if (verified_inline_entry_label != NULL) { 344 // Jump here from the scalarized entry points that already created the frame. 345 bind(*verified_inline_entry_label); 346 } 347 } 348 349 void C1_MacroAssembler::verified_entry(bool breakAtEntry) { 350 // If we have to make this method not-entrant we'll overwrite its 351 // first instruction with a jump. For this action to be legal we 352 // must ensure that this first instruction is a B, BL, NOP, BKPT, 353 // SVC, HVC, or SMC. Make it a NOP. 354 nop(); 355 if (C1Breakpoint) brk(1); 356 } 357 358 int C1_MacroAssembler::scalarized_entry(const CompiledEntrySignature* ces, int frame_size_in_bytes, int bang_size_in_bytes, int sp_offset_for_orig_pc, Label& verified_inline_entry_label, bool is_inline_ro_entry) { 359 assert(InlineTypePassFieldsAsArgs, "sanity"); 360 // Make sure there is enough stack space for this method's activation. 361 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect"); 362 generate_stack_overflow_check(bang_size_in_bytes); 363 364 GrowableArray<SigEntry>* sig = ces->sig(); 365 GrowableArray<SigEntry>* sig_cc = is_inline_ro_entry ? ces->sig_cc_ro() : ces->sig_cc(); 366 VMRegPair* regs = ces->regs(); 367 VMRegPair* regs_cc = is_inline_ro_entry ? ces->regs_cc_ro() : ces->regs_cc(); 368 int args_on_stack = ces->args_on_stack(); 369 int args_on_stack_cc = is_inline_ro_entry ? ces->args_on_stack_cc_ro() : ces->args_on_stack_cc(); 370 371 assert(sig->length() <= sig_cc->length(), "Zero-sized inline class not allowed!"); 372 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, sig_cc->length()); 373 int args_passed = sig->length(); 374 int args_passed_cc = SigEntry::fill_sig_bt(sig_cc, sig_bt); 375 376 // Create a temp frame so we can call into the runtime. It must be properly set up to accommodate GC. 377 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, 0, true, ces->c1_needs_stack_repair()); 378 379 // The runtime call might safepoint, make sure nmethod entry barrier is executed 380 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 381 // C1 code is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub 382 bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */); 383 384 // FIXME -- call runtime only if we cannot in-line allocate all the incoming inline type args. 385 mov(r19, (intptr_t) ces->method()); 386 if (is_inline_ro_entry) { 387 far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::buffer_inline_args_no_receiver_id))); 388 } else { 389 far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::buffer_inline_args_id))); 390 } 391 int rt_call_offset = offset(); 392 393 // The runtime call returns the new array in r20 instead of the usual r0 394 // because r0 is also j_rarg7 which may be holding a live argument here. 395 Register val_array = r20; 396 397 // Remove the temp frame 398 MacroAssembler::remove_frame(frame_size_in_bytes); 399 400 // Check if we need to extend the stack for packing 401 int sp_inc = 0; 402 if (args_on_stack > args_on_stack_cc) { 403 sp_inc = extend_stack_for_inline_args(args_on_stack); 404 } 405 406 shuffle_inline_args(true, is_inline_ro_entry, sig_cc, 407 args_passed_cc, args_on_stack_cc, regs_cc, // from 408 args_passed, args_on_stack, regs, // to 409 sp_inc, val_array); 410 411 // Create the real frame. Below jump will then skip over the stack banging and frame 412 // setup code in the verified_inline_entry (which has a different real_frame_size). 413 build_frame_helper(frame_size_in_bytes, sp_offset_for_orig_pc, sp_inc, false, ces->c1_needs_stack_repair()); 414 415 b(verified_inline_entry_label); 416 return rt_call_offset; 417 } 418 419 420 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) { 421 // rfp, + 0: link 422 // + 1: return address 423 // + 2: argument with offset 0 424 // + 3: argument with offset 1 425 // + 4: ... 426 427 ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord)); 428 } 429 430 #ifndef PRODUCT 431 432 void C1_MacroAssembler::verify_stack_oop(int stack_offset) { 433 if (!VerifyOops) return; 434 verify_oop_addr(Address(sp, stack_offset)); 435 } 436 437 void C1_MacroAssembler::verify_not_null_oop(Register r) { 438 if (!VerifyOops) return; 439 Label not_null; 440 cbnz(r, not_null); 441 stop("non-null oop required"); 442 bind(not_null); 443 verify_oop(r); 444 } 445 446 void C1_MacroAssembler::invalidate_registers(bool inv_r0, bool inv_r19, bool inv_r2, bool inv_r3, bool inv_r4, bool inv_r5) { 447 #ifdef ASSERT 448 static int nn; 449 if (inv_r0) mov(r0, 0xDEAD); 450 if (inv_r19) mov(r19, 0xDEAD); 451 if (inv_r2) mov(r2, nn++); 452 if (inv_r3) mov(r3, 0xDEAD); 453 if (inv_r4) mov(r4, 0xDEAD); 454 if (inv_r5) mov(r5, 0xDEAD); 455 #endif 456 } 457 #endif // ifndef PRODUCT