1 /* 2 * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/macroAssembler.inline.hpp" 28 #include "c1/c1_CodeStubs.hpp" 29 #include "c1/c1_FrameMap.hpp" 30 #include "c1/c1_LIRAssembler.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "classfile/javaClasses.hpp" 34 #include "nativeInst_aarch64.hpp" 35 #include "runtime/sharedRuntime.hpp" 36 #include "vmreg_aarch64.inline.hpp" 37 38 39 #define __ ce->masm()-> 40 41 void C1SafepointPollStub::emit_code(LIR_Assembler* ce) { 42 __ bind(_entry); 43 InternalAddress safepoint_pc(ce->masm()->pc() - ce->masm()->offset() + safepoint_offset()); 44 __ adr(rscratch1, safepoint_pc); 45 __ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset())); 46 47 assert(SharedRuntime::polling_page_return_handler_blob() != NULL, 48 "polling page return stub not created yet"); 49 address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); 50 51 __ far_jump(RuntimeAddress(stub)); 52 } 53 54 void CounterOverflowStub::emit_code(LIR_Assembler* ce) { 55 __ bind(_entry); 56 Metadata *m = _method->as_constant_ptr()->as_metadata(); 57 __ mov_metadata(rscratch1, m); 58 ce->store_parameter(rscratch1, 1); 59 ce->store_parameter(_bci, 0); 60 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::counter_overflow_id))); 61 ce->add_call_info_here(_info); 62 ce->verify_oop_map(_info); 63 __ b(_continuation); 64 } 65 66 RangeCheckStub::RangeCheckStub(CodeEmitInfo* info, LIR_Opr index, LIR_Opr array) 67 : _index(index), _array(array), _throw_index_out_of_bounds_exception(false) { 68 assert(info != NULL, "must have info"); 69 _info = new CodeEmitInfo(info); 70 } 71 72 RangeCheckStub::RangeCheckStub(CodeEmitInfo* info, LIR_Opr index) 73 : _index(index), _array(), _throw_index_out_of_bounds_exception(true) { 74 assert(info != NULL, "must have info"); 75 _info = new CodeEmitInfo(info); 76 } 77 78 void RangeCheckStub::emit_code(LIR_Assembler* ce) { 79 __ bind(_entry); 80 if (_info->deoptimize_on_exception()) { 81 address a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); 82 __ far_call(RuntimeAddress(a)); 83 ce->add_call_info_here(_info); 84 ce->verify_oop_map(_info); 85 debug_only(__ should_not_reach_here()); 86 return; 87 } 88 89 if (_index->is_cpu_register()) { 90 __ mov(rscratch1, _index->as_register()); 91 } else { 92 __ mov(rscratch1, _index->as_jint()); 93 } 94 Runtime1::StubID stub_id; 95 if (_throw_index_out_of_bounds_exception) { 96 stub_id = Runtime1::throw_index_exception_id; 97 } else { 98 assert(_array != LIR_Opr::nullOpr(), "sanity"); 99 __ mov(rscratch2, _array->as_pointer_register()); 100 stub_id = Runtime1::throw_range_check_failed_id; 101 } 102 __ lea(lr, RuntimeAddress(Runtime1::entry_for(stub_id))); 103 __ blr(lr); 104 ce->add_call_info_here(_info); 105 ce->verify_oop_map(_info); 106 debug_only(__ should_not_reach_here()); 107 } 108 109 PredicateFailedStub::PredicateFailedStub(CodeEmitInfo* info) { 110 _info = new CodeEmitInfo(info); 111 } 112 113 void PredicateFailedStub::emit_code(LIR_Assembler* ce) { 114 __ bind(_entry); 115 address a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); 116 __ far_call(RuntimeAddress(a)); 117 ce->add_call_info_here(_info); 118 ce->verify_oop_map(_info); 119 debug_only(__ should_not_reach_here()); 120 } 121 122 void DivByZeroStub::emit_code(LIR_Assembler* ce) { 123 if (_offset != -1) { 124 ce->compilation()->implicit_exception_table()->append(_offset, __ offset()); 125 } 126 __ bind(_entry); 127 __ far_call(Address(Runtime1::entry_for(Runtime1::throw_div0_exception_id), relocInfo::runtime_call_type)); 128 ce->add_call_info_here(_info); 129 ce->verify_oop_map(_info); 130 #ifdef ASSERT 131 __ should_not_reach_here(); 132 #endif 133 } 134 135 // Implementation of LoadFlattenedArrayStub 136 137 LoadFlattenedArrayStub::LoadFlattenedArrayStub(LIR_Opr array, LIR_Opr index, LIR_Opr result, CodeEmitInfo* info) { 138 _array = array; 139 _index = index; 140 _result = result; 141 _scratch_reg = FrameMap::r0_oop_opr; 142 _info = new CodeEmitInfo(info); 143 } 144 145 void LoadFlattenedArrayStub::emit_code(LIR_Assembler* ce) { 146 assert(__ rsp_offset() == 0, "frame size should be fixed"); 147 __ bind(_entry); 148 ce->store_parameter(_array->as_register(), 1); 149 ce->store_parameter(_index->as_register(), 0); 150 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::load_flattened_array_id))); 151 ce->add_call_info_here(_info); 152 ce->verify_oop_map(_info); 153 if (_result->as_register() != r0) { 154 __ mov(_result->as_register(), r0); 155 } 156 __ b(_continuation); 157 } 158 159 160 // Implementation of StoreFlattenedArrayStub 161 162 StoreFlattenedArrayStub::StoreFlattenedArrayStub(LIR_Opr array, LIR_Opr index, LIR_Opr value, CodeEmitInfo* info) { 163 _array = array; 164 _index = index; 165 _value = value; 166 _scratch_reg = FrameMap::r0_oop_opr; 167 _info = new CodeEmitInfo(info); 168 } 169 170 171 void StoreFlattenedArrayStub::emit_code(LIR_Assembler* ce) { 172 assert(__ rsp_offset() == 0, "frame size should be fixed"); 173 __ bind(_entry); 174 ce->store_parameter(_array->as_register(), 2); 175 ce->store_parameter(_index->as_register(), 1); 176 ce->store_parameter(_value->as_register(), 0); 177 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::store_flattened_array_id))); 178 ce->add_call_info_here(_info); 179 ce->verify_oop_map(_info); 180 __ b(_continuation); 181 } 182 183 // Implementation of SubstitutabilityCheckStub 184 SubstitutabilityCheckStub::SubstitutabilityCheckStub(LIR_Opr left, LIR_Opr right, CodeEmitInfo* info) { 185 _left = left; 186 _right = right; 187 _scratch_reg = FrameMap::r0_oop_opr; 188 _info = new CodeEmitInfo(info); 189 } 190 191 void SubstitutabilityCheckStub::emit_code(LIR_Assembler* ce) { 192 assert(__ rsp_offset() == 0, "frame size should be fixed"); 193 __ bind(_entry); 194 ce->store_parameter(_left->as_register(), 1); 195 ce->store_parameter(_right->as_register(), 0); 196 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::substitutability_check_id))); 197 ce->add_call_info_here(_info); 198 ce->verify_oop_map(_info); 199 __ b(_continuation); 200 } 201 202 203 // Implementation of NewInstanceStub 204 205 NewInstanceStub::NewInstanceStub(LIR_Opr klass_reg, LIR_Opr result, ciInstanceKlass* klass, CodeEmitInfo* info, Runtime1::StubID stub_id) { 206 _result = result; 207 _klass = klass; 208 _klass_reg = klass_reg; 209 _info = new CodeEmitInfo(info); 210 assert(stub_id == Runtime1::new_instance_id || 211 stub_id == Runtime1::new_instance_no_inline_id || 212 stub_id == Runtime1::fast_new_instance_id || 213 stub_id == Runtime1::fast_new_instance_init_check_id, 214 "need new_instance id"); 215 _stub_id = stub_id; 216 } 217 218 void NewInstanceStub::emit_code(LIR_Assembler* ce) { 219 assert(__ rsp_offset() == 0, "frame size should be fixed"); 220 __ bind(_entry); 221 __ mov(r3, _klass_reg->as_register()); 222 __ far_call(RuntimeAddress(Runtime1::entry_for(_stub_id))); 223 ce->add_call_info_here(_info); 224 ce->verify_oop_map(_info); 225 assert(_result->as_register() == r0, "result must in r0,"); 226 __ b(_continuation); 227 } 228 229 230 // Implementation of NewTypeArrayStub 231 232 // Implementation of NewTypeArrayStub 233 234 NewTypeArrayStub::NewTypeArrayStub(LIR_Opr klass_reg, LIR_Opr length, LIR_Opr result, CodeEmitInfo* info) { 235 _klass_reg = klass_reg; 236 _length = length; 237 _result = result; 238 _info = new CodeEmitInfo(info); 239 } 240 241 242 void NewTypeArrayStub::emit_code(LIR_Assembler* ce) { 243 assert(__ rsp_offset() == 0, "frame size should be fixed"); 244 __ bind(_entry); 245 assert(_length->as_register() == r19, "length must in r19,"); 246 assert(_klass_reg->as_register() == r3, "klass_reg must in r3"); 247 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::new_type_array_id))); 248 ce->add_call_info_here(_info); 249 ce->verify_oop_map(_info); 250 assert(_result->as_register() == r0, "result must in r0"); 251 __ b(_continuation); 252 } 253 254 255 // Implementation of NewObjectArrayStub 256 257 NewObjectArrayStub::NewObjectArrayStub(LIR_Opr klass_reg, LIR_Opr length, LIR_Opr result, 258 CodeEmitInfo* info, bool is_null_free) { 259 _klass_reg = klass_reg; 260 _result = result; 261 _length = length; 262 _info = new CodeEmitInfo(info); 263 _is_null_free = is_null_free; 264 } 265 266 267 void NewObjectArrayStub::emit_code(LIR_Assembler* ce) { 268 assert(__ rsp_offset() == 0, "frame size should be fixed"); 269 __ bind(_entry); 270 assert(_length->as_register() == r19, "length must in r19,"); 271 assert(_klass_reg->as_register() == r3, "klass_reg must in r3"); 272 273 if (_is_null_free) { 274 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::new_flat_array_id))); 275 } else { 276 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::new_object_array_id))); 277 } 278 279 ce->add_call_info_here(_info); 280 ce->verify_oop_map(_info); 281 assert(_result->as_register() == r0, "result must in r0"); 282 __ b(_continuation); 283 } 284 // Implementation of MonitorAccessStubs 285 286 MonitorEnterStub::MonitorEnterStub(LIR_Opr obj_reg, LIR_Opr lock_reg, CodeEmitInfo* info, CodeStub* throw_imse_stub, LIR_Opr scratch_reg) 287 : MonitorAccessStub(obj_reg, lock_reg) 288 { 289 _info = new CodeEmitInfo(info); 290 _scratch_reg = scratch_reg; 291 _throw_imse_stub = throw_imse_stub; 292 if (_throw_imse_stub != NULL) { 293 assert(_scratch_reg != LIR_OprFact::illegalOpr, "must be"); 294 } 295 } 296 297 298 void MonitorEnterStub::emit_code(LIR_Assembler* ce) { 299 assert(__ rsp_offset() == 0, "frame size should be fixed"); 300 __ bind(_entry); 301 if (_throw_imse_stub != NULL) { 302 // When we come here, _obj_reg has already been checked to be non-null. 303 __ ldr(rscratch1, Address(_obj_reg->as_register(), oopDesc::mark_offset_in_bytes())); 304 __ mov(rscratch2, markWord::inline_type_pattern); 305 __ andr(rscratch1, rscratch1, rscratch2); 306 307 __ cmp(rscratch1, rscratch2); 308 __ br(Assembler::EQ, *_throw_imse_stub->entry()); 309 } 310 311 ce->store_parameter(_obj_reg->as_register(), 1); 312 ce->store_parameter(_lock_reg->as_register(), 0); 313 Runtime1::StubID enter_id; 314 if (ce->compilation()->has_fpu_code()) { 315 enter_id = Runtime1::monitorenter_id; 316 } else { 317 enter_id = Runtime1::monitorenter_nofpu_id; 318 } 319 __ far_call(RuntimeAddress(Runtime1::entry_for(enter_id))); 320 ce->add_call_info_here(_info); 321 ce->verify_oop_map(_info); 322 __ b(_continuation); 323 } 324 325 326 void MonitorExitStub::emit_code(LIR_Assembler* ce) { 327 __ bind(_entry); 328 if (_compute_lock) { 329 // lock_reg was destroyed by fast unlocking attempt => recompute it 330 ce->monitor_address(_monitor_ix, _lock_reg); 331 } 332 ce->store_parameter(_lock_reg->as_register(), 0); 333 // note: non-blocking leaf routine => no call info needed 334 Runtime1::StubID exit_id; 335 if (ce->compilation()->has_fpu_code()) { 336 exit_id = Runtime1::monitorexit_id; 337 } else { 338 exit_id = Runtime1::monitorexit_nofpu_id; 339 } 340 __ adr(lr, _continuation); 341 __ far_jump(RuntimeAddress(Runtime1::entry_for(exit_id))); 342 } 343 344 345 // Implementation of patching: 346 // - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes) 347 // - Replace original code with a call to the stub 348 // At Runtime: 349 // - call to stub, jump to runtime 350 // - in runtime: preserve all registers (rspecially objects, i.e., source and destination object) 351 // - in runtime: after initializing class, restore original code, reexecute instruction 352 353 int PatchingStub::_patch_info_offset = -NativeGeneralJump::instruction_size; 354 355 void PatchingStub::align_patch_site(MacroAssembler* masm) { 356 } 357 358 void PatchingStub::emit_code(LIR_Assembler* ce) { 359 assert(false, "AArch64 should not use C1 runtime patching"); 360 } 361 362 363 void DeoptimizeStub::emit_code(LIR_Assembler* ce) { 364 __ bind(_entry); 365 ce->store_parameter(_trap_request, 0); 366 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::deoptimize_id))); 367 ce->add_call_info_here(_info); 368 DEBUG_ONLY(__ should_not_reach_here()); 369 } 370 371 372 void ImplicitNullCheckStub::emit_code(LIR_Assembler* ce) { 373 address a; 374 if (_info->deoptimize_on_exception()) { 375 // Deoptimize, do not throw the exception, because it is probably wrong to do it here. 376 a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); 377 } else { 378 a = Runtime1::entry_for(Runtime1::throw_null_pointer_exception_id); 379 } 380 381 ce->compilation()->implicit_exception_table()->append(_offset, __ offset()); 382 __ bind(_entry); 383 __ far_call(RuntimeAddress(a)); 384 ce->add_call_info_here(_info); 385 ce->verify_oop_map(_info); 386 debug_only(__ should_not_reach_here()); 387 } 388 389 390 void SimpleExceptionStub::emit_code(LIR_Assembler* ce) { 391 assert(__ rsp_offset() == 0, "frame size should be fixed"); 392 393 __ bind(_entry); 394 // pass the object in a scratch register because all other registers 395 // must be preserved 396 if (_obj->is_cpu_register()) { 397 __ mov(rscratch1, _obj->as_register()); 398 } 399 __ far_call(RuntimeAddress(Runtime1::entry_for(_stub)), NULL, rscratch2); 400 ce->add_call_info_here(_info); 401 debug_only(__ should_not_reach_here()); 402 } 403 404 405 void ArrayCopyStub::emit_code(LIR_Assembler* ce) { 406 //---------------slow case: call to native----------------- 407 __ bind(_entry); 408 // Figure out where the args should go 409 // This should really convert the IntrinsicID to the Method* and signature 410 // but I don't know how to do that. 411 // 412 VMRegPair args[5]; 413 BasicType signature[5] = { T_OBJECT, T_INT, T_OBJECT, T_INT, T_INT}; 414 SharedRuntime::java_calling_convention(signature, args, 5); 415 416 // push parameters 417 // (src, src_pos, dest, destPos, length) 418 Register r[5]; 419 r[0] = src()->as_register(); 420 r[1] = src_pos()->as_register(); 421 r[2] = dst()->as_register(); 422 r[3] = dst_pos()->as_register(); 423 r[4] = length()->as_register(); 424 425 // next registers will get stored on the stack 426 for (int i = 0; i < 5 ; i++ ) { 427 VMReg r_1 = args[i].first(); 428 if (r_1->is_stack()) { 429 int st_off = r_1->reg2stack() * wordSize; 430 __ str (r[i], Address(sp, st_off)); 431 } else { 432 assert(r[i] == args[i].first()->as_Register(), "Wrong register for arg "); 433 } 434 } 435 436 ce->align_call(lir_static_call); 437 438 ce->emit_static_call_stub(); 439 if (ce->compilation()->bailed_out()) { 440 return; // CodeCache is full 441 } 442 Address resolve(SharedRuntime::get_resolve_static_call_stub(), 443 relocInfo::static_call_type); 444 address call = __ trampoline_call(resolve); 445 if (call == NULL) { 446 ce->bailout("trampoline stub overflow"); 447 return; 448 } 449 ce->add_call_info_here(info()); 450 451 #ifndef PRODUCT 452 if (PrintC1Statistics) { 453 __ lea(rscratch2, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt)); 454 __ incrementw(Address(rscratch2)); 455 } 456 #endif 457 458 __ b(_continuation); 459 } 460 461 #undef __