1 /* 2 * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/classLoaderData.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/barrierSetAssembler.hpp" 29 #include "gc/shared/barrierSetNMethod.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "interpreter/interp_masm.hpp" 32 #include "memory/universe.hpp" 33 #include "runtime/jniHandles.hpp" 34 #include "runtime/sharedRuntime.hpp" 35 #include "runtime/stubRoutines.hpp" 36 #include "runtime/thread.hpp" 37 38 #define __ masm-> 39 40 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 41 Register dst, Address src, Register tmp1, Register tmp_thread) { 42 bool in_heap = (decorators & IN_HEAP) != 0; 43 bool in_native = (decorators & IN_NATIVE) != 0; 44 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 45 bool atomic = (decorators & MO_RELAXED) != 0; 46 47 switch (type) { 48 case T_OBJECT: 49 case T_ARRAY: { 50 if (in_heap) { 51 #ifdef _LP64 52 if (UseCompressedOops) { 53 __ movl(dst, src); 54 if (is_not_null) { 55 __ decode_heap_oop_not_null(dst); 56 } else { 57 __ decode_heap_oop(dst); 58 } 59 } else 60 #endif 61 { 62 __ movptr(dst, src); 63 } 64 } else { 65 assert(in_native, "why else?"); 66 __ movptr(dst, src); 67 } 68 break; 69 } 70 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break; 71 case T_BYTE: __ load_signed_byte(dst, src); break; 72 case T_CHAR: __ load_unsigned_short(dst, src); break; 73 case T_SHORT: __ load_signed_short(dst, src); break; 74 case T_INT: __ movl (dst, src); break; 75 case T_ADDRESS: __ movptr(dst, src); break; 76 case T_FLOAT: 77 assert(dst == noreg, "only to ftos"); 78 __ load_float(src); 79 break; 80 case T_DOUBLE: 81 assert(dst == noreg, "only to dtos"); 82 __ load_double(src); 83 break; 84 case T_LONG: 85 assert(dst == noreg, "only to ltos"); 86 #ifdef _LP64 87 __ movq(rax, src); 88 #else 89 if (atomic) { 90 __ fild_d(src); // Must load atomically 91 __ subptr(rsp,2*wordSize); // Make space for store 92 __ fistp_d(Address(rsp,0)); 93 __ pop(rax); 94 __ pop(rdx); 95 } else { 96 __ movl(rax, src); 97 __ movl(rdx, src.plus_disp(wordSize)); 98 } 99 #endif 100 break; 101 default: Unimplemented(); 102 } 103 } 104 105 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 106 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 107 bool in_heap = (decorators & IN_HEAP) != 0; 108 bool in_native = (decorators & IN_NATIVE) != 0; 109 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 110 bool atomic = (decorators & MO_RELAXED) != 0; 111 112 switch (type) { 113 case T_OBJECT: 114 case T_ARRAY: { 115 if (in_heap) { 116 if (val == noreg) { 117 assert(!is_not_null, "inconsistent access"); 118 #ifdef _LP64 119 if (UseCompressedOops) { 120 __ movl(dst, (int32_t)NULL_WORD); 121 } else { 122 __ movslq(dst, (int32_t)NULL_WORD); 123 } 124 #else 125 __ movl(dst, (int32_t)NULL_WORD); 126 #endif 127 } else { 128 #ifdef _LP64 129 if (UseCompressedOops) { 130 assert(!dst.uses(val), "not enough registers"); 131 if (is_not_null) { 132 __ encode_heap_oop_not_null(val); 133 } else { 134 __ encode_heap_oop(val); 135 } 136 __ movl(dst, val); 137 } else 138 #endif 139 { 140 __ movptr(dst, val); 141 } 142 } 143 } else { 144 assert(in_native, "why else?"); 145 assert(val != noreg, "not supported"); 146 __ movptr(dst, val); 147 } 148 break; 149 } 150 case T_BOOLEAN: 151 __ andl(val, 0x1); // boolean is true if LSB is 1 152 __ movb(dst, val); 153 break; 154 case T_BYTE: 155 __ movb(dst, val); 156 break; 157 case T_SHORT: 158 __ movw(dst, val); 159 break; 160 case T_CHAR: 161 __ movw(dst, val); 162 break; 163 case T_INT: 164 __ movl(dst, val); 165 break; 166 case T_LONG: 167 assert(val == noreg, "only tos"); 168 #ifdef _LP64 169 __ movq(dst, rax); 170 #else 171 if (atomic) { 172 __ push(rdx); 173 __ push(rax); // Must update atomically with FIST 174 __ fild_d(Address(rsp,0)); // So load into FPU register 175 __ fistp_d(dst); // and put into memory atomically 176 __ addptr(rsp, 2*wordSize); 177 } else { 178 __ movptr(dst, rax); 179 __ movptr(dst.plus_disp(wordSize), rdx); 180 } 181 #endif 182 break; 183 case T_FLOAT: 184 assert(val == noreg, "only tos"); 185 __ store_float(dst); 186 break; 187 case T_DOUBLE: 188 assert(val == noreg, "only tos"); 189 __ store_double(dst); 190 break; 191 case T_ADDRESS: 192 __ movptr(dst, val); 193 break; 194 default: Unimplemented(); 195 } 196 } 197 198 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 199 Register obj, Register tmp, Label& slowpath) { 200 __ clear_jweak_tag(obj); 201 __ movptr(obj, Address(obj, 0)); 202 } 203 204 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, 205 Register thread, Register obj, 206 Register var_size_in_bytes, 207 int con_size_in_bytes, 208 Register t1, 209 Register t2, 210 Label& slow_case) { 211 assert_different_registers(obj, t1, t2); 212 assert_different_registers(obj, var_size_in_bytes, t1); 213 Register end = t2; 214 if (!thread->is_valid()) { 215 #ifdef _LP64 216 thread = r15_thread; 217 #else 218 assert(t1->is_valid(), "need temp reg"); 219 thread = t1; 220 __ get_thread(thread); 221 #endif 222 } 223 224 __ verify_tlab(); 225 226 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); 227 if (var_size_in_bytes == noreg) { 228 __ lea(end, Address(obj, con_size_in_bytes)); 229 } else { 230 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 231 } 232 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); 233 __ jcc(Assembler::above, slow_case); 234 235 // update the tlab top pointer 236 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); 237 238 // recover var_size_in_bytes if necessary 239 if (var_size_in_bytes == end) { 240 __ subptr(var_size_in_bytes, obj); 241 } 242 __ verify_tlab(); 243 } 244 245 // Defines obj, preserves var_size_in_bytes 246 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, 247 Register thread, Register obj, 248 Register var_size_in_bytes, 249 int con_size_in_bytes, 250 Register t1, 251 Label& slow_case) { 252 assert(obj == rax, "obj must be in rax, for cmpxchg"); 253 assert_different_registers(obj, var_size_in_bytes, t1); 254 if (!Universe::heap()->supports_inline_contig_alloc()) { 255 __ jmp(slow_case); 256 } else { 257 Register end = t1; 258 Label retry; 259 __ bind(retry); 260 ExternalAddress heap_top((address) Universe::heap()->top_addr()); 261 __ movptr(obj, heap_top); 262 if (var_size_in_bytes == noreg) { 263 __ lea(end, Address(obj, con_size_in_bytes)); 264 } else { 265 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 266 } 267 // if end < obj then we wrapped around => object too long => slow case 268 __ cmpptr(end, obj); 269 __ jcc(Assembler::below, slow_case); 270 __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr())); 271 __ jcc(Assembler::above, slow_case); 272 // Compare obj with the top addr, and if still equal, store the new top addr in 273 // end at the address of the top addr pointer. Sets ZF if was equal, and clears 274 // it otherwise. Use lock prefix for atomicity on MPs. 275 __ locked_cmpxchgptr(end, heap_top); 276 __ jcc(Assembler::notEqual, retry); 277 incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1); 278 } 279 } 280 281 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread, 282 Register var_size_in_bytes, 283 int con_size_in_bytes, 284 Register t1) { 285 if (!thread->is_valid()) { 286 #ifdef _LP64 287 thread = r15_thread; 288 #else 289 assert(t1->is_valid(), "need temp reg"); 290 thread = t1; 291 __ get_thread(thread); 292 #endif 293 } 294 295 #ifdef _LP64 296 if (var_size_in_bytes->is_valid()) { 297 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 298 } else { 299 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 300 } 301 #else 302 if (var_size_in_bytes->is_valid()) { 303 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 304 } else { 305 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 306 } 307 __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); 308 #endif 309 } 310 311 #ifdef _LP64 312 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { 313 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 314 if (bs_nm == NULL) { 315 return; 316 } 317 Label continuation; 318 Register thread = r15_thread; 319 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset())); 320 __ align(8); 321 __ cmpl(disarmed_addr, 0); 322 __ jcc(Assembler::equal, continuation); 323 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); 324 __ bind(continuation); 325 } 326 #else 327 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { 328 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 329 if (bs_nm == NULL) { 330 return; 331 } 332 333 Label continuation; 334 335 Register tmp = rdi; 336 __ push(tmp); 337 __ movptr(tmp, (intptr_t)bs_nm->disarmed_value_address()); 338 Address disarmed_addr(tmp, 0); 339 __ align(4); 340 __ cmpl(disarmed_addr, 0); 341 __ pop(tmp); 342 __ jcc(Assembler::equal, continuation); 343 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); 344 __ bind(continuation); 345 } 346 #endif 347 348 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { 349 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod(); 350 if (bs == NULL) { 351 return; 352 } 353 354 Label bad_call; 355 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters. 356 __ jcc(Assembler::equal, bad_call); 357 358 #ifdef _LP64 359 Register tmp1 = rscratch1; 360 Register tmp2 = rscratch2; 361 #else 362 Register tmp1 = rax; 363 Register tmp2 = rcx; 364 __ push(tmp1); 365 __ push(tmp2); 366 #endif // _LP64 367 368 // Pointer chase to the method holder to find out if the method is concurrently unloading. 369 Label method_live; 370 __ load_method_holder_cld(tmp1, rbx); 371 372 // Is it a strong CLD? 373 __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_offset()), 0); 374 __ jcc(Assembler::greater, method_live); 375 376 // Is it a weak but alive CLD? 377 __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset())); 378 __ resolve_weak_handle(tmp1, tmp2); 379 __ cmpptr(tmp1, 0); 380 __ jcc(Assembler::notEqual, method_live); 381 382 #ifndef _LP64 383 __ pop(tmp2); 384 __ pop(tmp1); 385 #endif 386 387 __ bind(bad_call); 388 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 389 __ bind(method_live); 390 391 #ifndef _LP64 392 __ pop(tmp2); 393 __ pop(tmp1); 394 #endif 395 }