1 /*
2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classLoaderData.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "gc/shared/collectedHeap.hpp"
31 #include "interpreter/interp_masm.hpp"
32 #include "memory/universe.hpp"
33 #include "runtime/javaThread.hpp"
34 #include "runtime/jniHandles.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37
38 #define __ masm->
39
40 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
41 Register dst, Address src, Register tmp1, Register tmp_thread) {
42 bool in_heap = (decorators & IN_HEAP) != 0;
43 bool in_native = (decorators & IN_NATIVE) != 0;
44 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
45 bool atomic = (decorators & MO_RELAXED) != 0;
46
47 switch (type) {
48 case T_OBJECT:
49 case T_ARRAY: {
50 if (in_heap) {
51 #ifdef _LP64
52 if (UseCompressedOops) {
53 __ movl(dst, src);
54 if (is_not_null) {
55 __ decode_heap_oop_not_null(dst);
56 } else {
57 __ decode_heap_oop(dst);
58 }
59 } else
60 #endif
61 {
62 __ movptr(dst, src);
63 }
64 } else {
65 assert(in_native, "why else?");
66 __ movptr(dst, src);
67 }
68 break;
69 }
70 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break;
71 case T_BYTE: __ load_signed_byte(dst, src); break;
72 case T_CHAR: __ load_unsigned_short(dst, src); break;
73 case T_SHORT: __ load_signed_short(dst, src); break;
74 case T_INT: __ movl (dst, src); break;
75 case T_ADDRESS: __ movptr(dst, src); break;
76 case T_FLOAT:
77 assert(dst == noreg, "only to ftos");
78 __ load_float(src);
79 break;
80 case T_DOUBLE:
81 assert(dst == noreg, "only to dtos");
82 __ load_double(src);
83 break;
84 case T_LONG:
85 assert(dst == noreg, "only to ltos");
86 #ifdef _LP64
87 __ movq(rax, src);
88 #else
89 if (atomic) {
90 __ fild_d(src); // Must load atomically
91 __ subptr(rsp,2*wordSize); // Make space for store
92 __ fistp_d(Address(rsp,0));
93 __ pop(rax);
94 __ pop(rdx);
95 } else {
96 __ movl(rax, src);
97 __ movl(rdx, src.plus_disp(wordSize));
98 }
99 #endif
100 break;
101 default: Unimplemented();
102 }
103 }
104
105 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
106 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
107 bool in_heap = (decorators & IN_HEAP) != 0;
108 bool in_native = (decorators & IN_NATIVE) != 0;
109 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
110 bool atomic = (decorators & MO_RELAXED) != 0;
111
112 switch (type) {
113 case T_OBJECT:
114 case T_ARRAY: {
115 if (in_heap) {
116 if (val == noreg) {
117 assert(!is_not_null, "inconsistent access");
118 #ifdef _LP64
119 if (UseCompressedOops) {
120 __ movl(dst, NULL_WORD);
121 } else {
122 __ movslq(dst, NULL_WORD);
123 }
124 #else
125 __ movl(dst, NULL_WORD);
126 #endif
127 } else {
128 #ifdef _LP64
129 if (UseCompressedOops) {
130 assert(!dst.uses(val), "not enough registers");
131 if (is_not_null) {
132 __ encode_heap_oop_not_null(val);
133 } else {
134 __ encode_heap_oop(val);
135 }
136 __ movl(dst, val);
137 } else
138 #endif
139 {
140 __ movptr(dst, val);
141 }
142 }
143 } else {
144 assert(in_native, "why else?");
145 assert(val != noreg, "not supported");
146 __ movptr(dst, val);
147 }
148 break;
149 }
150 case T_BOOLEAN:
151 __ andl(val, 0x1); // boolean is true if LSB is 1
152 __ movb(dst, val);
153 break;
154 case T_BYTE:
155 __ movb(dst, val);
156 break;
157 case T_SHORT:
158 __ movw(dst, val);
159 break;
160 case T_CHAR:
161 __ movw(dst, val);
162 break;
163 case T_INT:
164 __ movl(dst, val);
165 break;
166 case T_LONG:
167 assert(val == noreg, "only tos");
168 #ifdef _LP64
169 __ movq(dst, rax);
170 #else
171 if (atomic) {
172 __ push(rdx);
173 __ push(rax); // Must update atomically with FIST
174 __ fild_d(Address(rsp,0)); // So load into FPU register
175 __ fistp_d(dst); // and put into memory atomically
176 __ addptr(rsp, 2*wordSize);
177 } else {
178 __ movptr(dst, rax);
179 __ movptr(dst.plus_disp(wordSize), rdx);
180 }
181 #endif
182 break;
183 case T_FLOAT:
184 assert(val == noreg, "only tos");
185 __ store_float(dst);
186 break;
187 case T_DOUBLE:
188 assert(val == noreg, "only tos");
189 __ store_double(dst);
190 break;
191 case T_ADDRESS:
192 __ movptr(dst, val);
193 break;
194 default: Unimplemented();
195 }
196 }
197
198 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
199 DecoratorSet decorators,
200 BasicType type,
201 size_t bytes,
202 Register dst,
203 Address src,
204 Register tmp) {
205 assert(bytes <= 8, "can only deal with non-vector registers");
206 switch (bytes) {
207 case 1:
208 __ movb(dst, src);
209 break;
210 case 2:
211 __ movw(dst, src);
212 break;
213 case 4:
214 __ movl(dst, src);
215 break;
216 case 8:
217 #ifdef _LP64
218 __ movq(dst, src);
219 #else
220 fatal("No support for 8 bytes copy");
221 #endif
222 break;
223 default:
224 fatal("Unexpected size");
225 }
226 #ifdef _LP64
227 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
228 __ decode_heap_oop(dst);
229 }
230 #endif
231 }
232
233 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
234 DecoratorSet decorators,
235 BasicType type,
236 size_t bytes,
237 Address dst,
238 Register src,
239 Register tmp) {
240 #ifdef _LP64
241 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
242 __ encode_heap_oop(src);
243 }
244 #endif
245 assert(bytes <= 8, "can only deal with non-vector registers");
246 switch (bytes) {
247 case 1:
248 __ movb(dst, src);
249 break;
250 case 2:
251 __ movw(dst, src);
252 break;
253 case 4:
254 __ movl(dst, src);
255 break;
256 case 8:
257 #ifdef _LP64
258 __ movq(dst, src);
259 #else
260 fatal("No support for 8 bytes copy");
261 #endif
262 break;
263 default:
264 fatal("Unexpected size");
265 }
266 }
267
268 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
269 DecoratorSet decorators,
270 BasicType type,
271 size_t bytes,
272 XMMRegister dst,
273 Address src,
274 Register tmp,
275 XMMRegister xmm_tmp) {
276 assert(bytes > 8, "can only deal with vector registers");
277 if (bytes == 16) {
278 __ movdqu(dst, src);
279 } else if (bytes == 32) {
280 __ vmovdqu(dst, src);
281 } else {
282 fatal("No support for >32 bytes copy");
283 }
284 }
285
286 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
287 DecoratorSet decorators,
288 BasicType type,
289 size_t bytes,
290 Address dst,
291 XMMRegister src,
292 Register tmp1,
293 Register tmp2,
294 XMMRegister xmm_tmp) {
295 assert(bytes > 8, "can only deal with vector registers");
296 if (bytes == 16) {
297 __ movdqu(dst, src);
298 } else if (bytes == 32) {
299 __ vmovdqu(dst, src);
300 } else {
301 fatal("No support for >32 bytes copy");
302 }
303 }
304
305 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
306 Register obj, Register tmp, Label& slowpath) {
307 __ clear_jobject_tag(obj);
308 __ movptr(obj, Address(obj, 0));
309 }
310
311 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
312 Register thread, Register obj,
313 Register var_size_in_bytes,
314 int con_size_in_bytes,
315 Register t1,
316 Register t2,
317 Label& slow_case) {
318 assert_different_registers(obj, t1, t2);
319 assert_different_registers(obj, var_size_in_bytes, t1);
320 Register end = t2;
321 if (!thread->is_valid()) {
322 #ifdef _LP64
323 thread = r15_thread;
324 #else
325 assert(t1->is_valid(), "need temp reg");
326 thread = t1;
327 __ get_thread(thread);
328 #endif
329 }
330
331 __ verify_tlab();
332
333 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
334 if (var_size_in_bytes == noreg) {
335 __ lea(end, Address(obj, con_size_in_bytes));
336 } else {
337 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
338 }
339 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
340 __ jcc(Assembler::above, slow_case);
341
342 // update the tlab top pointer
343 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
344
345 // recover var_size_in_bytes if necessary
346 if (var_size_in_bytes == end) {
347 __ subptr(var_size_in_bytes, obj);
348 }
349 __ verify_tlab();
350 }
351
352 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
353 Register var_size_in_bytes,
354 int con_size_in_bytes,
355 Register t1) {
356 if (!thread->is_valid()) {
357 #ifdef _LP64
358 thread = r15_thread;
359 #else
360 assert(t1->is_valid(), "need temp reg");
361 thread = t1;
362 __ get_thread(thread);
363 #endif
364 }
365
366 #ifdef _LP64
367 if (var_size_in_bytes->is_valid()) {
368 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
369 } else {
370 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
371 }
372 #else
373 if (var_size_in_bytes->is_valid()) {
374 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
375 } else {
376 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
377 }
378 __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
379 #endif
380 }
381
382 #ifdef _LP64
383 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
384 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
385 if (bs_nm == nullptr) {
386 return;
387 }
388 Register thread = r15_thread;
389 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
390 // The immediate is the last 4 bytes, so if we align the start of the cmp
391 // instruction to 4 bytes, we know that the second half of it is also 4
392 // byte aligned, which means that the immediate will not cross a cache line
393 __ align(4);
394 uintptr_t before_cmp = (uintptr_t)__ pc();
395 __ cmpl_imm32(disarmed_addr, 0);
396 uintptr_t after_cmp = (uintptr_t)__ pc();
397 guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
398
399 if (slow_path != nullptr) {
400 __ jcc(Assembler::notEqual, *slow_path);
401 __ bind(*continuation);
402 } else {
403 Label done;
404 __ jccb(Assembler::equal, done);
405 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
406 __ bind(done);
407 }
408 }
409 #else
410 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label*, Label*) {
411 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
412 if (bs_nm == nullptr) {
413 return;
414 }
415
416 Label continuation;
417
418 Register tmp = rdi;
419 __ push(tmp);
420 __ movptr(tmp, (intptr_t)bs_nm->disarmed_guard_value_address());
421 Address disarmed_addr(tmp, 0);
422 __ align(4);
423 __ cmpl_imm32(disarmed_addr, 0);
424 __ pop(tmp);
425 __ jcc(Assembler::equal, continuation);
426 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
427 __ bind(continuation);
428 }
429 #endif
430
431 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
432 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
433 if (bs == nullptr) {
434 return;
435 }
436
437 Label bad_call;
438 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
439 __ jcc(Assembler::equal, bad_call);
440
441 Register tmp1 = LP64_ONLY( rscratch1 ) NOT_LP64( rax );
442 Register tmp2 = LP64_ONLY( rscratch2 ) NOT_LP64( rcx );
443 #ifndef _LP64
444 __ push(tmp1);
445 __ push(tmp2);
446 #endif // !_LP64
447
448 // Pointer chase to the method holder to find out if the method is concurrently unloading.
449 Label method_live;
450 __ load_method_holder_cld(tmp1, rbx);
451
452 // Is it a strong CLD?
453 __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_offset()), 0);
454 __ jcc(Assembler::greater, method_live);
455
456 // Is it a weak but alive CLD?
457 __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
458 __ resolve_weak_handle(tmp1, tmp2);
459 __ cmpptr(tmp1, 0);
460 __ jcc(Assembler::notEqual, method_live);
461
462 #ifndef _LP64
463 __ pop(tmp2);
464 __ pop(tmp1);
465 #endif
466
467 __ bind(bad_call);
468 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
469 __ bind(method_live);
470
471 #ifndef _LP64
472 __ pop(tmp2);
473 __ pop(tmp1);
474 #endif
475 }
476
477 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
478 // Check if the oop is in the right area of memory
479 __ movptr(tmp1, obj);
480 __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
481 __ andptr(tmp1, tmp2);
482 __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
483 __ cmpptr(tmp1, tmp2);
484 __ jcc(Assembler::notZero, error);
485
486 // make sure klass is 'reasonable', which is not zero.
487 __ load_klass(obj, obj, tmp1); // get klass
488 __ testptr(obj, obj);
489 __ jcc(Assembler::zero, error); // if klass is null it is broken
490 }