1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/classLoaderData.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/barrierSetAssembler.hpp"
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "interpreter/interp_masm.hpp"
31 #include "memory/universe.hpp"
32 #include "runtime/javaThread.hpp"
33 #include "runtime/jniHandles.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #ifdef COMPILER2
37 #include "gc/shared/c2/barrierSetC2.hpp"
38 #endif // COMPILER2
39
40 #define __ masm->
41
42 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
43 Register dst, Address src, Register tmp1) {
44 bool in_heap = (decorators & IN_HEAP) != 0;
45 bool in_native = (decorators & IN_NATIVE) != 0;
46 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
47 bool atomic = (decorators & MO_RELAXED) != 0;
48
49 switch (type) {
50 case T_OBJECT:
51 case T_ARRAY: {
52 if (in_heap) {
53 if (UseCompressedOops) {
54 __ movl(dst, src);
55 if (is_not_null) {
56 __ decode_heap_oop_not_null(dst);
57 } else {
58 __ decode_heap_oop(dst);
59 }
60 } else {
61 __ movptr(dst, src);
62 }
63 } else {
64 assert(in_native, "why else?");
65 __ movptr(dst, src);
66 }
67 break;
68 }
69 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break;
70 case T_BYTE: __ load_signed_byte(dst, src); break;
71 case T_CHAR: __ load_unsigned_short(dst, src); break;
72 case T_SHORT: __ load_signed_short(dst, src); break;
73 case T_INT: __ movl (dst, src); break;
74 case T_ADDRESS: __ movptr(dst, src); break;
75 case T_FLOAT:
76 assert(dst == noreg, "only to ftos");
77 __ movflt(xmm0, src);
78 break;
79 case T_DOUBLE:
80 assert(dst == noreg, "only to dtos");
81 __ movdbl(xmm0, src);
82 break;
83 case T_LONG:
84 assert(dst == noreg, "only to ltos");
85 __ movq(rax, src);
86 break;
87 default: Unimplemented();
88 }
89 }
90
91 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
92 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
93 bool in_heap = (decorators & IN_HEAP) != 0;
94 bool in_native = (decorators & IN_NATIVE) != 0;
95 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
96 bool atomic = (decorators & MO_RELAXED) != 0;
97
98 switch (type) {
99 case T_OBJECT:
100 case T_ARRAY: {
101 if (in_heap) {
102 if (val == noreg) {
103 assert(!is_not_null, "inconsistent access");
104 if (UseCompressedOops) {
105 __ movl(dst, NULL_WORD);
106 } else {
107 __ movslq(dst, NULL_WORD);
108 }
109 } else {
110 if (UseCompressedOops) {
111 assert(!dst.uses(val), "not enough registers");
112 if (is_not_null) {
113 __ encode_heap_oop_not_null(val);
114 } else {
115 __ encode_heap_oop(val);
116 }
117 __ movl(dst, val);
118 } else {
119 __ movptr(dst, val);
120 }
121 }
122 } else {
123 assert(in_native, "why else?");
124 assert(val != noreg, "not supported");
125 __ movptr(dst, val);
126 }
127 break;
128 }
129 case T_BOOLEAN:
130 __ andl(val, 0x1); // boolean is true if LSB is 1
131 __ movb(dst, val);
132 break;
133 case T_BYTE:
134 __ movb(dst, val);
135 break;
136 case T_SHORT:
137 __ movw(dst, val);
138 break;
139 case T_CHAR:
140 __ movw(dst, val);
141 break;
142 case T_INT:
143 __ movl(dst, val);
144 break;
145 case T_LONG:
146 assert(val == noreg, "only tos");
147 __ movq(dst, rax);
148 break;
149 case T_FLOAT:
150 assert(val == noreg, "only tos");
151 __ movflt(dst, xmm0);
152 break;
153 case T_DOUBLE:
154 assert(val == noreg, "only tos");
155 __ movdbl(dst, xmm0);
156 break;
157 case T_ADDRESS:
158 __ movptr(dst, val);
159 break;
160 default: Unimplemented();
161 }
162 }
163
164 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
165 DecoratorSet decorators,
166 BasicType type,
167 size_t bytes,
168 Register dst,
169 Address src,
170 Register tmp) {
171 assert(bytes <= 8, "can only deal with non-vector registers");
172 switch (bytes) {
173 case 1:
174 __ movb(dst, src);
175 break;
176 case 2:
177 __ movw(dst, src);
178 break;
179 case 4:
180 __ movl(dst, src);
181 break;
182 case 8:
183 __ movq(dst, src);
184 break;
185 default:
186 fatal("Unexpected size");
187 }
188 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
189 __ decode_heap_oop(dst);
190 }
191 }
192
193 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
194 DecoratorSet decorators,
195 BasicType type,
196 size_t bytes,
197 Address dst,
198 Register src,
199 Register tmp) {
200 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
201 __ encode_heap_oop(src);
202 }
203 assert(bytes <= 8, "can only deal with non-vector registers");
204 switch (bytes) {
205 case 1:
206 __ movb(dst, src);
207 break;
208 case 2:
209 __ movw(dst, src);
210 break;
211 case 4:
212 __ movl(dst, src);
213 break;
214 case 8:
215 __ movq(dst, src);
216 break;
217 default:
218 fatal("Unexpected size");
219 }
220 }
221
222 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
223 DecoratorSet decorators,
224 BasicType type,
225 size_t bytes,
226 XMMRegister dst,
227 Address src,
228 Register tmp,
229 XMMRegister xmm_tmp) {
230 assert(bytes > 8, "can only deal with vector registers");
231 if (bytes == 16) {
232 __ movdqu(dst, src);
233 } else if (bytes == 32) {
234 __ vmovdqu(dst, src);
235 } else {
236 fatal("No support for >32 bytes copy");
237 }
238 }
239
240 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
241 DecoratorSet decorators,
242 BasicType type,
243 size_t bytes,
244 Address dst,
245 XMMRegister src,
246 Register tmp1,
247 Register tmp2,
248 XMMRegister xmm_tmp) {
249 assert(bytes > 8, "can only deal with vector registers");
250 if (bytes == 16) {
251 __ movdqu(dst, src);
252 } else if (bytes == 32) {
253 __ vmovdqu(dst, src);
254 } else {
255 fatal("No support for >32 bytes copy");
256 }
257 }
258
259 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
260 Register obj, Register tmp, Label& slowpath) {
261 __ clear_jobject_tag(obj);
262 __ movptr(obj, Address(obj, 0));
263 }
264
265 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
266 Register obj,
267 Register var_size_in_bytes,
268 int con_size_in_bytes,
269 Register t1,
270 Register t2,
271 Label& slow_case) {
272 assert_different_registers(obj, t1, t2);
273 assert_different_registers(obj, var_size_in_bytes, t1);
274 Register end = t2;
275
276 const Register thread = r15_thread;
277
278 __ verify_tlab();
279
280 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
281 if (var_size_in_bytes == noreg) {
282 __ lea(end, Address(obj, con_size_in_bytes));
283 } else {
284 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
285 }
286 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
287 __ jcc(Assembler::above, slow_case);
288
289 // update the tlab top pointer
290 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
291
292 // recover var_size_in_bytes if necessary
293 if (var_size_in_bytes == end) {
294 __ subptr(var_size_in_bytes, obj);
295 }
296 __ verify_tlab();
297 }
298
299 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
300 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
301 Register thread = r15_thread;
302 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
303 // The immediate is the last 4 bytes, so if we align the start of the cmp
304 // instruction to 4 bytes, we know that the second half of it is also 4
305 // byte aligned, which means that the immediate will not cross a cache line
306 __ align(4);
307 uintptr_t before_cmp = (uintptr_t)__ pc();
308 __ cmpl_imm32(disarmed_addr, 0);
309 uintptr_t after_cmp = (uintptr_t)__ pc();
310 guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
311
312 if (slow_path != nullptr) {
313 __ jcc(Assembler::notEqual, *slow_path);
314 __ bind(*continuation);
315 } else {
316 Label done;
317 __ jccb(Assembler::equal, done);
318 __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
319 __ bind(done);
320 }
321 }
322
323 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
324 Label bad_call;
325 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
326 __ jcc(Assembler::equal, bad_call);
327
328 Register tmp1 = rscratch1;
329 Register tmp2 = rscratch2;
330
331 // Pointer chase to the method holder to find out if the method is concurrently unloading.
332 Label method_live;
333 __ load_method_holder_cld(tmp1, rbx);
334
335 // Is it a strong CLD?
336 __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_ref_count_offset()), 0);
337 __ jcc(Assembler::greater, method_live);
338
339 // Is it a weak but alive CLD?
340 __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
341 __ resolve_weak_handle(tmp1, tmp2);
342 __ cmpptr(tmp1, 0);
343 __ jcc(Assembler::notEqual, method_live);
344
345 __ bind(bad_call);
346 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
347 __ bind(method_live);
348 }
349
350 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
351 // Check if the oop is in the right area of memory
352 __ movptr(tmp1, obj);
353 __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
354 __ andptr(tmp1, tmp2);
355 __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
356 __ cmpptr(tmp1, tmp2);
357 __ jcc(Assembler::notZero, error);
358
359 // make sure klass is 'reasonable', which is not zero.
360 __ load_klass(obj, obj, tmp1); // get klass
361 __ testptr(obj, obj);
362 __ jcc(Assembler::zero, error); // if klass is null it is broken
363 }
364
365 #ifdef COMPILER2
366
367 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
368 if (!OptoReg::is_reg(opto_reg)) {
369 return OptoReg::Bad;
370 }
371
372 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
373 if (vm_reg->is_XMMRegister()) {
374 opto_reg &= ~15;
375 switch (node->ideal_reg()) {
376 case Op_VecX:
377 opto_reg |= 2;
378 break;
379 case Op_VecY:
380 opto_reg |= 4;
381 break;
382 case Op_VecZ:
383 opto_reg |= 8;
384 break;
385 default:
386 opto_reg |= 1;
387 break;
388 }
389 }
390
391 return opto_reg;
392 }
393
394 // We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
395 extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
396 int stack_offset, int reg, uint ireg, outputStream* st);
397
398 #undef __
399 #define __ _masm->
400
401 int SaveLiveRegisters::xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
402 if (left->_size == right->_size) {
403 return 0;
404 }
405
406 return (left->_size < right->_size) ? -1 : 1;
407 }
408
409 int SaveLiveRegisters::xmm_slot_size(OptoReg::Name opto_reg) {
410 // The low order 4 bytes denote what size of the XMM register is live
411 return (opto_reg & 15) << 3;
412 }
413
414 uint SaveLiveRegisters::xmm_ideal_reg_for_size(int reg_size) {
415 switch (reg_size) {
416 case 8:
417 return Op_VecD;
418 case 16:
419 return Op_VecX;
420 case 32:
421 return Op_VecY;
422 case 64:
423 return Op_VecZ;
424 default:
425 fatal("Invalid register size %d", reg_size);
426 return 0;
427 }
428 }
429
430 bool SaveLiveRegisters::xmm_needs_vzeroupper() const {
431 return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
432 }
433
434 void SaveLiveRegisters::xmm_register_save(const XMMRegisterData& reg_data) {
435 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
436 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
437 _spill_offset -= reg_data._size;
438 C2_MacroAssembler c2_masm(__ code());
439 vec_spill_helper(&c2_masm, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
440 }
441
442 void SaveLiveRegisters::xmm_register_restore(const XMMRegisterData& reg_data) {
443 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
444 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
445 C2_MacroAssembler c2_masm(__ code());
446 vec_spill_helper(&c2_masm, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
447 _spill_offset += reg_data._size;
448 }
449
450 void SaveLiveRegisters::gp_register_save(Register reg) {
451 _spill_offset -= 8;
452 __ movq(Address(rsp, _spill_offset), reg);
453 }
454
455 void SaveLiveRegisters::opmask_register_save(KRegister reg) {
456 _spill_offset -= 8;
457 __ kmov(Address(rsp, _spill_offset), reg);
458 }
459
460 void SaveLiveRegisters::gp_register_restore(Register reg) {
461 __ movq(reg, Address(rsp, _spill_offset));
462 _spill_offset += 8;
463 }
464
465 void SaveLiveRegisters::opmask_register_restore(KRegister reg) {
466 __ kmov(reg, Address(rsp, _spill_offset));
467 _spill_offset += 8;
468 }
469
470 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
471 // Create mask of caller saved registers that need to
472 // be saved/restored if live
473 RegMask caller_saved;
474 caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
475 caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
476 caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
477 caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
478 caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
479 caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
480 caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
481 caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
482 caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
483
484 if (UseAPX) {
485 caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
486 caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
487 caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
488 caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
489 caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
490 caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
491 caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
492 caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
493 caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
494 caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
495 caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
496 caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
497 caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
498 caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
499 caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
500 caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
501 }
502
503 int gp_spill_size = 0;
504 int opmask_spill_size = 0;
505 int xmm_spill_size = 0;
506
507 // Record registers that needs to be saved/restored
508 RegMaskIterator rmi(stub->preserve_set());
509 while (rmi.has_next()) {
510 const OptoReg::Name opto_reg = rmi.next();
511 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
512
513 if (vm_reg->is_Register()) {
514 if (caller_saved.Member(opto_reg)) {
515 _gp_registers.append(vm_reg->as_Register());
516 gp_spill_size += 8;
517 }
518 } else if (vm_reg->is_KRegister()) {
519 // All opmask registers are caller saved, thus spill the ones
520 // which are live.
521 if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
522 _opmask_registers.append(vm_reg->as_KRegister());
523 opmask_spill_size += 8;
524 }
525 } else if (vm_reg->is_XMMRegister()) {
526 // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
527 const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
528 const int reg_size = xmm_slot_size(opto_reg);
529 const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
530 const int reg_index = _xmm_registers.find(reg_data);
531 if (reg_index == -1) {
532 // Not previously appended
533 _xmm_registers.append(reg_data);
534 xmm_spill_size += reg_size;
535 } else {
536 // Previously appended, update size
537 const int reg_size_prev = _xmm_registers.at(reg_index)._size;
538 if (reg_size > reg_size_prev) {
539 _xmm_registers.at_put(reg_index, reg_data);
540 xmm_spill_size += reg_size - reg_size_prev;
541 }
542 }
543 } else {
544 fatal("Unexpected register type");
545 }
546 }
547
548 // Sort by size, largest first
549 _xmm_registers.sort(xmm_compare_register_size);
550
551 // On Windows, the caller reserves stack space for spilling register arguments
552 const int arg_spill_size = frame::arg_reg_save_area_bytes;
553
554 // Stack pointer must be 16 bytes aligned for the call
555 _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
556 }
557
558 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
559 : _masm(masm),
560 _gp_registers(),
561 _opmask_registers(),
562 _xmm_registers(),
563 _spill_size(0),
564 _spill_offset(0) {
565
566 //
567 // Stack layout after registers have been spilled:
568 //
569 // | ... | original rsp, 16 bytes aligned
570 // ------------------
571 // | zmm0 high |
572 // | ... |
573 // | zmm0 low | 16 bytes aligned
574 // | ... |
575 // | ymm1 high |
576 // | ... |
577 // | ymm1 low | 16 bytes aligned
578 // | ... |
579 // | xmmN high |
580 // | ... |
581 // | xmmN low | 8 bytes aligned
582 // | reg0 | 8 bytes aligned
583 // | reg1 |
584 // | ... |
585 // | regN | new rsp, if 16 bytes aligned
586 // | <padding> | else new rsp, 16 bytes aligned
587 // ------------------
588 //
589
590 // Figure out what registers to save/restore
591 initialize(stub);
592
593 // Allocate stack space
594 if (_spill_size > 0) {
595 __ subptr(rsp, _spill_size);
596 }
597
598 // Save XMM/YMM/ZMM registers
599 for (int i = 0; i < _xmm_registers.length(); i++) {
600 xmm_register_save(_xmm_registers.at(i));
601 }
602
603 if (xmm_needs_vzeroupper()) {
604 __ vzeroupper();
605 }
606
607 // Save general purpose registers
608 for (int i = 0; i < _gp_registers.length(); i++) {
609 gp_register_save(_gp_registers.at(i));
610 }
611
612 // Save opmask registers
613 for (int i = 0; i < _opmask_registers.length(); i++) {
614 opmask_register_save(_opmask_registers.at(i));
615 }
616 }
617
618 SaveLiveRegisters::~SaveLiveRegisters() {
619 // Restore opmask registers
620 for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
621 opmask_register_restore(_opmask_registers.at(i));
622 }
623
624 // Restore general purpose registers
625 for (int i = _gp_registers.length() - 1; i >= 0; i--) {
626 gp_register_restore(_gp_registers.at(i));
627 }
628
629 __ vzeroupper();
630
631 // Restore XMM/YMM/ZMM registers
632 for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
633 xmm_register_restore(_xmm_registers.at(i));
634 }
635
636 // Free stack space
637 if (_spill_size > 0) {
638 __ addptr(rsp, _spill_size);
639 }
640 }
641
642 #endif // COMPILER2