1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "classfile/classLoaderData.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "gc/shared/barrierSetRuntime.hpp"
31 #include "gc/shared/collectedHeap.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "memory/universe.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/jniHandles.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #ifdef COMPILER2
39 #include "gc/shared/c2/barrierSetC2.hpp"
40 #endif // COMPILER2
41
42 #define __ masm->
43
44 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
45 Register dst, Address src, Register tmp1) {
46 bool in_heap = (decorators & IN_HEAP) != 0;
47 bool in_native = (decorators & IN_NATIVE) != 0;
48 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
49 bool atomic = (decorators & MO_RELAXED) != 0;
50
51 switch (type) {
52 case T_OBJECT:
53 case T_ARRAY: {
54 if (in_heap) {
55 if (UseCompressedOops) {
56 __ movl(dst, src);
57 if (is_not_null) {
58 __ decode_heap_oop_not_null(dst);
59 } else {
60 __ decode_heap_oop(dst);
61 }
62 } else {
63 __ movptr(dst, src);
64 }
65 } else {
66 assert(in_native, "why else?");
67 __ movptr(dst, src);
68 }
69 break;
70 }
71 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break;
72 case T_BYTE: __ load_signed_byte(dst, src); break;
73 case T_CHAR: __ load_unsigned_short(dst, src); break;
74 case T_SHORT: __ load_signed_short(dst, src); break;
75 case T_INT: __ movl (dst, src); break;
76 case T_ADDRESS: __ movptr(dst, src); break;
77 case T_FLOAT:
78 assert(dst == noreg, "only to ftos");
79 __ movflt(xmm0, src);
80 break;
81 case T_DOUBLE:
82 assert(dst == noreg, "only to dtos");
83 __ movdbl(xmm0, src);
84 break;
85 case T_LONG:
86 assert(dst == noreg, "only to ltos");
87 __ movq(rax, src);
88 break;
89 default: Unimplemented();
90 }
91 }
92
93 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
94 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
95 bool in_heap = (decorators & IN_HEAP) != 0;
96 bool in_native = (decorators & IN_NATIVE) != 0;
97 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
98 bool atomic = (decorators & MO_RELAXED) != 0;
99
100 switch (type) {
101 case T_OBJECT:
102 case T_ARRAY: {
103 if (in_heap) {
104 if (val == noreg) {
105 assert(!is_not_null, "inconsistent access");
106 if (UseCompressedOops) {
107 __ movl(dst, NULL_WORD);
108 } else {
109 __ movslq(dst, NULL_WORD);
110 }
111 } else {
112 if (UseCompressedOops) {
113 assert(!dst.uses(val), "not enough registers");
114 if (is_not_null) {
115 __ encode_heap_oop_not_null(val);
116 } else {
117 __ encode_heap_oop(val);
118 }
119 __ movl(dst, val);
120 } else {
121 __ movptr(dst, val);
122 }
123 }
124 } else {
125 assert(in_native, "why else?");
126 assert(val != noreg, "not supported");
127 __ movptr(dst, val);
128 }
129 break;
130 }
131 case T_BOOLEAN:
132 __ andl(val, 0x1); // boolean is true if LSB is 1
133 __ movb(dst, val);
134 break;
135 case T_BYTE:
136 __ movb(dst, val);
137 break;
138 case T_SHORT:
139 __ movw(dst, val);
140 break;
141 case T_CHAR:
142 __ movw(dst, val);
143 break;
144 case T_INT:
145 __ movl(dst, val);
146 break;
147 case T_LONG:
148 assert(val == noreg, "only tos");
149 __ movq(dst, rax);
150 break;
151 case T_FLOAT:
152 assert(val == noreg, "only tos");
153 __ movflt(dst, xmm0);
154 break;
155 case T_DOUBLE:
156 assert(val == noreg, "only tos");
157 __ movdbl(dst, xmm0);
158 break;
159 case T_ADDRESS:
160 __ movptr(dst, val);
161 break;
162 default: Unimplemented();
163 }
164 }
165
166 void BarrierSetAssembler::flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
167 Register src, Register dst, Register inline_layout_info) {
168 // flat_field_copy implementation is fairly complex, and there are not any
169 // "short-cuts" to be made from asm. What there is, appears to have the same
170 // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
171 // of hand-rolled instructions...
172 if (decorators & IS_DEST_UNINITIALIZED) {
173 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, inline_layout_info);
174 } else {
175 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, inline_layout_info);
176 }
177 }
178
179 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
180 DecoratorSet decorators,
181 BasicType type,
182 size_t bytes,
183 Register dst,
184 Address src,
185 Register tmp) {
186 assert(bytes <= 8, "can only deal with non-vector registers");
187 switch (bytes) {
188 case 1:
189 __ movb(dst, src);
190 break;
191 case 2:
192 __ movw(dst, src);
193 break;
194 case 4:
195 __ movl(dst, src);
196 break;
197 case 8:
198 __ movq(dst, src);
199 break;
200 default:
201 fatal("Unexpected size");
202 }
203 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
204 __ decode_heap_oop(dst);
205 }
206 }
207
208 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
209 DecoratorSet decorators,
210 BasicType type,
211 size_t bytes,
212 Address dst,
213 Register src,
214 Register tmp) {
215 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
216 __ encode_heap_oop(src);
217 }
218 assert(bytes <= 8, "can only deal with non-vector registers");
219 switch (bytes) {
220 case 1:
221 __ movb(dst, src);
222 break;
223 case 2:
224 __ movw(dst, src);
225 break;
226 case 4:
227 __ movl(dst, src);
228 break;
229 case 8:
230 __ movq(dst, src);
231 break;
232 default:
233 fatal("Unexpected size");
234 }
235 }
236
237 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
238 DecoratorSet decorators,
239 BasicType type,
240 size_t bytes,
241 XMMRegister dst,
242 Address src,
243 Register tmp,
244 XMMRegister xmm_tmp) {
245 assert(bytes > 8, "can only deal with vector registers");
246 if (bytes == 16) {
247 __ movdqu(dst, src);
248 } else if (bytes == 32) {
249 __ vmovdqu(dst, src);
250 } else {
251 fatal("No support for >32 bytes copy");
252 }
253 }
254
255 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
256 DecoratorSet decorators,
257 BasicType type,
258 size_t bytes,
259 Address dst,
260 XMMRegister src,
261 Register tmp1,
262 Register tmp2,
263 XMMRegister xmm_tmp) {
264 assert(bytes > 8, "can only deal with vector registers");
265 if (bytes == 16) {
266 __ movdqu(dst, src);
267 } else if (bytes == 32) {
268 __ vmovdqu(dst, src);
269 } else {
270 fatal("No support for >32 bytes copy");
271 }
272 }
273
274 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
275 Register obj, Register tmp, Label& slowpath) {
276 __ clear_jobject_tag(obj);
277 __ movptr(obj, Address(obj, 0));
278 }
279
280 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
281 Register obj,
282 Register var_size_in_bytes,
283 int con_size_in_bytes,
284 Register t1,
285 Register t2,
286 Label& slow_case) {
287 assert_different_registers(obj, t1, t2);
288 assert_different_registers(obj, var_size_in_bytes, t1);
289 Register end = t2;
290
291 const Register thread = r15_thread;
292
293 __ verify_tlab();
294
295 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
296 if (var_size_in_bytes == noreg) {
297 __ lea(end, Address(obj, con_size_in_bytes));
298 } else {
299 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
300 }
301 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
302 __ jcc(Assembler::above, slow_case);
303
304 // update the tlab top pointer
305 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
306
307 // recover var_size_in_bytes if necessary
308 if (var_size_in_bytes == end) {
309 __ subptr(var_size_in_bytes, obj);
310 }
311 __ verify_tlab();
312 }
313
314 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
315 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
316 Register thread = r15_thread;
317 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
318 // The immediate is the last 4 bytes, so if we align the start of the cmp
319 // instruction to 4 bytes, we know that the second half of it is also 4
320 // byte aligned, which means that the immediate will not cross a cache line
321 __ align(4);
322 uintptr_t before_cmp = (uintptr_t)__ pc();
323 __ cmpl_imm32(disarmed_addr, 0);
324 uintptr_t after_cmp = (uintptr_t)__ pc();
325 guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
326
327 if (slow_path != nullptr) {
328 __ jcc(Assembler::notEqual, *slow_path);
329 __ bind(*continuation);
330 } else {
331 Label done;
332 __ jccb(Assembler::equal, done);
333 __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
334 __ bind(done);
335 }
336 }
337
338 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
339 Label bad_call;
340 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
341 __ jcc(Assembler::equal, bad_call);
342
343 Register tmp1 = rscratch1;
344 Register tmp2 = rscratch2;
345
346 // Pointer chase to the method holder to find out if the method is concurrently unloading.
347 Label method_live;
348 __ load_method_holder_cld(tmp1, rbx);
349
350 // Is it a strong CLD?
351 __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_ref_count_offset()), 0);
352 __ jcc(Assembler::greater, method_live);
353
354 // Is it a weak but alive CLD?
355 __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
356 __ resolve_weak_handle(tmp1, tmp2);
357 __ cmpptr(tmp1, 0);
358 __ jcc(Assembler::notEqual, method_live);
359
360 __ bind(bad_call);
361 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
362 __ bind(method_live);
363 }
364
365 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
366 // Check if the oop is in the right area of memory
367 __ movptr(tmp1, obj);
368 __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
369 __ andptr(tmp1, tmp2);
370 __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
371 __ cmpptr(tmp1, tmp2);
372 __ jcc(Assembler::notZero, error);
373
374 // make sure klass is 'reasonable', which is not zero.
375 __ load_klass(obj, obj, tmp1); // get klass
376 __ testptr(obj, obj);
377 __ jcc(Assembler::zero, error); // if klass is null it is broken
378 }
379
380 #ifdef COMPILER2
381
382 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
383 if (!OptoReg::is_reg(opto_reg)) {
384 return OptoReg::Bad;
385 }
386
387 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
388 if (vm_reg->is_XMMRegister()) {
389 opto_reg &= ~15;
390 switch (node->ideal_reg()) {
391 case Op_VecX:
392 opto_reg |= 2;
393 break;
394 case Op_VecY:
395 opto_reg |= 4;
396 break;
397 case Op_VecZ:
398 opto_reg |= 8;
399 break;
400 default:
401 opto_reg |= 1;
402 break;
403 }
404 }
405
406 return opto_reg;
407 }
408
409 // We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
410 extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
411 int stack_offset, int reg, uint ireg, outputStream* st);
412
413 #undef __
414 #define __ _masm->
415
416 int SaveLiveRegisters::xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
417 if (left->_size == right->_size) {
418 return 0;
419 }
420
421 return (left->_size < right->_size) ? -1 : 1;
422 }
423
424 int SaveLiveRegisters::xmm_slot_size(OptoReg::Name opto_reg) {
425 // The low order 4 bytes denote what size of the XMM register is live
426 return (opto_reg & 15) << 3;
427 }
428
429 uint SaveLiveRegisters::xmm_ideal_reg_for_size(int reg_size) {
430 switch (reg_size) {
431 case 8:
432 return Op_VecD;
433 case 16:
434 return Op_VecX;
435 case 32:
436 return Op_VecY;
437 case 64:
438 return Op_VecZ;
439 default:
440 fatal("Invalid register size %d", reg_size);
441 return 0;
442 }
443 }
444
445 bool SaveLiveRegisters::xmm_needs_vzeroupper() const {
446 return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
447 }
448
449 void SaveLiveRegisters::xmm_register_save(const XMMRegisterData& reg_data) {
450 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
451 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
452 _spill_offset -= reg_data._size;
453 C2_MacroAssembler c2_masm(__ code());
454 vec_spill_helper(&c2_masm, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
455 }
456
457 void SaveLiveRegisters::xmm_register_restore(const XMMRegisterData& reg_data) {
458 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
459 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
460 C2_MacroAssembler c2_masm(__ code());
461 vec_spill_helper(&c2_masm, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
462 _spill_offset += reg_data._size;
463 }
464
465 void SaveLiveRegisters::gp_register_save(Register reg) {
466 _spill_offset -= 8;
467 __ movq(Address(rsp, _spill_offset), reg);
468 }
469
470 void SaveLiveRegisters::opmask_register_save(KRegister reg) {
471 _spill_offset -= 8;
472 __ kmov(Address(rsp, _spill_offset), reg);
473 }
474
475 void SaveLiveRegisters::gp_register_restore(Register reg) {
476 __ movq(reg, Address(rsp, _spill_offset));
477 _spill_offset += 8;
478 }
479
480 void SaveLiveRegisters::opmask_register_restore(KRegister reg) {
481 __ kmov(reg, Address(rsp, _spill_offset));
482 _spill_offset += 8;
483 }
484
485 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
486 // Create mask of caller saved registers that need to
487 // be saved/restored if live
488 RegMask caller_saved;
489 caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
490 caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
491 caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
492 caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
493 caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
494 caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
495 caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
496 caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
497 caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
498
499 if (UseAPX) {
500 caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
501 caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
502 caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
503 caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
504 caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
505 caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
506 caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
507 caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
508 caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
509 caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
510 caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
511 caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
512 caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
513 caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
514 caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
515 caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
516 }
517
518 int gp_spill_size = 0;
519 int opmask_spill_size = 0;
520 int xmm_spill_size = 0;
521
522 // Record registers that needs to be saved/restored
523 RegMaskIterator rmi(stub->preserve_set());
524 while (rmi.has_next()) {
525 const OptoReg::Name opto_reg = rmi.next();
526 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
527
528 if (vm_reg->is_Register()) {
529 if (caller_saved.Member(opto_reg)) {
530 _gp_registers.append(vm_reg->as_Register());
531 gp_spill_size += 8;
532 }
533 } else if (vm_reg->is_KRegister()) {
534 // All opmask registers are caller saved, thus spill the ones
535 // which are live.
536 if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
537 _opmask_registers.append(vm_reg->as_KRegister());
538 opmask_spill_size += 8;
539 }
540 } else if (vm_reg->is_XMMRegister()) {
541 // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
542 const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
543 const int reg_size = xmm_slot_size(opto_reg);
544 const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
545 const int reg_index = _xmm_registers.find(reg_data);
546 if (reg_index == -1) {
547 // Not previously appended
548 _xmm_registers.append(reg_data);
549 xmm_spill_size += reg_size;
550 } else {
551 // Previously appended, update size
552 const int reg_size_prev = _xmm_registers.at(reg_index)._size;
553 if (reg_size > reg_size_prev) {
554 _xmm_registers.at_put(reg_index, reg_data);
555 xmm_spill_size += reg_size - reg_size_prev;
556 }
557 }
558 } else {
559 fatal("Unexpected register type");
560 }
561 }
562
563 // Sort by size, largest first
564 _xmm_registers.sort(xmm_compare_register_size);
565
566 // On Windows, the caller reserves stack space for spilling register arguments
567 const int arg_spill_size = frame::arg_reg_save_area_bytes;
568
569 // Stack pointer must be 16 bytes aligned for the call
570 _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
571 }
572
573 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
574 : _masm(masm),
575 _gp_registers(),
576 _opmask_registers(),
577 _xmm_registers(),
578 _spill_size(0),
579 _spill_offset(0) {
580
581 //
582 // Stack layout after registers have been spilled:
583 //
584 // | ... | original rsp, 16 bytes aligned
585 // ------------------
586 // | zmm0 high |
587 // | ... |
588 // | zmm0 low | 16 bytes aligned
589 // | ... |
590 // | ymm1 high |
591 // | ... |
592 // | ymm1 low | 16 bytes aligned
593 // | ... |
594 // | xmmN high |
595 // | ... |
596 // | xmmN low | 8 bytes aligned
597 // | reg0 | 8 bytes aligned
598 // | reg1 |
599 // | ... |
600 // | regN | new rsp, if 16 bytes aligned
601 // | <padding> | else new rsp, 16 bytes aligned
602 // ------------------
603 //
604
605 // Figure out what registers to save/restore
606 initialize(stub);
607
608 // Allocate stack space
609 if (_spill_size > 0) {
610 __ subptr(rsp, _spill_size);
611 }
612
613 // Save XMM/YMM/ZMM registers
614 for (int i = 0; i < _xmm_registers.length(); i++) {
615 xmm_register_save(_xmm_registers.at(i));
616 }
617
618 if (xmm_needs_vzeroupper()) {
619 __ vzeroupper();
620 }
621
622 // Save general purpose registers
623 for (int i = 0; i < _gp_registers.length(); i++) {
624 gp_register_save(_gp_registers.at(i));
625 }
626
627 // Save opmask registers
628 for (int i = 0; i < _opmask_registers.length(); i++) {
629 opmask_register_save(_opmask_registers.at(i));
630 }
631 }
632
633 SaveLiveRegisters::~SaveLiveRegisters() {
634 // Restore opmask registers
635 for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
636 opmask_register_restore(_opmask_registers.at(i));
637 }
638
639 // Restore general purpose registers
640 for (int i = _gp_registers.length() - 1; i >= 0; i--) {
641 gp_register_restore(_gp_registers.at(i));
642 }
643
644 __ vzeroupper();
645
646 // Restore XMM/YMM/ZMM registers
647 for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
648 xmm_register_restore(_xmm_registers.at(i));
649 }
650
651 // Free stack space
652 if (_spill_size > 0) {
653 __ addptr(rsp, _spill_size);
654 }
655 }
656
657 #endif // COMPILER2