1 /*
  2  * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "asm/macroAssembler.inline.hpp"
 26 #include "classfile/classLoaderData.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "gc/shared/barrierSetAssembler.hpp"
 29 #include "gc/shared/barrierSetNMethod.hpp"
 30 #include "gc/shared/barrierSetRuntime.hpp"
 31 #include "gc/shared/collectedHeap.hpp"
 32 #include "interpreter/interp_masm.hpp"
 33 #include "memory/universe.hpp"
 34 #include "runtime/javaThread.hpp"
 35 #include "runtime/jniHandles.hpp"
 36 #include "runtime/sharedRuntime.hpp"
 37 #include "runtime/stubRoutines.hpp"
 38 #ifdef COMPILER2
 39 #include "gc/shared/c2/barrierSetC2.hpp"
 40 #endif // COMPILER2
 41 
 42 #define __ masm->
 43 
 44 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 45                                   Register dst, Address src, Register tmp1) {
 46   bool in_heap = (decorators & IN_HEAP) != 0;
 47   bool in_native = (decorators & IN_NATIVE) != 0;
 48   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 49   bool atomic = (decorators & MO_RELAXED) != 0;
 50 
 51   switch (type) {
 52   case T_OBJECT:
 53   case T_ARRAY: {
 54     if (in_heap) {
 55       if (UseCompressedOops) {
 56         __ movl(dst, src);
 57         if (is_not_null) {
 58           __ decode_heap_oop_not_null(dst);
 59         } else {
 60           __ decode_heap_oop(dst);
 61         }
 62       } else {
 63         __ movptr(dst, src);
 64       }
 65     } else {
 66       assert(in_native, "why else?");
 67       __ movptr(dst, src);
 68     }
 69     break;
 70   }
 71   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
 72   case T_BYTE:    __ load_signed_byte(dst, src);    break;
 73   case T_CHAR:    __ load_unsigned_short(dst, src); break;
 74   case T_SHORT:   __ load_signed_short(dst, src);   break;
 75   case T_INT:     __ movl  (dst, src);              break;
 76   case T_ADDRESS: __ movptr(dst, src);              break;
 77   case T_FLOAT:
 78     assert(dst == noreg, "only to ftos");
 79     __ movflt(xmm0, src);
 80     break;
 81   case T_DOUBLE:
 82     assert(dst == noreg, "only to dtos");
 83     __ movdbl(xmm0, src);
 84     break;
 85   case T_LONG:
 86     assert(dst == noreg, "only to ltos");
 87     __ movq(rax, src);
 88     break;
 89   default: Unimplemented();
 90   }
 91 }
 92 
 93 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 94                                    Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 95   bool in_heap = (decorators & IN_HEAP) != 0;
 96   bool in_native = (decorators & IN_NATIVE) != 0;
 97   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 98   bool atomic = (decorators & MO_RELAXED) != 0;
 99 
100   switch (type) {
101   case T_OBJECT:
102   case T_ARRAY: {
103     if (in_heap) {
104       if (val == noreg) {
105         assert(!is_not_null, "inconsistent access");
106         if (UseCompressedOops) {
107           __ movl(dst, NULL_WORD);
108         } else {
109           __ movslq(dst, NULL_WORD);
110         }
111       } else {
112         if (UseCompressedOops) {
113           assert(!dst.uses(val), "not enough registers");
114           if (is_not_null) {
115             __ encode_heap_oop_not_null(val);
116           } else {
117             __ encode_heap_oop(val);
118           }
119           __ movl(dst, val);
120         } else {
121           __ movptr(dst, val);
122         }
123       }
124     } else {
125       assert(in_native, "why else?");
126       assert(val != noreg, "not supported");
127       __ movptr(dst, val);
128     }
129     break;
130   }
131   case T_BOOLEAN:
132     __ andl(val, 0x1);  // boolean is true if LSB is 1
133     __ movb(dst, val);
134     break;
135   case T_BYTE:
136     __ movb(dst, val);
137     break;
138   case T_SHORT:
139     __ movw(dst, val);
140     break;
141   case T_CHAR:
142     __ movw(dst, val);
143     break;
144   case T_INT:
145     __ movl(dst, val);
146     break;
147   case T_LONG:
148     assert(val == noreg, "only tos");
149     __ movq(dst, rax);
150     break;
151   case T_FLOAT:
152     assert(val == noreg, "only tos");
153     __ movflt(dst, xmm0);
154     break;
155   case T_DOUBLE:
156     assert(val == noreg, "only tos");
157     __ movdbl(dst, xmm0);
158     break;
159   case T_ADDRESS:
160     __ movptr(dst, val);
161     break;
162   default: Unimplemented();
163   }
164 }
165 
166 void BarrierSetAssembler::flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
167                                      Register src, Register dst, Register inline_layout_info) {
168   // flat_field_copy implementation is fairly complex, and there are not any
169   // "short-cuts" to be made from asm. What there is, appears to have the same
170   // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
171   // of hand-rolled instructions...
172   if (decorators & IS_DEST_UNINITIALIZED) {
173     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, inline_layout_info);
174   } else {
175     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, inline_layout_info);
176   }
177 }
178 
179 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
180                                        DecoratorSet decorators,
181                                        BasicType type,
182                                        size_t bytes,
183                                        Register dst,
184                                        Address src,
185                                        Register tmp) {
186   assert(bytes <= 8, "can only deal with non-vector registers");
187   switch (bytes) {
188   case 1:
189     __ movb(dst, src);
190     break;
191   case 2:
192     __ movw(dst, src);
193     break;
194   case 4:
195     __ movl(dst, src);
196     break;
197   case 8:
198     __ movq(dst, src);
199     break;
200   default:
201     fatal("Unexpected size");
202   }
203   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
204     __ decode_heap_oop(dst);
205   }
206 }
207 
208 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
209                                         DecoratorSet decorators,
210                                         BasicType type,
211                                         size_t bytes,
212                                         Address dst,
213                                         Register src,
214                                         Register tmp) {
215   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
216     __ encode_heap_oop(src);
217   }
218   assert(bytes <= 8, "can only deal with non-vector registers");
219   switch (bytes) {
220   case 1:
221     __ movb(dst, src);
222     break;
223   case 2:
224     __ movw(dst, src);
225     break;
226   case 4:
227     __ movl(dst, src);
228     break;
229   case 8:
230     __ movq(dst, src);
231     break;
232   default:
233     fatal("Unexpected size");
234   }
235 }
236 
237 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
238                                        DecoratorSet decorators,
239                                        BasicType type,
240                                        size_t bytes,
241                                        XMMRegister dst,
242                                        Address src,
243                                        Register tmp,
244                                        XMMRegister xmm_tmp) {
245   assert(bytes > 8, "can only deal with vector registers");
246   if (bytes == 16) {
247     __ movdqu(dst, src);
248   } else if (bytes == 32) {
249     __ vmovdqu(dst, src);
250   } else {
251     fatal("No support for >32 bytes copy");
252   }
253 }
254 
255 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
256                                         DecoratorSet decorators,
257                                         BasicType type,
258                                         size_t bytes,
259                                         Address dst,
260                                         XMMRegister src,
261                                         Register tmp1,
262                                         Register tmp2,
263                                         XMMRegister xmm_tmp) {
264   assert(bytes > 8, "can only deal with vector registers");
265   if (bytes == 16) {
266     __ movdqu(dst, src);
267   } else if (bytes == 32) {
268     __ vmovdqu(dst, src);
269   } else {
270     fatal("No support for >32 bytes copy");
271   }
272 }
273 
274 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
275                                                         Register obj, Register tmp, Label& slowpath) {
276   __ clear_jobject_tag(obj);
277   __ movptr(obj, Address(obj, 0));
278 }
279 
280 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
281                                         Register obj,
282                                         Register var_size_in_bytes,
283                                         int con_size_in_bytes,
284                                         Register t1,
285                                         Register t2,
286                                         Label& slow_case) {
287   assert_different_registers(obj, t1, t2);
288   assert_different_registers(obj, var_size_in_bytes, t1);
289   Register end = t2;
290 
291   const Register thread = r15_thread;
292 
293   __ verify_tlab();
294 
295   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
296   if (var_size_in_bytes == noreg) {
297     __ lea(end, Address(obj, con_size_in_bytes));
298   } else {
299     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
300   }
301   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
302   __ jcc(Assembler::above, slow_case);
303 
304   // update the tlab top pointer
305   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
306 
307   // recover var_size_in_bytes if necessary
308   if (var_size_in_bytes == end) {
309     __ subptr(var_size_in_bytes, obj);
310   }
311   __ verify_tlab();
312 }
313 
314 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
315   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
316   Register thread = r15_thread;
317   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
318   // The immediate is the last 4 bytes, so if we align the start of the cmp
319   // instruction to 4 bytes, we know that the second half of it is also 4
320   // byte aligned, which means that the immediate will not cross a cache line
321   __ align(4);
322   uintptr_t before_cmp = (uintptr_t)__ pc();
323   __ cmpl_imm32(disarmed_addr, 0);
324   uintptr_t after_cmp = (uintptr_t)__ pc();
325   guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
326 
327   if (slow_path != nullptr) {
328     __ jcc(Assembler::notEqual, *slow_path);
329     __ bind(*continuation);
330   } else {
331     Label done;
332     __ jccb(Assembler::equal, done);
333     __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
334     __ bind(done);
335   }
336 }
337 
338 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
339   Label bad_call;
340   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
341   __ jcc(Assembler::equal, bad_call);
342 
343   Register tmp1 = rscratch1;
344   Register tmp2 = rscratch2;
345 
346   // Pointer chase to the method holder to find out if the method is concurrently unloading.
347   Label method_live;
348   __ load_method_holder_cld(tmp1, rbx);
349 
350    // Is it a strong CLD?
351   __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_ref_count_offset()), 0);
352   __ jcc(Assembler::greater, method_live);
353 
354    // Is it a weak but alive CLD?
355   __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
356   __ resolve_weak_handle(tmp1, tmp2);
357   __ cmpptr(tmp1, 0);
358   __ jcc(Assembler::notEqual, method_live);
359 
360   __ bind(bad_call);
361   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
362   __ bind(method_live);
363 }
364 
365 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
366   // Check if the oop is in the right area of memory
367   __ movptr(tmp1, obj);
368   __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
369   __ andptr(tmp1, tmp2);
370   __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
371   __ cmpptr(tmp1, tmp2);
372   __ jcc(Assembler::notZero, error);
373 
374   // make sure klass is 'reasonable', which is not zero.
375   __ load_klass(obj, obj, tmp1);  // get klass
376   __ testptr(obj, obj);
377   __ jcc(Assembler::zero, error); // if klass is null it is broken
378 }
379 
380 #ifdef COMPILER2
381 
382 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
383   if (!OptoReg::is_reg(opto_reg)) {
384     return OptoReg::Bad;
385   }
386 
387   const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
388   if (vm_reg->is_XMMRegister()) {
389     opto_reg &= ~15;
390     switch (node->ideal_reg()) {
391     case Op_VecX:
392       opto_reg |= 2;
393       break;
394     case Op_VecY:
395       opto_reg |= 4;
396       break;
397     case Op_VecZ:
398       opto_reg |= 8;
399       break;
400     default:
401       opto_reg |= 1;
402       break;
403     }
404   }
405 
406   return opto_reg;
407 }
408 
409 // We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
410 extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
411                             int stack_offset, int reg, uint ireg, outputStream* st);
412 
413 #undef __
414 #define __ _masm->
415 
416 int SaveLiveRegisters::xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
417   if (left->_size == right->_size) {
418     return 0;
419   }
420 
421   return (left->_size < right->_size) ? -1 : 1;
422 }
423 
424 int SaveLiveRegisters::xmm_slot_size(OptoReg::Name opto_reg) {
425   // The low order 4 bytes denote what size of the XMM register is live
426   return (opto_reg & 15) << 3;
427 }
428 
429 uint SaveLiveRegisters::xmm_ideal_reg_for_size(int reg_size) {
430   switch (reg_size) {
431   case 8:
432     return Op_VecD;
433   case 16:
434     return Op_VecX;
435   case 32:
436     return Op_VecY;
437   case 64:
438     return Op_VecZ;
439   default:
440     fatal("Invalid register size %d", reg_size);
441     return 0;
442   }
443 }
444 
445 bool SaveLiveRegisters::xmm_needs_vzeroupper() const {
446   return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
447 }
448 
449 void SaveLiveRegisters::xmm_register_save(const XMMRegisterData& reg_data) {
450   const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
451   const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
452   _spill_offset -= reg_data._size;
453   C2_MacroAssembler c2_masm(__ code());
454   vec_spill_helper(&c2_masm, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
455 }
456 
457 void SaveLiveRegisters::xmm_register_restore(const XMMRegisterData& reg_data) {
458   const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
459   const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
460   C2_MacroAssembler c2_masm(__ code());
461   vec_spill_helper(&c2_masm, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
462   _spill_offset += reg_data._size;
463 }
464 
465 void SaveLiveRegisters::gp_register_save(Register reg) {
466   _spill_offset -= 8;
467   __ movq(Address(rsp, _spill_offset), reg);
468 }
469 
470 void SaveLiveRegisters::opmask_register_save(KRegister reg) {
471   _spill_offset -= 8;
472   __ kmov(Address(rsp, _spill_offset), reg);
473 }
474 
475 void SaveLiveRegisters::gp_register_restore(Register reg) {
476   __ movq(reg, Address(rsp, _spill_offset));
477   _spill_offset += 8;
478 }
479 
480 void SaveLiveRegisters::opmask_register_restore(KRegister reg) {
481   __ kmov(reg, Address(rsp, _spill_offset));
482   _spill_offset += 8;
483 }
484 
485 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
486   // Create mask of caller saved registers that need to
487   // be saved/restored if live
488   RegMask caller_saved;
489   caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
490   caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
491   caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
492   caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
493   caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
494   caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
495   caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
496   caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
497   caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
498 
499   if (UseAPX) {
500     caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
501     caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
502     caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
503     caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
504     caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
505     caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
506     caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
507     caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
508     caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
509     caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
510     caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
511     caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
512     caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
513     caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
514     caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
515     caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
516   }
517 
518   int gp_spill_size = 0;
519   int opmask_spill_size = 0;
520   int xmm_spill_size = 0;
521 
522   // Record registers that needs to be saved/restored
523   RegMaskIterator rmi(stub->preserve_set());
524   while (rmi.has_next()) {
525     const OptoReg::Name opto_reg = rmi.next();
526     const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
527 
528     if (vm_reg->is_Register()) {
529       if (caller_saved.Member(opto_reg)) {
530         _gp_registers.append(vm_reg->as_Register());
531         gp_spill_size += 8;
532       }
533     } else if (vm_reg->is_KRegister()) {
534       // All opmask registers are caller saved, thus spill the ones
535       // which are live.
536       if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
537         _opmask_registers.append(vm_reg->as_KRegister());
538         opmask_spill_size += 8;
539       }
540     } else if (vm_reg->is_XMMRegister()) {
541       // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
542       const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
543       const int reg_size = xmm_slot_size(opto_reg);
544       const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
545       const int reg_index = _xmm_registers.find(reg_data);
546       if (reg_index == -1) {
547         // Not previously appended
548         _xmm_registers.append(reg_data);
549         xmm_spill_size += reg_size;
550       } else {
551         // Previously appended, update size
552         const int reg_size_prev = _xmm_registers.at(reg_index)._size;
553         if (reg_size > reg_size_prev) {
554           _xmm_registers.at_put(reg_index, reg_data);
555           xmm_spill_size += reg_size - reg_size_prev;
556         }
557       }
558     } else {
559       fatal("Unexpected register type");
560     }
561   }
562 
563   // Sort by size, largest first
564   _xmm_registers.sort(xmm_compare_register_size);
565 
566   // On Windows, the caller reserves stack space for spilling register arguments
567   const int arg_spill_size = frame::arg_reg_save_area_bytes;
568 
569   // Stack pointer must be 16 bytes aligned for the call
570   _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
571 }
572 
573 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
574   : _masm(masm),
575     _gp_registers(),
576     _opmask_registers(),
577     _xmm_registers(),
578     _spill_size(0),
579     _spill_offset(0) {
580 
581   //
582   // Stack layout after registers have been spilled:
583   //
584   // | ...            | original rsp, 16 bytes aligned
585   // ------------------
586   // | zmm0 high      |
587   // | ...            |
588   // | zmm0 low       | 16 bytes aligned
589   // | ...            |
590   // | ymm1 high      |
591   // | ...            |
592   // | ymm1 low       | 16 bytes aligned
593   // | ...            |
594   // | xmmN high      |
595   // | ...            |
596   // | xmmN low       | 8 bytes aligned
597   // | reg0           | 8 bytes aligned
598   // | reg1           |
599   // | ...            |
600   // | regN           | new rsp, if 16 bytes aligned
601   // | <padding>      | else new rsp, 16 bytes aligned
602   // ------------------
603   //
604 
605   // Figure out what registers to save/restore
606   initialize(stub);
607 
608   // Allocate stack space
609   if (_spill_size > 0) {
610     __ subptr(rsp, _spill_size);
611   }
612 
613   // Save XMM/YMM/ZMM registers
614   for (int i = 0; i < _xmm_registers.length(); i++) {
615     xmm_register_save(_xmm_registers.at(i));
616   }
617 
618   if (xmm_needs_vzeroupper()) {
619     __ vzeroupper();
620   }
621 
622   // Save general purpose registers
623   for (int i = 0; i < _gp_registers.length(); i++) {
624     gp_register_save(_gp_registers.at(i));
625   }
626 
627   // Save opmask registers
628   for (int i = 0; i < _opmask_registers.length(); i++) {
629     opmask_register_save(_opmask_registers.at(i));
630   }
631 }
632 
633 SaveLiveRegisters::~SaveLiveRegisters() {
634   // Restore opmask registers
635   for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
636     opmask_register_restore(_opmask_registers.at(i));
637   }
638 
639   // Restore general purpose registers
640   for (int i = _gp_registers.length() - 1; i >= 0; i--) {
641     gp_register_restore(_gp_registers.at(i));
642   }
643 
644   __ vzeroupper();
645 
646   // Restore XMM/YMM/ZMM registers
647   for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
648     xmm_register_restore(_xmm_registers.at(i));
649   }
650 
651   // Free stack space
652   if (_spill_size > 0) {
653     __ addptr(rsp, _spill_size);
654   }
655 }
656 
657 #endif // COMPILER2