1 /*
  2  * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "classfile/classLoaderData.hpp"
 26 #include "gc/shared/barrierSet.hpp"
 27 #include "gc/shared/barrierSetAssembler.hpp"
 28 #include "gc/shared/barrierSetNMethod.hpp"
 29 #include "gc/shared/collectedHeap.hpp"
 30 #include "interpreter/interp_masm.hpp"
 31 #include "memory/universe.hpp"
 32 #include "runtime/javaThread.hpp"
 33 #include "runtime/jniHandles.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "runtime/stubRoutines.hpp"
 36 #ifdef COMPILER2
 37 #include "gc/shared/c2/barrierSetC2.hpp"
 38 #endif // COMPILER2
 39 
 40 #define __ masm->
 41 
 42 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 43                                   Register dst, Address src, Register tmp1) {
 44   bool in_heap = (decorators & IN_HEAP) != 0;
 45   bool in_native = (decorators & IN_NATIVE) != 0;
 46   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 47   bool atomic = (decorators & MO_RELAXED) != 0;
 48 
 49   switch (type) {
 50   case T_OBJECT:
 51   case T_ARRAY: {
 52     if (in_heap) {
 53       if (UseCompressedOops) {
 54         __ movl(dst, src);
 55         if (is_not_null) {
 56           __ decode_heap_oop_not_null(dst);
 57         } else {
 58           __ decode_heap_oop(dst);
 59         }
 60       } else {
 61         __ movptr(dst, src);
 62       }
 63     } else {
 64       assert(in_native, "why else?");
 65       __ movptr(dst, src);
 66     }
 67     break;
 68   }
 69   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
 70   case T_BYTE:    __ load_signed_byte(dst, src);    break;
 71   case T_CHAR:    __ load_unsigned_short(dst, src); break;
 72   case T_SHORT:   __ load_signed_short(dst, src);   break;
 73   case T_INT:     __ movl  (dst, src);              break;
 74   case T_ADDRESS: __ movptr(dst, src);              break;
 75   case T_FLOAT:
 76     assert(dst == noreg, "only to ftos");
 77     __ movflt(xmm0, src);
 78     break;
 79   case T_DOUBLE:
 80     assert(dst == noreg, "only to dtos");
 81     __ movdbl(xmm0, src);
 82     break;
 83   case T_LONG:
 84     assert(dst == noreg, "only to ltos");
 85     __ movq(rax, src);
 86     break;
 87   default: Unimplemented();
 88   }
 89 }
 90 
 91 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 92                                    Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 93   bool in_heap = (decorators & IN_HEAP) != 0;
 94   bool in_native = (decorators & IN_NATIVE) != 0;
 95   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 96   bool atomic = (decorators & MO_RELAXED) != 0;
 97 
 98   switch (type) {
 99   case T_OBJECT:
100   case T_ARRAY: {
101     if (in_heap) {
102       if (val == noreg) {
103         assert(!is_not_null, "inconsistent access");
104         if (UseCompressedOops) {
105           __ movl(dst, NULL_WORD);
106         } else {
107           __ movslq(dst, NULL_WORD);
108         }
109       } else {
110         if (UseCompressedOops) {
111           assert(!dst.uses(val), "not enough registers");
112           if (is_not_null) {
113             __ encode_heap_oop_not_null(val);
114           } else {
115             __ encode_heap_oop(val);
116           }
117           __ movl(dst, val);
118         } else {
119           __ movptr(dst, val);
120         }
121       }
122     } else {
123       assert(in_native, "why else?");
124       assert(val != noreg, "not supported");
125       __ movptr(dst, val);
126     }
127     break;
128   }
129   case T_BOOLEAN:
130     __ andl(val, 0x1);  // boolean is true if LSB is 1
131     __ movb(dst, val);
132     break;
133   case T_BYTE:
134     __ movb(dst, val);
135     break;
136   case T_SHORT:
137     __ movw(dst, val);
138     break;
139   case T_CHAR:
140     __ movw(dst, val);
141     break;
142   case T_INT:
143     __ movl(dst, val);
144     break;
145   case T_LONG:
146     assert(val == noreg, "only tos");
147     __ movq(dst, rax);
148     break;
149   case T_FLOAT:
150     assert(val == noreg, "only tos");
151     __ movflt(dst, xmm0);
152     break;
153   case T_DOUBLE:
154     assert(val == noreg, "only tos");
155     __ movdbl(dst, xmm0);
156     break;
157   case T_ADDRESS:
158     __ movptr(dst, val);
159     break;
160   default: Unimplemented();
161   }
162 }
163 
164 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
165                                        DecoratorSet decorators,
166                                        BasicType type,
167                                        size_t bytes,
168                                        Register dst,
169                                        Address src,
170                                        Register tmp) {
171   assert(bytes <= 8, "can only deal with non-vector registers");
172   switch (bytes) {
173   case 1:
174     __ movb(dst, src);
175     break;
176   case 2:
177     __ movw(dst, src);
178     break;
179   case 4:
180     __ movl(dst, src);
181     break;
182   case 8:
183     __ movq(dst, src);
184     break;
185   default:
186     fatal("Unexpected size");
187   }
188   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
189     __ decode_heap_oop(dst);
190   }
191 }
192 
193 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
194                                         DecoratorSet decorators,
195                                         BasicType type,
196                                         size_t bytes,
197                                         Address dst,
198                                         Register src,
199                                         Register tmp) {
200   if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
201     __ encode_heap_oop(src);
202   }
203   assert(bytes <= 8, "can only deal with non-vector registers");
204   switch (bytes) {
205   case 1:
206     __ movb(dst, src);
207     break;
208   case 2:
209     __ movw(dst, src);
210     break;
211   case 4:
212     __ movl(dst, src);
213     break;
214   case 8:
215     __ movq(dst, src);
216     break;
217   default:
218     fatal("Unexpected size");
219   }
220 }
221 
222 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
223                                        DecoratorSet decorators,
224                                        BasicType type,
225                                        size_t bytes,
226                                        XMMRegister dst,
227                                        Address src,
228                                        Register tmp,
229                                        XMMRegister xmm_tmp) {
230   assert(bytes > 8, "can only deal with vector registers");
231   if (bytes == 16) {
232     __ movdqu(dst, src);
233   } else if (bytes == 32) {
234     __ vmovdqu(dst, src);
235   } else {
236     fatal("No support for >32 bytes copy");
237   }
238 }
239 
240 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
241                                         DecoratorSet decorators,
242                                         BasicType type,
243                                         size_t bytes,
244                                         Address dst,
245                                         XMMRegister src,
246                                         Register tmp1,
247                                         Register tmp2,
248                                         XMMRegister xmm_tmp) {
249   assert(bytes > 8, "can only deal with vector registers");
250   if (bytes == 16) {
251     __ movdqu(dst, src);
252   } else if (bytes == 32) {
253     __ vmovdqu(dst, src);
254   } else {
255     fatal("No support for >32 bytes copy");
256   }
257 }
258 
259 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
260                                                         Register obj, Register tmp, Label& slowpath) {
261   __ clear_jobject_tag(obj);
262   __ movptr(obj, Address(obj, 0));
263 }
264 
265 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
266                                         Register obj,
267                                         Register var_size_in_bytes,
268                                         int con_size_in_bytes,
269                                         Register t1,
270                                         Register t2,
271                                         Label& slow_case) {
272   assert_different_registers(obj, t1, t2);
273   assert_different_registers(obj, var_size_in_bytes, t1);
274   Register end = t2;
275 
276   const Register thread = r15_thread;
277 
278   __ verify_tlab();
279 
280   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
281   if (var_size_in_bytes == noreg) {
282     __ lea(end, Address(obj, con_size_in_bytes));
283   } else {
284     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
285   }
286   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
287   __ jcc(Assembler::above, slow_case);
288 
289   // update the tlab top pointer
290   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
291 
292   // recover var_size_in_bytes if necessary
293   if (var_size_in_bytes == end) {
294     __ subptr(var_size_in_bytes, obj);
295   }
296   __ verify_tlab();
297 }
298 
299 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation) {
300   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
301   Register thread = r15_thread;
302   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
303   // The immediate is the last 4 bytes, so if we align the start of the cmp
304   // instruction to 4 bytes, we know that the second half of it is also 4
305   // byte aligned, which means that the immediate will not cross a cache line
306   __ align(4);
307   uintptr_t before_cmp = (uintptr_t)__ pc();
308   __ cmpl_imm32(disarmed_addr, 0);
309   uintptr_t after_cmp = (uintptr_t)__ pc();
310   guarantee(after_cmp - before_cmp == 8, "Wrong assumed instruction length");
311 
312   if (slow_path != nullptr) {
313     __ jcc(Assembler::notEqual, *slow_path);
314     __ bind(*continuation);
315   } else {
316     Label done;
317     __ jccb(Assembler::equal, done);
318     __ call(RuntimeAddress(StubRoutines::method_entry_barrier()));
319     __ bind(done);
320   }
321 }
322 
323 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
324   Label bad_call;
325   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
326   __ jcc(Assembler::equal, bad_call);
327 
328   Register tmp1 = rscratch1;
329   Register tmp2 = rscratch2;
330 
331   // Pointer chase to the method holder to find out if the method is concurrently unloading.
332   Label method_live;
333   __ load_method_holder_cld(tmp1, rbx);
334 
335    // Is it a strong CLD?
336   __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_ref_count_offset()), 0);
337   __ jcc(Assembler::greater, method_live);
338 
339    // Is it a weak but alive CLD?
340   __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset()));
341   __ resolve_weak_handle(tmp1, tmp2);
342   __ cmpptr(tmp1, 0);
343   __ jcc(Assembler::notEqual, method_live);
344 
345   __ bind(bad_call);
346   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
347   __ bind(method_live);
348 }
349 
350 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
351   // Check if the oop is in the right area of memory
352   __ movptr(tmp1, obj);
353   __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
354   __ andptr(tmp1, tmp2);
355   __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
356   __ cmpptr(tmp1, tmp2);
357   __ jcc(Assembler::notZero, error);
358 
359   // make sure klass is 'reasonable', which is not zero.
360   __ load_klass(obj, obj, tmp1);  // get klass
361   __ testptr(obj, obj);
362   __ jcc(Assembler::zero, error); // if klass is null it is broken
363 }
364 
365 #ifdef COMPILER2
366 
367 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
368   if (!OptoReg::is_reg(opto_reg)) {
369     return OptoReg::Bad;
370   }
371 
372   const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
373   if (vm_reg->is_XMMRegister()) {
374     opto_reg &= ~15;
375     switch (node->ideal_reg()) {
376     case Op_VecX:
377       opto_reg |= 2;
378       break;
379     case Op_VecY:
380       opto_reg |= 4;
381       break;
382     case Op_VecZ:
383       opto_reg |= 8;
384       break;
385     default:
386       opto_reg |= 1;
387       break;
388     }
389   }
390 
391   return opto_reg;
392 }
393 
394 // We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
395 extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
396                             int stack_offset, int reg, uint ireg, outputStream* st);
397 
398 #undef __
399 #define __ _masm->
400 
401 int SaveLiveRegisters::xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
402   if (left->_size == right->_size) {
403     return 0;
404   }
405 
406   return (left->_size < right->_size) ? -1 : 1;
407 }
408 
409 int SaveLiveRegisters::xmm_slot_size(OptoReg::Name opto_reg) {
410   // The low order 4 bytes denote what size of the XMM register is live
411   return (opto_reg & 15) << 3;
412 }
413 
414 uint SaveLiveRegisters::xmm_ideal_reg_for_size(int reg_size) {
415   switch (reg_size) {
416   case 8:
417     return Op_VecD;
418   case 16:
419     return Op_VecX;
420   case 32:
421     return Op_VecY;
422   case 64:
423     return Op_VecZ;
424   default:
425     fatal("Invalid register size %d", reg_size);
426     return 0;
427   }
428 }
429 
430 bool SaveLiveRegisters::xmm_needs_vzeroupper() const {
431   return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
432 }
433 
434 void SaveLiveRegisters::xmm_register_save(const XMMRegisterData& reg_data) {
435   const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
436   const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
437   _spill_offset -= reg_data._size;
438   C2_MacroAssembler c2_masm(__ code());
439   vec_spill_helper(&c2_masm, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
440 }
441 
442 void SaveLiveRegisters::xmm_register_restore(const XMMRegisterData& reg_data) {
443   const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
444   const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
445   C2_MacroAssembler c2_masm(__ code());
446   vec_spill_helper(&c2_masm, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
447   _spill_offset += reg_data._size;
448 }
449 
450 void SaveLiveRegisters::gp_register_save(Register reg) {
451   _spill_offset -= 8;
452   __ movq(Address(rsp, _spill_offset), reg);
453 }
454 
455 void SaveLiveRegisters::opmask_register_save(KRegister reg) {
456   _spill_offset -= 8;
457   __ kmov(Address(rsp, _spill_offset), reg);
458 }
459 
460 void SaveLiveRegisters::gp_register_restore(Register reg) {
461   __ movq(reg, Address(rsp, _spill_offset));
462   _spill_offset += 8;
463 }
464 
465 void SaveLiveRegisters::opmask_register_restore(KRegister reg) {
466   __ kmov(reg, Address(rsp, _spill_offset));
467   _spill_offset += 8;
468 }
469 
470 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
471   // Create mask of caller saved registers that need to
472   // be saved/restored if live
473   RegMask caller_saved;
474   caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
475   caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
476   caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
477   caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
478   caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
479   caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
480   caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
481   caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
482   caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
483 
484   if (UseAPX) {
485     caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
486     caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
487     caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
488     caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
489     caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
490     caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
491     caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
492     caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
493     caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
494     caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
495     caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
496     caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
497     caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
498     caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
499     caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
500     caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
501   }
502 
503   int gp_spill_size = 0;
504   int opmask_spill_size = 0;
505   int xmm_spill_size = 0;
506 
507   // Record registers that needs to be saved/restored
508   RegMaskIterator rmi(stub->preserve_set());
509   while (rmi.has_next()) {
510     const OptoReg::Name opto_reg = rmi.next();
511     const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
512 
513     if (vm_reg->is_Register()) {
514       if (caller_saved.Member(opto_reg)) {
515         _gp_registers.append(vm_reg->as_Register());
516         gp_spill_size += 8;
517       }
518     } else if (vm_reg->is_KRegister()) {
519       // All opmask registers are caller saved, thus spill the ones
520       // which are live.
521       if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
522         _opmask_registers.append(vm_reg->as_KRegister());
523         opmask_spill_size += 8;
524       }
525     } else if (vm_reg->is_XMMRegister()) {
526       // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
527       const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
528       const int reg_size = xmm_slot_size(opto_reg);
529       const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
530       const int reg_index = _xmm_registers.find(reg_data);
531       if (reg_index == -1) {
532         // Not previously appended
533         _xmm_registers.append(reg_data);
534         xmm_spill_size += reg_size;
535       } else {
536         // Previously appended, update size
537         const int reg_size_prev = _xmm_registers.at(reg_index)._size;
538         if (reg_size > reg_size_prev) {
539           _xmm_registers.at_put(reg_index, reg_data);
540           xmm_spill_size += reg_size - reg_size_prev;
541         }
542       }
543     } else {
544       fatal("Unexpected register type");
545     }
546   }
547 
548   // Sort by size, largest first
549   _xmm_registers.sort(xmm_compare_register_size);
550 
551   // On Windows, the caller reserves stack space for spilling register arguments
552   const int arg_spill_size = frame::arg_reg_save_area_bytes;
553 
554   // Stack pointer must be 16 bytes aligned for the call
555   _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
556 }
557 
558 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
559   : _masm(masm),
560     _gp_registers(),
561     _opmask_registers(),
562     _xmm_registers(),
563     _spill_size(0),
564     _spill_offset(0) {
565 
566   //
567   // Stack layout after registers have been spilled:
568   //
569   // | ...            | original rsp, 16 bytes aligned
570   // ------------------
571   // | zmm0 high      |
572   // | ...            |
573   // | zmm0 low       | 16 bytes aligned
574   // | ...            |
575   // | ymm1 high      |
576   // | ...            |
577   // | ymm1 low       | 16 bytes aligned
578   // | ...            |
579   // | xmmN high      |
580   // | ...            |
581   // | xmmN low       | 8 bytes aligned
582   // | reg0           | 8 bytes aligned
583   // | reg1           |
584   // | ...            |
585   // | regN           | new rsp, if 16 bytes aligned
586   // | <padding>      | else new rsp, 16 bytes aligned
587   // ------------------
588   //
589 
590   // Figure out what registers to save/restore
591   initialize(stub);
592 
593   // Allocate stack space
594   if (_spill_size > 0) {
595     __ subptr(rsp, _spill_size);
596   }
597 
598   // Save XMM/YMM/ZMM registers
599   for (int i = 0; i < _xmm_registers.length(); i++) {
600     xmm_register_save(_xmm_registers.at(i));
601   }
602 
603   if (xmm_needs_vzeroupper()) {
604     __ vzeroupper();
605   }
606 
607   // Save general purpose registers
608   for (int i = 0; i < _gp_registers.length(); i++) {
609     gp_register_save(_gp_registers.at(i));
610   }
611 
612   // Save opmask registers
613   for (int i = 0; i < _opmask_registers.length(); i++) {
614     opmask_register_save(_opmask_registers.at(i));
615   }
616 }
617 
618 SaveLiveRegisters::~SaveLiveRegisters() {
619   // Restore opmask registers
620   for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
621     opmask_register_restore(_opmask_registers.at(i));
622   }
623 
624   // Restore general purpose registers
625   for (int i = _gp_registers.length() - 1; i >= 0; i--) {
626     gp_register_restore(_gp_registers.at(i));
627   }
628 
629   __ vzeroupper();
630 
631   // Restore XMM/YMM/ZMM registers
632   for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
633     xmm_register_restore(_xmm_registers.at(i));
634   }
635 
636   // Free stack space
637   if (_spill_size > 0) {
638     __ addptr(rsp, _spill_size);
639   }
640 }
641 
642 #endif // COMPILER2