1 /*
2 * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "memory/universe.hpp"
37 #include "nativeInst_s390.hpp"
38 #include "oops/objArrayKlass.hpp"
39 #include "runtime/frame.inline.hpp"
40 #include "runtime/safepointMechanism.inline.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "utilities/macros.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_s390.inline.hpp"
46
47 #define __ _masm->
48
49 #ifndef PRODUCT
50 #undef __
51 #define __ (Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm) : _masm)->
52 #endif
53
54 //------------------------------------------------------------
55
56 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
57 // Not used on ZARCH_64
58 ShouldNotCallThis();
59 return false;
60 }
61
62 LIR_Opr LIR_Assembler::receiverOpr() {
63 return FrameMap::Z_R2_oop_opr;
64 }
65
66 LIR_Opr LIR_Assembler::osrBufferPointer() {
67 return FrameMap::Z_R2_opr;
68 }
69
70 int LIR_Assembler::initial_frame_size_in_bytes() const {
71 return in_bytes(frame_map()->framesize_in_bytes());
72 }
73
74 // Inline cache check: done before the frame is built.
75 // The inline cached class is in Z_inline_cache(Z_R9).
76 // We fetch the class of the receiver and compare it with the cached class.
77 // If they do not match we jump to the slow case.
78 int LIR_Assembler::check_icache() {
79 return __ ic_check(CodeEntryAlignment);
80 }
81
82 void LIR_Assembler::clinit_barrier(ciMethod* method) {
83 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
84
85 Label L_skip_barrier;
86 Register klass = Z_R1_scratch;
87
88 metadata2reg(method->holder()->constant_encoding(), klass);
89 __ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
90
91 __ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
92 __ z_br(klass);
93
94 __ bind(L_skip_barrier);
95 }
96
97 void LIR_Assembler::osr_entry() {
98 // On-stack-replacement entry sequence (interpreter frame layout described in frame_s390.hpp):
99 //
100 // 1. Create a new compiled activation.
101 // 2. Initialize local variables in the compiled activation. The expression stack must be empty
102 // at the osr_bci; it is not initialized.
103 // 3. Jump to the continuation address in compiled code to resume execution.
104
105 // OSR entry point
106 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
107 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
108 ValueStack* entry_state = osr_entry->end()->state();
109 int number_of_locks = entry_state->locks_size();
110
111 // Create a frame for the compiled activation.
112 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
113
114 // OSR buffer is
115 //
116 // locals[nlocals-1..0]
117 // monitors[number_of_locks-1..0]
118 //
119 // Locals is a direct copy of the interpreter frame so in the osr buffer
120 // the first slot in the local array is the last local from the interpreter
121 // and the last slot is local[0] (receiver) from the interpreter
122 //
123 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
124 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
125 // in the interpreter frame (the method lock if a sync method)
126
127 // Initialize monitors in the compiled activation.
128 // I0: pointer to osr buffer
129 //
130 // All other registers are dead at this point and the locals will be
131 // copied into place by code emitted in the IR.
132
133 Register OSR_buf = osrBufferPointer()->as_register();
134 {
135 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
136
137 const int locals_space = BytesPerWord * method() -> max_locals();
138 int monitor_offset = locals_space + (2 * BytesPerWord) * (number_of_locks - 1);
139 bool large_offset = !Immediate::is_simm20(monitor_offset + BytesPerWord) && number_of_locks > 0;
140
141 if (large_offset) {
142 // z_lg can only handle displacement upto 20bit signed binary integer
143 __ z_algfi(OSR_buf, locals_space);
144 monitor_offset -= locals_space;
145 }
146
147 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
148 // the OSR buffer using 2 word entries: first the lock and then
149 // the oop.
150 for (int i = 0; i < number_of_locks; i++) {
151 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
152 // Verify the interpreter's monitor has a non-null object.
153 __ asm_assert_mem8_isnot_zero(slot_offset + 1*BytesPerWord, OSR_buf, "locked object is null", __LINE__);
154 // Copy the lock field into the compiled activation.
155 __ z_lg(Z_R1_scratch, slot_offset + 0, OSR_buf);
156 __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_lock(i));
157 __ z_lg(Z_R1_scratch, slot_offset + 1*BytesPerWord, OSR_buf);
158 __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_object(i));
159 }
160
161 if (large_offset) {
162 __ z_slgfi(OSR_buf, locals_space);
163 }
164 }
165 }
166
167 // --------------------------------------------------------------------------------------------
168
169 address LIR_Assembler::emit_call_c(address a) {
170 __ align_call_far_patchable(__ pc());
171 address call_addr = __ call_c_opt(a);
172 if (call_addr == nullptr) {
173 bailout("const section overflow");
174 }
175 return call_addr;
176 }
177
178 int LIR_Assembler::emit_exception_handler() {
179 // Generate code for exception handler.
180 address handler_base = __ start_a_stub(exception_handler_size());
181 if (handler_base == nullptr) {
182 // Not enough space left for the handler.
183 bailout("exception handler overflow");
184 return -1;
185 }
186
187 int offset = code_offset();
188
189 address a = Runtime1::entry_for (C1StubId::handle_exception_from_callee_id);
190 address call_addr = emit_call_c(a);
191 CHECK_BAILOUT_(-1);
192 __ should_not_reach_here();
193 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
194 __ end_a_stub();
195
196 return offset;
197 }
198
199 // Emit the code to remove the frame from the stack in the exception
200 // unwind path.
201 int LIR_Assembler::emit_unwind_handler() {
202 #ifndef PRODUCT
203 if (CommentedAssembly) {
204 _masm->block_comment("Unwind handler");
205 }
206 #endif
207
208 int offset = code_offset();
209 Register exception_oop_callee_saved = Z_R10; // Z_R10 is callee-saved.
210 Register Rtmp1 = Z_R11;
211 Register Rtmp2 = Z_R12;
212
213 // Fetch the exception from TLS and clear out exception related thread state.
214 Address exc_oop_addr = Address(Z_thread, JavaThread::exception_oop_offset());
215 Address exc_pc_addr = Address(Z_thread, JavaThread::exception_pc_offset());
216 __ z_lg(Z_EXC_OOP, exc_oop_addr);
217 __ clear_mem(exc_oop_addr, sizeof(oop));
218 __ clear_mem(exc_pc_addr, sizeof(intptr_t));
219
220 __ bind(_unwind_handler_entry);
221 __ verify_not_null_oop(Z_EXC_OOP);
222 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
223 __ lgr_if_needed(exception_oop_callee_saved, Z_EXC_OOP); // Preserve the exception.
224 }
225
226 // Perform needed unlocking.
227 MonitorExitStub* stub = nullptr;
228 if (method()->is_synchronized()) {
229 // C1StubId::monitorexit_id expects lock address in Z_R1_scratch.
230 LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
231 monitor_address(0, lock);
232 stub = new MonitorExitStub(lock, true, 0);
233 if (LockingMode == LM_MONITOR) {
234 __ branch_optimized(Assembler::bcondAlways, *stub->entry());
235 } else {
236 __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
237 }
238 __ bind(*stub->continuation());
239 }
240
241 if (compilation()->env()->dtrace_method_probes()) {
242 ShouldNotReachHere(); // Not supported.
243 #if 0
244 __ mov(rdi, r15_thread);
245 __ mov_metadata(rsi, method()->constant_encoding());
246 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
247 #endif
248 }
249
250 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
251 __ lgr_if_needed(Z_EXC_OOP, exception_oop_callee_saved); // Restore the exception.
252 }
253
254 // Remove the activation and dispatch to the unwind handler.
255 __ pop_frame();
256 __ z_lg(Z_EXC_PC, _z_common_abi(return_pc), Z_SP);
257
258 // Z_EXC_OOP: exception oop
259 // Z_EXC_PC: exception pc
260
261 // Dispatch to the unwind logic.
262 __ load_const_optimized(Z_R5, Runtime1::entry_for (C1StubId::unwind_exception_id));
263 __ z_br(Z_R5);
264
265 // Emit the slow path assembly.
266 if (stub != nullptr) {
267 stub->emit_code(this);
268 }
269
270 return offset;
271 }
272
273 int LIR_Assembler::emit_deopt_handler() {
274 // Generate code for exception handler.
275 address handler_base = __ start_a_stub(deopt_handler_size());
276 if (handler_base == nullptr) {
277 // Not enough space left for the handler.
278 bailout("deopt handler overflow");
279 return -1;
280 } int offset = code_offset();
281 // Size must be constant (see HandlerImpl::emit_deopt_handler).
282 __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack());
283 __ call(Z_R1_scratch);
284 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
285 __ end_a_stub();
286
287 return offset;
288 }
289
290 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
291 if (o == nullptr) {
292 __ clear_reg(reg, true/*64bit*/, false/*set cc*/); // Must not kill cc set by cmove.
293 } else {
294 AddressLiteral a = __ allocate_oop_address(o);
295 bool success = __ load_oop_from_toc(reg, a, reg);
296 if (!success) {
297 bailout("const section overflow");
298 }
299 }
300 }
301
302 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
303 // Allocate a new index in table to hold the object once it's been patched.
304 int oop_index = __ oop_recorder()->allocate_oop_index(nullptr);
305 PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
306
307 AddressLiteral addrlit((intptr_t)0, oop_Relocation::spec(oop_index));
308 assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
309 // The null will be dynamically patched later so the sequence to
310 // load the address literal must not be optimized.
311 __ load_const(reg, addrlit);
312
313 patching_epilog(patch, lir_patch_normal, reg, info);
314 }
315
316 void LIR_Assembler::metadata2reg(Metadata* md, Register reg) {
317 bool success = __ set_metadata_constant(md, reg);
318 if (!success) {
319 bailout("const section overflow");
320 return;
321 }
322 }
323
324 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
325 // Allocate a new index in table to hold the klass once it's been patched.
326 int index = __ oop_recorder()->allocate_metadata_index(nullptr);
327 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
328 AddressLiteral addrlit((intptr_t)0, metadata_Relocation::spec(index));
329 assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
330 // The null will be dynamically patched later so the sequence to
331 // load the address literal must not be optimized.
332 __ load_const(reg, addrlit);
333
334 patching_epilog(patch, lir_patch_normal, reg, info);
335 }
336
337 void LIR_Assembler::emit_op3(LIR_Op3* op) {
338 switch (op->code()) {
339 case lir_idiv:
340 case lir_irem:
341 arithmetic_idiv(op->code(),
342 op->in_opr1(),
343 op->in_opr2(),
344 op->in_opr3(),
345 op->result_opr(),
346 op->info());
347 break;
348 case lir_fmad: {
349 const FloatRegister opr1 = op->in_opr1()->as_double_reg(),
350 opr2 = op->in_opr2()->as_double_reg(),
351 opr3 = op->in_opr3()->as_double_reg(),
352 res = op->result_opr()->as_double_reg();
353 __ z_madbr(opr3, opr1, opr2);
354 if (res != opr3) { __ z_ldr(res, opr3); }
355 } break;
356 case lir_fmaf: {
357 const FloatRegister opr1 = op->in_opr1()->as_float_reg(),
358 opr2 = op->in_opr2()->as_float_reg(),
359 opr3 = op->in_opr3()->as_float_reg(),
360 res = op->result_opr()->as_float_reg();
361 __ z_maebr(opr3, opr1, opr2);
362 if (res != opr3) { __ z_ler(res, opr3); }
363 } break;
364 default: ShouldNotReachHere(); break;
365 }
366 }
367
368
369 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
370 #ifdef ASSERT
371 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
372 if (op->block() != nullptr) { _branch_target_blocks.append(op->block()); }
373 if (op->ublock() != nullptr) { _branch_target_blocks.append(op->ublock()); }
374 #endif
375
376 if (op->cond() == lir_cond_always) {
377 if (op->info() != nullptr) { add_debug_info_for_branch(op->info()); }
378 __ branch_optimized(Assembler::bcondAlways, *(op->label()));
379 } else {
380 Assembler::branch_condition acond = Assembler::bcondZero;
381 if (op->code() == lir_cond_float_branch) {
382 assert(op->ublock() != nullptr, "must have unordered successor");
383 __ branch_optimized(Assembler::bcondNotOrdered, *(op->ublock()->label()));
384 }
385 switch (op->cond()) {
386 case lir_cond_equal: acond = Assembler::bcondEqual; break;
387 case lir_cond_notEqual: acond = Assembler::bcondNotEqual; break;
388 case lir_cond_less: acond = Assembler::bcondLow; break;
389 case lir_cond_lessEqual: acond = Assembler::bcondNotHigh; break;
390 case lir_cond_greaterEqual: acond = Assembler::bcondNotLow; break;
391 case lir_cond_greater: acond = Assembler::bcondHigh; break;
392 case lir_cond_belowEqual: acond = Assembler::bcondNotHigh; break;
393 case lir_cond_aboveEqual: acond = Assembler::bcondNotLow; break;
394 default: ShouldNotReachHere();
395 }
396 __ branch_optimized(acond,*(op->label()));
397 }
398 }
399
400
401 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
402 LIR_Opr src = op->in_opr();
403 LIR_Opr dest = op->result_opr();
404
405 switch (op->bytecode()) {
406 case Bytecodes::_i2l:
407 __ move_reg_if_needed(dest->as_register_lo(), T_LONG, src->as_register(), T_INT);
408 break;
409
410 case Bytecodes::_l2i:
411 __ move_reg_if_needed(dest->as_register(), T_INT, src->as_register_lo(), T_LONG);
412 break;
413
414 case Bytecodes::_i2b:
415 __ move_reg_if_needed(dest->as_register(), T_BYTE, src->as_register(), T_INT);
416 break;
417
418 case Bytecodes::_i2c:
419 __ move_reg_if_needed(dest->as_register(), T_CHAR, src->as_register(), T_INT);
420 break;
421
422 case Bytecodes::_i2s:
423 __ move_reg_if_needed(dest->as_register(), T_SHORT, src->as_register(), T_INT);
424 break;
425
426 case Bytecodes::_f2d:
427 assert(dest->is_double_fpu(), "check");
428 __ move_freg_if_needed(dest->as_double_reg(), T_DOUBLE, src->as_float_reg(), T_FLOAT);
429 break;
430
431 case Bytecodes::_d2f:
432 assert(dest->is_single_fpu(), "check");
433 __ move_freg_if_needed(dest->as_float_reg(), T_FLOAT, src->as_double_reg(), T_DOUBLE);
434 break;
435
436 case Bytecodes::_i2f:
437 __ z_cefbr(dest->as_float_reg(), src->as_register());
438 break;
439
440 case Bytecodes::_i2d:
441 __ z_cdfbr(dest->as_double_reg(), src->as_register());
442 break;
443
444 case Bytecodes::_l2f:
445 __ z_cegbr(dest->as_float_reg(), src->as_register_lo());
446 break;
447 case Bytecodes::_l2d:
448 __ z_cdgbr(dest->as_double_reg(), src->as_register_lo());
449 break;
450
451 case Bytecodes::_f2i:
452 case Bytecodes::_f2l: {
453 Label done;
454 FloatRegister Rsrc = src->as_float_reg();
455 Register Rdst = (op->bytecode() == Bytecodes::_f2i ? dest->as_register() : dest->as_register_lo());
456 __ clear_reg(Rdst, true, false);
457 __ z_cebr(Rsrc, Rsrc);
458 __ z_brno(done); // NaN -> 0
459 if (op->bytecode() == Bytecodes::_f2i) {
460 __ z_cfebr(Rdst, Rsrc, Assembler::to_zero);
461 } else { // op->bytecode() == Bytecodes::_f2l
462 __ z_cgebr(Rdst, Rsrc, Assembler::to_zero);
463 }
464 __ bind(done);
465 }
466 break;
467
468 case Bytecodes::_d2i:
469 case Bytecodes::_d2l: {
470 Label done;
471 FloatRegister Rsrc = src->as_double_reg();
472 Register Rdst = (op->bytecode() == Bytecodes::_d2i ? dest->as_register() : dest->as_register_lo());
473 __ clear_reg(Rdst, true, false); // Don't set CC.
474 __ z_cdbr(Rsrc, Rsrc);
475 __ z_brno(done); // NaN -> 0
476 if (op->bytecode() == Bytecodes::_d2i) {
477 __ z_cfdbr(Rdst, Rsrc, Assembler::to_zero);
478 } else { // Bytecodes::_d2l
479 __ z_cgdbr(Rdst, Rsrc, Assembler::to_zero);
480 }
481 __ bind(done);
482 }
483 break;
484
485 default: ShouldNotReachHere();
486 }
487 }
488
489 void LIR_Assembler::align_call(LIR_Code code) {
490 // End of call instruction must be 4 byte aligned.
491 int offset = __ offset();
492 switch (code) {
493 case lir_icvirtual_call:
494 offset += MacroAssembler::load_const_from_toc_size();
495 // no break
496 case lir_static_call:
497 case lir_optvirtual_call:
498 case lir_dynamic_call:
499 offset += NativeCall::call_far_pcrelative_displacement_offset;
500 break;
501 default: ShouldNotReachHere();
502 }
503 if ((offset & (NativeCall::call_far_pcrelative_displacement_alignment-1)) != 0) {
504 __ nop();
505 }
506 }
507
508 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
509 assert((__ offset() + NativeCall::call_far_pcrelative_displacement_offset) % NativeCall::call_far_pcrelative_displacement_alignment == 0,
510 "must be aligned (offset=%d)", __ offset());
511 assert(rtype == relocInfo::none ||
512 rtype == relocInfo::opt_virtual_call_type ||
513 rtype == relocInfo::static_call_type, "unexpected rtype");
514 // Prepend each BRASL with a nop.
515 __ relocate(rtype);
516 __ z_nop();
517 __ z_brasl(Z_R14, op->addr());
518 add_call_info(code_offset(), op->info());
519 }
520
521 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
522 address virtual_call_oop_addr = nullptr;
523 AddressLiteral empty_ic((address) Universe::non_oop_word());
524 virtual_call_oop_addr = __ pc();
525 bool success = __ load_const_from_toc(Z_inline_cache, empty_ic);
526 if (!success) {
527 bailout("const section overflow");
528 return;
529 }
530
531 // CALL to fixup routine. Fixup routine uses ScopeDesc info
532 // to determine who we intended to call.
533 __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr));
534 call(op, relocInfo::none);
535 }
536
537 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
538 if (from_reg != to_reg) __ z_lgr(to_reg, from_reg);
539 }
540
541 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
542 assert(src->is_constant(), "should not call otherwise");
543 assert(dest->is_stack(), "should not call otherwise");
544 LIR_Const* c = src->as_constant_ptr();
545
546 unsigned int lmem = 0;
547 unsigned int lcon = 0;
548 int64_t cbits = 0;
549 Address dest_addr;
550 switch (c->type()) {
551 case T_INT: // fall through
552 case T_FLOAT:
553 dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
554 lmem = 4; lcon = 4; cbits = c->as_jint_bits();
555 break;
556
557 case T_ADDRESS:
558 dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
559 lmem = 8; lcon = 4; cbits = c->as_jint_bits();
560 break;
561
562 case T_OBJECT:
563 dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
564 if (c->as_jobject() == nullptr) {
565 __ store_const(dest_addr, (int64_t)NULL_WORD, 8, 8);
566 } else {
567 jobject2reg(c->as_jobject(), Z_R1_scratch);
568 __ reg2mem_opt(Z_R1_scratch, dest_addr, true);
569 }
570 return;
571
572 case T_LONG: // fall through
573 case T_DOUBLE:
574 dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
575 lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
576 break;
577
578 default:
579 ShouldNotReachHere();
580 }
581
582 __ store_const(dest_addr, cbits, lmem, lcon);
583 }
584
585 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
586 assert(src->is_constant(), "should not call otherwise");
587 assert(dest->is_address(), "should not call otherwise");
588
589 LIR_Const* c = src->as_constant_ptr();
590 Address addr = as_Address(dest->as_address_ptr());
591
592 int store_offset = -1;
593
594 if (dest->as_address_ptr()->index()->is_valid()) {
595 switch (type) {
596 case T_INT: // fall through
597 case T_FLOAT:
598 __ load_const_optimized(Z_R0_scratch, c->as_jint_bits());
599 store_offset = __ offset();
600 if (Immediate::is_uimm12(addr.disp())) {
601 __ z_st(Z_R0_scratch, addr);
602 } else {
603 __ z_sty(Z_R0_scratch, addr);
604 }
605 break;
606
607 case T_ADDRESS:
608 __ load_const_optimized(Z_R1_scratch, c->as_jint_bits());
609 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
610 break;
611
612 case T_OBJECT: // fall through
613 case T_ARRAY:
614 if (c->as_jobject() == nullptr) {
615 if (UseCompressedOops && !wide) {
616 __ clear_reg(Z_R1_scratch, false);
617 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
618 } else {
619 __ clear_reg(Z_R1_scratch, true);
620 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
621 }
622 } else {
623 jobject2reg(c->as_jobject(), Z_R1_scratch);
624 if (UseCompressedOops && !wide) {
625 __ encode_heap_oop(Z_R1_scratch);
626 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
627 } else {
628 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
629 }
630 }
631 assert(store_offset >= 0, "check");
632 break;
633
634 case T_LONG: // fall through
635 case T_DOUBLE:
636 __ load_const_optimized(Z_R1_scratch, (int64_t)(c->as_jlong_bits()));
637 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
638 break;
639
640 case T_BOOLEAN: // fall through
641 case T_BYTE:
642 __ load_const_optimized(Z_R0_scratch, (int8_t)(c->as_jint()));
643 store_offset = __ offset();
644 if (Immediate::is_uimm12(addr.disp())) {
645 __ z_stc(Z_R0_scratch, addr);
646 } else {
647 __ z_stcy(Z_R0_scratch, addr);
648 }
649 break;
650
651 case T_CHAR: // fall through
652 case T_SHORT:
653 __ load_const_optimized(Z_R0_scratch, (int16_t)(c->as_jint()));
654 store_offset = __ offset();
655 if (Immediate::is_uimm12(addr.disp())) {
656 __ z_sth(Z_R0_scratch, addr);
657 } else {
658 __ z_sthy(Z_R0_scratch, addr);
659 }
660 break;
661
662 default:
663 ShouldNotReachHere();
664 }
665
666 } else { // no index
667
668 unsigned int lmem = 0;
669 unsigned int lcon = 0;
670 int64_t cbits = 0;
671
672 switch (type) {
673 case T_INT: // fall through
674 case T_FLOAT:
675 lmem = 4; lcon = 4; cbits = c->as_jint_bits();
676 break;
677
678 case T_ADDRESS:
679 lmem = 8; lcon = 4; cbits = c->as_jint_bits();
680 break;
681
682 case T_OBJECT: // fall through
683 case T_ARRAY:
684 if (c->as_jobject() == nullptr) {
685 if (UseCompressedOops && !wide) {
686 store_offset = __ store_const(addr, (int32_t)NULL_WORD, 4, 4);
687 } else {
688 store_offset = __ store_const(addr, (int64_t)NULL_WORD, 8, 8);
689 }
690 } else {
691 jobject2reg(c->as_jobject(), Z_R1_scratch);
692 if (UseCompressedOops && !wide) {
693 __ encode_heap_oop(Z_R1_scratch);
694 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
695 } else {
696 store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
697 }
698 }
699 assert(store_offset >= 0, "check");
700 break;
701
702 case T_LONG: // fall through
703 case T_DOUBLE:
704 lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
705 break;
706
707 case T_BOOLEAN: // fall through
708 case T_BYTE:
709 lmem = 1; lcon = 1; cbits = (int8_t)(c->as_jint());
710 break;
711
712 case T_CHAR: // fall through
713 case T_SHORT:
714 lmem = 2; lcon = 2; cbits = (int16_t)(c->as_jint());
715 break;
716
717 default:
718 ShouldNotReachHere();
719 }
720
721 if (store_offset == -1) {
722 store_offset = __ store_const(addr, cbits, lmem, lcon);
723 assert(store_offset >= 0, "check");
724 }
725 }
726
727 if (info != nullptr) {
728 add_debug_info_for_null_check(store_offset, info);
729 }
730 }
731
732 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
733 assert(src->is_constant(), "should not call otherwise");
734 assert(dest->is_register(), "should not call otherwise");
735 LIR_Const* c = src->as_constant_ptr();
736
737 switch (c->type()) {
738 case T_INT: {
739 assert(patch_code == lir_patch_none, "no patching handled here");
740 __ load_const_optimized(dest->as_register(), c->as_jint());
741 break;
742 }
743
744 case T_ADDRESS: {
745 assert(patch_code == lir_patch_none, "no patching handled here");
746 __ load_const_optimized(dest->as_register(), c->as_jint());
747 break;
748 }
749
750 case T_LONG: {
751 assert(patch_code == lir_patch_none, "no patching handled here");
752 __ load_const_optimized(dest->as_register_lo(), (intptr_t)c->as_jlong());
753 break;
754 }
755
756 case T_OBJECT: {
757 if (patch_code != lir_patch_none) {
758 jobject2reg_with_patching(dest->as_register(), info);
759 } else {
760 jobject2reg(c->as_jobject(), dest->as_register());
761 }
762 break;
763 }
764
765 case T_METADATA: {
766 if (patch_code != lir_patch_none) {
767 klass2reg_with_patching(dest->as_register(), info);
768 } else {
769 metadata2reg(c->as_metadata(), dest->as_register());
770 }
771 break;
772 }
773
774 case T_FLOAT: {
775 Register toc_reg = Z_R1_scratch;
776 __ load_toc(toc_reg);
777 address const_addr = __ float_constant(c->as_jfloat());
778 if (const_addr == nullptr) {
779 bailout("const section overflow");
780 break;
781 }
782 int displ = const_addr - _masm->code()->consts()->start();
783 if (dest->is_single_fpu()) {
784 __ z_ley(dest->as_float_reg(), displ, toc_reg);
785 } else {
786 assert(dest->is_single_cpu(), "Must be a cpu register.");
787 __ z_ly(dest->as_register(), displ, toc_reg);
788 }
789 }
790 break;
791
792 case T_DOUBLE: {
793 Register toc_reg = Z_R1_scratch;
794 __ load_toc(toc_reg);
795 address const_addr = __ double_constant(c->as_jdouble());
796 if (const_addr == nullptr) {
797 bailout("const section overflow");
798 break;
799 }
800 int displ = const_addr - _masm->code()->consts()->start();
801 if (dest->is_double_fpu()) {
802 __ z_ldy(dest->as_double_reg(), displ, toc_reg);
803 } else {
804 assert(dest->is_double_cpu(), "Must be a long register.");
805 __ z_lg(dest->as_register_lo(), displ, toc_reg);
806 }
807 }
808 break;
809
810 default:
811 ShouldNotReachHere();
812 }
813 }
814
815 Address LIR_Assembler::as_Address(LIR_Address* addr) {
816 if (addr->base()->is_illegal()) {
817 Unimplemented();
818 }
819
820 Register base = addr->base()->as_pointer_register();
821
822 if (addr->index()->is_illegal()) {
823 return Address(base, addr->disp());
824 } else if (addr->index()->is_cpu_register()) {
825 Register index = addr->index()->as_pointer_register();
826 return Address(base, index, addr->disp());
827 } else if (addr->index()->is_constant()) {
828 intptr_t addr_offset = addr->index()->as_constant_ptr()->as_jint() + addr->disp();
829 return Address(base, addr_offset);
830 } else {
831 ShouldNotReachHere();
832 return Address();
833 }
834 }
835
836 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
837 switch (type) {
838 case T_INT:
839 case T_FLOAT: {
840 Register tmp = Z_R1_scratch;
841 Address from = frame_map()->address_for_slot(src->single_stack_ix());
842 Address to = frame_map()->address_for_slot(dest->single_stack_ix());
843 __ mem2reg_opt(tmp, from, false);
844 __ reg2mem_opt(tmp, to, false);
845 break;
846 }
847 case T_ADDRESS:
848 case T_OBJECT: {
849 Register tmp = Z_R1_scratch;
850 Address from = frame_map()->address_for_slot(src->single_stack_ix());
851 Address to = frame_map()->address_for_slot(dest->single_stack_ix());
852 __ mem2reg_opt(tmp, from, true);
853 __ reg2mem_opt(tmp, to, true);
854 break;
855 }
856 case T_LONG:
857 case T_DOUBLE: {
858 Register tmp = Z_R1_scratch;
859 Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
860 Address to = frame_map()->address_for_double_slot(dest->double_stack_ix());
861 __ mem2reg_opt(tmp, from, true);
862 __ reg2mem_opt(tmp, to, true);
863 break;
864 }
865
866 default:
867 ShouldNotReachHere();
868 }
869 }
870
871 // 4-byte accesses only! Don't use it to access 8 bytes!
872 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
873 ShouldNotCallThis();
874 return Address(); // unused
875 }
876
877 // 4-byte accesses only! Don't use it to access 8 bytes!
878 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
879 ShouldNotCallThis();
880 return Address(); // unused
881 }
882
883 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code,
884 CodeEmitInfo* info, bool wide) {
885
886 assert(type != T_METADATA, "load of metadata ptr not supported");
887 LIR_Address* addr = src_opr->as_address_ptr();
888 LIR_Opr to_reg = dest;
889
890 Register src = addr->base()->as_pointer_register();
891 Register disp_reg = Z_R0;
892 int disp_value = addr->disp();
893 bool needs_patching = (patch_code != lir_patch_none);
894
895 if (addr->base()->type() == T_OBJECT) {
896 __ verify_oop(src, FILE_AND_LINE);
897 }
898
899 PatchingStub* patch = nullptr;
900 if (needs_patching) {
901 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
902 assert(!to_reg->is_double_cpu() ||
903 patch_code == lir_patch_none ||
904 patch_code == lir_patch_normal, "patching doesn't match register");
905 }
906
907 if (addr->index()->is_illegal()) {
908 if (!Immediate::is_simm20(disp_value)) {
909 if (needs_patching) {
910 __ load_const(Z_R1_scratch, (intptr_t)0);
911 } else {
912 __ load_const_optimized(Z_R1_scratch, disp_value);
913 }
914 disp_reg = Z_R1_scratch;
915 disp_value = 0;
916 }
917 } else {
918 if (!Immediate::is_simm20(disp_value)) {
919 __ load_const_optimized(Z_R1_scratch, disp_value);
920 __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
921 disp_reg = Z_R1_scratch;
922 disp_value = 0;
923 }
924 disp_reg = addr->index()->as_pointer_register();
925 }
926
927 // Remember the offset of the load. The patching_epilog must be done
928 // before the call to add_debug_info, otherwise the PcDescs don't get
929 // entered in increasing order.
930 int offset = code_offset();
931
932 assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
933
934 bool short_disp = Immediate::is_uimm12(disp_value);
935
936 switch (type) {
937 case T_BOOLEAN: // fall through
938 case T_BYTE : __ z_lb(dest->as_register(), disp_value, disp_reg, src); break;
939 case T_CHAR : __ z_llgh(dest->as_register(), disp_value, disp_reg, src); break;
940 case T_SHORT :
941 if (short_disp) {
942 __ z_lh(dest->as_register(), disp_value, disp_reg, src);
943 } else {
944 __ z_lhy(dest->as_register(), disp_value, disp_reg, src);
945 }
946 break;
947 case T_INT :
948 if (short_disp) {
949 __ z_l(dest->as_register(), disp_value, disp_reg, src);
950 } else {
951 __ z_ly(dest->as_register(), disp_value, disp_reg, src);
952 }
953 break;
954 case T_ADDRESS:
955 __ z_lg(dest->as_register(), disp_value, disp_reg, src);
956 break;
957 case T_ARRAY : // fall through
958 case T_OBJECT:
959 {
960 if (UseCompressedOops && !wide) {
961 __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
962 __ oop_decoder(dest->as_register(), dest->as_register(), true);
963 } else {
964 __ z_lg(dest->as_register(), disp_value, disp_reg, src);
965 }
966 __ verify_oop(dest->as_register(), FILE_AND_LINE);
967 break;
968 }
969 case T_FLOAT:
970 if (short_disp) {
971 __ z_le(dest->as_float_reg(), disp_value, disp_reg, src);
972 } else {
973 __ z_ley(dest->as_float_reg(), disp_value, disp_reg, src);
974 }
975 break;
976 case T_DOUBLE:
977 if (short_disp) {
978 __ z_ld(dest->as_double_reg(), disp_value, disp_reg, src);
979 } else {
980 __ z_ldy(dest->as_double_reg(), disp_value, disp_reg, src);
981 }
982 break;
983 case T_LONG : __ z_lg(dest->as_register_lo(), disp_value, disp_reg, src); break;
984 default : ShouldNotReachHere();
985 }
986
987 if (patch != nullptr) {
988 patching_epilog(patch, patch_code, src, info);
989 }
990 if (info != nullptr) add_debug_info_for_null_check(offset, info);
991 }
992
993 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
994 assert(src->is_stack(), "should not call otherwise");
995 assert(dest->is_register(), "should not call otherwise");
996
997 if (dest->is_single_cpu()) {
998 if (is_reference_type(type)) {
999 __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
1000 __ verify_oop(dest->as_register(), FILE_AND_LINE);
1001 } else if (type == T_METADATA || type == T_ADDRESS) {
1002 __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
1003 } else {
1004 __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), false);
1005 }
1006 } else if (dest->is_double_cpu()) {
1007 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix());
1008 __ mem2reg_opt(dest->as_register_lo(), src_addr_LO, true);
1009 } else if (dest->is_single_fpu()) {
1010 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1011 __ mem2freg_opt(dest->as_float_reg(), src_addr, false);
1012 } else if (dest->is_double_fpu()) {
1013 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1014 __ mem2freg_opt(dest->as_double_reg(), src_addr, true);
1015 } else {
1016 ShouldNotReachHere();
1017 }
1018 }
1019
1020 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1021 assert(src->is_register(), "should not call otherwise");
1022 assert(dest->is_stack(), "should not call otherwise");
1023
1024 if (src->is_single_cpu()) {
1025 const Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
1026 if (is_reference_type(type)) {
1027 __ verify_oop(src->as_register(), FILE_AND_LINE);
1028 __ reg2mem_opt(src->as_register(), dst, true);
1029 } else if (type == T_METADATA || type == T_ADDRESS) {
1030 __ reg2mem_opt(src->as_register(), dst, true);
1031 } else {
1032 __ reg2mem_opt(src->as_register(), dst, false);
1033 }
1034 } else if (src->is_double_cpu()) {
1035 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix());
1036 __ reg2mem_opt(src->as_register_lo(), dstLO, true);
1037 } else if (src->is_single_fpu()) {
1038 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
1039 __ freg2mem_opt(src->as_float_reg(), dst_addr, false);
1040 } else if (src->is_double_fpu()) {
1041 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
1042 __ freg2mem_opt(src->as_double_reg(), dst_addr, true);
1043 } else {
1044 ShouldNotReachHere();
1045 }
1046 }
1047
1048 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1049 if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1050 if (from_reg->is_double_fpu()) {
1051 // double to double moves
1052 assert(to_reg->is_double_fpu(), "should match");
1053 __ z_ldr(to_reg->as_double_reg(), from_reg->as_double_reg());
1054 } else {
1055 // float to float moves
1056 assert(to_reg->is_single_fpu(), "should match");
1057 __ z_ler(to_reg->as_float_reg(), from_reg->as_float_reg());
1058 }
1059 } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1060 if (from_reg->is_double_cpu()) {
1061 __ z_lgr(to_reg->as_pointer_register(), from_reg->as_pointer_register());
1062 } else if (to_reg->is_double_cpu()) {
1063 // int to int moves
1064 __ z_lgr(to_reg->as_register_lo(), from_reg->as_register());
1065 } else {
1066 // int to int moves
1067 __ z_lgr(to_reg->as_register(), from_reg->as_register());
1068 }
1069 } else {
1070 ShouldNotReachHere();
1071 }
1072 if (is_reference_type(to_reg->type())) {
1073 __ verify_oop(to_reg->as_register(), FILE_AND_LINE);
1074 }
1075 }
1076
1077 void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
1078 LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1079 bool wide) {
1080 assert(type != T_METADATA, "store of metadata ptr not supported");
1081 LIR_Address* addr = dest_opr->as_address_ptr();
1082
1083 Register dest = addr->base()->as_pointer_register();
1084 Register disp_reg = Z_R0;
1085 int disp_value = addr->disp();
1086 bool needs_patching = (patch_code != lir_patch_none);
1087
1088 if (addr->base()->is_oop_register()) {
1089 __ verify_oop(dest, FILE_AND_LINE);
1090 }
1091
1092 PatchingStub* patch = nullptr;
1093 if (needs_patching) {
1094 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1095 assert(!from->is_double_cpu() ||
1096 patch_code == lir_patch_none ||
1097 patch_code == lir_patch_normal, "patching doesn't match register");
1098 }
1099
1100 assert(!needs_patching || (!Immediate::is_simm20(disp_value) && addr->index()->is_illegal()), "assumption");
1101 if (addr->index()->is_illegal()) {
1102 if (!Immediate::is_simm20(disp_value)) {
1103 if (needs_patching) {
1104 __ load_const(Z_R1_scratch, (intptr_t)0);
1105 } else {
1106 __ load_const_optimized(Z_R1_scratch, disp_value);
1107 }
1108 disp_reg = Z_R1_scratch;
1109 disp_value = 0;
1110 }
1111 } else {
1112 if (!Immediate::is_simm20(disp_value)) {
1113 __ load_const_optimized(Z_R1_scratch, disp_value);
1114 __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
1115 disp_reg = Z_R1_scratch;
1116 disp_value = 0;
1117 }
1118 disp_reg = addr->index()->as_pointer_register();
1119 }
1120
1121 assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
1122
1123 if (is_reference_type(type)) {
1124 __ verify_oop(from->as_register(), FILE_AND_LINE);
1125 }
1126
1127 bool short_disp = Immediate::is_uimm12(disp_value);
1128
1129 // Remember the offset of the store. The patching_epilog must be done
1130 // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1131 // entered in increasing order.
1132 int offset = code_offset();
1133 switch (type) {
1134 case T_BOOLEAN: // fall through
1135 case T_BYTE :
1136 if (short_disp) {
1137 __ z_stc(from->as_register(), disp_value, disp_reg, dest);
1138 } else {
1139 __ z_stcy(from->as_register(), disp_value, disp_reg, dest);
1140 }
1141 break;
1142 case T_CHAR : // fall through
1143 case T_SHORT :
1144 if (short_disp) {
1145 __ z_sth(from->as_register(), disp_value, disp_reg, dest);
1146 } else {
1147 __ z_sthy(from->as_register(), disp_value, disp_reg, dest);
1148 }
1149 break;
1150 case T_INT :
1151 if (short_disp) {
1152 __ z_st(from->as_register(), disp_value, disp_reg, dest);
1153 } else {
1154 __ z_sty(from->as_register(), disp_value, disp_reg, dest);
1155 }
1156 break;
1157 case T_LONG : __ z_stg(from->as_register_lo(), disp_value, disp_reg, dest); break;
1158 case T_ADDRESS: __ z_stg(from->as_register(), disp_value, disp_reg, dest); break;
1159 break;
1160 case T_ARRAY : // fall through
1161 case T_OBJECT:
1162 {
1163 if (UseCompressedOops && !wide) {
1164 Register compressed_src = Z_R14;
1165 __ oop_encoder(compressed_src, from->as_register(), true, (disp_reg != Z_R1) ? Z_R1 : Z_R0, -1, true);
1166 offset = code_offset();
1167 if (short_disp) {
1168 __ z_st(compressed_src, disp_value, disp_reg, dest);
1169 } else {
1170 __ z_sty(compressed_src, disp_value, disp_reg, dest);
1171 }
1172 } else {
1173 __ z_stg(from->as_register(), disp_value, disp_reg, dest);
1174 }
1175 break;
1176 }
1177 case T_FLOAT :
1178 if (short_disp) {
1179 __ z_ste(from->as_float_reg(), disp_value, disp_reg, dest);
1180 } else {
1181 __ z_stey(from->as_float_reg(), disp_value, disp_reg, dest);
1182 }
1183 break;
1184 case T_DOUBLE:
1185 if (short_disp) {
1186 __ z_std(from->as_double_reg(), disp_value, disp_reg, dest);
1187 } else {
1188 __ z_stdy(from->as_double_reg(), disp_value, disp_reg, dest);
1189 }
1190 break;
1191 default: ShouldNotReachHere();
1192 }
1193
1194 if (patch != nullptr) {
1195 patching_epilog(patch, patch_code, dest, info);
1196 }
1197
1198 if (info != nullptr) add_debug_info_for_null_check(offset, info);
1199 }
1200
1201
1202 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
1203 assert(result->is_illegal() ||
1204 (result->is_single_cpu() && result->as_register() == Z_R2) ||
1205 (result->is_double_cpu() && result->as_register_lo() == Z_R2) ||
1206 (result->is_single_fpu() && result->as_float_reg() == Z_F0) ||
1207 (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention");
1208
1209 __ z_lg(Z_R1_scratch, Address(Z_thread, JavaThread::polling_page_offset()));
1210
1211 // Pop the frame before the safepoint code.
1212 __ pop_frame_restore_retPC(initial_frame_size_in_bytes());
1213
1214 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
1215 __ reserved_stack_check(Z_R14);
1216 }
1217
1218 // We need to mark the code position where the load from the safepoint
1219 // polling page was emitted as relocInfo::poll_return_type here.
1220 __ relocate(relocInfo::poll_return_type);
1221 __ load_from_polling_page(Z_R1_scratch);
1222
1223 __ z_br(Z_R14); // Return to caller.
1224 }
1225
1226 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1227 const Register poll_addr = tmp->as_register_lo();
1228 __ z_lg(poll_addr, Address(Z_thread, JavaThread::polling_page_offset()));
1229 guarantee(info != nullptr, "Shouldn't be null");
1230 add_debug_info_for_branch(info);
1231 int offset = __ offset();
1232 __ relocate(relocInfo::poll_type);
1233 __ load_from_polling_page(poll_addr);
1234 return offset;
1235 }
1236
1237 void LIR_Assembler::emit_static_call_stub() {
1238
1239 // Stub is fixed up when the corresponding call is converted from calling
1240 // compiled code to calling interpreted code.
1241
1242 address call_pc = __ pc();
1243 address stub = __ start_a_stub(call_stub_size());
1244 if (stub == nullptr) {
1245 bailout("static call stub overflow");
1246 return;
1247 }
1248
1249 int start = __ offset();
1250
1251 __ relocate(static_stub_Relocation::spec(call_pc));
1252
1253 // See also Matcher::interpreter_method_reg().
1254 AddressLiteral meta = __ allocate_metadata_address(nullptr);
1255 bool success = __ load_const_from_toc(Z_method, meta);
1256
1257 __ set_inst_mark();
1258 AddressLiteral a((address)-1);
1259 success = success && __ load_const_from_toc(Z_R1, a);
1260 if (!success) {
1261 bailout("const section overflow");
1262 return;
1263 }
1264
1265 __ z_br(Z_R1);
1266 assert(__ offset() - start <= call_stub_size(), "stub too big");
1267 __ end_a_stub(); // Update current stubs pointer and restore insts_end.
1268 }
1269
1270 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1271 bool unsigned_comp = condition == lir_cond_belowEqual || condition == lir_cond_aboveEqual;
1272 if (opr1->is_single_cpu()) {
1273 Register reg1 = opr1->as_register();
1274 if (opr2->is_single_cpu()) {
1275 // cpu register - cpu register
1276 if (is_reference_type(opr1->type())) {
1277 __ z_clgr(reg1, opr2->as_register());
1278 } else {
1279 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1280 if (unsigned_comp) {
1281 __ z_clr(reg1, opr2->as_register());
1282 } else {
1283 __ z_cr(reg1, opr2->as_register());
1284 }
1285 }
1286 } else if (opr2->is_stack()) {
1287 // cpu register - stack
1288 if (is_reference_type(opr1->type())) {
1289 __ z_cg(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1290 } else {
1291 if (unsigned_comp) {
1292 __ z_cly(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1293 } else {
1294 __ z_cy(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1295 }
1296 }
1297 } else if (opr2->is_constant()) {
1298 // cpu register - constant
1299 LIR_Const* c = opr2->as_constant_ptr();
1300 if (c->type() == T_INT) {
1301 if (unsigned_comp) {
1302 __ z_clfi(reg1, c->as_jint());
1303 } else {
1304 __ z_cfi(reg1, c->as_jint());
1305 }
1306 } else if (c->type() == T_METADATA) {
1307 // We only need, for now, comparison with null for metadata.
1308 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
1309 Metadata* m = c->as_metadata();
1310 if (m == nullptr) {
1311 __ z_cghi(reg1, 0);
1312 } else {
1313 ShouldNotReachHere();
1314 }
1315 } else if (is_reference_type(c->type())) {
1316 // In 64bit oops are single register.
1317 jobject o = c->as_jobject();
1318 if (o == nullptr) {
1319 __ z_ltgr(reg1, reg1);
1320 } else {
1321 jobject2reg(o, Z_R1_scratch);
1322 __ z_cgr(reg1, Z_R1_scratch);
1323 }
1324 } else {
1325 fatal("unexpected type: %s", basictype_to_str(c->type()));
1326 }
1327 // cpu register - address
1328 } else if (opr2->is_address()) {
1329 if (op->info() != nullptr) {
1330 add_debug_info_for_null_check_here(op->info());
1331 }
1332 if (unsigned_comp) {
1333 __ z_cly(reg1, as_Address(opr2->as_address_ptr()));
1334 } else {
1335 __ z_cy(reg1, as_Address(opr2->as_address_ptr()));
1336 }
1337 } else {
1338 ShouldNotReachHere();
1339 }
1340
1341 } else if (opr1->is_double_cpu()) {
1342 assert(!unsigned_comp, "unexpected");
1343 Register xlo = opr1->as_register_lo();
1344 Register xhi = opr1->as_register_hi();
1345 if (opr2->is_double_cpu()) {
1346 __ z_cgr(xlo, opr2->as_register_lo());
1347 } else if (opr2->is_constant()) {
1348 // cpu register - constant 0
1349 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
1350 __ z_ltgr(xlo, xlo);
1351 } else {
1352 ShouldNotReachHere();
1353 }
1354
1355 } else if (opr1->is_single_fpu()) {
1356 if (opr2->is_single_fpu()) {
1357 __ z_cebr(opr1->as_float_reg(), opr2->as_float_reg());
1358 } else {
1359 // stack slot
1360 Address addr = frame_map()->address_for_slot(opr2->single_stack_ix());
1361 if (Immediate::is_uimm12(addr.disp())) {
1362 __ z_ceb(opr1->as_float_reg(), addr);
1363 } else {
1364 __ z_ley(Z_fscratch_1, addr);
1365 __ z_cebr(opr1->as_float_reg(), Z_fscratch_1);
1366 }
1367 }
1368 } else if (opr1->is_double_fpu()) {
1369 if (opr2->is_double_fpu()) {
1370 __ z_cdbr(opr1->as_double_reg(), opr2->as_double_reg());
1371 } else {
1372 // stack slot
1373 Address addr = frame_map()->address_for_slot(opr2->double_stack_ix());
1374 if (Immediate::is_uimm12(addr.disp())) {
1375 __ z_cdb(opr1->as_double_reg(), addr);
1376 } else {
1377 __ z_ldy(Z_fscratch_1, addr);
1378 __ z_cdbr(opr1->as_double_reg(), Z_fscratch_1);
1379 }
1380 }
1381 } else {
1382 ShouldNotReachHere();
1383 }
1384 }
1385
1386 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1387 Label done;
1388 Register dreg = dst->as_register();
1389
1390 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1391 assert((left->is_single_fpu() && right->is_single_fpu()) ||
1392 (left->is_double_fpu() && right->is_double_fpu()), "unexpected operand types");
1393 bool is_single = left->is_single_fpu();
1394 bool is_unordered_less = (code == lir_ucmp_fd2i);
1395 FloatRegister lreg = is_single ? left->as_float_reg() : left->as_double_reg();
1396 FloatRegister rreg = is_single ? right->as_float_reg() : right->as_double_reg();
1397 if (is_single) {
1398 __ z_cebr(lreg, rreg);
1399 } else {
1400 __ z_cdbr(lreg, rreg);
1401 }
1402 if (VM_Version::has_LoadStoreConditional()) {
1403 Register one = Z_R0_scratch;
1404 Register minus_one = Z_R1_scratch;
1405 __ z_lghi(minus_one, -1);
1406 __ z_lghi(one, 1);
1407 __ z_lghi(dreg, 0);
1408 __ z_locgr(dreg, one, is_unordered_less ? Assembler::bcondHigh : Assembler::bcondHighOrNotOrdered);
1409 __ z_locgr(dreg, minus_one, is_unordered_less ? Assembler::bcondLowOrNotOrdered : Assembler::bcondLow);
1410 } else {
1411 __ clear_reg(dreg, true, false);
1412 __ z_bre(done); // if (left == right) dst = 0
1413
1414 // if (left > right || ((code ~= cmpg) && (left <> right)) dst := 1
1415 __ z_lhi(dreg, 1);
1416 __ z_brc(is_unordered_less ? Assembler::bcondHigh : Assembler::bcondHighOrNotOrdered, done);
1417
1418 // if (left < right || ((code ~= cmpl) && (left <> right)) dst := -1
1419 __ z_lhi(dreg, -1);
1420 }
1421 } else {
1422 assert(code == lir_cmp_l2i, "check");
1423 if (VM_Version::has_LoadStoreConditional()) {
1424 Register one = Z_R0_scratch;
1425 Register minus_one = Z_R1_scratch;
1426 __ z_cgr(left->as_register_lo(), right->as_register_lo());
1427 __ z_lghi(minus_one, -1);
1428 __ z_lghi(one, 1);
1429 __ z_lghi(dreg, 0);
1430 __ z_locgr(dreg, one, Assembler::bcondHigh);
1431 __ z_locgr(dreg, minus_one, Assembler::bcondLow);
1432 } else {
1433 __ z_cgr(left->as_register_lo(), right->as_register_lo());
1434 __ z_lghi(dreg, 0); // eq value
1435 __ z_bre(done);
1436 __ z_lghi(dreg, 1); // gt value
1437 __ z_brh(done);
1438 __ z_lghi(dreg, -1); // lt value
1439 }
1440 }
1441 __ bind(done);
1442 }
1443
1444 // result = condition ? opr1 : opr2
1445 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1446 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1447 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on s390");
1448
1449 Assembler::branch_condition acond = Assembler::bcondEqual, ncond = Assembler::bcondNotEqual;
1450 switch (condition) {
1451 case lir_cond_equal: acond = Assembler::bcondEqual; ncond = Assembler::bcondNotEqual; break;
1452 case lir_cond_notEqual: acond = Assembler::bcondNotEqual; ncond = Assembler::bcondEqual; break;
1453 case lir_cond_less: acond = Assembler::bcondLow; ncond = Assembler::bcondNotLow; break;
1454 case lir_cond_lessEqual: acond = Assembler::bcondNotHigh; ncond = Assembler::bcondHigh; break;
1455 case lir_cond_greaterEqual: acond = Assembler::bcondNotLow; ncond = Assembler::bcondLow; break;
1456 case lir_cond_greater: acond = Assembler::bcondHigh; ncond = Assembler::bcondNotHigh; break;
1457 case lir_cond_belowEqual: acond = Assembler::bcondNotHigh; ncond = Assembler::bcondHigh; break;
1458 case lir_cond_aboveEqual: acond = Assembler::bcondNotLow; ncond = Assembler::bcondLow; break;
1459 default: ShouldNotReachHere();
1460 }
1461
1462 if (opr1->is_cpu_register()) {
1463 reg2reg(opr1, result);
1464 } else if (opr1->is_stack()) {
1465 stack2reg(opr1, result, result->type());
1466 } else if (opr1->is_constant()) {
1467 const2reg(opr1, result, lir_patch_none, nullptr);
1468 } else {
1469 ShouldNotReachHere();
1470 }
1471
1472 if (VM_Version::has_LoadStoreConditional() && !opr2->is_constant()) {
1473 // Optimized version that does not require a branch.
1474 if (opr2->is_single_cpu()) {
1475 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1476 __ z_locgr(result->as_register(), opr2->as_register(), ncond);
1477 } else if (opr2->is_double_cpu()) {
1478 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1479 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1480 __ z_locgr(result->as_register_lo(), opr2->as_register_lo(), ncond);
1481 } else if (opr2->is_single_stack()) {
1482 __ z_loc(result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()), ncond);
1483 } else if (opr2->is_double_stack()) {
1484 __ z_locg(result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix()), ncond);
1485 } else {
1486 ShouldNotReachHere();
1487 }
1488 } else {
1489 Label skip;
1490 __ z_brc(acond, skip);
1491 if (opr2->is_cpu_register()) {
1492 reg2reg(opr2, result);
1493 } else if (opr2->is_stack()) {
1494 stack2reg(opr2, result, result->type());
1495 } else if (opr2->is_constant()) {
1496 const2reg(opr2, result, lir_patch_none, nullptr);
1497 } else {
1498 ShouldNotReachHere();
1499 }
1500 __ bind(skip);
1501 }
1502 }
1503
1504 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest,
1505 CodeEmitInfo* info, bool pop_fpu_stack) {
1506 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1507
1508 if (left->is_single_cpu()) {
1509 assert(left == dest, "left and dest must be equal");
1510 Register lreg = left->as_register();
1511
1512 if (right->is_single_cpu()) {
1513 // cpu register - cpu register
1514 Register rreg = right->as_register();
1515 switch (code) {
1516 case lir_add: __ z_ar (lreg, rreg); break;
1517 case lir_sub: __ z_sr (lreg, rreg); break;
1518 case lir_mul: __ z_msr(lreg, rreg); break;
1519 default: ShouldNotReachHere();
1520 }
1521
1522 } else if (right->is_stack()) {
1523 // cpu register - stack
1524 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1525 switch (code) {
1526 case lir_add: __ z_ay(lreg, raddr); break;
1527 case lir_sub: __ z_sy(lreg, raddr); break;
1528 default: ShouldNotReachHere();
1529 }
1530
1531 } else if (right->is_constant()) {
1532 // cpu register - constant
1533 jint c = right->as_constant_ptr()->as_jint();
1534 switch (code) {
1535 case lir_add: __ z_agfi(lreg, c); break;
1536 case lir_sub: __ z_agfi(lreg, -c); break; // note: -min_jint == min_jint
1537 case lir_mul: __ z_msfi(lreg, c); break;
1538 default: ShouldNotReachHere();
1539 }
1540
1541 } else {
1542 ShouldNotReachHere();
1543 }
1544
1545 } else if (left->is_double_cpu()) {
1546 assert(left == dest, "left and dest must be equal");
1547 Register lreg_lo = left->as_register_lo();
1548 Register lreg_hi = left->as_register_hi();
1549
1550 if (right->is_double_cpu()) {
1551 // cpu register - cpu register
1552 Register rreg_lo = right->as_register_lo();
1553 Register rreg_hi = right->as_register_hi();
1554 assert_different_registers(lreg_lo, rreg_lo);
1555 switch (code) {
1556 case lir_add:
1557 __ z_agr(lreg_lo, rreg_lo);
1558 break;
1559 case lir_sub:
1560 __ z_sgr(lreg_lo, rreg_lo);
1561 break;
1562 case lir_mul:
1563 __ z_msgr(lreg_lo, rreg_lo);
1564 break;
1565 default:
1566 ShouldNotReachHere();
1567 }
1568
1569 } else if (right->is_constant()) {
1570 // cpu register - constant
1571 jlong c = right->as_constant_ptr()->as_jlong_bits();
1572 switch (code) {
1573 case lir_add: __ z_agfi(lreg_lo, c); break;
1574 case lir_sub:
1575 if (c != min_jint) {
1576 __ z_agfi(lreg_lo, -c);
1577 } else {
1578 // -min_jint cannot be represented as simm32 in z_agfi
1579 // min_jint sign extended: 0xffffffff80000000
1580 // -min_jint as 64 bit integer: 0x0000000080000000
1581 // 0x80000000 can be represented as uimm32 in z_algfi
1582 // lreg_lo := lreg_lo + -min_jint == lreg_lo + 0x80000000
1583 __ z_algfi(lreg_lo, UCONST64(0x80000000));
1584 }
1585 break;
1586 case lir_mul: __ z_msgfi(lreg_lo, c); break;
1587 default:
1588 ShouldNotReachHere();
1589 }
1590
1591 } else {
1592 ShouldNotReachHere();
1593 }
1594
1595 } else if (left->is_single_fpu()) {
1596 assert(left == dest, "left and dest must be equal");
1597 FloatRegister lreg = left->as_float_reg();
1598 FloatRegister rreg = right->is_single_fpu() ? right->as_float_reg() : fnoreg;
1599 Address raddr;
1600
1601 if (rreg == fnoreg) {
1602 assert(right->is_single_stack(), "constants should be loaded into register");
1603 raddr = frame_map()->address_for_slot(right->single_stack_ix());
1604 if (!Immediate::is_uimm12(raddr.disp())) {
1605 __ mem2freg_opt(rreg = Z_fscratch_1, raddr, false);
1606 }
1607 }
1608
1609 if (rreg != fnoreg) {
1610 switch (code) {
1611 case lir_add: __ z_aebr(lreg, rreg); break;
1612 case lir_sub: __ z_sebr(lreg, rreg); break;
1613 case lir_mul: __ z_meebr(lreg, rreg); break;
1614 case lir_div: __ z_debr(lreg, rreg); break;
1615 default: ShouldNotReachHere();
1616 }
1617 } else {
1618 switch (code) {
1619 case lir_add: __ z_aeb(lreg, raddr); break;
1620 case lir_sub: __ z_seb(lreg, raddr); break;
1621 case lir_mul: __ z_meeb(lreg, raddr); break;
1622 case lir_div: __ z_deb(lreg, raddr); break;
1623 default: ShouldNotReachHere();
1624 }
1625 }
1626 } else if (left->is_double_fpu()) {
1627 assert(left == dest, "left and dest must be equal");
1628 FloatRegister lreg = left->as_double_reg();
1629 FloatRegister rreg = right->is_double_fpu() ? right->as_double_reg() : fnoreg;
1630 Address raddr;
1631
1632 if (rreg == fnoreg) {
1633 assert(right->is_double_stack(), "constants should be loaded into register");
1634 raddr = frame_map()->address_for_slot(right->double_stack_ix());
1635 if (!Immediate::is_uimm12(raddr.disp())) {
1636 __ mem2freg_opt(rreg = Z_fscratch_1, raddr, true);
1637 }
1638 }
1639
1640 if (rreg != fnoreg) {
1641 switch (code) {
1642 case lir_add: __ z_adbr(lreg, rreg); break;
1643 case lir_sub: __ z_sdbr(lreg, rreg); break;
1644 case lir_mul: __ z_mdbr(lreg, rreg); break;
1645 case lir_div: __ z_ddbr(lreg, rreg); break;
1646 default: ShouldNotReachHere();
1647 }
1648 } else {
1649 switch (code) {
1650 case lir_add: __ z_adb(lreg, raddr); break;
1651 case lir_sub: __ z_sdb(lreg, raddr); break;
1652 case lir_mul: __ z_mdb(lreg, raddr); break;
1653 case lir_div: __ z_ddb(lreg, raddr); break;
1654 default: ShouldNotReachHere();
1655 }
1656 }
1657 } else if (left->is_address()) {
1658 assert(left == dest, "left and dest must be equal");
1659 assert(code == lir_add, "unsupported operation");
1660 assert(right->is_constant(), "unsupported operand");
1661 jint c = right->as_constant_ptr()->as_jint();
1662 LIR_Address* lir_addr = left->as_address_ptr();
1663 Address addr = as_Address(lir_addr);
1664 switch (lir_addr->type()) {
1665 case T_INT:
1666 __ add2mem_32(addr, c, Z_R1_scratch);
1667 break;
1668 case T_LONG:
1669 __ add2mem_64(addr, c, Z_R1_scratch);
1670 break;
1671 default:
1672 ShouldNotReachHere();
1673 }
1674 } else {
1675 ShouldNotReachHere();
1676 }
1677 }
1678
1679 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1680 switch (code) {
1681 case lir_sqrt: {
1682 assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1683 FloatRegister src_reg = value->as_double_reg();
1684 FloatRegister dst_reg = dest->as_double_reg();
1685 __ z_sqdbr(dst_reg, src_reg);
1686 break;
1687 }
1688 case lir_abs: {
1689 assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1690 FloatRegister src_reg = value->as_double_reg();
1691 FloatRegister dst_reg = dest->as_double_reg();
1692 __ z_lpdbr(dst_reg, src_reg);
1693 break;
1694 }
1695 default: {
1696 ShouldNotReachHere();
1697 break;
1698 }
1699 }
1700 }
1701
1702 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1703 if (left->is_single_cpu()) {
1704 Register reg = left->as_register();
1705 if (right->is_constant()) {
1706 int val = right->as_constant_ptr()->as_jint();
1707 switch (code) {
1708 case lir_logic_and: __ z_nilf(reg, val); break;
1709 case lir_logic_or: __ z_oilf(reg, val); break;
1710 case lir_logic_xor: __ z_xilf(reg, val); break;
1711 default: ShouldNotReachHere();
1712 }
1713 } else if (right->is_stack()) {
1714 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1715 switch (code) {
1716 case lir_logic_and: __ z_ny(reg, raddr); break;
1717 case lir_logic_or: __ z_oy(reg, raddr); break;
1718 case lir_logic_xor: __ z_xy(reg, raddr); break;
1719 default: ShouldNotReachHere();
1720 }
1721 } else {
1722 Register rright = right->as_register();
1723 switch (code) {
1724 case lir_logic_and: __ z_nr(reg, rright); break;
1725 case lir_logic_or : __ z_or(reg, rright); break;
1726 case lir_logic_xor: __ z_xr(reg, rright); break;
1727 default: ShouldNotReachHere();
1728 }
1729 }
1730 move_regs(reg, dst->as_register());
1731 } else {
1732 Register l_lo = left->as_register_lo();
1733 if (right->is_constant()) {
1734 __ load_const_optimized(Z_R1_scratch, right->as_constant_ptr()->as_jlong());
1735 switch (code) {
1736 case lir_logic_and:
1737 __ z_ngr(l_lo, Z_R1_scratch);
1738 break;
1739 case lir_logic_or:
1740 __ z_ogr(l_lo, Z_R1_scratch);
1741 break;
1742 case lir_logic_xor:
1743 __ z_xgr(l_lo, Z_R1_scratch);
1744 break;
1745 default: ShouldNotReachHere();
1746 }
1747 } else {
1748 Register r_lo;
1749 if (is_reference_type(right->type())) {
1750 r_lo = right->as_register();
1751 } else {
1752 r_lo = right->as_register_lo();
1753 }
1754 switch (code) {
1755 case lir_logic_and:
1756 __ z_ngr(l_lo, r_lo);
1757 break;
1758 case lir_logic_or:
1759 __ z_ogr(l_lo, r_lo);
1760 break;
1761 case lir_logic_xor:
1762 __ z_xgr(l_lo, r_lo);
1763 break;
1764 default: ShouldNotReachHere();
1765 }
1766 }
1767
1768 Register dst_lo = dst->as_register_lo();
1769
1770 move_regs(l_lo, dst_lo);
1771 }
1772 }
1773
1774 // See operand selection in LIRGenerator::do_ArithmeticOp_Int().
1775 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1776 if (left->is_double_cpu()) {
1777 // 64 bit integer case
1778 assert(left->is_double_cpu(), "left must be register");
1779 assert(right->is_double_cpu() || is_power_of_2(right->as_jlong()),
1780 "right must be register or power of 2 constant");
1781 assert(result->is_double_cpu(), "result must be register");
1782
1783 Register lreg = left->as_register_lo();
1784 Register dreg = result->as_register_lo();
1785
1786 if (right->is_constant()) {
1787 // Convert division by a power of two into some shifts and logical operations.
1788 Register treg1 = Z_R0_scratch;
1789 Register treg2 = Z_R1_scratch;
1790 jlong divisor = right->as_jlong();
1791 jlong log_divisor = log2i_exact(right->as_jlong());
1792
1793 if (divisor == min_jlong) {
1794 // Min_jlong is special. Result is '0' except for min_jlong/min_jlong = 1.
1795 if (dreg == lreg) {
1796 NearLabel done;
1797 __ load_const_optimized(treg2, min_jlong);
1798 __ z_cgr(lreg, treg2);
1799 __ z_lghi(dreg, 0); // Preserves condition code.
1800 __ z_brne(done);
1801 __ z_lghi(dreg, 1); // min_jlong / min_jlong = 1
1802 __ bind(done);
1803 } else {
1804 assert_different_registers(dreg, lreg);
1805 NearLabel done;
1806 __ z_lghi(dreg, 0);
1807 __ compare64_and_branch(lreg, min_jlong, Assembler::bcondNotEqual, done);
1808 __ z_lghi(dreg, 1);
1809 __ bind(done);
1810 }
1811 return;
1812 }
1813 __ move_reg_if_needed(dreg, T_LONG, lreg, T_LONG);
1814 if (divisor == 2) {
1815 __ z_srlg(treg2, dreg, 63); // dividend < 0 ? 1 : 0
1816 } else {
1817 __ z_srag(treg2, dreg, 63); // dividend < 0 ? -1 : 0
1818 __ and_imm(treg2, divisor - 1, treg1, true);
1819 }
1820 if (code == lir_idiv) {
1821 __ z_agr(dreg, treg2);
1822 __ z_srag(dreg, dreg, log_divisor);
1823 } else {
1824 assert(code == lir_irem, "check");
1825 __ z_agr(treg2, dreg);
1826 __ and_imm(treg2, ~(divisor - 1), treg1, true);
1827 __ z_sgr(dreg, treg2);
1828 }
1829 return;
1830 }
1831
1832 // Divisor is not a power of 2 constant.
1833 Register rreg = right->as_register_lo();
1834 Register treg = temp->as_register_lo();
1835 assert(right->is_double_cpu(), "right must be register");
1836 assert(lreg == Z_R11, "see ldivInOpr()");
1837 assert(rreg != lreg, "right register must not be same as left register");
1838 assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10) ||
1839 (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see ldivInOpr(), ldivOutOpr(), lremOutOpr()");
1840
1841 Register R1 = lreg->predecessor();
1842 Register R2 = rreg;
1843 assert(code != lir_idiv || lreg==dreg, "see code below");
1844 if (code == lir_idiv) {
1845 __ z_lcgr(lreg, lreg);
1846 } else {
1847 __ clear_reg(dreg, true, false);
1848 }
1849 NearLabel done;
1850 __ compare64_and_branch(R2, -1, Assembler::bcondEqual, done);
1851 if (code == lir_idiv) {
1852 __ z_lcgr(lreg, lreg); // Revert lcgr above.
1853 }
1854 if (ImplicitDiv0Checks) {
1855 // No debug info because the idiv won't trap.
1856 // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1857 // which is unnecessary, too.
1858 add_debug_info_for_div0(__ offset(), info);
1859 }
1860 __ z_dsgr(R1, R2);
1861 __ bind(done);
1862 return;
1863 }
1864
1865 // 32 bit integer case
1866
1867 assert(left->is_single_cpu(), "left must be register");
1868 assert(right->is_single_cpu() || is_power_of_2(right->as_jint()), "right must be register or power of 2 constant");
1869 assert(result->is_single_cpu(), "result must be register");
1870
1871 Register lreg = left->as_register();
1872 Register dreg = result->as_register();
1873
1874 if (right->is_constant()) {
1875 // Convert division by a power of two into some shifts and logical operations.
1876 Register treg1 = Z_R0_scratch;
1877 Register treg2 = Z_R1_scratch;
1878 jlong divisor = right->as_jint();
1879 jlong log_divisor = log2i_exact(right->as_jint());
1880 __ move_reg_if_needed(dreg, T_LONG, lreg, T_INT); // sign extend
1881 if (divisor == 2) {
1882 __ z_srlg(treg2, dreg, 63); // dividend < 0 ? 1 : 0
1883 } else {
1884 __ z_srag(treg2, dreg, 63); // dividend < 0 ? -1 : 0
1885 __ and_imm(treg2, divisor - 1, treg1, true);
1886 }
1887 if (code == lir_idiv) {
1888 __ z_agr(dreg, treg2);
1889 __ z_srag(dreg, dreg, log_divisor);
1890 } else {
1891 assert(code == lir_irem, "check");
1892 __ z_agr(treg2, dreg);
1893 __ and_imm(treg2, ~(divisor - 1), treg1, true);
1894 __ z_sgr(dreg, treg2);
1895 }
1896 return;
1897 }
1898
1899 // Divisor is not a power of 2 constant.
1900 Register rreg = right->as_register();
1901 Register treg = temp->as_register();
1902 assert(right->is_single_cpu(), "right must be register");
1903 assert(lreg == Z_R11, "left register must be rax,");
1904 assert(rreg != lreg, "right register must not be same as left register");
1905 assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10)
1906 || (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see divInOpr(), divOutOpr(), remOutOpr()");
1907
1908 Register R1 = lreg->predecessor();
1909 Register R2 = rreg;
1910 __ move_reg_if_needed(lreg, T_LONG, lreg, T_INT); // sign extend
1911 if (ImplicitDiv0Checks) {
1912 // No debug info because the idiv won't trap.
1913 // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1914 // which is unnecessary, too.
1915 add_debug_info_for_div0(__ offset(), info);
1916 }
1917 __ z_dsgfr(R1, R2);
1918 }
1919
1920 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1921 assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1922 assert(exceptionPC->as_register() == Z_EXC_PC, "should match");
1923
1924 // Exception object is not added to oop map by LinearScan
1925 // (LinearScan assumes that no oops are in fixed registers).
1926 info->add_register_oop(exceptionOop);
1927
1928 // Reuse the debug info from the safepoint poll for the throw op itself.
1929 __ get_PC(Z_EXC_PC);
1930 add_call_info(__ offset(), info); // for exception handler
1931 address stub = Runtime1::entry_for (compilation()->has_fpu_code() ? C1StubId::handle_exception_id
1932 : C1StubId::handle_exception_nofpu_id);
1933 emit_call_c(stub);
1934 }
1935
1936 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1937 assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1938
1939 __ branch_optimized(Assembler::bcondAlways, _unwind_handler_entry);
1940 }
1941
1942 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1943 ciArrayKlass* default_type = op->expected_type();
1944 Register src = op->src()->as_register();
1945 Register dst = op->dst()->as_register();
1946 Register src_pos = op->src_pos()->as_register();
1947 Register dst_pos = op->dst_pos()->as_register();
1948 Register length = op->length()->as_register();
1949 Register tmp = op->tmp()->as_register();
1950
1951 CodeStub* stub = op->stub();
1952 int flags = op->flags();
1953 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
1954 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1955
1956 // If we don't know anything, just go through the generic arraycopy.
1957 if (default_type == nullptr) {
1958 address copyfunc_addr = StubRoutines::generic_arraycopy();
1959
1960 if (copyfunc_addr == nullptr) {
1961 // Take a slow path for generic arraycopy.
1962 __ branch_optimized(Assembler::bcondAlways, *stub->entry());
1963 __ bind(*stub->continuation());
1964 return;
1965 }
1966
1967 // Save outgoing arguments in callee saved registers (C convention) in case
1968 // a call to System.arraycopy is needed.
1969 Register callee_saved_src = Z_R10;
1970 Register callee_saved_src_pos = Z_R11;
1971 Register callee_saved_dst = Z_R12;
1972 Register callee_saved_dst_pos = Z_R13;
1973 Register callee_saved_length = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
1974
1975 __ lgr_if_needed(callee_saved_src, src);
1976 __ lgr_if_needed(callee_saved_src_pos, src_pos);
1977 __ lgr_if_needed(callee_saved_dst, dst);
1978 __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
1979 __ lgr_if_needed(callee_saved_length, length);
1980
1981 // C function requires 64 bit values.
1982 __ z_lgfr(src_pos, src_pos);
1983 __ z_lgfr(dst_pos, dst_pos);
1984 __ z_lgfr(length, length);
1985
1986 // Pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint.
1987
1988 // The arguments are in the corresponding registers.
1989 assert(Z_ARG1 == src, "assumption");
1990 assert(Z_ARG2 == src_pos, "assumption");
1991 assert(Z_ARG3 == dst, "assumption");
1992 assert(Z_ARG4 == dst_pos, "assumption");
1993 assert(Z_ARG5 == length, "assumption");
1994 #ifndef PRODUCT
1995 if (PrintC1Statistics) {
1996 __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_generic_arraycopystub_cnt);
1997 __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
1998 }
1999 #endif
2000 emit_call_c(copyfunc_addr);
2001 CHECK_BAILOUT();
2002
2003 __ compare32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2004
2005 __ z_lgr(tmp, Z_RET);
2006 __ z_xilf(tmp, -1);
2007
2008 // Restore values from callee saved registers so they are where the stub
2009 // expects them.
2010 __ lgr_if_needed(src, callee_saved_src);
2011 __ lgr_if_needed(src_pos, callee_saved_src_pos);
2012 __ lgr_if_needed(dst, callee_saved_dst);
2013 __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2014 __ lgr_if_needed(length, callee_saved_length);
2015
2016 __ z_sr(length, tmp);
2017 __ z_ar(src_pos, tmp);
2018 __ z_ar(dst_pos, tmp);
2019 __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2020
2021 __ bind(*stub->continuation());
2022 return;
2023 }
2024
2025 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2026
2027 int elem_size = type2aelembytes(basic_type);
2028 int shift_amount;
2029
2030 switch (elem_size) {
2031 case 1 :
2032 shift_amount = 0;
2033 break;
2034 case 2 :
2035 shift_amount = 1;
2036 break;
2037 case 4 :
2038 shift_amount = 2;
2039 break;
2040 case 8 :
2041 shift_amount = 3;
2042 break;
2043 default:
2044 shift_amount = -1;
2045 ShouldNotReachHere();
2046 }
2047
2048 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2049 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2050 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2051 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2052
2053 // Length and pos's are all sign extended at this point on 64bit.
2054
2055 // test for null
2056 if (flags & LIR_OpArrayCopy::src_null_check) {
2057 __ compareU64_and_branch(src, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2058 }
2059 if (flags & LIR_OpArrayCopy::dst_null_check) {
2060 __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2061 }
2062
2063 // Check if negative.
2064 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2065 __ compare32_and_branch(src_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2066 }
2067 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2068 __ compare32_and_branch(dst_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2069 }
2070
2071 // If the compiler was not able to prove that exact type of the source or the destination
2072 // of the arraycopy is an array type, check at runtime if the source or the destination is
2073 // an instance type.
2074 if (flags & LIR_OpArrayCopy::type_check) {
2075 assert(Klass::_lh_neutral_value == 0, "or replace z_lt instructions");
2076
2077 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2078 __ load_klass(tmp, dst);
2079 __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2080 __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2081 }
2082
2083 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2084 __ load_klass(tmp, src);
2085 __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2086 __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2087 }
2088 }
2089
2090 if (flags & LIR_OpArrayCopy::src_range_check) {
2091 __ z_la(tmp, Address(src_pos, length));
2092 __ z_cl(tmp, src_length_addr);
2093 __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2094 }
2095 if (flags & LIR_OpArrayCopy::dst_range_check) {
2096 __ z_la(tmp, Address(dst_pos, length));
2097 __ z_cl(tmp, dst_length_addr);
2098 __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2099 }
2100
2101 if (flags & LIR_OpArrayCopy::length_positive_check) {
2102 __ z_ltr(length, length);
2103 __ branch_optimized(Assembler::bcondNegative, *stub->entry());
2104 }
2105
2106 // Stubs require 64 bit values.
2107 __ z_lgfr(src_pos, src_pos); // int -> long
2108 __ z_lgfr(dst_pos, dst_pos); // int -> long
2109 __ z_lgfr(length, length); // int -> long
2110
2111 if (flags & LIR_OpArrayCopy::type_check) {
2112 // We don't know the array types are compatible.
2113 if (basic_type != T_OBJECT) {
2114 // Simple test for basic type arrays.
2115 if (UseCompressedClassPointers) {
2116 __ z_l(tmp, src_klass_addr);
2117 __ z_c(tmp, dst_klass_addr);
2118 } else {
2119 __ z_lg(tmp, src_klass_addr);
2120 __ z_cg(tmp, dst_klass_addr);
2121 }
2122 __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2123 } else {
2124 // For object arrays, if src is a sub class of dst then we can
2125 // safely do the copy.
2126 NearLabel cont, slow;
2127 Register src_klass = Z_R1_scratch;
2128 Register dst_klass = Z_R10;
2129
2130 __ load_klass(src_klass, src);
2131 __ load_klass(dst_klass, dst);
2132
2133 __ check_klass_subtype_fast_path(src_klass, dst_klass, tmp, &cont, &slow, nullptr);
2134
2135 store_parameter(src_klass, 0); // sub
2136 store_parameter(dst_klass, 1); // super
2137 emit_call_c(Runtime1::entry_for (C1StubId::slow_subtype_check_id));
2138 CHECK_BAILOUT2(cont, slow);
2139 // Sets condition code 0 for match (2 otherwise).
2140 __ branch_optimized(Assembler::bcondEqual, cont);
2141
2142 __ bind(slow);
2143
2144 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2145 if (copyfunc_addr != nullptr) { // use stub if available
2146 // Src is not a sub class of dst so we have to do a
2147 // per-element check.
2148
2149 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2150 if ((flags & mask) != mask) {
2151 // Check that at least both of them object arrays.
2152 assert(flags & mask, "one of the two should be known to be an object array");
2153
2154 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2155 __ load_klass(tmp, src);
2156 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2157 __ load_klass(tmp, dst);
2158 }
2159 Address klass_lh_addr(tmp, Klass::layout_helper_offset());
2160 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2161 __ load_const_optimized(Z_R1_scratch, objArray_lh);
2162 __ z_c(Z_R1_scratch, klass_lh_addr);
2163 __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2164 }
2165
2166 // Save outgoing arguments in callee saved registers (C convention) in case
2167 // a call to System.arraycopy is needed.
2168 Register callee_saved_src = Z_R10;
2169 Register callee_saved_src_pos = Z_R11;
2170 Register callee_saved_dst = Z_R12;
2171 Register callee_saved_dst_pos = Z_R13;
2172 Register callee_saved_length = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
2173
2174 __ lgr_if_needed(callee_saved_src, src);
2175 __ lgr_if_needed(callee_saved_src_pos, src_pos);
2176 __ lgr_if_needed(callee_saved_dst, dst);
2177 __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
2178 __ lgr_if_needed(callee_saved_length, length);
2179
2180 __ z_llgfr(length, length); // Higher 32bits must be null.
2181
2182 __ z_sllg(Z_ARG1, src_pos, shift_amount); // index -> byte offset
2183 __ z_sllg(Z_ARG2, dst_pos, shift_amount); // index -> byte offset
2184
2185 __ z_la(Z_ARG1, Address(src, Z_ARG1, arrayOopDesc::base_offset_in_bytes(basic_type)));
2186 assert_different_registers(Z_ARG1, dst, dst_pos, length);
2187 __ z_la(Z_ARG2, Address(dst, Z_ARG2, arrayOopDesc::base_offset_in_bytes(basic_type)));
2188 assert_different_registers(Z_ARG2, dst, length);
2189
2190 __ z_lgr(Z_ARG3, length);
2191 assert_different_registers(Z_ARG3, dst);
2192
2193 __ load_klass(Z_ARG5, dst);
2194 __ z_lg(Z_ARG5, Address(Z_ARG5, ObjArrayKlass::element_klass_offset()));
2195 __ z_lg(Z_ARG4, Address(Z_ARG5, Klass::super_check_offset_offset()));
2196 emit_call_c(copyfunc_addr);
2197 CHECK_BAILOUT2(cont, slow);
2198
2199 #ifndef PRODUCT
2200 if (PrintC1Statistics) {
2201 NearLabel failed;
2202 __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondNotEqual, failed);
2203 __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_cnt);
2204 __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2205 __ bind(failed);
2206 }
2207 #endif
2208
2209 __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2210
2211 #ifndef PRODUCT
2212 if (PrintC1Statistics) {
2213 __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_attempt_cnt);
2214 __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2215 }
2216 #endif
2217
2218 __ z_lgr(tmp, Z_RET);
2219 __ z_xilf(tmp, -1);
2220
2221 // Restore previously spilled arguments
2222 __ lgr_if_needed(src, callee_saved_src);
2223 __ lgr_if_needed(src_pos, callee_saved_src_pos);
2224 __ lgr_if_needed(dst, callee_saved_dst);
2225 __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2226 __ lgr_if_needed(length, callee_saved_length);
2227
2228 __ z_sr(length, tmp);
2229 __ z_ar(src_pos, tmp);
2230 __ z_ar(dst_pos, tmp);
2231 }
2232
2233 __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2234
2235 __ bind(cont);
2236 }
2237 }
2238
2239 #ifdef ASSERT
2240 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2241 // Sanity check the known type with the incoming class. For the
2242 // primitive case the types must match exactly with src.klass and
2243 // dst.klass each exactly matching the default type. For the
2244 // object array case, if no type check is needed then either the
2245 // dst type is exactly the expected type and the src type is a
2246 // subtype which we can't check or src is the same array as dst
2247 // but not necessarily exactly of type default_type.
2248 NearLabel known_ok, halt;
2249 metadata2reg(default_type->constant_encoding(), tmp);
2250 if (UseCompressedClassPointers) {
2251 __ encode_klass_not_null(tmp);
2252 }
2253
2254 if (basic_type != T_OBJECT) {
2255 if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); }
2256 else { __ z_cg(tmp, dst_klass_addr); }
2257 __ branch_optimized(Assembler::bcondNotEqual, halt);
2258 if (UseCompressedClassPointers) { __ z_c (tmp, src_klass_addr); }
2259 else { __ z_cg(tmp, src_klass_addr); }
2260 __ branch_optimized(Assembler::bcondEqual, known_ok);
2261 } else {
2262 if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); }
2263 else { __ z_cg(tmp, dst_klass_addr); }
2264 __ branch_optimized(Assembler::bcondEqual, known_ok);
2265 __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
2266 }
2267 __ bind(halt);
2268 __ stop("incorrect type information in arraycopy");
2269 __ bind(known_ok);
2270 }
2271 #endif
2272
2273 #ifndef PRODUCT
2274 if (PrintC1Statistics) {
2275 __ load_const_optimized(Z_R1_scratch, Runtime1::arraycopy_count_address(basic_type));
2276 __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2277 }
2278 #endif
2279
2280 __ z_sllg(tmp, src_pos, shift_amount); // index -> byte offset
2281 __ z_sllg(Z_R1_scratch, dst_pos, shift_amount); // index -> byte offset
2282
2283 assert_different_registers(Z_ARG1, dst, dst_pos, length);
2284 __ z_la(Z_ARG1, Address(src, tmp, arrayOopDesc::base_offset_in_bytes(basic_type)));
2285 assert_different_registers(Z_ARG2, length);
2286 __ z_la(Z_ARG2, Address(dst, Z_R1_scratch, arrayOopDesc::base_offset_in_bytes(basic_type)));
2287 __ lgr_if_needed(Z_ARG3, length);
2288
2289 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2290 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2291 const char *name;
2292 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2293 __ call_VM_leaf(entry);
2294
2295 if (stub != nullptr) {
2296 __ bind(*stub->continuation());
2297 }
2298 }
2299
2300 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2301 if (dest->is_single_cpu()) {
2302 if (left->type() == T_OBJECT) {
2303 switch (code) {
2304 case lir_shl: __ z_sllg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2305 case lir_shr: __ z_srag (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2306 case lir_ushr: __ z_srlg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2307 default: ShouldNotReachHere();
2308 }
2309 } else {
2310 assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2311 Register masked_count = Z_R1_scratch;
2312 __ z_lr(masked_count, count->as_register());
2313 __ z_nill(masked_count, 31);
2314 switch (code) {
2315 case lir_shl: __ z_sllg (dest->as_register(), left->as_register(), 0, masked_count); break;
2316 case lir_shr: __ z_sra (dest->as_register(), 0, masked_count); break;
2317 case lir_ushr: __ z_srl (dest->as_register(), 0, masked_count); break;
2318 default: ShouldNotReachHere();
2319 }
2320 }
2321 } else {
2322 switch (code) {
2323 case lir_shl: __ z_sllg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2324 case lir_shr: __ z_srag (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2325 case lir_ushr: __ z_srlg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2326 default: ShouldNotReachHere();
2327 }
2328 }
2329 }
2330
2331 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2332 if (left->type() == T_OBJECT) {
2333 count = count & 63; // Shouldn't shift by more than sizeof(intptr_t).
2334 Register l = left->as_register();
2335 Register d = dest->as_register_lo();
2336 switch (code) {
2337 case lir_shl: __ z_sllg (d, l, count); break;
2338 case lir_shr: __ z_srag (d, l, count); break;
2339 case lir_ushr: __ z_srlg (d, l, count); break;
2340 default: ShouldNotReachHere();
2341 }
2342 return;
2343 }
2344 if (dest->is_single_cpu()) {
2345 assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2346 count = count & 0x1F; // Java spec
2347 switch (code) {
2348 case lir_shl: __ z_sllg (dest->as_register(), left->as_register(), count); break;
2349 case lir_shr: __ z_sra (dest->as_register(), count); break;
2350 case lir_ushr: __ z_srl (dest->as_register(), count); break;
2351 default: ShouldNotReachHere();
2352 }
2353 } else if (dest->is_double_cpu()) {
2354 count = count & 63; // Java spec
2355 Register l = left->as_pointer_register();
2356 Register d = dest->as_pointer_register();
2357 switch (code) {
2358 case lir_shl: __ z_sllg (d, l, count); break;
2359 case lir_shr: __ z_srag (d, l, count); break;
2360 case lir_ushr: __ z_srlg (d, l, count); break;
2361 default: ShouldNotReachHere();
2362 }
2363 } else {
2364 ShouldNotReachHere();
2365 }
2366 }
2367
2368 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2369 if (op->init_check()) {
2370 // Make sure klass is initialized & doesn't have finalizer.
2371 // init_state needs acquire, but S390 is TSO, and so we are already good.
2372 const int state_offset = in_bytes(InstanceKlass::init_state_offset());
2373 Register iklass = op->klass()->as_register();
2374 add_debug_info_for_null_check_here(op->stub()->info());
2375 if (Immediate::is_uimm12(state_offset)) {
2376 __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
2377 } else {
2378 __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
2379 }
2380 __ branch_optimized(Assembler::bcondNotEqual, *op->stub()->entry()); // Use long branch, because slow_case might be far.
2381 }
2382 __ allocate_object(op->obj()->as_register(),
2383 op->tmp1()->as_register(),
2384 op->tmp2()->as_register(),
2385 op->header_size(),
2386 op->object_size(),
2387 op->klass()->as_register(),
2388 *op->stub()->entry());
2389 __ bind(*op->stub()->continuation());
2390 __ verify_oop(op->obj()->as_register(), FILE_AND_LINE);
2391 }
2392
2393 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2394 Register len = op->len()->as_register();
2395 __ move_reg_if_needed(len, T_LONG, len, T_INT); // sign extend
2396
2397 if (UseSlowPath ||
2398 (!UseFastNewObjectArray && (is_reference_type(op->type()))) ||
2399 (!UseFastNewTypeArray && (!is_reference_type(op->type())))) {
2400 __ z_brul(*op->stub()->entry());
2401 } else {
2402 __ allocate_array(op->obj()->as_register(),
2403 op->len()->as_register(),
2404 op->tmp1()->as_register(),
2405 op->tmp2()->as_register(),
2406 arrayOopDesc::base_offset_in_bytes(op->type()),
2407 type2aelembytes(op->type()),
2408 op->klass()->as_register(),
2409 *op->stub()->entry(),
2410 op->zero_array());
2411 }
2412 __ bind(*op->stub()->continuation());
2413 }
2414
2415 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
2416 Register recv, Register tmp1, Label* update_done) {
2417 uint i;
2418 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2419 Label next_test;
2420 // See if the receiver is receiver[n].
2421 Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2422 __ z_cg(recv, receiver_addr);
2423 __ z_brne(next_test);
2424 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
2425 __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2426 __ branch_optimized(Assembler::bcondAlways, *update_done);
2427 __ bind(next_test);
2428 }
2429
2430 // Didn't find receiver; find next empty slot and fill it in.
2431 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2432 Label next_test;
2433 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2434 __ z_ltg(Z_R0_scratch, recv_addr);
2435 __ z_brne(next_test);
2436 __ z_stg(recv, recv_addr);
2437 __ load_const_optimized(tmp1, DataLayout::counter_increment);
2438 __ z_stg(tmp1, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)), mdo);
2439 __ branch_optimized(Assembler::bcondAlways, *update_done);
2440 __ bind(next_test);
2441 }
2442 }
2443
2444 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2445 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2446 Unimplemented();
2447 }
2448
2449 void LIR_Assembler::store_parameter(Register r, int param_num) {
2450 assert(param_num >= 0, "invalid num");
2451 int offset_in_bytes = param_num * BytesPerWord;
2452 check_reserved_argument_area(offset_in_bytes);
2453 offset_in_bytes += FrameMap::first_available_sp_in_frame;
2454 __ z_stg(r, offset_in_bytes, Z_SP);
2455 }
2456
2457 void LIR_Assembler::store_parameter(jint c, int param_num) {
2458 assert(param_num >= 0, "invalid num");
2459 int offset_in_bytes = param_num * BytesPerWord;
2460 check_reserved_argument_area(offset_in_bytes);
2461 offset_in_bytes += FrameMap::first_available_sp_in_frame;
2462 __ store_const(Address(Z_SP, offset_in_bytes), c, Z_R1_scratch, true);
2463 }
2464
2465 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2466 // We always need a stub for the failure case.
2467 CodeStub* stub = op->stub();
2468 Register obj = op->object()->as_register();
2469 Register k_RInfo = op->tmp1()->as_register();
2470 Register klass_RInfo = op->tmp2()->as_register();
2471 Register dst = op->result_opr()->as_register();
2472 Register Rtmp1 = Z_R1_scratch;
2473 ciKlass* k = op->klass();
2474
2475 assert(!op->tmp3()->is_valid(), "tmp3's not needed");
2476
2477 // Check if it needs to be profiled.
2478 ciMethodData* md = nullptr;
2479 ciProfileData* data = nullptr;
2480
2481 if (op->should_profile()) {
2482 ciMethod* method = op->profiled_method();
2483 assert(method != nullptr, "Should have method");
2484 int bci = op->profiled_bci();
2485 md = method->method_data_or_null();
2486 assert(md != nullptr, "Sanity");
2487 data = md->bci_to_data(bci);
2488 assert(data != nullptr, "need data for type check");
2489 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2490 }
2491
2492 // Temp operands do not overlap with inputs, if this is their last
2493 // use (end of range is exclusive), so a register conflict is possible.
2494 if (obj == k_RInfo) {
2495 k_RInfo = dst;
2496 } else if (obj == klass_RInfo) {
2497 klass_RInfo = dst;
2498 }
2499 assert_different_registers(obj, k_RInfo, klass_RInfo);
2500
2501 if (op->should_profile()) {
2502 Register mdo = klass_RInfo;
2503 metadata2reg(md->constant_encoding(), mdo);
2504 NearLabel not_null;
2505 __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2506 // Object is null; update MDO and exit.
2507 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2508 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2509 __ or2mem_8(data_addr, header_bits);
2510 __ branch_optimized(Assembler::bcondAlways, *obj_is_null);
2511 __ bind(not_null);
2512
2513 NearLabel update_done;
2514 Register recv = k_RInfo;
2515 __ load_klass(recv, obj);
2516 type_profile_helper(mdo, md, data, recv, Rtmp1, &update_done);
2517 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2518 __ add2mem_64(counter_addr, DataLayout::counter_increment, Rtmp1);
2519 __ bind(update_done);
2520 } else {
2521 __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondEqual, *obj_is_null);
2522 }
2523
2524 Label *failure_target = failure;
2525 Label *success_target = success;
2526
2527 // Patching may screw with our temporaries,
2528 // so let's do it before loading the class.
2529 if (k->is_loaded()) {
2530 metadata2reg(k->constant_encoding(), k_RInfo);
2531 } else {
2532 klass2reg_with_patching(k_RInfo, op->info_for_patch());
2533 }
2534 assert(obj != k_RInfo, "must be different");
2535
2536 __ verify_oop(obj, FILE_AND_LINE);
2537
2538 // Get object class.
2539 // Not a safepoint as obj null check happens earlier.
2540 if (op->fast_check()) {
2541 if (UseCompressedClassPointers) {
2542 __ load_klass(klass_RInfo, obj);
2543 __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
2544 } else {
2545 __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
2546 __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2547 }
2548 // Successful cast, fall through to profile or jump.
2549 } else {
2550 bool need_slow_path = !k->is_loaded() ||
2551 ((int) k->super_check_offset() == in_bytes(Klass::secondary_super_cache_offset()));
2552 intptr_t super_check_offset = k->is_loaded() ? k->super_check_offset() : -1L;
2553 __ load_klass(klass_RInfo, obj);
2554 // Perform the fast part of the checking logic.
2555 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1,
2556 (need_slow_path ? success_target : nullptr),
2557 failure_target, nullptr,
2558 RegisterOrConstant(super_check_offset));
2559 if (need_slow_path) {
2560 // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2561 address a = Runtime1::entry_for (C1StubId::slow_subtype_check_id);
2562 store_parameter(klass_RInfo, 0); // sub
2563 store_parameter(k_RInfo, 1); // super
2564 emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2565 __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2566 // Fall through to success case.
2567 }
2568 }
2569
2570 __ branch_optimized(Assembler::bcondAlways, *success);
2571 }
2572
2573 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2574 LIR_Code code = op->code();
2575 if (code == lir_store_check) {
2576 Register value = op->object()->as_register();
2577 Register array = op->array()->as_register();
2578 Register k_RInfo = op->tmp1()->as_register();
2579 Register klass_RInfo = op->tmp2()->as_register();
2580 Register Rtmp1 = Z_R1_scratch;
2581
2582 CodeStub* stub = op->stub();
2583
2584 // Check if it needs to be profiled.
2585 ciMethodData* md = nullptr;
2586 ciProfileData* data = nullptr;
2587
2588 assert_different_registers(value, k_RInfo, klass_RInfo);
2589
2590 if (op->should_profile()) {
2591 ciMethod* method = op->profiled_method();
2592 assert(method != nullptr, "Should have method");
2593 int bci = op->profiled_bci();
2594 md = method->method_data_or_null();
2595 assert(md != nullptr, "Sanity");
2596 data = md->bci_to_data(bci);
2597 assert(data != nullptr, "need data for type check");
2598 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2599 }
2600 NearLabel done;
2601 Label *success_target = &done;
2602 Label *failure_target = stub->entry();
2603
2604 if (op->should_profile()) {
2605 Register mdo = klass_RInfo;
2606 metadata2reg(md->constant_encoding(), mdo);
2607 NearLabel not_null;
2608 __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2609 // Object is null; update MDO and exit.
2610 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2611 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2612 __ or2mem_8(data_addr, header_bits);
2613 __ branch_optimized(Assembler::bcondAlways, done);
2614 __ bind(not_null);
2615
2616 NearLabel update_done;
2617 Register recv = k_RInfo;
2618 __ load_klass(recv, value);
2619 type_profile_helper(mdo, md, data, recv, Rtmp1, &update_done);
2620 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2621 __ add2mem_64(counter_addr, DataLayout::counter_increment, Rtmp1);
2622 __ bind(update_done);
2623 } else {
2624 __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondEqual, done);
2625 }
2626
2627 add_debug_info_for_null_check_here(op->info_for_exception());
2628 __ load_klass(k_RInfo, array);
2629 __ load_klass(klass_RInfo, value);
2630
2631 // Get instance klass (it's already uncompressed).
2632 __ z_lg(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2633 // Perform the fast part of the checking logic.
2634 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
2635 // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2636 address a = Runtime1::entry_for (C1StubId::slow_subtype_check_id);
2637 store_parameter(klass_RInfo, 0); // sub
2638 store_parameter(k_RInfo, 1); // super
2639 emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2640 __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2641 // Fall through to success case.
2642
2643 __ bind(done);
2644 } else {
2645 if (code == lir_checkcast) {
2646 Register obj = op->object()->as_register();
2647 Register dst = op->result_opr()->as_register();
2648 NearLabel success;
2649 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2650 __ bind(success);
2651 __ lgr_if_needed(dst, obj);
2652 } else {
2653 if (code == lir_instanceof) {
2654 Register obj = op->object()->as_register();
2655 Register dst = op->result_opr()->as_register();
2656 NearLabel success, failure, done;
2657 emit_typecheck_helper(op, &success, &failure, &failure);
2658 __ bind(failure);
2659 __ clear_reg(dst);
2660 __ branch_optimized(Assembler::bcondAlways, done);
2661 __ bind(success);
2662 __ load_const_optimized(dst, 1);
2663 __ bind(done);
2664 } else {
2665 ShouldNotReachHere();
2666 }
2667 }
2668 }
2669 }
2670
2671 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2672 Register addr = op->addr()->as_pointer_register();
2673 Register t1_cmp = Z_R1_scratch;
2674 if (op->code() == lir_cas_long) {
2675 Register cmp_value_lo = op->cmp_value()->as_register_lo();
2676 Register new_value_lo = op->new_value()->as_register_lo();
2677 __ z_lgr(t1_cmp, cmp_value_lo);
2678 // Perform the compare and swap operation.
2679 __ z_csg(t1_cmp, new_value_lo, 0, addr);
2680 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2681 Register cmp_value = op->cmp_value()->as_register();
2682 Register new_value = op->new_value()->as_register();
2683 if (op->code() == lir_cas_obj) {
2684 if (UseCompressedOops) {
2685 t1_cmp = op->tmp1()->as_register();
2686 Register t2_new = op->tmp2()->as_register();
2687 assert_different_registers(cmp_value, new_value, addr, t1_cmp, t2_new);
2688 __ oop_encoder(t1_cmp, cmp_value, true /*maybe null*/);
2689 __ oop_encoder(t2_new, new_value, true /*maybe null*/);
2690 __ z_cs(t1_cmp, t2_new, 0, addr);
2691 } else {
2692 __ z_lgr(t1_cmp, cmp_value);
2693 __ z_csg(t1_cmp, new_value, 0, addr);
2694 }
2695 } else {
2696 __ z_lr(t1_cmp, cmp_value);
2697 __ z_cs(t1_cmp, new_value, 0, addr);
2698 }
2699 } else {
2700 ShouldNotReachHere(); // new lir_cas_??
2701 }
2702 }
2703
2704 void LIR_Assembler::breakpoint() {
2705 Unimplemented();
2706 // __ breakpoint_trap();
2707 }
2708
2709 void LIR_Assembler::push(LIR_Opr opr) {
2710 ShouldNotCallThis(); // unused
2711 }
2712
2713 void LIR_Assembler::pop(LIR_Opr opr) {
2714 ShouldNotCallThis(); // unused
2715 }
2716
2717 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2718 Address addr = frame_map()->address_for_monitor_lock(monitor_no);
2719 __ add2reg(dst_opr->as_register(), addr.disp(), addr.base());
2720 }
2721
2722 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2723 Register obj = op->obj_opr()->as_register(); // May not be an oop.
2724 Register hdr = op->hdr_opr()->as_register();
2725 Register lock = op->lock_opr()->as_register();
2726 if (LockingMode == LM_MONITOR) {
2727 if (op->info() != nullptr) {
2728 add_debug_info_for_null_check_here(op->info());
2729 __ null_check(obj);
2730 }
2731 __ branch_optimized(Assembler::bcondAlways, *op->stub()->entry());
2732 } else if (op->code() == lir_lock) {
2733 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2734 // Add debug info for NullPointerException only if one is possible.
2735 if (op->info() != nullptr) {
2736 add_debug_info_for_null_check_here(op->info());
2737 }
2738 __ lock_object(hdr, obj, lock, *op->stub()->entry());
2739 // done
2740 } else if (op->code() == lir_unlock) {
2741 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2742 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2743 } else {
2744 ShouldNotReachHere();
2745 }
2746 __ bind(*op->stub()->continuation());
2747 }
2748
2749 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2750 Register obj = op->obj()->as_pointer_register();
2751 Register result = op->result_opr()->as_pointer_register();
2752
2753 CodeEmitInfo* info = op->info();
2754 if (info != nullptr) {
2755 add_debug_info_for_null_check_here(info);
2756 }
2757
2758 if (UseCompressedClassPointers) {
2759 __ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes()));
2760 __ decode_klass_not_null(result);
2761 } else {
2762 __ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes()));
2763 }
2764 }
2765 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2766 ciMethod* method = op->profiled_method();
2767 int bci = op->profiled_bci();
2768 ciMethod* callee = op->profiled_callee();
2769
2770 // Update counter for all call types.
2771 ciMethodData* md = method->method_data_or_null();
2772 assert(md != nullptr, "Sanity");
2773 ciProfileData* data = md->bci_to_data(bci);
2774 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2775 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2776 Register mdo = op->mdo()->as_register();
2777 assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2778 Register tmp1 = op->tmp1()->as_register_lo();
2779 metadata2reg(md->constant_encoding(), mdo);
2780
2781 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2782 // Perform additional virtual call profiling for invokevirtual and
2783 // invokeinterface bytecodes
2784 if (op->should_profile_receiver_type()) {
2785 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2786 Register recv = op->recv()->as_register();
2787 assert_different_registers(mdo, tmp1, recv);
2788 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2789 ciKlass* known_klass = op->known_holder();
2790 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2791 // We know the type that will be seen at this call site; we can
2792 // statically update the MethodData* rather than needing to do
2793 // dynamic tests on the receiver type.
2794
2795 // NOTE: we should probably put a lock around this search to
2796 // avoid collisions by concurrent compilations.
2797 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2798 uint i;
2799 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2800 ciKlass* receiver = vc_data->receiver(i);
2801 if (known_klass->equals(receiver)) {
2802 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2803 __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2804 return;
2805 }
2806 }
2807
2808 // Receiver type not found in profile data. Select an empty slot.
2809
2810 // Note that this is less efficient than it should be because it
2811 // always does a write to the receiver part of the
2812 // VirtualCallData rather than just the first time.
2813 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2814 ciKlass* receiver = vc_data->receiver(i);
2815 if (receiver == nullptr) {
2816 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2817 metadata2reg(known_klass->constant_encoding(), tmp1);
2818 __ z_stg(tmp1, recv_addr);
2819 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2820 __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2821 return;
2822 }
2823 }
2824 } else {
2825 __ load_klass(recv, recv);
2826 NearLabel update_done;
2827 type_profile_helper(mdo, md, data, recv, tmp1, &update_done);
2828 // Receiver did not match any saved receiver and there is no empty row for it.
2829 // Increment total counter to indicate polymorphic case.
2830 __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2831 __ bind(update_done);
2832 }
2833 } else {
2834 // static call
2835 __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2836 }
2837 }
2838
2839 void LIR_Assembler::align_backward_branch_target() {
2840 __ align(OptoLoopAlignment);
2841 }
2842
2843 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2844 ShouldNotCallThis(); // There are no delay slots on ZARCH_64.
2845 }
2846
2847 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2848 // tmp must be unused
2849 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2850 assert(left->is_register(), "can only handle registers");
2851
2852 if (left->is_single_cpu()) {
2853 __ z_lcr(dest->as_register(), left->as_register());
2854 } else if (left->is_single_fpu()) {
2855 __ z_lcebr(dest->as_float_reg(), left->as_float_reg());
2856 } else if (left->is_double_fpu()) {
2857 __ z_lcdbr(dest->as_double_reg(), left->as_double_reg());
2858 } else {
2859 assert(left->is_double_cpu(), "Must be a long");
2860 __ z_lcgr(dest->as_register_lo(), left->as_register_lo());
2861 }
2862 }
2863
2864 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
2865 const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2866 assert(!tmp->is_valid(), "don't need temporary");
2867 emit_call_c(dest);
2868 CHECK_BAILOUT();
2869 if (info != nullptr) {
2870 add_call_info_here(info);
2871 }
2872 }
2873
2874 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2875 ShouldNotCallThis(); // not needed on ZARCH_64
2876 }
2877
2878 void LIR_Assembler::membar() {
2879 __ z_fence();
2880 }
2881
2882 void LIR_Assembler::membar_acquire() {
2883 __ z_acquire();
2884 }
2885
2886 void LIR_Assembler::membar_release() {
2887 __ z_release();
2888 }
2889
2890 void LIR_Assembler::membar_loadload() {
2891 __ z_acquire();
2892 }
2893
2894 void LIR_Assembler::membar_storestore() {
2895 __ z_release();
2896 }
2897
2898 void LIR_Assembler::membar_loadstore() {
2899 __ z_acquire();
2900 }
2901
2902 void LIR_Assembler::membar_storeload() {
2903 __ z_fence();
2904 }
2905
2906 void LIR_Assembler::on_spin_wait() {
2907 Unimplemented();
2908 }
2909
2910 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2911 assert(patch_code == lir_patch_none, "Patch code not supported");
2912 LIR_Address* addr = addr_opr->as_address_ptr();
2913 assert(addr->scale() == LIR_Address::times_1, "scaling unsupported");
2914 __ load_address(dest->as_pointer_register(), as_Address(addr));
2915 }
2916
2917 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2918 ShouldNotCallThis(); // unused
2919 }
2920
2921 #ifdef ASSERT
2922 // Emit run-time assertion.
2923 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2924 Unimplemented();
2925 }
2926 #endif
2927
2928 void LIR_Assembler::peephole(LIR_List*) {
2929 // Do nothing for now.
2930 }
2931
2932 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
2933 assert(code == lir_xadd, "lir_xchg not supported");
2934 Address src_addr = as_Address(src->as_address_ptr());
2935 Register base = src_addr.base();
2936 intptr_t disp = src_addr.disp();
2937 if (src_addr.index()->is_valid()) {
2938 // LAA and LAAG do not support index register.
2939 __ load_address(Z_R1_scratch, src_addr);
2940 base = Z_R1_scratch;
2941 disp = 0;
2942 }
2943 if (data->type() == T_INT) {
2944 __ z_laa(dest->as_register(), data->as_register(), disp, base);
2945 } else if (data->type() == T_LONG) {
2946 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
2947 __ z_laag(dest->as_register_lo(), data->as_register_lo(), disp, base);
2948 } else {
2949 ShouldNotReachHere();
2950 }
2951 }
2952
2953 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2954 Register obj = op->obj()->as_register();
2955 Register tmp1 = op->tmp()->as_pointer_register();
2956 Register tmp2 = Z_R1_scratch;
2957 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2958 ciKlass* exact_klass = op->exact_klass();
2959 intptr_t current_klass = op->current_klass();
2960 bool not_null = op->not_null();
2961 bool no_conflict = op->no_conflict();
2962
2963 Label update, next, none, null_seen, init_klass;
2964
2965 bool do_null = !not_null;
2966 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2967 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2968
2969 assert(do_null || do_update, "why are we here?");
2970 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2971
2972 __ verify_oop(obj, FILE_AND_LINE);
2973
2974 if (do_null || tmp1 != obj DEBUG_ONLY(|| true)) {
2975 __ z_ltgr(tmp1, obj);
2976 }
2977 if (do_null) {
2978 __ z_brnz(update);
2979 if (!TypeEntries::was_null_seen(current_klass)) {
2980 __ z_lg(tmp1, mdo_addr);
2981 __ z_oill(tmp1, TypeEntries::null_seen);
2982 __ z_stg(tmp1, mdo_addr);
2983 }
2984 if (do_update) {
2985 __ z_bru(next);
2986 }
2987 } else {
2988 __ asm_assert(Assembler::bcondNotZero, "unexpected null obj", __LINE__);
2989 }
2990
2991 __ bind(update);
2992
2993 if (do_update) {
2994 #ifdef ASSERT
2995 if (exact_klass != nullptr) {
2996 __ load_klass(tmp1, tmp1);
2997 metadata2reg(exact_klass->constant_encoding(), tmp2);
2998 __ z_cgr(tmp1, tmp2);
2999 __ asm_assert(Assembler::bcondEqual, "exact klass and actual klass differ", __LINE__);
3000 }
3001 #endif
3002
3003 Label do_update;
3004 __ z_lg(tmp2, mdo_addr);
3005
3006 if (!no_conflict) {
3007 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3008 if (exact_klass != nullptr) {
3009 metadata2reg(exact_klass->constant_encoding(), tmp1);
3010 } else {
3011 __ load_klass(tmp1, tmp1);
3012 }
3013
3014 // Klass seen before: nothing to do (regardless of unknown bit).
3015 __ z_lgr(Z_R0_scratch, tmp2);
3016 assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
3017 __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
3018 __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
3019
3020 // Already unknown: Nothing to do anymore.
3021 __ z_tmll(tmp2, TypeEntries::type_unknown);
3022 __ z_brc(Assembler::bcondAllOne, next);
3023
3024 if (TypeEntries::is_type_none(current_klass)) {
3025 __ z_lgr(Z_R0_scratch, tmp2);
3026 assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3027 __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3028 __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, init_klass);
3029 }
3030 } else {
3031 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3032 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3033
3034 // Already unknown: Nothing to do anymore.
3035 __ z_tmll(tmp2, TypeEntries::type_unknown);
3036 __ z_brc(Assembler::bcondAllOne, next);
3037 }
3038
3039 // Different than before. Cannot keep accurate profile.
3040 __ z_oill(tmp2, TypeEntries::type_unknown);
3041 __ z_bru(do_update);
3042 } else {
3043 // There's a single possible klass at this profile point.
3044 assert(exact_klass != nullptr, "should be");
3045 if (TypeEntries::is_type_none(current_klass)) {
3046 metadata2reg(exact_klass->constant_encoding(), tmp1);
3047 __ z_lgr(Z_R0_scratch, tmp2);
3048 assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
3049 __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
3050 __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
3051 #ifdef ASSERT
3052 {
3053 Label ok;
3054 __ z_lgr(Z_R0_scratch, tmp2);
3055 assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3056 __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3057 __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, ok);
3058 __ stop("unexpected profiling mismatch");
3059 __ bind(ok);
3060 }
3061 #endif
3062
3063 } else {
3064 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3065 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3066
3067 // Already unknown: Nothing to do anymore.
3068 __ z_tmll(tmp2, TypeEntries::type_unknown);
3069 __ z_brc(Assembler::bcondAllOne, next);
3070 __ z_oill(tmp2, TypeEntries::type_unknown);
3071 __ z_bru(do_update);
3072 }
3073 }
3074
3075 __ bind(init_klass);
3076 // Combine klass and null_seen bit (only used if (tmp & type_mask)==0).
3077 __ z_ogr(tmp2, tmp1);
3078
3079 __ bind(do_update);
3080 __ z_stg(tmp2, mdo_addr);
3081
3082 __ bind(next);
3083 }
3084 }
3085
3086 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3087 assert(op->crc()->is_single_cpu(), "crc must be register");
3088 assert(op->val()->is_single_cpu(), "byte value must be register");
3089 assert(op->result_opr()->is_single_cpu(), "result must be register");
3090 Register crc = op->crc()->as_register();
3091 Register val = op->val()->as_register();
3092 Register res = op->result_opr()->as_register();
3093
3094 assert_different_registers(val, crc, res);
3095
3096 __ load_const_optimized(res, StubRoutines::crc_table_addr());
3097 __ kernel_crc32_singleByteReg(crc, val, res, true);
3098 __ z_lgfr(res, crc);
3099 }
3100
3101 #undef __