1 /*
2 * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
4 * Copyright 2024, 2026 IBM Corporation. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/codeBuffer.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "code/compiledIC.hpp"
30 #include "compiler/disassembler.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/collectedHeap.inline.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "memory/universe.hpp"
38 #include "oops/accessDecorators.hpp"
39 #include "oops/compressedKlass.inline.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "oops/methodData.hpp"
43 #include "prims/methodHandles.hpp"
44 #include "registerSaver_s390.hpp"
45 #include "runtime/icache.hpp"
46 #include "runtime/interfaceSupport.inline.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/objectMonitorTable.hpp"
49 #include "runtime/os.hpp"
50 #include "runtime/safepoint.hpp"
51 #include "runtime/safepointMechanism.hpp"
52 #include "runtime/sharedRuntime.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "utilities/events.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/powerOfTwo.hpp"
57
58 #include <ucontext.h>
59
60 #define BLOCK_COMMENT(str) block_comment(str)
61 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
62
63 // Move 32-bit register if destination and source are different.
64 void MacroAssembler::lr_if_needed(Register rd, Register rs) {
65 if (rs != rd) { z_lr(rd, rs); }
66 }
67
68 // Move register if destination and source are different.
69 void MacroAssembler::lgr_if_needed(Register rd, Register rs) {
70 if (rs != rd) { z_lgr(rd, rs); }
71 }
72
73 // Zero-extend 32-bit register into 64-bit register if destination and source are different.
74 void MacroAssembler::llgfr_if_needed(Register rd, Register rs) {
75 if (rs != rd) { z_llgfr(rd, rs); }
76 }
77
78 // Move float register if destination and source are different.
79 void MacroAssembler::ldr_if_needed(FloatRegister rd, FloatRegister rs) {
80 if (rs != rd) { z_ldr(rd, rs); }
81 }
82
83 // Move integer register if destination and source are different.
84 // It is assumed that shorter-than-int types are already
85 // appropriately sign-extended.
86 void MacroAssembler::move_reg_if_needed(Register dst, BasicType dst_type, Register src,
87 BasicType src_type) {
88 assert((dst_type != T_FLOAT) && (dst_type != T_DOUBLE), "use move_freg for float types");
89 assert((src_type != T_FLOAT) && (src_type != T_DOUBLE), "use move_freg for float types");
90
91 if (dst_type == src_type) {
92 lgr_if_needed(dst, src); // Just move all 64 bits.
93 return;
94 }
95
96 switch (dst_type) {
97 // Do not support these types for now.
98 // case T_BOOLEAN:
99 case T_BYTE: // signed byte
100 switch (src_type) {
101 case T_INT:
102 z_lgbr(dst, src);
103 break;
104 default:
105 ShouldNotReachHere();
106 }
107 return;
108
109 case T_CHAR:
110 case T_SHORT:
111 switch (src_type) {
112 case T_INT:
113 if (dst_type == T_CHAR) {
114 z_llghr(dst, src);
115 } else {
116 z_lghr(dst, src);
117 }
118 break;
119 default:
120 ShouldNotReachHere();
121 }
122 return;
123
124 case T_INT:
125 switch (src_type) {
126 case T_BOOLEAN:
127 case T_BYTE:
128 case T_CHAR:
129 case T_SHORT:
130 case T_INT:
131 case T_LONG:
132 case T_OBJECT:
133 case T_ARRAY:
134 case T_VOID:
135 case T_ADDRESS:
136 lr_if_needed(dst, src);
137 // llgfr_if_needed(dst, src); // zero-extend (in case we need to find a bug).
138 return;
139
140 default:
141 assert(false, "non-integer src type");
142 return;
143 }
144 case T_LONG:
145 switch (src_type) {
146 case T_BOOLEAN:
147 case T_BYTE:
148 case T_CHAR:
149 case T_SHORT:
150 case T_INT:
151 z_lgfr(dst, src); // sign extension
152 return;
153
154 case T_LONG:
155 case T_OBJECT:
156 case T_ARRAY:
157 case T_VOID:
158 case T_ADDRESS:
159 lgr_if_needed(dst, src);
160 return;
161
162 default:
163 assert(false, "non-integer src type");
164 return;
165 }
166 return;
167 case T_OBJECT:
168 case T_ARRAY:
169 case T_VOID:
170 case T_ADDRESS:
171 switch (src_type) {
172 // These types don't make sense to be converted to pointers:
173 // case T_BOOLEAN:
174 // case T_BYTE:
175 // case T_CHAR:
176 // case T_SHORT:
177
178 case T_INT:
179 z_llgfr(dst, src); // zero extension
180 return;
181
182 case T_LONG:
183 case T_OBJECT:
184 case T_ARRAY:
185 case T_VOID:
186 case T_ADDRESS:
187 lgr_if_needed(dst, src);
188 return;
189
190 default:
191 assert(false, "non-integer src type");
192 return;
193 }
194 return;
195 default:
196 assert(false, "non-integer dst type");
197 return;
198 }
199 }
200
201 // Move float register if destination and source are different.
202 void MacroAssembler::move_freg_if_needed(FloatRegister dst, BasicType dst_type,
203 FloatRegister src, BasicType src_type) {
204 assert((dst_type == T_FLOAT) || (dst_type == T_DOUBLE), "use move_reg for int types");
205 assert((src_type == T_FLOAT) || (src_type == T_DOUBLE), "use move_reg for int types");
206 if (dst_type == src_type) {
207 ldr_if_needed(dst, src); // Just move all 64 bits.
208 } else {
209 switch (dst_type) {
210 case T_FLOAT:
211 assert(src_type == T_DOUBLE, "invalid float type combination");
212 z_ledbr(dst, src);
213 return;
214 case T_DOUBLE:
215 assert(src_type == T_FLOAT, "invalid float type combination");
216 z_ldebr(dst, src);
217 return;
218 default:
219 assert(false, "non-float dst type");
220 return;
221 }
222 }
223 }
224
225 // Optimized emitter for reg to mem operations.
226 // Uses modern instructions if running on modern hardware, classic instructions
227 // otherwise. Prefers (usually shorter) classic instructions if applicable.
228 // Data register (reg) cannot be used as work register.
229 //
230 // Don't rely on register locking, instead pass a scratch register (Z_R0 by default).
231 // CAUTION! Passing registers >= Z_R2 may produce bad results on old CPUs!
232 void MacroAssembler::freg2mem_opt(FloatRegister reg,
233 int64_t disp,
234 Register index,
235 Register base,
236 void (MacroAssembler::*modern) (FloatRegister, int64_t, Register, Register),
237 void (MacroAssembler::*classic)(FloatRegister, int64_t, Register, Register),
238 Register scratch) {
239 index = (index == noreg) ? Z_R0 : index;
240 if (Displacement::is_shortDisp(disp)) {
241 (this->*classic)(reg, disp, index, base);
242 } else {
243 if (Displacement::is_validDisp(disp)) {
244 (this->*modern)(reg, disp, index, base);
245 } else {
246 if (scratch != Z_R0 && scratch != Z_R1) {
247 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
248 } else {
249 if (scratch != Z_R0) { // scratch == Z_R1
250 if ((scratch == index) || (index == base)) {
251 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
252 } else {
253 add2reg(scratch, disp, base);
254 (this->*classic)(reg, 0, index, scratch);
255 if (base == scratch) {
256 add2reg(base, -disp); // Restore base.
257 }
258 }
259 } else { // scratch == Z_R0
260 z_lgr(scratch, base);
261 add2reg(base, disp);
262 (this->*classic)(reg, 0, index, base);
263 z_lgr(base, scratch); // Restore base.
264 }
265 }
266 }
267 }
268 }
269
270 void MacroAssembler::freg2mem_opt(FloatRegister reg, const Address &a, bool is_double) {
271 if (is_double) {
272 freg2mem_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_FFUN(z_stdy), CLASSIC_FFUN(z_std));
273 } else {
274 freg2mem_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_FFUN(z_stey), CLASSIC_FFUN(z_ste));
275 }
276 }
277
278 // Optimized emitter for mem to reg operations.
279 // Uses modern instructions if running on modern hardware, classic instructions
280 // otherwise. Prefers (usually shorter) classic instructions if applicable.
281 // data register (reg) cannot be used as work register.
282 //
283 // Don't rely on register locking, instead pass a scratch register (Z_R0 by default).
284 // CAUTION! Passing registers >= Z_R2 may produce bad results on old CPUs!
285 void MacroAssembler::mem2freg_opt(FloatRegister reg,
286 int64_t disp,
287 Register index,
288 Register base,
289 void (MacroAssembler::*modern) (FloatRegister, int64_t, Register, Register),
290 void (MacroAssembler::*classic)(FloatRegister, int64_t, Register, Register),
291 Register scratch) {
292 index = (index == noreg) ? Z_R0 : index;
293 if (Displacement::is_shortDisp(disp)) {
294 (this->*classic)(reg, disp, index, base);
295 } else {
296 if (Displacement::is_validDisp(disp)) {
297 (this->*modern)(reg, disp, index, base);
298 } else {
299 if (scratch != Z_R0 && scratch != Z_R1) {
300 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
301 } else {
302 if (scratch != Z_R0) { // scratch == Z_R1
303 if ((scratch == index) || (index == base)) {
304 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
305 } else {
306 add2reg(scratch, disp, base);
307 (this->*classic)(reg, 0, index, scratch);
308 if (base == scratch) {
309 add2reg(base, -disp); // Restore base.
310 }
311 }
312 } else { // scratch == Z_R0
313 z_lgr(scratch, base);
314 add2reg(base, disp);
315 (this->*classic)(reg, 0, index, base);
316 z_lgr(base, scratch); // Restore base.
317 }
318 }
319 }
320 }
321 }
322
323 void MacroAssembler::mem2freg_opt(FloatRegister reg, const Address &a, bool is_double) {
324 if (is_double) {
325 mem2freg_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_FFUN(z_ldy), CLASSIC_FFUN(z_ld));
326 } else {
327 mem2freg_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_FFUN(z_ley), CLASSIC_FFUN(z_le));
328 }
329 }
330
331 // Optimized emitter for reg to mem operations.
332 // Uses modern instructions if running on modern hardware, classic instructions
333 // otherwise. Prefers (usually shorter) classic instructions if applicable.
334 // Data register (reg) cannot be used as work register.
335 //
336 // Don't rely on register locking, instead pass a scratch register
337 // (Z_R0 by default)
338 // CAUTION! passing registers >= Z_R2 may produce bad results on old CPUs!
339 void MacroAssembler::reg2mem_opt(Register reg,
340 int64_t disp,
341 Register index,
342 Register base,
343 void (MacroAssembler::*modern) (Register, int64_t, Register, Register),
344 void (MacroAssembler::*classic)(Register, int64_t, Register, Register),
345 Register scratch) {
346 index = (index == noreg) ? Z_R0 : index;
347 if (Displacement::is_shortDisp(disp)) {
348 (this->*classic)(reg, disp, index, base);
349 } else {
350 if (Displacement::is_validDisp(disp)) {
351 (this->*modern)(reg, disp, index, base);
352 } else {
353 if (scratch != Z_R0 && scratch != Z_R1) {
354 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
355 } else {
356 if (scratch != Z_R0) { // scratch == Z_R1
357 if ((scratch == index) || (index == base)) {
358 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
359 } else {
360 add2reg(scratch, disp, base);
361 (this->*classic)(reg, 0, index, scratch);
362 if (base == scratch) {
363 add2reg(base, -disp); // Restore base.
364 }
365 }
366 } else { // scratch == Z_R0
367 if ((scratch == reg) || (scratch == base) || (reg == base)) {
368 (this->*modern)(reg, disp, index, base); // Will fail with disp out of range.
369 } else {
370 z_lgr(scratch, base);
371 add2reg(base, disp);
372 (this->*classic)(reg, 0, index, base);
373 z_lgr(base, scratch); // Restore base.
374 }
375 }
376 }
377 }
378 }
379 }
380
381 int MacroAssembler::reg2mem_opt(Register reg, const Address &a, bool is_double) {
382 int store_offset = offset();
383 if (is_double) {
384 reg2mem_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_IFUN(z_stg), CLASSIC_IFUN(z_stg));
385 } else {
386 reg2mem_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_IFUN(z_sty), CLASSIC_IFUN(z_st));
387 }
388 return store_offset;
389 }
390
391 // Optimized emitter for mem to reg operations.
392 // Uses modern instructions if running on modern hardware, classic instructions
393 // otherwise. Prefers (usually shorter) classic instructions if applicable.
394 // Data register (reg) will be used as work register where possible.
395 void MacroAssembler::mem2reg_opt(Register reg,
396 int64_t disp,
397 Register index,
398 Register base,
399 void (MacroAssembler::*modern) (Register, int64_t, Register, Register),
400 void (MacroAssembler::*classic)(Register, int64_t, Register, Register)) {
401 index = (index == noreg) ? Z_R0 : index;
402 if (Displacement::is_shortDisp(disp)) {
403 (this->*classic)(reg, disp, index, base);
404 } else {
405 if (Displacement::is_validDisp(disp)) {
406 (this->*modern)(reg, disp, index, base);
407 } else {
408 if ((reg == index) && (reg == base)) {
409 z_sllg(reg, reg, 1);
410 add2reg(reg, disp);
411 (this->*classic)(reg, 0, noreg, reg);
412 } else if ((reg == index) && (reg != Z_R0)) {
413 add2reg(reg, disp);
414 (this->*classic)(reg, 0, reg, base);
415 } else if (reg == base) {
416 add2reg(reg, disp);
417 (this->*classic)(reg, 0, index, reg);
418 } else if (reg != Z_R0) {
419 add2reg(reg, disp, base);
420 (this->*classic)(reg, 0, index, reg);
421 } else { // reg == Z_R0 && reg != base here
422 add2reg(base, disp);
423 (this->*classic)(reg, 0, index, base);
424 add2reg(base, -disp);
425 }
426 }
427 }
428 }
429
430 void MacroAssembler::mem2reg_opt(Register reg, const Address &a, bool is_double) {
431 if (is_double) {
432 z_lg(reg, a);
433 } else {
434 mem2reg_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_IFUN(z_ly), CLASSIC_IFUN(z_l));
435 }
436 }
437
438 void MacroAssembler::mem2reg_signed_opt(Register reg, const Address &a) {
439 mem2reg_opt(reg, a.disp20(), a.indexOrR0(), a.baseOrR0(), MODERN_IFUN(z_lgf), CLASSIC_IFUN(z_lgf));
440 }
441
442 void MacroAssembler::and_imm(Register r, long mask,
443 Register tmp /* = Z_R0 */,
444 bool wide /* = false */) {
445 assert(wide || Immediate::is_simm32(mask), "mask value too large");
446
447 if (!wide) {
448 z_nilf(r, mask);
449 return;
450 }
451
452 assert(r != tmp, " need a different temporary register !");
453 load_const_optimized(tmp, mask);
454 z_ngr(r, tmp);
455 }
456
457 // Calculate the 1's complement.
458 // Note: The condition code is neither preserved nor correctly set by this code!!!
459 // Note: (wide == false) does not protect the high order half of the target register
460 // from alteration. It only serves as optimization hint for 32-bit results.
461 void MacroAssembler::not_(Register r1, Register r2, bool wide) {
462
463 if ((r2 == noreg) || (r2 == r1)) { // Calc 1's complement in place.
464 z_xilf(r1, -1);
465 if (wide) {
466 z_xihf(r1, -1);
467 }
468 } else { // Distinct src and dst registers.
469 load_const_optimized(r1, -1);
470 z_xgr(r1, r2);
471 }
472 }
473
474 unsigned long MacroAssembler::create_mask(int lBitPos, int rBitPos) {
475 assert(lBitPos >= 0, "zero is leftmost bit position");
476 assert(rBitPos <= 63, "63 is rightmost bit position");
477 assert(lBitPos <= rBitPos, "inverted selection interval");
478 return (lBitPos == 0 ? (unsigned long)(-1L) : ((1UL<<(63-lBitPos+1))-1)) & (~((1UL<<(63-rBitPos))-1));
479 }
480
481 // Helper function for the "Rotate_then_<logicalOP>" emitters.
482 // Rotate src, then mask register contents such that only bits in range survive.
483 // For oneBits == false, all bits not in range are set to 0. Useful for deleting all bits outside range.
484 // For oneBits == true, all bits not in range are set to 1. Useful for preserving all bits outside range.
485 // The caller must ensure that the selected range only contains bits with defined value.
486 void MacroAssembler::rotate_then_mask(Register dst, Register src, int lBitPos, int rBitPos,
487 int nRotate, bool src32bit, bool dst32bit, bool oneBits) {
488 assert(!(dst32bit && lBitPos < 32), "selection interval out of range for int destination");
489 bool sll4rll = (nRotate >= 0) && (nRotate <= (63-rBitPos)); // Substitute SLL(G) for RLL(G).
490 bool srl4rll = (nRotate < 0) && (-nRotate <= lBitPos); // Substitute SRL(G) for RLL(G).
491 // Pre-determine which parts of dst will be zero after shift/rotate.
492 bool llZero = sll4rll && (nRotate >= 16);
493 bool lhZero = (sll4rll && (nRotate >= 32)) || (srl4rll && (nRotate <= -48));
494 bool lfZero = llZero && lhZero;
495 bool hlZero = (sll4rll && (nRotate >= 48)) || (srl4rll && (nRotate <= -32));
496 bool hhZero = (srl4rll && (nRotate <= -16));
497 bool hfZero = hlZero && hhZero;
498
499 // rotate then mask src operand.
500 // if oneBits == true, all bits outside selected range are 1s.
501 // if oneBits == false, all bits outside selected range are 0s.
502 if (src32bit) { // There might be garbage in the upper 32 bits which will get masked away.
503 if (dst32bit) {
504 z_rll(dst, src, nRotate); // Copy and rotate, upper half of reg remains undisturbed.
505 } else {
506 if (sll4rll) { z_sllg(dst, src, nRotate); }
507 else if (srl4rll) { z_srlg(dst, src, -nRotate); }
508 else { z_rllg(dst, src, nRotate); }
509 }
510 } else {
511 if (sll4rll) { z_sllg(dst, src, nRotate); }
512 else if (srl4rll) { z_srlg(dst, src, -nRotate); }
513 else { z_rllg(dst, src, nRotate); }
514 }
515
516 unsigned long range_mask = create_mask(lBitPos, rBitPos);
517 unsigned int range_mask_h = (unsigned int)(range_mask >> 32);
518 unsigned int range_mask_l = (unsigned int)range_mask;
519 unsigned short range_mask_hh = (unsigned short)(range_mask >> 48);
520 unsigned short range_mask_hl = (unsigned short)(range_mask >> 32);
521 unsigned short range_mask_lh = (unsigned short)(range_mask >> 16);
522 unsigned short range_mask_ll = (unsigned short)range_mask;
523 // Works for z9 and newer H/W.
524 if (oneBits) {
525 if ((~range_mask_l) != 0) { z_oilf(dst, ~range_mask_l); } // All bits outside range become 1s.
526 if (((~range_mask_h) != 0) && !dst32bit) { z_oihf(dst, ~range_mask_h); }
527 } else {
528 // All bits outside range become 0s
529 if (((~range_mask_l) != 0) && !lfZero) {
530 z_nilf(dst, range_mask_l);
531 }
532 if (((~range_mask_h) != 0) && !dst32bit && !hfZero) {
533 z_nihf(dst, range_mask_h);
534 }
535 }
536 }
537
538 // Rotate src, then insert selected range from rotated src into dst.
539 // Clear dst before, if requested.
540 void MacroAssembler::rotate_then_insert(Register dst, Register src, int lBitPos, int rBitPos,
541 int nRotate, bool clear_dst) {
542 // This version does not depend on src being zero-extended int2long.
543 nRotate &= 0x003f; // For risbg, pretend it's an unsigned value.
544 z_risbg(dst, src, lBitPos, rBitPos, nRotate, clear_dst); // Rotate, then insert selected, clear the rest.
545 }
546
547 // Rotate src, then and selected range from rotated src into dst.
548 // Set condition code only if so requested. Otherwise it is unpredictable.
549 // See performance note in macroAssembler_s390.hpp for important information.
550 void MacroAssembler::rotate_then_and(Register dst, Register src, int lBitPos, int rBitPos,
551 int nRotate, bool test_only) {
552 guarantee(!test_only, "Emitter not fit for test_only instruction variant.");
553 // This version does not depend on src being zero-extended int2long.
554 nRotate &= 0x003f; // For risbg, pretend it's an unsigned value.
555 z_rxsbg(dst, src, lBitPos, rBitPos, nRotate, test_only); // Rotate, then xor selected.
556 }
557
558 // Rotate src, then or selected range from rotated src into dst.
559 // Set condition code only if so requested. Otherwise it is unpredictable.
560 // See performance note in macroAssembler_s390.hpp for important information.
561 void MacroAssembler::rotate_then_or(Register dst, Register src, int lBitPos, int rBitPos,
562 int nRotate, bool test_only) {
563 guarantee(!test_only, "Emitter not fit for test_only instruction variant.");
564 // This version does not depend on src being zero-extended int2long.
565 nRotate &= 0x003f; // For risbg, pretend it's an unsigned value.
566 z_rosbg(dst, src, lBitPos, rBitPos, nRotate, test_only); // Rotate, then xor selected.
567 }
568
569 // Rotate src, then xor selected range from rotated src into dst.
570 // Set condition code only if so requested. Otherwise it is unpredictable.
571 // See performance note in macroAssembler_s390.hpp for important information.
572 void MacroAssembler::rotate_then_xor(Register dst, Register src, int lBitPos, int rBitPos,
573 int nRotate, bool test_only) {
574 guarantee(!test_only, "Emitter not fit for test_only instruction variant.");
575 // This version does not depend on src being zero-extended int2long.
576 nRotate &= 0x003f; // For risbg, pretend it's an unsigned value.
577 z_rxsbg(dst, src, lBitPos, rBitPos, nRotate, test_only); // Rotate, then xor selected.
578 }
579
580 void MacroAssembler::add64(Register r1, RegisterOrConstant inc) {
581 if (inc.is_register()) {
582 z_agr(r1, inc.as_register());
583 } else { // constant
584 intptr_t imm = inc.as_constant();
585 add2reg(r1, imm);
586 }
587 }
588 // Helper function to multiply the 64bit contents of a register by a 16bit constant.
589 // The optimization tries to avoid the mghi instruction, since it uses the FPU for
590 // calculation and is thus rather slow.
591 //
592 // There is no handling for special cases, e.g. cval==0 or cval==1.
593 //
594 // Returns len of generated code block.
595 unsigned int MacroAssembler::mul_reg64_const16(Register rval, Register work, int cval) {
596 int block_start = offset();
597
598 bool sign_flip = cval < 0;
599 cval = sign_flip ? -cval : cval;
600
601 BLOCK_COMMENT("Reg64*Con16 {");
602
603 int bit1 = cval & -cval;
604 if (bit1 == cval) {
605 z_sllg(rval, rval, exact_log2(bit1));
606 if (sign_flip) { z_lcgr(rval, rval); }
607 } else {
608 int bit2 = (cval-bit1) & -(cval-bit1);
609 if ((bit1+bit2) == cval) {
610 z_sllg(work, rval, exact_log2(bit1));
611 z_sllg(rval, rval, exact_log2(bit2));
612 z_agr(rval, work);
613 if (sign_flip) { z_lcgr(rval, rval); }
614 } else {
615 if (sign_flip) { z_mghi(rval, -cval); }
616 else { z_mghi(rval, cval); }
617 }
618 }
619 BLOCK_COMMENT("} Reg64*Con16");
620
621 int block_end = offset();
622 return block_end - block_start;
623 }
624
625 // Generic operation r1 := r2 + imm.
626 //
627 // Should produce the best code for each supported CPU version.
628 // r2 == noreg yields r1 := r1 + imm
629 // imm == 0 emits either no instruction or r1 := r2 !
630 // NOTES: 1) Don't use this function where fixed sized
631 // instruction sequences are required!!!
632 // 2) Don't use this function if condition code
633 // setting is required!
634 // 3) Despite being declared as int64_t, the parameter imm
635 // must be a simm_32 value (= signed 32-bit integer).
636 void MacroAssembler::add2reg(Register r1, int64_t imm, Register r2) {
637 assert(Immediate::is_simm32(imm), "probably an implicit conversion went wrong");
638
639 if (r2 == noreg) { r2 = r1; }
640
641 // Handle special case imm == 0.
642 if (imm == 0) {
643 lgr_if_needed(r1, r2);
644 // Nothing else to do.
645 return;
646 }
647
648 if (!PreferLAoverADD || (r2 == Z_R0)) {
649 bool distinctOpnds = VM_Version::has_DistinctOpnds();
650
651 // Can we encode imm in 16 bits signed?
652 if (Immediate::is_simm16(imm)) {
653 if (r1 == r2) {
654 z_aghi(r1, imm);
655 return;
656 }
657 if (distinctOpnds) {
658 z_aghik(r1, r2, imm);
659 return;
660 }
661 lgr_if_needed(r1, r2);
662 z_aghi(r1, imm);
663 return;
664 }
665 } else {
666 // Can we encode imm in 12 bits unsigned?
667 if (Displacement::is_shortDisp(imm)) {
668 z_la(r1, imm, r2);
669 return;
670 }
671 // Can we encode imm in 20 bits signed?
672 if (Displacement::is_validDisp(imm)) {
673 // Always use LAY instruction, so we don't need the tmp register.
674 z_lay(r1, imm, r2);
675 return;
676 }
677
678 }
679
680 // Can handle it (all possible values) with long immediates.
681 lgr_if_needed(r1, r2);
682 z_agfi(r1, imm);
683 }
684
685 void MacroAssembler::add2reg_32(Register r1, int64_t imm, Register r2) {
686 assert(Immediate::is_simm32(imm), "probably an implicit conversion went wrong");
687
688 if (r2 == noreg) { r2 = r1; }
689
690 // Handle special case imm == 0.
691 if (imm == 0) {
692 lr_if_needed(r1, r2);
693 // Nothing else to do.
694 return;
695 }
696
697 if (Immediate::is_simm16(imm)) {
698 if (r1 == r2){
699 z_ahi(r1, imm);
700 return;
701 }
702 if (VM_Version::has_DistinctOpnds()) {
703 z_ahik(r1, r2, imm);
704 return;
705 }
706 lr_if_needed(r1, r2);
707 z_ahi(r1, imm);
708 return;
709 }
710
711 // imm is simm32
712 lr_if_needed(r1, r2);
713 z_afi(r1, imm);
714 }
715
716 // Generic operation r := b + x + d
717 //
718 // Addition of several operands with address generation semantics - sort of:
719 // - no restriction on the registers. Any register will do for any operand.
720 // - x == noreg: operand will be disregarded.
721 // - b == noreg: will use (contents of) result reg as operand (r := r + d).
722 // - x == Z_R0: just disregard
723 // - b == Z_R0: use as operand. This is not address generation semantics!!!
724 //
725 // The same restrictions as on add2reg() are valid!!!
726 void MacroAssembler::add2reg_with_index(Register r, int64_t d, Register x, Register b) {
727 assert(Immediate::is_simm32(d), "probably an implicit conversion went wrong");
728
729 if (x == noreg) { x = Z_R0; }
730 if (b == noreg) { b = r; }
731
732 // Handle special case x == R0.
733 if (x == Z_R0) {
734 // Can simply add the immediate value to the base register.
735 add2reg(r, d, b);
736 return;
737 }
738
739 if (!PreferLAoverADD || (b == Z_R0)) {
740 bool distinctOpnds = VM_Version::has_DistinctOpnds();
741 // Handle special case d == 0.
742 if (d == 0) {
743 if (b == x) { z_sllg(r, b, 1); return; }
744 if (r == x) { z_agr(r, b); return; }
745 if (r == b) { z_agr(r, x); return; }
746 if (distinctOpnds) { z_agrk(r, x, b); return; }
747 z_lgr(r, b);
748 z_agr(r, x);
749 } else {
750 if (x == b) { z_sllg(r, x, 1); }
751 else if (r == x) { z_agr(r, b); }
752 else if (r == b) { z_agr(r, x); }
753 else if (distinctOpnds) { z_agrk(r, x, b); }
754 else {
755 z_lgr(r, b);
756 z_agr(r, x);
757 }
758 add2reg(r, d);
759 }
760 } else {
761 // Can we encode imm in 12 bits unsigned?
762 if (Displacement::is_shortDisp(d)) {
763 z_la(r, d, x, b);
764 return;
765 }
766 // Can we encode imm in 20 bits signed?
767 if (Displacement::is_validDisp(d)) {
768 z_lay(r, d, x, b);
769 return;
770 }
771 z_la(r, 0, x, b);
772 add2reg(r, d);
773 }
774 }
775
776 // Generic emitter (32bit) for direct memory increment.
777 // For optimal code, do not specify Z_R0 as temp register.
778 void MacroAssembler::add2mem_32(const Address &a, int64_t imm, Register tmp) {
779 if (VM_Version::has_MemWithImmALUOps() && Immediate::is_simm8(imm)) {
780 z_asi(a, imm);
781 } else {
782 z_lgf(tmp, a);
783 add2reg(tmp, imm);
784 z_st(tmp, a);
785 }
786 }
787
788 void MacroAssembler::add2mem_64(const Address &a, int64_t imm, Register tmp) {
789 if (VM_Version::has_MemWithImmALUOps() && Immediate::is_simm8(imm)) {
790 z_agsi(a, imm);
791 } else {
792 z_lg(tmp, a);
793 add2reg(tmp, imm);
794 z_stg(tmp, a);
795 }
796 }
797
798 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) {
799 switch (size_in_bytes) {
800 case 8: z_lg(dst, src); break;
801 case 4: is_signed ? z_lgf(dst, src) : z_llgf(dst, src); break;
802 case 2: is_signed ? z_lgh(dst, src) : z_llgh(dst, src); break;
803 case 1: is_signed ? z_lgb(dst, src) : z_llgc(dst, src); break;
804 default: ShouldNotReachHere();
805 }
806 }
807
808 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes) {
809 switch (size_in_bytes) {
810 case 8: z_stg(src, dst); break;
811 case 4: z_st(src, dst); break;
812 case 2: z_sth(src, dst); break;
813 case 1: z_stc(src, dst); break;
814 default: ShouldNotReachHere();
815 }
816 }
817
818 // Split a si20 offset (20bit, signed) into an ui12 offset (12bit, unsigned) and
819 // a high-order summand in register tmp.
820 //
821 // return value: < 0: No split required, si20 actually has property uimm12.
822 // >= 0: Split performed. Use return value as uimm12 displacement and
823 // tmp as index register.
824 int MacroAssembler::split_largeoffset(int64_t si20_offset, Register tmp, bool fixed_codelen, bool accumulate) {
825 assert(Immediate::is_simm20(si20_offset), "sanity");
826 int lg_off = (int)si20_offset & 0x0fff; // Punch out low-order 12 bits, always positive.
827 int ll_off = (int)si20_offset & ~0x0fff; // Force low-order 12 bits to zero.
828 assert((Displacement::is_shortDisp(si20_offset) && (ll_off == 0)) ||
829 !Displacement::is_shortDisp(si20_offset), "unexpected offset values");
830 assert((lg_off+ll_off) == si20_offset, "offset splitup error");
831
832 Register work = accumulate? Z_R0 : tmp;
833
834 if (fixed_codelen) { // Len of code = 10 = 4 + 6.
835 z_lghi(work, ll_off>>12); // Implicit sign extension.
836 z_slag(work, work, 12);
837 } else { // Len of code = 0..10.
838 if (ll_off == 0) { return -1; }
839 // ll_off has 8 significant bits (at most) plus sign.
840 if ((ll_off & 0x0000f000) == 0) { // Non-zero bits only in upper halfbyte.
841 z_llilh(work, ll_off >> 16);
842 if (ll_off < 0) { // Sign-extension required.
843 z_lgfr(work, work);
844 }
845 } else {
846 if ((ll_off & 0x000f0000) == 0) { // Non-zero bits only in lower halfbyte.
847 z_llill(work, ll_off);
848 } else { // Non-zero bits in both halfbytes.
849 z_lghi(work, ll_off>>12); // Implicit sign extension.
850 z_slag(work, work, 12);
851 }
852 }
853 }
854 if (accumulate) { z_algr(tmp, work); } // len of code += 4
855 return lg_off;
856 }
857
858 void MacroAssembler::load_float_largeoffset(FloatRegister t, int64_t si20, Register a, Register tmp) {
859 if (Displacement::is_validDisp(si20)) {
860 z_ley(t, si20, a);
861 } else {
862 // Fixed_codelen = true is a simple way to ensure that the size of load_float_largeoffset
863 // does not depend on si20 (scratch buffer emit size == code buffer emit size for constant
864 // pool loads).
865 bool accumulate = true;
866 bool fixed_codelen = true;
867 Register work;
868
869 if (fixed_codelen) {
870 z_lgr(tmp, a); // Lgr_if_needed not applicable due to fixed_codelen.
871 } else {
872 accumulate = (a == tmp);
873 }
874 work = tmp;
875
876 int disp12 = split_largeoffset(si20, work, fixed_codelen, accumulate);
877 if (disp12 < 0) {
878 z_le(t, si20, work);
879 } else {
880 if (accumulate) {
881 z_le(t, disp12, work);
882 } else {
883 z_le(t, disp12, work, a);
884 }
885 }
886 }
887 }
888
889 void MacroAssembler::load_double_largeoffset(FloatRegister t, int64_t si20, Register a, Register tmp) {
890 if (Displacement::is_validDisp(si20)) {
891 z_ldy(t, si20, a);
892 } else {
893 // Fixed_codelen = true is a simple way to ensure that the size of load_double_largeoffset
894 // does not depend on si20 (scratch buffer emit size == code buffer emit size for constant
895 // pool loads).
896 bool accumulate = true;
897 bool fixed_codelen = true;
898 Register work;
899
900 if (fixed_codelen) {
901 z_lgr(tmp, a); // Lgr_if_needed not applicable due to fixed_codelen.
902 } else {
903 accumulate = (a == tmp);
904 }
905 work = tmp;
906
907 int disp12 = split_largeoffset(si20, work, fixed_codelen, accumulate);
908 if (disp12 < 0) {
909 z_ld(t, si20, work);
910 } else {
911 if (accumulate) {
912 z_ld(t, disp12, work);
913 } else {
914 z_ld(t, disp12, work, a);
915 }
916 }
917 }
918 }
919
920 // PCrelative TOC access.
921 // Returns distance (in bytes) from current position to start of consts section.
922 // Returns 0 (zero) if no consts section exists or if it has size zero.
923 long MacroAssembler::toc_distance() {
924 CodeSection* cs = code()->consts();
925 return (long)((cs != nullptr) ? cs->start()-pc() : 0);
926 }
927
928 // Implementation on x86/sparc assumes that constant and instruction section are
929 // adjacent, but this doesn't hold. Two special situations may occur, that we must
930 // be able to handle:
931 // 1. const section may be located apart from the inst section.
932 // 2. const section may be empty
933 // In both cases, we use the const section's start address to compute the "TOC",
934 // this seems to occur only temporarily; in the final step we always seem to end up
935 // with the pc-relatice variant.
936 //
937 // PC-relative offset could be +/-2**32 -> use long for disp
938 // Furthermore: makes no sense to have special code for
939 // adjacent const and inst sections.
940 void MacroAssembler::load_toc(Register Rtoc) {
941 // Simply use distance from start of const section (should be patched in the end).
942 long disp = toc_distance();
943
944 RelocationHolder rspec = internal_word_Relocation::spec(pc() + disp);
945 relocate(rspec);
946 z_larl(Rtoc, RelAddr::pcrel_off32(disp)); // Offset is in halfwords.
947 }
948
949 // PCrelative TOC access.
950 // Load from anywhere pcrelative (with relocation of load instr)
951 void MacroAssembler::load_long_pcrelative(Register Rdst, address dataLocation) {
952 address pc = this->pc();
953 ptrdiff_t total_distance = dataLocation - pc;
954 RelocationHolder rspec = internal_word_Relocation::spec(dataLocation);
955
956 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
957 assert(total_distance != 0, "sanity");
958
959 // Some extra safety net.
960 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
961 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
962 }
963
964 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
965 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
966 }
967
968
969 // PCrelative TOC access.
970 // Load from anywhere pcrelative (with relocation of load instr)
971 // loaded addr has to be relocated when added to constant pool.
972 void MacroAssembler::load_addr_pcrelative(Register Rdst, address addrLocation) {
973 address pc = this->pc();
974 ptrdiff_t total_distance = addrLocation - pc;
975 RelocationHolder rspec = internal_word_Relocation::spec(addrLocation);
976
977 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
978
979 // Some extra safety net.
980 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
981 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
982 }
983
984 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
985 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
986 }
987
988 // Generic operation: load a value from memory and test.
989 // CondCode indicates the sign (<0, ==0, >0) of the loaded value.
990 void MacroAssembler::load_and_test_byte(Register dst, const Address &a) {
991 z_lb(dst, a);
992 z_ltr(dst, dst);
993 }
994
995 void MacroAssembler::load_and_test_short(Register dst, const Address &a) {
996 int64_t disp = a.disp20();
997 if (Displacement::is_shortDisp(disp)) {
998 z_lh(dst, a);
999 } else if (Displacement::is_longDisp(disp)) {
1000 z_lhy(dst, a);
1001 } else {
1002 guarantee(false, "displacement out of range");
1003 }
1004 z_ltr(dst, dst);
1005 }
1006
1007 void MacroAssembler::load_and_test_int(Register dst, const Address &a) {
1008 z_lt(dst, a);
1009 }
1010
1011 void MacroAssembler::load_and_test_int2long(Register dst, const Address &a) {
1012 z_ltgf(dst, a);
1013 }
1014
1015 void MacroAssembler::load_and_test_long(Register dst, const Address &a) {
1016 z_ltg(dst, a);
1017 }
1018
1019 // Test a bit in memory for 2 byte datatype.
1020 void MacroAssembler::testbit_ushort(const Address &a, unsigned int bit) {
1021 assert(a.index() == noreg, "no index reg allowed in testbit");
1022 if (bit <= 7) {
1023 z_tm(a.disp() + 1, a.base(), 1 << bit);
1024 } else if (bit <= 15) {
1025 z_tm(a.disp() + 0, a.base(), 1 << (bit - 8));
1026 } else {
1027 ShouldNotReachHere();
1028 }
1029 }
1030
1031 // Test a bit in memory.
1032 void MacroAssembler::testbit(const Address &a, unsigned int bit) {
1033 assert(a.index() == noreg, "no index reg allowed in testbit");
1034 if (bit <= 7) {
1035 z_tm(a.disp() + 3, a.base(), 1 << bit);
1036 } else if (bit <= 15) {
1037 z_tm(a.disp() + 2, a.base(), 1 << (bit - 8));
1038 } else if (bit <= 23) {
1039 z_tm(a.disp() + 1, a.base(), 1 << (bit - 16));
1040 } else if (bit <= 31) {
1041 z_tm(a.disp() + 0, a.base(), 1 << (bit - 24));
1042 } else {
1043 ShouldNotReachHere();
1044 }
1045 }
1046
1047 // Test a bit in a register. Result is reflected in CC.
1048 void MacroAssembler::testbit(Register r, unsigned int bitPos) {
1049 if (bitPos < 16) {
1050 z_tmll(r, 1U<<bitPos);
1051 } else if (bitPos < 32) {
1052 z_tmlh(r, 1U<<(bitPos-16));
1053 } else if (bitPos < 48) {
1054 z_tmhl(r, 1U<<(bitPos-32));
1055 } else if (bitPos < 64) {
1056 z_tmhh(r, 1U<<(bitPos-48));
1057 } else {
1058 ShouldNotReachHere();
1059 }
1060 }
1061
1062 void MacroAssembler::prefetch_read(Address a) {
1063 z_pfd(1, a.disp20(), a.indexOrR0(), a.base());
1064 }
1065 void MacroAssembler::prefetch_update(Address a) {
1066 z_pfd(2, a.disp20(), a.indexOrR0(), a.base());
1067 }
1068
1069 // Clear a register, i.e. load const zero into reg.
1070 // Return len (in bytes) of generated instruction(s).
1071 // whole_reg: Clear 64 bits if true, 32 bits otherwise.
1072 // set_cc: Use instruction that sets the condition code, if true.
1073 int MacroAssembler::clear_reg(Register r, bool whole_reg, bool set_cc) {
1074 unsigned int start_off = offset();
1075 if (whole_reg) {
1076 set_cc ? z_xgr(r, r) : z_laz(r, 0, Z_R0);
1077 } else { // Only 32bit register.
1078 set_cc ? z_xr(r, r) : z_lhi(r, 0);
1079 }
1080 return offset() - start_off;
1081 }
1082
1083 #ifdef ASSERT
1084 int MacroAssembler::preset_reg(Register r, unsigned long pattern, int pattern_len) {
1085 switch (pattern_len) {
1086 case 1:
1087 pattern = (pattern & 0x000000ff) | ((pattern & 0x000000ff)<<8);
1088 case 2:
1089 pattern = (pattern & 0x0000ffff) | ((pattern & 0x0000ffff)<<16);
1090 case 4:
1091 pattern = (pattern & 0xffffffffL) | ((pattern & 0xffffffffL)<<32);
1092 case 8:
1093 return load_const_optimized_rtn_len(r, pattern, true);
1094 break;
1095 default:
1096 guarantee(false, "preset_reg: bad len");
1097 }
1098 return 0;
1099 }
1100 #endif
1101
1102 // addr: Address descriptor of memory to clear. Index register will not be used!
1103 // size: Number of bytes to clear.
1104 // condition code will not be preserved.
1105 // !!! DO NOT USE THEM FOR ATOMIC MEMORY CLEARING !!!
1106 // !!! Use store_const() instead !!!
1107 void MacroAssembler::clear_mem(const Address& addr, unsigned int size) {
1108 guarantee((addr.disp() + size) <= 4096, "MacroAssembler::clear_mem: size too large");
1109
1110 switch (size) {
1111 case 0:
1112 return;
1113 case 1:
1114 z_mvi(addr, 0);
1115 return;
1116 case 2:
1117 z_mvhhi(addr, 0);
1118 return;
1119 case 4:
1120 z_mvhi(addr, 0);
1121 return;
1122 case 8:
1123 z_mvghi(addr, 0);
1124 return;
1125 default: ; // Fallthru to xc.
1126 }
1127
1128 // Caution: the emitter with Address operands does implicitly decrement the length
1129 if (size <= 256) {
1130 z_xc(addr, size, addr);
1131 } else {
1132 unsigned int offset = addr.disp();
1133 unsigned int incr = 256;
1134 for (unsigned int i = 0; i <= size-incr; i += incr) {
1135 z_xc(offset, incr - 1, addr.base(), offset, addr.base());
1136 offset += incr;
1137 }
1138 unsigned int rest = size - (offset - addr.disp());
1139 if (size > 0) {
1140 z_xc(offset, rest-1, addr.base(), offset, addr.base());
1141 }
1142 }
1143 }
1144
1145 void MacroAssembler::align(int modulus) {
1146 align(modulus, offset());
1147 }
1148
1149 void MacroAssembler::align(int modulus, int target) {
1150 assert(((modulus % 2 == 0) && (target % 2 == 0)), "needs to be even");
1151 int delta = target - offset();
1152 while ((offset() + delta) % modulus != 0) z_nop();
1153 }
1154
1155 // Special version for non-relocateable code if required alignment
1156 // is larger than CodeEntryAlignment.
1157 void MacroAssembler::align_address(int modulus) {
1158 while ((uintptr_t)pc() % modulus != 0) z_nop();
1159 }
1160
1161 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
1162 Register temp_reg,
1163 int64_t extra_slot_offset) {
1164 // On Z, we can have index and disp in an Address. So don't call argument_offset,
1165 // which issues an unnecessary add instruction.
1166 int stackElementSize = Interpreter::stackElementSize;
1167 int64_t offset = extra_slot_offset * stackElementSize;
1168 const Register argbase = Z_esp;
1169 if (arg_slot.is_constant()) {
1170 offset += arg_slot.as_constant() * stackElementSize;
1171 return Address(argbase, offset);
1172 }
1173 // else
1174 assert(temp_reg != noreg, "must specify");
1175 assert(temp_reg != Z_ARG1, "base and index are conflicting");
1176 z_sllg(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize)); // tempreg = arg_slot << 3
1177 return Address(argbase, temp_reg, offset);
1178 }
1179
1180
1181 //===================================================================
1182 //=== START C O N S T A N T S I N C O D E S T R E A M ===
1183 //===================================================================
1184 //=== P A T CH A B L E C O N S T A N T S ===
1185 //===================================================================
1186
1187
1188 //---------------------------------------------------
1189 // Load (patchable) constant into register
1190 //---------------------------------------------------
1191
1192
1193 // Load absolute address (and try to optimize).
1194 // Note: This method is usable only for position-fixed code,
1195 // referring to a position-fixed target location.
1196 // If not so, relocations and patching must be used.
1197 void MacroAssembler::load_absolute_address(Register d, address addr) {
1198 assert(addr != nullptr, "should not happen");
1199 BLOCK_COMMENT("load_absolute_address:");
1200 if (addr == nullptr) {
1201 z_larl(d, pc()); // Dummy emit for size calc.
1202 return;
1203 }
1204
1205 if (RelAddr::is_in_range_of_RelAddr32(addr, pc())) {
1206 z_larl(d, addr);
1207 return;
1208 }
1209
1210 load_const_optimized(d, (long)addr);
1211 }
1212
1213 // Load a 64bit constant.
1214 // Patchable code sequence, but not atomically patchable.
1215 // Make sure to keep code size constant -> no value-dependent optimizations.
1216 // Do not kill condition code.
1217 void MacroAssembler::load_const(Register t, long x) {
1218 // Note: Right shift is only cleanly defined for unsigned types
1219 // or for signed types with nonnegative values.
1220 Assembler::z_iihf(t, (long)((unsigned long)x >> 32));
1221 Assembler::z_iilf(t, (long)((unsigned long)x & 0xffffffffUL));
1222 }
1223
1224 // Load a 32bit constant into a 64bit register, sign-extend or zero-extend.
1225 // Patchable code sequence, but not atomically patchable.
1226 // Make sure to keep code size constant -> no value-dependent optimizations.
1227 // Do not kill condition code.
1228 void MacroAssembler::load_const_32to64(Register t, int64_t x, bool sign_extend) {
1229 if (sign_extend) { Assembler::z_lgfi(t, x); }
1230 else { Assembler::z_llilf(t, x); }
1231 }
1232
1233 // Load narrow oop constant, no decompression.
1234 void MacroAssembler::load_narrow_oop(Register t, narrowOop a) {
1235 assert(UseCompressedOops, "must be on to call this method");
1236 load_const_32to64(t, CompressedOops::narrow_oop_value(a), false /*sign_extend*/);
1237 }
1238
1239 // Load narrow klass constant, compression required.
1240 void MacroAssembler::load_narrow_klass(Register t, Klass* k) {
1241 narrowKlass encoded_k = CompressedKlassPointers::encode(k);
1242 load_const_32to64(t, encoded_k, false /*sign_extend*/);
1243 }
1244
1245 //------------------------------------------------------
1246 // Compare (patchable) constant with register.
1247 //------------------------------------------------------
1248
1249 // Compare narrow oop in reg with narrow oop constant, no decompression.
1250 void MacroAssembler::compare_immediate_narrow_oop(Register oop1, narrowOop oop2) {
1251 assert(UseCompressedOops, "must be on to call this method");
1252
1253 Assembler::z_clfi(oop1, CompressedOops::narrow_oop_value(oop2));
1254 }
1255
1256 // Compare narrow oop in reg with narrow oop constant, no decompression.
1257 void MacroAssembler::compare_immediate_narrow_klass(Register klass1, Klass* klass2) {
1258 narrowKlass encoded_k = CompressedKlassPointers::encode(klass2);
1259
1260 Assembler::z_clfi(klass1, encoded_k);
1261 }
1262
1263 //----------------------------------------------------------
1264 // Check which kind of load_constant we have here.
1265 //----------------------------------------------------------
1266
1267 // Detection of CPU version dependent load_const sequence.
1268 // The detection is valid only for code sequences generated by load_const,
1269 // not load_const_optimized.
1270 bool MacroAssembler::is_load_const(address a) {
1271 unsigned long inst1, inst2;
1272 unsigned int len1, len2;
1273
1274 len1 = get_instruction(a, &inst1);
1275 len2 = get_instruction(a + len1, &inst2);
1276
1277 return is_z_iihf(inst1) && is_z_iilf(inst2);
1278 }
1279
1280 // Detection of CPU version dependent load_const_32to64 sequence.
1281 // Mostly used for narrow oops and narrow Klass pointers.
1282 // The detection is valid only for code sequences generated by load_const_32to64.
1283 bool MacroAssembler::is_load_const_32to64(address pos) {
1284 unsigned long inst1, inst2;
1285 unsigned int len1;
1286
1287 len1 = get_instruction(pos, &inst1);
1288 return is_z_llilf(inst1);
1289 }
1290
1291 // Detection of compare_immediate_narrow sequence.
1292 // The detection is valid only for code sequences generated by compare_immediate_narrow_oop.
1293 bool MacroAssembler::is_compare_immediate32(address pos) {
1294 return is_equal(pos, CLFI_ZOPC, RIL_MASK);
1295 }
1296
1297 // Detection of compare_immediate_narrow sequence.
1298 // The detection is valid only for code sequences generated by compare_immediate_narrow_oop.
1299 bool MacroAssembler::is_compare_immediate_narrow_oop(address pos) {
1300 return is_compare_immediate32(pos);
1301 }
1302
1303 // Detection of compare_immediate_narrow sequence.
1304 // The detection is valid only for code sequences generated by compare_immediate_narrow_klass.
1305 bool MacroAssembler::is_compare_immediate_narrow_klass(address pos) {
1306 return is_compare_immediate32(pos);
1307 }
1308
1309 //-----------------------------------
1310 // patch the load_constant
1311 //-----------------------------------
1312
1313 // CPU-version dependent patching of load_const.
1314 void MacroAssembler::patch_const(address a, long x) {
1315 assert(is_load_const(a), "not a load of a constant");
1316 // Note: Right shift is only cleanly defined for unsigned types
1317 // or for signed types with nonnegative values.
1318 set_imm32((address)a, (long)((unsigned long)x >> 32));
1319 set_imm32((address)(a + 6), (long)((unsigned long)x & 0xffffffffUL));
1320 }
1321
1322 // Patching the value of CPU version dependent load_const_32to64 sequence.
1323 // The passed ptr MUST be in compressed format!
1324 int MacroAssembler::patch_load_const_32to64(address pos, int64_t np) {
1325 assert(is_load_const_32to64(pos), "not a load of a narrow ptr (oop or klass)");
1326
1327 set_imm32(pos, np);
1328 return 6;
1329 }
1330
1331 // Patching the value of CPU version dependent compare_immediate_narrow sequence.
1332 // The passed ptr MUST be in compressed format!
1333 int MacroAssembler::patch_compare_immediate_32(address pos, int64_t np) {
1334 assert(is_compare_immediate32(pos), "not a compressed ptr compare");
1335
1336 set_imm32(pos, np);
1337 return 6;
1338 }
1339
1340 // Patching the immediate value of CPU version dependent load_narrow_oop sequence.
1341 // The passed ptr must NOT be in compressed format!
1342 int MacroAssembler::patch_load_narrow_oop(address pos, oop o) {
1343 assert(UseCompressedOops, "Can only patch compressed oops");
1344 return patch_load_const_32to64(pos, CompressedOops::narrow_oop_value(o));
1345 }
1346
1347 // Patching the immediate value of CPU version dependent load_narrow_klass sequence.
1348 // The passed ptr must NOT be in compressed format!
1349 int MacroAssembler::patch_load_narrow_klass(address pos, Klass* k) {
1350 narrowKlass nk = CompressedKlassPointers::encode(k);
1351 return patch_load_const_32to64(pos, nk);
1352 }
1353
1354 // Patching the immediate value of CPU version dependent compare_immediate_narrow_oop sequence.
1355 // The passed ptr must NOT be in compressed format!
1356 int MacroAssembler::patch_compare_immediate_narrow_oop(address pos, oop o) {
1357 assert(UseCompressedOops, "Can only patch compressed oops");
1358 return patch_compare_immediate_32(pos, CompressedOops::narrow_oop_value(o));
1359 }
1360
1361 // Patching the immediate value of CPU version dependent compare_immediate_narrow_klass sequence.
1362 // The passed ptr must NOT be in compressed format!
1363 int MacroAssembler::patch_compare_immediate_narrow_klass(address pos, Klass* k) {
1364 narrowKlass nk = CompressedKlassPointers::encode(k);
1365 return patch_compare_immediate_32(pos, nk);
1366 }
1367
1368 //------------------------------------------------------------------------
1369 // Extract the constant from a load_constant instruction stream.
1370 //------------------------------------------------------------------------
1371
1372 // Get constant from a load_const sequence.
1373 long MacroAssembler::get_const(address a) {
1374 assert(is_load_const(a), "not a load of a constant");
1375 unsigned long x;
1376 x = (((unsigned long) (get_imm32(a,0) & 0xffffffff)) << 32);
1377 x |= (((unsigned long) (get_imm32(a,1) & 0xffffffff)));
1378 return (long) x;
1379 }
1380
1381 //--------------------------------------
1382 // Store a constant in memory.
1383 //--------------------------------------
1384
1385 // General emitter to move a constant to memory.
1386 // The store is atomic.
1387 // o Address must be given in RS format (no index register)
1388 // o Displacement should be 12bit unsigned for efficiency. 20bit signed also supported.
1389 // o Constant can be 1, 2, 4, or 8 bytes, signed or unsigned.
1390 // o Memory slot can be 1, 2, 4, or 8 bytes, signed or unsigned.
1391 // o Memory slot must be at least as wide as constant, will assert otherwise.
1392 // o Signed constants will sign-extend, unsigned constants will zero-extend to slot width.
1393 int MacroAssembler::store_const(const Address &dest, long imm,
1394 unsigned int lm, unsigned int lc,
1395 Register scratch) {
1396 int64_t disp = dest.disp();
1397 Register base = dest.base();
1398 assert(!dest.has_index(), "not supported");
1399 assert((lm==1)||(lm==2)||(lm==4)||(lm==8), "memory length not supported");
1400 assert((lc==1)||(lc==2)||(lc==4)||(lc==8), "constant length not supported");
1401 assert(lm>=lc, "memory slot too small");
1402 assert(lc==8 || Immediate::is_simm(imm, lc*8), "const out of range");
1403 assert(Displacement::is_validDisp(disp), "displacement out of range");
1404
1405 bool is_shortDisp = Displacement::is_shortDisp(disp);
1406 int store_offset = -1;
1407
1408 // For target len == 1 it's easy.
1409 if (lm == 1) {
1410 store_offset = offset();
1411 if (is_shortDisp) {
1412 z_mvi(disp, base, imm);
1413 return store_offset;
1414 } else {
1415 z_mviy(disp, base, imm);
1416 return store_offset;
1417 }
1418 }
1419
1420 // All the "good stuff" takes an unsigned displacement.
1421 if (is_shortDisp) {
1422 // NOTE: Cannot use clear_mem for imm==0, because it is not atomic.
1423
1424 store_offset = offset();
1425 switch (lm) {
1426 case 2: // Lc == 1 handled correctly here, even for unsigned. Instruction does no widening.
1427 z_mvhhi(disp, base, imm);
1428 return store_offset;
1429 case 4:
1430 if (Immediate::is_simm16(imm)) {
1431 z_mvhi(disp, base, imm);
1432 return store_offset;
1433 }
1434 break;
1435 case 8:
1436 if (Immediate::is_simm16(imm)) {
1437 z_mvghi(disp, base, imm);
1438 return store_offset;
1439 }
1440 break;
1441 default:
1442 ShouldNotReachHere();
1443 break;
1444 }
1445 }
1446
1447 // Can't optimize, so load value and store it.
1448 guarantee(scratch != noreg, " need a scratch register here !");
1449 if (imm != 0) {
1450 load_const_optimized(scratch, imm); // Preserves CC anyway.
1451 } else {
1452 // Leave CC alone!!
1453 (void) clear_reg(scratch, true, false); // Indicate unused result.
1454 }
1455
1456 store_offset = offset();
1457 if (is_shortDisp) {
1458 switch (lm) {
1459 case 2:
1460 z_sth(scratch, disp, Z_R0, base);
1461 return store_offset;
1462 case 4:
1463 z_st(scratch, disp, Z_R0, base);
1464 return store_offset;
1465 case 8:
1466 z_stg(scratch, disp, Z_R0, base);
1467 return store_offset;
1468 default:
1469 ShouldNotReachHere();
1470 break;
1471 }
1472 } else {
1473 switch (lm) {
1474 case 2:
1475 z_sthy(scratch, disp, Z_R0, base);
1476 return store_offset;
1477 case 4:
1478 z_sty(scratch, disp, Z_R0, base);
1479 return store_offset;
1480 case 8:
1481 z_stg(scratch, disp, Z_R0, base);
1482 return store_offset;
1483 default:
1484 ShouldNotReachHere();
1485 break;
1486 }
1487 }
1488 return -1; // should not reach here
1489 }
1490
1491 //===================================================================
1492 //=== N O T P A T CH A B L E C O N S T A N T S ===
1493 //===================================================================
1494
1495 // Load constant x into register t with a fast instruction sequence
1496 // depending on the bits in x. Preserves CC under all circumstances.
1497 int MacroAssembler::load_const_optimized_rtn_len(Register t, long x, bool emit) {
1498 if (x == 0) {
1499 int len;
1500 if (emit) {
1501 len = clear_reg(t, true, false);
1502 } else {
1503 len = 4;
1504 }
1505 return len;
1506 }
1507
1508 if (Immediate::is_simm16(x)) {
1509 if (emit) { z_lghi(t, x); }
1510 return 4;
1511 }
1512
1513 // 64 bit value: | part1 | part2 | part3 | part4 |
1514 // At least one part is not zero!
1515 // Note: Right shift is only cleanly defined for unsigned types
1516 // or for signed types with nonnegative values.
1517 int part1 = (int)((unsigned long)x >> 48) & 0x0000ffff;
1518 int part2 = (int)((unsigned long)x >> 32) & 0x0000ffff;
1519 int part3 = (int)((unsigned long)x >> 16) & 0x0000ffff;
1520 int part4 = (int)x & 0x0000ffff;
1521 int part12 = (int)((unsigned long)x >> 32);
1522 int part34 = (int)x;
1523
1524 // Lower word only (unsigned).
1525 if (part12 == 0) {
1526 if (part3 == 0) {
1527 if (emit) z_llill(t, part4);
1528 return 4;
1529 }
1530 if (part4 == 0) {
1531 if (emit) z_llilh(t, part3);
1532 return 4;
1533 }
1534 if (emit) z_llilf(t, part34);
1535 return 6;
1536 }
1537
1538 // Upper word only.
1539 if (part34 == 0) {
1540 if (part1 == 0) {
1541 if (emit) z_llihl(t, part2);
1542 return 4;
1543 }
1544 if (part2 == 0) {
1545 if (emit) z_llihh(t, part1);
1546 return 4;
1547 }
1548 if (emit) z_llihf(t, part12);
1549 return 6;
1550 }
1551
1552 // Lower word only (signed).
1553 if ((part1 == 0x0000ffff) && (part2 == 0x0000ffff) && ((part3 & 0x00008000) != 0)) {
1554 if (emit) z_lgfi(t, part34);
1555 return 6;
1556 }
1557
1558 int len = 0;
1559
1560 if ((part1 == 0) || (part2 == 0)) {
1561 if (part1 == 0) {
1562 if (emit) z_llihl(t, part2);
1563 len += 4;
1564 } else {
1565 if (emit) z_llihh(t, part1);
1566 len += 4;
1567 }
1568 } else {
1569 if (emit) z_llihf(t, part12);
1570 len += 6;
1571 }
1572
1573 if ((part3 == 0) || (part4 == 0)) {
1574 if (part3 == 0) {
1575 if (emit) z_iill(t, part4);
1576 len += 4;
1577 } else {
1578 if (emit) z_iilh(t, part3);
1579 len += 4;
1580 }
1581 } else {
1582 if (emit) z_iilf(t, part34);
1583 len += 6;
1584 }
1585 return len;
1586 }
1587
1588 //=====================================================================
1589 //=== H I G H E R L E V E L B R A N C H E M I T T E R S ===
1590 //=====================================================================
1591
1592 // Note: In the worst case, one of the scratch registers is destroyed!!!
1593 void MacroAssembler::compare32_and_branch(Register r1, RegisterOrConstant x2, branch_condition cond, Label& lbl) {
1594 // Right operand is constant.
1595 if (x2.is_constant()) {
1596 jlong value = x2.as_constant();
1597 compare_and_branch_optimized(r1, value, cond, lbl, /*len64=*/false, /*has_sign=*/true);
1598 return;
1599 }
1600
1601 // Right operand is in register.
1602 compare_and_branch_optimized(r1, x2.as_register(), cond, lbl, /*len64=*/false, /*has_sign=*/true);
1603 }
1604
1605 // Note: In the worst case, one of the scratch registers is destroyed!!!
1606 void MacroAssembler::compareU32_and_branch(Register r1, RegisterOrConstant x2, branch_condition cond, Label& lbl) {
1607 // Right operand is constant.
1608 if (x2.is_constant()) {
1609 jlong value = x2.as_constant();
1610 compare_and_branch_optimized(r1, value, cond, lbl, /*len64=*/false, /*has_sign=*/false);
1611 return;
1612 }
1613
1614 // Right operand is in register.
1615 compare_and_branch_optimized(r1, x2.as_register(), cond, lbl, /*len64=*/false, /*has_sign=*/false);
1616 }
1617
1618 // Note: In the worst case, one of the scratch registers is destroyed!!!
1619 void MacroAssembler::compare64_and_branch(Register r1, RegisterOrConstant x2, branch_condition cond, Label& lbl) {
1620 // Right operand is constant.
1621 if (x2.is_constant()) {
1622 jlong value = x2.as_constant();
1623 compare_and_branch_optimized(r1, value, cond, lbl, /*len64=*/true, /*has_sign=*/true);
1624 return;
1625 }
1626
1627 // Right operand is in register.
1628 compare_and_branch_optimized(r1, x2.as_register(), cond, lbl, /*len64=*/true, /*has_sign=*/true);
1629 }
1630
1631 void MacroAssembler::compareU64_and_branch(Register r1, RegisterOrConstant x2, branch_condition cond, Label& lbl) {
1632 // Right operand is constant.
1633 if (x2.is_constant()) {
1634 jlong value = x2.as_constant();
1635 compare_and_branch_optimized(r1, value, cond, lbl, /*len64=*/true, /*has_sign=*/false);
1636 return;
1637 }
1638
1639 // Right operand is in register.
1640 compare_and_branch_optimized(r1, x2.as_register(), cond, lbl, /*len64=*/true, /*has_sign=*/false);
1641 }
1642
1643 // Generate an optimal branch to the branch target.
1644 // Optimal means that a relative branch (brc or brcl) is used if the
1645 // branch distance is short enough. Loading the target address into a
1646 // register and branching via reg is used as fallback only.
1647 //
1648 // Used registers:
1649 // Z_R1 - work reg. Holds branch target address.
1650 // Used in fallback case only.
1651 //
1652 // This version of branch_optimized is good for cases where the target address is known
1653 // and constant, i.e. is never changed (no relocation, no patching).
1654 void MacroAssembler::branch_optimized(Assembler::branch_condition cond, address branch_addr) {
1655 address branch_origin = pc();
1656
1657 if (RelAddr::is_in_range_of_RelAddr16(branch_addr, branch_origin)) {
1658 z_brc(cond, branch_addr);
1659 } else if (RelAddr::is_in_range_of_RelAddr32(branch_addr, branch_origin)) {
1660 z_brcl(cond, branch_addr);
1661 } else {
1662 load_const_optimized(Z_R1, branch_addr); // CC must not get killed by load_const_optimized.
1663 z_bcr(cond, Z_R1);
1664 }
1665 }
1666
1667 // This version of branch_optimized is good for cases where the target address
1668 // is potentially not yet known at the time the code is emitted.
1669 //
1670 // One very common case is a branch to an unbound label which is handled here.
1671 // The caller might know (or hope) that the branch distance is short enough
1672 // to be encoded in a 16bit relative address. In this case he will pass a
1673 // NearLabel branch_target.
1674 // Care must be taken with unbound labels. Each call to target(label) creates
1675 // an entry in the patch queue for that label to patch all references of the label
1676 // once it gets bound. Those recorded patch locations must be patchable. Otherwise,
1677 // an assertion fires at patch time.
1678 void MacroAssembler::branch_optimized(Assembler::branch_condition cond, Label& branch_target) {
1679 if (branch_target.is_bound()) {
1680 address branch_addr = target(branch_target);
1681 branch_optimized(cond, branch_addr);
1682 } else if (branch_target.is_near()) {
1683 z_brc(cond, branch_target); // Caller assures that the target will be in range for z_brc.
1684 } else {
1685 z_brcl(cond, branch_target); // Let's hope target is in range. Otherwise, we will abort at patch time.
1686 }
1687 }
1688
1689 // Generate an optimal compare and branch to the branch target.
1690 // Optimal means that a relative branch (clgrj, brc or brcl) is used if the
1691 // branch distance is short enough. Loading the target address into a
1692 // register and branching via reg is used as fallback only.
1693 //
1694 // Input:
1695 // r1 - left compare operand
1696 // r2 - right compare operand
1697 void MacroAssembler::compare_and_branch_optimized(Register r1,
1698 Register r2,
1699 Assembler::branch_condition cond,
1700 address branch_addr,
1701 bool len64,
1702 bool has_sign) {
1703 unsigned int casenum = (len64?2:0)+(has_sign?0:1);
1704
1705 address branch_origin = pc();
1706 if (VM_Version::has_CompareBranch() && RelAddr::is_in_range_of_RelAddr16(branch_addr, branch_origin)) {
1707 switch (casenum) {
1708 case 0: z_crj( r1, r2, cond, branch_addr); break;
1709 case 1: z_clrj (r1, r2, cond, branch_addr); break;
1710 case 2: z_cgrj(r1, r2, cond, branch_addr); break;
1711 case 3: z_clgrj(r1, r2, cond, branch_addr); break;
1712 default: ShouldNotReachHere(); break;
1713 }
1714 } else {
1715 switch (casenum) {
1716 case 0: z_cr( r1, r2); break;
1717 case 1: z_clr(r1, r2); break;
1718 case 2: z_cgr(r1, r2); break;
1719 case 3: z_clgr(r1, r2); break;
1720 default: ShouldNotReachHere(); break;
1721 }
1722 branch_optimized(cond, branch_addr);
1723 }
1724 }
1725
1726 // Generate an optimal compare and branch to the branch target.
1727 // Optimal means that a relative branch (clgij, brc or brcl) is used if the
1728 // branch distance is short enough. Loading the target address into a
1729 // register and branching via reg is used as fallback only.
1730 //
1731 // Input:
1732 // r1 - left compare operand (in register)
1733 // x2 - right compare operand (immediate)
1734 void MacroAssembler::compare_and_branch_optimized(Register r1,
1735 jlong x2,
1736 Assembler::branch_condition cond,
1737 Label& branch_target,
1738 bool len64,
1739 bool has_sign) {
1740 address branch_origin = pc();
1741 bool x2_imm8 = (has_sign && Immediate::is_simm8(x2)) || (!has_sign && Immediate::is_uimm8(x2));
1742 bool is_RelAddr16 = branch_target.is_near() ||
1743 (branch_target.is_bound() &&
1744 RelAddr::is_in_range_of_RelAddr16(target(branch_target), branch_origin));
1745 unsigned int casenum = (len64?2:0)+(has_sign?0:1);
1746
1747 if (VM_Version::has_CompareBranch() && is_RelAddr16 && x2_imm8) {
1748 switch (casenum) {
1749 case 0: z_cij( r1, x2, cond, branch_target); break;
1750 case 1: z_clij(r1, x2, cond, branch_target); break;
1751 case 2: z_cgij(r1, x2, cond, branch_target); break;
1752 case 3: z_clgij(r1, x2, cond, branch_target); break;
1753 default: ShouldNotReachHere(); break;
1754 }
1755 return;
1756 }
1757
1758 if (x2 == 0) {
1759 switch (casenum) {
1760 case 0: z_ltr(r1, r1); break;
1761 case 1: z_ltr(r1, r1); break; // Caution: unsigned test only provides zero/notZero indication!
1762 case 2: z_ltgr(r1, r1); break;
1763 case 3: z_ltgr(r1, r1); break; // Caution: unsigned test only provides zero/notZero indication!
1764 default: ShouldNotReachHere(); break;
1765 }
1766 } else {
1767 if ((has_sign && Immediate::is_simm16(x2)) || (!has_sign && Immediate::is_uimm(x2, 15))) {
1768 switch (casenum) {
1769 case 0: z_chi(r1, x2); break;
1770 case 1: z_chi(r1, x2); break; // positive immediate < 2**15
1771 case 2: z_cghi(r1, x2); break;
1772 case 3: z_cghi(r1, x2); break; // positive immediate < 2**15
1773 default: break;
1774 }
1775 } else if ( (has_sign && Immediate::is_simm32(x2)) || (!has_sign && Immediate::is_uimm32(x2)) ) {
1776 switch (casenum) {
1777 case 0: z_cfi( r1, x2); break;
1778 case 1: z_clfi(r1, x2); break;
1779 case 2: z_cgfi(r1, x2); break;
1780 case 3: z_clgfi(r1, x2); break;
1781 default: ShouldNotReachHere(); break;
1782 }
1783 } else {
1784 // No instruction with immediate operand possible, so load into register.
1785 Register scratch = (r1 != Z_R0) ? Z_R0 : Z_R1;
1786 load_const_optimized(scratch, x2);
1787 switch (casenum) {
1788 case 0: z_cr( r1, scratch); break;
1789 case 1: z_clr(r1, scratch); break;
1790 case 2: z_cgr(r1, scratch); break;
1791 case 3: z_clgr(r1, scratch); break;
1792 default: ShouldNotReachHere(); break;
1793 }
1794 }
1795 }
1796 branch_optimized(cond, branch_target);
1797 }
1798
1799 // Generate an optimal compare and branch to the branch target.
1800 // Optimal means that a relative branch (clgrj, brc or brcl) is used if the
1801 // branch distance is short enough. Loading the target address into a
1802 // register and branching via reg is used as fallback only.
1803 //
1804 // Input:
1805 // r1 - left compare operand
1806 // r2 - right compare operand
1807 void MacroAssembler::compare_and_branch_optimized(Register r1,
1808 Register r2,
1809 Assembler::branch_condition cond,
1810 Label& branch_target,
1811 bool len64,
1812 bool has_sign) {
1813 unsigned int casenum = (len64 ? 2 : 0) + (has_sign ? 0 : 1);
1814
1815 if (branch_target.is_bound()) {
1816 address branch_addr = target(branch_target);
1817 compare_and_branch_optimized(r1, r2, cond, branch_addr, len64, has_sign);
1818 } else {
1819 if (VM_Version::has_CompareBranch() && branch_target.is_near()) {
1820 switch (casenum) {
1821 case 0: z_crj( r1, r2, cond, branch_target); break;
1822 case 1: z_clrj( r1, r2, cond, branch_target); break;
1823 case 2: z_cgrj( r1, r2, cond, branch_target); break;
1824 case 3: z_clgrj(r1, r2, cond, branch_target); break;
1825 default: ShouldNotReachHere(); break;
1826 }
1827 } else {
1828 switch (casenum) {
1829 case 0: z_cr( r1, r2); break;
1830 case 1: z_clr(r1, r2); break;
1831 case 2: z_cgr(r1, r2); break;
1832 case 3: z_clgr(r1, r2); break;
1833 default: ShouldNotReachHere(); break;
1834 }
1835 branch_optimized(cond, branch_target);
1836 }
1837 }
1838 }
1839
1840 //===========================================================================
1841 //=== END H I G H E R L E V E L B R A N C H E M I T T E R S ===
1842 //===========================================================================
1843
1844 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) {
1845 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
1846 int index = oop_recorder()->allocate_metadata_index(obj);
1847 RelocationHolder rspec = metadata_Relocation::spec(index);
1848 return AddressLiteral((address)obj, rspec);
1849 }
1850
1851 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) {
1852 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
1853 int index = oop_recorder()->find_index(obj);
1854 RelocationHolder rspec = metadata_Relocation::spec(index);
1855 return AddressLiteral((address)obj, rspec);
1856 }
1857
1858 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
1859 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
1860 int oop_index = oop_recorder()->allocate_oop_index(obj);
1861 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
1862 }
1863
1864 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
1865 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
1866 int oop_index = oop_recorder()->find_index(obj);
1867 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
1868 }
1869
1870 // NOTE: destroys r
1871 void MacroAssembler::c2bool(Register r, Register t) {
1872 z_lcr(t, r); // t = -r
1873 z_or(r, t); // r = -r OR r
1874 z_srl(r, 31); // Yields 0 if r was 0, 1 otherwise.
1875 }
1876
1877 // Patch instruction `inst' at offset `inst_pos' to refer to `dest_pos'
1878 // and return the resulting instruction.
1879 // Dest_pos and inst_pos are 32 bit only. These parms can only designate
1880 // relative positions.
1881 // Use correct argument types. Do not pre-calculate distance.
1882 unsigned long MacroAssembler::patched_branch(address dest_pos, unsigned long inst, address inst_pos) {
1883 int c = 0;
1884 unsigned long patched_inst = 0;
1885 if (is_call_pcrelative_short(inst) ||
1886 is_branch_pcrelative_short(inst) ||
1887 is_branchoncount_pcrelative_short(inst) ||
1888 is_branchonindex32_pcrelative_short(inst)) {
1889 c = 1;
1890 int m = fmask(15, 0); // simm16(-1, 16, 32);
1891 int v = simm16(RelAddr::pcrel_off16(dest_pos, inst_pos), 16, 32);
1892 patched_inst = (inst & ~m) | v;
1893 } else if (is_compareandbranch_pcrelative_short(inst)) {
1894 c = 2;
1895 long m = fmask(31, 16); // simm16(-1, 16, 48);
1896 long v = simm16(RelAddr::pcrel_off16(dest_pos, inst_pos), 16, 48);
1897 patched_inst = (inst & ~m) | v;
1898 } else if (is_branchonindex64_pcrelative_short(inst)) {
1899 c = 3;
1900 long m = fmask(31, 16); // simm16(-1, 16, 48);
1901 long v = simm16(RelAddr::pcrel_off16(dest_pos, inst_pos), 16, 48);
1902 patched_inst = (inst & ~m) | v;
1903 } else if (is_call_pcrelative_long(inst) || is_branch_pcrelative_long(inst)) {
1904 c = 4;
1905 long m = fmask(31, 0); // simm32(-1, 16, 48);
1906 long v = simm32(RelAddr::pcrel_off32(dest_pos, inst_pos), 16, 48);
1907 patched_inst = (inst & ~m) | v;
1908 } else if (is_pcrelative_long(inst)) { // These are the non-branch pc-relative instructions.
1909 c = 5;
1910 long m = fmask(31, 0); // simm32(-1, 16, 48);
1911 long v = simm32(RelAddr::pcrel_off32(dest_pos, inst_pos), 16, 48);
1912 patched_inst = (inst & ~m) | v;
1913 } else {
1914 print_dbg_msg(tty, inst, "not a relative branch", 0);
1915 dump_code_range(tty, inst_pos, 32, "not a pcrelative branch");
1916 ShouldNotReachHere();
1917 }
1918
1919 long new_off = get_pcrel_offset(patched_inst);
1920 if (new_off != (dest_pos-inst_pos)) {
1921 tty->print_cr("case %d: dest_pos = %p, inst_pos = %p, disp = %ld(%12.12lx)", c, dest_pos, inst_pos, new_off, new_off);
1922 print_dbg_msg(tty, inst, "<- original instruction: branch patching error", 0);
1923 print_dbg_msg(tty, patched_inst, "<- patched instruction: branch patching error", 0);
1924 #ifdef LUCY_DBG
1925 VM_Version::z_SIGSEGV();
1926 #endif
1927 ShouldNotReachHere();
1928 }
1929 return patched_inst;
1930 }
1931
1932 // Only called when binding labels (share/vm/asm/assembler.cpp)
1933 // Pass arguments as intended. Do not pre-calculate distance.
1934 void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) {
1935 unsigned long stub_inst;
1936 int inst_len = get_instruction(branch, &stub_inst);
1937
1938 set_instruction(branch, patched_branch(target, stub_inst, branch), inst_len);
1939 }
1940
1941
1942 // Extract relative address (aka offset).
1943 // inv_simm16 works for 4-byte instructions only.
1944 // compare and branch instructions are 6-byte and have a 16bit offset "in the middle".
1945 long MacroAssembler::get_pcrel_offset(unsigned long inst) {
1946
1947 if (MacroAssembler::is_pcrelative_short(inst)) {
1948 if (((inst&0xFFFFffff00000000UL) == 0) && ((inst&0x00000000FFFF0000UL) != 0)) {
1949 return RelAddr::inv_pcrel_off16(inv_simm16(inst));
1950 } else {
1951 return RelAddr::inv_pcrel_off16(inv_simm16_48(inst));
1952 }
1953 }
1954
1955 if (MacroAssembler::is_pcrelative_long(inst)) {
1956 return RelAddr::inv_pcrel_off32(inv_simm32(inst));
1957 }
1958
1959 print_dbg_msg(tty, inst, "not a pcrelative instruction", 6);
1960 #ifdef LUCY_DBG
1961 VM_Version::z_SIGSEGV();
1962 #else
1963 ShouldNotReachHere();
1964 #endif
1965 return -1;
1966 }
1967
1968 long MacroAssembler::get_pcrel_offset(address pc) {
1969 unsigned long inst;
1970 unsigned int len = get_instruction(pc, &inst);
1971
1972 #ifdef ASSERT
1973 long offset;
1974 if (MacroAssembler::is_pcrelative_short(inst) || MacroAssembler::is_pcrelative_long(inst)) {
1975 offset = get_pcrel_offset(inst);
1976 } else {
1977 offset = -1;
1978 }
1979
1980 if (offset == -1) {
1981 dump_code_range(tty, pc, 32, "not a pcrelative instruction");
1982 #ifdef LUCY_DBG
1983 VM_Version::z_SIGSEGV();
1984 #else
1985 ShouldNotReachHere();
1986 #endif
1987 }
1988 return offset;
1989 #else
1990 return get_pcrel_offset(inst);
1991 #endif // ASSERT
1992 }
1993
1994 // Get target address from pc-relative instructions.
1995 address MacroAssembler::get_target_addr_pcrel(address pc) {
1996 assert(is_pcrelative_long(pc), "not a pcrelative instruction");
1997 return pc + get_pcrel_offset(pc);
1998 }
1999
2000 // Patch pc relative load address.
2001 void MacroAssembler::patch_target_addr_pcrel(address pc, address con) {
2002 unsigned long inst;
2003 // Offset is +/- 2**32 -> use long.
2004 ptrdiff_t distance = con - pc;
2005
2006 get_instruction(pc, &inst);
2007
2008 if (is_pcrelative_short(inst)) {
2009 *(short *)(pc+2) = RelAddr::pcrel_off16(con, pc); // Instructions are at least 2-byte aligned, no test required.
2010
2011 // Some extra safety net.
2012 if (!RelAddr::is_in_range_of_RelAddr16(distance)) {
2013 print_dbg_msg(tty, inst, "distance out of range (16bit)", 4);
2014 dump_code_range(tty, pc, 32, "distance out of range (16bit)");
2015 guarantee(RelAddr::is_in_range_of_RelAddr16(distance), "too far away (more than +/- 2**16");
2016 }
2017 return;
2018 }
2019
2020 if (is_pcrelative_long(inst)) {
2021 *(int *)(pc+2) = RelAddr::pcrel_off32(con, pc);
2022
2023 // Some Extra safety net.
2024 if (!RelAddr::is_in_range_of_RelAddr32(distance)) {
2025 print_dbg_msg(tty, inst, "distance out of range (32bit)", 6);
2026 dump_code_range(tty, pc, 32, "distance out of range (32bit)");
2027 guarantee(RelAddr::is_in_range_of_RelAddr32(distance), "too far away (more than +/- 2**32");
2028 }
2029 return;
2030 }
2031
2032 guarantee(false, "not a pcrelative instruction to patch!");
2033 }
2034
2035 // "Current PC" here means the address just behind the basr instruction.
2036 address MacroAssembler::get_PC(Register result) {
2037 z_basr(result, Z_R0); // Don't branch, just save next instruction address in result.
2038 return pc();
2039 }
2040
2041 // Get current PC + offset.
2042 // Offset given in bytes, must be even!
2043 // "Current PC" here means the address of the larl instruction plus the given offset.
2044 address MacroAssembler::get_PC(Register result, int64_t offset) {
2045 address here = pc();
2046 z_larl(result, offset/2); // Save target instruction address in result.
2047 return here + offset;
2048 }
2049
2050 void MacroAssembler::instr_size(Register size, Register pc) {
2051 // Extract 2 most significant bits of current instruction.
2052 z_llgc(size, Address(pc));
2053 z_srl(size, 6);
2054 // Compute (x+3)&6 which translates 0->2, 1->4, 2->4, 3->6.
2055 z_ahi(size, 3);
2056 z_nill(size, 6);
2057 }
2058
2059 // Resize_frame with SP(new) = SP(old) - [offset].
2060 void MacroAssembler::resize_frame_sub(Register offset, Register fp, bool load_fp)
2061 {
2062 assert_different_registers(offset, fp, Z_SP);
2063 if (load_fp) { z_lg(fp, _z_abi(callers_sp), Z_SP); }
2064
2065 z_sgr(Z_SP, offset);
2066 z_stg(fp, _z_abi(callers_sp), Z_SP);
2067 }
2068
2069 // Resize_frame with SP(new) = [newSP] + offset.
2070 // This emitter is useful if we already have calculated a pointer
2071 // into the to-be-allocated stack space, e.g. with special alignment properties,
2072 // but need some additional space, e.g. for spilling.
2073 // newSP is the pre-calculated pointer. It must not be modified.
2074 // fp holds, or is filled with, the frame pointer.
2075 // offset is the additional increment which is added to addr to form the new SP.
2076 // Note: specify a negative value to reserve more space!
2077 // load_fp == true only indicates that fp is not pre-filled with the frame pointer.
2078 // It does not guarantee that fp contains the frame pointer at the end.
2079 void MacroAssembler::resize_frame_abs_with_offset(Register newSP, Register fp, int offset, bool load_fp) {
2080 assert_different_registers(newSP, fp, Z_SP);
2081
2082 if (load_fp) {
2083 z_lg(fp, _z_abi(callers_sp), Z_SP);
2084 }
2085
2086 add2reg(Z_SP, offset, newSP);
2087 z_stg(fp, _z_abi(callers_sp), Z_SP);
2088 }
2089
2090 // Resize_frame with SP(new) = [newSP].
2091 // load_fp == true only indicates that fp is not pre-filled with the frame pointer.
2092 // It does not guarantee that fp contains the frame pointer at the end.
2093 void MacroAssembler::resize_frame_absolute(Register newSP, Register fp, bool load_fp) {
2094 assert_different_registers(newSP, fp, Z_SP);
2095
2096 if (load_fp) {
2097 z_lg(fp, _z_abi(callers_sp), Z_SP); // need to use load/store.
2098 }
2099
2100 z_lgr(Z_SP, newSP);
2101 if (newSP != Z_R0) { // make sure we generate correct code, no matter what register newSP uses.
2102 z_stg(fp, _z_abi(callers_sp), newSP);
2103 } else {
2104 z_stg(fp, _z_abi(callers_sp), Z_SP);
2105 }
2106 }
2107
2108 // Resize_frame with SP(new) = SP(old) + offset.
2109 void MacroAssembler::resize_frame(RegisterOrConstant offset, Register fp, bool load_fp) {
2110 assert_different_registers(fp, Z_SP);
2111
2112 if (load_fp) {
2113 z_lg(fp, _z_abi(callers_sp), Z_SP);
2114 }
2115 add64(Z_SP, offset);
2116 z_stg(fp, _z_abi(callers_sp), Z_SP);
2117 }
2118
2119 void MacroAssembler::push_frame(Register bytes, Register old_sp, bool copy_sp, bool bytes_with_inverted_sign) {
2120 #ifdef ASSERT
2121 assert_different_registers(bytes, old_sp, Z_SP);
2122 if (!copy_sp) {
2123 z_cgr(old_sp, Z_SP);
2124 asm_assert(bcondEqual, "[old_sp]!=[Z_SP]", 0x211);
2125 }
2126 #endif
2127 if (copy_sp) { z_lgr(old_sp, Z_SP); }
2128 if (bytes_with_inverted_sign) {
2129 z_agr(Z_SP, bytes);
2130 } else {
2131 z_sgr(Z_SP, bytes); // Z_sgfr sufficient, but probably not faster.
2132 }
2133 z_stg(old_sp, _z_abi(callers_sp), Z_SP);
2134 }
2135
2136 unsigned int MacroAssembler::push_frame(unsigned int bytes, Register scratch) {
2137 long offset = Assembler::align(bytes, frame::alignment_in_bytes);
2138 assert(offset > 0, "should push a frame with positive size, size = %ld.", offset);
2139 assert(Displacement::is_validDisp(-offset), "frame size out of range, size = %ld", offset);
2140
2141 // We must not write outside the current stack bounds (given by Z_SP).
2142 // Thus, we have to first update Z_SP and then store the previous SP as stack linkage.
2143 // We rely on Z_R0 by default to be available as scratch.
2144 z_lgr(scratch, Z_SP);
2145 add2reg(Z_SP, -offset);
2146 z_stg(scratch, _z_abi(callers_sp), Z_SP);
2147 #ifdef ASSERT
2148 // Just make sure nobody uses the value in the default scratch register.
2149 // When another register is used, the caller might rely on it containing the frame pointer.
2150 if (scratch == Z_R0) {
2151 z_iihf(scratch, 0xbaadbabe);
2152 z_iilf(scratch, 0xdeadbeef);
2153 }
2154 #endif
2155 return offset;
2156 }
2157
2158 // Push a frame of size `bytes' plus abi160 on top.
2159 unsigned int MacroAssembler::push_frame_abi160(unsigned int bytes) {
2160 BLOCK_COMMENT("push_frame_abi160 {");
2161 unsigned int res = push_frame(bytes + frame::z_abi_160_size);
2162 BLOCK_COMMENT("} push_frame_abi160");
2163 return res;
2164 }
2165
2166 // Pop current C frame.
2167 void MacroAssembler::pop_frame() {
2168 BLOCK_COMMENT("pop_frame {");
2169 Assembler::z_lg(Z_SP, _z_abi(callers_sp), Z_SP);
2170 BLOCK_COMMENT("} pop_frame");
2171 }
2172
2173 // Pop current C frame and restore return PC register (Z_R14).
2174 void MacroAssembler::pop_frame_restore_retPC(int frame_size_in_bytes) {
2175 BLOCK_COMMENT("pop_frame_restore_retPC:");
2176 int retPC_offset = _z_common_abi(return_pc) + frame_size_in_bytes;
2177 // If possible, pop frame by add instead of load (a penny saved is a penny got :-).
2178 if (Displacement::is_validDisp(retPC_offset)) {
2179 z_lg(Z_R14, retPC_offset, Z_SP);
2180 add2reg(Z_SP, frame_size_in_bytes);
2181 } else {
2182 add2reg(Z_SP, frame_size_in_bytes);
2183 restore_return_pc();
2184 }
2185 }
2186
2187 void MacroAssembler::call_VM_leaf_base(address entry_point, bool allow_relocation) {
2188 if (allow_relocation) {
2189 call_c(entry_point);
2190 } else {
2191 call_c_static(entry_point);
2192 }
2193 }
2194
2195 void MacroAssembler::call_VM_leaf_base(address entry_point) {
2196 bool allow_relocation = true;
2197 call_VM_leaf_base(entry_point, allow_relocation);
2198 }
2199
2200 int MacroAssembler::ic_check_size() {
2201 int ic_size = 24;
2202 if (!ImplicitNullChecks) {
2203 ic_size += 6;
2204 }
2205 if (UseCompactObjectHeaders) {
2206 ic_size += 12;
2207 } else {
2208 ic_size += 6; // either z_llgf or z_lg
2209 }
2210 return ic_size;
2211 }
2212
2213 int MacroAssembler::ic_check(int end_alignment) {
2214 Register R2_receiver = Z_ARG1;
2215 Register R0_scratch = Z_R0_scratch;
2216 Register R1_scratch = Z_R1_scratch;
2217 Register R9_data = Z_inline_cache;
2218 Label success, failure;
2219
2220 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
2221 // before the inline cache check, so we don't have to execute any nop instructions when dispatching
2222 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
2223 // before the inline cache check here, and not after
2224 align(end_alignment, offset() + ic_check_size());
2225
2226 int uep_offset = offset();
2227 if (!ImplicitNullChecks) {
2228 z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure);
2229 }
2230
2231 if (UseCompactObjectHeaders) {
2232 load_narrow_klass_compact(R1_scratch, R2_receiver);
2233 } else {
2234 z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
2235 }
2236 z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
2237 z_bre(success);
2238
2239 bind(failure);
2240 load_const(R1_scratch, AddressLiteral(SharedRuntime::get_ic_miss_stub()));
2241 z_br(R1_scratch);
2242 bind(success);
2243
2244 assert((offset() % end_alignment) == 0, "Misaligned verified entry point, offset() = %d, end_alignment = %d", offset(), end_alignment);
2245 return uep_offset;
2246 }
2247
2248 void MacroAssembler::call_VM_base(Register oop_result,
2249 Register last_java_sp,
2250 address entry_point,
2251 bool allow_relocation,
2252 bool check_exceptions) { // Defaults to true.
2253 // Allow_relocation indicates, if true, that the generated code shall
2254 // be fit for code relocation or referenced data relocation. In other
2255 // words: all addresses must be considered variable. PC-relative addressing
2256 // is not possible then.
2257 // On the other hand, if (allow_relocation == false), addresses and offsets
2258 // may be considered stable, enabling us to take advantage of some PC-relative
2259 // addressing tweaks. These might improve performance and reduce code size.
2260
2261 // Determine last_java_sp register.
2262 if (!last_java_sp->is_valid()) {
2263 last_java_sp = Z_SP; // Load Z_SP as SP.
2264 }
2265
2266 set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, Z_R1, allow_relocation);
2267
2268 // ARG1 must hold thread address.
2269 z_lgr(Z_ARG1, Z_thread);
2270
2271 address return_pc = nullptr;
2272 if (allow_relocation) {
2273 return_pc = call_c(entry_point);
2274 } else {
2275 return_pc = call_c_static(entry_point);
2276 }
2277
2278 reset_last_Java_frame(allow_relocation);
2279
2280 // C++ interp handles this in the interpreter.
2281 check_and_handle_popframe(Z_thread);
2282 check_and_handle_earlyret(Z_thread);
2283
2284 // Check for pending exceptions.
2285 if (check_exceptions) {
2286 // Check for pending exceptions (java_thread is set upon return).
2287 load_and_test_long(Z_R0_scratch, Address(Z_thread, Thread::pending_exception_offset()));
2288
2289 // This used to conditionally jump to forward_exception however it is
2290 // possible if we relocate that the branch will not reach. So we must jump
2291 // around so we can always reach.
2292
2293 Label ok;
2294 z_bre(ok); // Bcondequal is the same as bcondZero.
2295 call_stub(StubRoutines::forward_exception_entry());
2296 bind(ok);
2297 }
2298
2299 // Get oop result if there is one and reset the value in the thread.
2300 if (oop_result->is_valid()) {
2301 get_vm_result_oop(oop_result);
2302 }
2303
2304 _last_calls_return_pc = return_pc; // Wipe out other (error handling) calls.
2305 }
2306
2307 void MacroAssembler::call_VM_base(Register oop_result,
2308 Register last_java_sp,
2309 address entry_point,
2310 bool check_exceptions) { // Defaults to true.
2311 bool allow_relocation = true;
2312 call_VM_base(oop_result, last_java_sp, entry_point, allow_relocation, check_exceptions);
2313 }
2314
2315 // VM calls without explicit last_java_sp.
2316
2317 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
2318 // Call takes possible detour via InterpreterMacroAssembler.
2319 call_VM_base(oop_result, noreg, entry_point, true, check_exceptions);
2320 }
2321
2322 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) {
2323 // Z_ARG1 is reserved for the thread.
2324 lgr_if_needed(Z_ARG2, arg_1);
2325 call_VM(oop_result, entry_point, check_exceptions);
2326 }
2327
2328 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
2329 // Z_ARG1 is reserved for the thread.
2330 assert_different_registers(arg_2, Z_ARG2);
2331 lgr_if_needed(Z_ARG2, arg_1);
2332 lgr_if_needed(Z_ARG3, arg_2);
2333 call_VM(oop_result, entry_point, check_exceptions);
2334 }
2335
2336 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2,
2337 Register arg_3, bool check_exceptions) {
2338 // Z_ARG1 is reserved for the thread.
2339 assert_different_registers(arg_3, Z_ARG2, Z_ARG3);
2340 assert_different_registers(arg_2, Z_ARG2);
2341 lgr_if_needed(Z_ARG2, arg_1);
2342 lgr_if_needed(Z_ARG3, arg_2);
2343 lgr_if_needed(Z_ARG4, arg_3);
2344 call_VM(oop_result, entry_point, check_exceptions);
2345 }
2346
2347 // VM static calls without explicit last_java_sp.
2348
2349 void MacroAssembler::call_VM_static(Register oop_result, address entry_point, bool check_exceptions) {
2350 // Call takes possible detour via InterpreterMacroAssembler.
2351 call_VM_base(oop_result, noreg, entry_point, false, check_exceptions);
2352 }
2353
2354 void MacroAssembler::call_VM_static(Register oop_result, address entry_point, Register arg_1, Register arg_2,
2355 Register arg_3, bool check_exceptions) {
2356 // Z_ARG1 is reserved for the thread.
2357 assert_different_registers(arg_3, Z_ARG2, Z_ARG3);
2358 assert_different_registers(arg_2, Z_ARG2);
2359 lgr_if_needed(Z_ARG2, arg_1);
2360 lgr_if_needed(Z_ARG3, arg_2);
2361 lgr_if_needed(Z_ARG4, arg_3);
2362 call_VM_static(oop_result, entry_point, check_exceptions);
2363 }
2364
2365 // VM calls with explicit last_java_sp.
2366
2367 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, bool check_exceptions) {
2368 // Call takes possible detour via InterpreterMacroAssembler.
2369 call_VM_base(oop_result, last_java_sp, entry_point, true, check_exceptions);
2370 }
2371
2372 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) {
2373 // Z_ARG1 is reserved for the thread.
2374 lgr_if_needed(Z_ARG2, arg_1);
2375 call_VM(oop_result, last_java_sp, entry_point, check_exceptions);
2376 }
2377
2378 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1,
2379 Register arg_2, bool check_exceptions) {
2380 // Z_ARG1 is reserved for the thread.
2381 assert_different_registers(arg_2, Z_ARG2);
2382 lgr_if_needed(Z_ARG2, arg_1);
2383 lgr_if_needed(Z_ARG3, arg_2);
2384 call_VM(oop_result, last_java_sp, entry_point, check_exceptions);
2385 }
2386
2387 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1,
2388 Register arg_2, Register arg_3, bool check_exceptions) {
2389 // Z_ARG1 is reserved for the thread.
2390 assert_different_registers(arg_3, Z_ARG2, Z_ARG3);
2391 assert_different_registers(arg_2, Z_ARG2);
2392 lgr_if_needed(Z_ARG2, arg_1);
2393 lgr_if_needed(Z_ARG3, arg_2);
2394 lgr_if_needed(Z_ARG4, arg_3);
2395 call_VM(oop_result, last_java_sp, entry_point, check_exceptions);
2396 }
2397
2398 // VM leaf calls.
2399
2400 void MacroAssembler::call_VM_leaf(address entry_point) {
2401 // Call takes possible detour via InterpreterMacroAssembler.
2402 call_VM_leaf_base(entry_point, true);
2403 }
2404
2405 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
2406 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2407 call_VM_leaf(entry_point);
2408 }
2409
2410 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
2411 assert_different_registers(arg_2, Z_ARG1);
2412 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2413 if (arg_2 != noreg) lgr_if_needed(Z_ARG2, arg_2);
2414 call_VM_leaf(entry_point);
2415 }
2416
2417 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
2418 assert_different_registers(arg_3, Z_ARG1, Z_ARG2);
2419 assert_different_registers(arg_2, Z_ARG1);
2420 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2421 if (arg_2 != noreg) lgr_if_needed(Z_ARG2, arg_2);
2422 if (arg_3 != noreg) lgr_if_needed(Z_ARG3, arg_3);
2423 call_VM_leaf(entry_point);
2424 }
2425
2426 // Static VM leaf calls.
2427 // Really static VM leaf calls are never patched.
2428
2429 void MacroAssembler::call_VM_leaf_static(address entry_point) {
2430 // Call takes possible detour via InterpreterMacroAssembler.
2431 call_VM_leaf_base(entry_point, false);
2432 }
2433
2434 void MacroAssembler::call_VM_leaf_static(address entry_point, Register arg_1) {
2435 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2436 call_VM_leaf_static(entry_point);
2437 }
2438
2439 void MacroAssembler::call_VM_leaf_static(address entry_point, Register arg_1, Register arg_2) {
2440 assert_different_registers(arg_2, Z_ARG1);
2441 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2442 if (arg_2 != noreg) lgr_if_needed(Z_ARG2, arg_2);
2443 call_VM_leaf_static(entry_point);
2444 }
2445
2446 void MacroAssembler::call_VM_leaf_static(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
2447 assert_different_registers(arg_3, Z_ARG1, Z_ARG2);
2448 assert_different_registers(arg_2, Z_ARG1);
2449 if (arg_1 != noreg) lgr_if_needed(Z_ARG1, arg_1);
2450 if (arg_2 != noreg) lgr_if_needed(Z_ARG2, arg_2);
2451 if (arg_3 != noreg) lgr_if_needed(Z_ARG3, arg_3);
2452 call_VM_leaf_static(entry_point);
2453 }
2454
2455 // Don't use detour via call_c(reg).
2456 address MacroAssembler::call_c(address function_entry) {
2457 load_const(Z_R1, function_entry);
2458 return call(Z_R1);
2459 }
2460
2461 // Variant for really static (non-relocatable) calls which are never patched.
2462 address MacroAssembler::call_c_static(address function_entry) {
2463 load_absolute_address(Z_R1, function_entry);
2464 #if 0 // def ASSERT
2465 // Verify that call site did not move.
2466 load_const_optimized(Z_R0, function_entry);
2467 z_cgr(Z_R1, Z_R0);
2468 z_brc(bcondEqual, 3);
2469 z_illtrap(0xba);
2470 #endif
2471 return call(Z_R1);
2472 }
2473
2474 address MacroAssembler::call_c_opt(address function_entry) {
2475 bool success = call_far_patchable(function_entry, -2 /* emit relocation + constant */);
2476 _last_calls_return_pc = success ? pc() : nullptr;
2477 return _last_calls_return_pc;
2478 }
2479
2480 // Identify a call_far_patchable instruction: LARL + LG + BASR
2481 //
2482 // nop ; optionally, if required for alignment
2483 // lgrl rx,A(TOC entry) ; PC-relative access into constant pool
2484 // basr Z_R14,rx ; end of this instruction must be aligned to a word boundary
2485 //
2486 // Code pattern will eventually get patched into variant2 (see below for detection code).
2487 //
2488 bool MacroAssembler::is_call_far_patchable_variant0_at(address instruction_addr) {
2489 address iaddr = instruction_addr;
2490
2491 // Check for the actual load instruction.
2492 if (!is_load_const_from_toc(iaddr)) { return false; }
2493 iaddr += load_const_from_toc_size();
2494
2495 // Check for the call (BASR) instruction, finally.
2496 assert(iaddr-instruction_addr+call_byregister_size() == call_far_patchable_size(), "size mismatch");
2497 return is_call_byregister(iaddr);
2498 }
2499
2500 // Identify a call_far_patchable instruction: BRASL
2501 //
2502 // Code pattern to suits atomic patching:
2503 // nop ; Optionally, if required for alignment.
2504 // nop ... ; Multiple filler nops to compensate for size difference (variant0 is longer).
2505 // nop ; For code pattern detection: Prepend each BRASL with a nop.
2506 // brasl Z_R14,<reladdr> ; End of code must be 4-byte aligned !
2507 bool MacroAssembler::is_call_far_patchable_variant2_at(address instruction_addr) {
2508 const address call_addr = (address)((intptr_t)instruction_addr + call_far_patchable_size() - call_far_pcrelative_size());
2509
2510 // Check for correct number of leading nops.
2511 address iaddr;
2512 for (iaddr = instruction_addr; iaddr < call_addr; iaddr += nop_size()) {
2513 if (!is_z_nop(iaddr)) { return false; }
2514 }
2515 assert(iaddr == call_addr, "sanity");
2516
2517 // --> Check for call instruction.
2518 if (is_call_far_pcrelative(call_addr)) {
2519 assert(call_addr-instruction_addr+call_far_pcrelative_size() == call_far_patchable_size(), "size mismatch");
2520 return true;
2521 }
2522
2523 return false;
2524 }
2525
2526 // Emit a NOT mt-safely patchable 64 bit absolute call.
2527 // If toc_offset == -2, then the destination of the call (= target) is emitted
2528 // to the constant pool and a runtime_call relocation is added
2529 // to the code buffer.
2530 // If toc_offset != -2, target must already be in the constant pool at
2531 // _ctableStart+toc_offset (a caller can retrieve toc_offset
2532 // from the runtime_call relocation).
2533 // Special handling of emitting to scratch buffer when there is no constant pool.
2534 // Slightly changed code pattern. We emit an additional nop if we would
2535 // not end emitting at a word aligned address. This is to ensure
2536 // an atomically patchable displacement in brasl instructions.
2537 //
2538 // A call_far_patchable comes in different flavors:
2539 // - LARL(CP) / LG(CP) / BR (address in constant pool, access via CP register)
2540 // - LGRL(CP) / BR (address in constant pool, pc-relative access)
2541 // - BRASL (relative address of call target coded in instruction)
2542 // All flavors occupy the same amount of space. Length differences are compensated
2543 // by leading nops, such that the instruction sequence always ends at the same
2544 // byte offset. This is required to keep the return offset constant.
2545 // Furthermore, the return address (the end of the instruction sequence) is forced
2546 // to be on a 4-byte boundary. This is required for atomic patching, should we ever
2547 // need to patch the call target of the BRASL flavor.
2548 // RETURN value: false, if no constant pool entry could be allocated, true otherwise.
2549 bool MacroAssembler::call_far_patchable(address target, int64_t tocOffset) {
2550 // Get current pc and ensure word alignment for end of instr sequence.
2551 const address start_pc = pc();
2552 const intptr_t start_off = offset();
2553 assert(!call_far_patchable_requires_alignment_nop(start_pc), "call_far_patchable requires aligned address");
2554 const ptrdiff_t dist = (ptrdiff_t)(target - (start_pc + 2)); // Prepend each BRASL with a nop.
2555 const bool emit_target_to_pool = (tocOffset == -2) && !code_section()->scratch_emit();
2556 const bool emit_relative_call = !emit_target_to_pool &&
2557 RelAddr::is_in_range_of_RelAddr32(dist) &&
2558 ReoptimizeCallSequences &&
2559 !code_section()->scratch_emit();
2560
2561 if (emit_relative_call) {
2562 // Add padding to get the same size as below.
2563 const unsigned int padding = call_far_patchable_size() - call_far_pcrelative_size();
2564 unsigned int current_padding;
2565 for (current_padding = 0; current_padding < padding; current_padding += nop_size()) { z_nop(); }
2566 assert(current_padding == padding, "sanity");
2567
2568 // relative call: len = 2(nop) + 6 (brasl)
2569 // CodeBlob resize cannot occur in this case because
2570 // this call is emitted into pre-existing space.
2571 z_nop(); // Prepend each BRASL with a nop.
2572 z_brasl(Z_R14, target);
2573 } else {
2574 // absolute call: Get address from TOC.
2575 // len = (load TOC){6|0} + (load from TOC){6} + (basr){2} = {14|8}
2576 if (emit_target_to_pool) {
2577 // When emitting the call for the first time, we do not need to use
2578 // the pc-relative version. It will be patched anyway, when the code
2579 // buffer is copied.
2580 // Relocation is not needed when !ReoptimizeCallSequences.
2581 relocInfo::relocType rt = ReoptimizeCallSequences ? relocInfo::runtime_call_w_cp_type : relocInfo::none;
2582 AddressLiteral dest(target, rt);
2583 // Store_oop_in_toc() adds dest to the constant table. As side effect, this kills
2584 // inst_mark(). Reset if possible.
2585 bool reset_mark = (inst_mark() == pc());
2586 tocOffset = store_oop_in_toc(dest);
2587 if (reset_mark) { set_inst_mark(); }
2588 if (tocOffset == -1) {
2589 return false; // Couldn't create constant pool entry.
2590 }
2591 }
2592 assert(offset() == start_off, "emit no code before this point!");
2593
2594 address tocPos = pc() + tocOffset;
2595 if (emit_target_to_pool) {
2596 tocPos = code()->consts()->start() + tocOffset;
2597 }
2598 load_long_pcrelative(Z_R14, tocPos);
2599 z_basr(Z_R14, Z_R14);
2600 }
2601
2602 #ifdef ASSERT
2603 // Assert that we can identify the emitted call.
2604 assert(is_call_far_patchable_at(addr_at(start_off)), "can't identify emitted call");
2605 assert(offset() == start_off+call_far_patchable_size(), "wrong size");
2606
2607 if (emit_target_to_pool) {
2608 assert(get_dest_of_call_far_patchable_at(addr_at(start_off), code()->consts()->start()) == target,
2609 "wrong encoding of dest address");
2610 }
2611 #endif
2612 return true; // success
2613 }
2614
2615 // Identify a call_far_patchable instruction.
2616 // For more detailed information see header comment of call_far_patchable.
2617 bool MacroAssembler::is_call_far_patchable_at(address instruction_addr) {
2618 return is_call_far_patchable_variant2_at(instruction_addr) || // short version: BRASL
2619 is_call_far_patchable_variant0_at(instruction_addr); // long version LARL + LG + BASR
2620 }
2621
2622 // Does the call_far_patchable instruction use a pc-relative encoding
2623 // of the call destination?
2624 bool MacroAssembler::is_call_far_patchable_pcrelative_at(address instruction_addr) {
2625 // Variant 2 is pc-relative.
2626 return is_call_far_patchable_variant2_at(instruction_addr);
2627 }
2628
2629 bool MacroAssembler::is_call_far_pcrelative(address instruction_addr) {
2630 // Prepend each BRASL with a nop.
2631 return is_z_nop(instruction_addr) && is_z_brasl(instruction_addr + nop_size()); // Match at position after one nop required.
2632 }
2633
2634 // Set destination address of a call_far_patchable instruction.
2635 void MacroAssembler::set_dest_of_call_far_patchable_at(address instruction_addr, address dest, int64_t tocOffset) {
2636 ResourceMark rm;
2637
2638 // Now that CP entry is verified, patch call to a pc-relative call (if circumstances permit).
2639 int code_size = MacroAssembler::call_far_patchable_size();
2640 CodeBuffer buf(instruction_addr, code_size);
2641 MacroAssembler masm(&buf);
2642 masm.call_far_patchable(dest, tocOffset);
2643 ICache::invalidate_range(instruction_addr, code_size); // Empty on z.
2644 }
2645
2646 // Get dest address of a call_far_patchable instruction.
2647 address MacroAssembler::get_dest_of_call_far_patchable_at(address instruction_addr, address ctable) {
2648 // Dynamic TOC: absolute address in constant pool.
2649 // Check variant2 first, it is more frequent.
2650
2651 // Relative address encoded in call instruction.
2652 if (is_call_far_patchable_variant2_at(instruction_addr)) {
2653 return MacroAssembler::get_target_addr_pcrel(instruction_addr + nop_size()); // Prepend each BRASL with a nop.
2654
2655 // Absolute address in constant pool.
2656 } else if (is_call_far_patchable_variant0_at(instruction_addr)) {
2657 address iaddr = instruction_addr;
2658
2659 long tocOffset = get_load_const_from_toc_offset(iaddr);
2660 address tocLoc = iaddr + tocOffset;
2661 return *(address *)(tocLoc);
2662 } else {
2663 fprintf(stderr, "MacroAssembler::get_dest_of_call_far_patchable_at has a problem at %p:\n", instruction_addr);
2664 fprintf(stderr, "not a call_far_patchable: %16.16lx %16.16lx, len = %d\n",
2665 *(unsigned long*)instruction_addr,
2666 *(unsigned long*)(instruction_addr+8),
2667 call_far_patchable_size());
2668 Disassembler::decode(instruction_addr, instruction_addr+call_far_patchable_size());
2669 ShouldNotReachHere();
2670 return nullptr;
2671 }
2672 }
2673
2674 void MacroAssembler::align_call_far_patchable(address pc) {
2675 if (call_far_patchable_requires_alignment_nop(pc)) { z_nop(); }
2676 }
2677
2678 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2679 }
2680
2681 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2682 }
2683
2684 // Read from the polling page.
2685 // Use TM or TMY instruction, depending on read offset.
2686 // offset = 0: Use TM, safepoint polling.
2687 // offset < 0: Use TMY, profiling safepoint polling.
2688 void MacroAssembler::load_from_polling_page(Register polling_page_address, int64_t offset) {
2689 if (Immediate::is_uimm12(offset)) {
2690 z_tm(offset, polling_page_address, mask_safepoint);
2691 } else {
2692 z_tmy(offset, polling_page_address, mask_profiling);
2693 }
2694 }
2695
2696 // Check whether z_instruction is a read access to the polling page
2697 // which was emitted by load_from_polling_page(..).
2698 bool MacroAssembler::is_load_from_polling_page(address instr_loc) {
2699 unsigned long z_instruction;
2700 unsigned int ilen = get_instruction(instr_loc, &z_instruction);
2701
2702 if (ilen == 2) { return false; } // It's none of the allowed instructions.
2703
2704 if (ilen == 4) {
2705 if (!is_z_tm(z_instruction)) { return false; } // It's len=4, but not a z_tm. fail.
2706
2707 int ms = inv_mask(z_instruction,8,32); // mask
2708 int ra = inv_reg(z_instruction,16,32); // base register
2709 int ds = inv_uimm12(z_instruction); // displacement
2710
2711 if (!(ds == 0 && ra != 0 && ms == mask_safepoint)) {
2712 return false; // It's not a z_tm(0, ra, mask_safepoint). Fail.
2713 }
2714
2715 } else { /* if (ilen == 6) */
2716
2717 assert(!is_z_lg(z_instruction), "old form (LG) polling page access. Please fix and use TM(Y).");
2718
2719 if (!is_z_tmy(z_instruction)) { return false; } // It's len=6, but not a z_tmy. fail.
2720
2721 int ms = inv_mask(z_instruction,8,48); // mask
2722 int ra = inv_reg(z_instruction,16,48); // base register
2723 int ds = inv_simm20(z_instruction); // displacement
2724 }
2725
2726 return true;
2727 }
2728
2729 // Extract poll address from instruction and ucontext.
2730 address MacroAssembler::get_poll_address(address instr_loc, void* ucontext) {
2731 assert(ucontext != nullptr, "must have ucontext");
2732 ucontext_t* uc = (ucontext_t*) ucontext;
2733 unsigned long z_instruction;
2734 unsigned int ilen = get_instruction(instr_loc, &z_instruction);
2735
2736 if (ilen == 4 && is_z_tm(z_instruction)) {
2737 int ra = inv_reg(z_instruction, 16, 32); // base register
2738 int ds = inv_uimm12(z_instruction); // displacement
2739 address addr = (address)uc->uc_mcontext.gregs[ra];
2740 return addr + ds;
2741 } else if (ilen == 6 && is_z_tmy(z_instruction)) {
2742 int ra = inv_reg(z_instruction, 16, 48); // base register
2743 int ds = inv_simm20(z_instruction); // displacement
2744 address addr = (address)uc->uc_mcontext.gregs[ra];
2745 return addr + ds;
2746 }
2747
2748 ShouldNotReachHere();
2749 return nullptr;
2750 }
2751
2752 // Extract poll register from instruction.
2753 uint MacroAssembler::get_poll_register(address instr_loc) {
2754 unsigned long z_instruction;
2755 unsigned int ilen = get_instruction(instr_loc, &z_instruction);
2756
2757 if (ilen == 4 && is_z_tm(z_instruction)) {
2758 return (uint)inv_reg(z_instruction, 16, 32); // base register
2759 } else if (ilen == 6 && is_z_tmy(z_instruction)) {
2760 return (uint)inv_reg(z_instruction, 16, 48); // base register
2761 }
2762
2763 ShouldNotReachHere();
2764 return 0;
2765 }
2766
2767 void MacroAssembler::safepoint_poll(Label& slow_path, Register temp_reg) {
2768 const Address poll_byte_addr(Z_thread, in_bytes(JavaThread::polling_word_offset()) + 7 /* Big Endian */);
2769 // Armed page has poll_bit set.
2770 z_tm(poll_byte_addr, SafepointMechanism::poll_bit());
2771 z_brnaz(slow_path);
2772 }
2773
2774 // Don't rely on register locking, always use Z_R1 as scratch register instead.
2775 void MacroAssembler::bang_stack_with_offset(int offset) {
2776 // Stack grows down, caller passes positive offset.
2777 assert(offset > 0, "must bang with positive offset");
2778 if (Displacement::is_validDisp(-offset)) {
2779 z_tmy(-offset, Z_SP, mask_stackbang);
2780 } else {
2781 add2reg(Z_R1, -offset, Z_SP); // Do not destroy Z_SP!!!
2782 z_tm(0, Z_R1, mask_stackbang); // Just banging.
2783 }
2784 }
2785
2786 void MacroAssembler::reserved_stack_check(Register return_pc) {
2787 // Test if reserved zone needs to be enabled.
2788 Label no_reserved_zone_enabling;
2789 assert(return_pc == Z_R14, "Return pc must be in R14 before z_br() to StackOverflow stub.");
2790 BLOCK_COMMENT("reserved_stack_check {");
2791
2792 z_clg(Z_SP, Address(Z_thread, JavaThread::reserved_stack_activation_offset()));
2793 z_brl(no_reserved_zone_enabling);
2794
2795 // Enable reserved zone again, throw stack overflow exception.
2796 save_return_pc();
2797 push_frame_abi160(0);
2798 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), Z_thread);
2799 pop_frame();
2800 restore_return_pc();
2801
2802 load_const_optimized(Z_R1, SharedRuntime::throw_delayed_StackOverflowError_entry());
2803 // Don't use call() or z_basr(), they will invalidate Z_R14 which contains the return pc.
2804 z_br(Z_R1);
2805
2806 should_not_reach_here();
2807
2808 bind(no_reserved_zone_enabling);
2809 BLOCK_COMMENT("} reserved_stack_check");
2810 }
2811
2812 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
2813 void MacroAssembler::tlab_allocate(Register obj,
2814 Register var_size_in_bytes,
2815 int con_size_in_bytes,
2816 Register t1,
2817 Label& slow_case) {
2818 assert_different_registers(obj, var_size_in_bytes, t1);
2819 Register end = t1;
2820 Register thread = Z_thread;
2821
2822 z_lg(obj, Address(thread, JavaThread::tlab_top_offset()));
2823 if (var_size_in_bytes == noreg) {
2824 z_lay(end, Address(obj, con_size_in_bytes));
2825 } else {
2826 z_lay(end, Address(obj, var_size_in_bytes));
2827 }
2828 z_cg(end, Address(thread, JavaThread::tlab_end_offset()));
2829 branch_optimized(bcondHigh, slow_case);
2830
2831 // Update the tlab top pointer.
2832 z_stg(end, Address(thread, JavaThread::tlab_top_offset()));
2833
2834 // Recover var_size_in_bytes if necessary.
2835 if (var_size_in_bytes == end) {
2836 z_sgr(var_size_in_bytes, obj);
2837 }
2838 }
2839
2840 // Emitter for interface method lookup.
2841 // input: recv_klass, intf_klass, itable_index
2842 // output: method_result
2843 // kills: itable_index, temp1_reg, Z_R0, Z_R1
2844 // TODO: Temp2_reg is unused. we may use this emitter also in the itable stubs.
2845 // If the register is still not needed then, remove it.
2846 void MacroAssembler::lookup_interface_method(Register recv_klass,
2847 Register intf_klass,
2848 RegisterOrConstant itable_index,
2849 Register method_result,
2850 Register temp1_reg,
2851 Label& no_such_interface,
2852 bool return_method) {
2853
2854 const Register vtable_len = temp1_reg; // Used to compute itable_entry_addr.
2855 const Register itable_entry_addr = Z_R1_scratch;
2856 const Register itable_interface = Z_R0_scratch;
2857
2858 BLOCK_COMMENT("lookup_interface_method {");
2859
2860 // Load start of itable entries into itable_entry_addr.
2861 z_llgf(vtable_len, Address(recv_klass, Klass::vtable_length_offset()));
2862 z_sllg(vtable_len, vtable_len, exact_log2(vtableEntry::size_in_bytes()));
2863
2864 // Loop over all itable entries until desired interfaceOop(Rinterface) found.
2865 add2reg_with_index(itable_entry_addr,
2866 in_bytes(Klass::vtable_start_offset() + itableOffsetEntry::interface_offset()),
2867 recv_klass, vtable_len);
2868
2869 const int itable_offset_search_inc = itableOffsetEntry::size() * wordSize;
2870 Label search;
2871
2872 bind(search);
2873
2874 // Handle IncompatibleClassChangeError.
2875 // If the entry is null then we've reached the end of the table
2876 // without finding the expected interface, so throw an exception.
2877 load_and_test_long(itable_interface, Address(itable_entry_addr));
2878 z_bre(no_such_interface);
2879
2880 add2reg(itable_entry_addr, itable_offset_search_inc);
2881 z_cgr(itable_interface, intf_klass);
2882 z_brne(search);
2883
2884 // Entry found and itable_entry_addr points to it, get offset of vtable for interface.
2885 if (return_method) {
2886 const int vtable_offset_offset = in_bytes(itableOffsetEntry::offset_offset() -
2887 itableOffsetEntry::interface_offset()) -
2888 itable_offset_search_inc;
2889
2890 // Compute itableMethodEntry and get method and entry point
2891 // we use addressing with index and displacement, since the formula
2892 // for computing the entry's offset has a fixed and a dynamic part,
2893 // the latter depending on the matched interface entry and on the case,
2894 // that the itable index has been passed as a register, not a constant value.
2895 int method_offset = in_bytes(itableMethodEntry::method_offset());
2896 // Fixed part (displacement), common operand.
2897 Register itable_offset = method_result; // Dynamic part (index register).
2898
2899 if (itable_index.is_register()) {
2900 // Compute the method's offset in that register, for the formula, see the
2901 // else-clause below.
2902 z_sllg(itable_offset, itable_index.as_register(), exact_log2(itableMethodEntry::size() * wordSize));
2903 z_agf(itable_offset, vtable_offset_offset, itable_entry_addr);
2904 } else {
2905 // Displacement increases.
2906 method_offset += itableMethodEntry::size() * wordSize * itable_index.as_constant();
2907
2908 // Load index from itable.
2909 z_llgf(itable_offset, vtable_offset_offset, itable_entry_addr);
2910 }
2911
2912 // Finally load the method's oop.
2913 z_lg(method_result, method_offset, itable_offset, recv_klass);
2914 }
2915 BLOCK_COMMENT("} lookup_interface_method");
2916 }
2917
2918 // Lookup for virtual method invocation.
2919 void MacroAssembler::lookup_virtual_method(Register recv_klass,
2920 RegisterOrConstant vtable_index,
2921 Register method_result) {
2922 assert_different_registers(recv_klass, vtable_index.register_or_noreg());
2923 assert(vtableEntry::size() * wordSize == wordSize,
2924 "else adjust the scaling in the code below");
2925
2926 BLOCK_COMMENT("lookup_virtual_method {");
2927
2928 const int base = in_bytes(Klass::vtable_start_offset());
2929
2930 if (vtable_index.is_constant()) {
2931 // Load with base + disp.
2932 Address vtable_entry_addr(recv_klass,
2933 vtable_index.as_constant() * wordSize +
2934 base +
2935 in_bytes(vtableEntry::method_offset()));
2936
2937 z_lg(method_result, vtable_entry_addr);
2938 } else {
2939 // Shift index properly and load with base + index + disp.
2940 Register vindex = vtable_index.as_register();
2941 Address vtable_entry_addr(recv_klass, vindex,
2942 base + in_bytes(vtableEntry::method_offset()));
2943
2944 z_sllg(vindex, vindex, exact_log2(wordSize));
2945 z_lg(method_result, vtable_entry_addr);
2946 }
2947 BLOCK_COMMENT("} lookup_virtual_method");
2948 }
2949
2950 // Factor out code to call ic_miss_handler.
2951 // Generate code to call the inline cache miss handler.
2952 //
2953 // In most cases, this code will be generated out-of-line.
2954 // The method parameters are intended to provide some variability.
2955 // ICM - Label which has to be bound to the start of useful code (past any traps).
2956 // trapMarker - Marking byte for the generated illtrap instructions (if any).
2957 // Any value except 0x00 is supported.
2958 // = 0x00 - do not generate illtrap instructions.
2959 // use nops to fill unused space.
2960 // requiredSize - required size of the generated code. If the actually
2961 // generated code is smaller, use padding instructions to fill up.
2962 // = 0 - no size requirement, no padding.
2963 // scratch - scratch register to hold branch target address.
2964 //
2965 // The method returns the code offset of the bound label.
2966 unsigned int MacroAssembler::call_ic_miss_handler(Label& ICM, int trapMarker, int requiredSize, Register scratch) {
2967 intptr_t startOffset = offset();
2968
2969 // Prevent entry at content_begin().
2970 if (trapMarker != 0) {
2971 z_illtrap(trapMarker);
2972 }
2973
2974 // Load address of inline cache miss code into scratch register
2975 // and branch to cache miss handler.
2976 BLOCK_COMMENT("IC miss handler {");
2977 BIND(ICM);
2978 unsigned int labelOffset = offset();
2979 AddressLiteral icmiss(SharedRuntime::get_ic_miss_stub());
2980
2981 load_const_optimized(scratch, icmiss);
2982 z_br(scratch);
2983
2984 // Fill unused space.
2985 if (requiredSize > 0) {
2986 while ((offset() - startOffset) < requiredSize) {
2987 if (trapMarker == 0) {
2988 z_nop();
2989 } else {
2990 z_illtrap(trapMarker);
2991 }
2992 }
2993 }
2994 BLOCK_COMMENT("} IC miss handler");
2995 return labelOffset;
2996 }
2997
2998 void MacroAssembler::nmethod_UEP(Label& ic_miss) {
2999 Register ic_reg = Z_inline_cache;
3000 int klass_offset = oopDesc::klass_offset_in_bytes();
3001 if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
3002 if (VM_Version::has_CompareBranch()) {
3003 z_cgij(Z_ARG1, 0, Assembler::bcondEqual, ic_miss);
3004 } else {
3005 z_ltgr(Z_ARG1, Z_ARG1);
3006 z_bre(ic_miss);
3007 }
3008 }
3009 // Compare cached class against klass from receiver.
3010 compare_klass_ptr(ic_reg, klass_offset, Z_ARG1, false);
3011 z_brne(ic_miss);
3012 }
3013
3014 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3015 Register super_klass,
3016 Register temp1_reg,
3017 Label* L_success,
3018 Label* L_failure,
3019 Label* L_slow_path,
3020 Register super_check_offset) {
3021 // Input registers must not overlap.
3022 assert_different_registers(sub_klass, super_klass, temp1_reg, super_check_offset);
3023
3024 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
3025 bool must_load_sco = ! super_check_offset->is_valid();
3026
3027 // Input registers must not overlap.
3028 if (must_load_sco) {
3029 assert(temp1_reg != noreg, "supply either a temp or a register offset");
3030 }
3031
3032 const Register Rsuper_check_offset = temp1_reg;
3033
3034 NearLabel L_fallthrough;
3035 int label_nulls = 0;
3036 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
3037 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
3038 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
3039 assert(label_nulls <= 1 || (L_slow_path == &L_fallthrough && label_nulls <= 2), "at most one null in the batch, usually");
3040
3041 BLOCK_COMMENT("check_klass_subtype_fast_path {");
3042 // If the pointers are equal, we are done (e.g., String[] elements).
3043 // This self-check enables sharing of secondary supertype arrays among
3044 // non-primary types such as array-of-interface. Otherwise, each such
3045 // type would need its own customized SSA.
3046 // We move this check to the front of the fast path because many
3047 // type checks are in fact trivially successful in this manner,
3048 // so we get a nicely predicted branch right at the start of the check.
3049 compare64_and_branch(sub_klass, super_klass, bcondEqual, *L_success);
3050
3051 // Check the supertype display, which is uint.
3052 if (must_load_sco) {
3053 z_llgf(Rsuper_check_offset, sco_offset, super_klass);
3054 super_check_offset = Rsuper_check_offset;
3055 }
3056
3057 Address super_check_addr(sub_klass, super_check_offset, 0);
3058 z_cg(super_klass, super_check_addr); // compare w/ displayed supertype
3059 branch_optimized(Assembler::bcondEqual, *L_success);
3060
3061 // This check has worked decisively for primary supers.
3062 // Secondary supers are sought in the super_cache ('super_cache_addr').
3063 // (Secondary supers are interfaces and very deeply nested subtypes.)
3064 // This works in the same check above because of a tricky aliasing
3065 // between the super_cache and the primary super display elements.
3066 // (The 'super_check_addr' can address either, as the case requires.)
3067 // Note that the cache is updated below if it does not help us find
3068 // what we need immediately.
3069 // So if it was a primary super, we can just fail immediately.
3070 // Otherwise, it's the slow path for us (no success at this point).
3071
3072 // Hacked jmp, which may only be used just before L_fallthrough.
3073 #define final_jmp(label) \
3074 if (&(label) == &L_fallthrough) { /*do nothing*/ } \
3075 else { branch_optimized(Assembler::bcondAlways, label); } /*omit semicolon*/
3076
3077 z_cfi(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
3078 if (L_failure == &L_fallthrough) {
3079 branch_optimized(Assembler::bcondEqual, *L_slow_path);
3080 } else {
3081 branch_optimized(Assembler::bcondNotEqual, *L_failure);
3082 final_jmp(*L_slow_path);
3083 }
3084
3085 bind(L_fallthrough);
3086 #undef final_jmp
3087 BLOCK_COMMENT("} check_klass_subtype_fast_path");
3088 // fallthru (to slow path)
3089 }
3090
3091 void MacroAssembler::check_klass_subtype_slow_path_linear(Register Rsubklass,
3092 Register Rsuperklass,
3093 Register Rarray_ptr, // tmp
3094 Register Rlength, // tmp
3095 Label* L_success,
3096 Label* L_failure,
3097 bool set_cond_codes /* unused */) {
3098 // Input registers must not overlap.
3099 // Also check for R1 which is explicitly used here.
3100 assert_different_registers(Z_R1, Rsubklass, Rsuperklass, Rarray_ptr, Rlength);
3101 NearLabel L_fallthrough;
3102 int label_nulls = 0;
3103 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
3104 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
3105 assert(label_nulls <= 1, "at most one null in the batch");
3106
3107 const int ss_offset = in_bytes(Klass::secondary_supers_offset());
3108 const int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3109
3110 const int length_offset = Array<Klass*>::length_offset_in_bytes();
3111 const int base_offset = Array<Klass*>::base_offset_in_bytes();
3112
3113 // Hacked jmp, which may only be used just before L_fallthrough.
3114 #define final_jmp(label) \
3115 if (&(label) == &L_fallthrough) { /*do nothing*/ } \
3116 else branch_optimized(Assembler::bcondAlways, label) /*omit semicolon*/
3117
3118 NearLabel loop_iterate, loop_count, match;
3119
3120 BLOCK_COMMENT("check_klass_subtype_slow_path_linear {");
3121 z_lg(Rarray_ptr, ss_offset, Rsubklass);
3122
3123 load_and_test_int(Rlength, Address(Rarray_ptr, length_offset));
3124 branch_optimized(Assembler::bcondZero, *L_failure);
3125
3126 // Oops in table are NO MORE compressed.
3127 z_cg(Rsuperklass, base_offset, Rarray_ptr); // Check array element for match.
3128 z_bre(match); // Shortcut for array length = 1.
3129
3130 // No match yet, so we must walk the array's elements.
3131 z_lngfr(Rlength, Rlength);
3132 z_sllg(Rlength, Rlength, LogBytesPerWord); // -#bytes of cache array
3133 z_llill(Z_R1, BytesPerWord); // Set increment/end index.
3134 add2reg(Rlength, 2 * BytesPerWord); // start index = -(n-2)*BytesPerWord
3135 z_slgr(Rarray_ptr, Rlength); // start addr: += (n-2)*BytesPerWord
3136 z_bru(loop_count);
3137
3138 BIND(loop_iterate);
3139 z_cg(Rsuperklass, base_offset, Rlength, Rarray_ptr); // Check array element for match.
3140 z_bre(match);
3141 BIND(loop_count);
3142 z_brxlg(Rlength, Z_R1, loop_iterate);
3143
3144 // Rsuperklass not found among secondary super classes -> failure.
3145 branch_optimized(Assembler::bcondAlways, *L_failure);
3146
3147 // Got a hit. Return success (zero result). Set cache.
3148 // Cache load doesn't happen here. For speed, it is directly emitted by the compiler.
3149
3150 BIND(match);
3151
3152 if (UseSecondarySupersCache) {
3153 z_stg(Rsuperklass, sc_offset, Rsubklass); // Save result to cache.
3154 }
3155 final_jmp(*L_success);
3156
3157 // Exit to the surrounding code.
3158 BIND(L_fallthrough);
3159 #undef final_jmp
3160 BLOCK_COMMENT("} check_klass_subtype_slow_path_linear");
3161 }
3162
3163 // If Register r is invalid, remove a new register from
3164 // available_regs, and add new register to regs_to_push.
3165 Register MacroAssembler::allocate_if_noreg(Register r,
3166 RegSetIterator<Register> &available_regs,
3167 RegSet ®s_to_push) {
3168 if (!r->is_valid()) {
3169 r = *available_regs++;
3170 regs_to_push += r;
3171 }
3172 return r;
3173 }
3174
3175 // check_klass_subtype_slow_path_table() looks for super_klass in the
3176 // hash table belonging to super_klass, branching to L_success or
3177 // L_failure as appropriate. This is essentially a shim which
3178 // allocates registers as necessary and then calls
3179 // lookup_secondary_supers_table() to do the work. Any of the temp
3180 // regs may be noreg, in which case this logic will choose some
3181 // registers push and pop them from the stack.
3182 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
3183 Register super_klass,
3184 Register temp_reg,
3185 Register temp2_reg,
3186 Register temp3_reg,
3187 Register temp4_reg,
3188 Register result_reg,
3189 Label* L_success,
3190 Label* L_failure,
3191 bool set_cond_codes) {
3192 BLOCK_COMMENT("check_klass_subtype_slow_path_table {");
3193
3194 RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg, temp4_reg);
3195
3196 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp4_reg);
3197
3198 Label L_fallthrough;
3199 int label_nulls = 0;
3200 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
3201 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
3202 assert(label_nulls <= 1, "at most one null in the batch");
3203
3204 RegSetIterator<Register> available_regs
3205 // Z_R0 will be used to hold Z_R15(Z_SP) while pushing a new frame, So don't use that here.
3206 // Z_R1 will be used to hold r_bitmap in lookup_secondary_supers_table_var, so can't be used
3207 // Z_R2, Z_R3, Z_R4 will be used in secondary_supers_verify, for the failure reporting
3208 = (RegSet::range(Z_R0, Z_R15) - temps - sub_klass - super_klass - Z_R1_scratch - Z_R0_scratch - Z_R2 - Z_R3 - Z_R4).begin();
3209
3210 RegSet pushed_regs;
3211
3212 temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
3213 temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
3214 temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);;
3215 temp4_reg = allocate_if_noreg(temp4_reg, available_regs, pushed_regs);
3216 result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
3217
3218 const int frame_size = pushed_regs.size() * BytesPerWord + frame::z_abi_160_size;
3219
3220 // Push & save registers
3221 {
3222 int i = 0;
3223 save_return_pc();
3224 push_frame(frame_size);
3225
3226 for (auto it = pushed_regs.begin(); *it != noreg; i++) {
3227 z_stg(*it++, i * BytesPerWord + frame::z_abi_160_size, Z_SP);
3228 }
3229 assert(i * BytesPerWord + frame::z_abi_160_size == frame_size, "sanity");
3230 }
3231
3232 lookup_secondary_supers_table_var(sub_klass,
3233 super_klass,
3234 temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
3235
3236 // NOTE: Condition Code should not be altered before jump instruction below !!!!
3237 z_cghi(result_reg, 0);
3238
3239 {
3240 int i = 0;
3241 for (auto it = pushed_regs.begin(); *it != noreg; ++i) {
3242 z_lg(*it++, i * BytesPerWord + frame::z_abi_160_size, Z_SP);
3243 }
3244 assert(i * BytesPerWord + frame::z_abi_160_size == frame_size, "sanity");
3245 pop_frame();
3246 restore_return_pc();
3247 }
3248
3249 // NB! Callers may assume that, when set_cond_codes is true, this
3250 // code sets temp2_reg to a nonzero value.
3251 if (set_cond_codes) {
3252 z_lghi(temp2_reg, 1);
3253 }
3254
3255 branch_optimized(bcondNotEqual, *L_failure);
3256
3257 if(L_success != &L_fallthrough) {
3258 z_bru(*L_success);
3259 }
3260
3261 bind(L_fallthrough);
3262 BLOCK_COMMENT("} check_klass_subtype_slow_path_table");
3263 }
3264
3265 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3266 Register super_klass,
3267 Register temp_reg,
3268 Register temp2_reg,
3269 Label* L_success,
3270 Label* L_failure,
3271 bool set_cond_codes) {
3272 BLOCK_COMMENT("check_klass_subtype_slow_path {");
3273 if (UseSecondarySupersTable) {
3274 check_klass_subtype_slow_path_table(sub_klass,
3275 super_klass,
3276 temp_reg,
3277 temp2_reg,
3278 /*temp3*/noreg,
3279 /*temp4*/noreg,
3280 /*result*/noreg,
3281 L_success,
3282 L_failure,
3283 set_cond_codes);
3284 } else {
3285 check_klass_subtype_slow_path_linear(sub_klass,
3286 super_klass,
3287 temp_reg,
3288 temp2_reg,
3289 L_success,
3290 L_failure,
3291 set_cond_codes);
3292 }
3293 BLOCK_COMMENT("} check_klass_subtype_slow_path");
3294 }
3295
3296 // Emitter for combining fast and slow path.
3297 void MacroAssembler::check_klass_subtype(Register sub_klass,
3298 Register super_klass,
3299 Register temp1_reg,
3300 Register temp2_reg,
3301 Label& L_success) {
3302 NearLabel failure;
3303 BLOCK_COMMENT(err_msg("check_klass_subtype(%s subclass of %s) {", sub_klass->name(), super_klass->name()));
3304 check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg,
3305 &L_success, &failure, nullptr);
3306 check_klass_subtype_slow_path(sub_klass, super_klass,
3307 temp1_reg, temp2_reg, &L_success, nullptr);
3308 BIND(failure);
3309 BLOCK_COMMENT("} check_klass_subtype");
3310 }
3311
3312 // scans r_count pointer sized words at [r_addr] for occurrence of r_value,
3313 // generic (r_count must be >0)
3314 // iff found: CC eq, r_result == 0
3315 void MacroAssembler::repne_scan(Register r_addr, Register r_value, Register r_count, Register r_result) {
3316 NearLabel L_loop, L_exit;
3317
3318 BLOCK_COMMENT("repne_scan {");
3319 #ifdef ASSERT
3320 z_chi(r_count, 0);
3321 asm_assert(bcondHigh, "count must be positive", 11);
3322 #endif
3323
3324 clear_reg(r_result, true /* whole_reg */, false /* set_cc */); // sets r_result=0, let's hope that search will be successful
3325
3326 bind(L_loop);
3327 z_cg(r_value, Address(r_addr));
3328 z_bre(L_exit); // branch on success
3329 z_la(r_addr, wordSize, r_addr);
3330 z_brct(r_count, L_loop);
3331
3332 // z_brct above doesn't change CC.
3333 // If we reach here, then the value in r_value is not present. Set r_result to 1.
3334 z_lghi(r_result, 1);
3335
3336 bind(L_exit);
3337 BLOCK_COMMENT("} repne_scan");
3338 }
3339
3340 // Ensure that the inline code and the stub are using the same registers.
3341 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS \
3342 do { \
3343 assert(r_super_klass == Z_ARG1 && \
3344 r_array_base == Z_ARG5 && \
3345 r_array_length == Z_ARG4 && \
3346 (r_array_index == Z_ARG3 || r_array_index == noreg) && \
3347 (r_sub_klass == Z_ARG2 || r_sub_klass == noreg) && \
3348 (r_bitmap == Z_R10 || r_bitmap == noreg) && \
3349 (r_result == Z_R11 || r_result == noreg), "registers must match s390.ad"); \
3350 } while(0)
3351
3352 // Note: this method also kills Z_R1_scratch register on machines older than z15
3353 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
3354 Register r_super_klass,
3355 Register r_temp1,
3356 Register r_temp2,
3357 Register r_temp3,
3358 Register r_temp4,
3359 Register r_result,
3360 u1 super_klass_slot) {
3361 NearLabel L_done, L_failure;
3362
3363 BLOCK_COMMENT("lookup_secondary_supers_table_const {");
3364
3365 const Register
3366 r_array_base = r_temp1,
3367 r_array_length = r_temp2,
3368 r_array_index = r_temp3,
3369 r_bitmap = r_temp4;
3370
3371 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
3372
3373 z_lg(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
3374
3375 // First check the bitmap to see if super_klass might be present. If
3376 // the bit is zero, we are certain that super_klass is not one of
3377 // the secondary supers.
3378 u1 bit = super_klass_slot;
3379 int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
3380
3381 z_sllg(r_array_index, r_bitmap, shift_count); // take the bit to 63rd location
3382
3383 // Initialize r_result with 0 (indicating success). If searching fails, r_result will be loaded
3384 // with 1 (failure) at the end of this method.
3385 clear_reg(r_result, true /* whole_reg */, false /* set_cc */); // r_result = 0
3386
3387 // We test the MSB of r_array_index, i.e., its sign bit
3388 testbit(r_array_index, 63);
3389 z_bfalse(L_failure); // if not set, then jump!!!
3390
3391 // We will consult the secondary-super array.
3392 z_lg(r_array_base, Address(r_sub_klass, Klass::secondary_supers_offset()));
3393
3394 // The value i in r_array_index is >= 1, so even though r_array_base
3395 // points to the length, we don't need to adjust it to point to the
3396 // data.
3397 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
3398
3399 // Get the first array index that can contain super_klass.
3400 if (bit != 0) {
3401 pop_count_long(r_array_index, r_array_index, Z_R1_scratch); // kills Z_R1_scratch on machines older than z15
3402
3403 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
3404 z_sllg(r_array_index, r_array_index, LogBytesPerWord); // scale
3405 } else {
3406 // Actually use index 0, but r_array_base and r_array_index are off by 1 word
3407 // such that the sum is precise.
3408 z_lghi(r_array_index, BytesPerWord); // for slow path (scaled)
3409 }
3410
3411 z_cg(r_super_klass, Address(r_array_base, r_array_index));
3412 branch_optimized(bcondEqual, L_done); // found a match; success
3413
3414 // Is there another entry to check? Consult the bitmap.
3415 testbit(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
3416 z_bfalse(L_failure);
3417
3418 // Linear probe. Rotate the bitmap so that the next bit to test is
3419 // in Bit 2 for the look-ahead check in the slow path.
3420 if (bit != 0) {
3421 z_rllg(r_bitmap, r_bitmap, 64-bit); // rotate right
3422 }
3423
3424 // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
3425 // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
3426 // Kills: r_array_length.
3427 // Returns: r_result
3428
3429 call_stub(StubRoutines::lookup_secondary_supers_table_slow_path_stub());
3430
3431 z_bru(L_done); // pass whatever result we got from a slow path
3432
3433 bind(L_failure);
3434
3435 z_lghi(r_result, 1);
3436
3437 bind(L_done);
3438 BLOCK_COMMENT("} lookup_secondary_supers_table_const");
3439
3440 if (VerifySecondarySupers) {
3441 verify_secondary_supers_table(r_sub_klass, r_super_klass, r_result,
3442 r_temp1, r_temp2, r_temp3);
3443 }
3444 }
3445
3446 // At runtime, return 0 in result if r_super_klass is a superclass of
3447 // r_sub_klass, otherwise return nonzero. Use this version of
3448 // lookup_secondary_supers_table() if you don't know ahead of time
3449 // which superclass will be searched for. Used by interpreter and
3450 // runtime stubs. It is larger and has somewhat greater latency than
3451 // the version above, which takes a constant super_klass_slot.
3452 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
3453 Register r_super_klass,
3454 Register temp1,
3455 Register temp2,
3456 Register temp3,
3457 Register temp4,
3458 Register result) {
3459 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result, Z_R1_scratch);
3460
3461 Label L_done, L_failure;
3462
3463 BLOCK_COMMENT("lookup_secondary_supers_table_var {");
3464
3465 const Register
3466 r_array_index = temp3,
3467 slot = temp4, // NOTE: "slot" can't be Z_R0 otherwise z_sllg and z_rllg instructions below will mess up!!!!
3468 r_bitmap = Z_R1_scratch;
3469
3470 z_llgc(slot, Address(r_super_klass, Klass::hash_slot_offset()));
3471
3472 // Initialize r_result with 0 (indicating success). If searching fails, r_result will be loaded
3473 // with 1 (failure) at the end of this method.
3474 clear_reg(result, true /* whole_reg */, false /* set_cc */); // result = 0
3475
3476 z_lg(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
3477
3478 // First check the bitmap to see if super_klass might be present. If
3479 // the bit is zero, we are certain that super_klass is not one of
3480 // the secondary supers.
3481 z_xilf(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
3482 z_sllg(r_array_index, r_bitmap, /*d2 = */ 0, /* b2 = */ slot);
3483
3484 testbit(r_array_index, Klass::SECONDARY_SUPERS_TABLE_SIZE - 1);
3485 branch_optimized(bcondAllZero, L_failure);
3486
3487 const Register
3488 r_array_base = temp1,
3489 r_array_length = temp2;
3490
3491 // Get the first array index that can contain super_klass into r_array_index.
3492 // NOTE: Z_R1_scratch is holding bitmap (look above for r_bitmap). So let's try to save it.
3493 // On the other hand, r_array_base/temp1 is free at current moment (look at the load operation below).
3494 pop_count_long(r_array_index, r_array_index, temp1); // kills r_array_base/temp1 on machines older than z15
3495
3496 // The value i in r_array_index is >= 1, so even though r_array_base
3497 // points to the length, we don't need to adjust it to point to the data.
3498 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
3499 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
3500
3501 // We will consult the secondary-super array.
3502 z_lg(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
3503
3504 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
3505 z_sllg(r_array_index, r_array_index, LogBytesPerWord); // scale, r_array_index is loaded by popcnt above
3506
3507 z_cg(r_super_klass, Address(r_array_base, r_array_index));
3508 branch_optimized(bcondEqual, L_done); // found a match
3509
3510 // Note: this is a small hack:
3511 //
3512 // The operation "(slot ^ 63) === 63 - slot (mod 64)" has already been performed above.
3513 // Since we lack a rotate-right instruction, we achieve the same effect by rotating left
3514 // by "64 - slot" positions. This produces the result equivalent to a right rotation by "slot" positions.
3515 //
3516 // => initial slot value
3517 // => slot = 63 - slot // done above with that z_xilf instruction
3518 // => slot = 64 - slot // need to do for rotating right by "slot" positions
3519 // => slot = 64 - (63 - slot)
3520 // => slot = slot - 63 + 64
3521 // => slot = slot + 1
3522 //
3523 // So instead of rotating-left by 64-slot times, we can, for now, just rotate left by slot+1 and it would be fine.
3524
3525 // Linear probe. Rotate the bitmap so that the next bit to test is
3526 // in Bit 1.
3527 z_aghi(slot, 1); // slot = slot + 1
3528
3529 z_rllg(r_bitmap, r_bitmap, /*d2=*/ 0, /*b2=*/ slot);
3530 testbit(r_bitmap, 1);
3531 branch_optimized(bcondAllZero, L_failure);
3532
3533 // The slot we just inspected is at secondary_supers[r_array_index - 1].
3534 // The next slot to be inspected, by the logic we're about to call,
3535 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap
3536 // have been checked.
3537 lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index,
3538 r_bitmap, /*temp=*/ r_array_length, result, /*is_stub*/false);
3539
3540 // pass whatever we got from slow path
3541 z_bru(L_done);
3542
3543 bind(L_failure);
3544 z_lghi(result, 1); // load 1 to represent failure
3545
3546 bind(L_done);
3547
3548 BLOCK_COMMENT("} lookup_secondary_supers_table_var");
3549
3550 if (VerifySecondarySupers) {
3551 verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
3552 temp1, temp2, temp3);
3553 }
3554 }
3555
3556 // Called by code generated by check_klass_subtype_slow_path
3557 // above. This is called when there is a collision in the hashed
3558 // lookup in the secondary supers array.
3559 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
3560 Register r_array_base,
3561 Register r_array_index,
3562 Register r_bitmap,
3563 Register r_temp,
3564 Register r_result,
3565 bool is_stub) {
3566 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, r_result, r_temp);
3567
3568 const Register
3569 r_array_length = r_temp,
3570 r_sub_klass = noreg;
3571
3572 if(is_stub) {
3573 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
3574 }
3575
3576 BLOCK_COMMENT("lookup_secondary_supers_table_slow_path {");
3577 NearLabel L_done, L_failure;
3578
3579 // Load the array length.
3580 z_llgf(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
3581
3582 // And adjust the array base to point to the data.
3583 // NB!
3584 // Effectively increments the current slot index by 1.
3585 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
3586 add2reg(r_array_base, Array<Klass*>::base_offset_in_bytes());
3587
3588 // Linear probe
3589 NearLabel L_huge;
3590
3591 // The bitmap is full to bursting.
3592 z_chi(r_array_length, Klass::SECONDARY_SUPERS_BITMAP_FULL - 2);
3593 z_brh(L_huge);
3594
3595 // NB! Our caller has checked bits 0 and 1 in the bitmap. The
3596 // current slot (at secondary_supers[r_array_index]) has not yet
3597 // been inspected, and r_array_index may be out of bounds if we
3598 // wrapped around the end of the array.
3599
3600 { // This is conventional linear probing, but instead of terminating
3601 // when a null entry is found in the table, we maintain a bitmap
3602 // in which a 0 indicates missing entries.
3603 // As long as the bitmap is not completely full,
3604 // array_length == popcount(bitmap). The array_length check above
3605 // guarantees there are 0s in the bitmap, so the loop eventually
3606 // terminates.
3607
3608 #ifdef ASSERT
3609 // r_result is set to 0 by lookup_secondary_supers_table.
3610 // clear_reg(r_result, true /* whole_reg */, false /* set_cc */);
3611 z_cghi(r_result, 0);
3612 asm_assert(bcondEqual, "r_result required to be 0, used by z_locgr", 44);
3613
3614 // We should only reach here after having found a bit in the bitmap.
3615 z_ltgr(r_array_length, r_array_length);
3616 asm_assert(bcondHigh, "array_length > 0, should hold", 22);
3617 #endif // ASSERT
3618
3619 // Compute limit in r_array_length
3620 add2reg(r_array_length, -1);
3621 z_sllg(r_array_length, r_array_length, LogBytesPerWord);
3622
3623 NearLabel L_loop;
3624 bind(L_loop);
3625
3626 // Check for wraparound.
3627 z_cgr(r_array_index, r_array_length);
3628 z_locgr(r_array_index, r_result, bcondHigh); // r_result is containing 0
3629
3630 z_cg(r_super_klass, Address(r_array_base, r_array_index));
3631 z_bre(L_done); // success
3632
3633 // look-ahead check: if Bit 2 is 0, we're done
3634 testbit(r_bitmap, 2);
3635 z_bfalse(L_failure);
3636
3637 z_rllg(r_bitmap, r_bitmap, 64-1); // rotate right
3638 add2reg(r_array_index, BytesPerWord);
3639
3640 z_bru(L_loop);
3641 }
3642
3643 { // Degenerate case: more than 64 secondary supers.
3644 // FIXME: We could do something smarter here, maybe a vectorized
3645 // comparison or a binary search, but is that worth any added
3646 // complexity?
3647
3648 bind(L_huge);
3649 repne_scan(r_array_base, r_super_klass, r_array_length, r_result);
3650
3651 z_bru(L_done); // forward the result we got from repne_scan
3652 }
3653
3654 bind(L_failure);
3655 z_lghi(r_result, 1);
3656
3657 bind(L_done);
3658 BLOCK_COMMENT("} lookup_secondary_supers_table_slow_path");
3659 }
3660
3661 // Make sure that the hashed lookup and a linear scan agree.
3662 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
3663 Register r_super_klass,
3664 Register r_result /* expected */,
3665 Register r_temp1,
3666 Register r_temp2,
3667 Register r_temp3) {
3668 assert_different_registers(r_sub_klass, r_super_klass, r_result, r_temp1, r_temp2, r_temp3);
3669
3670 const Register
3671 r_array_base = r_temp1,
3672 r_array_length = r_temp2,
3673 r_array_index = r_temp3,
3674 r_bitmap = noreg; // unused
3675
3676 BLOCK_COMMENT("verify_secondary_supers_table {");
3677
3678 Label L_passed, L_failure;
3679
3680 // We will consult the secondary-super array.
3681 z_lg(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
3682
3683 // Load the array length.
3684 z_llgf(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
3685
3686 // And adjust the array base to point to the data.
3687 z_aghi(r_array_base, Array<Klass*>::base_offset_in_bytes());
3688
3689 const Register r_linear_result = r_array_index; // reuse
3690 z_chi(r_array_length, 0);
3691 load_on_condition_imm_32(r_linear_result, 1, bcondNotHigh); // load failure if array_length <= 0
3692 z_brc(bcondNotHigh, L_failure);
3693 repne_scan(r_array_base, r_super_klass, r_array_length, r_linear_result);
3694 bind(L_failure);
3695
3696 z_cr(r_result, r_linear_result);
3697 z_bre(L_passed);
3698
3699 // report fatal error and terminate VM
3700
3701 // Argument shuffle
3702 // Z_F1, Z_F3, Z_F5 are volatile regs
3703 z_ldgr(Z_F1, r_super_klass);
3704 z_ldgr(Z_F3, r_sub_klass);
3705 z_ldgr(Z_F5, r_linear_result);
3706
3707 z_lgr(Z_ARG4, r_result);
3708
3709 z_lgdr(Z_ARG1, Z_F1); // r_super_klass
3710 z_lgdr(Z_ARG2, Z_F3); // r_sub_klass
3711 z_lgdr(Z_ARG3, Z_F5); // r_linear_result
3712
3713 const char* msg = "mismatch";
3714 load_const_optimized(Z_ARG5, (address)msg);
3715
3716 call_VM_leaf(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure));
3717 should_not_reach_here();
3718
3719 bind(L_passed);
3720
3721 BLOCK_COMMENT("} verify_secondary_supers_table");
3722 }
3723
3724 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
3725 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
3726
3727 Label L_fallthrough;
3728 if (L_fast_path == nullptr) {
3729 L_fast_path = &L_fallthrough;
3730 } else if (L_slow_path == nullptr) {
3731 L_slow_path = &L_fallthrough;
3732 }
3733
3734 // Fast path check: class is fully initialized.
3735 // init_state needs acquire, but S390 is TSO, and so we are already good.
3736 z_cli(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
3737 z_bre(*L_fast_path);
3738
3739 // Fast path check: current thread is initializer thread
3740 z_cg(thread, Address(klass, InstanceKlass::init_thread_offset()));
3741 if (L_slow_path == &L_fallthrough) {
3742 z_bre(*L_fast_path);
3743 } else if (L_fast_path == &L_fallthrough) {
3744 z_brne(*L_slow_path);
3745 } else {
3746 Unimplemented();
3747 }
3748
3749 bind(L_fallthrough);
3750 }
3751
3752 // Increment a counter at counter_address when the eq condition code is
3753 // set. Kills registers tmp1_reg and tmp2_reg and preserves the condition code.
3754 void MacroAssembler::increment_counter_eq(address counter_address, Register tmp1_reg, Register tmp2_reg) {
3755 Label l;
3756 z_brne(l);
3757 load_const(tmp1_reg, counter_address);
3758 add2mem_32(Address(tmp1_reg), 1, tmp2_reg);
3759 z_cr(tmp1_reg, tmp1_reg); // Set cc to eq.
3760 bind(l);
3761 }
3762
3763 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) {
3764 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3765 bs->resolve_jobject(this, value, tmp1, tmp2);
3766 }
3767
3768 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) {
3769 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3770 bs->resolve_global_jobject(this, value, tmp1, tmp2);
3771 }
3772
3773 // Last_Java_sp must comply to the rules in frame_s390.hpp.
3774 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc, bool allow_relocation) {
3775 BLOCK_COMMENT("set_last_Java_frame {");
3776
3777 // Always set last_Java_pc and flags first because once last_Java_sp
3778 // is visible has_last_Java_frame is true and users will look at the
3779 // rest of the fields. (Note: flags should always be zero before we
3780 // get here so doesn't need to be set.)
3781
3782 // Verify that last_Java_pc was zeroed on return to Java.
3783 if (allow_relocation) {
3784 asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()),
3785 Z_thread,
3786 "last_Java_pc not zeroed before leaving Java",
3787 0x200);
3788 } else {
3789 asm_assert_mem8_is_zero_static(in_bytes(JavaThread::last_Java_pc_offset()),
3790 Z_thread,
3791 "last_Java_pc not zeroed before leaving Java",
3792 0x200);
3793 }
3794
3795 // When returning from calling out from Java mode the frame anchor's
3796 // last_Java_pc will always be set to null. It is set here so that
3797 // if we are doing a call to native (not VM) that we capture the
3798 // known pc and don't have to rely on the native call having a
3799 // standard frame linkage where we can find the pc.
3800 if (last_Java_pc!=noreg) {
3801 z_stg(last_Java_pc, Address(Z_thread, JavaThread::last_Java_pc_offset()));
3802 }
3803
3804 // This membar release is not required on z/Architecture, since the sequence of stores
3805 // in maintained. Nevertheless, we leave it in to document the required ordering.
3806 // The implementation of z_release() should be empty.
3807 // z_release();
3808
3809 z_stg(last_Java_sp, Address(Z_thread, JavaThread::last_Java_sp_offset()));
3810 BLOCK_COMMENT("} set_last_Java_frame");
3811 }
3812
3813 void MacroAssembler::reset_last_Java_frame(bool allow_relocation) {
3814 BLOCK_COMMENT("reset_last_Java_frame {");
3815
3816 if (allow_relocation) {
3817 asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()),
3818 Z_thread,
3819 "SP was not set, still zero",
3820 0x202);
3821 } else {
3822 asm_assert_mem8_isnot_zero_static(in_bytes(JavaThread::last_Java_sp_offset()),
3823 Z_thread,
3824 "SP was not set, still zero",
3825 0x202);
3826 }
3827
3828 // _last_Java_sp = 0
3829 // Clearing storage must be atomic here, so don't use clear_mem()!
3830 store_const(Address(Z_thread, JavaThread::last_Java_sp_offset()), 0);
3831
3832 // _last_Java_pc = 0
3833 store_const(Address(Z_thread, JavaThread::last_Java_pc_offset()), 0);
3834
3835 BLOCK_COMMENT("} reset_last_Java_frame");
3836 return;
3837 }
3838
3839 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, bool allow_relocation) {
3840 assert_different_registers(sp, tmp1);
3841
3842 // We cannot trust that code generated by the C++ compiler saves R14
3843 // to z_abi_160.return_pc, because sometimes it spills R14 using stmg at
3844 // z_abi_160.gpr14 (e.g. InterpreterRuntime::_new()).
3845 // Therefore we load the PC into tmp1 and let set_last_Java_frame() save
3846 // it into the frame anchor.
3847 get_PC(tmp1);
3848 set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1, allow_relocation);
3849 }
3850
3851 void MacroAssembler::set_thread_state(JavaThreadState new_state) {
3852 z_release();
3853
3854 assert(Immediate::is_uimm16(_thread_max_state), "enum value out of range for instruction");
3855 assert(sizeof(JavaThreadState) == sizeof(int), "enum value must have base type int");
3856 store_const(Address(Z_thread, JavaThread::thread_state_offset()), new_state, Z_R0, false);
3857 }
3858
3859 void MacroAssembler::get_vm_result_oop(Register oop_result) {
3860 z_lg(oop_result, Address(Z_thread, JavaThread::vm_result_oop_offset()));
3861 clear_mem(Address(Z_thread, JavaThread::vm_result_oop_offset()), sizeof(void*));
3862
3863 verify_oop(oop_result, FILE_AND_LINE);
3864 }
3865
3866 void MacroAssembler::get_vm_result_metadata(Register result) {
3867 z_lg(result, Address(Z_thread, JavaThread::vm_result_metadata_offset()));
3868 clear_mem(Address(Z_thread, JavaThread::vm_result_metadata_offset()), sizeof(void*));
3869 }
3870
3871 // We require that C code which does not return a value in vm_result will
3872 // leave it undisturbed.
3873 void MacroAssembler::set_vm_result(Register oop_result) {
3874 z_stg(oop_result, Address(Z_thread, JavaThread::vm_result_oop_offset()));
3875 }
3876
3877 // Explicit null checks (used for method handle code).
3878 void MacroAssembler::null_check(Register reg, Register tmp, int64_t offset) {
3879 if (!ImplicitNullChecks) {
3880 NearLabel ok;
3881
3882 compare64_and_branch(reg, (intptr_t) 0, Assembler::bcondNotEqual, ok);
3883
3884 // We just put the address into reg if it was 0 (tmp==Z_R0 is allowed so we can't use it for the address).
3885 address exception_entry = Interpreter::throw_NullPointerException_entry();
3886 load_absolute_address(reg, exception_entry);
3887 z_br(reg);
3888
3889 bind(ok);
3890 } else {
3891 if (needs_explicit_null_check((intptr_t)offset)) {
3892 // Provoke OS null exception if reg is null by
3893 // accessing M[reg] w/o changing any registers.
3894 z_lg(tmp, 0, reg);
3895 }
3896 // else
3897 // Nothing to do, (later) access of M[reg + offset]
3898 // will provoke OS null exception if reg is null.
3899 }
3900 }
3901
3902 //-------------------------------------
3903 // Compressed Klass Pointers
3904 //-------------------------------------
3905
3906 // Klass oop manipulations if compressed.
3907 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3908 Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided. (dst == src) also possible.
3909 address base = CompressedKlassPointers::base();
3910 int shift = CompressedKlassPointers::shift();
3911 bool need_zero_extend = base != nullptr;
3912
3913 BLOCK_COMMENT("cKlass encoder {");
3914
3915 #ifdef ASSERT
3916 Label ok;
3917 z_tmll(current, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
3918 z_brc(Assembler::bcondAllZero, ok);
3919 // The plain disassembler does not recognize illtrap. It instead displays
3920 // a 32-bit value. Issuing two illtraps assures the disassembler finds
3921 // the proper beginning of the next instruction.
3922 z_illtrap(0xee);
3923 z_illtrap(0xee);
3924 bind(ok);
3925 #endif
3926
3927 // Scale down the incoming klass pointer first.
3928 // We then can be sure we calculate an offset that fits into 32 bit.
3929 // More generally speaking: all subsequent calculations are purely 32-bit.
3930 if (shift != 0) {
3931 z_srlg(dst, current, shift);
3932 current = dst;
3933 }
3934
3935 if (base != nullptr) {
3936 // Use scaled-down base address parts to match scaled-down klass pointer.
3937 unsigned int base_h = ((unsigned long)base)>>(32+shift);
3938 unsigned int base_l = (unsigned int)(((unsigned long)base)>>shift);
3939
3940 // General considerations:
3941 // - when calculating (current_h - base_h), all digits must cancel (become 0).
3942 // Otherwise, we would end up with a compressed klass pointer which doesn't
3943 // fit into 32-bit.
3944 // - Only bit#33 of the difference could potentially be non-zero. For that
3945 // to happen, (current_l < base_l) must hold. In this case, the subtraction
3946 // will create a borrow out of bit#32, nicely killing bit#33.
3947 // - With the above, we only need to consider current_l and base_l to
3948 // calculate the result.
3949 // - Both values are treated as unsigned. The unsigned subtraction is
3950 // replaced by adding (unsigned) the 2's complement of the subtrahend.
3951
3952 if (base_l == 0) {
3953 // - By theory, the calculation to be performed here (current_h - base_h) MUST
3954 // cancel all high-word bits. Otherwise, we would end up with an offset
3955 // (i.e. compressed klass pointer) that does not fit into 32 bit.
3956 // - current_l remains unchanged.
3957 // - Therefore, we can replace all calculation with just a
3958 // zero-extending load 32 to 64 bit.
3959 // - Even that can be replaced with a conditional load if dst != current.
3960 // (this is a local view. The shift step may have requested zero-extension).
3961 } else {
3962 if ((base_h == 0) && is_uimm(base_l, 31)) {
3963 // If we happen to find that (base_h == 0), and that base_l is within the range
3964 // which can be represented by a signed int, then we can use 64bit signed add with
3965 // (-base_l) as 32bit signed immediate operand. The add will take care of the
3966 // upper 32 bits of the result, saving us the need of an extra zero extension.
3967 // For base_l to be in the required range, it must not have the most significant
3968 // bit (aka sign bit) set.
3969 lgr_if_needed(dst, current); // no zero/sign extension in this case!
3970 z_agfi(dst, -(int)base_l); // base_l must be passed as signed.
3971 need_zero_extend = false;
3972 current = dst;
3973 } else {
3974 // To begin with, we may need to copy and/or zero-extend the register operand.
3975 // We have to calculate (current_l - base_l). Because there is no unsigend
3976 // subtract instruction with immediate operand, we add the 2's complement of base_l.
3977 if (need_zero_extend) {
3978 z_llgfr(dst, current);
3979 need_zero_extend = false;
3980 } else {
3981 llgfr_if_needed(dst, current);
3982 }
3983 current = dst;
3984 z_alfi(dst, -base_l);
3985 }
3986 }
3987 }
3988
3989 if (need_zero_extend) {
3990 // We must zero-extend the calculated result. It may have some leftover bits in
3991 // the hi-word because we only did optimized calculations.
3992 z_llgfr(dst, current);
3993 } else {
3994 llgfr_if_needed(dst, current); // zero-extension while copying comes at no extra cost.
3995 }
3996
3997 BLOCK_COMMENT("} cKlass encoder");
3998 }
3999
4000 // This function calculates the size of the code generated by
4001 // decode_klass_not_null(register dst, Register src)
4002 // when Universe::heap() isn't null. Hence, if the instructions
4003 // it generates change, then this method needs to be updated.
4004 int MacroAssembler::instr_size_for_decode_klass_not_null() {
4005 address base = CompressedKlassPointers::base();
4006 int shift_size = CompressedKlassPointers::shift() == 0 ? 0 : 6; /* sllg */
4007 int addbase_size = 0;
4008
4009 if (base != nullptr) {
4010 unsigned int base_h = ((unsigned long)base)>>32;
4011 unsigned int base_l = (unsigned int)((unsigned long)base);
4012 if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
4013 addbase_size += 6; /* aih */
4014 } else if ((base_h == 0) && (base_l != 0)) {
4015 addbase_size += 6; /* algfi */
4016 } else {
4017 addbase_size += load_const_size();
4018 addbase_size += 4; /* algr */
4019 }
4020 }
4021 #ifdef ASSERT
4022 addbase_size += 10;
4023 addbase_size += 2; // Extra sigill.
4024 #endif
4025 return addbase_size + shift_size;
4026 }
4027
4028 // !!! If the instructions that get generated here change
4029 // then function instr_size_for_decode_klass_not_null()
4030 // needs to get updated.
4031 // This variant of decode_klass_not_null() must generate predictable code!
4032 // The code must only depend on globally known parameters.
4033 void MacroAssembler::decode_klass_not_null(Register dst) {
4034 address base = CompressedKlassPointers::base();
4035 int shift = CompressedKlassPointers::shift();
4036 int beg_off = offset();
4037
4038 BLOCK_COMMENT("cKlass decoder (const size) {");
4039
4040 if (shift != 0) { // Shift required?
4041 z_sllg(dst, dst, shift);
4042 }
4043 if (base != nullptr) {
4044 unsigned int base_h = ((unsigned long)base)>>32;
4045 unsigned int base_l = (unsigned int)((unsigned long)base);
4046 if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
4047 z_aih(dst, base_h); // Base has no set bits in lower half.
4048 } else if ((base_h == 0) && (base_l != 0)) {
4049 z_algfi(dst, base_l); // Base has no set bits in upper half.
4050 } else {
4051 load_const(Z_R0, base); // Base has set bits everywhere.
4052 z_algr(dst, Z_R0);
4053 }
4054 }
4055
4056 #ifdef ASSERT
4057 Label ok;
4058 z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
4059 z_brc(Assembler::bcondAllZero, ok);
4060 // The plain disassembler does not recognize illtrap. It instead displays
4061 // a 32-bit value. Issuing two illtraps assures the disassembler finds
4062 // the proper beginning of the next instruction.
4063 z_illtrap(0xd1);
4064 z_illtrap(0xd1);
4065 bind(ok);
4066 #endif
4067 assert(offset() == beg_off + instr_size_for_decode_klass_not_null(), "Code gen mismatch.");
4068
4069 BLOCK_COMMENT("} cKlass decoder (const size)");
4070 }
4071
4072 // This variant of decode_klass_not_null() is for cases where
4073 // 1) the size of the generated instructions may vary
4074 // 2) the result is (potentially) stored in a register different from the source.
4075 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
4076 address base = CompressedKlassPointers::base();
4077 int shift = CompressedKlassPointers::shift();
4078
4079 BLOCK_COMMENT("cKlass decoder {");
4080
4081 if (src == noreg) src = dst;
4082
4083 if (shift != 0) { // Shift or at least move required?
4084 z_sllg(dst, src, shift);
4085 } else {
4086 lgr_if_needed(dst, src);
4087 }
4088
4089 if (base != nullptr) {
4090 unsigned int base_h = ((unsigned long)base)>>32;
4091 unsigned int base_l = (unsigned int)((unsigned long)base);
4092 if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
4093 z_aih(dst, base_h); // Base has not set bits in lower half.
4094 } else if ((base_h == 0) && (base_l != 0)) {
4095 z_algfi(dst, base_l); // Base has no set bits in upper half.
4096 } else {
4097 load_const_optimized(Z_R0, base); // Base has set bits everywhere.
4098 z_algr(dst, Z_R0);
4099 }
4100 }
4101
4102 #ifdef ASSERT
4103 Label ok;
4104 z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
4105 z_brc(Assembler::bcondAllZero, ok);
4106 // The plain disassembler does not recognize illtrap. It instead displays
4107 // a 32-bit value. Issuing two illtraps assures the disassembler finds
4108 // the proper beginning of the next instruction.
4109 z_illtrap(0xd2);
4110 z_illtrap(0xd2);
4111 bind(ok);
4112 #endif
4113 BLOCK_COMMENT("} cKlass decoder");
4114 }
4115
4116 void MacroAssembler::load_klass(Register klass, Address mem) {
4117 z_llgf(klass, mem);
4118 // Attention: no null check here!
4119 decode_klass_not_null(klass);
4120 }
4121
4122 // Loads the obj's Klass* into dst.
4123 // Input:
4124 // src - the oop we want to load the klass from.
4125 // dst - output nklass.
4126 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
4127 BLOCK_COMMENT("load_narrow_klass_compact {");
4128 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
4129 z_lg(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4130 z_srlg(dst, dst, markWord::klass_shift);
4131 BLOCK_COMMENT("} load_narrow_klass_compact");
4132 }
4133
4134 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
4135 BLOCK_COMMENT("cmp_klass {");
4136 assert_different_registers(obj, klass, tmp);
4137 if (UseCompactObjectHeaders) {
4138 assert(tmp != noreg, "required");
4139 assert_different_registers(klass, obj, tmp);
4140 load_narrow_klass_compact(tmp, obj);
4141 z_cr(klass, tmp);
4142 } else {
4143 z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4144 }
4145 BLOCK_COMMENT("} cmp_klass");
4146 }
4147
4148 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
4149 BLOCK_COMMENT("cmp_klasses_from_objects {");
4150 if (UseCompactObjectHeaders) {
4151 assert(tmp1 != noreg && tmp2 != noreg, "required");
4152 assert_different_registers(obj1, obj2, tmp1, tmp2);
4153 load_narrow_klass_compact(tmp1, obj1);
4154 load_narrow_klass_compact(tmp2, obj2);
4155 z_cr(tmp1, tmp2);
4156 } else {
4157 z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
4158 z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
4159 }
4160 BLOCK_COMMENT("} cmp_klasses_from_objects");
4161 }
4162
4163 void MacroAssembler::load_klass(Register klass, Register src_oop) {
4164 if (UseCompactObjectHeaders) {
4165 load_narrow_klass_compact(klass, src_oop);
4166 decode_klass_not_null(klass);
4167 } else {
4168 z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
4169 decode_klass_not_null(klass);
4170 }
4171 }
4172
4173 void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
4174 assert(!UseCompactObjectHeaders, "Don't use with compact headers");
4175 assert_different_registers(dst_oop, klass, Z_R0);
4176 if (ck == noreg) ck = klass;
4177 encode_klass_not_null(ck, klass);
4178 z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
4179 }
4180
4181 void MacroAssembler::store_klass_gap(Register s, Register d) {
4182 assert(!UseCompactObjectHeaders, "Don't use with compact headers");
4183 assert(s != d, "not enough registers");
4184 // Support s = noreg.
4185 if (s != noreg) {
4186 z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
4187 } else {
4188 z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
4189 }
4190 }
4191
4192 // Compare klass ptr in memory against klass ptr in register.
4193 //
4194 // Rop1 - klass in register, always uncompressed.
4195 // disp - Offset of klass in memory, compressed/uncompressed, depending on runtime flag.
4196 // Rbase - Base address of cKlass in memory.
4197 // maybenull - True if Rop1 possibly is a null.
4198 void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rbase, bool maybenull) {
4199
4200 BLOCK_COMMENT("compare klass ptr {");
4201
4202 const int shift = CompressedKlassPointers::shift();
4203 address base = CompressedKlassPointers::base();
4204
4205 if (UseCompactObjectHeaders) {
4206 assert(shift >= 3, "cKlass encoder detected bad shift");
4207 } else {
4208 assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
4209 }
4210 assert_different_registers(Rop1, Z_R0);
4211 assert_different_registers(Rop1, Rbase, Z_R1);
4212
4213 // First encode register oop and then compare with cOop in memory.
4214 // This sequence saves an unnecessary cOop load and decode.
4215 if (base == nullptr) {
4216 if (shift == 0) {
4217 z_cl(Rop1, disp, Rbase); // Unscaled
4218 } else {
4219 z_srlg(Z_R0, Rop1, shift); // ZeroBased
4220 z_cl(Z_R0, disp, Rbase);
4221 }
4222 } else { // HeapBased
4223 #ifdef ASSERT
4224 bool used_R0 = true;
4225 bool used_R1 = true;
4226 #endif
4227 Register current = Rop1;
4228 Label done;
4229
4230 if (maybenull) { // null pointer must be preserved!
4231 z_ltgr(Z_R0, current);
4232 z_bre(done);
4233 current = Z_R0;
4234 }
4235
4236 unsigned int base_h = ((unsigned long)base)>>32;
4237 unsigned int base_l = (unsigned int)((unsigned long)base);
4238 if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
4239 lgr_if_needed(Z_R0, current);
4240 z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
4241 } else if ((base_h == 0) && (base_l != 0)) {
4242 lgr_if_needed(Z_R0, current);
4243 z_agfi(Z_R0, -(int)base_l);
4244 } else {
4245 int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
4246 add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
4247 }
4248
4249 if (shift != 0) {
4250 z_srlg(Z_R0, Z_R0, shift);
4251 }
4252 bind(done);
4253 z_cl(Z_R0, disp, Rbase);
4254 #ifdef ASSERT
4255 if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
4256 if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
4257 #endif
4258 }
4259
4260 BLOCK_COMMENT("} compare klass ptr");
4261 }
4262
4263 //---------------------------
4264 // Compressed oops
4265 //---------------------------
4266
4267 void MacroAssembler::encode_heap_oop(Register oop) {
4268 oop_encoder(oop, oop, true /*maybe null*/);
4269 }
4270
4271 void MacroAssembler::encode_heap_oop_not_null(Register oop) {
4272 oop_encoder(oop, oop, false /*not null*/);
4273 }
4274
4275 // Called with something derived from the oop base. e.g. oop_base>>3.
4276 int MacroAssembler::get_oop_base_pow2_offset(uint64_t oop_base) {
4277 unsigned int oop_base_ll = ((unsigned int)(oop_base >> 0)) & 0xffff;
4278 unsigned int oop_base_lh = ((unsigned int)(oop_base >> 16)) & 0xffff;
4279 unsigned int oop_base_hl = ((unsigned int)(oop_base >> 32)) & 0xffff;
4280 unsigned int oop_base_hh = ((unsigned int)(oop_base >> 48)) & 0xffff;
4281 unsigned int n_notzero_parts = (oop_base_ll == 0 ? 0:1)
4282 + (oop_base_lh == 0 ? 0:1)
4283 + (oop_base_hl == 0 ? 0:1)
4284 + (oop_base_hh == 0 ? 0:1);
4285
4286 assert(oop_base != 0, "This is for HeapBased cOops only");
4287
4288 if (n_notzero_parts != 1) { // Check if oop_base is just a few pages shy of a power of 2.
4289 uint64_t pow2_offset = 0x10000 - oop_base_ll;
4290 if (pow2_offset < 0x8000) { // This might not be necessary.
4291 uint64_t oop_base2 = oop_base + pow2_offset;
4292
4293 oop_base_ll = ((unsigned int)(oop_base2 >> 0)) & 0xffff;
4294 oop_base_lh = ((unsigned int)(oop_base2 >> 16)) & 0xffff;
4295 oop_base_hl = ((unsigned int)(oop_base2 >> 32)) & 0xffff;
4296 oop_base_hh = ((unsigned int)(oop_base2 >> 48)) & 0xffff;
4297 n_notzero_parts = (oop_base_ll == 0 ? 0:1) +
4298 (oop_base_lh == 0 ? 0:1) +
4299 (oop_base_hl == 0 ? 0:1) +
4300 (oop_base_hh == 0 ? 0:1);
4301 if (n_notzero_parts == 1) {
4302 assert(-(int64_t)pow2_offset != (int64_t)-1, "We use -1 to signal uninitialized base register");
4303 return -pow2_offset;
4304 }
4305 }
4306 }
4307 return 0;
4308 }
4309
4310 // If base address is offset from a straight power of two by just a few pages,
4311 // return this offset to the caller for a possible later composite add.
4312 // TODO/FIX: will only work correctly for 4k pages.
4313 int MacroAssembler::get_oop_base(Register Rbase, uint64_t oop_base) {
4314 int pow2_offset = get_oop_base_pow2_offset(oop_base);
4315
4316 load_const_optimized(Rbase, oop_base - pow2_offset); // Best job possible.
4317
4318 return pow2_offset;
4319 }
4320
4321 int MacroAssembler::get_oop_base_complement(Register Rbase, uint64_t oop_base) {
4322 int offset = get_oop_base(Rbase, oop_base);
4323 z_lcgr(Rbase, Rbase);
4324 return -offset;
4325 }
4326
4327 // Compare compressed oop in memory against oop in register.
4328 // Rop1 - Oop in register.
4329 // disp - Offset of cOop in memory.
4330 // Rbase - Base address of cOop in memory.
4331 // maybenull - True if Rop1 possibly is a null.
4332 // maybenulltarget - Branch target for Rop1 == nullptr, if flow control shall NOT continue with compare instruction.
4333 void MacroAssembler::compare_heap_oop(Register Rop1, Address mem, bool maybenull) {
4334 Register Rbase = mem.baseOrR0();
4335 Register Rindex = mem.indexOrR0();
4336 int64_t disp = mem.disp();
4337
4338 const int shift = CompressedOops::shift();
4339 address base = CompressedOops::base();
4340
4341 assert(UseCompressedOops, "must be on to call this method");
4342 assert(Universe::heap() != nullptr, "java heap must be initialized to call this method");
4343 assert((shift == 0) || (shift == LogMinObjAlignmentInBytes), "cOop encoder detected bad shift");
4344 assert_different_registers(Rop1, Z_R0);
4345 assert_different_registers(Rop1, Rbase, Z_R1);
4346 assert_different_registers(Rop1, Rindex, Z_R1);
4347
4348 BLOCK_COMMENT("compare heap oop {");
4349
4350 // First encode register oop and then compare with cOop in memory.
4351 // This sequence saves an unnecessary cOop load and decode.
4352 if (base == nullptr) {
4353 if (shift == 0) {
4354 z_cl(Rop1, disp, Rindex, Rbase); // Unscaled
4355 } else {
4356 z_srlg(Z_R0, Rop1, shift); // ZeroBased
4357 z_cl(Z_R0, disp, Rindex, Rbase);
4358 }
4359 } else { // HeapBased
4360 #ifdef ASSERT
4361 bool used_R0 = true;
4362 bool used_R1 = true;
4363 #endif
4364 Label done;
4365 int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
4366
4367 if (maybenull) { // null pointer must be preserved!
4368 z_ltgr(Z_R0, Rop1);
4369 z_bre(done);
4370 }
4371
4372 add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1);
4373 z_srlg(Z_R0, Z_R0, shift);
4374
4375 bind(done);
4376 z_cl(Z_R0, disp, Rindex, Rbase);
4377 #ifdef ASSERT
4378 if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
4379 if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
4380 #endif
4381 }
4382 BLOCK_COMMENT("} compare heap oop");
4383 }
4384
4385 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4386 const Address& addr, Register val,
4387 Register tmp1, Register tmp2, Register tmp3) {
4388 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
4389 ON_UNKNOWN_OOP_REF)) == 0, "unsupported decorator");
4390 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4391 decorators = AccessInternal::decorator_fixup(decorators, type);
4392 bool as_raw = (decorators & AS_RAW) != 0;
4393 if (as_raw) {
4394 bs->BarrierSetAssembler::store_at(this, decorators, type,
4395 addr, val,
4396 tmp1, tmp2, tmp3);
4397 } else {
4398 bs->store_at(this, decorators, type,
4399 addr, val,
4400 tmp1, tmp2, tmp3);
4401 }
4402 }
4403
4404 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
4405 const Address& addr, Register dst,
4406 Register tmp1, Register tmp2, Label *is_null) {
4407 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
4408 ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator");
4409 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4410 decorators = AccessInternal::decorator_fixup(decorators, type);
4411 bool as_raw = (decorators & AS_RAW) != 0;
4412 if (as_raw) {
4413 bs->BarrierSetAssembler::load_at(this, decorators, type,
4414 addr, dst,
4415 tmp1, tmp2, is_null);
4416 } else {
4417 bs->load_at(this, decorators, type,
4418 addr, dst,
4419 tmp1, tmp2, is_null);
4420 }
4421 }
4422
4423 void MacroAssembler::load_heap_oop(Register dest, const Address &a,
4424 Register tmp1, Register tmp2,
4425 DecoratorSet decorators, Label *is_null) {
4426 access_load_at(T_OBJECT, IN_HEAP | decorators, a, dest, tmp1, tmp2, is_null);
4427 }
4428
4429 void MacroAssembler::store_heap_oop(Register Roop, const Address &a,
4430 Register tmp1, Register tmp2, Register tmp3,
4431 DecoratorSet decorators) {
4432 access_store_at(T_OBJECT, IN_HEAP | decorators, a, Roop, tmp1, tmp2, tmp3);
4433 }
4434
4435 //-------------------------------------------------
4436 // Encode compressed oop. Generally usable encoder.
4437 //-------------------------------------------------
4438 // Rsrc - contains regular oop on entry. It remains unchanged.
4439 // Rdst - contains compressed oop on exit.
4440 // Rdst and Rsrc may indicate same register, in which case Rsrc does not remain unchanged.
4441 //
4442 // Rdst must not indicate scratch register Z_R1 (Z_R1_scratch) for functionality.
4443 // Rdst should not indicate scratch register Z_R0 (Z_R0_scratch) for performance.
4444 //
4445 // only32bitValid is set, if later code only uses the lower 32 bits. In this
4446 // case we must not fix the upper 32 bits.
4447 void MacroAssembler::oop_encoder(Register Rdst, Register Rsrc, bool maybenull,
4448 Register Rbase, int pow2_offset, bool only32bitValid) {
4449
4450 const address oop_base = CompressedOops::base();
4451 const int oop_shift = CompressedOops::shift();
4452 const bool disjoint = CompressedOops::base_disjoint();
4453
4454 assert(UseCompressedOops, "must be on to call this method");
4455 assert(Universe::heap() != nullptr, "java heap must be initialized to call this encoder");
4456 assert((oop_shift == 0) || (oop_shift == LogMinObjAlignmentInBytes), "cOop encoder detected bad shift");
4457
4458 if (disjoint || (oop_base == nullptr)) {
4459 BLOCK_COMMENT("cOop encoder zeroBase {");
4460 if (oop_shift == 0) {
4461 if (oop_base != nullptr && !only32bitValid) {
4462 z_llgfr(Rdst, Rsrc); // Clear upper bits in case the register will be decoded again.
4463 } else {
4464 lgr_if_needed(Rdst, Rsrc);
4465 }
4466 } else {
4467 z_srlg(Rdst, Rsrc, oop_shift);
4468 if (oop_base != nullptr && !only32bitValid) {
4469 z_llgfr(Rdst, Rdst); // Clear upper bits in case the register will be decoded again.
4470 }
4471 }
4472 BLOCK_COMMENT("} cOop encoder zeroBase");
4473 return;
4474 }
4475
4476 bool used_R0 = false;
4477 bool used_R1 = false;
4478
4479 BLOCK_COMMENT("cOop encoder general {");
4480 assert_different_registers(Rdst, Z_R1);
4481 assert_different_registers(Rsrc, Rbase);
4482 if (maybenull) {
4483 Label done;
4484 // We reorder shifting and subtracting, so that we can compare
4485 // and shift in parallel:
4486 //
4487 // cycle 0: potential LoadN, base = <const>
4488 // cycle 1: base = !base dst = src >> 3, cmp cr = (src != 0)
4489 // cycle 2: if (cr) br, dst = dst + base + offset
4490
4491 // Get oop_base components.
4492 if (pow2_offset == -1) {
4493 if (Rdst == Rbase) {
4494 if (Rdst == Z_R1 || Rsrc == Z_R1) {
4495 Rbase = Z_R0;
4496 used_R0 = true;
4497 } else {
4498 Rdst = Z_R1;
4499 used_R1 = true;
4500 }
4501 }
4502 if (Rbase == Z_R1) {
4503 used_R1 = true;
4504 }
4505 pow2_offset = get_oop_base_complement(Rbase, ((uint64_t)(intptr_t)oop_base) >> oop_shift);
4506 }
4507 assert_different_registers(Rdst, Rbase);
4508
4509 // Check for null oop (must be left alone) and shift.
4510 if (oop_shift != 0) { // Shift out alignment bits
4511 if (((intptr_t)oop_base&0xc000000000000000L) == 0L) { // We are sure: no single address will have the leftmost bit set.
4512 z_srag(Rdst, Rsrc, oop_shift); // Arithmetic shift sets the condition code.
4513 } else {
4514 z_srlg(Rdst, Rsrc, oop_shift);
4515 z_ltgr(Rsrc, Rsrc); // This is the recommended way of testing for zero.
4516 // This probably is faster, as it does not write a register. No!
4517 // z_cghi(Rsrc, 0);
4518 }
4519 } else {
4520 z_ltgr(Rdst, Rsrc); // Move null to result register.
4521 }
4522 z_bre(done);
4523
4524 // Subtract oop_base components.
4525 if ((Rdst == Z_R0) || (Rbase == Z_R0)) {
4526 z_algr(Rdst, Rbase);
4527 if (pow2_offset != 0) { add2reg(Rdst, pow2_offset); }
4528 } else {
4529 add2reg_with_index(Rdst, pow2_offset, Rbase, Rdst);
4530 }
4531 if (!only32bitValid) {
4532 z_llgfr(Rdst, Rdst); // Clear upper bits in case the register will be decoded again.
4533 }
4534 bind(done);
4535
4536 } else { // not null
4537 // Get oop_base components.
4538 if (pow2_offset == -1) {
4539 pow2_offset = get_oop_base_complement(Rbase, (uint64_t)(intptr_t)oop_base);
4540 }
4541
4542 // Subtract oop_base components and shift.
4543 if (Rdst == Z_R0 || Rsrc == Z_R0 || Rbase == Z_R0) {
4544 // Don't use lay instruction.
4545 if (Rdst == Rsrc) {
4546 z_algr(Rdst, Rbase);
4547 } else {
4548 lgr_if_needed(Rdst, Rbase);
4549 z_algr(Rdst, Rsrc);
4550 }
4551 if (pow2_offset != 0) add2reg(Rdst, pow2_offset);
4552 } else {
4553 add2reg_with_index(Rdst, pow2_offset, Rbase, Rsrc);
4554 }
4555 if (oop_shift != 0) { // Shift out alignment bits.
4556 z_srlg(Rdst, Rdst, oop_shift);
4557 }
4558 if (!only32bitValid) {
4559 z_llgfr(Rdst, Rdst); // Clear upper bits in case the register will be decoded again.
4560 }
4561 }
4562 #ifdef ASSERT
4563 if (used_R0 && Rdst != Z_R0 && Rsrc != Z_R0) { preset_reg(Z_R0, 0xb01bUL, 2); }
4564 if (used_R1 && Rdst != Z_R1 && Rsrc != Z_R1) { preset_reg(Z_R1, 0xb02bUL, 2); }
4565 #endif
4566 BLOCK_COMMENT("} cOop encoder general");
4567 }
4568
4569 //-------------------------------------------------
4570 // decode compressed oop. Generally usable decoder.
4571 //-------------------------------------------------
4572 // Rsrc - contains compressed oop on entry.
4573 // Rdst - contains regular oop on exit.
4574 // Rdst and Rsrc may indicate same register.
4575 // Rdst must not be the same register as Rbase, if Rbase was preloaded (before call).
4576 // Rdst can be the same register as Rbase. Then, either Z_R0 or Z_R1 must be available as scratch.
4577 // Rbase - register to use for the base
4578 // pow2_offset - offset of base to nice value. If -1, base must be loaded.
4579 // For performance, it is good to
4580 // - avoid Z_R0 for any of the argument registers.
4581 // - keep Rdst and Rsrc distinct from Rbase. Rdst == Rsrc is ok for performance.
4582 // - avoid Z_R1 for Rdst if Rdst == Rbase.
4583 void MacroAssembler::oop_decoder(Register Rdst, Register Rsrc, bool maybenull, Register Rbase, int pow2_offset) {
4584
4585 const address oop_base = CompressedOops::base();
4586 const int oop_shift = CompressedOops::shift();
4587 const bool disjoint = CompressedOops::base_disjoint();
4588
4589 assert(UseCompressedOops, "must be on to call this method");
4590 assert(Universe::heap() != nullptr, "java heap must be initialized to call this decoder");
4591 assert((oop_shift == 0) || (oop_shift == LogMinObjAlignmentInBytes),
4592 "cOop encoder detected bad shift");
4593
4594 // cOops are always loaded zero-extended from memory. No explicit zero-extension necessary.
4595
4596 if (oop_base != nullptr) {
4597 unsigned int oop_base_hl = ((unsigned int)((uint64_t)(intptr_t)oop_base >> 32)) & 0xffff;
4598 unsigned int oop_base_hh = ((unsigned int)((uint64_t)(intptr_t)oop_base >> 48)) & 0xffff;
4599 unsigned int oop_base_hf = ((unsigned int)((uint64_t)(intptr_t)oop_base >> 32)) & 0xFFFFffff;
4600 if (disjoint && (oop_base_hl == 0 || oop_base_hh == 0)) {
4601 BLOCK_COMMENT("cOop decoder disjointBase {");
4602 // We do not need to load the base. Instead, we can install the upper bits
4603 // with an OR instead of an ADD.
4604 Label done;
4605
4606 // Rsrc contains a narrow oop. Thus we are sure the leftmost <oop_shift> bits will never be set.
4607 if (maybenull) { // null pointer must be preserved!
4608 z_slag(Rdst, Rsrc, oop_shift); // Arithmetic shift sets the condition code.
4609 z_bre(done);
4610 } else {
4611 z_sllg(Rdst, Rsrc, oop_shift); // Logical shift leaves condition code alone.
4612 }
4613 if ((oop_base_hl != 0) && (oop_base_hh != 0)) {
4614 z_oihf(Rdst, oop_base_hf);
4615 } else if (oop_base_hl != 0) {
4616 z_oihl(Rdst, oop_base_hl);
4617 } else {
4618 assert(oop_base_hh != 0, "not heapbased mode");
4619 z_oihh(Rdst, oop_base_hh);
4620 }
4621 bind(done);
4622 BLOCK_COMMENT("} cOop decoder disjointBase");
4623 } else {
4624 BLOCK_COMMENT("cOop decoder general {");
4625 // There are three decode steps:
4626 // scale oop offset (shift left)
4627 // get base (in reg) and pow2_offset (constant)
4628 // add base, pow2_offset, and oop offset
4629 // The following register overlap situations may exist:
4630 // Rdst == Rsrc, Rbase any other
4631 // not a problem. Scaling in-place leaves Rbase undisturbed.
4632 // Loading Rbase does not impact the scaled offset.
4633 // Rdst == Rbase, Rsrc any other
4634 // scaling would destroy a possibly preloaded Rbase. Loading Rbase
4635 // would destroy the scaled offset.
4636 // Remedy: use Rdst_tmp if Rbase has been preloaded.
4637 // use Rbase_tmp if base has to be loaded.
4638 // Rsrc == Rbase, Rdst any other
4639 // Only possible without preloaded Rbase.
4640 // Loading Rbase does not destroy compressed oop because it was scaled into Rdst before.
4641 // Rsrc == Rbase, Rdst == Rbase
4642 // Only possible without preloaded Rbase.
4643 // Loading Rbase would destroy compressed oop. Scaling in-place is ok.
4644 // Remedy: use Rbase_tmp.
4645 //
4646 Label done;
4647 Register Rdst_tmp = Rdst;
4648 Register Rbase_tmp = Rbase;
4649 bool used_R0 = false;
4650 bool used_R1 = false;
4651 bool base_preloaded = pow2_offset >= 0;
4652 guarantee(!(base_preloaded && (Rsrc == Rbase)), "Register clash, check caller");
4653 assert(oop_shift != 0, "room for optimization");
4654
4655 // Check if we need to use scratch registers.
4656 if (Rdst == Rbase) {
4657 assert(!(((Rdst == Z_R0) && (Rsrc == Z_R1)) || ((Rdst == Z_R1) && (Rsrc == Z_R0))), "need a scratch reg");
4658 if (Rdst != Rsrc) {
4659 if (base_preloaded) { Rdst_tmp = (Rdst == Z_R1) ? Z_R0 : Z_R1; }
4660 else { Rbase_tmp = (Rdst == Z_R1) ? Z_R0 : Z_R1; }
4661 } else {
4662 Rbase_tmp = (Rdst == Z_R1) ? Z_R0 : Z_R1;
4663 }
4664 }
4665 if (base_preloaded) lgr_if_needed(Rbase_tmp, Rbase);
4666
4667 // Scale oop and check for null.
4668 // Rsrc contains a narrow oop. Thus we are sure the leftmost <oop_shift> bits will never be set.
4669 if (maybenull) { // null pointer must be preserved!
4670 z_slag(Rdst_tmp, Rsrc, oop_shift); // Arithmetic shift sets the condition code.
4671 z_bre(done);
4672 } else {
4673 z_sllg(Rdst_tmp, Rsrc, oop_shift); // Logical shift leaves condition code alone.
4674 }
4675
4676 // Get oop_base components.
4677 if (!base_preloaded) {
4678 pow2_offset = get_oop_base(Rbase_tmp, (uint64_t)(intptr_t)oop_base);
4679 }
4680
4681 // Add up all components.
4682 if ((Rbase_tmp == Z_R0) || (Rdst_tmp == Z_R0)) {
4683 z_algr(Rdst_tmp, Rbase_tmp);
4684 if (pow2_offset != 0) { add2reg(Rdst_tmp, pow2_offset); }
4685 } else {
4686 add2reg_with_index(Rdst_tmp, pow2_offset, Rbase_tmp, Rdst_tmp);
4687 }
4688
4689 bind(done);
4690 lgr_if_needed(Rdst, Rdst_tmp);
4691 #ifdef ASSERT
4692 if (used_R0 && Rdst != Z_R0 && Rsrc != Z_R0) { preset_reg(Z_R0, 0xb03bUL, 2); }
4693 if (used_R1 && Rdst != Z_R1 && Rsrc != Z_R1) { preset_reg(Z_R1, 0xb04bUL, 2); }
4694 #endif
4695 BLOCK_COMMENT("} cOop decoder general");
4696 }
4697 } else {
4698 BLOCK_COMMENT("cOop decoder zeroBase {");
4699 if (oop_shift == 0) {
4700 lgr_if_needed(Rdst, Rsrc);
4701 } else {
4702 z_sllg(Rdst, Rsrc, oop_shift);
4703 }
4704 BLOCK_COMMENT("} cOop decoder zeroBase");
4705 }
4706 }
4707
4708 // ((OopHandle)result).resolve();
4709 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
4710 access_load_at(T_OBJECT, IN_NATIVE, Address(result, 0), result, tmp1, tmp2);
4711 }
4712
4713 void MacroAssembler::load_method_holder(Register holder, Register method) {
4714 mem2reg_opt(holder, Address(method, Method::const_offset()));
4715 mem2reg_opt(holder, Address(holder, ConstMethod::constants_offset()));
4716 mem2reg_opt(holder, Address(holder, ConstantPool::pool_holder_offset()));
4717 }
4718
4719 //---------------------------------------------------------------
4720 //--- Operations on arrays.
4721 //---------------------------------------------------------------
4722
4723 // Compiler ensures base is doubleword aligned and cnt is #doublewords.
4724 // Emitter does not KILL cnt and base arguments, since they need to be copied to
4725 // work registers anyway.
4726 // Actually, only r0, r1, and r5 are killed.
4727 unsigned int MacroAssembler::Clear_Array(Register cnt_arg, Register base_pointer_arg, Register odd_tmp_reg) {
4728
4729 int block_start = offset();
4730 Register dst_len = Z_R1; // Holds dst len for MVCLE.
4731 Register dst_addr = Z_R0; // Holds dst addr for MVCLE.
4732
4733 Label doXC, doMVCLE, done;
4734
4735 BLOCK_COMMENT("Clear_Array {");
4736
4737 // Check for zero len and convert to long.
4738 z_ltgfr(odd_tmp_reg, cnt_arg);
4739 z_bre(done); // Nothing to do if len == 0.
4740
4741 // Prefetch data to be cleared.
4742 if (VM_Version::has_Prefetch()) {
4743 z_pfd(0x02, 0, Z_R0, base_pointer_arg);
4744 z_pfd(0x02, 256, Z_R0, base_pointer_arg);
4745 }
4746
4747 z_sllg(dst_len, odd_tmp_reg, 3); // #bytes to clear.
4748 z_cghi(odd_tmp_reg, 32); // Check for len <= 256 bytes (<=32 DW).
4749 z_brnh(doXC); // If so, use executed XC to clear.
4750
4751 // MVCLE: initialize long arrays (general case).
4752 bind(doMVCLE);
4753 z_lgr(dst_addr, base_pointer_arg);
4754 // Pass 0 as source length to MVCLE: destination will be filled with padding byte 0.
4755 // The even register of the register pair is not killed.
4756 clear_reg(odd_tmp_reg, true, false);
4757 MacroAssembler::move_long_ext(dst_addr, as_Register(odd_tmp_reg->encoding()-1), 0);
4758 z_bru(done);
4759
4760 // XC: initialize short arrays.
4761 Label XC_template; // Instr template, never exec directly!
4762 bind(XC_template);
4763 z_xc(0,0,base_pointer_arg,0,base_pointer_arg);
4764
4765 bind(doXC);
4766 add2reg(dst_len, -1); // Get #bytes-1 for EXECUTE.
4767 if (VM_Version::has_ExecuteExtensions()) {
4768 z_exrl(dst_len, XC_template); // Execute XC with var. len.
4769 } else {
4770 z_larl(odd_tmp_reg, XC_template);
4771 z_ex(dst_len,0,Z_R0,odd_tmp_reg); // Execute XC with var. len.
4772 }
4773 // z_bru(done); // fallthru
4774
4775 bind(done);
4776
4777 BLOCK_COMMENT("} Clear_Array");
4778
4779 int block_end = offset();
4780 return block_end - block_start;
4781 }
4782
4783 // Compiler ensures base is doubleword aligned and cnt is count of doublewords.
4784 // Emitter does not KILL any arguments nor work registers.
4785 // Emitter generates up to 16 XC instructions, depending on the array length.
4786 unsigned int MacroAssembler::Clear_Array_Const(long cnt, Register base) {
4787 int block_start = offset();
4788 int off;
4789 int lineSize_Bytes = AllocatePrefetchStepSize;
4790 int lineSize_DW = AllocatePrefetchStepSize>>LogBytesPerWord;
4791 bool doPrefetch = VM_Version::has_Prefetch();
4792 int XC_maxlen = 256;
4793 int numXCInstr = cnt > 0 ? (cnt*BytesPerWord-1)/XC_maxlen+1 : 0;
4794
4795 BLOCK_COMMENT("Clear_Array_Const {");
4796 assert(cnt*BytesPerWord <= 4096, "ClearArrayConst can handle 4k only");
4797
4798 // Do less prefetching for very short arrays.
4799 if (numXCInstr > 0) {
4800 // Prefetch only some cache lines, then begin clearing.
4801 if (doPrefetch) {
4802 if (cnt*BytesPerWord <= lineSize_Bytes/4) { // If less than 1/4 of a cache line to clear,
4803 z_pfd(0x02, 0, Z_R0, base); // prefetch just the first cache line.
4804 } else {
4805 assert(XC_maxlen == lineSize_Bytes, "ClearArrayConst needs 256B cache lines");
4806 for (off = 0; (off < AllocatePrefetchLines) && (off <= numXCInstr); off ++) {
4807 z_pfd(0x02, off*lineSize_Bytes, Z_R0, base);
4808 }
4809 }
4810 }
4811
4812 for (off=0; off<(numXCInstr-1); off++) {
4813 z_xc(off*XC_maxlen, XC_maxlen-1, base, off*XC_maxlen, base);
4814
4815 // Prefetch some cache lines in advance.
4816 if (doPrefetch && (off <= numXCInstr-AllocatePrefetchLines)) {
4817 z_pfd(0x02, (off+AllocatePrefetchLines)*lineSize_Bytes, Z_R0, base);
4818 }
4819 }
4820 if (off*XC_maxlen < cnt*BytesPerWord) {
4821 z_xc(off*XC_maxlen, (cnt*BytesPerWord-off*XC_maxlen)-1, base, off*XC_maxlen, base);
4822 }
4823 }
4824 BLOCK_COMMENT("} Clear_Array_Const");
4825
4826 int block_end = offset();
4827 return block_end - block_start;
4828 }
4829
4830 // Compiler ensures base is doubleword aligned and cnt is #doublewords.
4831 // Emitter does not KILL cnt and base arguments, since they need to be copied to
4832 // work registers anyway.
4833 // Actually, only r0, r1, (which are work registers) and odd_tmp_reg are killed.
4834 //
4835 // For very large arrays, exploit MVCLE H/W support.
4836 // MVCLE instruction automatically exploits H/W-optimized page mover.
4837 // - Bytes up to next page boundary are cleared with a series of XC to self.
4838 // - All full pages are cleared with the page mover H/W assist.
4839 // - Remaining bytes are again cleared by a series of XC to self.
4840 //
4841 unsigned int MacroAssembler::Clear_Array_Const_Big(long cnt, Register base_pointer_arg, Register odd_tmp_reg) {
4842
4843 int block_start = offset();
4844 Register dst_len = Z_R1; // Holds dst len for MVCLE.
4845 Register dst_addr = Z_R0; // Holds dst addr for MVCLE.
4846
4847 BLOCK_COMMENT("Clear_Array_Const_Big {");
4848
4849 // Get len to clear.
4850 load_const_optimized(dst_len, (long)cnt*8L); // in Bytes = #DW*8
4851
4852 // Prepare other args to MVCLE.
4853 z_lgr(dst_addr, base_pointer_arg);
4854 // Pass 0 as source length to MVCLE: destination will be filled with padding byte 0.
4855 // The even register of the register pair is not killed.
4856 (void) clear_reg(odd_tmp_reg, true, false); // Src len of MVCLE is zero.
4857 MacroAssembler::move_long_ext(dst_addr, as_Register(odd_tmp_reg->encoding() - 1), 0);
4858 BLOCK_COMMENT("} Clear_Array_Const_Big");
4859
4860 int block_end = offset();
4861 return block_end - block_start;
4862 }
4863
4864 // Allocator.
4865 unsigned int MacroAssembler::CopyRawMemory_AlignedDisjoint(Register src_reg, Register dst_reg,
4866 Register cnt_reg,
4867 Register tmp1_reg, Register tmp2_reg) {
4868 // Tmp1 is oddReg.
4869 // Tmp2 is evenReg.
4870
4871 int block_start = offset();
4872 Label doMVC, doMVCLE, done, MVC_template;
4873
4874 BLOCK_COMMENT("CopyRawMemory_AlignedDisjoint {");
4875
4876 // Check for zero len and convert to long.
4877 z_ltgfr(cnt_reg, cnt_reg); // Remember casted value for doSTG case.
4878 z_bre(done); // Nothing to do if len == 0.
4879
4880 z_sllg(Z_R1, cnt_reg, 3); // Dst len in bytes. calc early to have the result ready.
4881
4882 z_cghi(cnt_reg, 32); // Check for len <= 256 bytes (<=32 DW).
4883 z_brnh(doMVC); // If so, use executed MVC to clear.
4884
4885 bind(doMVCLE); // A lot of data (more than 256 bytes).
4886 // Prep dest reg pair.
4887 z_lgr(Z_R0, dst_reg); // dst addr
4888 // Dst len already in Z_R1.
4889 // Prep src reg pair.
4890 z_lgr(tmp2_reg, src_reg); // src addr
4891 z_lgr(tmp1_reg, Z_R1); // Src len same as dst len.
4892
4893 // Do the copy.
4894 move_long_ext(Z_R0, tmp2_reg, 0xb0); // Bypass cache.
4895 z_bru(done); // All done.
4896
4897 bind(MVC_template); // Just some data (not more than 256 bytes).
4898 z_mvc(0, 0, dst_reg, 0, src_reg);
4899
4900 bind(doMVC);
4901
4902 if (VM_Version::has_ExecuteExtensions()) {
4903 add2reg(Z_R1, -1);
4904 } else {
4905 add2reg(tmp1_reg, -1, Z_R1);
4906 z_larl(Z_R1, MVC_template);
4907 }
4908
4909 if (VM_Version::has_Prefetch()) {
4910 z_pfd(1, 0,Z_R0,src_reg);
4911 z_pfd(2, 0,Z_R0,dst_reg);
4912 // z_pfd(1,256,Z_R0,src_reg); // Assume very short copy.
4913 // z_pfd(2,256,Z_R0,dst_reg);
4914 }
4915
4916 if (VM_Version::has_ExecuteExtensions()) {
4917 z_exrl(Z_R1, MVC_template);
4918 } else {
4919 z_ex(tmp1_reg, 0, Z_R0, Z_R1);
4920 }
4921
4922 bind(done);
4923
4924 BLOCK_COMMENT("} CopyRawMemory_AlignedDisjoint");
4925
4926 int block_end = offset();
4927 return block_end - block_start;
4928 }
4929
4930 //-------------------------------------------------
4931 // Constants (scalar and oop) in constant pool
4932 //-------------------------------------------------
4933
4934 // Add a non-relocated constant to the CP.
4935 int MacroAssembler::store_const_in_toc(AddressLiteral& val) {
4936 long value = val.value();
4937 address tocPos = long_constant(value);
4938
4939 if (tocPos != nullptr) {
4940 int tocOffset = (int)(tocPos - code()->consts()->start());
4941 return tocOffset;
4942 }
4943 // Address_constant returned null, so no constant entry has been created.
4944 // In that case, we return a "fatal" offset, just in case that subsequently
4945 // generated access code is executed.
4946 return -1;
4947 }
4948
4949 // Returns the TOC offset where the address is stored.
4950 // Add a relocated constant to the CP.
4951 int MacroAssembler::store_oop_in_toc(AddressLiteral& oop) {
4952 // Use RelocationHolder::none for the constant pool entry.
4953 // Otherwise we will end up with a failing NativeCall::verify(x),
4954 // where x is the address of the constant pool entry.
4955 address tocPos = address_constant((address)oop.value(), RelocationHolder::none);
4956
4957 if (tocPos != nullptr) {
4958 int tocOffset = (int)(tocPos - code()->consts()->start());
4959 RelocationHolder rsp = oop.rspec();
4960 Relocation *rel = rsp.reloc();
4961
4962 // Store toc_offset in relocation, used by call_far_patchable.
4963 if ((relocInfo::relocType)rel->type() == relocInfo::runtime_call_w_cp_type) {
4964 ((runtime_call_w_cp_Relocation *)(rel))->set_constant_pool_offset(tocOffset);
4965 }
4966 // Relocate at the load's pc.
4967 relocate(rsp);
4968
4969 return tocOffset;
4970 }
4971 // Address_constant returned null, so no constant entry has been created
4972 // in that case, we return a "fatal" offset, just in case that subsequently
4973 // generated access code is executed.
4974 return -1;
4975 }
4976
4977 bool MacroAssembler::load_const_from_toc(Register dst, AddressLiteral& a, Register Rtoc) {
4978 int tocOffset = store_const_in_toc(a);
4979 if (tocOffset == -1) return false;
4980 address tocPos = tocOffset + code()->consts()->start();
4981 assert((address)code()->consts()->start() != nullptr, "Please add CP address");
4982 relocate(a.rspec());
4983 load_long_pcrelative(dst, tocPos);
4984 return true;
4985 }
4986
4987 bool MacroAssembler::load_oop_from_toc(Register dst, AddressLiteral& a, Register Rtoc) {
4988 int tocOffset = store_oop_in_toc(a);
4989 if (tocOffset == -1) return false;
4990 address tocPos = tocOffset + code()->consts()->start();
4991 assert((address)code()->consts()->start() != nullptr, "Please add CP address");
4992
4993 load_addr_pcrelative(dst, tocPos);
4994 return true;
4995 }
4996
4997 // If the instruction sequence at the given pc is a load_const_from_toc
4998 // sequence, return the value currently stored at the referenced position
4999 // in the TOC.
5000 intptr_t MacroAssembler::get_const_from_toc(address pc) {
5001
5002 assert(is_load_const_from_toc(pc), "must be load_const_from_pool");
5003
5004 long offset = get_load_const_from_toc_offset(pc);
5005 address dataLoc = nullptr;
5006 if (is_load_const_from_toc_pcrelative(pc)) {
5007 dataLoc = pc + offset;
5008 } else {
5009 CodeBlob* cb = CodeCache::find_blob(pc);
5010 assert(cb && cb->is_nmethod(), "sanity");
5011 nmethod* nm = (nmethod*)cb;
5012 dataLoc = nm->ctable_begin() + offset;
5013 }
5014 return *(intptr_t *)dataLoc;
5015 }
5016
5017 // If the instruction sequence at the given pc is a load_const_from_toc
5018 // sequence, copy the passed-in new_data value into the referenced
5019 // position in the TOC.
5020 void MacroAssembler::set_const_in_toc(address pc, unsigned long new_data, CodeBlob *cb) {
5021 assert(is_load_const_from_toc(pc), "must be load_const_from_pool");
5022
5023 long offset = MacroAssembler::get_load_const_from_toc_offset(pc);
5024 address dataLoc = nullptr;
5025 if (is_load_const_from_toc_pcrelative(pc)) {
5026 dataLoc = pc+offset;
5027 } else {
5028 nmethod* nm = CodeCache::find_nmethod(pc);
5029 assert((cb == nullptr) || (nm == (nmethod*)cb), "instruction address should be in CodeBlob");
5030 dataLoc = nm->ctable_begin() + offset;
5031 }
5032 if (*(unsigned long *)dataLoc != new_data) { // Prevent cache invalidation: update only if necessary.
5033 *(unsigned long *)dataLoc = new_data;
5034 }
5035 }
5036
5037 // Dynamic TOC. Getter must only be called if "a" is a load_const_from_toc
5038 // site. Verify by calling is_load_const_from_toc() before!!
5039 // Offset is +/- 2**32 -> use long.
5040 long MacroAssembler::get_load_const_from_toc_offset(address a) {
5041 assert(is_load_const_from_toc_pcrelative(a), "expected pc relative load");
5042 // expected code sequence:
5043 // z_lgrl(t, simm32); len = 6
5044 unsigned long inst;
5045 unsigned int len = get_instruction(a, &inst);
5046 return get_pcrel_offset(inst);
5047 }
5048
5049 //**********************************************************************************
5050 // inspection of generated instruction sequences for a particular pattern
5051 //**********************************************************************************
5052
5053 bool MacroAssembler::is_load_const_from_toc_pcrelative(address a) {
5054 #ifdef ASSERT
5055 unsigned long inst;
5056 unsigned int len = get_instruction(a+2, &inst);
5057 if ((len == 6) && is_load_pcrelative_long(a) && is_call_pcrelative_long(inst)) {
5058 const int range = 128;
5059 Assembler::dump_code_range(tty, a, range, "instr(a) == z_lgrl && instr(a+2) == z_brasl");
5060 VM_Version::z_SIGSEGV();
5061 }
5062 #endif
5063 // expected code sequence:
5064 // z_lgrl(t, relAddr32); len = 6
5065 //TODO: verify accessed data is in CP, if possible.
5066 return is_load_pcrelative_long(a); // TODO: might be too general. Currently, only lgrl is used.
5067 }
5068
5069 bool MacroAssembler::is_load_const_from_toc_call(address a) {
5070 return is_load_const_from_toc(a) && is_call_byregister(a + load_const_from_toc_size());
5071 }
5072
5073 bool MacroAssembler::is_load_const_call(address a) {
5074 return is_load_const(a) && is_call_byregister(a + load_const_size());
5075 }
5076
5077 //-------------------------------------------------
5078 // Emitters for some really CICS instructions
5079 //-------------------------------------------------
5080
5081 void MacroAssembler::move_long_ext(Register dst, Register src, unsigned int pad) {
5082 assert(dst->encoding()%2==0, "must be an even/odd register pair");
5083 assert(src->encoding()%2==0, "must be an even/odd register pair");
5084 assert(pad<256, "must be a padding BYTE");
5085
5086 Label retry;
5087 bind(retry);
5088 Assembler::z_mvcle(dst, src, pad);
5089 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5090 }
5091
5092 void MacroAssembler::compare_long_ext(Register left, Register right, unsigned int pad) {
5093 assert(left->encoding() % 2 == 0, "must be an even/odd register pair");
5094 assert(right->encoding() % 2 == 0, "must be an even/odd register pair");
5095 assert(pad<256, "must be a padding BYTE");
5096
5097 Label retry;
5098 bind(retry);
5099 Assembler::z_clcle(left, right, pad, Z_R0);
5100 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5101 }
5102
5103 void MacroAssembler::compare_long_uni(Register left, Register right, unsigned int pad) {
5104 assert(left->encoding() % 2 == 0, "must be an even/odd register pair");
5105 assert(right->encoding() % 2 == 0, "must be an even/odd register pair");
5106 assert(pad<=0xfff, "must be a padding HALFWORD");
5107 assert(VM_Version::has_ETF2(), "instruction must be available");
5108
5109 Label retry;
5110 bind(retry);
5111 Assembler::z_clclu(left, right, pad, Z_R0);
5112 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5113 }
5114
5115 void MacroAssembler::search_string(Register end, Register start) {
5116 assert(end->encoding() != 0, "end address must not be in R0");
5117 assert(start->encoding() != 0, "start address must not be in R0");
5118
5119 Label retry;
5120 bind(retry);
5121 Assembler::z_srst(end, start);
5122 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5123 }
5124
5125 void MacroAssembler::search_string_uni(Register end, Register start) {
5126 assert(end->encoding() != 0, "end address must not be in R0");
5127 assert(start->encoding() != 0, "start address must not be in R0");
5128 assert(VM_Version::has_ETF3(), "instruction must be available");
5129
5130 Label retry;
5131 bind(retry);
5132 Assembler::z_srstu(end, start);
5133 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5134 }
5135
5136 void MacroAssembler::kmac(Register srcBuff) {
5137 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5138 assert(srcBuff->encoding() % 2 == 0, "src buffer/len must be an even/odd register pair");
5139
5140 Label retry;
5141 bind(retry);
5142 Assembler::z_kmac(Z_R0, srcBuff);
5143 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5144 }
5145
5146 void MacroAssembler::kimd(Register srcBuff) {
5147 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5148 assert(srcBuff->encoding() % 2 == 0, "src buffer/len must be an even/odd register pair");
5149
5150 Label retry;
5151 bind(retry);
5152 Assembler::z_kimd(Z_R0, srcBuff);
5153 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5154 }
5155
5156 void MacroAssembler::klmd(Register srcBuff) {
5157 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5158 assert(srcBuff->encoding() % 2 == 0, "src buffer/len must be an even/odd register pair");
5159
5160 Label retry;
5161 bind(retry);
5162 Assembler::z_klmd(Z_R0, srcBuff);
5163 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5164 }
5165
5166 void MacroAssembler::km(Register dstBuff, Register srcBuff) {
5167 // DstBuff and srcBuff are allowed to be the same register (encryption in-place).
5168 // DstBuff and srcBuff storage must not overlap destructively, and neither must overlap the parameter block.
5169 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5170 assert(dstBuff->encoding() % 2 == 0, "dst buffer addr must be an even register");
5171 assert(srcBuff->encoding() % 2 == 0, "src buffer addr/len must be an even/odd register pair");
5172
5173 Label retry;
5174 bind(retry);
5175 Assembler::z_km(dstBuff, srcBuff);
5176 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5177 }
5178
5179 void MacroAssembler::kmc(Register dstBuff, Register srcBuff) {
5180 // DstBuff and srcBuff are allowed to be the same register (encryption in-place).
5181 // DstBuff and srcBuff storage must not overlap destructively, and neither must overlap the parameter block.
5182 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5183 assert(dstBuff->encoding() % 2 == 0, "dst buffer addr must be an even register");
5184 assert(srcBuff->encoding() % 2 == 0, "src buffer addr/len must be an even/odd register pair");
5185
5186 Label retry;
5187 bind(retry);
5188 Assembler::z_kmc(dstBuff, srcBuff);
5189 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5190 }
5191
5192 void MacroAssembler::kmctr(Register dstBuff, Register ctrBuff, Register srcBuff) {
5193 // DstBuff and srcBuff are allowed to be the same register (encryption in-place).
5194 // DstBuff and srcBuff storage must not overlap destructively, and neither must overlap the parameter block.
5195 assert(srcBuff->encoding() != 0, "src buffer address can't be in Z_R0");
5196 assert(dstBuff->encoding() != 0, "dst buffer address can't be in Z_R0");
5197 assert(ctrBuff->encoding() != 0, "ctr buffer address can't be in Z_R0");
5198 assert(ctrBuff->encoding() % 2 == 0, "ctr buffer addr must be an even register");
5199 assert(dstBuff->encoding() % 2 == 0, "dst buffer addr must be an even register");
5200 assert(srcBuff->encoding() % 2 == 0, "src buffer addr/len must be an even/odd register pair");
5201
5202 Label retry;
5203 bind(retry);
5204 Assembler::z_kmctr(dstBuff, ctrBuff, srcBuff);
5205 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5206 }
5207
5208 void MacroAssembler::cksm(Register crcBuff, Register srcBuff) {
5209 assert(srcBuff->encoding() % 2 == 0, "src buffer addr/len must be an even/odd register pair");
5210
5211 Label retry;
5212 bind(retry);
5213 Assembler::z_cksm(crcBuff, srcBuff);
5214 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5215 }
5216
5217 void MacroAssembler::translate_oo(Register r1, Register r2, uint m3) {
5218 assert(r1->encoding() % 2 == 0, "dst addr/src len must be an even/odd register pair");
5219 assert((m3 & 0b1110) == 0, "Unused mask bits must be zero");
5220
5221 Label retry;
5222 bind(retry);
5223 Assembler::z_troo(r1, r2, m3);
5224 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5225 }
5226
5227 void MacroAssembler::translate_ot(Register r1, Register r2, uint m3) {
5228 assert(r1->encoding() % 2 == 0, "dst addr/src len must be an even/odd register pair");
5229 assert((m3 & 0b1110) == 0, "Unused mask bits must be zero");
5230
5231 Label retry;
5232 bind(retry);
5233 Assembler::z_trot(r1, r2, m3);
5234 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5235 }
5236
5237 void MacroAssembler::translate_to(Register r1, Register r2, uint m3) {
5238 assert(r1->encoding() % 2 == 0, "dst addr/src len must be an even/odd register pair");
5239 assert((m3 & 0b1110) == 0, "Unused mask bits must be zero");
5240
5241 Label retry;
5242 bind(retry);
5243 Assembler::z_trto(r1, r2, m3);
5244 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5245 }
5246
5247 void MacroAssembler::translate_tt(Register r1, Register r2, uint m3) {
5248 assert(r1->encoding() % 2 == 0, "dst addr/src len must be an even/odd register pair");
5249 assert((m3 & 0b1110) == 0, "Unused mask bits must be zero");
5250
5251 Label retry;
5252 bind(retry);
5253 Assembler::z_trtt(r1, r2, m3);
5254 Assembler::z_brc(Assembler::bcondOverflow /* CC==3 (iterate) */, retry);
5255 }
5256
5257 //---------------------------------------
5258 // Helpers for Intrinsic Emitters
5259 //---------------------------------------
5260
5261 /**
5262 * uint32_t crc;
5263 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
5264 */
5265 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) {
5266 assert_different_registers(crc, table, tmp);
5267 assert_different_registers(val, table);
5268 if (crc == val) { // Must rotate first to use the unmodified value.
5269 rotate_then_insert(tmp, val, 56-2, 63-2, 2, true); // Insert byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
5270 z_srl(crc, 8); // Unsigned shift, clear leftmost 8 bits.
5271 } else {
5272 z_srl(crc, 8); // Unsigned shift, clear leftmost 8 bits.
5273 rotate_then_insert(tmp, val, 56-2, 63-2, 2, true); // Insert byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
5274 }
5275 z_x(crc, Address(table, tmp, 0));
5276 }
5277
5278 /**
5279 * uint32_t crc;
5280 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
5281 */
5282 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
5283 fold_byte_crc32(crc, crc, table, tmp);
5284 }
5285
5286 /**
5287 * Emits code to update CRC-32 with a byte value according to constants in table.
5288 *
5289 * @param [in,out]crc Register containing the crc.
5290 * @param [in]val Register containing the byte to fold into the CRC.
5291 * @param [in]table Register containing the table of crc constants.
5292 *
5293 * uint32_t crc;
5294 * val = crc_table[(val ^ crc) & 0xFF];
5295 * crc = val ^ (crc >> 8);
5296 */
5297 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
5298 z_xr(val, crc);
5299 fold_byte_crc32(crc, val, table, val);
5300 }
5301
5302
5303 /**
5304 * @param crc register containing existing CRC (32-bit)
5305 * @param buf register pointing to input byte buffer (byte*)
5306 * @param len register containing number of bytes
5307 * @param table register pointing to CRC table
5308 */
5309 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table, Register data) {
5310 assert_different_registers(crc, buf, len, table, data);
5311
5312 Label L_mainLoop, L_done;
5313 const int mainLoop_stepping = 1;
5314
5315 // Process all bytes in a single-byte loop.
5316 z_ltr(len, len);
5317 z_brnh(L_done);
5318
5319 bind(L_mainLoop);
5320 z_llgc(data, Address(buf, (intptr_t)0));// Current byte of input buffer (zero extended). Avoids garbage in upper half of register.
5321 add2reg(buf, mainLoop_stepping); // Advance buffer position.
5322 update_byte_crc32(crc, data, table);
5323 z_brct(len, L_mainLoop); // Iterate.
5324
5325 bind(L_done);
5326 }
5327
5328 /**
5329 * Emits code to update CRC-32 with a 4-byte value according to constants in table.
5330 * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c.
5331 *
5332 */
5333 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
5334 Register t0, Register t1, Register t2, Register t3) {
5335 // This is what we implement (the DOBIG4 part):
5336 //
5337 // #define DOBIG4 c ^= *++buf4; \
5338 // c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \
5339 // crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24]
5340 // #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4
5341 // Pre-calculate (constant) column offsets, use columns 4..7 for big-endian.
5342 const int ix0 = 4*(4*CRC32_COLUMN_SIZE);
5343 const int ix1 = 5*(4*CRC32_COLUMN_SIZE);
5344 const int ix2 = 6*(4*CRC32_COLUMN_SIZE);
5345 const int ix3 = 7*(4*CRC32_COLUMN_SIZE);
5346
5347 // XOR crc with next four bytes of buffer.
5348 lgr_if_needed(t0, crc);
5349 z_x(t0, Address(buf, bufDisp));
5350 if (bufInc != 0) {
5351 add2reg(buf, bufInc);
5352 }
5353
5354 // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices.
5355 rotate_then_insert(t3, t0, 56-2, 63-2, 2, true); // ((c >> 0) & 0xff) << 2
5356 rotate_then_insert(t2, t0, 56-2, 63-2, 2-8, true); // ((c >> 8) & 0xff) << 2
5357 rotate_then_insert(t1, t0, 56-2, 63-2, 2-16, true); // ((c >> 16) & 0xff) << 2
5358 rotate_then_insert(t0, t0, 56-2, 63-2, 2-24, true); // ((c >> 24) & 0xff) << 2
5359
5360 // XOR indexed table values to calculate updated crc.
5361 z_ly(t2, Address(table, t2, (intptr_t)ix1));
5362 z_ly(t0, Address(table, t0, (intptr_t)ix3));
5363 z_xy(t2, Address(table, t3, (intptr_t)ix0));
5364 z_xy(t0, Address(table, t1, (intptr_t)ix2));
5365 z_xr(t0, t2); // Now t0 contains the updated CRC value.
5366 lgr_if_needed(crc, t0);
5367 }
5368
5369 /**
5370 * @param crc register containing existing CRC (32-bit)
5371 * @param buf register pointing to input byte buffer (byte*)
5372 * @param len register containing number of bytes
5373 * @param table register pointing to CRC table
5374 *
5375 * uses Z_R10..Z_R13 as work register. Must be saved/restored by caller!
5376 */
5377 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
5378 Register t0, Register t1, Register t2, Register t3,
5379 bool invertCRC) {
5380 assert_different_registers(crc, buf, len, table);
5381
5382 Label L_mainLoop, L_tail;
5383 Register data = t0;
5384 Register ctr = Z_R0;
5385 const int mainLoop_stepping = 4;
5386 const int log_stepping = exact_log2(mainLoop_stepping);
5387
5388 // Don't test for len <= 0 here. This pathological case should not occur anyway.
5389 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles.
5390 // The situation itself is detected and handled correctly by the conditional branches
5391 // following aghi(len, -stepping) and aghi(len, +stepping).
5392
5393 if (invertCRC) {
5394 not_(crc, noreg, false); // 1s complement of crc
5395 }
5396
5397 // Check for short (<4 bytes) buffer.
5398 z_srag(ctr, len, log_stepping);
5399 z_brnh(L_tail);
5400
5401 z_lrvr(crc, crc); // Revert byte order because we are dealing with big-endian data.
5402 rotate_then_insert(len, len, 64-log_stepping, 63, 0, true); // #bytes for tailLoop
5403
5404 BIND(L_mainLoop);
5405 update_1word_crc32(crc, buf, table, 0, mainLoop_stepping, crc, t1, t2, t3);
5406 z_brct(ctr, L_mainLoop); // Iterate.
5407
5408 z_lrvr(crc, crc); // Revert byte order back to original.
5409
5410 // Process last few (<8) bytes of buffer.
5411 BIND(L_tail);
5412 update_byteLoop_crc32(crc, buf, len, table, data);
5413
5414 if (invertCRC) {
5415 not_(crc, noreg, false); // 1s complement of crc
5416 }
5417 }
5418
5419 /**
5420 * @param crc register containing existing CRC (32-bit)
5421 * @param buf register pointing to input byte buffer (byte*)
5422 * @param len register containing number of bytes
5423 * @param table register pointing to CRC table
5424 */
5425 void MacroAssembler::kernel_crc32_1byte(Register crc, Register buf, Register len, Register table,
5426 Register t0, Register t1, Register t2, Register t3,
5427 bool invertCRC) {
5428 assert_different_registers(crc, buf, len, table);
5429 Register data = t0;
5430
5431 if (invertCRC) {
5432 not_(crc, noreg, false); // 1s complement of crc
5433 }
5434
5435 update_byteLoop_crc32(crc, buf, len, table, data);
5436
5437 if (invertCRC) {
5438 not_(crc, noreg, false); // 1s complement of crc
5439 }
5440 }
5441
5442 void MacroAssembler::kernel_crc32_singleByte(Register crc, Register buf, Register len, Register table, Register tmp,
5443 bool invertCRC) {
5444 assert_different_registers(crc, buf, len, table, tmp);
5445
5446 if (invertCRC) {
5447 not_(crc, noreg, false); // 1s complement of crc
5448 }
5449
5450 z_llgc(tmp, Address(buf, (intptr_t)0)); // Current byte of input buffer (zero extended). Avoids garbage in upper half of register.
5451 update_byte_crc32(crc, tmp, table);
5452
5453 if (invertCRC) {
5454 not_(crc, noreg, false); // 1s complement of crc
5455 }
5456 }
5457
5458 void MacroAssembler::kernel_crc32_singleByteReg(Register crc, Register val, Register table,
5459 bool invertCRC) {
5460 assert_different_registers(crc, val, table);
5461
5462 if (invertCRC) {
5463 not_(crc, noreg, false); // 1s complement of crc
5464 }
5465
5466 update_byte_crc32(crc, val, table);
5467
5468 if (invertCRC) {
5469 not_(crc, noreg, false); // 1s complement of crc
5470 }
5471 }
5472
5473 //
5474 // Code for BigInteger::multiplyToLen() intrinsic.
5475 //
5476
5477 // dest_lo += src1 + src2
5478 // dest_hi += carry1 + carry2
5479 // Z_R7 is destroyed !
5480 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo,
5481 Register src1, Register src2) {
5482 clear_reg(Z_R7);
5483 z_algr(dest_lo, src1);
5484 z_alcgr(dest_hi, Z_R7);
5485 z_algr(dest_lo, src2);
5486 z_alcgr(dest_hi, Z_R7);
5487 }
5488
5489 // Multiply 64 bit by 64 bit first loop.
5490 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart,
5491 Register x_xstart,
5492 Register y, Register y_idx,
5493 Register z,
5494 Register carry,
5495 Register product,
5496 Register idx, Register kdx) {
5497 // jlong carry, x[], y[], z[];
5498 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
5499 // huge_128 product = y[idx] * x[xstart] + carry;
5500 // z[kdx] = (jlong)product;
5501 // carry = (jlong)(product >>> 64);
5502 // }
5503 // z[xstart] = carry;
5504
5505 Label L_first_loop, L_first_loop_exit;
5506 Label L_one_x, L_one_y, L_multiply;
5507
5508 z_aghi(xstart, -1);
5509 z_brl(L_one_x); // Special case: length of x is 1.
5510
5511 // Load next two integers of x.
5512 z_sllg(Z_R1_scratch, xstart, LogBytesPerInt);
5513 mem2reg_opt(x_xstart, Address(x, Z_R1_scratch, 0));
5514
5515
5516 bind(L_first_loop);
5517
5518 z_aghi(idx, -1);
5519 z_brl(L_first_loop_exit);
5520 z_aghi(idx, -1);
5521 z_brl(L_one_y);
5522
5523 // Load next two integers of y.
5524 z_sllg(Z_R1_scratch, idx, LogBytesPerInt);
5525 mem2reg_opt(y_idx, Address(y, Z_R1_scratch, 0));
5526
5527
5528 bind(L_multiply);
5529
5530 Register multiplicand = product->successor();
5531 Register product_low = multiplicand;
5532
5533 lgr_if_needed(multiplicand, x_xstart);
5534 z_mlgr(product, y_idx); // multiplicand * y_idx -> product::multiplicand
5535 clear_reg(Z_R7);
5536 z_algr(product_low, carry); // Add carry to result.
5537 z_alcgr(product, Z_R7); // Add carry of the last addition.
5538 add2reg(kdx, -2);
5539
5540 // Store result.
5541 z_sllg(Z_R7, kdx, LogBytesPerInt);
5542 reg2mem_opt(product_low, Address(z, Z_R7, 0));
5543 lgr_if_needed(carry, product);
5544 z_bru(L_first_loop);
5545
5546
5547 bind(L_one_y); // Load one 32 bit portion of y as (0,value).
5548
5549 clear_reg(y_idx);
5550 mem2reg_opt(y_idx, Address(y, (intptr_t) 0), false);
5551 z_bru(L_multiply);
5552
5553
5554 bind(L_one_x); // Load one 32 bit portion of x as (0,value).
5555
5556 clear_reg(x_xstart);
5557 mem2reg_opt(x_xstart, Address(x, (intptr_t) 0), false);
5558 z_bru(L_first_loop);
5559
5560 bind(L_first_loop_exit);
5561 }
5562
5563 // Multiply 64 bit by 64 bit and add 128 bit.
5564 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y,
5565 Register z,
5566 Register yz_idx, Register idx,
5567 Register carry, Register product,
5568 int offset) {
5569 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
5570 // z[kdx] = (jlong)product;
5571
5572 Register multiplicand = product->successor();
5573 Register product_low = multiplicand;
5574
5575 z_sllg(Z_R7, idx, LogBytesPerInt);
5576 mem2reg_opt(yz_idx, Address(y, Z_R7, offset));
5577
5578 lgr_if_needed(multiplicand, x_xstart);
5579 z_mlgr(product, yz_idx); // multiplicand * yz_idx -> product::multiplicand
5580 mem2reg_opt(yz_idx, Address(z, Z_R7, offset));
5581
5582 add2_with_carry(product, product_low, carry, yz_idx);
5583
5584 z_sllg(Z_R7, idx, LogBytesPerInt);
5585 reg2mem_opt(product_low, Address(z, Z_R7, offset));
5586
5587 }
5588
5589 // Multiply 128 bit by 128 bit. Unrolled inner loop.
5590 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart,
5591 Register y, Register z,
5592 Register yz_idx, Register idx,
5593 Register jdx,
5594 Register carry, Register product,
5595 Register carry2) {
5596 // jlong carry, x[], y[], z[];
5597 // int kdx = ystart+1;
5598 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
5599 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
5600 // z[kdx+idx+1] = (jlong)product;
5601 // jlong carry2 = (jlong)(product >>> 64);
5602 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
5603 // z[kdx+idx] = (jlong)product;
5604 // carry = (jlong)(product >>> 64);
5605 // }
5606 // idx += 2;
5607 // if (idx > 0) {
5608 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
5609 // z[kdx+idx] = (jlong)product;
5610 // carry = (jlong)(product >>> 64);
5611 // }
5612
5613 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
5614
5615 // scale the index
5616 lgr_if_needed(jdx, idx);
5617 and_imm(jdx, 0xfffffffffffffffcL);
5618 rshift(jdx, 2);
5619
5620
5621 bind(L_third_loop);
5622
5623 z_aghi(jdx, -1);
5624 z_brl(L_third_loop_exit);
5625 add2reg(idx, -4);
5626
5627 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
5628 lgr_if_needed(carry2, product);
5629
5630 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
5631 lgr_if_needed(carry, product);
5632 z_bru(L_third_loop);
5633
5634
5635 bind(L_third_loop_exit); // Handle any left-over operand parts.
5636
5637 and_imm(idx, 0x3);
5638 z_brz(L_post_third_loop_done);
5639
5640 Label L_check_1;
5641
5642 z_aghi(idx, -2);
5643 z_brl(L_check_1);
5644
5645 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
5646 lgr_if_needed(carry, product);
5647
5648
5649 bind(L_check_1);
5650
5651 add2reg(idx, 0x2);
5652 and_imm(idx, 0x1);
5653 z_aghi(idx, -1);
5654 z_brl(L_post_third_loop_done);
5655
5656 Register multiplicand = product->successor();
5657 Register product_low = multiplicand;
5658
5659 z_sllg(Z_R7, idx, LogBytesPerInt);
5660 clear_reg(yz_idx);
5661 mem2reg_opt(yz_idx, Address(y, Z_R7, 0), false);
5662 lgr_if_needed(multiplicand, x_xstart);
5663 z_mlgr(product, yz_idx); // multiplicand * yz_idx -> product::multiplicand
5664 clear_reg(yz_idx);
5665 mem2reg_opt(yz_idx, Address(z, Z_R7, 0), false);
5666
5667 add2_with_carry(product, product_low, yz_idx, carry);
5668
5669 z_sllg(Z_R7, idx, LogBytesPerInt);
5670 reg2mem_opt(product_low, Address(z, Z_R7, 0), false);
5671 rshift(product_low, 32);
5672
5673 lshift(product, 32);
5674 z_ogr(product_low, product);
5675 lgr_if_needed(carry, product_low);
5676
5677 bind(L_post_third_loop_done);
5678 }
5679
5680 void MacroAssembler::multiply_to_len(Register x, Register xlen,
5681 Register y, Register ylen,
5682 Register z,
5683 Register tmp1, Register tmp2,
5684 Register tmp3, Register tmp4,
5685 Register tmp5) {
5686 ShortBranchVerifier sbv(this);
5687
5688 assert_different_registers(x, xlen, y, ylen, z,
5689 tmp1, tmp2, tmp3, tmp4, tmp5, Z_R1_scratch, Z_R7);
5690 assert_different_registers(x, xlen, y, ylen, z,
5691 tmp1, tmp2, tmp3, tmp4, tmp5, Z_R8);
5692
5693 z_stmg(Z_R7, Z_R13, _z_abi(gpr7), Z_SP);
5694
5695 const Register idx = tmp1;
5696 const Register kdx = tmp2;
5697 const Register xstart = tmp3;
5698
5699 const Register y_idx = tmp4;
5700 const Register carry = tmp5;
5701 const Register product = Z_R0_scratch;
5702 const Register x_xstart = Z_R8;
5703
5704 // First Loop.
5705 //
5706 // final static long LONG_MASK = 0xffffffffL;
5707 // int xstart = xlen - 1;
5708 // int ystart = ylen - 1;
5709 // long carry = 0;
5710 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
5711 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
5712 // z[kdx] = (int)product;
5713 // carry = product >>> 32;
5714 // }
5715 // z[xstart] = (int)carry;
5716 //
5717
5718 lgr_if_needed(idx, ylen); // idx = ylen
5719 z_agrk(kdx, xlen, ylen); // kdx = xlen + ylen
5720 clear_reg(carry); // carry = 0
5721
5722 Label L_done;
5723
5724 lgr_if_needed(xstart, xlen);
5725 z_aghi(xstart, -1);
5726 z_brl(L_done);
5727
5728 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
5729
5730 NearLabel L_second_loop;
5731 compare64_and_branch(kdx, RegisterOrConstant((intptr_t) 0), bcondEqual, L_second_loop);
5732
5733 NearLabel L_carry;
5734 z_aghi(kdx, -1);
5735 z_brz(L_carry);
5736
5737 // Store lower 32 bits of carry.
5738 z_sllg(Z_R1_scratch, kdx, LogBytesPerInt);
5739 reg2mem_opt(carry, Address(z, Z_R1_scratch, 0), false);
5740 rshift(carry, 32);
5741 z_aghi(kdx, -1);
5742
5743
5744 bind(L_carry);
5745
5746 // Store upper 32 bits of carry.
5747 z_sllg(Z_R1_scratch, kdx, LogBytesPerInt);
5748 reg2mem_opt(carry, Address(z, Z_R1_scratch, 0), false);
5749
5750 // Second and third (nested) loops.
5751 //
5752 // for (int i = xstart-1; i >= 0; i--) { // Second loop
5753 // carry = 0;
5754 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
5755 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
5756 // (z[k] & LONG_MASK) + carry;
5757 // z[k] = (int)product;
5758 // carry = product >>> 32;
5759 // }
5760 // z[i] = (int)carry;
5761 // }
5762 //
5763 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
5764
5765 const Register jdx = tmp1;
5766
5767 bind(L_second_loop);
5768
5769 clear_reg(carry); // carry = 0;
5770 lgr_if_needed(jdx, ylen); // j = ystart+1
5771
5772 z_aghi(xstart, -1); // i = xstart-1;
5773 z_brl(L_done);
5774
5775 // Use free slots in the current stackframe instead of push/pop.
5776 Address zsave(Z_SP, _z_abi(carg_1));
5777 reg2mem_opt(z, zsave);
5778
5779
5780 Label L_last_x;
5781
5782 z_sllg(Z_R1_scratch, xstart, LogBytesPerInt);
5783 load_address(z, Address(z, Z_R1_scratch, 4)); // z = z + k - j
5784 z_aghi(xstart, -1); // i = xstart-1;
5785 z_brl(L_last_x);
5786
5787 z_sllg(Z_R1_scratch, xstart, LogBytesPerInt);
5788 mem2reg_opt(x_xstart, Address(x, Z_R1_scratch, 0));
5789
5790
5791 Label L_third_loop_prologue;
5792
5793 bind(L_third_loop_prologue);
5794
5795 Address xsave(Z_SP, _z_abi(carg_2));
5796 Address xlensave(Z_SP, _z_abi(carg_3));
5797 Address ylensave(Z_SP, _z_abi(carg_4));
5798
5799 reg2mem_opt(x, xsave);
5800 reg2mem_opt(xstart, xlensave);
5801 reg2mem_opt(ylen, ylensave);
5802
5803
5804 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
5805
5806 mem2reg_opt(z, zsave);
5807 mem2reg_opt(x, xsave);
5808 mem2reg_opt(xlen, xlensave); // This is the decrement of the loop counter!
5809 mem2reg_opt(ylen, ylensave);
5810
5811 add2reg(tmp3, 1, xlen);
5812 z_sllg(Z_R1_scratch, tmp3, LogBytesPerInt);
5813 reg2mem_opt(carry, Address(z, Z_R1_scratch, 0), false);
5814 z_aghi(tmp3, -1);
5815 z_brl(L_done);
5816
5817 rshift(carry, 32);
5818 z_sllg(Z_R1_scratch, tmp3, LogBytesPerInt);
5819 reg2mem_opt(carry, Address(z, Z_R1_scratch, 0), false);
5820 z_bru(L_second_loop);
5821
5822 // Next infrequent code is moved outside loops.
5823 bind(L_last_x);
5824
5825 clear_reg(x_xstart);
5826 mem2reg_opt(x_xstart, Address(x, (intptr_t) 0), false);
5827 z_bru(L_third_loop_prologue);
5828
5829 bind(L_done);
5830
5831 z_lmg(Z_R7, Z_R13, _z_abi(gpr7), Z_SP);
5832 }
5833
5834 void MacroAssembler::asm_assert(branch_condition cond, const char* msg, int id, bool is_static) {
5835 #ifdef ASSERT
5836 Label ok;
5837 z_brc(cond, ok);
5838 is_static ? stop_static(msg, id) : stop(msg, id);
5839 bind(ok);
5840 #endif // ASSERT
5841 }
5842
5843 // Assert if CC indicates "not equal" (check_equal==true) or "equal" (check_equal==false).
5844 void MacroAssembler::asm_assert(bool check_equal, const char *msg, int id) {
5845 #ifdef ASSERT
5846 asm_assert(check_equal ? bcondEqual : bcondNotEqual, msg, id);
5847 #endif // ASSERT
5848 }
5849
5850 void MacroAssembler::asm_assert_mems_zero(bool check_equal, bool allow_relocation, int size, int64_t mem_offset,
5851 Register mem_base, const char* msg, int id) {
5852 #ifdef ASSERT
5853 switch (size) {
5854 case 4:
5855 load_and_test_int(Z_R0, Address(mem_base, mem_offset));
5856 break;
5857 case 8:
5858 load_and_test_long(Z_R0, Address(mem_base, mem_offset));
5859 break;
5860 default:
5861 ShouldNotReachHere();
5862 }
5863 // if relocation is not allowed then stop_static() will be called otherwise call stop()
5864 asm_assert(check_equal ? bcondEqual : bcondNotEqual, msg, id, !allow_relocation);
5865 #endif // ASSERT
5866 }
5867
5868 // Check the condition
5869 // expected_size == FP - SP
5870 // after transformation:
5871 // expected_size - FP + SP == 0
5872 // Destroys Register expected_size if no tmp register is passed.
5873 void MacroAssembler::asm_assert_frame_size(Register expected_size, Register tmp, const char* msg, int id) {
5874 #ifdef ASSERT
5875 lgr_if_needed(tmp, expected_size);
5876 z_algr(tmp, Z_SP);
5877 z_slg(tmp, 0, Z_R0, Z_SP);
5878 asm_assert(bcondEqual, msg, id);
5879 #endif // ASSERT
5880 }
5881
5882 #ifdef ASSERT
5883 bool is_excluded(Register excluded_register[], Register reg, int n) {
5884 for (int i = 0; i < n; i++) {
5885 if (excluded_register[i] == reg) {
5886 return true;
5887 }
5888 }
5889 return false;
5890 }
5891
5892 void MacroAssembler::clobber_volatile_registers(Register excluded_register[], int n) {
5893 const int magic_number = 0x82;
5894
5895 for (int i = 0; i < 6 /* R0 to R5 */; i++) {
5896 Register reg = as_Register(i);
5897 if (!is_excluded(excluded_register, reg, n)) {
5898 load_const_optimized(reg, magic_number);
5899 }
5900 }
5901 }
5902 #endif // ASSERT
5903
5904 // Save and restore functions: Exclude Z_R0.
5905 void MacroAssembler::save_volatile_regs(Register dst, int offset, bool include_fp, bool include_flags) {
5906 z_stmg(Z_R1, Z_R5, offset, dst); offset += 5 * BytesPerWord;
5907 if (include_fp) {
5908 z_std(Z_F0, Address(dst, offset)); offset += BytesPerWord;
5909 z_std(Z_F1, Address(dst, offset)); offset += BytesPerWord;
5910 z_std(Z_F2, Address(dst, offset)); offset += BytesPerWord;
5911 z_std(Z_F3, Address(dst, offset)); offset += BytesPerWord;
5912 z_std(Z_F4, Address(dst, offset)); offset += BytesPerWord;
5913 z_std(Z_F5, Address(dst, offset)); offset += BytesPerWord;
5914 z_std(Z_F6, Address(dst, offset)); offset += BytesPerWord;
5915 z_std(Z_F7, Address(dst, offset)); offset += BytesPerWord;
5916 }
5917 if (include_flags) {
5918 Label done;
5919 z_mvi(Address(dst, offset), 2); // encoding: equal
5920 z_bre(done);
5921 z_mvi(Address(dst, offset), 4); // encoding: higher
5922 z_brh(done);
5923 z_mvi(Address(dst, offset), 1); // encoding: lower
5924 bind(done);
5925 }
5926 }
5927 void MacroAssembler::restore_volatile_regs(Register src, int offset, bool include_fp, bool include_flags) {
5928 z_lmg(Z_R1, Z_R5, offset, src); offset += 5 * BytesPerWord;
5929 if (include_fp) {
5930 z_ld(Z_F0, Address(src, offset)); offset += BytesPerWord;
5931 z_ld(Z_F1, Address(src, offset)); offset += BytesPerWord;
5932 z_ld(Z_F2, Address(src, offset)); offset += BytesPerWord;
5933 z_ld(Z_F3, Address(src, offset)); offset += BytesPerWord;
5934 z_ld(Z_F4, Address(src, offset)); offset += BytesPerWord;
5935 z_ld(Z_F5, Address(src, offset)); offset += BytesPerWord;
5936 z_ld(Z_F6, Address(src, offset)); offset += BytesPerWord;
5937 z_ld(Z_F7, Address(src, offset)); offset += BytesPerWord;
5938 }
5939 if (include_flags) {
5940 z_cli(Address(src, offset), 2); // see encoding above
5941 }
5942 }
5943
5944 // Plausibility check for oops.
5945 void MacroAssembler::verify_oop(Register oop, const char* msg) {
5946 if (!VerifyOops) return;
5947
5948 BLOCK_COMMENT("verify_oop {");
5949 unsigned int nbytes_save = (5 + 8 + 1) * BytesPerWord;
5950 address entry_addr = StubRoutines::verify_oop_subroutine_entry_address();
5951
5952 save_return_pc();
5953
5954 // Push frame, but preserve flags
5955 z_lgr(Z_R0, Z_SP);
5956 z_lay(Z_SP, -((int64_t)nbytes_save + frame::z_abi_160_size), Z_SP);
5957 z_stg(Z_R0, _z_abi(callers_sp), Z_SP);
5958
5959 save_volatile_regs(Z_SP, frame::z_abi_160_size, true, true);
5960
5961 lgr_if_needed(Z_ARG2, oop);
5962 load_const_optimized(Z_ARG1, (address)msg);
5963 load_const_optimized(Z_R1, entry_addr);
5964 z_lg(Z_R1, 0, Z_R1);
5965 call_c(Z_R1);
5966
5967 restore_volatile_regs(Z_SP, frame::z_abi_160_size, true, true);
5968 pop_frame();
5969 restore_return_pc();
5970
5971 BLOCK_COMMENT("} verify_oop ");
5972 }
5973
5974 void MacroAssembler::verify_oop_addr(Address addr, const char* msg) {
5975 if (!VerifyOops) return;
5976
5977 BLOCK_COMMENT("verify_oop {");
5978 unsigned int nbytes_save = (5 + 8) * BytesPerWord;
5979 address entry_addr = StubRoutines::verify_oop_subroutine_entry_address();
5980
5981 save_return_pc();
5982 unsigned int frame_size = push_frame_abi160(nbytes_save); // kills Z_R0
5983 save_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
5984
5985 z_lg(Z_ARG2, addr.plus_disp(frame_size));
5986 load_const_optimized(Z_ARG1, (address)msg);
5987 load_const_optimized(Z_R1, entry_addr);
5988 z_lg(Z_R1, 0, Z_R1);
5989 call_c(Z_R1);
5990
5991 restore_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
5992 pop_frame();
5993 restore_return_pc();
5994
5995 BLOCK_COMMENT("} verify_oop ");
5996 }
5997
5998 const char* MacroAssembler::stop_types[] = {
5999 "stop",
6000 "untested",
6001 "unimplemented",
6002 "shouldnotreachhere"
6003 };
6004
6005 static void stop_on_request(const char* tp, const char* msg) {
6006 tty->print("Z assembly code requires stop: (%s) %s\n", tp, msg);
6007 guarantee(false, "Z assembly code requires stop: %s", msg);
6008 }
6009
6010 void MacroAssembler::stop(int type, const char* msg, int id) {
6011 BLOCK_COMMENT(err_msg("stop: %s {", msg));
6012
6013 // Setup arguments.
6014 load_const(Z_ARG1, (void*) stop_types[type%stop_end]);
6015 load_const(Z_ARG2, (void*) msg);
6016 get_PC(Z_R14); // Following code pushes a frame without entering a new function. Use current pc as return address.
6017 save_return_pc(); // Saves return pc Z_R14.
6018 push_frame_abi160(0);
6019 call_VM_leaf(CAST_FROM_FN_PTR(address, stop_on_request), Z_ARG1, Z_ARG2);
6020 // The plain disassembler does not recognize illtrap. It instead displays
6021 // a 32-bit value. Issuing two illtraps assures the disassembler finds
6022 // the proper beginning of the next instruction.
6023 z_illtrap(id); // Illegal instruction.
6024 z_illtrap(id); // Illegal instruction.
6025
6026 BLOCK_COMMENT(" } stop");
6027 }
6028
6029 // Special version of stop() for code size reduction.
6030 // Reuses the previously generated call sequence, if any.
6031 // Generates the call sequence on its own, if necessary.
6032 // Note: This code will work only in non-relocatable code!
6033 // The relative address of the data elements (arg1, arg2) must not change.
6034 // The reentry point must not move relative to it's users. This prerequisite
6035 // should be given for "hand-written" code, if all chain calls are in the same code blob.
6036 // Generated code must not undergo any transformation, e.g. ShortenBranches, to be safe.
6037 address MacroAssembler::stop_chain(address reentry, int type, const char* msg, int id, bool allow_relocation) {
6038 BLOCK_COMMENT(err_msg("stop_chain(%s,%s): %s {", reentry==nullptr?"init":"cont", allow_relocation?"reloc ":"static", msg));
6039
6040 // Setup arguments.
6041 if (allow_relocation) {
6042 // Relocatable version (for comparison purposes). Remove after some time.
6043 load_const(Z_ARG1, (void*) stop_types[type%stop_end]);
6044 load_const(Z_ARG2, (void*) msg);
6045 } else {
6046 load_absolute_address(Z_ARG1, (address)stop_types[type%stop_end]);
6047 load_absolute_address(Z_ARG2, (address)msg);
6048 }
6049 if ((reentry != nullptr) && RelAddr::is_in_range_of_RelAddr16(reentry, pc())) {
6050 BLOCK_COMMENT("branch to reentry point:");
6051 z_brc(bcondAlways, reentry);
6052 } else {
6053 BLOCK_COMMENT("reentry point:");
6054 reentry = pc(); // Re-entry point for subsequent stop calls.
6055 save_return_pc(); // Saves return pc Z_R14.
6056 push_frame_abi160(0);
6057 if (allow_relocation) {
6058 reentry = nullptr; // Prevent reentry if code relocation is allowed.
6059 call_VM_leaf(CAST_FROM_FN_PTR(address, stop_on_request), Z_ARG1, Z_ARG2);
6060 } else {
6061 call_VM_leaf_static(CAST_FROM_FN_PTR(address, stop_on_request), Z_ARG1, Z_ARG2);
6062 }
6063 z_illtrap(id); // Illegal instruction as emergency stop, should the above call return.
6064 }
6065 BLOCK_COMMENT(" } stop_chain");
6066
6067 return reentry;
6068 }
6069
6070 // Special version of stop() for code size reduction.
6071 // Assumes constant relative addresses for data and runtime call.
6072 void MacroAssembler::stop_static(int type, const char* msg, int id) {
6073 stop_chain(nullptr, type, msg, id, false);
6074 }
6075
6076 void MacroAssembler::stop_subroutine() {
6077 unimplemented("stop_subroutine", 710);
6078 }
6079
6080 // Prints msg to stdout from within generated code..
6081 void MacroAssembler::warn(const char* msg) {
6082 RegisterSaver::save_live_registers(this, RegisterSaver::all_registers, Z_R14);
6083 load_absolute_address(Z_R1, (address) warning);
6084 load_absolute_address(Z_ARG1, (address) msg);
6085 (void) call(Z_R1);
6086 RegisterSaver::restore_live_registers(this, RegisterSaver::all_registers);
6087 }
6088
6089 #ifndef PRODUCT
6090
6091 // Write pattern 0x0101010101010101 in region [low-before, high+after].
6092 void MacroAssembler::zap_from_to(Register low, Register high, Register val, Register addr, int before, int after) {
6093 if (!ZapEmptyStackFields) return;
6094 BLOCK_COMMENT("zap memory region {");
6095 load_const_optimized(val, 0x0101010101010101);
6096 int size = before + after;
6097 if (low == high && size < 5 && size > 0) {
6098 int offset = -before*BytesPerWord;
6099 for (int i = 0; i < size; ++i) {
6100 z_stg(val, Address(low, offset));
6101 offset +=(1*BytesPerWord);
6102 }
6103 } else {
6104 add2reg(addr, -before*BytesPerWord, low);
6105 if (after) {
6106 #ifdef ASSERT
6107 jlong check = after * BytesPerWord;
6108 assert(Immediate::is_simm32(check) && Immediate::is_simm32(-check), "value not encodable !");
6109 #endif
6110 add2reg(high, after * BytesPerWord);
6111 }
6112 NearLabel loop;
6113 bind(loop);
6114 z_stg(val, Address(addr));
6115 add2reg(addr, 8);
6116 compare64_and_branch(addr, high, bcondNotHigh, loop);
6117 if (after) {
6118 add2reg(high, -after * BytesPerWord);
6119 }
6120 }
6121 BLOCK_COMMENT("} zap memory region");
6122 }
6123 #endif // !PRODUCT
6124
6125 // Implements fast-locking.
6126 // - obj: the object to be locked, contents preserved.
6127 // - temp1, temp2: temporary registers, contents destroyed.
6128 // Note: make sure Z_R1 is not manipulated here when C2 compiler is in play
6129 void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register temp1, Register temp2, Label& slow) {
6130
6131 assert_different_registers(basic_lock, obj, temp1, temp2);
6132
6133 Label push;
6134 const Register top = temp1;
6135 const Register mark = temp2;
6136 const int mark_offset = oopDesc::mark_offset_in_bytes();
6137 const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
6138
6139 // Preload the markWord. It is important that this is the first
6140 // instruction emitted as it is part of C1's null check semantics.
6141 z_lg(mark, Address(obj, mark_offset));
6142
6143 if (UseObjectMonitorTable) {
6144 // Clear cache in case fast locking succeeds or we need to take the slow-path.
6145 const Address om_cache_addr = Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes())));
6146 z_mvghi(om_cache_addr, 0);
6147 }
6148
6149 if (DiagnoseSyncOnValueBasedClasses != 0) {
6150 load_klass(temp1, obj);
6151 z_tm(Address(temp1, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
6152 z_brne(slow);
6153 }
6154
6155 // First we need to check if the lock-stack has room for pushing the object reference.
6156 z_lgf(top, Address(Z_thread, ls_top_offset));
6157
6158 compareU32_and_branch(top, (unsigned)LockStack::end_offset(), bcondNotLow, slow);
6159
6160 // The underflow check is elided. The recursive check will always fail
6161 // when the lock stack is empty because of the _bad_oop_sentinel field.
6162
6163 // Check for recursion:
6164 z_aghi(top, -oopSize);
6165 z_cg(obj, Address(Z_thread, top));
6166 z_bre(push);
6167
6168 // Check header for monitor (0b10).
6169 z_tmll(mark, markWord::monitor_value);
6170 branch_optimized(bcondNotAllZero, slow);
6171
6172 { // Try to lock. Transition lock bits 0b01 => 0b00
6173 const Register locked_obj = top;
6174 z_oill(mark, markWord::unlocked_value);
6175 z_lgr(locked_obj, mark);
6176 // Clear lock-bits from locked_obj (locked state)
6177 z_xilf(locked_obj, markWord::unlocked_value);
6178 z_csg(mark, locked_obj, mark_offset, obj);
6179 branch_optimized(Assembler::bcondNotEqual, slow);
6180 }
6181
6182 bind(push);
6183
6184 // After successful lock, push object on lock-stack
6185 z_lgf(top, Address(Z_thread, ls_top_offset));
6186 z_stg(obj, Address(Z_thread, top));
6187 z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
6188 }
6189
6190 // Implements fast-unlocking.
6191 // - obj: the object to be unlocked
6192 // - temp1, temp2: temporary registers, will be destroyed
6193 // - Z_R1_scratch: will be killed in case of Interpreter & C1 Compiler
6194 void MacroAssembler::fast_unlock(Register obj, Register temp1, Register temp2, Label& slow) {
6195
6196 assert_different_registers(obj, temp1, temp2);
6197
6198 Label unlocked, push_and_slow;
6199 const Register mark = temp1;
6200 const Register top = temp2;
6201 const int mark_offset = oopDesc::mark_offset_in_bytes();
6202 const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
6203
6204 #ifdef ASSERT
6205 {
6206 // The following checks rely on the fact that LockStack is only ever modified by
6207 // its owning thread, even if the lock got inflated concurrently; removal of LockStack
6208 // entries after inflation will happen delayed in that case.
6209
6210 // Check for lock-stack underflow.
6211 NearLabel stack_ok;
6212 z_lgf(top, Address(Z_thread, ls_top_offset));
6213 compareU32_and_branch(top, (unsigned)LockStack::start_offset(), bcondNotLow, stack_ok);
6214 stop("Lock-stack underflow");
6215 bind(stack_ok);
6216 }
6217 #endif // ASSERT
6218
6219 // Check if obj is top of lock-stack.
6220 z_lgf(top, Address(Z_thread, ls_top_offset));
6221 z_aghi(top, -oopSize);
6222 z_cg(obj, Address(Z_thread, top));
6223 branch_optimized(bcondNotEqual, slow);
6224
6225 // pop object from lock-stack
6226 #ifdef ASSERT
6227 const Register temp_top = temp1; // mark is not yet loaded, but be careful
6228 z_agrk(temp_top, top, Z_thread);
6229 z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry
6230 #endif // ASSERT
6231 z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object
6232
6233 // The underflow check is elided. The recursive check will always fail
6234 // when the lock stack is empty because of the _bad_oop_sentinel field.
6235
6236 // Check if recursive. (this is a check for the 2nd object on the stack)
6237 z_aghi(top, -oopSize);
6238 z_cg(obj, Address(Z_thread, top));
6239 branch_optimized(bcondEqual, unlocked);
6240
6241 // Not recursive. Check header for monitor (0b10).
6242 z_lg(mark, Address(obj, mark_offset));
6243 z_tmll(mark, markWord::monitor_value);
6244 z_brnaz(push_and_slow);
6245
6246 #ifdef ASSERT
6247 // Check header not unlocked (0b01).
6248 NearLabel not_unlocked;
6249 z_tmll(mark, markWord::unlocked_value);
6250 z_braz(not_unlocked);
6251 stop("fast_unlock already unlocked");
6252 bind(not_unlocked);
6253 #endif // ASSERT
6254
6255 { // Try to unlock. Transition lock bits 0b00 => 0b01
6256 Register unlocked_obj = top;
6257 z_lgr(unlocked_obj, mark);
6258 z_oill(unlocked_obj, markWord::unlocked_value);
6259 z_csg(mark, unlocked_obj, mark_offset, obj);
6260 branch_optimized(Assembler::bcondEqual, unlocked);
6261 }
6262
6263 bind(push_and_slow);
6264
6265 // Restore lock-stack and handle the unlock in runtime.
6266 z_lgf(top, Address(Z_thread, ls_top_offset));
6267 DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));)
6268 z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
6269 // set CC to NE
6270 z_ltgr(obj, obj); // object shouldn't be null at this point
6271 branch_optimized(bcondAlways, slow);
6272
6273 bind(unlocked);
6274 }
6275
6276 void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Register tmp1, Register tmp2) {
6277 assert_different_registers(obj, box, tmp1, tmp2, Z_R0_scratch);
6278
6279 // Handle inflated monitor.
6280 NearLabel inflated;
6281 // Finish fast lock successfully. MUST reach to with flag == NE
6282 NearLabel locked;
6283 // Finish fast lock unsuccessfully. MUST branch to with flag == EQ
6284 NearLabel slow_path;
6285
6286 if (UseObjectMonitorTable) {
6287 // Clear cache in case fast locking succeeds or we need to take the slow-path.
6288 z_mvghi(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), 0);
6289 }
6290
6291 if (DiagnoseSyncOnValueBasedClasses != 0) {
6292 load_klass(tmp1, obj);
6293 z_tm(Address(tmp1, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
6294 z_brne(slow_path);
6295 }
6296
6297 const Register mark = tmp1;
6298 const int mark_offset = oopDesc::mark_offset_in_bytes();
6299 const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
6300
6301 BLOCK_COMMENT("compiler_fast_locking {");
6302 { // Fast locking
6303
6304 // Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ
6305 NearLabel push;
6306
6307 const Register top = tmp2;
6308
6309 // Check if lock-stack is full.
6310 z_lgf(top, Address(Z_thread, ls_top_offset));
6311 compareU32_and_branch(top, (unsigned) LockStack::end_offset() - 1, bcondHigh, slow_path);
6312
6313 // The underflow check is elided. The recursive check will always fail
6314 // when the lock stack is empty because of the _bad_oop_sentinel field.
6315
6316 // Check if recursive.
6317 z_aghi(top, -oopSize);
6318 z_cg(obj, Address(Z_thread, top));
6319 z_bre(push);
6320
6321 // Check for monitor (0b10)
6322 z_lg(mark, Address(obj, mark_offset));
6323 z_tmll(mark, markWord::monitor_value);
6324 z_brnaz(inflated);
6325
6326 // not inflated
6327
6328 { // Try to lock. Transition lock bits 0b01 => 0b00
6329 assert(mark_offset == 0, "required to avoid a lea");
6330 const Register locked_obj = top;
6331 z_oill(mark, markWord::unlocked_value);
6332 z_lgr(locked_obj, mark);
6333 // Clear lock-bits from locked_obj (locked state)
6334 z_xilf(locked_obj, markWord::unlocked_value);
6335 z_csg(mark, locked_obj, mark_offset, obj);
6336 branch_optimized(Assembler::bcondNotEqual, slow_path);
6337 }
6338
6339 bind(push);
6340
6341 // After successful lock, push object on lock-stack.
6342 z_lgf(top, Address(Z_thread, ls_top_offset));
6343 z_stg(obj, Address(Z_thread, top));
6344 z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
6345
6346 z_cgr(obj, obj); // set the CC to EQ, as it could be changed by alsi
6347 z_bru(locked);
6348 }
6349 BLOCK_COMMENT("} compiler_fast_locking");
6350
6351 BLOCK_COMMENT("handle_inflated_monitor_locking {");
6352 { // Handle inflated monitor.
6353 bind(inflated);
6354
6355 const Register tmp1_monitor = tmp1;
6356 if (!UseObjectMonitorTable) {
6357 assert(tmp1_monitor == mark, "should be the same here");
6358 } else {
6359 const Register tmp1_bucket = tmp1;
6360 const Register hash = Z_R0_scratch;
6361 NearLabel monitor_found;
6362
6363 // Save the mark, we might need it to extract the hash.
6364 z_lgr(hash, mark);
6365
6366 // Look for the monitor in the om_cache.
6367
6368 ByteSize cache_offset = JavaThread::om_cache_oops_offset();
6369 ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
6370 const int num_unrolled = OMCache::CAPACITY;
6371 for (int i = 0; i < num_unrolled; i++) {
6372 z_lg(tmp1_monitor, Address(Z_thread, cache_offset + monitor_offset));
6373 z_cg(obj, Address(Z_thread, cache_offset));
6374 z_bre(monitor_found);
6375 cache_offset = cache_offset + OMCache::oop_to_oop_difference();
6376 }
6377
6378 // Get the hash code.
6379 z_srlg(hash, hash, markWord::hash_shift);
6380
6381 // Get the table and calculate the bucket's address.
6382 load_const_optimized(tmp2, ObjectMonitorTable::current_table_address());
6383 z_lg(tmp2, Address(tmp2));
6384 z_ng(hash, Address(tmp2, ObjectMonitorTable::table_capacity_mask_offset()));
6385 z_lg(tmp1_bucket, Address(tmp2, ObjectMonitorTable::table_buckets_offset()));
6386 z_sllg(hash, hash, LogBytesPerWord);
6387 z_agr(tmp1_bucket, hash);
6388
6389 // Read the monitor from the bucket.
6390 z_lg(tmp1_monitor, Address(tmp1_bucket));
6391
6392 // Check if the monitor in the bucket is special (empty, tombstone or removed).
6393 z_clgfi(tmp1_monitor, ObjectMonitorTable::SpecialPointerValues::below_is_special);
6394 z_brl(slow_path);
6395
6396 // Check if object matches.
6397 z_lg(tmp2, Address(tmp1_monitor, ObjectMonitor::object_offset()));
6398 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
6399 bs_asm->try_peek_weak_handle_in_nmethod(this, tmp2, tmp2, Z_R0_scratch, slow_path);
6400 z_cgr(obj, tmp2);
6401 z_brne(slow_path);
6402
6403 bind(monitor_found);
6404 }
6405 NearLabel monitor_locked;
6406 // lock the monitor
6407
6408 const Register zero = tmp2;
6409
6410 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
6411 const Address owner_address(tmp1_monitor, ObjectMonitor::owner_offset() - monitor_tag);
6412 const Address recursions_address(tmp1_monitor, ObjectMonitor::recursions_offset() - monitor_tag);
6413
6414
6415 // Try to CAS owner (no owner => current thread's _monitor_owner_id).
6416 // If csg succeeds then CR=EQ, otherwise, register zero is filled
6417 // with the current owner.
6418 z_lghi(zero, 0);
6419 z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::monitor_owner_id_offset()));
6420 z_csg(zero, Z_R0_scratch, owner_address);
6421 z_bre(monitor_locked);
6422
6423 // Check if recursive.
6424 z_cgr(Z_R0_scratch, zero); // zero contains the owner from z_csg instruction
6425 z_brne(slow_path);
6426
6427 // Recursive
6428 z_agsi(recursions_address, 1ll);
6429
6430 bind(monitor_locked);
6431 if (UseObjectMonitorTable) {
6432 // Cache the monitor for unlock
6433 z_stg(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
6434 }
6435 // set the CC now
6436 z_cgr(obj, obj);
6437 }
6438 BLOCK_COMMENT("} handle_inflated_monitor_locking");
6439
6440 bind(locked);
6441
6442 #ifdef ASSERT
6443 // Check that locked label is reached with flag == EQ.
6444 NearLabel flag_correct;
6445 z_bre(flag_correct);
6446 stop("CC is not set to EQ, it should be - lock");
6447 #endif // ASSERT
6448
6449 bind(slow_path);
6450
6451 #ifdef ASSERT
6452 // Check that slow_path label is reached with flag == NE.
6453 z_brne(flag_correct);
6454 stop("CC is not set to NE, it should be - lock");
6455 bind(flag_correct);
6456 #endif // ASSERT
6457
6458 // C2 uses the value of flag (NE vs EQ) to determine the continuation.
6459 }
6460
6461 void MacroAssembler::compiler_fast_unlock_object(Register obj, Register box, Register tmp1, Register tmp2) {
6462 assert_different_registers(obj, box, tmp1, tmp2);
6463
6464 // Handle inflated monitor.
6465 NearLabel inflated, inflated_load_mark;
6466 // Finish fast unlock successfully. MUST reach to with flag == EQ.
6467 NearLabel unlocked;
6468 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE.
6469 NearLabel slow_path;
6470
6471 const Register mark = tmp1;
6472 const Register top = tmp2;
6473 const int mark_offset = oopDesc::mark_offset_in_bytes();
6474 const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
6475
6476 BLOCK_COMMENT("compiler_fast_unlock {");
6477 { // Fast Unlock
6478 NearLabel push_and_slow_path;
6479
6480 // Check if obj is top of lock-stack.
6481 z_lgf(top, Address(Z_thread, ls_top_offset));
6482
6483 z_aghi(top, -oopSize);
6484 z_cg(obj, Address(Z_thread, top));
6485 branch_optimized(bcondNotEqual, inflated_load_mark);
6486
6487 // Pop lock-stack.
6488 #ifdef ASSERT
6489 const Register temp_top = tmp1; // let's not kill top here, we can use for recursive check
6490 z_agrk(temp_top, top, Z_thread);
6491 z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry
6492 #endif
6493 z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object
6494
6495 // The underflow check is elided. The recursive check will always fail
6496 // when the lock stack is empty because of the _bad_oop_sentinel field.
6497
6498 // Check if recursive.
6499 z_aghi(top, -oopSize);
6500 z_cg(obj, Address(Z_thread, top));
6501 z_bre(unlocked);
6502
6503 // Not recursive
6504
6505 // Check for monitor (0b10).
6506 // Because we got here by popping (meaning we pushed in locked)
6507 // there will be no monitor in the box. So we need to push back the obj
6508 // so that the runtime can fix any potential anonymous owner.
6509 z_lg(mark, Address(obj, mark_offset));
6510 z_tmll(mark, markWord::monitor_value);
6511 if (!UseObjectMonitorTable) {
6512 z_brnaz(inflated);
6513 } else {
6514 z_brnaz(push_and_slow_path);
6515 }
6516
6517 #ifdef ASSERT
6518 // Check header not unlocked (0b01).
6519 NearLabel not_unlocked;
6520 z_tmll(mark, markWord::unlocked_value);
6521 z_braz(not_unlocked);
6522 stop("fast_unlock already unlocked");
6523 bind(not_unlocked);
6524 #endif // ASSERT
6525
6526 { // Try to unlock. Transition lock bits 0b00 => 0b01
6527 Register unlocked_obj = top;
6528 z_lgr(unlocked_obj, mark);
6529 z_oill(unlocked_obj, markWord::unlocked_value);
6530 z_csg(mark, unlocked_obj, mark_offset, obj);
6531 branch_optimized(Assembler::bcondEqual, unlocked);
6532 }
6533
6534 bind(push_and_slow_path);
6535 // Restore lock-stack and handle the unlock in runtime.
6536 z_lgf(top, Address(Z_thread, ls_top_offset));
6537 DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));)
6538 z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
6539 // set CC to NE
6540 z_ltgr(obj, obj); // object is not null here
6541 z_bru(slow_path);
6542 }
6543 BLOCK_COMMENT("} compiler_fast_unlock");
6544
6545 { // Handle inflated monitor.
6546
6547 bind(inflated_load_mark);
6548
6549 z_lg(mark, Address(obj, mark_offset));
6550
6551 #ifdef ASSERT
6552 z_tmll(mark, markWord::monitor_value);
6553 z_brnaz(inflated);
6554 stop("Fast Unlock not monitor");
6555 #endif // ASSERT
6556
6557 bind(inflated);
6558
6559 #ifdef ASSERT
6560 NearLabel check_done, loop;
6561 z_lgf(top, Address(Z_thread, ls_top_offset));
6562 bind(loop);
6563 z_aghi(top, -oopSize);
6564 compareU32_and_branch(top, in_bytes(JavaThread::lock_stack_base_offset()),
6565 bcondLow, check_done);
6566 z_cg(obj, Address(Z_thread, top));
6567 z_brne(loop);
6568 stop("Fast Unlock lock on stack");
6569 bind(check_done);
6570 #endif // ASSERT
6571
6572 const Register tmp1_monitor = tmp1;
6573
6574 if (!UseObjectMonitorTable) {
6575 assert(tmp1_monitor == mark, "should be the same here");
6576 } else {
6577 // Uses ObjectMonitorTable. Look for the monitor in our BasicLock on the stack.
6578 z_lg(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
6579 // null check with ZF == 0, no valid pointer below alignof(ObjectMonitor*)
6580 z_cghi(tmp1_monitor, alignof(ObjectMonitor*));
6581
6582 z_brl(slow_path);
6583 }
6584
6585 // mark contains the tagged ObjectMonitor*.
6586 const Register monitor = mark;
6587
6588 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
6589 const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
6590 const Address succ_address{monitor, ObjectMonitor::succ_offset() - monitor_tag};
6591 const Address entry_list_address{monitor, ObjectMonitor::entry_list_offset() - monitor_tag};
6592 const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
6593
6594 NearLabel not_recursive;
6595 const Register recursions = tmp2;
6596
6597 // Check if recursive.
6598 load_and_test_long(recursions, recursions_address);
6599 z_bre(not_recursive); // if 0 then jump, it's not recursive locking
6600
6601 // Recursive unlock
6602 z_agsi(recursions_address, -1ll);
6603 z_cgr(monitor, monitor); // set the CC to EQUAL
6604 z_bru(unlocked);
6605
6606 bind(not_recursive);
6607
6608 NearLabel set_eq_unlocked;
6609
6610 // Set owner to null.
6611 // Release to satisfy the JMM
6612 z_release();
6613 z_lghi(tmp2, 0);
6614 z_stg(tmp2 /*=0*/, owner_address);
6615 // We need a full fence after clearing owner to avoid stranding.
6616 z_fence();
6617
6618 // Check if the entry_list is empty.
6619 load_and_test_long(tmp2, entry_list_address);
6620 z_bre(unlocked); // If so we are done.
6621
6622 // Check if there is a successor.
6623 load_and_test_long(tmp2, succ_address);
6624 z_brne(set_eq_unlocked); // If so we are done.
6625
6626 // Save the monitor pointer in the current thread, so we can try to
6627 // reacquire the lock in SharedRuntime::monitor_exit_helper().
6628 if (!UseObjectMonitorTable) {
6629 z_xilf(monitor, markWord::monitor_value);
6630 }
6631 z_stg(monitor, Address(Z_thread, JavaThread::unlocked_inflated_monitor_offset()));
6632
6633 z_ltgr(obj, obj); // Set flag = NE
6634 z_bru(slow_path);
6635
6636 bind(set_eq_unlocked);
6637 z_cr(tmp2, tmp2); // Set flag = EQ
6638 }
6639
6640 bind(unlocked);
6641
6642 #ifdef ASSERT
6643 // Check that unlocked label is reached with flag == EQ.
6644 NearLabel flag_correct;
6645 z_bre(flag_correct);
6646 stop("CC is not set to EQ, it should be - unlock");
6647 #endif // ASSERT
6648
6649 bind(slow_path);
6650
6651 #ifdef ASSERT
6652 // Check that slow_path label is reached with flag == NE.
6653 z_brne(flag_correct);
6654 stop("CC is not set to NE, it should be - unlock");
6655 bind(flag_correct);
6656 #endif // ASSERT
6657
6658 // C2 uses the value of flag (NE vs EQ) to determine the continuation.
6659 }
6660
6661 void MacroAssembler::pop_count_int(Register r_dst, Register r_src, Register r_tmp) {
6662 BLOCK_COMMENT("pop_count_int {");
6663
6664 assert(r_tmp != noreg, "temp register required for pop_count_int, as code may run on machine older than z15");
6665 assert_different_registers(r_dst, r_tmp); // if r_src is same as r_tmp, it should be fine
6666
6667 if (VM_Version::has_MiscInstrExt3()) {
6668 pop_count_int_with_ext3(r_dst, r_src);
6669 } else {
6670 pop_count_int_without_ext3(r_dst, r_src, r_tmp);
6671 }
6672
6673 BLOCK_COMMENT("} pop_count_int");
6674 }
6675
6676 void MacroAssembler::pop_count_long(Register r_dst, Register r_src, Register r_tmp) {
6677 BLOCK_COMMENT("pop_count_long {");
6678
6679 assert(r_tmp != noreg, "temp register required for pop_count_long, as code may run on machine older than z15");
6680 assert_different_registers(r_dst, r_tmp); // if r_src is same as r_tmp, it should be fine
6681
6682 if (VM_Version::has_MiscInstrExt3()) {
6683 pop_count_long_with_ext3(r_dst, r_src);
6684 } else {
6685 pop_count_long_without_ext3(r_dst, r_src, r_tmp);
6686 }
6687
6688 BLOCK_COMMENT("} pop_count_long");
6689 }
6690
6691 void MacroAssembler::pop_count_int_without_ext3(Register r_dst, Register r_src, Register r_tmp) {
6692 BLOCK_COMMENT("pop_count_int_without_ext3 {");
6693
6694 assert(r_tmp != noreg, "temp register required for popcnt, for machines < z15");
6695 assert_different_registers(r_dst, r_tmp); // if r_src is same as r_tmp, it should be fine
6696
6697 z_popcnt(r_dst, r_src, 0);
6698 z_srlg(r_tmp, r_dst, 16);
6699 z_alr(r_dst, r_tmp);
6700 z_srlg(r_tmp, r_dst, 8);
6701 z_alr(r_dst, r_tmp);
6702 z_llgcr(r_dst, r_dst);
6703
6704 BLOCK_COMMENT("} pop_count_int_without_ext3");
6705 }
6706
6707 void MacroAssembler::pop_count_long_without_ext3(Register r_dst, Register r_src, Register r_tmp) {
6708 BLOCK_COMMENT("pop_count_long_without_ext3 {");
6709
6710 assert(r_tmp != noreg, "temp register required for popcnt, for machines < z15");
6711 assert_different_registers(r_dst, r_tmp); // if r_src is same as r_tmp, it should be fine
6712
6713 z_popcnt(r_dst, r_src, 0);
6714 z_ahhlr(r_dst, r_dst, r_dst);
6715 z_sllg(r_tmp, r_dst, 16);
6716 z_algr(r_dst, r_tmp);
6717 z_sllg(r_tmp, r_dst, 8);
6718 z_algr(r_dst, r_tmp);
6719 z_srlg(r_dst, r_dst, 56);
6720
6721 BLOCK_COMMENT("} pop_count_long_without_ext3");
6722 }
6723
6724 void MacroAssembler::pop_count_long_with_ext3(Register r_dst, Register r_src) {
6725 BLOCK_COMMENT("pop_count_long_with_ext3 {");
6726
6727 guarantee(VM_Version::has_MiscInstrExt3(),
6728 "this hardware doesn't support miscellaneous-instruction-extensions facility 3, still pop_count_long_with_ext3 is used");
6729 z_popcnt(r_dst, r_src, 8);
6730
6731 BLOCK_COMMENT("} pop_count_long_with_ext3");
6732 }
6733
6734 void MacroAssembler::pop_count_int_with_ext3(Register r_dst, Register r_src) {
6735 BLOCK_COMMENT("pop_count_int_with_ext3 {");
6736
6737 guarantee(VM_Version::has_MiscInstrExt3(),
6738 "this hardware doesn't support miscellaneous-instruction-extensions facility 3, still pop_count_long_with_ext3 is used");
6739 z_llgfr(r_dst, r_src);
6740 z_popcnt(r_dst, r_dst, 8);
6741
6742 BLOCK_COMMENT("} pop_count_int_with_ext3");
6743 }
6744
6745 // LOAD HALFWORD IMMEDIATE ON CONDITION (32 <- 16)
6746 void MacroAssembler::load_on_condition_imm_32(Register dst, int64_t i2, branch_condition cc) {
6747 if (VM_Version::has_LoadStoreConditional2()) { // z_lochi works on z13 or above
6748 assert(Assembler::is_simm16(i2), "sanity");
6749 z_lochi(dst, i2, cc);
6750 } else {
6751 NearLabel done;
6752 z_brc(Assembler::inverse_condition(cc), done);
6753 z_lhi(dst, i2);
6754 bind(done);
6755 }
6756 }
6757
6758 // LOAD HALFWORD IMMEDIATE ON CONDITION (64 <- 16)
6759 void MacroAssembler::load_on_condition_imm_64(Register dst, int64_t i2, branch_condition cc) {
6760 if (VM_Version::has_LoadStoreConditional2()) { // z_locghi works on z13 or above
6761 assert(Assembler::is_simm16(i2), "sanity");
6762 z_locghi(dst, i2, cc);
6763 } else {
6764 NearLabel done;
6765 z_brc(Assembler::inverse_condition(cc), done);
6766 z_lghi(dst, i2);
6767 bind(done);
6768 }
6769 }
6770
6771 // Handle the receiver type profile update given the "recv" klass.
6772 //
6773 // Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset".
6774 // If there are no matching or claimable receiver entries in RD, updates
6775 // the polymorphic counter.
6776 //
6777 // This code expected to run by either the interpreter or JIT-ed code, without
6778 // extra synchronization. For safety, receiver cells are claimed atomically, which
6779 // avoids grossly misrepresenting the profiles under concurrent updates. For speed,
6780 // counter updates are not atomic.
6781 //
6782 void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset, Register scratch) {
6783 Register r0_tmp = Z_R0_scratch; // cannot be used in address calculation
6784 assert_different_registers(recv, mdp, scratch, r0_tmp);
6785
6786 int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0));
6787 int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit()));
6788 int poly_count_offset = in_bytes(CounterData::count_offset());
6789 int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset;
6790 int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset;
6791
6792 // Adjust for MDP offsets.
6793 base_receiver_offset += mdp_offset;
6794 end_receiver_offset += mdp_offset;
6795 poly_count_offset += mdp_offset;
6796
6797 #ifdef ASSERT
6798 // We are about to walk the MDO slots without asking for offsets.
6799 // Check that our math hits all the right spots.
6800 for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) {
6801 int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c));
6802 int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c));
6803 int offset = base_receiver_offset + receiver_step*c;
6804 int count_offset = offset + receiver_to_count_step;
6805 assert(offset == real_recv_offset, "receiver slot math");
6806 assert(count_offset == real_count_offset, "receiver count math");
6807 }
6808 int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset());
6809 assert(poly_count_offset == real_poly_count_offset, "poly counter math");
6810 #endif
6811
6812 // Corner case: no profile table. Increment poly counter and exit.
6813 if (ReceiverTypeData::row_limit() == 0) {
6814 add2mem_64(Address(mdp, poly_count_offset), DataLayout::counter_increment, scratch);
6815 return;
6816 }
6817
6818 NearLabel L_loop_search_receiver, L_loop_search_empty;
6819 NearLabel L_restart, L_found_recv, L_found_empty, L_count_update;
6820 Register offset = scratch;
6821
6822 // The code here recognizes three major cases:
6823 // A. Fastest: receiver found in the table
6824 // B. Fast: no receiver in the table, and the table is full
6825 // C. Slow: no receiver in the table, free slots in the table
6826 //
6827 // The case A performance is most important, as perfectly-behaved code would end up
6828 // there, especially with larger TypeProfileWidth. The case B performance is
6829 // important as well, this is where bulk of code would land for normally megamorphic
6830 // cases. The case C performance is not essential, its job is to deal with installation
6831 // races, we optimize for code density instead. Case C needs to make sure that receiver
6832 // rows are only claimed once. This makes sure we never overwrite a row for another
6833 // receiver and never duplicate the receivers in the list, making profile type-accurate.
6834 //
6835 // It is very tempting to handle these cases in a single loop, and claim the first slot
6836 // without checking the rest of the table. But, profiling code should tolerate free slots
6837 // in the table, as class unloading can clear them. After such cleanup, the receiver
6838 // we need might be _after_ the free slot. Therefore, we need to let at least full scan
6839 // to complete, before trying to install new slots. Splitting the code in several tight
6840 // loops also helpfully optimizes for cases A and B.
6841 //
6842 // This code is effectively:
6843 //
6844 // restart:
6845 // // Fastest: receiver is already installed
6846 // for (i = 0; i < receiver_count(); i++) {
6847 // if (receiver(i) == recv) goto found_recv(i);
6848 // }
6849 //
6850 // // Fast: no receiver, but profile is not full
6851 // for (i = 0; i < receiver_count(); i++) {
6852 // if (receiver(i) == null) goto found_null(i);
6853 // }
6854 // goto polymorphic
6855 //
6856 // // Slow: try to install receiver
6857 // found_null(i):
6858 // CAS(&receiver(i), null, recv);
6859 // goto restart
6860 //
6861 // polymorphic:
6862 // count++;
6863 // return
6864 //
6865 // found_recv(i):
6866 // *receiver_count(i)++
6867 //
6868
6869 bind(L_restart);
6870
6871 // Fastest: receiver is already installed
6872 load_const_optimized(offset, base_receiver_offset);
6873
6874 bind(L_loop_search_receiver);
6875 z_cg(recv, Address(mdp, offset));
6876 z_bre(L_found_recv);
6877 add2reg(offset, receiver_step);
6878 compare64_and_branch(offset, end_receiver_offset, bcondNotEqual, L_loop_search_receiver);
6879
6880 // Fast: no receiver, but profile is not full
6881 load_const_optimized(offset, base_receiver_offset);
6882
6883 bind(L_loop_search_empty);
6884 z_ltg(r0_tmp, Address(mdp, offset));
6885 z_brz(L_found_empty);
6886 add2reg(offset, receiver_step);
6887 compare64_and_branch(offset, end_receiver_offset, bcondNotEqual, L_loop_search_empty);
6888
6889 // Slow: Receiver is not found and table is full.
6890 // Increment polymorphic counter instead of receiver slot.
6891 load_const_optimized(offset, poly_count_offset);
6892 z_bru(L_count_update);
6893
6894 // Slowest: try to install receiver
6895 bind(L_found_empty);
6896
6897 {
6898 // Atomically swing receiver slot: null -> recv.
6899 // Use compare-and-swap to claim the slot.
6900 Register receiver_addr = offset;
6901 z_agr(receiver_addr, mdp); // receiver_addr = mdp + offset
6902
6903 // r0_tmp is used as expected value (0), recv is the new value
6904 z_lghi(r0_tmp, 0);
6905 z_csg(r0_tmp, recv, 0, receiver_addr);
6906 }
6907
6908 // CAS success means the slot now has the receiver we want. CAS failure means
6909 // something had claimed the slot concurrently: it can be the same receiver we want,
6910 // or something else. Since this is a slow path, we can optimize for code density,
6911 // and just restart the search from the beginning.
6912 z_bru(L_restart);
6913
6914 // Found a receiver, convert its slot offset to corresponding count offset.
6915 bind(L_found_recv);
6916 add2reg(offset, receiver_to_count_step);
6917
6918 // Finally, update the counter
6919 bind(L_count_update);
6920 z_agr(offset, mdp);
6921 add2mem_64(Address(offset), DataLayout::counter_increment, r0_tmp);
6922 }
6923
6924 // Unimplemented methods for inline types.
6925 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
6926 Unimplemented();
6927 }
6928
6929 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
6930 Unimplemented();
6931 }
6932
6933 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
6934 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
6935 RegState reg_state[]) {
6936 Unimplemented();
6937 }
6938
6939 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
6940 VMRegPair* from, int from_count, int& from_index, VMReg to,
6941 RegState reg_state[], Register val_array) {
6942 Unimplemented();
6943 }
6944
6945 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
6946 Unimplemented();
6947 }
6948
6949 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
6950 Unimplemented();
6951 }