1 /*
2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2023 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "code/compiledIC.hpp"
29 #include "compiler/disassembler.hpp"
30 #include "gc/shared/collectedHeap.inline.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "interpreter/interpreter.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "nativeInst_ppc.hpp"
36 #include "oops/compressedKlass.inline.hpp"
37 #include "oops/compressedOops.inline.hpp"
38 #include "oops/klass.inline.hpp"
39 #include "oops/methodData.hpp"
40 #include "prims/methodHandles.hpp"
41 #include "register_ppc.hpp"
42 #include "runtime/icache.hpp"
43 #include "runtime/interfaceSupport.inline.hpp"
44 #include "runtime/objectMonitor.hpp"
45 #include "runtime/os.hpp"
46 #include "runtime/safepoint.hpp"
47 #include "runtime/safepointMechanism.hpp"
48 #include "runtime/sharedRuntime.hpp"
49 #include "runtime/stubRoutines.hpp"
50 #include "runtime/vm_version.hpp"
51 #include "utilities/macros.hpp"
52 #include "utilities/powerOfTwo.hpp"
53
54 #ifdef PRODUCT
55 #define BLOCK_COMMENT(str) // nothing
56 #else
57 #define BLOCK_COMMENT(str) block_comment(str)
58 #endif
59 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
60
61 #ifdef ASSERT
62 // On RISC, there's no benefit to verifying instruction boundaries.
63 bool AbstractAssembler::pd_check_instruction_mark() { return false; }
64 #endif
65
66 void MacroAssembler::ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop) {
67 assert(Assembler::is_simm(si31, 31) && si31 >= 0, "si31 out of range");
68 if (Assembler::is_simm(si31, 16)) {
69 ld(d, si31, a);
70 if (emit_filler_nop) nop();
71 } else {
72 const int hi = MacroAssembler::largeoffset_si16_si16_hi(si31);
73 const int lo = MacroAssembler::largeoffset_si16_si16_lo(si31);
74 addis(d, a, hi);
75 ld(d, lo, d);
76 }
77 }
78
79 void MacroAssembler::ld_largeoffset(Register d, int si31, Register a, int emit_filler_nop) {
80 assert_different_registers(d, a);
81 ld_largeoffset_unchecked(d, si31, a, emit_filler_nop);
82 }
83
84 void MacroAssembler::load_sized_value(Register dst, RegisterOrConstant offs, Register base,
85 size_t size_in_bytes, bool is_signed) {
86 switch (size_in_bytes) {
87 case 8: ld(dst, offs, base); break;
88 case 4: is_signed ? lwa(dst, offs, base) : lwz(dst, offs, base); break;
89 case 2: is_signed ? lha(dst, offs, base) : lhz(dst, offs, base); break;
90 case 1: lbz(dst, offs, base); if (is_signed) extsb(dst, dst); break; // lba doesn't exist :(
91 default: ShouldNotReachHere();
92 }
93 }
94
95 void MacroAssembler::store_sized_value(Register dst, RegisterOrConstant offs, Register base,
96 size_t size_in_bytes) {
97 switch (size_in_bytes) {
98 case 8: std(dst, offs, base); break;
99 case 4: stw(dst, offs, base); break;
100 case 2: sth(dst, offs, base); break;
101 case 1: stb(dst, offs, base); break;
102 default: ShouldNotReachHere();
103 }
104 }
105
106 void MacroAssembler::align(int modulus, int max, int rem) {
107 int padding = (rem + modulus - (offset() % modulus)) % modulus;
108 if (padding > max) return;
109 for (int c = (padding >> 2); c > 0; --c) { nop(); }
110 }
111
112 void MacroAssembler::align_prefix() {
113 if (is_aligned(offset() + BytesPerInstWord, 64)) { nop(); }
114 }
115
116 // Issue instructions that calculate given TOC from global TOC.
117 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16,
118 bool add_relocation, bool emit_dummy_addr) {
119 int offset = -1;
120 if (emit_dummy_addr) {
121 offset = -128; // dummy address
122 } else if (addr != (address)(intptr_t)-1) {
123 offset = MacroAssembler::offset_to_global_toc(addr);
124 }
125
126 if (hi16) {
127 addis(dst, R29_TOC, MacroAssembler::largeoffset_si16_si16_hi(offset));
128 }
129 if (lo16) {
130 if (add_relocation) {
131 // Relocate at the addi to avoid confusion with a load from the method's TOC.
132 relocate(internal_word_Relocation::spec(addr));
133 }
134 addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset));
135 }
136 }
137
138 address MacroAssembler::patch_calculate_address_from_global_toc_at(address a, address bound, address addr) {
139 const int offset = MacroAssembler::offset_to_global_toc(addr);
140
141 const address inst2_addr = a;
142 const int inst2 = *(int *)inst2_addr;
143
144 // The relocation points to the second instruction, the addi,
145 // and the addi reads and writes the same register dst.
146 const int dst = inv_rt_field(inst2);
147 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
148
149 // Now, find the preceding addis which writes to dst.
150 int inst1 = 0;
151 address inst1_addr = inst2_addr - BytesPerInstWord;
152 while (inst1_addr >= bound) {
153 inst1 = *(int *) inst1_addr;
154 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
155 // Stop, found the addis which writes dst.
156 break;
157 }
158 inst1_addr -= BytesPerInstWord;
159 }
160
161 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
162 set_imm((int *)inst1_addr, MacroAssembler::largeoffset_si16_si16_hi(offset));
163 set_imm((int *)inst2_addr, MacroAssembler::largeoffset_si16_si16_lo(offset));
164 return inst1_addr;
165 }
166
167 address MacroAssembler::get_address_of_calculate_address_from_global_toc_at(address a, address bound) {
168 const address inst2_addr = a;
169 const int inst2 = *(int *)inst2_addr;
170
171 // The relocation points to the second instruction, the addi,
172 // and the addi reads and writes the same register dst.
173 const int dst = inv_rt_field(inst2);
174 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
175
176 // Now, find the preceding addis which writes to dst.
177 int inst1 = 0;
178 address inst1_addr = inst2_addr - BytesPerInstWord;
179 while (inst1_addr >= bound) {
180 inst1 = *(int *) inst1_addr;
181 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
182 // stop, found the addis which writes dst
183 break;
184 }
185 inst1_addr -= BytesPerInstWord;
186 }
187
188 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
189
190 int offset = (get_imm(inst1_addr, 0) << 16) + get_imm(inst2_addr, 0);
191 // -1 is a special case
192 if (offset == -1) {
193 return (address)(intptr_t)-1;
194 } else {
195 return global_toc() + offset;
196 }
197 }
198
199 #ifdef _LP64
200 // Patch compressed oops or klass constants.
201 // Assembler sequence is
202 // 1) compressed oops:
203 // lis rx = const.hi
204 // ori rx = rx | const.lo
205 // 2) compressed klass:
206 // lis rx = const.hi
207 // clrldi rx = rx & 0xFFFFffff // clearMS32b, optional
208 // ori rx = rx | const.lo
209 // Clrldi will be passed by.
210 address MacroAssembler::patch_set_narrow_oop(address a, address bound, narrowOop data) {
211 assert(UseCompressedOops, "Should only patch compressed oops");
212
213 const address inst2_addr = a;
214 const int inst2 = *(int *)inst2_addr;
215
216 // The relocation points to the second instruction, the ori,
217 // and the ori reads and writes the same register dst.
218 const int dst = inv_rta_field(inst2);
219 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
220 // Now, find the preceding addis which writes to dst.
221 int inst1 = 0;
222 address inst1_addr = inst2_addr - BytesPerInstWord;
223 bool inst1_found = false;
224 while (inst1_addr >= bound) {
225 inst1 = *(int *)inst1_addr;
226 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break; }
227 inst1_addr -= BytesPerInstWord;
228 }
229 assert(inst1_found, "inst is not lis");
230
231 uint32_t data_value = CompressedOops::narrow_oop_value(data);
232 int xc = (data_value >> 16) & 0xffff;
233 int xd = (data_value >> 0) & 0xffff;
234
235 set_imm((int *)inst1_addr, (short)(xc)); // see enc_load_con_narrow_hi/_lo
236 set_imm((int *)inst2_addr, (xd)); // unsigned int
237 return inst1_addr;
238 }
239
240 // Get compressed oop constant.
241 narrowOop MacroAssembler::get_narrow_oop(address a, address bound) {
242 assert(UseCompressedOops, "Should only patch compressed oops");
243
244 const address inst2_addr = a;
245 const int inst2 = *(int *)inst2_addr;
246
247 // The relocation points to the second instruction, the ori,
248 // and the ori reads and writes the same register dst.
249 const int dst = inv_rta_field(inst2);
250 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
251 // Now, find the preceding lis which writes to dst.
252 int inst1 = 0;
253 address inst1_addr = inst2_addr - BytesPerInstWord;
254 bool inst1_found = false;
255
256 while (inst1_addr >= bound) {
257 inst1 = *(int *) inst1_addr;
258 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break;}
259 inst1_addr -= BytesPerInstWord;
260 }
261 assert(inst1_found, "inst is not lis");
262
263 uint xl = ((unsigned int) (get_imm(inst2_addr, 0) & 0xffff));
264 uint xh = (((get_imm(inst1_addr, 0)) & 0xffff) << 16);
265
266 return CompressedOops::narrow_oop_cast(xl | xh);
267 }
268 #endif // _LP64
269
270 // Returns true if successful.
271 bool MacroAssembler::load_const_from_method_toc(Register dst, AddressLiteral& a,
272 Register toc, bool fixed_size) {
273 int toc_offset = 0;
274 // Use RelocationHolder::none for the constant pool entry, otherwise
275 // we will end up with a failing NativeCall::verify(x) where x is
276 // the address of the constant pool entry.
277 // FIXME: We should insert relocation information for oops at the constant
278 // pool entries instead of inserting it at the loads; patching of a constant
279 // pool entry should be less expensive.
280 address const_address = address_constant((address)a.value(), RelocationHolder::none);
281 if (const_address == nullptr) { return false; } // allocation failure
282 // Relocate at the pc of the load.
283 relocate(a.rspec());
284 toc_offset = (int)(const_address - code()->consts()->start());
285 ld_largeoffset_unchecked(dst, toc_offset, toc, fixed_size);
286 return true;
287 }
288
289 bool MacroAssembler::is_load_const_from_method_toc_at(address a) {
290 const address inst1_addr = a;
291 const int inst1 = *(int *)inst1_addr;
292
293 // The relocation points to the ld or the addis.
294 return (is_ld(inst1)) ||
295 (is_addis(inst1) && inv_ra_field(inst1) != 0);
296 }
297
298 int MacroAssembler::get_offset_of_load_const_from_method_toc_at(address a) {
299 assert(is_load_const_from_method_toc_at(a), "must be load_const_from_method_toc");
300
301 const address inst1_addr = a;
302 const int inst1 = *(int *)inst1_addr;
303
304 if (is_ld(inst1)) {
305 return inv_d1_field(inst1);
306 } else if (is_addis(inst1)) {
307 const int dst = inv_rt_field(inst1);
308
309 // Now, find the succeeding ld which reads and writes to dst.
310 address inst2_addr = inst1_addr + BytesPerInstWord;
311 int inst2 = 0;
312 while (true) {
313 inst2 = *(int *) inst2_addr;
314 if (is_ld(inst2) && inv_ra_field(inst2) == dst && inv_rt_field(inst2) == dst) {
315 // Stop, found the ld which reads and writes dst.
316 break;
317 }
318 inst2_addr += BytesPerInstWord;
319 }
320 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
321 }
322 ShouldNotReachHere();
323 return 0;
324 }
325
326 // Get the constant from a `load_const' sequence.
327 long MacroAssembler::get_const(address a) {
328 assert(is_load_const_at(a), "not a load of a constant");
329 const int *p = (const int*) a;
330 unsigned long x = (((unsigned long) (get_imm(a,0) & 0xffff)) << 48);
331 if (is_ori(*(p+1))) {
332 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 32);
333 x |= (((unsigned long) (get_imm(a,3) & 0xffff)) << 16);
334 x |= (((unsigned long) (get_imm(a,4) & 0xffff)));
335 } else if (is_lis(*(p+1))) {
336 x |= (((unsigned long) (get_imm(a,2) & 0xffff)) << 32);
337 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 16);
338 x |= (((unsigned long) (get_imm(a,3) & 0xffff)));
339 } else {
340 ShouldNotReachHere();
341 return (long) 0;
342 }
343 return (long) x;
344 }
345
346 // Patch the 64 bit constant of a `load_const' sequence. This is a low
347 // level procedure. It neither flushes the instruction cache nor is it
348 // mt safe.
349 void MacroAssembler::patch_const(address a, long x) {
350 assert(is_load_const_at(a), "not a load of a constant");
351 int *p = (int*) a;
352 if (is_ori(*(p+1))) {
353 set_imm(0 + p, (x >> 48) & 0xffff);
354 set_imm(1 + p, (x >> 32) & 0xffff);
355 set_imm(3 + p, (x >> 16) & 0xffff);
356 set_imm(4 + p, x & 0xffff);
357 } else if (is_lis(*(p+1))) {
358 set_imm(0 + p, (x >> 48) & 0xffff);
359 set_imm(2 + p, (x >> 32) & 0xffff);
360 set_imm(1 + p, (x >> 16) & 0xffff);
361 set_imm(3 + p, x & 0xffff);
362 } else {
363 ShouldNotReachHere();
364 }
365 }
366
367 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) {
368 assert(oop_recorder() != nullptr, "this assembler needs a Recorder");
369 int index = oop_recorder()->allocate_metadata_index(obj);
370 RelocationHolder rspec = metadata_Relocation::spec(index);
371 return AddressLiteral((address)obj, rspec);
372 }
373
374 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) {
375 assert(oop_recorder() != nullptr, "this assembler needs a Recorder");
376 int index = oop_recorder()->find_index(obj);
377 RelocationHolder rspec = metadata_Relocation::spec(index);
378 return AddressLiteral((address)obj, rspec);
379 }
380
381 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
382 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
383 int oop_index = oop_recorder()->allocate_oop_index(obj);
384 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
385 }
386
387 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
388 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
389 int oop_index = oop_recorder()->find_index(obj);
390 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
391 }
392
393 #ifndef PRODUCT
394 void MacroAssembler::pd_print_patched_instruction(address branch) {
395 Unimplemented(); // TODO: PPC port
396 }
397 #endif // ndef PRODUCT
398
399 // Conditional far branch for destinations encodable in 24+2 bits.
400 void MacroAssembler::bc_far(int boint, int biint, Label& dest, int optimize) {
401
402 // If requested by flag optimize, relocate the bc_far as a
403 // runtime_call and prepare for optimizing it when the code gets
404 // relocated.
405 if (optimize == bc_far_optimize_on_relocate) {
406 relocate(relocInfo::runtime_call_type);
407 }
408
409 // variant 2:
410 //
411 // b!cxx SKIP
412 // bxx DEST
413 // SKIP:
414 //
415
416 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
417 opposite_bcond(inv_boint_bcond(boint)));
418
419 // We emit two branches.
420 // First, a conditional branch which jumps around the far branch.
421 const address not_taken_pc = pc() + 2 * BytesPerInstWord;
422 const address bc_pc = pc();
423 bc(opposite_boint, biint, not_taken_pc);
424
425 const int bc_instr = *(int*)bc_pc;
426 assert(not_taken_pc == (address)inv_bd_field(bc_instr, (intptr_t)bc_pc), "postcondition");
427 assert(opposite_boint == inv_bo_field(bc_instr), "postcondition");
428 assert(boint == add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(bc_instr))),
429 opposite_bcond(inv_boint_bcond(inv_bo_field(bc_instr)))),
430 "postcondition");
431 assert(biint == inv_bi_field(bc_instr), "postcondition");
432
433 // Second, an unconditional far branch which jumps to dest.
434 // Note: target(dest) remembers the current pc (see CodeSection::target)
435 // and returns the current pc if the label is not bound yet; when
436 // the label gets bound, the unconditional far branch will be patched.
437 const address target_pc = target(dest);
438 const address b_pc = pc();
439 b(target_pc);
440
441 assert(not_taken_pc == pc(), "postcondition");
442 assert(dest.is_bound() || target_pc == b_pc, "postcondition");
443 }
444
445 // 1 or 2 instructions
446 void MacroAssembler::bc_far_optimized(int boint, int biint, Label& dest) {
447 if (dest.is_bound() && is_within_range_of_bcxx(target(dest), pc())) {
448 bc(boint, biint, dest);
449 } else {
450 bc_far(boint, biint, dest, MacroAssembler::bc_far_optimize_on_relocate);
451 }
452 }
453
454 bool MacroAssembler::is_bc_far_at(address instruction_addr) {
455 return is_bc_far_variant1_at(instruction_addr) ||
456 is_bc_far_variant2_at(instruction_addr) ||
457 is_bc_far_variant3_at(instruction_addr);
458 }
459
460 address MacroAssembler::get_dest_of_bc_far_at(address instruction_addr) {
461 if (is_bc_far_variant1_at(instruction_addr)) {
462 const address instruction_1_addr = instruction_addr;
463 const int instruction_1 = *(int*)instruction_1_addr;
464 return (address)inv_bd_field(instruction_1, (intptr_t)instruction_1_addr);
465 } else if (is_bc_far_variant2_at(instruction_addr)) {
466 const address instruction_2_addr = instruction_addr + 4;
467 return bxx_destination(instruction_2_addr);
468 } else if (is_bc_far_variant3_at(instruction_addr)) {
469 return instruction_addr + 8;
470 }
471 // variant 4 ???
472 ShouldNotReachHere();
473 return nullptr;
474 }
475 void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address dest) {
476
477 if (is_bc_far_variant3_at(instruction_addr)) {
478 // variant 3, far cond branch to the next instruction, already patched to nops:
479 //
480 // nop
481 // endgroup
482 // SKIP/DEST:
483 //
484 return;
485 }
486
487 // first, extract boint and biint from the current branch
488 int boint = 0;
489 int biint = 0;
490
491 ResourceMark rm;
492 const int code_size = 2 * BytesPerInstWord;
493 CodeBuffer buf(instruction_addr, code_size);
494 MacroAssembler masm(&buf);
495 if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
496 // Far branch to next instruction: Optimize it by patching nops (produce variant 3).
497 masm.nop();
498 masm.endgroup();
499 } else {
500 if (is_bc_far_variant1_at(instruction_addr)) {
501 // variant 1, the 1st instruction contains the destination address:
502 //
503 // bcxx DEST
504 // nop
505 //
506 const int instruction_1 = *(int*)(instruction_addr);
507 boint = inv_bo_field(instruction_1);
508 biint = inv_bi_field(instruction_1);
509 } else if (is_bc_far_variant2_at(instruction_addr)) {
510 // variant 2, the 2nd instruction contains the destination address:
511 //
512 // b!cxx SKIP
513 // bxx DEST
514 // SKIP:
515 //
516 const int instruction_1 = *(int*)(instruction_addr);
517 boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(instruction_1))),
518 opposite_bcond(inv_boint_bcond(inv_bo_field(instruction_1))));
519 biint = inv_bi_field(instruction_1);
520 } else {
521 // variant 4???
522 ShouldNotReachHere();
523 }
524
525 // second, set the new branch destination and optimize the code
526 if (dest != instruction_addr + 4 && // the bc_far is still unbound!
527 masm.is_within_range_of_bcxx(dest, instruction_addr)) {
528 // variant 1:
529 //
530 // bcxx DEST
531 // nop
532 //
533 masm.bc(boint, biint, dest);
534 masm.nop();
535 } else {
536 // variant 2:
537 //
538 // b!cxx SKIP
539 // bxx DEST
540 // SKIP:
541 //
542 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
543 opposite_bcond(inv_boint_bcond(boint)));
544 const address not_taken_pc = masm.pc() + 2 * BytesPerInstWord;
545 masm.bc(opposite_boint, biint, not_taken_pc);
546 masm.b(dest);
547 }
548 }
549 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
550 }
551
552 // Emit a NOT mt-safe patchable 64 bit absolute call/jump.
553 void MacroAssembler::bxx64_patchable(address dest, relocInfo::relocType rt, bool link) {
554 // get current pc
555 uint64_t start_pc = (uint64_t) pc();
556
557 const address pc_of_bl = (address) (start_pc + (6*BytesPerInstWord)); // bl is last
558 const address pc_of_b = (address) (start_pc + (0*BytesPerInstWord)); // b is first
559
560 // relocate here
561 if (rt != relocInfo::none) {
562 relocate(rt);
563 }
564
565 if ( ReoptimizeCallSequences &&
566 (( link && is_within_range_of_b(dest, pc_of_bl)) ||
567 (!link && is_within_range_of_b(dest, pc_of_b)))) {
568 // variant 2:
569 // Emit an optimized, pc-relative call/jump.
570
571 if (link) {
572 // some padding
573 nop();
574 nop();
575 nop();
576 nop();
577 nop();
578 nop();
579
580 // do the call
581 assert(pc() == pc_of_bl, "just checking");
582 bl(dest, relocInfo::none);
583 } else {
584 // do the jump
585 assert(pc() == pc_of_b, "just checking");
586 b(dest, relocInfo::none);
587
588 // some padding
589 nop();
590 nop();
591 nop();
592 nop();
593 nop();
594 nop();
595 }
596
597 // Assert that we can identify the emitted call/jump.
598 assert(is_bxx64_patchable_variant2_at((address)start_pc, link),
599 "can't identify emitted call");
600 } else {
601 // variant 1:
602 mr(R0, R11); // spill R11 -> R0.
603
604 // Load the destination address into CTR,
605 // calculate destination relative to global toc.
606 calculate_address_from_global_toc(R11, dest, true, true, false);
607
608 mtctr(R11);
609 mr(R11, R0); // spill R11 <- R0.
610 nop();
611
612 // do the call/jump
613 if (link) {
614 bctrl();
615 } else{
616 bctr();
617 }
618 // Assert that we can identify the emitted call/jump.
619 assert(is_bxx64_patchable_variant1b_at((address)start_pc, link),
620 "can't identify emitted call");
621 }
622
623 // Assert that we can identify the emitted call/jump.
624 assert(is_bxx64_patchable_at((address)start_pc, link),
625 "can't identify emitted call");
626 assert(get_dest_of_bxx64_patchable_at((address)start_pc, link) == dest,
627 "wrong encoding of dest address");
628 }
629
630 // Identify a bxx64_patchable instruction.
631 bool MacroAssembler::is_bxx64_patchable_at(address instruction_addr, bool link) {
632 return is_bxx64_patchable_variant1b_at(instruction_addr, link)
633 //|| is_bxx64_patchable_variant1_at(instruction_addr, link)
634 || is_bxx64_patchable_variant2_at(instruction_addr, link);
635 }
636
637 // Does the call64_patchable instruction use a pc-relative encoding of
638 // the call destination?
639 bool MacroAssembler::is_bxx64_patchable_pcrelative_at(address instruction_addr, bool link) {
640 // variant 2 is pc-relative
641 return is_bxx64_patchable_variant2_at(instruction_addr, link);
642 }
643
644 // Identify variant 1.
645 bool MacroAssembler::is_bxx64_patchable_variant1_at(address instruction_addr, bool link) {
646 unsigned int* instr = (unsigned int*) instruction_addr;
647 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
648 && is_mtctr(instr[5]) // mtctr
649 && is_load_const_at(instruction_addr);
650 }
651
652 // Identify variant 1b: load destination relative to global toc.
653 bool MacroAssembler::is_bxx64_patchable_variant1b_at(address instruction_addr, bool link) {
654 unsigned int* instr = (unsigned int*) instruction_addr;
655 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
656 && is_mtctr(instr[3]) // mtctr
657 && is_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, instruction_addr);
658 }
659
660 // Identify variant 2.
661 bool MacroAssembler::is_bxx64_patchable_variant2_at(address instruction_addr, bool link) {
662 unsigned int* instr = (unsigned int*) instruction_addr;
663 if (link) {
664 return is_bl (instr[6]) // bl dest is last
665 && is_nop(instr[0]) // nop
666 && is_nop(instr[1]) // nop
667 && is_nop(instr[2]) // nop
668 && is_nop(instr[3]) // nop
669 && is_nop(instr[4]) // nop
670 && is_nop(instr[5]); // nop
671 } else {
672 return is_b (instr[0]) // b dest is first
673 && is_nop(instr[1]) // nop
674 && is_nop(instr[2]) // nop
675 && is_nop(instr[3]) // nop
676 && is_nop(instr[4]) // nop
677 && is_nop(instr[5]) // nop
678 && is_nop(instr[6]); // nop
679 }
680 }
681
682 // Set dest address of a bxx64_patchable instruction.
683 void MacroAssembler::set_dest_of_bxx64_patchable_at(address instruction_addr, address dest, bool link) {
684 ResourceMark rm;
685 int code_size = MacroAssembler::bxx64_patchable_size;
686 CodeBuffer buf(instruction_addr, code_size);
687 MacroAssembler masm(&buf);
688 masm.bxx64_patchable(dest, relocInfo::none, link);
689 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
690 }
691
692 // Get dest address of a bxx64_patchable instruction.
693 address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, bool link) {
694 if (is_bxx64_patchable_variant1_at(instruction_addr, link)) {
695 return (address) (unsigned long) get_const(instruction_addr);
696 } else if (is_bxx64_patchable_variant2_at(instruction_addr, link)) {
697 unsigned int* instr = (unsigned int*) instruction_addr;
698 if (link) {
699 const int instr_idx = 6; // bl is last
700 int branchoffset = branch_destination(instr[instr_idx], 0);
701 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
702 } else {
703 const int instr_idx = 0; // b is first
704 int branchoffset = branch_destination(instr[instr_idx], 0);
705 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
706 }
707 // Load dest relative to global toc.
708 } else if (is_bxx64_patchable_variant1b_at(instruction_addr, link)) {
709 return get_address_of_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord,
710 instruction_addr);
711 } else {
712 ShouldNotReachHere();
713 return nullptr;
714 }
715 }
716
717 void MacroAssembler::clobber_volatile_gprs(Register excluded_register) {
718 const int magic_number = 0x42;
719
720 // Preserve stack pointer register (R1_SP) and system thread id register (R13);
721 // although they're technically volatile
722 for (int i = 2; i < 13; i++) {
723 Register reg = as_Register(i);
724 if (reg == excluded_register) {
725 continue;
726 }
727
728 li(reg, magic_number);
729 }
730 }
731
732 void MacroAssembler::clobber_carg_stack_slots(Register tmp) {
733 const int magic_number = 0x43;
734
735 li(tmp, magic_number);
736 for (int m = 0; m <= 7; m++) {
737 std(tmp, frame::native_abi_minframe_size + m * 8, R1_SP);
738 }
739 }
740
741 // Uses ordering which corresponds to ABI:
742 // _savegpr0_14: std r14,-144(r1)
743 // _savegpr0_15: std r15,-136(r1)
744 // _savegpr0_16: std r16,-128(r1)
745 void MacroAssembler::save_nonvolatile_gprs(Register dst, int offset) {
746 std(R14, offset, dst); offset += 8;
747 std(R15, offset, dst); offset += 8;
748 std(R16, offset, dst); offset += 8;
749 std(R17, offset, dst); offset += 8;
750 std(R18, offset, dst); offset += 8;
751 std(R19, offset, dst); offset += 8;
752 std(R20, offset, dst); offset += 8;
753 std(R21, offset, dst); offset += 8;
754 std(R22, offset, dst); offset += 8;
755 std(R23, offset, dst); offset += 8;
756 std(R24, offset, dst); offset += 8;
757 std(R25, offset, dst); offset += 8;
758 std(R26, offset, dst); offset += 8;
759 std(R27, offset, dst); offset += 8;
760 std(R28, offset, dst); offset += 8;
761 std(R29, offset, dst); offset += 8;
762 std(R30, offset, dst); offset += 8;
763 std(R31, offset, dst); offset += 8;
764
765 stfd(F14, offset, dst); offset += 8;
766 stfd(F15, offset, dst); offset += 8;
767 stfd(F16, offset, dst); offset += 8;
768 stfd(F17, offset, dst); offset += 8;
769 stfd(F18, offset, dst); offset += 8;
770 stfd(F19, offset, dst); offset += 8;
771 stfd(F20, offset, dst); offset += 8;
772 stfd(F21, offset, dst); offset += 8;
773 stfd(F22, offset, dst); offset += 8;
774 stfd(F23, offset, dst); offset += 8;
775 stfd(F24, offset, dst); offset += 8;
776 stfd(F25, offset, dst); offset += 8;
777 stfd(F26, offset, dst); offset += 8;
778 stfd(F27, offset, dst); offset += 8;
779 stfd(F28, offset, dst); offset += 8;
780 stfd(F29, offset, dst); offset += 8;
781 stfd(F30, offset, dst); offset += 8;
782 stfd(F31, offset, dst);
783 }
784
785 // Uses ordering which corresponds to ABI:
786 // _restgpr0_14: ld r14,-144(r1)
787 // _restgpr0_15: ld r15,-136(r1)
788 // _restgpr0_16: ld r16,-128(r1)
789 void MacroAssembler::restore_nonvolatile_gprs(Register src, int offset) {
790 ld(R14, offset, src); offset += 8;
791 ld(R15, offset, src); offset += 8;
792 ld(R16, offset, src); offset += 8;
793 ld(R17, offset, src); offset += 8;
794 ld(R18, offset, src); offset += 8;
795 ld(R19, offset, src); offset += 8;
796 ld(R20, offset, src); offset += 8;
797 ld(R21, offset, src); offset += 8;
798 ld(R22, offset, src); offset += 8;
799 ld(R23, offset, src); offset += 8;
800 ld(R24, offset, src); offset += 8;
801 ld(R25, offset, src); offset += 8;
802 ld(R26, offset, src); offset += 8;
803 ld(R27, offset, src); offset += 8;
804 ld(R28, offset, src); offset += 8;
805 ld(R29, offset, src); offset += 8;
806 ld(R30, offset, src); offset += 8;
807 ld(R31, offset, src); offset += 8;
808
809 // FP registers
810 lfd(F14, offset, src); offset += 8;
811 lfd(F15, offset, src); offset += 8;
812 lfd(F16, offset, src); offset += 8;
813 lfd(F17, offset, src); offset += 8;
814 lfd(F18, offset, src); offset += 8;
815 lfd(F19, offset, src); offset += 8;
816 lfd(F20, offset, src); offset += 8;
817 lfd(F21, offset, src); offset += 8;
818 lfd(F22, offset, src); offset += 8;
819 lfd(F23, offset, src); offset += 8;
820 lfd(F24, offset, src); offset += 8;
821 lfd(F25, offset, src); offset += 8;
822 lfd(F26, offset, src); offset += 8;
823 lfd(F27, offset, src); offset += 8;
824 lfd(F28, offset, src); offset += 8;
825 lfd(F29, offset, src); offset += 8;
826 lfd(F30, offset, src); offset += 8;
827 lfd(F31, offset, src);
828 }
829
830 // For verify_oops.
831 void MacroAssembler::save_volatile_gprs(Register dst, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
832 std(R2, offset, dst); offset += 8;
833 if (include_R3_RET_reg) {
834 std(R3, offset, dst); offset += 8;
835 }
836 std(R4, offset, dst); offset += 8;
837 std(R5, offset, dst); offset += 8;
838 std(R6, offset, dst); offset += 8;
839 std(R7, offset, dst); offset += 8;
840 std(R8, offset, dst); offset += 8;
841 std(R9, offset, dst); offset += 8;
842 std(R10, offset, dst); offset += 8;
843 std(R11, offset, dst); offset += 8;
844 std(R12, offset, dst); offset += 8;
845
846 if (include_fp_regs) {
847 stfd(F0, offset, dst); offset += 8;
848 stfd(F1, offset, dst); offset += 8;
849 stfd(F2, offset, dst); offset += 8;
850 stfd(F3, offset, dst); offset += 8;
851 stfd(F4, offset, dst); offset += 8;
852 stfd(F5, offset, dst); offset += 8;
853 stfd(F6, offset, dst); offset += 8;
854 stfd(F7, offset, dst); offset += 8;
855 stfd(F8, offset, dst); offset += 8;
856 stfd(F9, offset, dst); offset += 8;
857 stfd(F10, offset, dst); offset += 8;
858 stfd(F11, offset, dst); offset += 8;
859 stfd(F12, offset, dst); offset += 8;
860 stfd(F13, offset, dst);
861 }
862 }
863
864 // For verify_oops.
865 void MacroAssembler::restore_volatile_gprs(Register src, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
866 ld(R2, offset, src); offset += 8;
867 if (include_R3_RET_reg) {
868 ld(R3, offset, src); offset += 8;
869 }
870 ld(R4, offset, src); offset += 8;
871 ld(R5, offset, src); offset += 8;
872 ld(R6, offset, src); offset += 8;
873 ld(R7, offset, src); offset += 8;
874 ld(R8, offset, src); offset += 8;
875 ld(R9, offset, src); offset += 8;
876 ld(R10, offset, src); offset += 8;
877 ld(R11, offset, src); offset += 8;
878 ld(R12, offset, src); offset += 8;
879
880 if (include_fp_regs) {
881 lfd(F0, offset, src); offset += 8;
882 lfd(F1, offset, src); offset += 8;
883 lfd(F2, offset, src); offset += 8;
884 lfd(F3, offset, src); offset += 8;
885 lfd(F4, offset, src); offset += 8;
886 lfd(F5, offset, src); offset += 8;
887 lfd(F6, offset, src); offset += 8;
888 lfd(F7, offset, src); offset += 8;
889 lfd(F8, offset, src); offset += 8;
890 lfd(F9, offset, src); offset += 8;
891 lfd(F10, offset, src); offset += 8;
892 lfd(F11, offset, src); offset += 8;
893 lfd(F12, offset, src); offset += 8;
894 lfd(F13, offset, src);
895 }
896 }
897
898 void MacroAssembler::save_LR_CR(Register tmp) {
899 mfcr(tmp);
900 std(tmp, _abi0(cr), R1_SP);
901 mflr(tmp);
902 std(tmp, _abi0(lr), R1_SP);
903 // Tmp must contain lr on exit! (see return_addr and prolog in ppc64.ad)
904 }
905
906 void MacroAssembler::restore_LR_CR(Register tmp) {
907 assert(tmp != R1_SP, "must be distinct");
908 ld(tmp, _abi0(lr), R1_SP);
909 mtlr(tmp);
910 ld(tmp, _abi0(cr), R1_SP);
911 mtcr(tmp);
912 }
913
914 address MacroAssembler::get_PC_trash_LR(Register result) {
915 Label L;
916 bl(L);
917 bind(L);
918 address lr_pc = pc();
919 mflr(result);
920 return lr_pc;
921 }
922
923 void MacroAssembler::resize_frame(Register offset, Register tmp) {
924 #ifdef ASSERT
925 assert_different_registers(offset, tmp, R1_SP);
926 andi_(tmp, offset, frame::alignment_in_bytes-1);
927 asm_assert_eq("resize_frame: unaligned");
928 #endif
929
930 // tmp <- *(SP)
931 ld(tmp, _abi0(callers_sp), R1_SP);
932 // addr <- SP + offset;
933 // *(addr) <- tmp;
934 // SP <- addr
935 stdux(tmp, R1_SP, offset);
936 }
937
938 void MacroAssembler::resize_frame(int offset, Register tmp) {
939 assert(is_simm(offset, 16), "too big an offset");
940 assert_different_registers(tmp, R1_SP);
941 assert((offset & (frame::alignment_in_bytes-1))==0, "resize_frame: unaligned");
942 // tmp <- *(SP)
943 ld(tmp, _abi0(callers_sp), R1_SP);
944 // addr <- SP + offset;
945 // *(addr) <- tmp;
946 // SP <- addr
947 stdu(tmp, offset, R1_SP);
948 }
949
950 void MacroAssembler::resize_frame_absolute(Register addr, Register tmp1, Register tmp2) {
951 // (addr == tmp1) || (addr == tmp2) is allowed here!
952 assert(tmp1 != tmp2, "must be distinct");
953
954 // compute offset w.r.t. current stack pointer
955 // tmp_1 <- addr - SP (!)
956 subf(tmp1, R1_SP, addr);
957
958 // atomically update SP keeping back link.
959 resize_frame(tmp1/* offset */, tmp2/* tmp */);
960 }
961
962 void MacroAssembler::push_frame(Register bytes, Register tmp) {
963 #ifdef ASSERT
964 assert(bytes != R0, "r0 not allowed here");
965 andi_(R0, bytes, frame::alignment_in_bytes-1);
966 asm_assert_eq("push_frame(Reg, Reg): unaligned");
967 #endif
968 neg(tmp, bytes);
969 stdux(R1_SP, R1_SP, tmp);
970 }
971
972 // Push a frame of size `bytes'.
973 void MacroAssembler::push_frame(unsigned int bytes, Register tmp) {
974 long offset = align_addr(bytes, frame::alignment_in_bytes);
975 if (is_simm(-offset, 16)) {
976 stdu(R1_SP, -offset, R1_SP);
977 } else {
978 load_const_optimized(tmp, -offset);
979 stdux(R1_SP, R1_SP, tmp);
980 }
981 }
982
983 // Push a frame of size `bytes' plus native_abi_reg_args on top.
984 void MacroAssembler::push_frame_reg_args(unsigned int bytes, Register tmp) {
985 push_frame(bytes + frame::native_abi_reg_args_size, tmp);
986 }
987
988 // Setup up a new C frame with a spill area for non-volatile GPRs and
989 // additional space for local variables.
990 void MacroAssembler::push_frame_reg_args_nonvolatiles(unsigned int bytes,
991 Register tmp) {
992 push_frame(bytes + frame::native_abi_reg_args_size + frame::spill_nonvolatiles_size, tmp);
993 }
994
995 // Pop current C frame.
996 void MacroAssembler::pop_frame() {
997 ld(R1_SP, _abi0(callers_sp), R1_SP);
998 }
999
1000 #if defined(ABI_ELFv2)
1001 address MacroAssembler::branch_to(Register r_function_entry, bool and_link) {
1002 // TODO(asmundak): make sure the caller uses R12 as function descriptor
1003 // most of the times.
1004 if (R12 != r_function_entry) {
1005 mr(R12, r_function_entry);
1006 }
1007 mtctr(R12);
1008 // Do a call or a branch.
1009 if (and_link) {
1010 bctrl();
1011 } else {
1012 bctr();
1013 }
1014 _last_calls_return_pc = pc();
1015
1016 return _last_calls_return_pc;
1017 }
1018
1019 // Call a C function via a function descriptor and use full C
1020 // calling conventions. Updates and returns _last_calls_return_pc.
1021 address MacroAssembler::call_c(Register r_function_entry) {
1022 return branch_to(r_function_entry, /*and_link=*/true);
1023 }
1024
1025 // For tail calls: only branch, don't link, so callee returns to caller of this function.
1026 address MacroAssembler::call_c_and_return_to_caller(Register r_function_entry) {
1027 return branch_to(r_function_entry, /*and_link=*/false);
1028 }
1029
1030 address MacroAssembler::call_c(address function_entry, relocInfo::relocType rt) {
1031 load_const(R12, function_entry, R0);
1032 return branch_to(R12, /*and_link=*/true);
1033 }
1034
1035 #else
1036 // Generic version of a call to C function via a function descriptor
1037 // with variable support for C calling conventions (TOC, ENV, etc.).
1038 // Updates and returns _last_calls_return_pc.
1039 address MacroAssembler::branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call,
1040 bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee) {
1041 // we emit standard ptrgl glue code here
1042 assert((function_descriptor != R0), "function_descriptor cannot be R0");
1043
1044 // retrieve necessary entries from the function descriptor
1045 ld(R0, in_bytes(FunctionDescriptor::entry_offset()), function_descriptor);
1046 mtctr(R0);
1047
1048 if (load_toc_of_callee) {
1049 ld(R2_TOC, in_bytes(FunctionDescriptor::toc_offset()), function_descriptor);
1050 }
1051 if (load_env_of_callee) {
1052 ld(R11, in_bytes(FunctionDescriptor::env_offset()), function_descriptor);
1053 } else if (load_toc_of_callee) {
1054 li(R11, 0);
1055 }
1056
1057 // do a call or a branch
1058 if (and_link) {
1059 bctrl();
1060 } else {
1061 bctr();
1062 }
1063 _last_calls_return_pc = pc();
1064
1065 return _last_calls_return_pc;
1066 }
1067
1068 // Call a C function via a function descriptor and use full C calling
1069 // conventions.
1070 // We don't use the TOC in generated code, so there is no need to save
1071 // and restore its value.
1072 address MacroAssembler::call_c(Register fd) {
1073 return branch_to(fd, /*and_link=*/true,
1074 /*save toc=*/false,
1075 /*restore toc=*/false,
1076 /*load toc=*/true,
1077 /*load env=*/true);
1078 }
1079
1080 address MacroAssembler::call_c_and_return_to_caller(Register fd) {
1081 return branch_to(fd, /*and_link=*/false,
1082 /*save toc=*/false,
1083 /*restore toc=*/false,
1084 /*load toc=*/true,
1085 /*load env=*/true);
1086 }
1087
1088 address MacroAssembler::call_c(const FunctionDescriptor* fd, relocInfo::relocType rt) {
1089 if (rt != relocInfo::none) {
1090 // this call needs to be relocatable
1091 if (!ReoptimizeCallSequences
1092 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1093 || fd == nullptr // support code-size estimation
1094 || !fd->is_friend_function()
1095 || fd->entry() == nullptr) {
1096 // it's not a friend function as defined by class FunctionDescriptor,
1097 // so do a full call-c here.
1098 load_const(R11, (address)fd, R0);
1099
1100 bool has_env = (fd != nullptr && fd->env() != nullptr);
1101 return branch_to(R11, /*and_link=*/true,
1102 /*save toc=*/false,
1103 /*restore toc=*/false,
1104 /*load toc=*/true,
1105 /*load env=*/has_env);
1106 } else {
1107 // It's a friend function. Load the entry point and don't care about
1108 // toc and env. Use an optimizable call instruction, but ensure the
1109 // same code-size as in the case of a non-friend function.
1110 nop();
1111 nop();
1112 nop();
1113 bl64_patchable(fd->entry(), rt);
1114 _last_calls_return_pc = pc();
1115 return _last_calls_return_pc;
1116 }
1117 } else {
1118 // This call does not need to be relocatable, do more aggressive
1119 // optimizations.
1120 if (!ReoptimizeCallSequences
1121 || !fd->is_friend_function()) {
1122 // It's not a friend function as defined by class FunctionDescriptor,
1123 // so do a full call-c here.
1124 load_const(R11, (address)fd, R0);
1125 return branch_to(R11, /*and_link=*/true,
1126 /*save toc=*/false,
1127 /*restore toc=*/false,
1128 /*load toc=*/true,
1129 /*load env=*/true);
1130 } else {
1131 // it's a friend function, load the entry point and don't care about
1132 // toc and env.
1133 address dest = fd->entry();
1134 if (is_within_range_of_b(dest, pc())) {
1135 bl(dest);
1136 } else {
1137 bl64_patchable(dest, rt);
1138 }
1139 _last_calls_return_pc = pc();
1140 return _last_calls_return_pc;
1141 }
1142 }
1143 }
1144
1145 // Call a C function. All constants needed reside in TOC.
1146 //
1147 // Read the address to call from the TOC.
1148 // Read env from TOC, if fd specifies an env.
1149 // Read new TOC from TOC.
1150 address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd,
1151 relocInfo::relocType rt, Register toc) {
1152 if (!ReoptimizeCallSequences
1153 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1154 || !fd->is_friend_function()) {
1155 // It's not a friend function as defined by class FunctionDescriptor,
1156 // so do a full call-c here.
1157 assert(fd->entry() != nullptr, "function must be linked");
1158
1159 AddressLiteral fd_entry(fd->entry());
1160 bool success = load_const_from_method_toc(R11, fd_entry, toc, /*fixed_size*/ true);
1161 mtctr(R11);
1162 if (fd->env() == nullptr) {
1163 li(R11, 0);
1164 nop();
1165 } else {
1166 AddressLiteral fd_env(fd->env());
1167 success = success && load_const_from_method_toc(R11, fd_env, toc, /*fixed_size*/ true);
1168 }
1169 AddressLiteral fd_toc(fd->toc());
1170 // Set R2_TOC (load from toc)
1171 success = success && load_const_from_method_toc(R2_TOC, fd_toc, toc, /*fixed_size*/ true);
1172 bctrl();
1173 _last_calls_return_pc = pc();
1174 if (!success) { return nullptr; }
1175 } else {
1176 // It's a friend function, load the entry point and don't care about
1177 // toc and env. Use an optimizable call instruction, but ensure the
1178 // same code-size as in the case of a non-friend function.
1179 nop();
1180 bl64_patchable(fd->entry(), rt);
1181 _last_calls_return_pc = pc();
1182 }
1183 return _last_calls_return_pc;
1184 }
1185 #endif // ABI_ELFv2
1186
1187 void MacroAssembler::post_call_nop() {
1188 // Make inline again when loom is always enabled.
1189 if (!Continuations::enabled()) {
1190 return;
1191 }
1192 // We use CMPI/CMPLI instructions to encode post call nops.
1193 // Refer to NativePostCallNop for details.
1194 relocate(post_call_nop_Relocation::spec());
1195 InlineSkippedInstructionsCounter skipCounter(this);
1196 Assembler::emit_int32(Assembler::CMPLI_OPCODE | Assembler::opp_u_field(1, 9, 9));
1197 assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found");
1198 }
1199
1200 int MacroAssembler::ic_check_size() {
1201 bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
1202 use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
1203 use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
1204
1205 int num_ins;
1206 if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
1207 num_ins = 3;
1208 if (use_trap_based_null_check) num_ins += 1;
1209 } else {
1210 num_ins = 7;
1211 if (!implicit_null_checks_available) num_ins += 2;
1212 }
1213 return num_ins * BytesPerInstWord;
1214 }
1215
1216 int MacroAssembler::ic_check(int end_alignment) {
1217 bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
1218 use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
1219 use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
1220
1221 Register receiver = R3_ARG1;
1222 Register data = R19_inline_cache_reg;
1223 Register tmp1 = R11_scratch1;
1224 Register tmp2 = R12_scratch2;
1225
1226 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
1227 // before the inline cache check, so we don't have to execute any nop instructions when dispatching
1228 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
1229 // before the inline cache check here, and not after
1230 align(end_alignment, end_alignment, end_alignment - ic_check_size());
1231
1232 int uep_offset = offset();
1233
1234 if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
1235 // Fast version which uses SIGTRAP
1236
1237 if (use_trap_based_null_check) {
1238 trap_null_check(receiver);
1239 }
1240 if (UseCompressedClassPointers) {
1241 lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1242 } else {
1243 ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1244 }
1245 ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
1246 trap_ic_miss_check(tmp1, tmp2);
1247
1248 } else {
1249 // Slower version which doesn't use SIGTRAP
1250
1251 // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
1252 calculate_address_from_global_toc(tmp1, SharedRuntime::get_ic_miss_stub(),
1253 true, true, false); // 2 instructions
1254 mtctr(tmp1);
1255
1256 if (!implicit_null_checks_available) {
1257 cmpdi(CCR0, receiver, 0);
1258 beqctr(CCR0);
1259 }
1260 if (UseCompressedClassPointers) {
1261 lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1262 } else {
1263 ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1264 }
1265 ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
1266 cmpd(CCR0, tmp1, tmp2);
1267 bnectr(CCR0);
1268 }
1269
1270 assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
1271
1272 return uep_offset;
1273 }
1274
1275 void MacroAssembler::call_VM_base(Register oop_result,
1276 Register last_java_sp,
1277 address entry_point,
1278 bool check_exceptions) {
1279 BLOCK_COMMENT("call_VM {");
1280 // Determine last_java_sp register.
1281 if (!last_java_sp->is_valid()) {
1282 last_java_sp = R1_SP;
1283 }
1284 set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1);
1285
1286 // ARG1 must hold thread address.
1287 mr(R3_ARG1, R16_thread);
1288 #if defined(ABI_ELFv2)
1289 address return_pc = call_c(entry_point, relocInfo::none);
1290 #else
1291 address return_pc = call_c((FunctionDescriptor*)entry_point, relocInfo::none);
1292 #endif
1293
1294 reset_last_Java_frame();
1295
1296 // Check for pending exceptions.
1297 if (check_exceptions) {
1298 // We don't check for exceptions here.
1299 ShouldNotReachHere();
1300 }
1301
1302 // Get oop result if there is one and reset the value in the thread.
1303 if (oop_result->is_valid()) {
1304 get_vm_result(oop_result);
1305 }
1306
1307 _last_calls_return_pc = return_pc;
1308 BLOCK_COMMENT("} call_VM");
1309 }
1310
1311 void MacroAssembler::call_VM_leaf_base(address entry_point) {
1312 BLOCK_COMMENT("call_VM_leaf {");
1313 #if defined(ABI_ELFv2)
1314 call_c(entry_point, relocInfo::none);
1315 #else
1316 call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::none);
1317 #endif
1318 BLOCK_COMMENT("} call_VM_leaf");
1319 }
1320
1321 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
1322 call_VM_base(oop_result, noreg, entry_point, check_exceptions);
1323 }
1324
1325 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1,
1326 bool check_exceptions) {
1327 // R3_ARG1 is reserved for the thread.
1328 mr_if_needed(R4_ARG2, arg_1);
1329 call_VM(oop_result, entry_point, check_exceptions);
1330 }
1331
1332 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2,
1333 bool check_exceptions) {
1334 // R3_ARG1 is reserved for the thread
1335 assert_different_registers(arg_2, R4_ARG2);
1336 mr_if_needed(R4_ARG2, arg_1);
1337 mr_if_needed(R5_ARG3, arg_2);
1338 call_VM(oop_result, entry_point, check_exceptions);
1339 }
1340
1341 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3,
1342 bool check_exceptions) {
1343 // R3_ARG1 is reserved for the thread
1344 assert_different_registers(arg_2, R4_ARG2);
1345 assert_different_registers(arg_3, R4_ARG2, R5_ARG3);
1346 mr_if_needed(R4_ARG2, arg_1);
1347 mr_if_needed(R5_ARG3, arg_2);
1348 mr_if_needed(R6_ARG4, arg_3);
1349 call_VM(oop_result, entry_point, check_exceptions);
1350 }
1351
1352 void MacroAssembler::call_VM_leaf(address entry_point) {
1353 call_VM_leaf_base(entry_point);
1354 }
1355
1356 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
1357 mr_if_needed(R3_ARG1, arg_1);
1358 call_VM_leaf(entry_point);
1359 }
1360
1361 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
1362 assert_different_registers(arg_2, R3_ARG1);
1363 mr_if_needed(R3_ARG1, arg_1);
1364 mr_if_needed(R4_ARG2, arg_2);
1365 call_VM_leaf(entry_point);
1366 }
1367
1368 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
1369 assert_different_registers(arg_2, R3_ARG1);
1370 assert_different_registers(arg_3, R3_ARG1, R4_ARG2);
1371 mr_if_needed(R3_ARG1, arg_1);
1372 mr_if_needed(R4_ARG2, arg_2);
1373 mr_if_needed(R5_ARG3, arg_3);
1374 call_VM_leaf(entry_point);
1375 }
1376
1377 // Check whether instruction is a read access to the polling page
1378 // which was emitted by load_from_polling_page(..).
1379 bool MacroAssembler::is_load_from_polling_page(int instruction, void* ucontext,
1380 address* polling_address_ptr) {
1381 if (!is_ld(instruction))
1382 return false; // It's not a ld. Fail.
1383
1384 int rt = inv_rt_field(instruction);
1385 int ra = inv_ra_field(instruction);
1386 int ds = inv_ds_field(instruction);
1387 if (!(ds == 0 && ra != 0 && rt == 0)) {
1388 return false; // It's not a ld(r0, X, ra). Fail.
1389 }
1390
1391 if (!ucontext) {
1392 // Set polling address.
1393 if (polling_address_ptr != nullptr) {
1394 *polling_address_ptr = nullptr;
1395 }
1396 return true; // No ucontext given. Can't check value of ra. Assume true.
1397 }
1398
1399 #ifdef LINUX
1400 // Ucontext given. Check that register ra contains the address of
1401 // the safepoing polling page.
1402 ucontext_t* uc = (ucontext_t*) ucontext;
1403 // Set polling address.
1404 address addr = (address)uc->uc_mcontext.regs->gpr[ra] + (ssize_t)ds;
1405 if (polling_address_ptr != nullptr) {
1406 *polling_address_ptr = addr;
1407 }
1408 return SafepointMechanism::is_poll_address(addr);
1409 #else
1410 // Not on Linux, ucontext must be null.
1411 ShouldNotReachHere();
1412 return false;
1413 #endif
1414 }
1415
1416 void MacroAssembler::bang_stack_with_offset(int offset) {
1417 // When increasing the stack, the old stack pointer will be written
1418 // to the new top of stack according to the PPC64 abi.
1419 // Therefore, stack banging is not necessary when increasing
1420 // the stack by <= os::vm_page_size() bytes.
1421 // When increasing the stack by a larger amount, this method is
1422 // called repeatedly to bang the intermediate pages.
1423
1424 // Stack grows down, caller passes positive offset.
1425 assert(offset > 0, "must bang with positive offset");
1426
1427 long stdoffset = -offset;
1428
1429 if (is_simm(stdoffset, 16)) {
1430 // Signed 16 bit offset, a simple std is ok.
1431 if (UseLoadInstructionsForStackBangingPPC64) {
1432 ld(R0, (int)(signed short)stdoffset, R1_SP);
1433 } else {
1434 std(R0,(int)(signed short)stdoffset, R1_SP);
1435 }
1436 } else if (is_simm(stdoffset, 31)) {
1437 const int hi = MacroAssembler::largeoffset_si16_si16_hi(stdoffset);
1438 const int lo = MacroAssembler::largeoffset_si16_si16_lo(stdoffset);
1439
1440 Register tmp = R11;
1441 addis(tmp, R1_SP, hi);
1442 if (UseLoadInstructionsForStackBangingPPC64) {
1443 ld(R0, lo, tmp);
1444 } else {
1445 std(R0, lo, tmp);
1446 }
1447 } else {
1448 ShouldNotReachHere();
1449 }
1450 }
1451
1452 // If instruction is a stack bang of the form
1453 // std R0, x(Ry), (see bang_stack_with_offset())
1454 // stdu R1_SP, x(R1_SP), (see push_frame(), resize_frame())
1455 // or stdux R1_SP, Rx, R1_SP (see push_frame(), resize_frame())
1456 // return the banged address. Otherwise, return 0.
1457 address MacroAssembler::get_stack_bang_address(int instruction, void *ucontext) {
1458 #ifdef LINUX
1459 ucontext_t* uc = (ucontext_t*) ucontext;
1460 int rs = inv_rs_field(instruction);
1461 int ra = inv_ra_field(instruction);
1462 if ( (is_ld(instruction) && rs == 0 && UseLoadInstructionsForStackBangingPPC64)
1463 || (is_std(instruction) && rs == 0 && !UseLoadInstructionsForStackBangingPPC64)
1464 || (is_stdu(instruction) && rs == 1)) {
1465 int ds = inv_ds_field(instruction);
1466 // return banged address
1467 return ds+(address)uc->uc_mcontext.regs->gpr[ra];
1468 } else if (is_stdux(instruction) && rs == 1) {
1469 int rb = inv_rb_field(instruction);
1470 address sp = (address)uc->uc_mcontext.regs->gpr[1];
1471 long rb_val = (long)uc->uc_mcontext.regs->gpr[rb];
1472 return ra != 1 || rb_val >= 0 ? nullptr // not a stack bang
1473 : sp + rb_val; // banged address
1474 }
1475 return nullptr; // not a stack bang
1476 #else
1477 // workaround not needed on !LINUX :-)
1478 ShouldNotCallThis();
1479 return nullptr;
1480 #endif
1481 }
1482
1483 void MacroAssembler::reserved_stack_check(Register return_pc) {
1484 // Test if reserved zone needs to be enabled.
1485 Label no_reserved_zone_enabling;
1486
1487 ld_ptr(R0, JavaThread::reserved_stack_activation_offset(), R16_thread);
1488 cmpld(CCR0, R1_SP, R0);
1489 blt_predict_taken(CCR0, no_reserved_zone_enabling);
1490
1491 // Enable reserved zone again, throw stack overflow exception.
1492 push_frame_reg_args(0, R0);
1493 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), R16_thread);
1494 pop_frame();
1495 mtlr(return_pc);
1496 load_const_optimized(R0, StubRoutines::throw_delayed_StackOverflowError_entry());
1497 mtctr(R0);
1498 bctr();
1499
1500 should_not_reach_here();
1501
1502 bind(no_reserved_zone_enabling);
1503 }
1504
1505 void MacroAssembler::getandsetd(Register dest_current_value, Register exchange_value, Register addr_base,
1506 bool cmpxchgx_hint) {
1507 Label retry;
1508 bind(retry);
1509 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1510 stdcx_(exchange_value, addr_base);
1511 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1512 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1513 } else {
1514 bne( CCR0, retry); // StXcx_ sets CCR0.
1515 }
1516 }
1517
1518 void MacroAssembler::getandaddd(Register dest_current_value, Register inc_value, Register addr_base,
1519 Register tmp, bool cmpxchgx_hint) {
1520 Label retry;
1521 bind(retry);
1522 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1523 add(tmp, dest_current_value, inc_value);
1524 stdcx_(tmp, addr_base);
1525 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1526 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1527 } else {
1528 bne( CCR0, retry); // StXcx_ sets CCR0.
1529 }
1530 }
1531
1532 // Word/sub-word atomic helper functions
1533
1534 // Temps and addr_base are killed if size < 4 and processor does not support respective instructions.
1535 // Only signed types are supported with size < 4.
1536 // Atomic add always kills tmp1.
1537 void MacroAssembler::atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value,
1538 Register addr_base, Register tmp1, Register tmp2, Register tmp3,
1539 bool cmpxchgx_hint, bool is_add, int size) {
1540 // Sub-word instructions are available since Power 8.
1541 // For older processors, instruction_type != size holds, and we
1542 // emulate the sub-word instructions by constructing a 4-byte value
1543 // that leaves the other bytes unchanged.
1544 const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1545
1546 Label retry;
1547 Register shift_amount = noreg,
1548 val32 = dest_current_value,
1549 modval = is_add ? tmp1 : exchange_value;
1550
1551 if (instruction_type != size) {
1552 assert_different_registers(tmp1, tmp2, tmp3, dest_current_value, exchange_value, addr_base);
1553 modval = tmp1;
1554 shift_amount = tmp2;
1555 val32 = tmp3;
1556 // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1557 #ifdef VM_LITTLE_ENDIAN
1558 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1559 clrrdi(addr_base, addr_base, 2);
1560 #else
1561 xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1562 clrrdi(addr_base, addr_base, 2);
1563 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1564 #endif
1565 }
1566
1567 // atomic emulation loop
1568 bind(retry);
1569
1570 switch (instruction_type) {
1571 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1572 case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1573 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1574 default: ShouldNotReachHere();
1575 }
1576
1577 if (instruction_type != size) {
1578 srw(dest_current_value, val32, shift_amount);
1579 }
1580
1581 if (is_add) { add(modval, dest_current_value, exchange_value); }
1582
1583 if (instruction_type != size) {
1584 // Transform exchange value such that the replacement can be done by one xor instruction.
1585 xorr(modval, dest_current_value, is_add ? modval : exchange_value);
1586 clrldi(modval, modval, (size == 1) ? 56 : 48);
1587 slw(modval, modval, shift_amount);
1588 xorr(modval, val32, modval);
1589 }
1590
1591 switch (instruction_type) {
1592 case 4: stwcx_(modval, addr_base); break;
1593 case 2: sthcx_(modval, addr_base); break;
1594 case 1: stbcx_(modval, addr_base); break;
1595 default: ShouldNotReachHere();
1596 }
1597
1598 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1599 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1600 } else {
1601 bne( CCR0, retry); // StXcx_ sets CCR0.
1602 }
1603
1604 // l?arx zero-extends, but Java wants byte/short values sign-extended.
1605 if (size == 1) {
1606 extsb(dest_current_value, dest_current_value);
1607 } else if (size == 2) {
1608 extsh(dest_current_value, dest_current_value);
1609 };
1610 }
1611
1612 // Temps, addr_base and exchange_value are killed if size < 4 and processor does not support respective instructions.
1613 // Only signed types are supported with size < 4.
1614 void MacroAssembler::cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value,
1615 Register compare_value, Register exchange_value,
1616 Register addr_base, Register tmp1, Register tmp2,
1617 Label &retry, Label &failed, bool cmpxchgx_hint, int size) {
1618 // Sub-word instructions are available since Power 8.
1619 // For older processors, instruction_type != size holds, and we
1620 // emulate the sub-word instructions by constructing a 4-byte value
1621 // that leaves the other bytes unchanged.
1622 const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1623
1624 Register shift_amount = noreg,
1625 val32 = dest_current_value,
1626 modval = exchange_value;
1627
1628 if (instruction_type != size) {
1629 assert_different_registers(tmp1, tmp2, dest_current_value, compare_value, exchange_value, addr_base);
1630 shift_amount = tmp1;
1631 val32 = tmp2;
1632 modval = tmp2;
1633 // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1634 #ifdef VM_LITTLE_ENDIAN
1635 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1636 clrrdi(addr_base, addr_base, 2);
1637 #else
1638 xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1639 clrrdi(addr_base, addr_base, 2);
1640 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1641 #endif
1642 // Transform exchange value such that the replacement can be done by one xor instruction.
1643 xorr(exchange_value, compare_value, exchange_value);
1644 clrldi(exchange_value, exchange_value, (size == 1) ? 56 : 48);
1645 slw(exchange_value, exchange_value, shift_amount);
1646 }
1647
1648 // atomic emulation loop
1649 bind(retry);
1650
1651 switch (instruction_type) {
1652 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1653 case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1654 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1655 default: ShouldNotReachHere();
1656 }
1657
1658 if (instruction_type != size) {
1659 srw(dest_current_value, val32, shift_amount);
1660 }
1661 if (size == 1) {
1662 extsb(dest_current_value, dest_current_value);
1663 } else if (size == 2) {
1664 extsh(dest_current_value, dest_current_value);
1665 };
1666
1667 cmpw(flag, dest_current_value, compare_value);
1668 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1669 bne_predict_not_taken(flag, failed);
1670 } else {
1671 bne( flag, failed);
1672 }
1673 // branch to done => (flag == ne), (dest_current_value != compare_value)
1674 // fall through => (flag == eq), (dest_current_value == compare_value)
1675
1676 if (instruction_type != size) {
1677 xorr(modval, val32, exchange_value);
1678 }
1679
1680 switch (instruction_type) {
1681 case 4: stwcx_(modval, addr_base); break;
1682 case 2: sthcx_(modval, addr_base); break;
1683 case 1: stbcx_(modval, addr_base); break;
1684 default: ShouldNotReachHere();
1685 }
1686 }
1687
1688 // CmpxchgX sets condition register to cmpX(current, compare).
1689 void MacroAssembler::cmpxchg_generic(ConditionRegister flag, Register dest_current_value,
1690 Register compare_value, Register exchange_value,
1691 Register addr_base, Register tmp1, Register tmp2,
1692 int semantics, bool cmpxchgx_hint,
1693 Register int_flag_success, bool contention_hint, bool weak, int size) {
1694 Label retry;
1695 Label failed;
1696 Label done;
1697
1698 // Save one branch if result is returned via register and
1699 // result register is different from the other ones.
1700 bool use_result_reg = (int_flag_success != noreg);
1701 bool preset_result_reg = (int_flag_success != dest_current_value && int_flag_success != compare_value &&
1702 int_flag_success != exchange_value && int_flag_success != addr_base &&
1703 int_flag_success != tmp1 && int_flag_success != tmp2);
1704 assert(!weak || flag == CCR0, "weak only supported with CCR0");
1705 assert(size == 1 || size == 2 || size == 4, "unsupported");
1706
1707 if (use_result_reg && preset_result_reg) {
1708 li(int_flag_success, 0); // preset (assume cas failed)
1709 }
1710
1711 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1712 if (contention_hint) { // Don't try to reserve if cmp fails.
1713 switch (size) {
1714 case 1: lbz(dest_current_value, 0, addr_base); extsb(dest_current_value, dest_current_value); break;
1715 case 2: lha(dest_current_value, 0, addr_base); break;
1716 case 4: lwz(dest_current_value, 0, addr_base); break;
1717 default: ShouldNotReachHere();
1718 }
1719 cmpw(flag, dest_current_value, compare_value);
1720 bne(flag, failed);
1721 }
1722
1723 // release/fence semantics
1724 if (semantics & MemBarRel) {
1725 release();
1726 }
1727
1728 cmpxchg_loop_body(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
1729 retry, failed, cmpxchgx_hint, size);
1730 if (!weak || use_result_reg) {
1731 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1732 bne_predict_not_taken(CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1733 } else {
1734 bne( CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1735 }
1736 }
1737 // fall through => (flag == eq), (dest_current_value == compare_value), (swapped)
1738
1739 // Result in register (must do this at the end because int_flag_success can be the
1740 // same register as one above).
1741 if (use_result_reg) {
1742 li(int_flag_success, 1);
1743 }
1744
1745 if (semantics & MemBarFenceAfter) {
1746 fence();
1747 } else if (semantics & MemBarAcq) {
1748 isync();
1749 }
1750
1751 if (use_result_reg && !preset_result_reg) {
1752 b(done);
1753 }
1754
1755 bind(failed);
1756 if (use_result_reg && !preset_result_reg) {
1757 li(int_flag_success, 0);
1758 }
1759
1760 bind(done);
1761 // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1762 // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1763 }
1764
1765 // Performs atomic compare exchange:
1766 // if (compare_value == *addr_base)
1767 // *addr_base = exchange_value
1768 // int_flag_success = 1;
1769 // else
1770 // int_flag_success = 0;
1771 //
1772 // ConditionRegister flag = cmp(compare_value, *addr_base)
1773 // Register dest_current_value = *addr_base
1774 // Register compare_value Used to compare with value in memory
1775 // Register exchange_value Written to memory if compare_value == *addr_base
1776 // Register addr_base The memory location to compareXChange
1777 // Register int_flag_success Set to 1 if exchange_value was written to *addr_base
1778 //
1779 // To avoid the costly compare exchange the value is tested beforehand.
1780 // Several special cases exist to avoid that unnecessary information is generated.
1781 //
1782 void MacroAssembler::cmpxchgd(ConditionRegister flag,
1783 Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value,
1784 Register addr_base, int semantics, bool cmpxchgx_hint,
1785 Register int_flag_success, Label* failed_ext, bool contention_hint, bool weak) {
1786 Label retry;
1787 Label failed_int;
1788 Label& failed = (failed_ext != nullptr) ? *failed_ext : failed_int;
1789 Label done;
1790
1791 // Save one branch if result is returned via register and result register is different from the other ones.
1792 bool use_result_reg = (int_flag_success!=noreg);
1793 bool preset_result_reg = (int_flag_success!=dest_current_value && int_flag_success!=compare_value.register_or_noreg() &&
1794 int_flag_success!=exchange_value && int_flag_success!=addr_base);
1795 assert(!weak || flag == CCR0, "weak only supported with CCR0");
1796 assert(int_flag_success == noreg || failed_ext == nullptr, "cannot have both");
1797
1798 if (use_result_reg && preset_result_reg) {
1799 li(int_flag_success, 0); // preset (assume cas failed)
1800 }
1801
1802 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1803 if (contention_hint) { // Don't try to reserve if cmp fails.
1804 ld(dest_current_value, 0, addr_base);
1805 cmpd(flag, compare_value, dest_current_value);
1806 bne(flag, failed);
1807 }
1808
1809 // release/fence semantics
1810 if (semantics & MemBarRel) {
1811 release();
1812 }
1813
1814 // atomic emulation loop
1815 bind(retry);
1816
1817 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1818 cmpd(flag, compare_value, dest_current_value);
1819 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1820 bne_predict_not_taken(flag, failed);
1821 } else {
1822 bne( flag, failed);
1823 }
1824
1825 stdcx_(exchange_value, addr_base);
1826 if (!weak || use_result_reg || failed_ext) {
1827 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1828 bne_predict_not_taken(CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1829 } else {
1830 bne( CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1831 }
1832 }
1833
1834 // result in register (must do this at the end because int_flag_success can be the same register as one above)
1835 if (use_result_reg) {
1836 li(int_flag_success, 1);
1837 }
1838
1839 if (semantics & MemBarFenceAfter) {
1840 fence();
1841 } else if (semantics & MemBarAcq) {
1842 isync();
1843 }
1844
1845 if (use_result_reg && !preset_result_reg) {
1846 b(done);
1847 }
1848
1849 bind(failed_int);
1850 if (use_result_reg && !preset_result_reg) {
1851 li(int_flag_success, 0);
1852 }
1853
1854 bind(done);
1855 // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1856 // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1857 }
1858
1859 // Look up the method for a megamorphic invokeinterface call.
1860 // The target method is determined by <intf_klass, itable_index>.
1861 // The receiver klass is in recv_klass.
1862 // On success, the result will be in method_result, and execution falls through.
1863 // On failure, execution transfers to the given label.
1864 void MacroAssembler::lookup_interface_method(Register recv_klass,
1865 Register intf_klass,
1866 RegisterOrConstant itable_index,
1867 Register method_result,
1868 Register scan_temp,
1869 Register temp2,
1870 Label& L_no_such_interface,
1871 bool return_method) {
1872 assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
1873
1874 // Compute start of first itableOffsetEntry (which is at the end of the vtable).
1875 int vtable_base = in_bytes(Klass::vtable_start_offset());
1876 int itentry_off = in_bytes(itableMethodEntry::method_offset());
1877 int logMEsize = exact_log2(itableMethodEntry::size() * wordSize);
1878 int scan_step = itableOffsetEntry::size() * wordSize;
1879 int log_vte_size= exact_log2(vtableEntry::size_in_bytes());
1880
1881 lwz(scan_temp, in_bytes(Klass::vtable_length_offset()), recv_klass);
1882 // We should store the aligned, prescaled offset in the klass.
1883 // Then the next several instructions would fold away.
1884
1885 sldi(scan_temp, scan_temp, log_vte_size);
1886 addi(scan_temp, scan_temp, vtable_base);
1887 add(scan_temp, recv_klass, scan_temp);
1888
1889 // Adjust recv_klass by scaled itable_index, so we can free itable_index.
1890 if (return_method) {
1891 if (itable_index.is_register()) {
1892 Register itable_offset = itable_index.as_register();
1893 sldi(method_result, itable_offset, logMEsize);
1894 if (itentry_off) { addi(method_result, method_result, itentry_off); }
1895 add(method_result, method_result, recv_klass);
1896 } else {
1897 long itable_offset = (long)itable_index.as_constant();
1898 // static address, no relocation
1899 add_const_optimized(method_result, recv_klass, (itable_offset << logMEsize) + itentry_off, temp2);
1900 }
1901 }
1902
1903 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
1904 // if (scan->interface() == intf) {
1905 // result = (klass + scan->offset() + itable_index);
1906 // }
1907 // }
1908 Label search, found_method;
1909
1910 for (int peel = 1; peel >= 0; peel--) {
1911 // %%%% Could load both offset and interface in one ldx, if they were
1912 // in the opposite order. This would save a load.
1913 ld(temp2, in_bytes(itableOffsetEntry::interface_offset()), scan_temp);
1914
1915 // Check that this entry is non-null. A null entry means that
1916 // the receiver class doesn't implement the interface, and wasn't the
1917 // same as when the caller was compiled.
1918 cmpd(CCR0, temp2, intf_klass);
1919
1920 if (peel) {
1921 beq(CCR0, found_method);
1922 } else {
1923 bne(CCR0, search);
1924 // (invert the test to fall through to found_method...)
1925 }
1926
1927 if (!peel) break;
1928
1929 bind(search);
1930
1931 cmpdi(CCR0, temp2, 0);
1932 beq(CCR0, L_no_such_interface);
1933 addi(scan_temp, scan_temp, scan_step);
1934 }
1935
1936 bind(found_method);
1937
1938 // Got a hit.
1939 if (return_method) {
1940 int ito_offset = in_bytes(itableOffsetEntry::offset_offset());
1941 lwz(scan_temp, ito_offset, scan_temp);
1942 ldx(method_result, scan_temp, method_result);
1943 }
1944 }
1945
1946 // virtual method calling
1947 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1948 RegisterOrConstant vtable_index,
1949 Register method_result) {
1950
1951 assert_different_registers(recv_klass, method_result, vtable_index.register_or_noreg());
1952
1953 const ByteSize base = Klass::vtable_start_offset();
1954 assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1955
1956 if (vtable_index.is_register()) {
1957 sldi(vtable_index.as_register(), vtable_index.as_register(), LogBytesPerWord);
1958 add(recv_klass, vtable_index.as_register(), recv_klass);
1959 } else {
1960 addi(recv_klass, recv_klass, vtable_index.as_constant() << LogBytesPerWord);
1961 }
1962 ld(R19_method, in_bytes(base + vtableEntry::method_offset()), recv_klass);
1963 }
1964
1965 /////////////////////////////////////////// subtype checking ////////////////////////////////////////////
1966 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1967 Register super_klass,
1968 Register temp1_reg,
1969 Register temp2_reg,
1970 Label* L_success,
1971 Label* L_failure,
1972 Label* L_slow_path,
1973 RegisterOrConstant super_check_offset) {
1974
1975 const Register check_cache_offset = temp1_reg;
1976 const Register cached_super = temp2_reg;
1977
1978 assert_different_registers(sub_klass, super_klass, check_cache_offset, cached_super);
1979
1980 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1981 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1982
1983 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1984 bool need_slow_path = (must_load_sco || super_check_offset.constant_or_zero() == sco_offset);
1985
1986 Label L_fallthrough;
1987 int label_nulls = 0;
1988 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
1989 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
1990 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
1991 assert(label_nulls <= 1 ||
1992 (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
1993 "at most one null in the batch, usually");
1994
1995 // If the pointers are equal, we are done (e.g., String[] elements).
1996 // This self-check enables sharing of secondary supertype arrays among
1997 // non-primary types such as array-of-interface. Otherwise, each such
1998 // type would need its own customized SSA.
1999 // We move this check to the front of the fast path because many
2000 // type checks are in fact trivially successful in this manner,
2001 // so we get a nicely predicted branch right at the start of the check.
2002 cmpd(CCR0, sub_klass, super_klass);
2003 beq(CCR0, *L_success);
2004
2005 // Check the supertype display:
2006 if (must_load_sco) {
2007 // The super check offset is always positive...
2008 lwz(check_cache_offset, sco_offset, super_klass);
2009 super_check_offset = RegisterOrConstant(check_cache_offset);
2010 // super_check_offset is register.
2011 assert_different_registers(sub_klass, super_klass, cached_super, super_check_offset.as_register());
2012 }
2013 // The loaded value is the offset from Klass.
2014
2015 ld(cached_super, super_check_offset, sub_klass);
2016 cmpd(CCR0, cached_super, super_klass);
2017
2018 // This check has worked decisively for primary supers.
2019 // Secondary supers are sought in the super_cache ('super_cache_addr').
2020 // (Secondary supers are interfaces and very deeply nested subtypes.)
2021 // This works in the same check above because of a tricky aliasing
2022 // between the super_cache and the primary super display elements.
2023 // (The 'super_check_addr' can address either, as the case requires.)
2024 // Note that the cache is updated below if it does not help us find
2025 // what we need immediately.
2026 // So if it was a primary super, we can just fail immediately.
2027 // Otherwise, it's the slow path for us (no success at this point).
2028
2029 #define FINAL_JUMP(label) if (&(label) != &L_fallthrough) { b(label); }
2030
2031 if (super_check_offset.is_register()) {
2032 beq(CCR0, *L_success);
2033 cmpwi(CCR0, super_check_offset.as_register(), sc_offset);
2034 if (L_failure == &L_fallthrough) {
2035 beq(CCR0, *L_slow_path);
2036 } else {
2037 bne(CCR0, *L_failure);
2038 FINAL_JUMP(*L_slow_path);
2039 }
2040 } else {
2041 if (super_check_offset.as_constant() == sc_offset) {
2042 // Need a slow path; fast failure is impossible.
2043 if (L_slow_path == &L_fallthrough) {
2044 beq(CCR0, *L_success);
2045 } else {
2046 bne(CCR0, *L_slow_path);
2047 FINAL_JUMP(*L_success);
2048 }
2049 } else {
2050 // No slow path; it's a fast decision.
2051 if (L_failure == &L_fallthrough) {
2052 beq(CCR0, *L_success);
2053 } else {
2054 bne(CCR0, *L_failure);
2055 FINAL_JUMP(*L_success);
2056 }
2057 }
2058 }
2059
2060 bind(L_fallthrough);
2061 #undef FINAL_JUMP
2062 }
2063
2064 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
2065 Register super_klass,
2066 Register temp1_reg,
2067 Register temp2_reg,
2068 Label* L_success,
2069 Register result_reg) {
2070 const Register array_ptr = temp1_reg; // current value from cache array
2071 const Register temp = temp2_reg;
2072
2073 assert_different_registers(sub_klass, super_klass, array_ptr, temp);
2074
2075 int source_offset = in_bytes(Klass::secondary_supers_offset());
2076 int target_offset = in_bytes(Klass::secondary_super_cache_offset());
2077
2078 int length_offset = Array<Klass*>::length_offset_in_bytes();
2079 int base_offset = Array<Klass*>::base_offset_in_bytes();
2080
2081 Label hit, loop, failure, fallthru;
2082
2083 ld(array_ptr, source_offset, sub_klass);
2084
2085 // TODO: PPC port: assert(4 == arrayOopDesc::length_length_in_bytes(), "precondition violated.");
2086 lwz(temp, length_offset, array_ptr);
2087 cmpwi(CCR0, temp, 0);
2088 beq(CCR0, result_reg!=noreg ? failure : fallthru); // length 0
2089
2090 mtctr(temp); // load ctr
2091
2092 bind(loop);
2093 // Oops in table are NO MORE compressed.
2094 ld(temp, base_offset, array_ptr);
2095 cmpd(CCR0, temp, super_klass);
2096 beq(CCR0, hit);
2097 addi(array_ptr, array_ptr, BytesPerWord);
2098 bdnz(loop);
2099
2100 bind(failure);
2101 if (result_reg!=noreg) li(result_reg, 1); // load non-zero result (indicates a miss)
2102 b(fallthru);
2103
2104 bind(hit);
2105 std(super_klass, target_offset, sub_klass); // save result to cache
2106 if (result_reg != noreg) { li(result_reg, 0); } // load zero result (indicates a hit)
2107 if (L_success != nullptr) { b(*L_success); }
2108 else if (result_reg == noreg) { blr(); } // return with CR0.eq if neither label nor result reg provided
2109
2110 bind(fallthru);
2111 }
2112
2113 // Try fast path, then go to slow one if not successful
2114 void MacroAssembler::check_klass_subtype(Register sub_klass,
2115 Register super_klass,
2116 Register temp1_reg,
2117 Register temp2_reg,
2118 Label& L_success) {
2119 Label L_failure;
2120 check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success, &L_failure);
2121 check_klass_subtype_slow_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success);
2122 bind(L_failure); // Fallthru if not successful.
2123 }
2124
2125 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
2126 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
2127
2128 Label L_fallthrough;
2129 if (L_fast_path == nullptr) {
2130 L_fast_path = &L_fallthrough;
2131 } else if (L_slow_path == nullptr) {
2132 L_slow_path = &L_fallthrough;
2133 }
2134
2135 // Fast path check: class is fully initialized
2136 lbz(R0, in_bytes(InstanceKlass::init_state_offset()), klass);
2137 cmpwi(CCR0, R0, InstanceKlass::fully_initialized);
2138 beq(CCR0, *L_fast_path);
2139
2140 // Fast path check: current thread is initializer thread
2141 ld(R0, in_bytes(InstanceKlass::init_thread_offset()), klass);
2142 cmpd(CCR0, thread, R0);
2143 if (L_slow_path == &L_fallthrough) {
2144 beq(CCR0, *L_fast_path);
2145 } else if (L_fast_path == &L_fallthrough) {
2146 bne(CCR0, *L_slow_path);
2147 } else {
2148 Unimplemented();
2149 }
2150
2151 bind(L_fallthrough);
2152 }
2153
2154 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
2155 Register temp_reg,
2156 int extra_slot_offset) {
2157 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
2158 int stackElementSize = Interpreter::stackElementSize;
2159 int offset = extra_slot_offset * stackElementSize;
2160 if (arg_slot.is_constant()) {
2161 offset += arg_slot.as_constant() * stackElementSize;
2162 return offset;
2163 } else {
2164 assert(temp_reg != noreg, "must specify");
2165 sldi(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize));
2166 if (offset != 0)
2167 addi(temp_reg, temp_reg, offset);
2168 return temp_reg;
2169 }
2170 }
2171
2172 void MacroAssembler::tlab_allocate(
2173 Register obj, // result: pointer to object after successful allocation
2174 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
2175 int con_size_in_bytes, // object size in bytes if known at compile time
2176 Register t1, // temp register
2177 Label& slow_case // continuation point if fast allocation fails
2178 ) {
2179 // make sure arguments make sense
2180 assert_different_registers(obj, var_size_in_bytes, t1);
2181 assert(0 <= con_size_in_bytes && is_simm16(con_size_in_bytes), "illegal object size");
2182 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
2183
2184 const Register new_top = t1;
2185 //verify_tlab(); not implemented
2186
2187 ld(obj, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2188 ld(R0, in_bytes(JavaThread::tlab_end_offset()), R16_thread);
2189 if (var_size_in_bytes == noreg) {
2190 addi(new_top, obj, con_size_in_bytes);
2191 } else {
2192 add(new_top, obj, var_size_in_bytes);
2193 }
2194 cmpld(CCR0, new_top, R0);
2195 bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
2196
2197 #ifdef ASSERT
2198 // make sure new free pointer is properly aligned
2199 {
2200 Label L;
2201 andi_(R0, new_top, MinObjAlignmentInBytesMask);
2202 beq(CCR0, L);
2203 stop("updated TLAB free is not properly aligned");
2204 bind(L);
2205 }
2206 #endif // ASSERT
2207
2208 // update the tlab top pointer
2209 std(new_top, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2210 //verify_tlab(); not implemented
2211 }
2212 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2) {
2213 unimplemented("incr_allocated_bytes");
2214 }
2215
2216 address MacroAssembler::emit_trampoline_stub(int destination_toc_offset,
2217 int insts_call_instruction_offset, Register Rtoc) {
2218 // Start the stub.
2219 address stub = start_a_stub(64);
2220 if (stub == nullptr) { return nullptr; } // CodeCache full: bail out
2221
2222 // Create a trampoline stub relocation which relates this trampoline stub
2223 // with the call instruction at insts_call_instruction_offset in the
2224 // instructions code-section.
2225 relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset));
2226 const int stub_start_offset = offset();
2227
2228 // For java_to_interp stubs we use R11_scratch1 as scratch register
2229 // and in call trampoline stubs we use R12_scratch2. This way we
2230 // can distinguish them (see is_NativeCallTrampolineStub_at()).
2231 Register reg_scratch = R12_scratch2;
2232
2233 // Now, create the trampoline stub's code:
2234 // - load the TOC
2235 // - load the call target from the constant pool
2236 // - call
2237 if (Rtoc == noreg) {
2238 calculate_address_from_global_toc(reg_scratch, method_toc());
2239 Rtoc = reg_scratch;
2240 }
2241
2242 ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false);
2243 mtctr(reg_scratch);
2244 bctr();
2245
2246 const address stub_start_addr = addr_at(stub_start_offset);
2247
2248 // Assert that the encoded destination_toc_offset can be identified and that it is correct.
2249 assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(),
2250 "encoded offset into the constant pool must match");
2251 // Trampoline_stub_size should be good.
2252 assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size");
2253 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
2254
2255 // End the stub.
2256 end_a_stub();
2257 return stub;
2258 }
2259
2260 // "The box" is the space on the stack where we copy the object mark.
2261 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
2262 Register temp, Register displaced_header, Register current_header) {
2263 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight");
2264 assert_different_registers(oop, box, temp, displaced_header, current_header);
2265 Label object_has_monitor;
2266 Label cas_failed;
2267 Label success, failure;
2268
2269 // Load markWord from object into displaced_header.
2270 ld(displaced_header, oopDesc::mark_offset_in_bytes(), oop);
2271
2272 if (DiagnoseSyncOnValueBasedClasses != 0) {
2273 load_klass(temp, oop);
2274 lwz(temp, in_bytes(Klass::access_flags_offset()), temp);
2275 testbitdi(flag, R0, temp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
2276 bne(flag, failure);
2277 }
2278
2279 // Handle existing monitor.
2280 // The object has an existing monitor iff (mark & monitor_value) != 0.
2281 andi_(temp, displaced_header, markWord::monitor_value);
2282 bne(CCR0, object_has_monitor);
2283
2284 if (LockingMode == LM_MONITOR) {
2285 // Set NE to indicate 'failure' -> take slow-path.
2286 crandc(flag, Assembler::equal, flag, Assembler::equal);
2287 b(failure);
2288 } else {
2289 assert(LockingMode == LM_LEGACY, "must be");
2290 // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
2291 ori(displaced_header, displaced_header, markWord::unlocked_value);
2292
2293 // Load Compare Value application register.
2294
2295 // Initialize the box. (Must happen before we update the object mark!)
2296 std(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2297
2298 // Must fence, otherwise, preceding store(s) may float below cmpxchg.
2299 // Compare object markWord with mark and if equal exchange scratch1 with object markWord.
2300 cmpxchgd(/*flag=*/flag,
2301 /*current_value=*/current_header,
2302 /*compare_value=*/displaced_header,
2303 /*exchange_value=*/box,
2304 /*where=*/oop,
2305 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2306 MacroAssembler::cmpxchgx_hint_acquire_lock(),
2307 noreg,
2308 &cas_failed,
2309 /*check without membar and ldarx first*/true);
2310 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2311 // If the compare-and-exchange succeeded, then we found an unlocked
2312 // object and we have now locked it.
2313 b(success);
2314
2315 bind(cas_failed);
2316 // We did not see an unlocked object so try the fast recursive case.
2317
2318 // Check if the owner is self by comparing the value in the markWord of object
2319 // (current_header) with the stack pointer.
2320 sub(current_header, current_header, R1_SP);
2321 load_const_optimized(temp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
2322
2323 and_(R0/*==0?*/, current_header, temp);
2324 // If condition is true we are cont and hence we can store 0 as the
2325 // displaced header in the box, which indicates that it is a recursive lock.
2326 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2327
2328 if (flag != CCR0) {
2329 mcrf(flag, CCR0);
2330 }
2331 beq(CCR0, success);
2332 b(failure);
2333 }
2334
2335 // Handle existing monitor.
2336 bind(object_has_monitor);
2337 // The object's monitor m is unlocked iff m->owner is null,
2338 // otherwise m->owner may contain a thread or a stack address.
2339
2340 // Try to CAS m->owner from null to current thread.
2341 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2342 cmpxchgd(/*flag=*/flag,
2343 /*current_value=*/current_header,
2344 /*compare_value=*/(intptr_t)0,
2345 /*exchange_value=*/R16_thread,
2346 /*where=*/temp,
2347 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2348 MacroAssembler::cmpxchgx_hint_acquire_lock());
2349
2350 // Store a non-null value into the box.
2351 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2352 beq(flag, success);
2353
2354 // Check for recursive locking.
2355 cmpd(flag, current_header, R16_thread);
2356 bne(flag, failure);
2357
2358 // Current thread already owns the lock. Just increment recursions.
2359 Register recursions = displaced_header;
2360 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2361 addi(recursions, recursions, 1);
2362 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2363
2364 // flag == EQ indicates success, increment held monitor count
2365 // flag == NE indicates failure
2366 bind(success);
2367 inc_held_monitor_count(temp);
2368 bind(failure);
2369 }
2370
2371 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2372 Register temp, Register displaced_header, Register current_header) {
2373 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2374 assert_different_registers(oop, box, temp, displaced_header, current_header);
2375 Label success, failure, object_has_monitor, notRecursive;
2376
2377 if (LockingMode == LM_LEGACY) {
2378 // Find the lock address and load the displaced header from the stack.
2379 ld(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2380
2381 // If the displaced header is 0, we have a recursive unlock.
2382 cmpdi(flag, displaced_header, 0);
2383 beq(flag, success);
2384 }
2385
2386 // Handle existing monitor.
2387 // The object has an existing monitor iff (mark & monitor_value) != 0.
2388 ld(current_header, oopDesc::mark_offset_in_bytes(), oop);
2389 andi_(R0, current_header, markWord::monitor_value);
2390 bne(CCR0, object_has_monitor);
2391
2392 if (LockingMode == LM_MONITOR) {
2393 // Set NE to indicate 'failure' -> take slow-path.
2394 crandc(flag, Assembler::equal, flag, Assembler::equal);
2395 b(failure);
2396 } else {
2397 assert(LockingMode == LM_LEGACY, "must be");
2398 // Check if it is still a light weight lock, this is is true if we see
2399 // the stack address of the basicLock in the markWord of the object.
2400 // Cmpxchg sets flag to cmpd(current_header, box).
2401 cmpxchgd(/*flag=*/flag,
2402 /*current_value=*/current_header,
2403 /*compare_value=*/box,
2404 /*exchange_value=*/displaced_header,
2405 /*where=*/oop,
2406 MacroAssembler::MemBarRel,
2407 MacroAssembler::cmpxchgx_hint_release_lock(),
2408 noreg,
2409 &failure);
2410 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2411 b(success);
2412 }
2413
2414 // Handle existing monitor.
2415 bind(object_has_monitor);
2416 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2417 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2418 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2419
2420 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2421 // This is handled like owner thread mismatches: We take the slow path.
2422 cmpd(flag, temp, R16_thread);
2423 bne(flag, failure);
2424
2425 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2426
2427 addic_(displaced_header, displaced_header, -1);
2428 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2429 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2430 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2431 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2432 }
2433 b(success);
2434
2435 bind(notRecursive);
2436 ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2437 ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2438 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2439 cmpdi(flag, temp, 0);
2440 bne(flag, failure);
2441 release();
2442 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2443
2444 // flag == EQ indicates success, decrement held monitor count
2445 // flag == NE indicates failure
2446 bind(success);
2447 dec_held_monitor_count(temp);
2448 bind(failure);
2449 }
2450
2451 void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
2452 Register tmp2, Register tmp3) {
2453 assert_different_registers(obj, tmp1, tmp2, tmp3);
2454 assert(flag == CCR0, "bad condition register");
2455
2456 // Handle inflated monitor.
2457 Label inflated;
2458 // Finish fast lock successfully. MUST reach to with flag == NE
2459 Label locked;
2460 // Finish fast lock unsuccessfully. MUST branch to with flag == EQ
2461 Label slow_path;
2462
2463 if (DiagnoseSyncOnValueBasedClasses != 0) {
2464 load_klass(tmp1, obj);
2465 lwz(tmp1, in_bytes(Klass::access_flags_offset()), tmp1);
2466 testbitdi(flag, R0, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
2467 bne(flag, slow_path);
2468 }
2469
2470 const Register mark = tmp1;
2471 const Register t = tmp3; // Usage of R0 allowed!
2472
2473 { // Lightweight locking
2474
2475 // Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ
2476 Label push;
2477
2478 const Register top = tmp2;
2479
2480 // Check if lock-stack is full.
2481 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2482 cmplwi(flag, top, LockStack::end_offset() - 1);
2483 bgt(flag, slow_path);
2484
2485 // The underflow check is elided. The recursive check will always fail
2486 // when the lock stack is empty because of the _bad_oop_sentinel field.
2487
2488 // Check if recursive.
2489 subi(t, top, oopSize);
2490 ldx(t, R16_thread, t);
2491 cmpd(flag, obj, t);
2492 beq(flag, push);
2493
2494 // Check for monitor (0b10) or locked (0b00).
2495 ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2496 andi_(t, mark, markWord::lock_mask_in_place);
2497 cmpldi(flag, t, markWord::unlocked_value);
2498 bgt(flag, inflated);
2499 bne(flag, slow_path);
2500
2501 // Not inflated.
2502
2503 // Try to lock. Transition lock bits 0b00 => 0b01
2504 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
2505 atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow_path, MacroAssembler::MemBarAcq);
2506
2507 bind(push);
2508 // After successful lock, push object on lock-stack.
2509 stdx(obj, R16_thread, top);
2510 addi(top, top, oopSize);
2511 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2512 b(locked);
2513 }
2514
2515 { // Handle inflated monitor.
2516 bind(inflated);
2517
2518 // mark contains the tagged ObjectMonitor*.
2519 const Register tagged_monitor = mark;
2520 const uintptr_t monitor_tag = markWord::monitor_value;
2521 const Register owner_addr = tmp2;
2522
2523 // Compute owner address.
2524 addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2525
2526 // CAS owner (null => current thread).
2527 cmpxchgd(/*flag=*/flag,
2528 /*current_value=*/t,
2529 /*compare_value=*/(intptr_t)0,
2530 /*exchange_value=*/R16_thread,
2531 /*where=*/owner_addr,
2532 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2533 MacroAssembler::cmpxchgx_hint_acquire_lock());
2534 beq(flag, locked);
2535
2536 // Check if recursive.
2537 cmpd(flag, t, R16_thread);
2538 bne(flag, slow_path);
2539
2540 // Recursive.
2541 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2542 addi(tmp1, tmp1, 1);
2543 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2544 }
2545
2546 bind(locked);
2547 inc_held_monitor_count(tmp1);
2548
2549 #ifdef ASSERT
2550 // Check that locked label is reached with flag == EQ.
2551 Label flag_correct;
2552 beq(flag, flag_correct);
2553 stop("Fast Lock Flag != EQ");
2554 #endif
2555 bind(slow_path);
2556 #ifdef ASSERT
2557 // Check that slow_path label is reached with flag == NE.
2558 bne(flag, flag_correct);
2559 stop("Fast Lock Flag != NE");
2560 bind(flag_correct);
2561 #endif
2562 // C2 uses the value of flag (NE vs EQ) to determine the continuation.
2563 }
2564
2565 void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
2566 Register tmp2, Register tmp3) {
2567 assert_different_registers(obj, tmp1, tmp2, tmp3);
2568 assert(flag == CCR0, "bad condition register");
2569
2570 // Handle inflated monitor.
2571 Label inflated, inflated_load_monitor;
2572 // Finish fast unlock successfully. MUST reach to with flag == EQ.
2573 Label unlocked;
2574 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE.
2575 Label slow_path;
2576
2577 const Register mark = tmp1;
2578 const Register top = tmp2;
2579 const Register t = tmp3;
2580
2581 { // Lightweight unlock
2582 Label push_and_slow;
2583
2584 // Check if obj is top of lock-stack.
2585 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2586 subi(top, top, oopSize);
2587 ldx(t, R16_thread, top);
2588 cmpd(flag, obj, t);
2589 // Top of lock stack was not obj. Must be monitor.
2590 bne(flag, inflated_load_monitor);
2591
2592 // Pop lock-stack.
2593 DEBUG_ONLY(li(t, 0);)
2594 DEBUG_ONLY(stdx(t, R16_thread, top);)
2595 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2596
2597 // The underflow check is elided. The recursive check will always fail
2598 // when the lock stack is empty because of the _bad_oop_sentinel field.
2599
2600 // Check if recursive.
2601 subi(t, top, oopSize);
2602 ldx(t, R16_thread, t);
2603 cmpd(flag, obj, t);
2604 beq(flag, unlocked);
2605
2606 // Not recursive.
2607
2608 // Check for monitor (0b10).
2609 ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2610 andi_(t, mark, markWord::monitor_value);
2611 bne(CCR0, inflated);
2612
2613 #ifdef ASSERT
2614 // Check header not unlocked (0b01).
2615 Label not_unlocked;
2616 andi_(t, mark, markWord::unlocked_value);
2617 beq(CCR0, not_unlocked);
2618 stop("lightweight_unlock already unlocked");
2619 bind(not_unlocked);
2620 #endif
2621
2622 // Try to unlock. Transition lock bits 0b00 => 0b01
2623 atomically_flip_locked_state(/* is_unlock */ true, obj, mark, push_and_slow, MacroAssembler::MemBarRel);
2624 b(unlocked);
2625
2626 bind(push_and_slow);
2627 // Restore lock-stack and handle the unlock in runtime.
2628 DEBUG_ONLY(stdx(obj, R16_thread, top);)
2629 addi(top, top, oopSize);
2630 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2631 b(slow_path);
2632 }
2633
2634 { // Handle inflated monitor.
2635 bind(inflated_load_monitor);
2636 ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2637 #ifdef ASSERT
2638 andi_(t, mark, markWord::monitor_value);
2639 bne(CCR0, inflated);
2640 stop("Fast Unlock not monitor");
2641 #endif
2642
2643 bind(inflated);
2644
2645 #ifdef ASSERT
2646 Label check_done;
2647 subi(top, top, oopSize);
2648 cmplwi(CCR0, top, in_bytes(JavaThread::lock_stack_base_offset()));
2649 blt(CCR0, check_done);
2650 ldx(t, R16_thread, top);
2651 cmpd(flag, obj, t);
2652 bne(flag, inflated);
2653 stop("Fast Unlock lock on stack");
2654 bind(check_done);
2655 #endif
2656
2657 // mark contains the tagged ObjectMonitor*.
2658 const Register monitor = mark;
2659 const uintptr_t monitor_tag = markWord::monitor_value;
2660
2661 // Untag the monitor.
2662 subi(monitor, mark, monitor_tag);
2663
2664 const Register recursions = tmp2;
2665 Label not_recursive;
2666
2667 // Check if recursive.
2668 ld(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2669 addic_(recursions, recursions, -1);
2670 blt(CCR0, not_recursive);
2671
2672 // Recursive unlock.
2673 std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2674 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2675 b(unlocked);
2676
2677 bind(not_recursive);
2678
2679 Label release_;
2680 const Register t2 = tmp2;
2681
2682 // Check if the entry lists are empty.
2683 ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2684 ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2685 orr(t, t, t2);
2686 cmpdi(flag, t, 0);
2687 beq(flag, release_);
2688
2689 // The owner may be anonymous and we removed the last obj entry in
2690 // the lock-stack. This loses the information about the owner.
2691 // Write the thread to the owner field so the runtime knows the owner.
2692 std(R16_thread, in_bytes(ObjectMonitor::owner_offset()), monitor);
2693 b(slow_path);
2694
2695 bind(release_);
2696 // Set owner to null.
2697 release();
2698 // t contains 0
2699 std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2700 }
2701
2702 bind(unlocked);
2703 dec_held_monitor_count(t);
2704
2705 #ifdef ASSERT
2706 // Check that unlocked label is reached with flag == EQ.
2707 Label flag_correct;
2708 beq(flag, flag_correct);
2709 stop("Fast Lock Flag != EQ");
2710 #endif
2711 bind(slow_path);
2712 #ifdef ASSERT
2713 // Check that slow_path label is reached with flag == NE.
2714 bne(flag, flag_correct);
2715 stop("Fast Lock Flag != NE");
2716 bind(flag_correct);
2717 #endif
2718 // C2 uses the value of flag (NE vs EQ) to determine the continuation.
2719 }
2720
2721 void MacroAssembler::safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod) {
2722 ld(temp, in_bytes(JavaThread::polling_word_offset()), R16_thread);
2723
2724 if (at_return) {
2725 if (in_nmethod) {
2726 if (UseSIGTRAP) {
2727 // Use Signal Handler.
2728 relocate(relocInfo::poll_return_type);
2729 td(traptoGreaterThanUnsigned, R1_SP, temp);
2730 } else {
2731 cmpld(CCR0, R1_SP, temp);
2732 // Stub may be out of range for short conditional branch.
2733 bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_path);
2734 }
2735 } else { // Not in nmethod.
2736 // Frame still on stack, need to get fp.
2737 Register fp = R0;
2738 ld(fp, _abi0(callers_sp), R1_SP);
2739 cmpld(CCR0, fp, temp);
2740 bgt(CCR0, slow_path);
2741 }
2742 } else { // Normal safepoint poll. Not at return.
2743 assert(!in_nmethod, "should use load_from_polling_page");
2744 andi_(temp, temp, SafepointMechanism::poll_bit());
2745 bne(CCR0, slow_path);
2746 }
2747 }
2748
2749 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2,
2750 MacroAssembler::PreservationLevel preservation_level) {
2751 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2752 bs->resolve_jobject(this, value, tmp1, tmp2, preservation_level);
2753 }
2754
2755 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2,
2756 MacroAssembler::PreservationLevel preservation_level) {
2757 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2758 bs->resolve_global_jobject(this, value, tmp1, tmp2, preservation_level);
2759 }
2760
2761 // Values for last_Java_pc, and last_Java_sp must comply to the rules
2762 // in frame_ppc.hpp.
2763 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc) {
2764 // Always set last_Java_pc and flags first because once last_Java_sp
2765 // is visible has_last_Java_frame is true and users will look at the
2766 // rest of the fields. (Note: flags should always be zero before we
2767 // get here so doesn't need to be set.)
2768
2769 // Verify that last_Java_pc was zeroed on return to Java
2770 asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()), R16_thread,
2771 "last_Java_pc not zeroed before leaving Java");
2772
2773 // When returning from calling out from Java mode the frame anchor's
2774 // last_Java_pc will always be set to null. It is set here so that
2775 // if we are doing a call to native (not VM) that we capture the
2776 // known pc and don't have to rely on the native call having a
2777 // standard frame linkage where we can find the pc.
2778 if (last_Java_pc != noreg)
2779 std(last_Java_pc, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2780
2781 // Set last_Java_sp last.
2782 std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2783 }
2784
2785 void MacroAssembler::reset_last_Java_frame(void) {
2786 asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()),
2787 R16_thread, "SP was not set, still zero");
2788
2789 BLOCK_COMMENT("reset_last_Java_frame {");
2790 li(R0, 0);
2791
2792 // _last_Java_sp = 0
2793 std(R0, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2794
2795 // _last_Java_pc = 0
2796 std(R0, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2797 BLOCK_COMMENT("} reset_last_Java_frame");
2798 }
2799
2800 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) {
2801 assert_different_registers(sp, tmp1);
2802
2803 // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via
2804 // TOP_IJAVA_FRAME_ABI.
2805 // FIXME: assert that we really have a TOP_IJAVA_FRAME here!
2806 address entry = pc();
2807 load_const_optimized(tmp1, entry);
2808
2809 set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1);
2810 }
2811
2812 void MacroAssembler::get_vm_result(Register oop_result) {
2813 // Read:
2814 // R16_thread
2815 // R16_thread->in_bytes(JavaThread::vm_result_offset())
2816 //
2817 // Updated:
2818 // oop_result
2819 // R16_thread->in_bytes(JavaThread::vm_result_offset())
2820
2821 ld(oop_result, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2822 li(R0, 0);
2823 std(R0, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2824
2825 verify_oop(oop_result, FILE_AND_LINE);
2826 }
2827
2828 void MacroAssembler::get_vm_result_2(Register metadata_result) {
2829 // Read:
2830 // R16_thread
2831 // R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2832 //
2833 // Updated:
2834 // metadata_result
2835 // R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2836
2837 ld(metadata_result, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2838 li(R0, 0);
2839 std(R0, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2840 }
2841
2842 Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
2843 Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided.
2844 if (CompressedKlassPointers::base() != 0) {
2845 // Use dst as temp if it is free.
2846 sub_const_optimized(dst, current, CompressedKlassPointers::base(), R0);
2847 current = dst;
2848 }
2849 if (CompressedKlassPointers::shift() != 0) {
2850 srdi(dst, current, CompressedKlassPointers::shift());
2851 current = dst;
2852 }
2853 return current;
2854 }
2855
2856 void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
2857 if (UseCompressedClassPointers) {
2858 Register compressedKlass = encode_klass_not_null(ck, klass);
2859 stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
2860 } else {
2861 std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
2862 }
2863 }
2864
2865 void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
2866 if (UseCompressedClassPointers) {
2867 if (val == noreg) {
2868 val = R0;
2869 li(val, 0);
2870 }
2871 stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed
2872 }
2873 }
2874
2875 int MacroAssembler::instr_size_for_decode_klass_not_null() {
2876 static int computed_size = -1;
2877
2878 // Not yet computed?
2879 if (computed_size == -1) {
2880
2881 if (!UseCompressedClassPointers) {
2882 computed_size = 0;
2883 } else {
2884 // Determine by scratch emit.
2885 ResourceMark rm;
2886 int code_size = 8 * BytesPerInstWord;
2887 CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
2888 MacroAssembler* a = new MacroAssembler(&cb);
2889 a->decode_klass_not_null(R11_scratch1);
2890 computed_size = a->offset();
2891 }
2892 }
2893
2894 return computed_size;
2895 }
2896
2897 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
2898 assert(dst != R0, "Dst reg may not be R0, as R0 is used here.");
2899 if (src == noreg) src = dst;
2900 Register shifted_src = src;
2901 if (CompressedKlassPointers::shift() != 0 ||
2902 (CompressedKlassPointers::base() == 0 && src != dst)) { // Move required.
2903 shifted_src = dst;
2904 sldi(shifted_src, src, CompressedKlassPointers::shift());
2905 }
2906 if (CompressedKlassPointers::base() != 0) {
2907 add_const_optimized(dst, shifted_src, CompressedKlassPointers::base(), R0);
2908 }
2909 }
2910
2911 void MacroAssembler::load_klass(Register dst, Register src) {
2912 if (UseCompressedClassPointers) {
2913 lwz(dst, oopDesc::klass_offset_in_bytes(), src);
2914 // Attention: no null check here!
2915 decode_klass_not_null(dst, dst);
2916 } else {
2917 ld(dst, oopDesc::klass_offset_in_bytes(), src);
2918 }
2919 }
2920
2921 void MacroAssembler::load_klass_check_null(Register dst, Register src, Label* is_null) {
2922 null_check(src, oopDesc::klass_offset_in_bytes(), is_null);
2923 load_klass(dst, src);
2924 }
2925
2926 // ((OopHandle)result).resolve();
2927 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2,
2928 MacroAssembler::PreservationLevel preservation_level) {
2929 access_load_at(T_OBJECT, IN_NATIVE, result, noreg, result, tmp1, tmp2, preservation_level);
2930 }
2931
2932 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
2933 MacroAssembler::PreservationLevel preservation_level) {
2934 Label resolved;
2935
2936 // A null weak handle resolves to null.
2937 cmpdi(CCR0, result, 0);
2938 beq(CCR0, resolved);
2939
2940 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
2941 preservation_level);
2942 bind(resolved);
2943 }
2944
2945 void MacroAssembler::load_method_holder(Register holder, Register method) {
2946 ld(holder, in_bytes(Method::const_offset()), method);
2947 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
2948 ld(holder, ConstantPool::pool_holder_offset(), holder);
2949 }
2950
2951 // Clear Array
2952 // For very short arrays. tmp == R0 is allowed.
2953 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
2954 if (cnt_dwords > 0) { li(tmp, 0); }
2955 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
2956 }
2957
2958 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
2959 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
2960 if (cnt_dwords < 8) {
2961 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
2962 return;
2963 }
2964
2965 Label loop;
2966 const long loopcnt = cnt_dwords >> 1,
2967 remainder = cnt_dwords & 1;
2968
2969 li(tmp, loopcnt);
2970 mtctr(tmp);
2971 li(tmp, 0);
2972 bind(loop);
2973 std(tmp, 0, base_ptr);
2974 std(tmp, 8, base_ptr);
2975 addi(base_ptr, base_ptr, 16);
2976 bdnz(loop);
2977 if (remainder) { std(tmp, 0, base_ptr); }
2978 }
2979
2980 // Kills both input registers. tmp == R0 is allowed.
2981 void MacroAssembler::clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp, long const_cnt) {
2982 // Procedure for large arrays (uses data cache block zero instruction).
2983 Label startloop, fast, fastloop, small_rest, restloop, done;
2984 const int cl_size = VM_Version::L1_data_cache_line_size(),
2985 cl_dwords = cl_size >> 3,
2986 cl_dw_addr_bits = exact_log2(cl_dwords),
2987 dcbz_min = 1, // Min count of dcbz executions, needs to be >0.
2988 min_cnt = ((dcbz_min + 1) << cl_dw_addr_bits) - 1;
2989
2990 if (const_cnt >= 0) {
2991 // Constant case.
2992 if (const_cnt < min_cnt) {
2993 clear_memory_constlen(base_ptr, const_cnt, tmp);
2994 return;
2995 }
2996 load_const_optimized(cnt_dwords, const_cnt, tmp);
2997 } else {
2998 // cnt_dwords already loaded in register. Need to check size.
2999 cmpdi(CCR1, cnt_dwords, min_cnt); // Big enough? (ensure >= dcbz_min lines included).
3000 blt(CCR1, small_rest);
3001 }
3002 rldicl_(tmp, base_ptr, 64-3, 64-cl_dw_addr_bits); // Extract dword offset within first cache line.
3003 beq(CCR0, fast); // Already 128byte aligned.
3004
3005 subfic(tmp, tmp, cl_dwords);
3006 mtctr(tmp); // Set ctr to hit 128byte boundary (0<ctr<cl_dwords).
3007 subf(cnt_dwords, tmp, cnt_dwords); // rest.
3008 li(tmp, 0);
3009
3010 bind(startloop); // Clear at the beginning to reach 128byte boundary.
3011 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3012 addi(base_ptr, base_ptr, 8);
3013 bdnz(startloop);
3014
3015 bind(fast); // Clear 128byte blocks.
3016 srdi(tmp, cnt_dwords, cl_dw_addr_bits); // Loop count for 128byte loop (>0).
3017 andi(cnt_dwords, cnt_dwords, cl_dwords-1); // Rest in dwords.
3018 mtctr(tmp); // Load counter.
3019
3020 bind(fastloop);
3021 dcbz(base_ptr); // Clear 128byte aligned block.
3022 addi(base_ptr, base_ptr, cl_size);
3023 bdnz(fastloop);
3024
3025 bind(small_rest);
3026 cmpdi(CCR0, cnt_dwords, 0); // size 0?
3027 beq(CCR0, done); // rest == 0
3028 li(tmp, 0);
3029 mtctr(cnt_dwords); // Load counter.
3030
3031 bind(restloop); // Clear rest.
3032 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3033 addi(base_ptr, base_ptr, 8);
3034 bdnz(restloop);
3035
3036 bind(done);
3037 }
3038
3039 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3040
3041 // Helpers for Intrinsic Emitters
3042 //
3043 // Revert the byte order of a 32bit value in a register
3044 // src: 0x44556677
3045 // dst: 0x77665544
3046 // Three steps to obtain the result:
3047 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3048 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3049 // This value initializes dst.
3050 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3051 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3052 // This value is mask inserted into dst with a [0..23] mask of 1s.
3053 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3054 // This value is mask inserted into dst with a [8..15] mask of 1s.
3055 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3056 assert_different_registers(dst, src);
3057
3058 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
3059 rlwimi(dst, src, 3*8, 0, 23); // Insert byte 5 into position 6, 7 into 4, leave pos 7 alone.
3060 rlwimi(dst, src, 1*8, 8, 15); // Insert byte 6 into position 5, leave the rest alone.
3061 }
3062
3063 // Calculate the column addresses of the crc32 lookup table into distinct registers.
3064 // This loop-invariant calculation is moved out of the loop body, reducing the loop
3065 // body size from 20 to 16 instructions.
3066 // Returns the offset that was used to calculate the address of column tc3.
3067 // Due to register shortage, setting tc3 may overwrite table. With the return offset
3068 // at hand, the original table address can be easily reconstructed.
3069 int MacroAssembler::crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3) {
3070 assert(!VM_Version::has_vpmsumb(), "Vector version should be used instead!");
3071
3072 // Point to 4 byte folding tables (byte-reversed version for Big Endian)
3073 // Layout: See StubRoutines::ppc::generate_crc_constants.
3074 #ifdef VM_LITTLE_ENDIAN
3075 const int ix0 = 3 * CRC32_TABLE_SIZE;
3076 const int ix1 = 2 * CRC32_TABLE_SIZE;
3077 const int ix2 = 1 * CRC32_TABLE_SIZE;
3078 const int ix3 = 0 * CRC32_TABLE_SIZE;
3079 #else
3080 const int ix0 = 1 * CRC32_TABLE_SIZE;
3081 const int ix1 = 2 * CRC32_TABLE_SIZE;
3082 const int ix2 = 3 * CRC32_TABLE_SIZE;
3083 const int ix3 = 4 * CRC32_TABLE_SIZE;
3084 #endif
3085 assert_different_registers(table, tc0, tc1, tc2);
3086 assert(table == tc3, "must be!");
3087
3088 addi(tc0, table, ix0);
3089 addi(tc1, table, ix1);
3090 addi(tc2, table, ix2);
3091 if (ix3 != 0) addi(tc3, table, ix3);
3092
3093 return ix3;
3094 }
3095
3096 /**
3097 * uint32_t crc;
3098 * table[crc & 0xFF] ^ (crc >> 8);
3099 */
3100 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) {
3101 assert_different_registers(crc, table, tmp);
3102 assert_different_registers(val, table);
3103
3104 if (crc == val) { // Must rotate first to use the unmodified value.
3105 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3106 // As we use a word (4-byte) instruction, we have to adapt the mask bit positions.
3107 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits.
3108 } else {
3109 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits.
3110 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3111 }
3112 lwzx(tmp, table, tmp);
3113 xorr(crc, crc, tmp);
3114 }
3115
3116 /**
3117 * Emits code to update CRC-32 with a byte value according to constants in table.
3118 *
3119 * @param [in,out]crc Register containing the crc.
3120 * @param [in]val Register containing the byte to fold into the CRC.
3121 * @param [in]table Register containing the table of crc constants.
3122 *
3123 * uint32_t crc;
3124 * val = crc_table[(val ^ crc) & 0xFF];
3125 * crc = val ^ (crc >> 8);
3126 */
3127 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
3128 BLOCK_COMMENT("update_byte_crc32:");
3129 xorr(val, val, crc);
3130 fold_byte_crc32(crc, val, table, val);
3131 }
3132
3133 /**
3134 * @param crc register containing existing CRC (32-bit)
3135 * @param buf register pointing to input byte buffer (byte*)
3136 * @param len register containing number of bytes
3137 * @param table register pointing to CRC table
3138 */
3139 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table,
3140 Register data, bool loopAlignment) {
3141 assert_different_registers(crc, buf, len, table, data);
3142
3143 Label L_mainLoop, L_done;
3144 const int mainLoop_stepping = 1;
3145 const int mainLoop_alignment = loopAlignment ? 32 : 4; // (InputForNewCode > 4 ? InputForNewCode : 32) : 4;
3146
3147 // Process all bytes in a single-byte loop.
3148 clrldi_(len, len, 32); // Enforce 32 bit. Anything to do?
3149 beq(CCR0, L_done);
3150
3151 mtctr(len);
3152 align(mainLoop_alignment);
3153 BIND(L_mainLoop);
3154 lbz(data, 0, buf); // Byte from buffer, zero-extended.
3155 addi(buf, buf, mainLoop_stepping); // Advance buffer position.
3156 update_byte_crc32(crc, data, table);
3157 bdnz(L_mainLoop); // Iterate.
3158
3159 bind(L_done);
3160 }
3161
3162 /**
3163 * Emits code to update CRC-32 with a 4-byte value according to constants in table
3164 * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c
3165 */
3166 // A note on the lookup table address(es):
3167 // The implementation uses 4 table columns (byte-reversed versions for Big Endian).
3168 // To save the effort of adding the column offset to the table address each time
3169 // a table element is looked up, it is possible to pass the pre-calculated
3170 // column addresses.
3171 // Uses R9..R12 as work register. Must be saved/restored by caller, if necessary.
3172 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
3173 Register t0, Register t1, Register t2, Register t3,
3174 Register tc0, Register tc1, Register tc2, Register tc3) {
3175 assert_different_registers(crc, t3);
3176
3177 // XOR crc with next four bytes of buffer.
3178 lwz(t3, bufDisp, buf);
3179 if (bufInc != 0) {
3180 addi(buf, buf, bufInc);
3181 }
3182 xorr(t3, t3, crc);
3183
3184 // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices.
3185 rlwinm(t0, t3, 2, 24-2, 31-2); // ((t1 >> 0) & 0xff) << 2
3186 rlwinm(t1, t3, 32+(2- 8), 24-2, 31-2); // ((t1 >> 8) & 0xff) << 2
3187 rlwinm(t2, t3, 32+(2-16), 24-2, 31-2); // ((t1 >> 16) & 0xff) << 2
3188 rlwinm(t3, t3, 32+(2-24), 24-2, 31-2); // ((t1 >> 24) & 0xff) << 2
3189
3190 // Use the pre-calculated column addresses.
3191 // Load pre-calculated table values.
3192 lwzx(t0, tc0, t0);
3193 lwzx(t1, tc1, t1);
3194 lwzx(t2, tc2, t2);
3195 lwzx(t3, tc3, t3);
3196
3197 // Calculate new crc from table values.
3198 xorr(t0, t0, t1);
3199 xorr(t2, t2, t3);
3200 xorr(crc, t0, t2); // Now crc contains the final checksum value.
3201 }
3202
3203 /**
3204 * @param crc register containing existing CRC (32-bit)
3205 * @param buf register pointing to input byte buffer (byte*)
3206 * @param len register containing number of bytes
3207 * @param table register pointing to CRC table
3208 *
3209 * uses R9..R12 as work register. Must be saved/restored by caller!
3210 */
3211 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
3212 Register t0, Register t1, Register t2, Register t3,
3213 Register tc0, Register tc1, Register tc2, Register tc3,
3214 bool invertCRC) {
3215 assert_different_registers(crc, buf, len, table);
3216
3217 Label L_mainLoop, L_tail;
3218 Register tmp = t0;
3219 Register data = t0;
3220 Register tmp2 = t1;
3221 const int mainLoop_stepping = 4;
3222 const int tailLoop_stepping = 1;
3223 const int log_stepping = exact_log2(mainLoop_stepping);
3224 const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32;
3225 const int complexThreshold = 2*mainLoop_stepping;
3226
3227 // Don't test for len <= 0 here. This pathological case should not occur anyway.
3228 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles
3229 // for all well-behaved cases. The situation itself is detected and handled correctly
3230 // within update_byteLoop_crc32.
3231 assert(tailLoop_stepping == 1, "check tailLoop_stepping!");
3232
3233 BLOCK_COMMENT("kernel_crc32_1word {");
3234
3235 if (invertCRC) {
3236 nand(crc, crc, crc); // 1s complement of crc
3237 }
3238
3239 // Check for short (<mainLoop_stepping) buffer.
3240 cmpdi(CCR0, len, complexThreshold);
3241 blt(CCR0, L_tail);
3242
3243 // Pre-mainLoop alignment did show a slight (1%) positive effect on performance.
3244 // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions.
3245 {
3246 // Align buf addr to mainLoop_stepping boundary.
3247 neg(tmp2, buf); // Calculate # preLoop iterations for alignment.
3248 rldicl(tmp2, tmp2, 0, 64-log_stepping); // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63.
3249
3250 if (complexThreshold > mainLoop_stepping) {
3251 sub(len, len, tmp2); // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3252 } else {
3253 sub(tmp, len, tmp2); // Remaining bytes for main loop.
3254 cmpdi(CCR0, tmp, mainLoop_stepping);
3255 blt(CCR0, L_tail); // For less than one mainloop_stepping left, do only tail processing
3256 mr(len, tmp); // remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3257 }
3258 update_byteLoop_crc32(crc, buf, tmp2, table, data, false);
3259 }
3260
3261 srdi(tmp2, len, log_stepping); // #iterations for mainLoop
3262 andi(len, len, mainLoop_stepping-1); // remaining bytes for tailLoop
3263 mtctr(tmp2);
3264
3265 #ifdef VM_LITTLE_ENDIAN
3266 Register crc_rv = crc;
3267 #else
3268 Register crc_rv = tmp; // Load_reverse needs separate registers to work on.
3269 // Occupies tmp, but frees up crc.
3270 load_reverse_32(crc_rv, crc); // Revert byte order because we are dealing with big-endian data.
3271 tmp = crc;
3272 #endif
3273
3274 int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3);
3275
3276 align(mainLoop_alignment); // Octoword-aligned loop address. Shows 2% improvement.
3277 BIND(L_mainLoop);
3278 update_1word_crc32(crc_rv, buf, table, 0, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
3279 bdnz(L_mainLoop);
3280
3281 #ifndef VM_LITTLE_ENDIAN
3282 load_reverse_32(crc, crc_rv); // Revert byte order because we are dealing with big-endian data.
3283 tmp = crc_rv; // Tmp uses it's original register again.
3284 #endif
3285
3286 // Restore original table address for tailLoop.
3287 if (reconstructTableOffset != 0) {
3288 addi(table, table, -reconstructTableOffset);
3289 }
3290
3291 // Process last few (<complexThreshold) bytes of buffer.
3292 BIND(L_tail);
3293 update_byteLoop_crc32(crc, buf, len, table, data, false);
3294
3295 if (invertCRC) {
3296 nand(crc, crc, crc); // 1s complement of crc
3297 }
3298 BLOCK_COMMENT("} kernel_crc32_1word");
3299 }
3300
3301 /**
3302 * @param crc register containing existing CRC (32-bit)
3303 * @param buf register pointing to input byte buffer (byte*)
3304 * @param len register containing number of bytes
3305 * @param constants register pointing to precomputed constants
3306 * @param t0-t6 temp registers
3307 */
3308 void MacroAssembler::kernel_crc32_vpmsum(Register crc, Register buf, Register len, Register constants,
3309 Register t0, Register t1, Register t2, Register t3,
3310 Register t4, Register t5, Register t6, bool invertCRC) {
3311 assert_different_registers(crc, buf, len, constants);
3312
3313 Label L_tail;
3314
3315 BLOCK_COMMENT("kernel_crc32_vpmsum {");
3316
3317 if (invertCRC) {
3318 nand(crc, crc, crc); // 1s complement of crc
3319 }
3320
3321 // Enforce 32 bit.
3322 clrldi(len, len, 32);
3323
3324 // Align if we have enough bytes for the fast version.
3325 const int alignment = 16,
3326 threshold = 32;
3327 Register prealign = t0;
3328
3329 neg(prealign, buf);
3330 addi(t1, len, -threshold);
3331 andi(prealign, prealign, alignment - 1);
3332 cmpw(CCR0, t1, prealign);
3333 blt(CCR0, L_tail); // len - prealign < threshold?
3334
3335 subf(len, prealign, len);
3336 update_byteLoop_crc32(crc, buf, prealign, constants, t2, false);
3337
3338 // Calculate from first aligned address as far as possible.
3339 addi(constants, constants, CRC32_TABLE_SIZE); // Point to vector constants.
3340 kernel_crc32_vpmsum_aligned(crc, buf, len, constants, t0, t1, t2, t3, t4, t5, t6);
3341 addi(constants, constants, -CRC32_TABLE_SIZE); // Point to table again.
3342
3343 // Remaining bytes.
3344 BIND(L_tail);
3345 update_byteLoop_crc32(crc, buf, len, constants, t2, false);
3346
3347 if (invertCRC) {
3348 nand(crc, crc, crc); // 1s complement of crc
3349 }
3350
3351 BLOCK_COMMENT("} kernel_crc32_vpmsum");
3352 }
3353
3354 /**
3355 * @param crc register containing existing CRC (32-bit)
3356 * @param buf register pointing to input byte buffer (byte*)
3357 * @param len register containing number of bytes (will get updated to remaining bytes)
3358 * @param constants register pointing to CRC table for 128-bit aligned memory
3359 * @param t0-t6 temp registers
3360 */
3361 void MacroAssembler::kernel_crc32_vpmsum_aligned(Register crc, Register buf, Register len, Register constants,
3362 Register t0, Register t1, Register t2, Register t3, Register t4, Register t5, Register t6) {
3363
3364 // Save non-volatile vector registers (frameless).
3365 Register offset = t1;
3366 int offsetInt = 0;
3367 offsetInt -= 16; li(offset, offsetInt); stvx(VR20, offset, R1_SP);
3368 offsetInt -= 16; li(offset, offsetInt); stvx(VR21, offset, R1_SP);
3369 offsetInt -= 16; li(offset, offsetInt); stvx(VR22, offset, R1_SP);
3370 offsetInt -= 16; li(offset, offsetInt); stvx(VR23, offset, R1_SP);
3371 offsetInt -= 16; li(offset, offsetInt); stvx(VR24, offset, R1_SP);
3372 offsetInt -= 16; li(offset, offsetInt); stvx(VR25, offset, R1_SP);
3373 #ifndef VM_LITTLE_ENDIAN
3374 offsetInt -= 16; li(offset, offsetInt); stvx(VR26, offset, R1_SP);
3375 #endif
3376 offsetInt -= 8; std(R14, offsetInt, R1_SP);
3377 offsetInt -= 8; std(R15, offsetInt, R1_SP);
3378
3379 // Implementation uses an inner loop which uses between 256 and 16 * unroll_factor
3380 // bytes per iteration. The basic scheme is:
3381 // lvx: load vector (Big Endian needs reversal)
3382 // vpmsumw: carry-less 32 bit multiplications with constant representing a large CRC shift
3383 // vxor: xor partial results together to get unroll_factor2 vectors
3384
3385 // Outer loop performs the CRC shifts needed to combine the unroll_factor2 vectors.
3386
3387 // Using 16 * unroll_factor / unroll_factor_2 bytes for constants.
3388 const int unroll_factor = CRC32_UNROLL_FACTOR,
3389 unroll_factor2 = CRC32_UNROLL_FACTOR2;
3390
3391 const int outer_consts_size = (unroll_factor2 - 1) * 16,
3392 inner_consts_size = (unroll_factor / unroll_factor2) * 16;
3393
3394 // Support registers.
3395 Register offs[] = { noreg, t0, t1, t2, t3, t4, t5, t6 };
3396 Register num_bytes = R14,
3397 loop_count = R15,
3398 cur_const = crc; // will live in VCRC
3399 // Constant array for outer loop: unroll_factor2 - 1 registers,
3400 // Constant array for inner loop: unroll_factor / unroll_factor2 registers.
3401 VectorRegister consts0[] = { VR16, VR17, VR18, VR19, VR20, VR21, VR22 },
3402 consts1[] = { VR23, VR24 };
3403 // Data register arrays: 2 arrays with unroll_factor2 registers.
3404 VectorRegister data0[] = { VR0, VR1, VR2, VR3, VR4, VR5, VR6, VR7 },
3405 data1[] = { VR8, VR9, VR10, VR11, VR12, VR13, VR14, VR15 };
3406
3407 VectorRegister VCRC = data0[0];
3408 VectorRegister Vc = VR25;
3409 VectorRegister swap_bytes = VR26; // Only for Big Endian.
3410
3411 // We have at least 1 iteration (ensured by caller).
3412 Label L_outer_loop, L_inner_loop, L_last;
3413
3414 // If supported set DSCR pre-fetch to deepest.
3415 if (VM_Version::has_mfdscr()) {
3416 load_const_optimized(t0, VM_Version::_dscr_val | 7);
3417 mtdscr(t0);
3418 }
3419
3420 mtvrwz(VCRC, crc); // crc lives in VCRC, now
3421
3422 for (int i = 1; i < unroll_factor2; ++i) {
3423 li(offs[i], 16 * i);
3424 }
3425
3426 // Load consts for outer loop
3427 lvx(consts0[0], constants);
3428 for (int i = 1; i < unroll_factor2 - 1; ++i) {
3429 lvx(consts0[i], offs[i], constants);
3430 }
3431
3432 load_const_optimized(num_bytes, 16 * unroll_factor);
3433
3434 // Reuse data registers outside of the loop.
3435 VectorRegister Vtmp = data1[0];
3436 VectorRegister Vtmp2 = data1[1];
3437 VectorRegister zeroes = data1[2];
3438
3439 vspltisb(Vtmp, 0);
3440 vsldoi(VCRC, Vtmp, VCRC, 8); // 96 bit zeroes, 32 bit CRC.
3441
3442 // Load vector for vpermxor (to xor both 64 bit parts together)
3443 lvsl(Vtmp, buf); // 000102030405060708090a0b0c0d0e0f
3444 vspltisb(Vc, 4);
3445 vsl(Vc, Vtmp, Vc); // 00102030405060708090a0b0c0d0e0f0
3446 xxspltd(Vc->to_vsr(), Vc->to_vsr(), 0);
3447 vor(Vc, Vtmp, Vc); // 001122334455667708192a3b4c5d6e7f
3448
3449 #ifdef VM_LITTLE_ENDIAN
3450 #define BE_swap_bytes(x)
3451 #else
3452 vspltisb(Vtmp2, 0xf);
3453 vxor(swap_bytes, Vtmp, Vtmp2);
3454 #define BE_swap_bytes(x) vperm(x, x, x, swap_bytes)
3455 #endif
3456
3457 cmpd(CCR0, len, num_bytes);
3458 blt(CCR0, L_last);
3459
3460 addi(cur_const, constants, outer_consts_size); // Point to consts for inner loop
3461 load_const_optimized(loop_count, unroll_factor / (2 * unroll_factor2) - 1); // One double-iteration peeled off.
3462
3463 // ********** Main loop start **********
3464 align(32);
3465 bind(L_outer_loop);
3466
3467 // Begin of unrolled first iteration (no xor).
3468 lvx(data1[0], buf);
3469 for (int i = 1; i < unroll_factor2 / 2; ++i) {
3470 lvx(data1[i], offs[i], buf);
3471 }
3472 vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3473 lvx(consts1[0], cur_const);
3474 mtctr(loop_count);
3475 for (int i = 0; i < unroll_factor2 / 2; ++i) {
3476 BE_swap_bytes(data1[i]);
3477 if (i == 0) { vxor(data1[0], data1[0], VCRC); } // xor in previous CRC.
3478 lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3479 vpmsumw(data0[i], data1[i], consts1[0]);
3480 }
3481 addi(buf, buf, 16 * unroll_factor2);
3482 subf(len, num_bytes, len);
3483 lvx(consts1[1], offs[1], cur_const);
3484 addi(cur_const, cur_const, 32);
3485 // Begin of unrolled second iteration (head).
3486 for (int i = 0; i < unroll_factor2 / 2; ++i) {
3487 BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3488 if (i == 0) { lvx(data1[0], buf); } else { lvx(data1[i], offs[i], buf); }
3489 vpmsumw(data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[0]);
3490 }
3491 for (int i = 0; i < unroll_factor2 / 2; ++i) {
3492 BE_swap_bytes(data1[i]);
3493 lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3494 vpmsumw(data1[i], data1[i], consts1[1]);
3495 }
3496 addi(buf, buf, 16 * unroll_factor2);
3497
3498 // Generate most performance relevant code. Loads + half of the vpmsumw have been generated.
3499 // Double-iteration allows using the 2 constant registers alternatingly.
3500 align(32);
3501 bind(L_inner_loop);
3502 for (int j = 1; j < 3; ++j) { // j < unroll_factor / unroll_factor2 - 1 for complete unrolling.
3503 if (j & 1) {
3504 lvx(consts1[0], cur_const);
3505 } else {
3506 lvx(consts1[1], offs[1], cur_const);
3507 addi(cur_const, cur_const, 32);
3508 }
3509 for (int i = 0; i < unroll_factor2; ++i) {
3510 int idx = i + unroll_factor2 / 2, inc = 0; // For modulo-scheduled input.
3511 if (idx >= unroll_factor2) { idx -= unroll_factor2; inc = 1; }
3512 BE_swap_bytes(data1[idx]);
3513 vxor(data0[i], data0[i], data1[i]);
3514 if (i == 0) lvx(data1[0], buf); else lvx(data1[i], offs[i], buf);
3515 vpmsumw(data1[idx], data1[idx], consts1[(j + inc) & 1]);
3516 }
3517 addi(buf, buf, 16 * unroll_factor2);
3518 }
3519 bdnz(L_inner_loop);
3520
3521 addi(cur_const, constants, outer_consts_size); // Reset
3522
3523 // Tail of last iteration (no loads).
3524 for (int i = 0; i < unroll_factor2 / 2; ++i) {
3525 BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3526 vxor(data0[i], data0[i], data1[i]);
3527 vpmsumw(data1[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[1]);
3528 }
3529 for (int i = 0; i < unroll_factor2 / 2; ++i) {
3530 vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]); // First half of fixup shifts.
3531 vxor(data0[i + unroll_factor2 / 2], data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2]);
3532 }
3533
3534 // Last data register is ok, other ones need fixup shift.
3535 for (int i = unroll_factor2 / 2; i < unroll_factor2 - 1; ++i) {
3536 vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]);
3537 }
3538
3539 // Combine to 128 bit result vector VCRC = data0[0].
3540 for (int i = 1; i < unroll_factor2; i<<=1) {
3541 for (int j = 0; j <= unroll_factor2 - 2*i; j+=2*i) {
3542 vxor(data0[j], data0[j], data0[j+i]);
3543 }
3544 }
3545 cmpd(CCR0, len, num_bytes);
3546 bge(CCR0, L_outer_loop);
3547
3548 // Last chance with lower num_bytes.
3549 bind(L_last);
3550 srdi(loop_count, len, exact_log2(16 * 2 * unroll_factor2)); // Use double-iterations.
3551 // Point behind last const for inner loop.
3552 add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3553 sldi(R0, loop_count, exact_log2(16 * 2)); // Bytes of constants to be used.
3554 clrrdi(num_bytes, len, exact_log2(16 * 2 * unroll_factor2));
3555 subf(cur_const, R0, cur_const); // Point to constant to be used first.
3556
3557 addic_(loop_count, loop_count, -1); // One double-iteration peeled off.
3558 bgt(CCR0, L_outer_loop);
3559 // ********** Main loop end **********
3560
3561 // Restore DSCR pre-fetch value.
3562 if (VM_Version::has_mfdscr()) {
3563 load_const_optimized(t0, VM_Version::_dscr_val);
3564 mtdscr(t0);
3565 }
3566
3567 // ********** Simple loop for remaining 16 byte blocks **********
3568 {
3569 Label L_loop, L_done;
3570
3571 srdi_(t0, len, 4); // 16 bytes per iteration
3572 clrldi(len, len, 64-4);
3573 beq(CCR0, L_done);
3574
3575 // Point to const (same as last const for inner loop).
3576 add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size - 16);
3577 mtctr(t0);
3578 lvx(Vtmp2, cur_const);
3579
3580 align(32);
3581 bind(L_loop);
3582
3583 lvx(Vtmp, buf);
3584 addi(buf, buf, 16);
3585 vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3586 BE_swap_bytes(Vtmp);
3587 vxor(VCRC, VCRC, Vtmp);
3588 vpmsumw(VCRC, VCRC, Vtmp2);
3589 bdnz(L_loop);
3590
3591 bind(L_done);
3592 }
3593 // ********** Simple loop end **********
3594 #undef BE_swap_bytes
3595
3596 // Point to Barrett constants
3597 add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3598
3599 vspltisb(zeroes, 0);
3600
3601 // Combine to 64 bit result.
3602 vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3603
3604 // Reduce to 32 bit CRC: Remainder by multiply-high.
3605 lvx(Vtmp, cur_const);
3606 vsldoi(Vtmp2, zeroes, VCRC, 12); // Extract high 32 bit.
3607 vpmsumd(Vtmp2, Vtmp2, Vtmp); // Multiply by inverse long poly.
3608 vsldoi(Vtmp2, zeroes, Vtmp2, 12); // Extract high 32 bit.
3609 vsldoi(Vtmp, zeroes, Vtmp, 8);
3610 vpmsumd(Vtmp2, Vtmp2, Vtmp); // Multiply quotient by long poly.
3611 vxor(VCRC, VCRC, Vtmp2); // Remainder fits into 32 bit.
3612
3613 // Move result. len is already updated.
3614 vsldoi(VCRC, VCRC, zeroes, 8);
3615 mfvrd(crc, VCRC);
3616
3617 // Restore non-volatile Vector registers (frameless).
3618 offsetInt = 0;
3619 offsetInt -= 16; li(offset, offsetInt); lvx(VR20, offset, R1_SP);
3620 offsetInt -= 16; li(offset, offsetInt); lvx(VR21, offset, R1_SP);
3621 offsetInt -= 16; li(offset, offsetInt); lvx(VR22, offset, R1_SP);
3622 offsetInt -= 16; li(offset, offsetInt); lvx(VR23, offset, R1_SP);
3623 offsetInt -= 16; li(offset, offsetInt); lvx(VR24, offset, R1_SP);
3624 offsetInt -= 16; li(offset, offsetInt); lvx(VR25, offset, R1_SP);
3625 #ifndef VM_LITTLE_ENDIAN
3626 offsetInt -= 16; li(offset, offsetInt); lvx(VR26, offset, R1_SP);
3627 #endif
3628 offsetInt -= 8; ld(R14, offsetInt, R1_SP);
3629 offsetInt -= 8; ld(R15, offsetInt, R1_SP);
3630 }
3631
3632 void MacroAssembler::crc32(Register crc, Register buf, Register len, Register t0, Register t1, Register t2,
3633 Register t3, Register t4, Register t5, Register t6, Register t7, bool is_crc32c) {
3634 load_const_optimized(t0, is_crc32c ? StubRoutines::crc32c_table_addr()
3635 : StubRoutines::crc_table_addr() , R0);
3636
3637 if (VM_Version::has_vpmsumb()) {
3638 kernel_crc32_vpmsum(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, !is_crc32c);
3639 } else {
3640 kernel_crc32_1word(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, t0, !is_crc32c);
3641 }
3642 }
3643
3644 void MacroAssembler::kernel_crc32_singleByteReg(Register crc, Register val, Register table, bool invertCRC) {
3645 assert_different_registers(crc, val, table);
3646
3647 BLOCK_COMMENT("kernel_crc32_singleByteReg:");
3648 if (invertCRC) {
3649 nand(crc, crc, crc); // 1s complement of crc
3650 }
3651
3652 update_byte_crc32(crc, val, table);
3653
3654 if (invertCRC) {
3655 nand(crc, crc, crc); // 1s complement of crc
3656 }
3657 }
3658
3659 // dest_lo += src1 + src2
3660 // dest_hi += carry1 + carry2
3661 void MacroAssembler::add2_with_carry(Register dest_hi,
3662 Register dest_lo,
3663 Register src1, Register src2) {
3664 li(R0, 0);
3665 addc(dest_lo, dest_lo, src1);
3666 adde(dest_hi, dest_hi, R0);
3667 addc(dest_lo, dest_lo, src2);
3668 adde(dest_hi, dest_hi, R0);
3669 }
3670
3671 // Multiply 64 bit by 64 bit first loop.
3672 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart,
3673 Register x_xstart,
3674 Register y, Register y_idx,
3675 Register z,
3676 Register carry,
3677 Register product_high, Register product,
3678 Register idx, Register kdx,
3679 Register tmp) {
3680 // jlong carry, x[], y[], z[];
3681 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
3682 // huge_128 product = y[idx] * x[xstart] + carry;
3683 // z[kdx] = (jlong)product;
3684 // carry = (jlong)(product >>> 64);
3685 // }
3686 // z[xstart] = carry;
3687
3688 Label L_first_loop, L_first_loop_exit;
3689 Label L_one_x, L_one_y, L_multiply;
3690
3691 addic_(xstart, xstart, -1);
3692 blt(CCR0, L_one_x); // Special case: length of x is 1.
3693
3694 // Load next two integers of x.
3695 sldi(tmp, xstart, LogBytesPerInt);
3696 ldx(x_xstart, x, tmp);
3697 #ifdef VM_LITTLE_ENDIAN
3698 rldicl(x_xstart, x_xstart, 32, 0);
3699 #endif
3700
3701 align(32, 16);
3702 bind(L_first_loop);
3703
3704 cmpdi(CCR0, idx, 1);
3705 blt(CCR0, L_first_loop_exit);
3706 addi(idx, idx, -2);
3707 beq(CCR0, L_one_y);
3708
3709 // Load next two integers of y.
3710 sldi(tmp, idx, LogBytesPerInt);
3711 ldx(y_idx, y, tmp);
3712 #ifdef VM_LITTLE_ENDIAN
3713 rldicl(y_idx, y_idx, 32, 0);
3714 #endif
3715
3716
3717 bind(L_multiply);
3718 multiply64(product_high, product, x_xstart, y_idx);
3719
3720 li(tmp, 0);
3721 addc(product, product, carry); // Add carry to result.
3722 adde(product_high, product_high, tmp); // Add carry of the last addition.
3723 addi(kdx, kdx, -2);
3724
3725 // Store result.
3726 #ifdef VM_LITTLE_ENDIAN
3727 rldicl(product, product, 32, 0);
3728 #endif
3729 sldi(tmp, kdx, LogBytesPerInt);
3730 stdx(product, z, tmp);
3731 mr_if_needed(carry, product_high);
3732 b(L_first_loop);
3733
3734
3735 bind(L_one_y); // Load one 32 bit portion of y as (0,value).
3736
3737 lwz(y_idx, 0, y);
3738 b(L_multiply);
3739
3740
3741 bind(L_one_x); // Load one 32 bit portion of x as (0,value).
3742
3743 lwz(x_xstart, 0, x);
3744 b(L_first_loop);
3745
3746 bind(L_first_loop_exit);
3747 }
3748
3749 // Multiply 64 bit by 64 bit and add 128 bit.
3750 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y,
3751 Register z, Register yz_idx,
3752 Register idx, Register carry,
3753 Register product_high, Register product,
3754 Register tmp, int offset) {
3755
3756 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
3757 // z[kdx] = (jlong)product;
3758
3759 sldi(tmp, idx, LogBytesPerInt);
3760 if (offset) {
3761 addi(tmp, tmp, offset);
3762 }
3763 ldx(yz_idx, y, tmp);
3764 #ifdef VM_LITTLE_ENDIAN
3765 rldicl(yz_idx, yz_idx, 32, 0);
3766 #endif
3767
3768 multiply64(product_high, product, x_xstart, yz_idx);
3769 ldx(yz_idx, z, tmp);
3770 #ifdef VM_LITTLE_ENDIAN
3771 rldicl(yz_idx, yz_idx, 32, 0);
3772 #endif
3773
3774 add2_with_carry(product_high, product, carry, yz_idx);
3775
3776 sldi(tmp, idx, LogBytesPerInt);
3777 if (offset) {
3778 addi(tmp, tmp, offset);
3779 }
3780 #ifdef VM_LITTLE_ENDIAN
3781 rldicl(product, product, 32, 0);
3782 #endif
3783 stdx(product, z, tmp);
3784 }
3785
3786 // Multiply 128 bit by 128 bit. Unrolled inner loop.
3787 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart,
3788 Register y, Register z,
3789 Register yz_idx, Register idx, Register carry,
3790 Register product_high, Register product,
3791 Register carry2, Register tmp) {
3792
3793 // jlong carry, x[], y[], z[];
3794 // int kdx = ystart+1;
3795 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
3796 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
3797 // z[kdx+idx+1] = (jlong)product;
3798 // jlong carry2 = (jlong)(product >>> 64);
3799 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
3800 // z[kdx+idx] = (jlong)product;
3801 // carry = (jlong)(product >>> 64);
3802 // }
3803 // idx += 2;
3804 // if (idx > 0) {
3805 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
3806 // z[kdx+idx] = (jlong)product;
3807 // carry = (jlong)(product >>> 64);
3808 // }
3809
3810 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
3811 const Register jdx = R0;
3812
3813 // Scale the index.
3814 srdi_(jdx, idx, 2);
3815 beq(CCR0, L_third_loop_exit);
3816 mtctr(jdx);
3817
3818 align(32, 16);
3819 bind(L_third_loop);
3820
3821 addi(idx, idx, -4);
3822
3823 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 8);
3824 mr_if_needed(carry2, product_high);
3825
3826 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product_high, product, tmp, 0);
3827 mr_if_needed(carry, product_high);
3828 bdnz(L_third_loop);
3829
3830 bind(L_third_loop_exit); // Handle any left-over operand parts.
3831
3832 andi_(idx, idx, 0x3);
3833 beq(CCR0, L_post_third_loop_done);
3834
3835 Label L_check_1;
3836
3837 addic_(idx, idx, -2);
3838 blt(CCR0, L_check_1);
3839
3840 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 0);
3841 mr_if_needed(carry, product_high);
3842
3843 bind(L_check_1);
3844
3845 addi(idx, idx, 0x2);
3846 andi_(idx, idx, 0x1);
3847 addic_(idx, idx, -1);
3848 blt(CCR0, L_post_third_loop_done);
3849
3850 sldi(tmp, idx, LogBytesPerInt);
3851 lwzx(yz_idx, y, tmp);
3852 multiply64(product_high, product, x_xstart, yz_idx);
3853 lwzx(yz_idx, z, tmp);
3854
3855 add2_with_carry(product_high, product, yz_idx, carry);
3856
3857 sldi(tmp, idx, LogBytesPerInt);
3858 stwx(product, z, tmp);
3859 srdi(product, product, 32);
3860
3861 sldi(product_high, product_high, 32);
3862 orr(product, product, product_high);
3863 mr_if_needed(carry, product);
3864
3865 bind(L_post_third_loop_done);
3866 } // multiply_128_x_128_loop
3867
3868 void MacroAssembler::muladd(Register out, Register in,
3869 Register offset, Register len, Register k,
3870 Register tmp1, Register tmp2, Register carry) {
3871
3872 // Labels
3873 Label LOOP, SKIP;
3874
3875 // Make sure length is positive.
3876 cmpdi (CCR0, len, 0);
3877
3878 // Prepare variables
3879 subi (offset, offset, 4);
3880 li (carry, 0);
3881 ble (CCR0, SKIP);
3882
3883 mtctr (len);
3884 subi (len, len, 1 );
3885 sldi (len, len, 2 );
3886
3887 // Main loop
3888 bind(LOOP);
3889 lwzx (tmp1, len, in );
3890 lwzx (tmp2, offset, out );
3891 mulld (tmp1, tmp1, k );
3892 add (tmp2, carry, tmp2 );
3893 add (tmp2, tmp1, tmp2 );
3894 stwx (tmp2, offset, out );
3895 srdi (carry, tmp2, 32 );
3896 subi (offset, offset, 4 );
3897 subi (len, len, 4 );
3898 bdnz (LOOP);
3899 bind(SKIP);
3900 }
3901
3902 void MacroAssembler::multiply_to_len(Register x, Register xlen,
3903 Register y, Register ylen,
3904 Register z, Register zlen,
3905 Register tmp1, Register tmp2,
3906 Register tmp3, Register tmp4,
3907 Register tmp5, Register tmp6,
3908 Register tmp7, Register tmp8,
3909 Register tmp9, Register tmp10,
3910 Register tmp11, Register tmp12,
3911 Register tmp13) {
3912
3913 ShortBranchVerifier sbv(this);
3914
3915 assert_different_registers(x, xlen, y, ylen, z, zlen,
3916 tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
3917 assert_different_registers(x, xlen, y, ylen, z, zlen,
3918 tmp1, tmp2, tmp3, tmp4, tmp5, tmp7);
3919 assert_different_registers(x, xlen, y, ylen, z, zlen,
3920 tmp1, tmp2, tmp3, tmp4, tmp5, tmp8);
3921
3922 const Register idx = tmp1;
3923 const Register kdx = tmp2;
3924 const Register xstart = tmp3;
3925
3926 const Register y_idx = tmp4;
3927 const Register carry = tmp5;
3928 const Register product = tmp6;
3929 const Register product_high = tmp7;
3930 const Register x_xstart = tmp8;
3931 const Register tmp = tmp9;
3932
3933 // First Loop.
3934 //
3935 // final static long LONG_MASK = 0xffffffffL;
3936 // int xstart = xlen - 1;
3937 // int ystart = ylen - 1;
3938 // long carry = 0;
3939 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
3940 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
3941 // z[kdx] = (int)product;
3942 // carry = product >>> 32;
3943 // }
3944 // z[xstart] = (int)carry;
3945
3946 mr_if_needed(idx, ylen); // idx = ylen
3947 mr_if_needed(kdx, zlen); // kdx = xlen + ylen
3948 li(carry, 0); // carry = 0
3949
3950 Label L_done;
3951
3952 addic_(xstart, xlen, -1);
3953 blt(CCR0, L_done);
3954
3955 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z,
3956 carry, product_high, product, idx, kdx, tmp);
3957
3958 Label L_second_loop;
3959
3960 cmpdi(CCR0, kdx, 0);
3961 beq(CCR0, L_second_loop);
3962
3963 Label L_carry;
3964
3965 addic_(kdx, kdx, -1);
3966 beq(CCR0, L_carry);
3967
3968 // Store lower 32 bits of carry.
3969 sldi(tmp, kdx, LogBytesPerInt);
3970 stwx(carry, z, tmp);
3971 srdi(carry, carry, 32);
3972 addi(kdx, kdx, -1);
3973
3974
3975 bind(L_carry);
3976
3977 // Store upper 32 bits of carry.
3978 sldi(tmp, kdx, LogBytesPerInt);
3979 stwx(carry, z, tmp);
3980
3981 // Second and third (nested) loops.
3982 //
3983 // for (int i = xstart-1; i >= 0; i--) { // Second loop
3984 // carry = 0;
3985 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
3986 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
3987 // (z[k] & LONG_MASK) + carry;
3988 // z[k] = (int)product;
3989 // carry = product >>> 32;
3990 // }
3991 // z[i] = (int)carry;
3992 // }
3993 //
3994 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
3995
3996 bind(L_second_loop);
3997
3998 li(carry, 0); // carry = 0;
3999
4000 addic_(xstart, xstart, -1); // i = xstart-1;
4001 blt(CCR0, L_done);
4002
4003 Register zsave = tmp10;
4004
4005 mr(zsave, z);
4006
4007
4008 Label L_last_x;
4009
4010 sldi(tmp, xstart, LogBytesPerInt);
4011 add(z, z, tmp); // z = z + k - j
4012 addi(z, z, 4);
4013 addic_(xstart, xstart, -1); // i = xstart-1;
4014 blt(CCR0, L_last_x);
4015
4016 sldi(tmp, xstart, LogBytesPerInt);
4017 ldx(x_xstart, x, tmp);
4018 #ifdef VM_LITTLE_ENDIAN
4019 rldicl(x_xstart, x_xstart, 32, 0);
4020 #endif
4021
4022
4023 Label L_third_loop_prologue;
4024
4025 bind(L_third_loop_prologue);
4026
4027 Register xsave = tmp11;
4028 Register xlensave = tmp12;
4029 Register ylensave = tmp13;
4030
4031 mr(xsave, x);
4032 mr(xlensave, xstart);
4033 mr(ylensave, ylen);
4034
4035
4036 multiply_128_x_128_loop(x_xstart, y, z, y_idx, ylen,
4037 carry, product_high, product, x, tmp);
4038
4039 mr(z, zsave);
4040 mr(x, xsave);
4041 mr(xlen, xlensave); // This is the decrement of the loop counter!
4042 mr(ylen, ylensave);
4043
4044 addi(tmp3, xlen, 1);
4045 sldi(tmp, tmp3, LogBytesPerInt);
4046 stwx(carry, z, tmp);
4047 addic_(tmp3, tmp3, -1);
4048 blt(CCR0, L_done);
4049
4050 srdi(carry, carry, 32);
4051 sldi(tmp, tmp3, LogBytesPerInt);
4052 stwx(carry, z, tmp);
4053 b(L_second_loop);
4054
4055 // Next infrequent code is moved outside loops.
4056 bind(L_last_x);
4057
4058 lwz(x_xstart, 0, x);
4059 b(L_third_loop_prologue);
4060
4061 bind(L_done);
4062 } // multiply_to_len
4063
4064 void MacroAssembler::asm_assert(bool check_equal, const char *msg) {
4065 #ifdef ASSERT
4066 Label ok;
4067 if (check_equal) {
4068 beq(CCR0, ok);
4069 } else {
4070 bne(CCR0, ok);
4071 }
4072 stop(msg);
4073 bind(ok);
4074 #endif
4075 }
4076
4077 void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset,
4078 Register mem_base, const char* msg) {
4079 #ifdef ASSERT
4080 switch (size) {
4081 case 4:
4082 lwz(R0, mem_offset, mem_base);
4083 cmpwi(CCR0, R0, 0);
4084 break;
4085 case 8:
4086 ld(R0, mem_offset, mem_base);
4087 cmpdi(CCR0, R0, 0);
4088 break;
4089 default:
4090 ShouldNotReachHere();
4091 }
4092 asm_assert(check_equal, msg);
4093 #endif // ASSERT
4094 }
4095
4096 void MacroAssembler::verify_coop(Register coop, const char* msg) {
4097 if (!VerifyOops) { return; }
4098 if (UseCompressedOops) { decode_heap_oop(coop); }
4099 verify_oop(coop, msg);
4100 if (UseCompressedOops) { encode_heap_oop(coop, coop); }
4101 }
4102
4103 // READ: oop. KILL: R0. Volatile floats perhaps.
4104 void MacroAssembler::verify_oop(Register oop, const char* msg) {
4105 if (!VerifyOops) {
4106 return;
4107 }
4108
4109 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4110 const Register tmp = R11; // Will be preserved.
4111 const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4112
4113 BLOCK_COMMENT("verify_oop {");
4114
4115 save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4116
4117 mr_if_needed(R4_ARG2, oop);
4118 save_LR_CR(tmp); // save in old frame
4119 push_frame_reg_args(nbytes_save, tmp);
4120 // load FunctionDescriptor** / entry_address *
4121 load_const_optimized(tmp, fd, R0);
4122 // load FunctionDescriptor* / entry_address
4123 ld(tmp, 0, tmp);
4124 load_const_optimized(R3_ARG1, (address)msg, R0);
4125 // Call destination for its side effect.
4126 call_c(tmp);
4127
4128 pop_frame();
4129 restore_LR_CR(tmp);
4130 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4131
4132 BLOCK_COMMENT("} verify_oop");
4133 }
4134
4135 void MacroAssembler::verify_oop_addr(RegisterOrConstant offs, Register base, const char* msg) {
4136 if (!VerifyOops) {
4137 return;
4138 }
4139
4140 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4141 const Register tmp = R11; // Will be preserved.
4142 const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4143 save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4144
4145 ld(R4_ARG2, offs, base);
4146 save_LR_CR(tmp); // save in old frame
4147 push_frame_reg_args(nbytes_save, tmp);
4148 // load FunctionDescriptor** / entry_address *
4149 load_const_optimized(tmp, fd, R0);
4150 // load FunctionDescriptor* / entry_address
4151 ld(tmp, 0, tmp);
4152 load_const_optimized(R3_ARG1, (address)msg, R0);
4153 // Call destination for its side effect.
4154 call_c(tmp);
4155
4156 pop_frame();
4157 restore_LR_CR(tmp);
4158 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4159 }
4160
4161 // Call a C-function that prints output.
4162 void MacroAssembler::stop(int type, const char* msg) {
4163 bool msg_present = (msg != nullptr);
4164
4165 #ifndef PRODUCT
4166 block_comment(err_msg("stop(type %d): %s {", type, msg_present ? msg : "null"));
4167 #else
4168 block_comment("stop {");
4169 #endif
4170
4171 if (msg_present) {
4172 type |= stop_msg_present;
4173 }
4174 tdi_unchecked(traptoUnconditional, 0/*reg 0*/, type);
4175 if (msg_present) {
4176 emit_int64((uintptr_t)msg);
4177 }
4178
4179 block_comment("} stop;");
4180 }
4181
4182 #ifndef PRODUCT
4183 // Write pattern 0x0101010101010101 in memory region [low-before, high+after].
4184 // Val, addr are temp registers.
4185 // If low == addr, addr is killed.
4186 // High is preserved.
4187 void MacroAssembler::zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) {
4188 if (!ZapMemory) return;
4189
4190 assert_different_registers(low, val);
4191
4192 BLOCK_COMMENT("zap memory region {");
4193 load_const_optimized(val, 0x0101010101010101);
4194 int size = before + after;
4195 if (low == high && size < 5 && size > 0) {
4196 int offset = -before*BytesPerWord;
4197 for (int i = 0; i < size; ++i) {
4198 std(val, offset, low);
4199 offset += (1*BytesPerWord);
4200 }
4201 } else {
4202 addi(addr, low, -before*BytesPerWord);
4203 assert_different_registers(high, val);
4204 if (after) addi(high, high, after * BytesPerWord);
4205 Label loop;
4206 bind(loop);
4207 std(val, 0, addr);
4208 addi(addr, addr, 8);
4209 cmpd(CCR6, addr, high);
4210 ble(CCR6, loop);
4211 if (after) addi(high, high, -after * BytesPerWord); // Correct back to old value.
4212 }
4213 BLOCK_COMMENT("} zap memory region");
4214 }
4215
4216 #endif // !PRODUCT
4217
4218 void SkipIfEqualZero::skip_to_label_if_equal_zero(MacroAssembler* masm, Register temp,
4219 const bool* flag_addr, Label& label) {
4220 int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true);
4221 assert(sizeof(bool) == 1, "PowerPC ABI");
4222 masm->lbz(temp, simm16_offset, temp);
4223 masm->cmpwi(CCR0, temp, 0);
4224 masm->beq(CCR0, label);
4225 }
4226
4227 SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() {
4228 skip_to_label_if_equal_zero(masm, temp, flag_addr, _label);
4229 }
4230
4231 SkipIfEqualZero::~SkipIfEqualZero() {
4232 _masm->bind(_label);
4233 }
4234
4235 void MacroAssembler::cache_wb(Address line) {
4236 assert(line.index() == noreg, "index should be noreg");
4237 assert(line.disp() == 0, "displacement should be 0");
4238 assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support flush to persistent memory");
4239 // Data Cache Store, not really a flush, so it works like a sync of cache
4240 // line and persistent mem, i.e. copying the cache line to persistent whilst
4241 // not invalidating the cache line.
4242 dcbst(line.base());
4243 }
4244
4245 void MacroAssembler::cache_wbsync(bool is_presync) {
4246 assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support sync related to persistent memory");
4247 // We only need a post sync barrier. Post means _after_ a cache line flush or
4248 // store instruction, pre means a barrier emitted before such a instructions.
4249 if (!is_presync) {
4250 fence();
4251 }
4252 }
4253
4254 void MacroAssembler::push_cont_fastpath() {
4255 Label done;
4256 ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4257 cmpld(CCR0, R1_SP, R0);
4258 ble(CCR0, done);
4259 st_ptr(R1_SP, JavaThread::cont_fastpath_offset(), R16_thread);
4260 bind(done);
4261 }
4262
4263 void MacroAssembler::pop_cont_fastpath() {
4264 Label done;
4265 ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4266 cmpld(CCR0, R1_SP, R0);
4267 ble(CCR0, done);
4268 li(R0, 0);
4269 st_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4270 bind(done);
4271 }
4272
4273 // Note: Must preserve CCR0 EQ (invariant).
4274 void MacroAssembler::inc_held_monitor_count(Register tmp) {
4275 ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4276 #ifdef ASSERT
4277 Label ok;
4278 cmpdi(CCR0, tmp, 0);
4279 bge_predict_taken(CCR0, ok);
4280 stop("held monitor count is negativ at increment");
4281 bind(ok);
4282 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Restore CCR0 EQ
4283 #endif
4284 addi(tmp, tmp, 1);
4285 std(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4286 }
4287
4288 // Note: Must preserve CCR0 EQ (invariant).
4289 void MacroAssembler::dec_held_monitor_count(Register tmp) {
4290 ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4291 #ifdef ASSERT
4292 Label ok;
4293 cmpdi(CCR0, tmp, 0);
4294 bgt_predict_taken(CCR0, ok);
4295 stop("held monitor count is <= 0 at decrement");
4296 bind(ok);
4297 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Restore CCR0 EQ
4298 #endif
4299 addi(tmp, tmp, -1);
4300 std(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4301 }
4302
4303 // Function to flip between unlocked and locked state (fast locking).
4304 // Branches to failed if the state is not as expected with CCR0 NE.
4305 // Falls through upon success with CCR0 EQ.
4306 // This requires fewer instructions and registers and is easier to use than the
4307 // cmpxchg based implementation.
4308 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4309 assert_different_registers(obj, tmp, R0);
4310 Label retry;
4311
4312 if (semantics & MemBarRel) {
4313 release();
4314 }
4315
4316 bind(retry);
4317 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4318 if (!is_unlock) {
4319 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4320 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4321 andi_(R0, tmp, markWord::lock_mask_in_place);
4322 bne(CCR0, failed); // failed if new header doesn't contain locked_value (which is 0)
4323 } else {
4324 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4325 andi_(R0, tmp, markWord::lock_mask_in_place);
4326 bne(CCR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4327 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4328 }
4329 stdcx_(tmp, obj);
4330 bne(CCR0, retry);
4331
4332 if (semantics & MemBarFenceAfter) {
4333 fence();
4334 } else if (semantics & MemBarAcq) {
4335 isync();
4336 }
4337 }
4338
4339 // Implements lightweight-locking.
4340 //
4341 // - obj: the object to be locked
4342 // - t1, t2: temporary register
4343 void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Label& slow) {
4344 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4345 assert_different_registers(obj, t1, t2);
4346
4347 Label push;
4348 const Register top = t1;
4349 const Register mark = t2;
4350 const Register t = R0;
4351
4352 // Check if the lock-stack is full.
4353 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4354 cmplwi(CCR0, top, LockStack::end_offset());
4355 bge(CCR0, slow);
4356
4357 // The underflow check is elided. The recursive check will always fail
4358 // when the lock stack is empty because of the _bad_oop_sentinel field.
4359
4360 // Check for recursion.
4361 subi(t, top, oopSize);
4362 ldx(t, R16_thread, t);
4363 cmpd(CCR0, obj, t);
4364 beq(CCR0, push);
4365
4366 // Check header for monitor (0b10) or locked (0b00).
4367 ld(mark, oopDesc::mark_offset_in_bytes(), obj);
4368 xori(t, mark, markWord::unlocked_value);
4369 andi_(t, t, markWord::lock_mask_in_place);
4370 bne(CCR0, slow);
4371
4372 // Try to lock. Transition lock bits 0b00 => 0b01
4373 atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow, MacroAssembler::MemBarAcq);
4374
4375 bind(push);
4376 // After successful lock, push object on lock-stack
4377 stdx(obj, R16_thread, top);
4378 addi(top, top, oopSize);
4379 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4380 }
4381
4382 // Implements lightweight-unlocking.
4383 //
4384 // - obj: the object to be unlocked
4385 // - t1: temporary register
4386 void MacroAssembler::lightweight_unlock(Register obj, Register t1, Label& slow) {
4387 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4388 assert_different_registers(obj, t1);
4389
4390 #ifdef ASSERT
4391 {
4392 // The following checks rely on the fact that LockStack is only ever modified by
4393 // its owning thread, even if the lock got inflated concurrently; removal of LockStack
4394 // entries after inflation will happen delayed in that case.
4395
4396 // Check for lock-stack underflow.
4397 Label stack_ok;
4398 lwz(t1, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4399 cmplwi(CCR0, t1, LockStack::start_offset());
4400 bge(CCR0, stack_ok);
4401 stop("Lock-stack underflow");
4402 bind(stack_ok);
4403 }
4404 #endif
4405
4406 Label unlocked, push_and_slow;
4407 const Register top = t1;
4408 const Register mark = R0;
4409 Register t = R0;
4410
4411 // Check if obj is top of lock-stack.
4412 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4413 subi(top, top, oopSize);
4414 ldx(t, R16_thread, top);
4415 cmpd(CCR0, obj, t);
4416 bne(CCR0, slow);
4417
4418 // Pop lock-stack.
4419 DEBUG_ONLY(li(t, 0);)
4420 DEBUG_ONLY(stdx(t, R16_thread, top);)
4421 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4422
4423 // The underflow check is elided. The recursive check will always fail
4424 // when the lock stack is empty because of the _bad_oop_sentinel field.
4425
4426 // Check if recursive.
4427 subi(t, top, oopSize);
4428 ldx(t, R16_thread, t);
4429 cmpd(CCR0, obj, t);
4430 beq(CCR0, unlocked);
4431
4432 // Use top as tmp
4433 t = top;
4434
4435 // Not recursive. Check header for monitor (0b10).
4436 ld(mark, oopDesc::mark_offset_in_bytes(), obj);
4437 andi_(t, mark, markWord::monitor_value);
4438 bne(CCR0, push_and_slow);
4439
4440 #ifdef ASSERT
4441 // Check header not unlocked (0b01).
4442 Label not_unlocked;
4443 andi_(t, mark, markWord::unlocked_value);
4444 beq(CCR0, not_unlocked);
4445 stop("lightweight_unlock already unlocked");
4446 bind(not_unlocked);
4447 #endif
4448
4449 // Try to unlock. Transition lock bits 0b00 => 0b01
4450 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
4451 b(unlocked);
4452
4453 bind(push_and_slow);
4454
4455 // Restore lock-stack and handle the unlock in runtime.
4456 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4457 DEBUG_ONLY(stdx(obj, R16_thread, top);)
4458 addi(top, top, oopSize);
4459 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4460 b(slow);
4461
4462 bind(unlocked);
4463 }