1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/assembler.hpp"
27 #include "c1/c1_LIRAssembler.hpp"
28 #include "c1/c1_MacroAssembler.hpp"
29 #include "ci/ciArrayKlass.hpp"
30 #include "oops/objArrayKlass.hpp"
31 #include "runtime/stubRoutines.hpp"
32
33 #define __ _masm->
34
35
36 void LIR_Assembler::generic_arraycopy(Register src, Register src_pos, Register length,
37 Register dst, Register dst_pos, CodeStub *stub) {
38 assert(src == x11 && src_pos == x12, "mismatch in calling convention");
39 // Save the arguments in case the generic arraycopy fails and we
40 // have to fall back to the JNI stub
41 arraycopy_store_args(src, src_pos, length, dst, dst_pos);
42
43 address copyfunc_addr = StubRoutines::generic_arraycopy();
44 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
45
46 // The arguments are in java calling convention so we shift them
47 // to C convention
48 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
49 __ mv(c_rarg0, j_rarg0);
50 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
51 __ mv(c_rarg1, j_rarg1);
52 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
53 __ mv(c_rarg2, j_rarg2);
54 assert_different_registers(c_rarg3, j_rarg4);
55 __ mv(c_rarg3, j_rarg3);
56 __ mv(c_rarg4, j_rarg4);
57 #ifndef PRODUCT
58 if (PrintC1Statistics) {
59 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
60 }
61 #endif
62 __ far_call(RuntimeAddress(copyfunc_addr));
63 __ beqz(x10, *stub->continuation());
64 // Reload values from the stack so they are where the stub
65 // expects them.
66 arraycopy_load_args(src, src_pos, length, dst, dst_pos);
67
68 // x10 is -1^K where K == partial copied count
69 __ xori(t0, x10, -1);
70 // adjust length down and src/end pos up by partial copied count
71 __ subw(length, length, t0);
72 __ addw(src_pos, src_pos, t0);
73 __ addw(dst_pos, dst_pos, t0);
74 __ j(*stub->entry());
75
76 __ bind(*stub->continuation());
77 }
78
79 void LIR_Assembler::arraycopy_simple_check(Register src, Register src_pos, Register length,
80 Register dst, Register dst_pos, Register tmp,
81 CodeStub *stub, int flags) {
82 // test for null
83 if (flags & LIR_OpArrayCopy::src_null_check) {
84 __ beqz(src, *stub->entry(), /* is_far */ true);
85 }
86 if (flags & LIR_OpArrayCopy::dst_null_check) {
87 __ beqz(dst, *stub->entry(), /* is_far */ true);
88 }
89
90 // If the compiler was not able to prove that exact type of the source or the destination
91 // of the arraycopy is an array type, check at runtime if the source or the destination is
92 // an instance type.
93 if (flags & LIR_OpArrayCopy::type_check) {
94 assert(Klass::_lh_neutral_value == 0, "or replace bgez instructions");
95 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
96 __ load_klass(tmp, dst);
97 __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
98 __ bgez(t0, *stub->entry(), /* is_far */ true);
99 }
100
101 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
102 __ load_klass(tmp, src);
103 __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
104 __ bgez(t0, *stub->entry(), /* is_far */ true);
105 }
106 }
107
108 // check if negative
109 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
110 __ bltz(src_pos, *stub->entry(), /* is_far */ true);
111 }
112 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
113 __ bltz(dst_pos, *stub->entry(), /* is_far */ true);
114 }
115 if (flags & LIR_OpArrayCopy::length_positive_check) {
116 __ bltz(length, *stub->entry(), /* is_far */ true);
117 }
118
119 if (flags & LIR_OpArrayCopy::src_range_check) {
120 __ addw(tmp, src_pos, length);
121 __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes()));
122 __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
123 }
124 if (flags & LIR_OpArrayCopy::dst_range_check) {
125 __ addw(tmp, dst_pos, length);
126 __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes()));
127 __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
128 }
129 }
130
131 void LIR_Assembler::arraycopy_checkcast(Register src, Register src_pos, Register length,
132 Register dst, Register dst_pos, Register tmp,
133 CodeStub *stub, BasicType basic_type,
134 address copyfunc_addr, int flags) {
135 // src is not a sub class of dst so we have to do a
136 // per-element check.
137 int mask = LIR_OpArrayCopy::src_objarray | LIR_OpArrayCopy::dst_objarray;
138 if ((flags & mask) != mask) {
139 // Check that at least both of them object arrays.
140 assert(flags & mask, "one of the two should be known to be an object array");
141
142 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
143 __ load_klass(tmp, src);
144 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
145 __ load_klass(tmp, dst);
146 }
147 int lh_offset = in_bytes(Klass::layout_helper_offset());
148 Address klass_lh_addr(tmp, lh_offset);
149 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
150 __ lw(t0, klass_lh_addr);
151 __ mv(t1, objArray_lh);
152 __ bne(t0, t1, *stub->entry(), /* is_far */ true);
153 }
154
155 // Spill because stubs can use any register they like and it's
156 // easier to restore just those that we care about.
157 arraycopy_store_args(src, src_pos, length, dst, dst_pos);
158 arraycopy_checkcast_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
159 __ far_call(RuntimeAddress(copyfunc_addr));
160
161 #ifndef PRODUCT
162 if (PrintC1Statistics) {
163 Label failed;
164 __ bnez(x10, failed);
165 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
166 __ bind(failed);
167 }
168 #endif
169
170 __ beqz(x10, *stub->continuation());
171
172 #ifndef PRODUCT
173 if (PrintC1Statistics) {
174 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
175 }
176 #endif
177 assert_different_registers(dst, dst_pos, length, src_pos, src, x10, t0);
178
179 // Restore previously spilled arguments
180 arraycopy_load_args(src, src_pos, length, dst, dst_pos);
181
182 // return value is -1^K where K is partial copied count
183 __ xori(t0, x10, -1);
184 // adjust length down and src/end pos up by partial copied count
185 __ subw(length, length, t0);
186 __ addw(src_pos, src_pos, t0);
187 __ addw(dst_pos, dst_pos, t0);
188 }
189
190 void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Register length,
191 Register dst, Register dst_pos, Register tmp,
192 CodeStub *stub, BasicType basic_type, int flags) {
193 // We don't know the array types are compatible
194 if (basic_type != T_OBJECT) {
195 // Simple test for basic type arrays
196 if (UseCompactObjectHeaders) {
197 __ load_narrow_klass_compact(tmp, src);
198 __ load_narrow_klass_compact(t0, dst);
199 } else {
200 __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
201 __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
202 }
203 __ bne(tmp, t0, *stub->entry(), /* is_far */ true);
204 } else {
205 // For object arrays, if src is a sub class of dst then we can
206 // safely do the copy.
207 Label cont, slow;
208
209 #define PUSH(r1, r2) \
210 __ subi(sp, sp, 2 * wordSize); \
211 __ sd(r1, Address(sp, 1 * wordSize)); \
212 __ sd(r2, Address(sp, 0));
213
214 #define POP(r1, r2) \
215 __ ld(r1, Address(sp, 1 * wordSize)); \
216 __ ld(r2, Address(sp, 0)); \
217 __ addi(sp, sp, 2 * wordSize);
218
219 PUSH(src, dst);
220 __ load_klass(src, src);
221 __ load_klass(dst, dst);
222 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
223
224 PUSH(src, dst);
225 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
226 POP(src, dst);
227 __ bnez(dst, cont);
228
229 __ bind(slow);
230 POP(src, dst);
231
232 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
233 if (copyfunc_addr != nullptr) { // use stub if available
234 arraycopy_checkcast(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, copyfunc_addr, flags);
235 }
236
237 __ j(*stub->entry());
238 __ bind(cont);
239 POP(src, dst);
240 }
241 }
242
243 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
244 if (null_check) {
245 __ beqz(obj, *slow_path->entry(), /* is_far */ true);
246 }
247 if (is_dest) {
248 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
249 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
250 } else {
251 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
252 }
253 }
254
255 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
256 ciArrayKlass *default_type = op->expected_type();
257 Register src = op->src()->as_register();
258 Register dst = op->dst()->as_register();
259 Register src_pos = op->src_pos()->as_register();
260 Register dst_pos = op->dst_pos()->as_register();
261 Register length = op->length()->as_register();
262 Register tmp = op->tmp()->as_register();
263
264 CodeStub* stub = op->stub();
265 int flags = op->flags();
266 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
267 if (is_reference_type(basic_type)) { basic_type = T_OBJECT; }
268
269 if (flags & LIR_OpArrayCopy::always_slow_path) {
270 __ j(*stub->entry());
271 __ bind(*stub->continuation());
272 return;
273 }
274
275 // if we don't know anything, just go through the generic arraycopy
276 if (default_type == nullptr) {
277 generic_arraycopy(src, src_pos, length, dst, dst_pos, stub);
278 return;
279 }
280
281 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
282 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
283 }
284 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
285 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
286 }
287
288 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(),
289 "must be true at this point");
290
291 arraycopy_simple_check(src, src_pos, length, dst, dst_pos, tmp, stub, flags);
292
293 if (flags & LIR_OpArrayCopy::type_check) {
294 arraycopy_type_check(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, flags);
295 }
296
297 #ifdef ASSERT
298 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
299 // Sanity check the known type with the incoming class. For the
300 // primitive case the types must match exactly with src.klass and
301 // dst.klass each exactly matching the default type. For the
302 // object array case, if no type check is needed then either the
303 // dst type is exactly the expected type and the src type is a
304 // subtype which we can't check or src is the same array as dst
305 // but not necessarily exactly of type default_type.
306 Label known_ok, halt;
307 __ mov_metadata(tmp, default_type->constant_encoding());
308
309 if (basic_type != T_OBJECT) {
310 __ cmp_klass_bne(dst, tmp, t0, t1, halt);
311 __ cmp_klass_beq(src, tmp, t0, t1, known_ok);
312 } else {
313 __ cmp_klass_beq(dst, tmp, t0, t1, known_ok);
314 __ beq(src, dst, known_ok);
315 }
316 __ bind(halt);
317 __ stop("incorrect type information in arraycopy");
318 __ bind(known_ok);
319 }
320 #endif
321
322 #ifndef PRODUCT
323 if (PrintC1Statistics) {
324 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
325 }
326 #endif
327 arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
328
329 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
330 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
331 const char *name = nullptr;
332 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
333
334 if (CodeCache::contains(entry)) {
335 __ far_call(RuntimeAddress(entry));
336 } else {
337 const int args_num = 3;
338 __ call_VM_leaf(entry, args_num);
339 }
340
341 if (stub != nullptr) {
342 __ bind(*stub->continuation());
343 }
344 }
345
346
347 void LIR_Assembler::arraycopy_prepare_params(Register src, Register src_pos, Register length,
348 Register dst, Register dst_pos, BasicType basic_type) {
349 int scale = array_element_size(basic_type);
350 __ shadd(c_rarg0, src_pos, src, t0, scale);
351 __ addi(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
352 assert_different_registers(c_rarg0, dst, dst_pos, length);
353 __ shadd(c_rarg1, dst_pos, dst, t0, scale);
354 __ addi(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
355 assert_different_registers(c_rarg1, dst, length);
356 __ mv(c_rarg2, length);
357 assert_different_registers(c_rarg2, dst);
358 }
359
360 void LIR_Assembler::arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
361 Register dst, Register dst_pos, BasicType basic_type) {
362 arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
363 __ load_klass(c_rarg4, dst);
364 __ ld(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
365 __ lwu(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
366 }
367
368 void LIR_Assembler::arraycopy_store_args(Register src, Register src_pos, Register length,
369 Register dst, Register dst_pos) {
370 __ sd(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset
371 __ sd(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset
372 __ sd(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset
373 __ sd(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
374 __ sd(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset
375 }
376
377 void LIR_Assembler::arraycopy_load_args(Register src, Register src_pos, Register length,
378 Register dst, Register dst_pos) {
379 __ ld(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset
380 __ ld(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset
381 __ ld(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset
382 __ ld(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
383 __ ld(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset
384 }
385
386 #undef __