1 /*
2 * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_LIRAssembler.hpp"
29 #include "c1/c1_MacroAssembler.hpp"
30 #include "ci/ciArrayKlass.hpp"
31 #include "oops/objArrayKlass.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #define __ _masm->
35
36
37 void LIR_Assembler::generic_arraycopy(Register src, Register src_pos, Register length,
38 Register dst, Register dst_pos, CodeStub *stub) {
39 assert(src == x11 && src_pos == x12, "mismatch in calling convention");
40 // Save the arguments in case the generic arraycopy fails and we
41 // have to fall back to the JNI stub
42 arraycopy_store_args(src, src_pos, length, dst, dst_pos);
43
44 address copyfunc_addr = StubRoutines::generic_arraycopy();
45 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
46
47 // The arguments are in java calling convention so we shift them
48 // to C convention
49 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
50 __ mv(c_rarg0, j_rarg0);
51 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
52 __ mv(c_rarg1, j_rarg1);
53 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
54 __ mv(c_rarg2, j_rarg2);
55 assert_different_registers(c_rarg3, j_rarg4);
56 __ mv(c_rarg3, j_rarg3);
57 __ mv(c_rarg4, j_rarg4);
58 #ifndef PRODUCT
59 if (PrintC1Statistics) {
60 __ add_memory_int32(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), 1);
61 }
62 #endif
63 __ far_call(RuntimeAddress(copyfunc_addr));
64 __ beqz(x10, *stub->continuation());
65 // Reload values from the stack so they are where the stub
66 // expects them.
67 arraycopy_load_args(src, src_pos, length, dst, dst_pos);
68
69 // x10 is -1^K where K == partial copied count
70 __ xori(t0, x10, -1);
71 // adjust length down and src/end pos up by partial copied count
72 __ subw(length, length, t0);
73 __ addw(src_pos, src_pos, t0);
74 __ addw(dst_pos, dst_pos, t0);
75 __ j(*stub->entry());
76
77 __ bind(*stub->continuation());
78 }
79
80 void LIR_Assembler::arraycopy_simple_check(Register src, Register src_pos, Register length,
81 Register dst, Register dst_pos, Register tmp,
82 CodeStub *stub, int flags) {
83 // test for NULL
84 if (flags & LIR_OpArrayCopy::src_null_check) {
85 __ beqz(src, *stub->entry(), /* is_far */ true);
86 }
87 if (flags & LIR_OpArrayCopy::dst_null_check) {
88 __ beqz(dst, *stub->entry(), /* is_far */ true);
89 }
90
91 // If the compiler was not able to prove that exact type of the source or the destination
92 // of the arraycopy is an array type, check at runtime if the source or the destination is
93 // an instance type.
94 if (flags & LIR_OpArrayCopy::type_check) {
95 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
96 __ load_klass(tmp, dst);
97 __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
98 __ li(t1, Klass::_lh_neutral_value);
99 __ bge(t0, t1, *stub->entry(), /* is_far */ true);
100 }
101
102 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
103 __ load_klass(tmp, src);
104 __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
105 __ li(t1, Klass::_lh_neutral_value);
106 __ bge(t0, t1, *stub->entry(), /* is_far */ true);
107 }
108 }
109
110 // check if negative
111 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
112 __ bltz(src_pos, *stub->entry(), /* is_far */ true);
113 }
114 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
115 __ bltz(dst_pos, *stub->entry(), /* is_far */ true);
116 }
117 if (flags & LIR_OpArrayCopy::length_positive_check) {
118 __ bltz(length, *stub->entry(), /* is_far */ true);
119 }
120
121 if (flags & LIR_OpArrayCopy::src_range_check) {
122 __ addw(tmp, src_pos, length);
123 __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes()));
124 __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
125 }
126 if (flags & LIR_OpArrayCopy::dst_range_check) {
127 __ addw(tmp, dst_pos, length);
128 __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes()));
129 __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
130 }
131 }
132
133 void LIR_Assembler::arraycopy_checkcast(Register src, Register src_pos, Register length,
134 Register dst, Register dst_pos, Register tmp,
135 CodeStub *stub, BasicType basic_type,
136 address copyfunc_addr, int flags) {
137 // src is not a sub class of dst so we have to do a
138 // per-element check.
139 int mask = LIR_OpArrayCopy::src_objarray | LIR_OpArrayCopy::dst_objarray;
140 if ((flags & mask) != mask) {
141 // Check that at least both of them object arrays.
142 assert(flags & mask, "one of the two should be known to be an object array");
143
144 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
145 __ load_klass(tmp, src);
146 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
147 __ load_klass(tmp, dst);
148 }
149 int lh_offset = in_bytes(Klass::layout_helper_offset());
150 Address klass_lh_addr(tmp, lh_offset);
151 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
152 __ lw(t0, klass_lh_addr);
153 __ mvw(t1, objArray_lh);
154 __ bne(t0, t1, *stub->entry(), /* is_far */ true);
155 }
156
157 // Spill because stubs can use any register they like and it's
158 // easier to restore just those that we care about.
159 arraycopy_store_args(src, src_pos, length, dst, dst_pos);
160 arraycopy_checkcast_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
161 __ far_call(RuntimeAddress(copyfunc_addr));
162
163 #ifndef PRODUCT
164 if (PrintC1Statistics) {
165 Label failed;
166 __ bnez(x10, failed);
167 __ add_memory_int32(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), 1);
168 __ bind(failed);
169 }
170 #endif
171
172 __ beqz(x10, *stub->continuation());
173
174 #ifndef PRODUCT
175 if (PrintC1Statistics) {
176 __ add_memory_int32(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), 1);
177 }
178 #endif
179 assert_different_registers(dst, dst_pos, length, src_pos, src, x10, t0);
180
181 // Restore previously spilled arguments
182 arraycopy_load_args(src, src_pos, length, dst, dst_pos);
183
184 // return value is -1^K where K is partial copied count
185 __ xori(t0, x10, -1);
186 // adjust length down and src/end pos up by partial copied count
187 __ subw(length, length, t0);
188 __ addw(src_pos, src_pos, t0);
189 __ addw(dst_pos, dst_pos, t0);
190 }
191
192 void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Register length,
193 Register dst, Register dst_pos, Register tmp,
194 CodeStub *stub, BasicType basic_type, int flags) {
195 // We don't know the array types are compatible
196 if (basic_type != T_OBJECT) {
197 // Simple test for basic type arrays
198 if (UseCompressedClassPointers) {
199 __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
200 __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
201 } else {
202 __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
203 __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
204 }
205 __ bne(tmp, t0, *stub->entry(), /* is_far */ true);
206 } else {
207 // For object arrays, if src is a sub class of dst then we can
208 // safely do the copy.
209 Label cont, slow;
210
211 #define PUSH(r1, r2) \
212 __ addi(sp, sp, -2 * wordSize); \
213 __ sd(r1, Address(sp, 1 * wordSize)); \
214 __ sd(r2, Address(sp, 0));
215
216 #define POP(r1, r2) \
217 __ ld(r1, Address(sp, 1 * wordSize)); \
218 __ ld(r2, Address(sp, 0)); \
219 __ addi(sp, sp, 2 * wordSize);
220
221 PUSH(src, dst);
222 __ load_klass(src, src);
223 __ load_klass(dst, dst);
224 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
225
226 PUSH(src, dst);
227 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
228 POP(src, dst);
229 __ bnez(dst, cont);
230
231 __ bind(slow);
232 POP(src, dst);
233
234 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
235 if (copyfunc_addr != NULL) { // use stub if available
236 arraycopy_checkcast(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, copyfunc_addr, flags);
237 }
238
239 __ j(*stub->entry());
240 __ bind(cont);
241 POP(src, dst);
242 }
243 }
244
245 void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
246 assert(default_type != NULL, "NULL default_type!");
247 BasicType basic_type = default_type->element_type()->basic_type();
248
249 if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
250 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
251 // Sanity check the known type with the incoming class. For the
252 // primitive case the types must match exactly with src.klass and
253 // dst.klass each exactly matching the default type. For the
254 // object array case, if no type check is needed then either the
255 // dst type is exactly the expected type and the src type is a
256 // subtype which we can't check or src is the same array as dst
257 // but not necessarily exactly of type default_type.
258 Label known_ok, halt;
259 __ mov_metadata(tmp, default_type->constant_encoding());
260 if (UseCompressedClassPointers) {
261 __ encode_klass_not_null(tmp);
262 }
263
264 if (basic_type != T_OBJECT) {
265 if (UseCompressedClassPointers) {
266 __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
267 } else {
268 __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
269 }
270 __ bne(tmp, t0, halt);
271 if (UseCompressedClassPointers) {
272 __ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes()));
273 } else {
274 __ ld(t0, Address(src, oopDesc::klass_offset_in_bytes()));
275 }
276 __ beq(tmp, t0, known_ok);
277 } else {
278 if (UseCompressedClassPointers) {
279 __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
280 } else {
281 __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
282 }
283 __ beq(tmp, t0, known_ok);
284 __ beq(src, dst, known_ok);
285 }
286 __ bind(halt);
287 __ stop("incorrect type information in arraycopy");
288 __ bind(known_ok);
289 }
290 }
291
292 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
293 ciArrayKlass *default_type = op->expected_type();
294 Register src = op->src()->as_register();
295 Register dst = op->dst()->as_register();
296 Register src_pos = op->src_pos()->as_register();
297 Register dst_pos = op->dst_pos()->as_register();
298 Register length = op->length()->as_register();
299 Register tmp = op->tmp()->as_register();
300
301 CodeStub* stub = op->stub();
302 int flags = op->flags();
303 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
304 if (is_reference_type(basic_type)) { basic_type = T_OBJECT; }
305
306 // if we don't know anything, just go through the generic arraycopy
307 if (default_type == NULL) {
308 generic_arraycopy(src, src_pos, length, dst, dst_pos, stub);
309 return;
310 }
311
312 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(),
313 "must be true at this point");
314
315 arraycopy_simple_check(src, src_pos, length, dst, dst_pos, tmp, stub, flags);
316
317 if (flags & LIR_OpArrayCopy::type_check) {
318 arraycopy_type_check(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, flags);
319 }
320
321 #ifdef ASSERT
322 arraycopy_assert(src, dst, tmp, default_type, flags);
323 #endif
324
325 #ifndef PRODUCT
326 if (PrintC1Statistics) {
327 __ add_memory_int32(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), 1);
328 }
329 #endif
330 arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
331
332 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
333 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
334 const char *name = NULL;
335 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
336
337 CodeBlob *cb = CodeCache::find_blob(entry);
338 if (cb != NULL) {
339 __ far_call(RuntimeAddress(entry));
340 } else {
341 const int args_num = 3;
342 __ call_VM_leaf(entry, args_num);
343 }
344
345 __ bind(*stub->continuation());
346 }
347
348
349 void LIR_Assembler::arraycopy_prepare_params(Register src, Register src_pos, Register length,
350 Register dst, Register dst_pos, BasicType basic_type) {
351 int scale = array_element_size(basic_type);
352 __ shadd(c_rarg0, src_pos, src, t0, scale);
353 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
354 assert_different_registers(c_rarg0, dst, dst_pos, length);
355 __ shadd(c_rarg1, dst_pos, dst, t0, scale);
356 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
357 assert_different_registers(c_rarg1, dst, length);
358 __ mv(c_rarg2, length);
359 assert_different_registers(c_rarg2, dst);
360 }
361
362 void LIR_Assembler::arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
363 Register dst, Register dst_pos, BasicType basic_type) {
364 arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
365 __ load_klass(c_rarg4, dst);
366 __ ld(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
367 __ lwu(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
368 }
369
370 void LIR_Assembler::arraycopy_store_args(Register src, Register src_pos, Register length,
371 Register dst, Register dst_pos) {
372 __ sd(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset
373 __ sd(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset
374 __ sd(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset
375 __ sd(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
376 __ sd(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset
377 }
378
379 void LIR_Assembler::arraycopy_load_args(Register src, Register src_pos, Register length,
380 Register dst, Register dst_pos) {
381 __ ld(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset
382 __ ld(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset
383 __ ld(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset
384 __ ld(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
385 __ ld(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset
386 }
387
388 #undef __