1 /*
  2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  4  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
  5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  6  *
  7  * This code is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License version 2 only, as
  9  * published by the Free Software Foundation.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  *
 25  */
 26 
 27 #include "precompiled.hpp"
 28 #include "asm/assembler.hpp"
 29 #include "c1/c1_LIRAssembler.hpp"
 30 #include "c1/c1_MacroAssembler.hpp"
 31 #include "ci/ciArrayKlass.hpp"
 32 #include "oops/objArrayKlass.hpp"
 33 #include "runtime/stubRoutines.hpp"
 34 
 35 #define __ _masm->
 36 
 37 
 38 void LIR_Assembler::generic_arraycopy(Register src, Register src_pos, Register length,
 39                                       Register dst, Register dst_pos, CodeStub *stub) {
 40   assert(src == x11 && src_pos == x12, "mismatch in calling convention");
 41   // Save the arguments in case the generic arraycopy fails and we
 42   // have to fall back to the JNI stub
 43   arraycopy_store_args(src, src_pos, length, dst, dst_pos);
 44 
 45   address copyfunc_addr = StubRoutines::generic_arraycopy();
 46   assert(copyfunc_addr != NULL, "generic arraycopy stub required");
 47 
 48   // The arguments are in java calling convention so we shift them
 49   // to C convention
 50   assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
 51   __ mv(c_rarg0, j_rarg0);
 52   assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
 53   __ mv(c_rarg1, j_rarg1);
 54   assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
 55   __ mv(c_rarg2, j_rarg2);
 56   assert_different_registers(c_rarg3, j_rarg4);
 57   __ mv(c_rarg3, j_rarg3);
 58   __ mv(c_rarg4, j_rarg4);
 59 #ifndef PRODUCT
 60   if (PrintC1Statistics) {
 61     __ add_memory_int32(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), 1);
 62   }
 63 #endif
 64   __ far_call(RuntimeAddress(copyfunc_addr));
 65   __ beqz(x10, *stub->continuation());
 66   // Reload values from the stack so they are where the stub
 67   // expects them.
 68   arraycopy_load_args(src, src_pos, length, dst, dst_pos);
 69 
 70   // x10 is -1^K where K == partial copied count
 71   __ xori(t0, x10, -1);
 72   // adjust length down and src/end pos up by partial copied count
 73   __ subw(length, length, t0);
 74   __ addw(src_pos, src_pos, t0);
 75   __ addw(dst_pos, dst_pos, t0);
 76   __ j(*stub->entry());
 77 
 78   __ bind(*stub->continuation());
 79 }
 80 
 81 void LIR_Assembler::arraycopy_simple_check(Register src, Register src_pos, Register length,
 82                                            Register dst, Register dst_pos, Register tmp,
 83                                            CodeStub *stub, int flags) {
 84   // test for NULL
 85   if (flags & LIR_OpArrayCopy::src_null_check) {
 86     __ beqz(src, *stub->entry(), /* is_far */ true);
 87   }
 88   if (flags & LIR_OpArrayCopy::dst_null_check) {
 89     __ beqz(dst, *stub->entry(), /* is_far */ true);
 90   }
 91 
 92   // If the compiler was not able to prove that exact type of the source or the destination
 93   // of the arraycopy is an array type, check at runtime if the source or the destination is
 94   // an instance type.
 95   if (flags & LIR_OpArrayCopy::type_check) {
 96     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
 97       __ load_klass(tmp, dst);
 98       __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
 99       __ li(t1, Klass::_lh_neutral_value);
100       __ bge(t0, t1, *stub->entry(), /* is_far */ true);
101     }
102 
103     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
104       __ load_klass(tmp, src);
105       __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset())));
106       __ li(t1, Klass::_lh_neutral_value);
107       __ bge(t0, t1, *stub->entry(), /* is_far */ true);
108     }
109   }
110 
111   // check if negative
112   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
113     __ bltz(src_pos, *stub->entry(), /* is_far */ true);
114   }
115   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
116     __ bltz(dst_pos, *stub->entry(), /* is_far */ true);
117   }
118   if (flags & LIR_OpArrayCopy::length_positive_check) {
119     __ bltz(length, *stub->entry(), /* is_far */ true);
120   }
121 
122   if (flags & LIR_OpArrayCopy::src_range_check) {
123     __ addw(tmp, src_pos, length);
124     __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes()));
125     __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
126   }
127   if (flags & LIR_OpArrayCopy::dst_range_check) {
128     __ addw(tmp, dst_pos, length);
129     __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes()));
130     __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true);
131   }
132 }
133 
134 void LIR_Assembler::arraycopy_checkcast(Register src, Register src_pos, Register length,
135                                         Register dst, Register dst_pos, Register tmp,
136                                         CodeStub *stub, BasicType basic_type,
137                                         address copyfunc_addr, int flags) {
138   // src is not a sub class of dst so we have to do a
139   // per-element check.
140   int mask = LIR_OpArrayCopy::src_objarray | LIR_OpArrayCopy::dst_objarray;
141   if ((flags & mask) != mask) {
142     // Check that at least both of them object arrays.
143     assert(flags & mask, "one of the two should be known to be an object array");
144 
145     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
146       __ load_klass(tmp, src);
147     } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
148       __ load_klass(tmp, dst);
149     }
150     int lh_offset = in_bytes(Klass::layout_helper_offset());
151     Address klass_lh_addr(tmp, lh_offset);
152     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
153     __ lw(t0, klass_lh_addr);
154     __ mvw(t1, objArray_lh);
155     __ bne(t0, t1, *stub->entry(), /* is_far */ true);
156   }
157 
158   // Spill because stubs can use any register they like and it's
159   // easier to restore just those that we care about.
160   arraycopy_store_args(src, src_pos, length, dst, dst_pos);
161   arraycopy_checkcast_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
162   __ far_call(RuntimeAddress(copyfunc_addr));
163 
164 #ifndef PRODUCT
165   if (PrintC1Statistics) {
166     Label failed;
167     __ bnez(x10, failed);
168     __ add_memory_int32(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), 1);
169     __ bind(failed);
170   }
171 #endif
172 
173   __ beqz(x10, *stub->continuation());
174 
175 #ifndef PRODUCT
176   if (PrintC1Statistics) {
177     __ add_memory_int32(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), 1);
178   }
179 #endif
180   assert_different_registers(dst, dst_pos, length, src_pos, src, x10, t0);
181 
182   // Restore previously spilled arguments
183   arraycopy_load_args(src, src_pos, length, dst, dst_pos);
184 
185   // return value is -1^K where K is partial copied count
186   __ xori(t0, x10, -1);
187   // adjust length down and src/end pos up by partial copied count
188   __ subw(length, length, t0);
189   __ addw(src_pos, src_pos, t0);
190   __ addw(dst_pos, dst_pos, t0);
191 }
192 
193 void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Register length,
194                                          Register dst, Register dst_pos, Register tmp,
195                                          CodeStub *stub, BasicType basic_type, int flags) {
196   // We don't know the array types are compatible
197   if (basic_type != T_OBJECT) {
198     // Simple test for basic type arrays
199     if (UseCompressedClassPointers) {
200       __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
201       __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
202     } else {
203       __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
204       __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
205     }
206     __ bne(tmp, t0, *stub->entry(), /* is_far */ true);
207   } else {
208     // For object arrays, if src is a sub class of dst then we can
209     // safely do the copy.
210     Label cont, slow;
211 
212 #define PUSH(r1, r2)                                     \
213     __ addi(sp, sp, -2 * wordSize);                      \
214     __ sd(r1, Address(sp, 1 * wordSize));                \
215     __ sd(r2, Address(sp, 0));
216 
217 #define POP(r1, r2)                                      \
218     __ ld(r1, Address(sp, 1 * wordSize));                \
219     __ ld(r2, Address(sp, 0));                           \
220     __ addi(sp, sp, 2 * wordSize);
221 
222     PUSH(src, dst);
223     __ load_klass(src, src);
224     __ load_klass(dst, dst);
225     __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
226 
227     PUSH(src, dst);
228     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
229     POP(src, dst);
230     __ bnez(dst, cont);
231 
232     __ bind(slow);
233     POP(src, dst);
234 
235     address copyfunc_addr = StubRoutines::checkcast_arraycopy();
236     if (copyfunc_addr != NULL) { // use stub if available
237       arraycopy_checkcast(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, copyfunc_addr, flags);
238     }
239 
240     __ j(*stub->entry());
241     __ bind(cont);
242     POP(src, dst);
243   }
244 }
245 
246 void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
247   assert(default_type != NULL, "NULL default_type!");
248   BasicType basic_type = default_type->element_type()->basic_type();
249 
250   if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
251   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
252     // Sanity check the known type with the incoming class.  For the
253     // primitive case the types must match exactly with src.klass and
254     // dst.klass each exactly matching the default type.  For the
255     // object array case, if no type check is needed then either the
256     // dst type is exactly the expected type and the src type is a
257     // subtype which we can't check or src is the same array as dst
258     // but not necessarily exactly of type default_type.
259     Label known_ok, halt;
260     __ mov_metadata(tmp, default_type->constant_encoding());
261     if (UseCompressedClassPointers) {
262       __ encode_klass_not_null(tmp);
263     }
264 
265     if (basic_type != T_OBJECT) {
266       if (UseCompressedClassPointers) {
267         __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
268       } else {
269         __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
270       }
271       __ bne(tmp, t0, halt);
272       if (UseCompressedClassPointers) {
273         __ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes()));
274       } else {
275         __ ld(t0, Address(src, oopDesc::klass_offset_in_bytes()));
276       }
277       __ beq(tmp, t0, known_ok);
278     } else {
279       if (UseCompressedClassPointers) {
280         __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
281       } else {
282         __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
283       }
284       __ beq(tmp, t0, known_ok);
285       __ beq(src, dst, known_ok);
286     }
287     __ bind(halt);
288     __ stop("incorrect type information in arraycopy");
289     __ bind(known_ok);
290   }
291 }
292 
293 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
294   ciArrayKlass *default_type = op->expected_type();
295   Register src = op->src()->as_register();
296   Register dst = op->dst()->as_register();
297   Register src_pos = op->src_pos()->as_register();
298   Register dst_pos = op->dst_pos()->as_register();
299   Register length = op->length()->as_register();
300   Register tmp = op->tmp()->as_register();
301 
302   CodeStub* stub = op->stub();
303   int flags = op->flags();
304   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
305   if (is_reference_type(basic_type)) { basic_type = T_OBJECT; }
306 
307   // if we don't know anything, just go through the generic arraycopy
308   if (default_type == NULL) {
309     generic_arraycopy(src, src_pos, length, dst, dst_pos, stub);
310     return;
311   }
312 
313   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(),
314          "must be true at this point");
315 
316   arraycopy_simple_check(src, src_pos, length, dst, dst_pos, tmp, stub, flags);
317 
318   if (flags & LIR_OpArrayCopy::type_check) {
319     arraycopy_type_check(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, flags);
320   }
321 
322 #ifdef ASSERT
323   arraycopy_assert(src, dst, tmp, default_type, flags);
324 #endif
325 
326 #ifndef PRODUCT
327   if (PrintC1Statistics) {
328     __ add_memory_int32(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), 1);
329   }
330 #endif
331   arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
332 
333   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
334   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
335   const char *name = NULL;
336   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
337 
338   CodeBlob *cb = CodeCache::find_blob(entry);
339   if (cb != NULL) {
340     __ far_call(RuntimeAddress(entry));
341   } else {
342     const int args_num = 3;
343     __ call_VM_leaf(entry, args_num);
344   }
345 
346   __ bind(*stub->continuation());
347 }
348 
349 
350 void LIR_Assembler::arraycopy_prepare_params(Register src, Register src_pos, Register length,
351                                              Register dst, Register dst_pos, BasicType basic_type) {
352   int scale = array_element_size(basic_type);
353   __ slli(t0, src_pos, scale);
354   __ add(c_rarg0, src, t0);
355   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
356   assert_different_registers(c_rarg0, dst, dst_pos, length);
357   __ slli(t0, dst_pos, scale);
358   __ add(c_rarg1, dst, t0);
359   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
360   assert_different_registers(c_rarg1, dst, length);
361   __ mv(c_rarg2, length);
362   assert_different_registers(c_rarg2, dst);
363 }
364 
365 void LIR_Assembler::arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
366                                                        Register dst, Register dst_pos, BasicType basic_type) {
367   arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type);
368   __ load_klass(c_rarg4, dst);
369   __ ld(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
370   __ lwu(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
371 }
372 
373 void LIR_Assembler::arraycopy_store_args(Register src, Register src_pos, Register length,
374                                          Register dst, Register dst_pos) {
375   __ sd(dst_pos, Address(sp, 0));                // 0: dst_pos sp offset
376   __ sd(dst, Address(sp, 1 * BytesPerWord));     // 1: dst sp offset
377   __ sd(length, Address(sp, 2 * BytesPerWord));  // 2: length sp offset
378   __ sd(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
379   __ sd(src, Address(sp, 4 * BytesPerWord));     // 4: src sp offset
380 }
381 
382 void LIR_Assembler::arraycopy_load_args(Register src, Register src_pos, Register length,
383                                         Register dst, Register dst_pos) {
384   __ ld(dst_pos, Address(sp, 0));                // 0: dst_pos sp offset
385   __ ld(dst, Address(sp, 1 * BytesPerWord));     // 1: dst sp offset
386   __ ld(length, Address(sp, 2 * BytesPerWord));  // 2: length sp offset
387   __ ld(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset
388   __ ld(src, Address(sp, 4 * BytesPerWord));     // 4: src sp offset
389 }
390 
391 #undef __