1 /*
  2  * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "code/codeCache.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "gc/shared/collectedHeap.inline.hpp"
 29 #include "gc/shared/gcLocker.inline.hpp"
 30 #include "interpreter/interpreter.hpp"
 31 #include "logging/log.hpp"
 32 #include "memory/metaspaceClosure.hpp"
 33 #include "memory/metadataFactory.hpp"
 34 #include "oops/access.hpp"
 35 #include "oops/compressedOops.inline.hpp"
 36 #include "oops/fieldStreams.inline.hpp"
 37 #include "oops/flatArrayKlass.hpp"
 38 #include "oops/inlineKlass.inline.hpp"
 39 #include "oops/instanceKlass.inline.hpp"
 40 #include "oops/method.hpp"
 41 #include "oops/oop.inline.hpp"
 42 #include "oops/objArrayKlass.hpp"
 43 #include "runtime/fieldDescriptor.inline.hpp"
 44 #include "runtime/handles.inline.hpp"
 45 #include "runtime/safepointVerifiers.hpp"
 46 #include "runtime/sharedRuntime.hpp"
 47 #include "runtime/signature.hpp"
 48 #include "runtime/thread.inline.hpp"
 49 #include "utilities/copy.hpp"
 50 
 51   // Constructor
 52 InlineKlass::InlineKlass(const ClassFileParser& parser)
 53     : InstanceKlass(parser, InstanceKlass::_kind_inline_type, InstanceKlass::ID) {
 54   _adr_inlineklass_fixed_block = inlineklass_static_block();
 55   // Addresses used for inline type calling convention
 56   *((Array<SigEntry>**)adr_extended_sig()) = NULL;
 57   *((Array<VMRegPair>**)adr_return_regs()) = NULL;
 58   *((address*)adr_pack_handler()) = NULL;
 59   *((address*)adr_pack_handler_jobject()) = NULL;
 60   *((address*)adr_unpack_handler()) = NULL;
 61   assert(pack_handler() == NULL, "pack handler not null");
 62   *((int*)adr_default_value_offset()) = 0;
 63   *((address*)adr_null_free_inline_array_klasses()) = NULL;
 64   set_prototype_header(markWord::inline_type_prototype());
 65   assert(is_inline_type_klass(), "sanity");
 66   assert(prototype_header().is_inline_type(), "sanity");
 67 }
 68 
 69 oop InlineKlass::default_value() {
 70   oop val = java_mirror()->obj_field_acquire(default_value_offset());
 71   assert(oopDesc::is_oop(val), "Sanity check");
 72   assert(val->is_inline_type(), "Sanity check");
 73   assert(val->klass() == this, "sanity check");
 74   return val;
 75 }
 76 
 77 int InlineKlass::first_field_offset_old() {
 78 #ifdef ASSERT
 79   int first_offset = INT_MAX;
 80   for (AllFieldStream fs(this); !fs.done(); fs.next()) {
 81     if (fs.offset() < first_offset) first_offset= fs.offset();
 82   }
 83 #endif
 84   int base_offset = instanceOopDesc::base_offset_in_bytes();
 85   // The first field of line types is aligned on a long boundary
 86   base_offset = align_up(base_offset, BytesPerLong);
 87   assert(base_offset == first_offset, "inconsistent offsets");
 88   return base_offset;
 89 }
 90 
 91 instanceOop InlineKlass::allocate_instance(TRAPS) {
 92   int size = size_helper();  // Query before forming handle.
 93 
 94   instanceOop oop = (instanceOop)Universe::heap()->obj_allocate(this, size, CHECK_NULL);
 95   assert(oop->mark().is_inline_type(), "Expected inline type");
 96   return oop;
 97 }
 98 
 99 instanceOop InlineKlass::allocate_instance_buffer(TRAPS) {
100   int size = size_helper();  // Query before forming handle.
101 
102   instanceOop oop = (instanceOop)Universe::heap()->obj_buffer_allocate(this, size, CHECK_NULL);
103   assert(oop->mark().is_inline_type(), "Expected inline type");
104   return oop;
105 }
106 
107 int InlineKlass::nonstatic_oop_count() {
108   int oops = 0;
109   int map_count = nonstatic_oop_map_count();
110   OopMapBlock* block = start_of_nonstatic_oop_maps();
111   OopMapBlock* end = block + map_count;
112   while (block != end) {
113     oops += block->count();
114     block++;
115   }
116   return oops;
117 }
118 
119 oop InlineKlass::read_inlined_field(oop obj, int offset, TRAPS) {
120   oop res = NULL;
121   this->initialize(CHECK_NULL); // will throw an exception if in error state
122   if (is_empty_inline_type()) {
123     res = (instanceOop)default_value();
124   } else {
125     Handle obj_h(THREAD, obj);
126     res = allocate_instance_buffer(CHECK_NULL);
127     inline_copy_payload_to_new_oop(((char*)(oopDesc*)obj_h()) + offset, res);
128   }
129   assert(res != NULL, "Must be set in one of two paths above");
130   return res;
131 }
132 
133 void InlineKlass::write_inlined_field(oop obj, int offset, oop value, TRAPS) {
134   if (value == NULL) {
135     THROW(vmSymbols::java_lang_NullPointerException());
136   }
137   if (!is_empty_inline_type()) {
138     inline_copy_oop_to_payload(value, ((char*)(oopDesc*)obj) + offset);
139   }
140 }
141 
142 // Arrays of...
143 
144 bool InlineKlass::flatten_array() {
145   if (!UseFlatArray) {
146     return false;
147   }
148   // Too big
149   int elem_bytes = get_exact_size_in_bytes();
150   if ((FlatArrayElementMaxSize >= 0) && (elem_bytes > FlatArrayElementMaxSize)) {
151     return false;
152   }
153   // Too many embedded oops
154   if ((FlatArrayElementMaxOops >= 0) && (nonstatic_oop_count() > FlatArrayElementMaxOops)) {
155     return false;
156   }
157   // Declared atomic but not naturally atomic.
158   if (is_declared_atomic() && !is_naturally_atomic()) {
159     return false;
160   }
161   // VM enforcing InlineArrayAtomicAccess only...
162   if (InlineArrayAtomicAccess && (!is_naturally_atomic())) {
163     return false;
164   }
165   return true;
166 }
167 
168 Klass* InlineKlass::null_free_inline_array_klass(int n, TRAPS) {
169   if (Atomic::load_acquire(adr_null_free_inline_array_klasses()) == NULL) {
170     ResourceMark rm(THREAD);
171     JavaThread *jt = JavaThread::cast(THREAD);
172     {
173       // Atomic creation of array_klasses
174       MutexLocker ma(THREAD, MultiArray_lock);
175 
176       // Check if update has already taken place
177       if (null_free_inline_array_klasses() == NULL) {
178         ArrayKlass* k;
179         if (flatten_array()) {
180           k = FlatArrayKlass::allocate_klass(this, CHECK_NULL);
181         } else {
182           k = ObjArrayKlass::allocate_objArray_klass(class_loader_data(), 1, this, true, true, CHECK_NULL);
183 
184         }
185         // use 'release' to pair with lock-free load
186         Atomic::release_store(adr_null_free_inline_array_klasses(), k);
187       }
188     }
189   }
190   ArrayKlass* ak = null_free_inline_array_klasses();
191   return ak->array_klass(n, THREAD);
192 }
193 
194 Klass* InlineKlass::null_free_inline_array_klass_or_null(int n) {
195   // Need load-acquire for lock-free read
196   ArrayKlass* ak = Atomic::load_acquire(adr_null_free_inline_array_klasses());
197   if (ak == NULL) {
198     return NULL;
199   } else {
200     return ak->array_klass_or_null(n);
201   }
202 }
203 
204 Klass* InlineKlass::null_free_inline_array_klass(TRAPS) {
205   return null_free_inline_array_klass(1, THREAD);
206 }
207 
208 Klass* InlineKlass::null_free_inline_array_klass_or_null() {
209   return null_free_inline_array_klass_or_null(1);
210 }
211 
212 void InlineKlass::array_klasses_do(void f(Klass* k)) {
213   InstanceKlass::array_klasses_do(f);
214   if (null_free_inline_array_klasses() != NULL) {
215     null_free_inline_array_klasses()->array_klasses_do(f);
216   }
217 }
218 
219 void InlineKlass::array_klasses_do(void f(Klass* k, TRAPS), TRAPS) {
220   InstanceKlass::array_klasses_do(f, THREAD);
221   if (null_free_inline_array_klasses() != NULL) {
222     null_free_inline_array_klasses()->array_klasses_do(f, THREAD);
223   }
224 }
225 
226 // Inline type arguments are not passed by reference, instead each
227 // field of the inline type is passed as an argument. This helper
228 // function collects the inlined field (recursively)
229 // in a list. Included with the field's type is
230 // the offset of each field in the inline type: i2c and c2i adapters
231 // need that to load or store fields. Finally, the list of fields is
232 // sorted in order of increasing offsets: the adapters and the
233 // compiled code need to agree upon the order of fields.
234 //
235 // The list of basic types that is returned starts with a T_INLINE_TYPE
236 // and ends with an extra T_VOID. T_INLINE_TYPE/T_VOID pairs are used as
237 // delimiters. Every entry between the two is a field of the inline
238 // type. If there's an embedded inline type in the list, it also starts
239 // with a T_INLINE_TYPE and ends with a T_VOID. This is so we can
240 // generate a unique fingerprint for the method's adapters and we can
241 // generate the list of basic types from the interpreter point of view
242 // (inline types passed as reference: iterate on the list until a
243 // T_INLINE_TYPE, drop everything until and including the closing
244 // T_VOID) or the compiler point of view (each field of the inline
245 // types is an argument: drop all T_INLINE_TYPE/T_VOID from the list).
246 int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, int base_off) {
247   int count = 0;
248   SigEntry::add_entry(sig, T_INLINE_TYPE, name(), base_off);
249   for (JavaFieldStream fs(this); !fs.done(); fs.next()) {
250     if (fs.access_flags().is_static()) continue;
251     int offset = base_off + fs.offset() - (base_off > 0 ? first_field_offset() : 0);
252     if (fs.is_inlined()) {
253       // Resolve klass of inlined field and recursively collect fields
254       Klass* vk = get_inline_type_field_klass(fs.index());
255       count += InlineKlass::cast(vk)->collect_fields(sig, offset);
256     } else {
257       BasicType bt = Signature::basic_type(fs.signature());
258       if (bt == T_INLINE_TYPE) {
259         bt = T_OBJECT;
260       }
261       SigEntry::add_entry(sig, bt, fs.signature(), offset);
262       count += type2size[bt];
263     }
264   }
265   int offset = base_off + size_helper()*HeapWordSize - (base_off > 0 ? first_field_offset() : 0);
266   SigEntry::add_entry(sig, T_VOID, name(), offset);
267   if (base_off == 0) {
268     sig->sort(SigEntry::compare);
269   }
270   assert(sig->at(0)._bt == T_INLINE_TYPE && sig->at(sig->length()-1)._bt == T_VOID, "broken structure");
271   return count;
272 }
273 
274 void InlineKlass::initialize_calling_convention(TRAPS) {
275   // Because the pack and unpack handler addresses need to be loadable from generated code,
276   // they are stored at a fixed offset in the klass metadata. Since inline type klasses do
277   // not have a vtable, the vtable offset is used to store these addresses.
278   if (InlineTypeReturnedAsFields || InlineTypePassFieldsAsArgs) {
279     ResourceMark rm;
280     GrowableArray<SigEntry> sig_vk;
281     int nb_fields = collect_fields(&sig_vk);
282     Array<SigEntry>* extended_sig = MetadataFactory::new_array<SigEntry>(class_loader_data(), sig_vk.length(), CHECK);
283     *((Array<SigEntry>**)adr_extended_sig()) = extended_sig;
284     for (int i = 0; i < sig_vk.length(); i++) {
285       extended_sig->at_put(i, sig_vk.at(i));
286     }
287     if (can_be_returned_as_fields(/* init= */ true)) {
288       nb_fields++;
289       BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nb_fields);
290       sig_bt[0] = T_METADATA;
291       SigEntry::fill_sig_bt(&sig_vk, sig_bt+1);
292       VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, nb_fields);
293       int total = SharedRuntime::java_return_convention(sig_bt, regs, nb_fields);
294 
295       if (total > 0) {
296         Array<VMRegPair>* return_regs = MetadataFactory::new_array<VMRegPair>(class_loader_data(), nb_fields, CHECK);
297         *((Array<VMRegPair>**)adr_return_regs()) = return_regs;
298         for (int i = 0; i < nb_fields; i++) {
299           return_regs->at_put(i, regs[i]);
300         }
301 
302         BufferedInlineTypeBlob* buffered_blob = SharedRuntime::generate_buffered_inline_type_adapter(this);
303         *((address*)adr_pack_handler()) = buffered_blob->pack_fields();
304         *((address*)adr_pack_handler_jobject()) = buffered_blob->pack_fields_jobject();
305         *((address*)adr_unpack_handler()) = buffered_blob->unpack_fields();
306         assert(CodeCache::find_blob(pack_handler()) == buffered_blob, "lost track of blob");
307         assert(can_be_returned_as_fields(), "sanity");
308       }
309     }
310     if (!can_be_returned_as_fields() && !can_be_passed_as_fields()) {
311       MetadataFactory::free_array<SigEntry>(class_loader_data(), extended_sig);
312       assert(return_regs() == NULL, "sanity");
313     }
314   }
315 }
316 
317 void InlineKlass::deallocate_contents(ClassLoaderData* loader_data) {
318   if (extended_sig() != NULL) {
319     MetadataFactory::free_array<SigEntry>(loader_data, extended_sig());
320   }
321   if (return_regs() != NULL) {
322     MetadataFactory::free_array<VMRegPair>(loader_data, return_regs());
323   }
324   cleanup_blobs();
325   InstanceKlass::deallocate_contents(loader_data);
326 }
327 
328 void InlineKlass::cleanup(InlineKlass* ik) {
329   ik->cleanup_blobs();
330 }
331 
332 void InlineKlass::cleanup_blobs() {
333   if (pack_handler() != NULL) {
334     CodeBlob* buffered_blob = CodeCache::find_blob(pack_handler());
335     assert(buffered_blob->is_buffered_inline_type_blob(), "bad blob type");
336     BufferBlob::free((BufferBlob*)buffered_blob);
337     *((address*)adr_pack_handler()) = NULL;
338     *((address*)adr_pack_handler_jobject()) = NULL;
339     *((address*)adr_unpack_handler()) = NULL;
340   }
341 }
342 
343 // Can this inline type be passed as multiple values?
344 bool InlineKlass::can_be_passed_as_fields() const {
345   return InlineTypePassFieldsAsArgs;
346 }
347 
348 // Can this inline type be returned as multiple values?
349 bool InlineKlass::can_be_returned_as_fields(bool init) const {
350   return InlineTypeReturnedAsFields && (init || return_regs() != NULL);
351 }
352 
353 // Create handles for all oop fields returned in registers that are going to be live across a safepoint
354 void InlineKlass::save_oop_fields(const RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
355   Thread* thread = Thread::current();
356   const Array<SigEntry>* sig_vk = extended_sig();
357   const Array<VMRegPair>* regs = return_regs();
358   int j = 1;
359 
360   for (int i = 0; i < sig_vk->length(); i++) {
361     BasicType bt = sig_vk->at(i)._bt;
362     if (bt == T_OBJECT || bt == T_ARRAY) {
363       VMRegPair pair = regs->at(j);
364       address loc = reg_map.location(pair.first());
365       oop v = *(oop*)loc;
366       assert(v == NULL || oopDesc::is_oop(v), "not an oop?");
367       assert(Universe::heap()->is_in_or_null(v), "must be heap pointer");
368       handles.push(Handle(thread, v));
369     }
370     if (bt == T_INLINE_TYPE) {
371       continue;
372     }
373     if (bt == T_VOID &&
374         sig_vk->at(i-1)._bt != T_LONG &&
375         sig_vk->at(i-1)._bt != T_DOUBLE) {
376       continue;
377     }
378     j++;
379   }
380   assert(j == regs->length(), "missed a field?");
381 }
382 
383 // Update oop fields in registers from handles after a safepoint
384 void InlineKlass::restore_oop_results(RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
385   assert(InlineTypeReturnedAsFields, "inconsistent");
386   const Array<SigEntry>* sig_vk = extended_sig();
387   const Array<VMRegPair>* regs = return_regs();
388   assert(regs != NULL, "inconsistent");
389 
390   int j = 1;
391   for (int i = 0, k = 0; i < sig_vk->length(); i++) {
392     BasicType bt = sig_vk->at(i)._bt;
393     if (bt == T_OBJECT || bt == T_ARRAY) {
394       VMRegPair pair = regs->at(j);
395       address loc = reg_map.location(pair.first());
396       *(oop*)loc = handles.at(k++)();
397     }
398     if (bt == T_INLINE_TYPE) {
399       continue;
400     }
401     if (bt == T_VOID &&
402         sig_vk->at(i-1)._bt != T_LONG &&
403         sig_vk->at(i-1)._bt != T_DOUBLE) {
404       continue;
405     }
406     j++;
407   }
408   assert(j == regs->length(), "missed a field?");
409 }
410 
411 // Fields are in registers. Create an instance of the inline type and
412 // initialize it with the values of the fields.
413 oop InlineKlass::realloc_result(const RegisterMap& reg_map, const GrowableArray<Handle>& handles, TRAPS) {
414   oop new_vt = allocate_instance(CHECK_NULL);
415   const Array<SigEntry>* sig_vk = extended_sig();
416   const Array<VMRegPair>* regs = return_regs();
417 
418   int j = 1;
419   int k = 0;
420   for (int i = 0; i < sig_vk->length(); i++) {
421     BasicType bt = sig_vk->at(i)._bt;
422     if (bt == T_INLINE_TYPE) {
423       continue;
424     }
425     if (bt == T_VOID) {
426       if (sig_vk->at(i-1)._bt == T_LONG ||
427           sig_vk->at(i-1)._bt == T_DOUBLE) {
428         j++;
429       }
430       continue;
431     }
432     int off = sig_vk->at(i)._offset;
433     assert(off > 0, "offset in object should be positive");
434     VMRegPair pair = regs->at(j);
435     address loc = reg_map.location(pair.first());
436     switch(bt) {
437     case T_BOOLEAN: {
438       new_vt->bool_field_put(off, *(jboolean*)loc);
439       break;
440     }
441     case T_CHAR: {
442       new_vt->char_field_put(off, *(jchar*)loc);
443       break;
444     }
445     case T_BYTE: {
446       new_vt->byte_field_put(off, *(jbyte*)loc);
447       break;
448     }
449     case T_SHORT: {
450       new_vt->short_field_put(off, *(jshort*)loc);
451       break;
452     }
453     case T_INT: {
454       new_vt->int_field_put(off, *(jint*)loc);
455       break;
456     }
457     case T_LONG: {
458 #ifdef _LP64
459       new_vt->double_field_put(off,  *(jdouble*)loc);
460 #else
461       Unimplemented();
462 #endif
463       break;
464     }
465     case T_OBJECT:
466     case T_ARRAY: {
467       Handle handle = handles.at(k++);
468       new_vt->obj_field_put(off, handle());
469       break;
470     }
471     case T_FLOAT: {
472       new_vt->float_field_put(off,  *(jfloat*)loc);
473       break;
474     }
475     case T_DOUBLE: {
476       new_vt->double_field_put(off, *(jdouble*)loc);
477       break;
478     }
479     default:
480       ShouldNotReachHere();
481     }
482     *(intptr_t*)loc = 0xDEAD;
483     j++;
484   }
485   assert(j == regs->length(), "missed a field?");
486   assert(k == handles.length(), "missed an oop?");
487   return new_vt;
488 }
489 
490 // Check the return register for an InlineKlass oop
491 InlineKlass* InlineKlass::returned_inline_klass(const RegisterMap& map) {
492   BasicType bt = T_METADATA;
493   VMRegPair pair;
494   int nb = SharedRuntime::java_return_convention(&bt, &pair, 1);
495   assert(nb == 1, "broken");
496 
497   address loc = map.location(pair.first());
498   intptr_t ptr = *(intptr_t*)loc;
499   if (is_set_nth_bit(ptr, 0)) {
500     // Oop is tagged, must be an InlineKlass oop
501     clear_nth_bit(ptr, 0);
502     assert(Metaspace::contains((void*)ptr), "should be klass");
503     InlineKlass* vk = (InlineKlass*)ptr;
504     assert(vk->can_be_returned_as_fields(), "must be able to return as fields");
505     return vk;
506   }
507 #ifdef ASSERT
508   // Oop is not tagged, must be a valid oop
509   if (VerifyOops) {
510     oopDesc::verify(cast_to_oop(ptr));
511   }
512 #endif
513   return NULL;
514 }
515 
516 // CDS support
517 
518 void InlineKlass::metaspace_pointers_do(MetaspaceClosure* it) {
519   InstanceKlass::metaspace_pointers_do(it);
520 
521   InlineKlass* this_ptr = this;
522   it->push_internal_pointer(&this_ptr, (intptr_t*)&_adr_inlineklass_fixed_block);
523   it->push((Klass**)adr_null_free_inline_array_klasses());
524 }
525 
526 void InlineKlass::remove_unshareable_info() {
527   InstanceKlass::remove_unshareable_info();
528 
529   *((Array<SigEntry>**)adr_extended_sig()) = NULL;
530   *((Array<VMRegPair>**)adr_return_regs()) = NULL;
531   *((address*)adr_pack_handler()) = NULL;
532   *((address*)adr_pack_handler_jobject()) = NULL;
533   *((address*)adr_unpack_handler()) = NULL;
534   assert(pack_handler() == NULL, "pack handler not null");
535   if (null_free_inline_array_klasses() != NULL) {
536     null_free_inline_array_klasses()->remove_unshareable_info();
537   }
538 }
539 
540 void InlineKlass::remove_java_mirror() {
541   InstanceKlass::remove_java_mirror();
542   if (null_free_inline_array_klasses() != NULL) {
543     null_free_inline_array_klasses()->remove_java_mirror();
544   }
545 }
546 
547 void InlineKlass::restore_unshareable_info(ClassLoaderData* loader_data, Handle protection_domain, PackageEntry* pkg_entry, TRAPS) {
548   InstanceKlass::restore_unshareable_info(loader_data, protection_domain, pkg_entry, CHECK);
549   if (null_free_inline_array_klasses() != NULL) {
550     null_free_inline_array_klasses()->restore_unshareable_info(ClassLoaderData::the_null_class_loader_data(), Handle(), CHECK);
551   }
552 }
553 
554 // oop verify
555 
556 void InlineKlass::verify_on(outputStream* st) {
557   InstanceKlass::verify_on(st);
558   guarantee(prototype_header().is_inline_type(), "Prototype header is not inline type");
559 }
560 
561 void InlineKlass::oop_verify_on(oop obj, outputStream* st) {
562   InstanceKlass::oop_verify_on(obj, st);
563   guarantee(obj->mark().is_inline_type(), "Header is not inline type");
564 }