1 /*
  2  * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "code/codeCache.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "gc/shared/collectedHeap.inline.hpp"
 29 #include "gc/shared/gcLocker.inline.hpp"
 30 #include "interpreter/interpreter.hpp"
 31 #include "logging/log.hpp"
 32 #include "memory/metaspaceClosure.hpp"
 33 #include "memory/metadataFactory.hpp"
 34 #include "oops/access.hpp"
 35 #include "oops/compressedOops.inline.hpp"
 36 #include "oops/fieldStreams.inline.hpp"
 37 #include "oops/flatArrayKlass.hpp"
 38 #include "oops/inlineKlass.inline.hpp"
 39 #include "oops/instanceKlass.inline.hpp"
 40 #include "oops/method.hpp"
 41 #include "oops/oop.inline.hpp"
 42 #include "oops/objArrayKlass.hpp"
 43 #include "runtime/fieldDescriptor.inline.hpp"
 44 #include "runtime/handles.inline.hpp"
 45 #include "runtime/safepointVerifiers.hpp"
 46 #include "runtime/sharedRuntime.hpp"
 47 #include "runtime/signature.hpp"
 48 #include "runtime/thread.inline.hpp"
 49 #include "utilities/copy.hpp"
 50 
 51   // Constructor
 52 InlineKlass::InlineKlass(const ClassFileParser& parser)
 53     : InstanceKlass(parser, InstanceKlass::_kind_inline_type, InstanceKlass::ID) {
 54   _adr_inlineklass_fixed_block = inlineklass_static_block();
 55   // Addresses used for inline type calling convention
 56   *((Array<SigEntry>**)adr_extended_sig()) = NULL;
 57   *((Array<VMRegPair>**)adr_return_regs()) = NULL;
 58   *((address*)adr_pack_handler()) = NULL;
 59   *((address*)adr_pack_handler_jobject()) = NULL;
 60   *((address*)adr_unpack_handler()) = NULL;
 61   assert(pack_handler() == NULL, "pack handler not null");
 62   *((int*)adr_default_value_offset()) = 0;
 63   *((address*)adr_value_array_klasses()) = NULL;
 64   set_prototype_header(markWord::inline_type_prototype());
 65   assert(is_inline_type_klass(), "sanity");
 66   assert(prototype_header().is_inline_type(), "sanity");
 67 }
 68 
 69 oop InlineKlass::default_value() {
 70   assert(is_initialized() || is_being_initialized() || is_in_error_state(), "default value is set at the beginning of initialization");
 71   oop val = java_mirror()->obj_field_acquire(default_value_offset());
 72   assert(val != NULL, "Sanity check");
 73   assert(oopDesc::is_oop(val), "Sanity check");
 74   assert(val->is_inline_type(), "Sanity check");
 75   assert(val->klass() == this, "sanity check");
 76   return val;
 77 }
 78 
 79 int InlineKlass::first_field_offset_old() {
 80 #ifdef ASSERT
 81   int first_offset = INT_MAX;
 82   for (AllFieldStream fs(this); !fs.done(); fs.next()) {
 83     if (fs.offset() < first_offset) first_offset= fs.offset();
 84   }
 85 #endif
 86   int base_offset = instanceOopDesc::base_offset_in_bytes();
 87   // The first field of line types is aligned on a long boundary
 88   base_offset = align_up(base_offset, BytesPerLong);
 89   assert(base_offset == first_offset, "inconsistent offsets");
 90   return base_offset;
 91 }
 92 
 93 instanceOop InlineKlass::allocate_instance(TRAPS) {
 94   int size = size_helper();  // Query before forming handle.
 95 
 96   instanceOop oop = (instanceOop)Universe::heap()->obj_allocate(this, size, CHECK_NULL);
 97   assert(oop->mark().is_inline_type(), "Expected inline type");
 98   return oop;
 99 }
100 
101 instanceOop InlineKlass::allocate_instance_buffer(TRAPS) {
102   int size = size_helper();  // Query before forming handle.
103 
104   instanceOop oop = (instanceOop)Universe::heap()->obj_buffer_allocate(this, size, CHECK_NULL);
105   assert(oop->mark().is_inline_type(), "Expected inline type");
106   return oop;
107 }
108 
109 int InlineKlass::nonstatic_oop_count() {
110   int oops = 0;
111   int map_count = nonstatic_oop_map_count();
112   OopMapBlock* block = start_of_nonstatic_oop_maps();
113   OopMapBlock* end = block + map_count;
114   while (block != end) {
115     oops += block->count();
116     block++;
117   }
118   return oops;
119 }
120 
121 oop InlineKlass::read_inlined_field(oop obj, int offset, TRAPS) {
122   oop res = NULL;
123   assert(is_initialized() || is_being_initialized()|| is_in_error_state(),
124         "Must be initialized, initializing or in a corner case of an escaped instance of a class that failed its initialization");
125   if (is_empty_inline_type()) {
126     res = (instanceOop)default_value();
127   } else {
128     Handle obj_h(THREAD, obj);
129     res = allocate_instance_buffer(CHECK_NULL);
130     inline_copy_payload_to_new_oop(((char*)(oopDesc*)obj_h()) + offset, res);
131   }
132   assert(res != NULL, "Must be set in one of two paths above");
133   return res;
134 }
135 
136 void InlineKlass::write_inlined_field(oop obj, int offset, oop value, TRAPS) {
137   if (value == NULL) {
138     THROW(vmSymbols::java_lang_NullPointerException());
139   }
140   if (!is_empty_inline_type()) {
141     inline_copy_oop_to_payload(value, ((char*)(oopDesc*)obj) + offset);
142   }
143 }
144 
145 // Arrays of...
146 
147 bool InlineKlass::flatten_array() {
148   if (!UseFlatArray) {
149     return false;
150   }
151   // Too big
152   int elem_bytes = get_exact_size_in_bytes();
153   if ((FlatArrayElementMaxSize >= 0) && (elem_bytes > FlatArrayElementMaxSize)) {
154     return false;
155   }
156   // Too many embedded oops
157   if ((FlatArrayElementMaxOops >= 0) && (nonstatic_oop_count() > FlatArrayElementMaxOops)) {
158     return false;
159   }
160   // Declared atomic but not naturally atomic.
161   if (is_declared_atomic() && !is_naturally_atomic()) {
162     return false;
163   }
164   // VM enforcing InlineArrayAtomicAccess only...
165   if (InlineArrayAtomicAccess && (!is_naturally_atomic())) {
166     return false;
167   }
168   return true;
169 }
170 
171 Klass* InlineKlass::value_array_klass(int n, TRAPS) {
172   if (Atomic::load_acquire(adr_value_array_klasses()) == NULL) {
173     ResourceMark rm(THREAD);
174     JavaThread *jt = JavaThread::cast(THREAD);
175     {
176       // Atomic creation of array_klasses
177       MutexLocker ma(THREAD, MultiArray_lock);
178 
179       // Check if update has already taken place
180       if (value_array_klasses() == NULL) {
181         ArrayKlass* k;
182         if (flatten_array()) {
183           k = FlatArrayKlass::allocate_klass(this, CHECK_NULL);
184         } else {
185           k = ObjArrayKlass::allocate_objArray_klass(class_loader_data(), 1, this, true, true, CHECK_NULL);
186 
187         }
188         // use 'release' to pair with lock-free load
189         Atomic::release_store(adr_value_array_klasses(), k);
190       }
191     }
192   }
193   ArrayKlass* ak = value_array_klasses();
194   return ak->array_klass(n, THREAD);
195 }
196 
197 Klass* InlineKlass::value_array_klass_or_null(int n) {
198   // Need load-acquire for lock-free read
199   ArrayKlass* ak = Atomic::load_acquire(adr_value_array_klasses());
200   if (ak == NULL) {
201     return NULL;
202   } else {
203     return ak->array_klass_or_null(n);
204   }
205 }
206 
207 Klass* InlineKlass::value_array_klass(TRAPS) {
208   return value_array_klass(1, THREAD);
209 }
210 
211 Klass* InlineKlass::value_array_klass_or_null() {
212   return value_array_klass_or_null(1);
213 }
214 
215 // Inline type arguments are not passed by reference, instead each
216 // field of the inline type is passed as an argument. This helper
217 // function collects the inlined field (recursively)
218 // in a list. Included with the field's type is
219 // the offset of each field in the inline type: i2c and c2i adapters
220 // need that to load or store fields. Finally, the list of fields is
221 // sorted in order of increasing offsets: the adapters and the
222 // compiled code need to agree upon the order of fields.
223 //
224 // The list of basic types that is returned starts with a T_INLINE_TYPE
225 // and ends with an extra T_VOID. T_INLINE_TYPE/T_VOID pairs are used as
226 // delimiters. Every entry between the two is a field of the inline
227 // type. If there's an embedded inline type in the list, it also starts
228 // with a T_INLINE_TYPE and ends with a T_VOID. This is so we can
229 // generate a unique fingerprint for the method's adapters and we can
230 // generate the list of basic types from the interpreter point of view
231 // (inline types passed as reference: iterate on the list until a
232 // T_INLINE_TYPE, drop everything until and including the closing
233 // T_VOID) or the compiler point of view (each field of the inline
234 // types is an argument: drop all T_INLINE_TYPE/T_VOID from the list).
235 int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, int base_off) {
236   int count = 0;
237   SigEntry::add_entry(sig, T_INLINE_TYPE, name(), base_off);
238   for (JavaFieldStream fs(this); !fs.done(); fs.next()) {
239     if (fs.access_flags().is_static()) continue;
240     int offset = base_off + fs.offset() - (base_off > 0 ? first_field_offset() : 0);
241     if (fs.is_inlined()) {
242       // Resolve klass of inlined field and recursively collect fields
243       Klass* vk = get_inline_type_field_klass(fs.index());
244       count += InlineKlass::cast(vk)->collect_fields(sig, offset);
245     } else {
246       BasicType bt = Signature::basic_type(fs.signature());
247       if (bt == T_INLINE_TYPE) {
248         bt = T_OBJECT;
249       }
250       SigEntry::add_entry(sig, bt, fs.signature(), offset);
251       count += type2size[bt];
252     }
253   }
254   int offset = base_off + size_helper()*HeapWordSize - (base_off > 0 ? first_field_offset() : 0);
255   SigEntry::add_entry(sig, T_VOID, name(), offset);
256   if (base_off == 0) {
257     sig->sort(SigEntry::compare);
258   }
259   assert(sig->at(0)._bt == T_INLINE_TYPE && sig->at(sig->length()-1)._bt == T_VOID, "broken structure");
260   return count;
261 }
262 
263 void InlineKlass::initialize_calling_convention(TRAPS) {
264   // Because the pack and unpack handler addresses need to be loadable from generated code,
265   // they are stored at a fixed offset in the klass metadata. Since inline type klasses do
266   // not have a vtable, the vtable offset is used to store these addresses.
267   if (InlineTypeReturnedAsFields || InlineTypePassFieldsAsArgs) {
268     ResourceMark rm;
269     GrowableArray<SigEntry> sig_vk;
270     int nb_fields = collect_fields(&sig_vk);
271     Array<SigEntry>* extended_sig = MetadataFactory::new_array<SigEntry>(class_loader_data(), sig_vk.length(), CHECK);
272     *((Array<SigEntry>**)adr_extended_sig()) = extended_sig;
273     for (int i = 0; i < sig_vk.length(); i++) {
274       extended_sig->at_put(i, sig_vk.at(i));
275     }
276     if (can_be_returned_as_fields(/* init= */ true)) {
277       nb_fields++;
278       BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nb_fields);
279       sig_bt[0] = T_METADATA;
280       SigEntry::fill_sig_bt(&sig_vk, sig_bt+1);
281       VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, nb_fields);
282       int total = SharedRuntime::java_return_convention(sig_bt, regs, nb_fields);
283 
284       if (total > 0) {
285         Array<VMRegPair>* return_regs = MetadataFactory::new_array<VMRegPair>(class_loader_data(), nb_fields, CHECK);
286         *((Array<VMRegPair>**)adr_return_regs()) = return_regs;
287         for (int i = 0; i < nb_fields; i++) {
288           return_regs->at_put(i, regs[i]);
289         }
290 
291         BufferedInlineTypeBlob* buffered_blob = SharedRuntime::generate_buffered_inline_type_adapter(this);
292         *((address*)adr_pack_handler()) = buffered_blob->pack_fields();
293         *((address*)adr_pack_handler_jobject()) = buffered_blob->pack_fields_jobject();
294         *((address*)adr_unpack_handler()) = buffered_blob->unpack_fields();
295         assert(CodeCache::find_blob(pack_handler()) == buffered_blob, "lost track of blob");
296         assert(can_be_returned_as_fields(), "sanity");
297       }
298     }
299     if (!can_be_returned_as_fields() && !can_be_passed_as_fields()) {
300       MetadataFactory::free_array<SigEntry>(class_loader_data(), extended_sig);
301       assert(return_regs() == NULL, "sanity");
302     }
303   }
304 }
305 
306 void InlineKlass::deallocate_contents(ClassLoaderData* loader_data) {
307   if (extended_sig() != NULL) {
308     MetadataFactory::free_array<SigEntry>(loader_data, extended_sig());
309   }
310   if (return_regs() != NULL) {
311     MetadataFactory::free_array<VMRegPair>(loader_data, return_regs());
312   }
313   cleanup_blobs();
314   InstanceKlass::deallocate_contents(loader_data);
315 }
316 
317 void InlineKlass::cleanup(InlineKlass* ik) {
318   ik->cleanup_blobs();
319 }
320 
321 void InlineKlass::cleanup_blobs() {
322   if (pack_handler() != NULL) {
323     CodeBlob* buffered_blob = CodeCache::find_blob(pack_handler());
324     assert(buffered_blob->is_buffered_inline_type_blob(), "bad blob type");
325     BufferBlob::free((BufferBlob*)buffered_blob);
326     *((address*)adr_pack_handler()) = NULL;
327     *((address*)adr_pack_handler_jobject()) = NULL;
328     *((address*)adr_unpack_handler()) = NULL;
329   }
330 }
331 
332 // Can this inline type be passed as multiple values?
333 bool InlineKlass::can_be_passed_as_fields() const {
334   return InlineTypePassFieldsAsArgs;
335 }
336 
337 // Can this inline type be returned as multiple values?
338 bool InlineKlass::can_be_returned_as_fields(bool init) const {
339   return InlineTypeReturnedAsFields && (init || return_regs() != NULL);
340 }
341 
342 // Create handles for all oop fields returned in registers that are going to be live across a safepoint
343 void InlineKlass::save_oop_fields(const RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
344   Thread* thread = Thread::current();
345   const Array<SigEntry>* sig_vk = extended_sig();
346   const Array<VMRegPair>* regs = return_regs();
347   int j = 1;
348 
349   for (int i = 0; i < sig_vk->length(); i++) {
350     BasicType bt = sig_vk->at(i)._bt;
351     if (bt == T_OBJECT || bt == T_ARRAY) {
352       VMRegPair pair = regs->at(j);
353       address loc = reg_map.location(pair.first());
354       oop v = *(oop*)loc;
355       assert(v == NULL || oopDesc::is_oop(v), "not an oop?");
356       assert(Universe::heap()->is_in_or_null(v), "must be heap pointer");
357       handles.push(Handle(thread, v));
358     }
359     if (bt == T_INLINE_TYPE) {
360       continue;
361     }
362     if (bt == T_VOID &&
363         sig_vk->at(i-1)._bt != T_LONG &&
364         sig_vk->at(i-1)._bt != T_DOUBLE) {
365       continue;
366     }
367     j++;
368   }
369   assert(j == regs->length(), "missed a field?");
370 }
371 
372 // Update oop fields in registers from handles after a safepoint
373 void InlineKlass::restore_oop_results(RegisterMap& reg_map, GrowableArray<Handle>& handles) const {
374   assert(InlineTypeReturnedAsFields, "inconsistent");
375   const Array<SigEntry>* sig_vk = extended_sig();
376   const Array<VMRegPair>* regs = return_regs();
377   assert(regs != NULL, "inconsistent");
378 
379   int j = 1;
380   for (int i = 0, k = 0; i < sig_vk->length(); i++) {
381     BasicType bt = sig_vk->at(i)._bt;
382     if (bt == T_OBJECT || bt == T_ARRAY) {
383       VMRegPair pair = regs->at(j);
384       address loc = reg_map.location(pair.first());
385       *(oop*)loc = handles.at(k++)();
386     }
387     if (bt == T_INLINE_TYPE) {
388       continue;
389     }
390     if (bt == T_VOID &&
391         sig_vk->at(i-1)._bt != T_LONG &&
392         sig_vk->at(i-1)._bt != T_DOUBLE) {
393       continue;
394     }
395     j++;
396   }
397   assert(j == regs->length(), "missed a field?");
398 }
399 
400 // Fields are in registers. Create an instance of the inline type and
401 // initialize it with the values of the fields.
402 oop InlineKlass::realloc_result(const RegisterMap& reg_map, const GrowableArray<Handle>& handles, TRAPS) {
403   oop new_vt = allocate_instance(CHECK_NULL);
404   const Array<SigEntry>* sig_vk = extended_sig();
405   const Array<VMRegPair>* regs = return_regs();
406 
407   int j = 1;
408   int k = 0;
409   for (int i = 0; i < sig_vk->length(); i++) {
410     BasicType bt = sig_vk->at(i)._bt;
411     if (bt == T_INLINE_TYPE) {
412       continue;
413     }
414     if (bt == T_VOID) {
415       if (sig_vk->at(i-1)._bt == T_LONG ||
416           sig_vk->at(i-1)._bt == T_DOUBLE) {
417         j++;
418       }
419       continue;
420     }
421     int off = sig_vk->at(i)._offset;
422     assert(off > 0, "offset in object should be positive");
423     VMRegPair pair = regs->at(j);
424     address loc = reg_map.location(pair.first());
425     switch(bt) {
426     case T_BOOLEAN: {
427       new_vt->bool_field_put(off, *(jboolean*)loc);
428       break;
429     }
430     case T_CHAR: {
431       new_vt->char_field_put(off, *(jchar*)loc);
432       break;
433     }
434     case T_BYTE: {
435       new_vt->byte_field_put(off, *(jbyte*)loc);
436       break;
437     }
438     case T_SHORT: {
439       new_vt->short_field_put(off, *(jshort*)loc);
440       break;
441     }
442     case T_INT: {
443       new_vt->int_field_put(off, *(jint*)loc);
444       break;
445     }
446     case T_LONG: {
447 #ifdef _LP64
448       new_vt->double_field_put(off,  *(jdouble*)loc);
449 #else
450       Unimplemented();
451 #endif
452       break;
453     }
454     case T_OBJECT:
455     case T_ARRAY: {
456       Handle handle = handles.at(k++);
457       new_vt->obj_field_put(off, handle());
458       break;
459     }
460     case T_FLOAT: {
461       new_vt->float_field_put(off,  *(jfloat*)loc);
462       break;
463     }
464     case T_DOUBLE: {
465       new_vt->double_field_put(off, *(jdouble*)loc);
466       break;
467     }
468     default:
469       ShouldNotReachHere();
470     }
471     *(intptr_t*)loc = 0xDEAD;
472     j++;
473   }
474   assert(j == regs->length(), "missed a field?");
475   assert(k == handles.length(), "missed an oop?");
476   return new_vt;
477 }
478 
479 // Check the return register for an InlineKlass oop
480 InlineKlass* InlineKlass::returned_inline_klass(const RegisterMap& map) {
481   BasicType bt = T_METADATA;
482   VMRegPair pair;
483   int nb = SharedRuntime::java_return_convention(&bt, &pair, 1);
484   assert(nb == 1, "broken");
485 
486   address loc = map.location(pair.first());
487   intptr_t ptr = *(intptr_t*)loc;
488   if (is_set_nth_bit(ptr, 0)) {
489     // Oop is tagged, must be an InlineKlass oop
490     clear_nth_bit(ptr, 0);
491     assert(Metaspace::contains((void*)ptr), "should be klass");
492     InlineKlass* vk = (InlineKlass*)ptr;
493     assert(vk->can_be_returned_as_fields(), "must be able to return as fields");
494     return vk;
495   }
496 #ifdef ASSERT
497   // Oop is not tagged, must be a valid oop
498   if (VerifyOops) {
499     oopDesc::verify(cast_to_oop(ptr));
500   }
501 #endif
502   return NULL;
503 }
504 
505 // CDS support
506 
507 void InlineKlass::metaspace_pointers_do(MetaspaceClosure* it) {
508   InstanceKlass::metaspace_pointers_do(it);
509 
510   InlineKlass* this_ptr = this;
511   it->push_internal_pointer(&this_ptr, (intptr_t*)&_adr_inlineklass_fixed_block);
512   it->push((Klass**)adr_value_array_klasses());
513 }
514 
515 void InlineKlass::remove_unshareable_info() {
516   InstanceKlass::remove_unshareable_info();
517 
518   *((Array<SigEntry>**)adr_extended_sig()) = NULL;
519   *((Array<VMRegPair>**)adr_return_regs()) = NULL;
520   *((address*)adr_pack_handler()) = NULL;
521   *((address*)adr_pack_handler_jobject()) = NULL;
522   *((address*)adr_unpack_handler()) = NULL;
523   assert(pack_handler() == NULL, "pack handler not null");
524   if (value_array_klasses() != NULL) {
525     value_array_klasses()->remove_unshareable_info();
526   }
527 }
528 
529 void InlineKlass::remove_java_mirror() {
530   InstanceKlass::remove_java_mirror();
531   if (value_array_klasses() != NULL) {
532     value_array_klasses()->remove_java_mirror();
533   }
534 }
535 
536 void InlineKlass::restore_unshareable_info(ClassLoaderData* loader_data, Handle protection_domain, PackageEntry* pkg_entry, TRAPS) {
537   InstanceKlass::restore_unshareable_info(loader_data, protection_domain, pkg_entry, CHECK);
538   if (value_array_klasses() != NULL) {
539     value_array_klasses()->restore_unshareable_info(ClassLoaderData::the_null_class_loader_data(), Handle(), CHECK);
540   }
541 }
542 
543 // oop verify
544 
545 void InlineKlass::verify_on(outputStream* st) {
546   InstanceKlass::verify_on(st);
547   guarantee(prototype_header().is_inline_type(), "Prototype header is not inline type");
548 }
549 
550 void InlineKlass::oop_verify_on(oop obj, outputStream* st) {
551   InstanceKlass::oop_verify_on(obj, st);
552   guarantee(obj->mark().is_inline_type(), "Header is not inline type");
553 }