1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "code/vtableStubs.hpp" 27 #include "compiler/compileBroker.hpp" 28 #include "compiler/disassembler.hpp" 29 #include "logging/log.hpp" 30 #include "memory/allocation.inline.hpp" 31 #include "memory/resourceArea.hpp" 32 #include "oops/instanceKlass.hpp" 33 #include "oops/klass.inline.hpp" 34 #include "oops/klassVtable.hpp" 35 #include "oops/oop.inline.hpp" 36 #include "prims/forte.hpp" 37 #include "prims/jvmtiExport.hpp" 38 #include "runtime/handles.inline.hpp" 39 #include "runtime/mutexLocker.hpp" 40 #include "runtime/sharedRuntime.hpp" 41 #include "utilities/align.hpp" 42 #include "utilities/powerOfTwo.hpp" 43 #ifdef COMPILER2 44 #include "opto/matcher.hpp" 45 #endif 46 47 // ----------------------------------------------------------------------------------------- 48 // Implementation of VtableStub 49 50 address VtableStub::_chunk = nullptr; 51 address VtableStub::_chunk_end = nullptr; 52 VMReg VtableStub::_receiver_location = VMRegImpl::Bad(); 53 54 55 void* VtableStub::operator new(size_t size, int code_size) throw() { 56 assert_lock_strong(VtableStubs_lock); 57 assert(size == sizeof(VtableStub), "mismatched size"); 58 // compute real VtableStub size (rounded to nearest word) 59 const int real_size = align_up(code_size + (int)sizeof(VtableStub), wordSize); 60 // malloc them in chunks to minimize header overhead 61 const int chunk_factor = 32; 62 if (_chunk == nullptr || _chunk + real_size > _chunk_end) { 63 const int bytes = chunk_factor * real_size + pd_code_alignment(); 64 65 // There is a dependency on the name of the blob in src/share/vm/prims/jvmtiCodeBlobEvents.cpp 66 // If changing the name, update the other file accordingly. 67 VtableBlob* blob = VtableBlob::create("vtable chunks", bytes); 68 if (blob == nullptr) { 69 return nullptr; 70 } 71 _chunk = blob->content_begin(); 72 _chunk_end = _chunk + bytes; 73 Forte::register_stub("vtable stub", _chunk, _chunk_end); 74 align_chunk(); 75 } 76 assert(_chunk + real_size <= _chunk_end, "bad allocation"); 77 void* res = _chunk; 78 _chunk += real_size; 79 align_chunk(); 80 return res; 81 } 82 83 84 void VtableStub::print_on(outputStream* st) const { 85 st->print("vtable stub (index = %d, receiver_location = " INTX_FORMAT ", code = [" INTPTR_FORMAT ", " INTPTR_FORMAT "])", 86 index(), p2i(receiver_location()), p2i(code_begin()), p2i(code_end())); 87 } 88 89 void VtableStub::print() const { print_on(tty); } 90 91 // ----------------------------------------------------------------------------------------- 92 // Implementation of VtableStubs 93 // 94 // For each hash value there's a linked list of vtable stubs (with that 95 // hash value). Each list is anchored in a little hash _table, indexed 96 // by that hash value. 97 98 VtableStub* volatile VtableStubs::_table[VtableStubs::N]; 99 int VtableStubs::_vtab_stub_size = 0; 100 int VtableStubs::_itab_stub_size = 0; 101 102 #if defined(PRODUCT) 103 // These values are good for the PRODUCT case (no tracing). 104 static const int first_vtableStub_size = 64; 105 static const int first_itableStub_size = 256; 106 #else 107 // These values are good for the non-PRODUCT case (when tracing can be switched on). 108 // To find out, run test workload with 109 // -Xlog:vtablestubs=Trace -XX:+CountCompiledCalls -XX:+DebugVtables 110 // and use the reported "estimate" value. 111 // Here is a list of observed worst-case values: 112 // vtable itable 113 // aarch64: 460 324 114 // arm: ? ? 115 // ppc (linux, BE): 404 288 116 // ppc (linux, LE): 356 276 117 // ppc (AIX): 416 296 118 // s390x: 408 256 119 // Solaris-sparc: 792 348 120 // x86 (Linux): 670 309 121 // x86 (MacOS): 682 321 122 static const int first_vtableStub_size = 1024; 123 static const int first_itableStub_size = 512; 124 #endif 125 126 127 void VtableStubs::initialize() { 128 assert(VtableStub::_receiver_location == VMRegImpl::Bad(), "initialized multiple times?"); 129 130 VtableStub::_receiver_location = SharedRuntime::name_for_receiver(); 131 { 132 MutexLocker ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); 133 for (int i = 0; i < N; i++) { 134 Atomic::store(&_table[i], (VtableStub*)nullptr); 135 } 136 } 137 } 138 139 140 int VtableStubs::code_size_limit(bool is_vtable_stub) { 141 if (is_vtable_stub) { 142 return _vtab_stub_size > 0 ? _vtab_stub_size : first_vtableStub_size; 143 } else { // itable stub 144 return _itab_stub_size > 0 ? _itab_stub_size : first_itableStub_size; 145 } 146 } // code_size_limit 147 148 149 void VtableStubs::check_and_set_size_limit(bool is_vtable_stub, 150 int code_size, 151 int padding) { 152 const char* name = is_vtable_stub ? "vtable" : "itable"; 153 154 guarantee(code_size <= code_size_limit(is_vtable_stub), 155 "buffer overflow in %s stub, code_size is %d, limit is %d", name, code_size, code_size_limit(is_vtable_stub)); 156 157 if (is_vtable_stub) { 158 if (log_is_enabled(Trace, vtablestubs)) { 159 if ( (_vtab_stub_size > 0) && ((code_size + padding) > _vtab_stub_size) ) { 160 log_trace(vtablestubs)("%s size estimate needed adjustment from %d to %d bytes", 161 name, _vtab_stub_size, code_size + padding); 162 } 163 } 164 if ( (code_size + padding) > _vtab_stub_size ) { 165 _vtab_stub_size = code_size + padding; 166 } 167 } else { // itable stub 168 if (log_is_enabled(Trace, vtablestubs)) { 169 if ( (_itab_stub_size > 0) && ((code_size + padding) > _itab_stub_size) ) { 170 log_trace(vtablestubs)("%s size estimate needed adjustment from %d to %d bytes", 171 name, _itab_stub_size, code_size + padding); 172 } 173 } 174 if ( (code_size + padding) > _itab_stub_size ) { 175 _itab_stub_size = code_size + padding; 176 } 177 } 178 return; 179 } // check_and_set_size_limit 180 181 182 void VtableStubs::bookkeeping(MacroAssembler* masm, outputStream* out, VtableStub* s, 183 address npe_addr, address ame_addr, bool is_vtable_stub, 184 int index, int slop_bytes, int index_dependent_slop) { 185 const char* name = is_vtable_stub ? "vtable" : "itable"; 186 const int stub_length = code_size_limit(is_vtable_stub); 187 188 if (log_is_enabled(Trace, vtablestubs)) { 189 log_trace(vtablestubs)("%s #%d at " PTR_FORMAT ": size: %d, estimate: %d, slop area: %d", 190 name, index, p2i(s->code_begin()), 191 (int)(masm->pc() - s->code_begin()), 192 stub_length, 193 (int)(s->code_end() - masm->pc())); 194 } 195 guarantee(masm->pc() <= s->code_end(), "%s #%d: overflowed buffer, estimated len: %d, actual len: %d, overrun: %d", 196 name, index, stub_length, 197 (int)(masm->pc() - s->code_begin()), 198 (int)(masm->pc() - s->code_end())); 199 assert((masm->pc() + index_dependent_slop) <= s->code_end(), "%s #%d: spare space for 32-bit offset: required = %d, available = %d", 200 name, index, index_dependent_slop, 201 (int)(s->code_end() - masm->pc())); 202 203 // After the first vtable/itable stub is generated, we have a much 204 // better estimate for the stub size. Remember/update this 205 // estimate after some sanity checks. 206 check_and_set_size_limit(is_vtable_stub, masm->offset(), slop_bytes); 207 s->set_exception_points(npe_addr, ame_addr); 208 } 209 210 211 address VtableStubs::find_stub(bool is_vtable_stub, int vtable_index, bool caller_is_c1) { 212 assert(vtable_index >= 0, "must be positive"); 213 214 VtableStub* s; 215 { 216 MutexLocker ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); 217 s = lookup(is_vtable_stub, vtable_index, caller_is_c1); 218 if (s == nullptr) { 219 if (is_vtable_stub) { 220 s = create_vtable_stub(vtable_index, caller_is_c1); 221 } else { 222 s = create_itable_stub(vtable_index, caller_is_c1); 223 } 224 225 // Creation of vtable or itable can fail if there is not enough free space in the code cache. 226 if (s == nullptr) { 227 return nullptr; 228 } 229 230 enter(is_vtable_stub, vtable_index, caller_is_c1, s); 231 if (PrintAdapterHandlers) { 232 tty->print_cr("Decoding VtableStub (%s) %s[%d]@" PTR_FORMAT " [" PTR_FORMAT ", " PTR_FORMAT "] (" SIZE_FORMAT " bytes)", 233 caller_is_c1 ? "c1" : "full opt", 234 is_vtable_stub? "vtbl": "itbl", vtable_index, p2i(VtableStub::receiver_location()), 235 p2i(s->code_begin()), p2i(s->code_end()), pointer_delta(s->code_end(), s->code_begin(), 1)); 236 Disassembler::decode(s->code_begin(), s->code_end()); 237 } 238 // Notify JVMTI about this stub. The event will be recorded by the enclosing 239 // JvmtiDynamicCodeEventCollector and posted when this thread has released 240 // all locks. Only post this event if a new state is not required. Creating a new state would 241 // cause a safepoint and the caller of this code has a NoSafepointVerifier. 242 if (JvmtiExport::should_post_dynamic_code_generated()) { 243 JvmtiExport::post_dynamic_code_generated_while_holding_locks(is_vtable_stub? "vtable stub": "itable stub", // FIXME: need to pass caller_is_c1?? 244 s->code_begin(), s->code_end()); 245 } 246 } 247 } 248 return s->entry_point(); 249 } 250 251 252 inline uint VtableStubs::hash(bool is_vtable_stub, int vtable_index, bool caller_is_c1) { 253 // Assumption: receiver_location < 4 in most cases. 254 int hash = ((vtable_index << 2) ^ VtableStub::receiver_location()->value()) + vtable_index; 255 if (caller_is_c1) { 256 hash = 7 - hash; 257 } 258 return (is_vtable_stub ? ~hash : hash) & mask; 259 } 260 261 262 inline uint VtableStubs::unsafe_hash(address entry_point, bool caller_is_c1) { 263 // The entrypoint may or may not be a VtableStub. Generate a hash as if it was. 264 address vtable_stub_addr = entry_point - VtableStub::entry_offset(); 265 assert(CodeCache::contains(vtable_stub_addr), "assumed to always be the case"); 266 address vtable_type_addr = vtable_stub_addr + offset_of(VtableStub, _type); 267 address vtable_index_addr = vtable_stub_addr + offset_of(VtableStub, _index); 268 bool is_vtable_stub = *vtable_type_addr == static_cast<uint8_t>(VtableStub::Type::vtable_stub); 269 short vtable_index; 270 static_assert(sizeof(VtableStub::_index) == sizeof(vtable_index), "precondition"); 271 memcpy(&vtable_index, vtable_index_addr, sizeof(vtable_index)); 272 return hash(is_vtable_stub, vtable_index, caller_is_c1); 273 } 274 275 VtableStub* VtableStubs::lookup(bool is_vtable_stub, int vtable_index, bool caller_is_c1) { 276 assert_lock_strong(VtableStubs_lock); 277 unsigned hash = VtableStubs::hash(is_vtable_stub, vtable_index, caller_is_c1); 278 VtableStub* s = Atomic::load(&_table[hash]); 279 while( s && !s->matches(is_vtable_stub, vtable_index, caller_is_c1)) s = s->next(); 280 return s; 281 } 282 283 284 void VtableStubs::enter(bool is_vtable_stub, int vtable_index, bool caller_is_c1, VtableStub* s) { 285 assert_lock_strong(VtableStubs_lock); 286 assert(s->matches(is_vtable_stub, vtable_index, caller_is_c1), "bad vtable stub"); 287 unsigned int h = VtableStubs::hash(is_vtable_stub, vtable_index, caller_is_c1); 288 // Insert s at the beginning of the corresponding list. 289 s->set_next(Atomic::load(&_table[h])); 290 // Make sure that concurrent readers not taking the mutex observe the writing of "next". 291 Atomic::release_store(&_table[h], s); 292 } 293 294 VtableStub* VtableStubs::entry_point(address pc) { 295 // The pc may or may not be the entry point for a VtableStub. Use unsafe_hash 296 // to generate the hash that would have been used if it was. The lookup in the 297 // _table will only succeed if there is a VtableStub with an entry point at 298 // the pc. 299 MutexLocker ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); 300 VtableStub* stub = (VtableStub*)(pc - VtableStub::entry_offset()); 301 uint hash = VtableStubs::unsafe_hash(pc, stub->caller_is_c1()); 302 VtableStub* s; 303 for (s = Atomic::load(&_table[hash]); s != nullptr && s->entry_point() != pc; s = s->next()) {} 304 return (s != nullptr && s->entry_point() == pc) ? s : nullptr; 305 } 306 307 bool VtableStubs::contains(address pc) { 308 // simple solution for now - we may want to use 309 // a faster way if this function is called often 310 return stub_containing(pc) != nullptr; 311 } 312 313 314 VtableStub* VtableStubs::stub_containing(address pc) { 315 for (int i = 0; i < N; i++) { 316 for (VtableStub* s = Atomic::load_acquire(&_table[i]); s != nullptr; s = s->next()) { 317 if (s->contains(pc)) return s; 318 } 319 } 320 return nullptr; 321 } 322 323 void vtableStubs_init() { 324 VtableStubs::initialize(); 325 } 326 327 void VtableStubs::vtable_stub_do(void f(VtableStub*)) { 328 for (int i = 0; i < N; i++) { 329 for (VtableStub* s = Atomic::load_acquire(&_table[i]); s != nullptr; s = s->next()) { 330 f(s); 331 } 332 } 333 } 334 335 336 //----------------------------------------------------------------------------------------------------- 337 // Non-product code 338 #ifndef PRODUCT 339 340 extern "C" void bad_compiled_vtable_index(JavaThread* thread, oop receiver, int index) { 341 ResourceMark rm; 342 Klass* klass = receiver->klass(); 343 InstanceKlass* ik = InstanceKlass::cast(klass); 344 klassVtable vt = ik->vtable(); 345 ik->print(); 346 fatal("bad compiled vtable dispatch: receiver " INTPTR_FORMAT ", " 347 "index %d (vtable length %d)", 348 p2i(receiver), index, vt.length()); 349 } 350 351 #endif // PRODUCT --- EOF ---