16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.inline.hpp"
26 #include "code/aotCodeCache.hpp"
27 #include "code/compiledIC.hpp"
28 #include "code/debugInfo.hpp"
29 #include "code/debugInfoRec.hpp"
30 #include "compiler/compileBroker.hpp"
31 #include "compiler/compilerDirectives.hpp"
32 #include "compiler/disassembler.hpp"
33 #include "compiler/oopMap.hpp"
34 #include "gc/shared/barrierSet.hpp"
35 #include "gc/shared/c2/barrierSetC2.hpp"
36 #include "memory/allocation.hpp"
37 #include "opto/ad.hpp"
38 #include "opto/block.hpp"
39 #include "opto/c2_MacroAssembler.hpp"
40 #include "opto/c2compiler.hpp"
41 #include "opto/callnode.hpp"
42 #include "opto/cfgnode.hpp"
43 #include "opto/locknode.hpp"
44 #include "opto/machnode.hpp"
45 #include "opto/node.hpp"
46 #include "opto/optoreg.hpp"
47 #include "opto/output.hpp"
48 #include "opto/regalloc.hpp"
49 #include "opto/type.hpp"
50 #include "runtime/sharedRuntime.hpp"
51 #include "utilities/macros.hpp"
52 #include "utilities/powerOfTwo.hpp"
53 #include "utilities/xmlstream.hpp"
54
55 #ifndef PRODUCT
205 #endif
206
207 };
208
209 PhaseOutput::PhaseOutput()
210 : Phase(Phase::Output),
211 _code_buffer("Compile::Fill_buffer"),
212 _first_block_size(0),
213 _handler_table(),
214 _inc_table(),
215 _stub_list(),
216 _oop_map_set(nullptr),
217 _scratch_buffer_blob(nullptr),
218 _scratch_locs_memory(nullptr),
219 _scratch_const_size(-1),
220 _in_scratch_emit_size(false),
221 _frame_slots(0),
222 _code_offsets(),
223 _node_bundling_limit(0),
224 _node_bundling_base(nullptr),
225 _orig_pc_slot(0),
226 _orig_pc_slot_offset_in_bytes(0),
227 _buf_sizes(),
228 _block(nullptr),
229 _index(0) {
230 C->set_output(this);
231 if (C->stub_name() == nullptr) {
232 _orig_pc_slot = C->fixed_slots() - (sizeof(address) / VMRegImpl::stack_slot_size);
233 }
234 }
235
236 PhaseOutput::~PhaseOutput() {
237 C->set_output(nullptr);
238 if (_scratch_buffer_blob != nullptr) {
239 BufferBlob::free(_scratch_buffer_blob);
240 }
241 }
242
243 void PhaseOutput::perform_mach_node_analysis() {
244 // Late barrier analysis must be done after schedule and bundle
245 // Otherwise liveness based spilling will fail
246 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
247 bs->late_barrier_analysis();
248
249 pd_perform_mach_node_analysis();
250
251 C->print_method(CompilerPhaseType::PHASE_MACH_ANALYSIS, 3);
252 }
253
254 // Convert Nodes to instruction bits and pass off to the VM
255 void PhaseOutput::Output() {
256 // RootNode goes
257 assert( C->cfg()->get_root_block()->number_of_nodes() == 0, "" );
258
259 // The number of new nodes (mostly MachNop) is proportional to
260 // the number of java calls and inner loops which are aligned.
261 if ( C->check_node_count((NodeLimitFudgeFactor + C->java_calls()*3 +
262 C->inner_loops()*(OptoLoopAlignment-1)),
263 "out of nodes before code generation" ) ) {
264 return;
265 }
266 // Make sure I can find the Start Node
267 Block *entry = C->cfg()->get_block(1);
268 Block *broot = C->cfg()->get_root_block();
269
270 const StartNode *start = entry->head()->as_Start();
271
272 // Replace StartNode with prolog
273 MachPrologNode *prolog = new MachPrologNode();
274 entry->map_node(prolog, 0);
275 C->cfg()->map_node_to_block(prolog, entry);
276 C->cfg()->unmap_node_from_block(start); // start is no longer in any block
277
278 // Virtual methods need an unverified entry point
279
280 if( C->is_osr_compilation() ) {
281 if( PoisonOSREntry ) {
282 // TODO: Should use a ShouldNotReachHereNode...
283 C->cfg()->insert( broot, 0, new MachBreakpointNode() );
284 }
285 } else {
286 if( C->method() && !C->method()->flags().is_static() ) {
287 // Insert unvalidated entry point
288 C->cfg()->insert( broot, 0, new MachUEPNode() );
289 }
290
291 }
292
293 // Break before main entry point
294 if ((C->method() && C->directive()->BreakAtExecuteOption) ||
295 (OptoBreakpoint && C->is_method_compilation()) ||
296 (OptoBreakpointOSR && C->is_osr_compilation()) ||
297 (OptoBreakpointC2R && !C->method()) ) {
298 // checking for C->method() means that OptoBreakpoint does not apply to
299 // runtime stubs or frame converters
300 C->cfg()->insert( entry, 1, new MachBreakpointNode() );
301 }
302
303 // Insert epilogs before every return
304 for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
305 Block* block = C->cfg()->get_block(i);
306 if (!block->is_connector() && block->non_connector_successor(0) == C->cfg()->get_root_block()) { // Found a program exit point?
307 Node* m = block->end();
308 if (m->is_Mach() && m->as_Mach()->ideal_Opcode() != Op_Halt) {
309 MachEpilogNode* epilog = new MachEpilogNode(m->as_Mach()->ideal_Opcode() == Op_Return);
310 block->add_inst(epilog);
311 C->cfg()->map_node_to_block(epilog, block);
312 }
313 }
314 }
315
316 // Keeper of sizing aspects
317 _buf_sizes = BufferSizingData();
318
319 // Initialize code buffer
320 estimate_buffer_size(_buf_sizes._const);
321 if (C->failing()) return;
322
323 // Pre-compute the length of blocks and replace
324 // long branches with short if machine supports it.
325 // Must be done before ScheduleAndBundle due to SPARC delay slots
326 uint* blk_starts = NEW_RESOURCE_ARRAY(uint, C->cfg()->number_of_blocks() + 1);
327 blk_starts[0] = 0;
328 shorten_branches(blk_starts);
329
330 ScheduleAndBundle();
331 if (C->failing()) {
332 return;
333 }
334
335 perform_mach_node_analysis();
336
337 // Complete sizing of codebuffer
338 CodeBuffer* cb = init_buffer();
339 if (cb == nullptr || C->failing()) {
340 return;
341 }
342
343 BuildOopMaps();
344
345 if (C->failing()) {
346 return;
347 }
348
349 C2_MacroAssembler masm(cb);
350 fill_buffer(&masm, blk_starts);
351 }
352
353 bool PhaseOutput::need_stack_bang(int frame_size_in_bytes) const {
354 // Determine if we need to generate a stack overflow check.
355 // Do it if the method is not a stub function and
356 // has java calls or has frame size > vm_page_size/8.
357 // The debug VM checks that deoptimization doesn't trigger an
358 // unexpected stack overflow (compiled method stack banging should
359 // guarantee it doesn't happen) so we always need the stack bang in
360 // a debug VM.
361 return (C->stub_function() == nullptr &&
362 (C->has_java_calls() || frame_size_in_bytes > (int)(os::vm_page_size())>>3
363 DEBUG_ONLY(|| true)));
364 }
365
366 // Compute the size of first NumberOfLoopInstrToAlign instructions at the top
367 // of a loop. When aligning a loop we need to provide enough instructions
368 // in cpu's fetch buffer to feed decoders. The loop alignment could be
369 // avoided if we have enough instructions in fetch buffer at the head of a loop.
370 // By default, the size is set to 999999 by Block's constructor so that
462 // Sum all instruction sizes to compute block size
463 uint last_inst = block->number_of_nodes();
464 uint blk_size = 0;
465 for (uint j = 0; j < last_inst; j++) {
466 _index = j;
467 Node* nj = block->get_node(_index);
468 // Handle machine instruction nodes
469 if (nj->is_Mach()) {
470 MachNode* mach = nj->as_Mach();
471 blk_size += (mach->alignment_required() - 1) * relocInfo::addr_unit(); // assume worst case padding
472 reloc_size += mach->reloc();
473 if (mach->is_MachCall()) {
474 // add size information for trampoline stub
475 // class CallStubImpl is platform-specific and defined in the *.ad files.
476 stub_size += CallStubImpl::size_call_trampoline();
477 reloc_size += CallStubImpl::reloc_call_trampoline();
478
479 MachCallNode *mcall = mach->as_MachCall();
480 // This destination address is NOT PC-relative
481
482 mcall->method_set((intptr_t)mcall->entry_point());
483
484 if (mcall->is_MachCallJava() && mcall->as_MachCallJava()->_method) {
485 stub_size += CompiledDirectCall::to_interp_stub_size();
486 reloc_size += CompiledDirectCall::reloc_to_interp_stub();
487 }
488 } else if (mach->is_MachSafePoint()) {
489 // If call/safepoint are adjacent, account for possible
490 // nop to disambiguate the two safepoints.
491 // ScheduleAndBundle() can rearrange nodes in a block,
492 // check for all offsets inside this block.
493 if (last_call_adr >= blk_starts[i]) {
494 blk_size += nop_size;
495 }
496 }
497 if (mach->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
498 // Nop is inserted between "avoid back to back" instructions.
499 // ScheduleAndBundle() can rearrange nodes in a block,
500 // check for all offsets inside this block.
501 if (last_avoid_back_to_back_adr >= blk_starts[i]) {
502 blk_size += nop_size;
717 // New functionality:
718 // Assert if the local is not top. In product mode let the new node
719 // override the old entry.
720 assert(local == C->top(), "LocArray collision");
721 if (local == C->top()) {
722 return;
723 }
724 array->pop();
725 }
726 const Type *t = local->bottom_type();
727
728 // Is it a safepoint scalar object node?
729 if (local->is_SafePointScalarObject()) {
730 SafePointScalarObjectNode* spobj = local->as_SafePointScalarObject();
731
732 ObjectValue* sv = sv_for_node_id(objs, spobj->_idx);
733 if (sv == nullptr) {
734 ciKlass* cik = t->is_oopptr()->exact_klass();
735 assert(cik->is_instance_klass() ||
736 cik->is_array_klass(), "Not supported allocation.");
737 sv = new ObjectValue(spobj->_idx,
738 new ConstantOopWriteValue(cik->java_mirror()->constant_encoding()));
739 set_sv_for_object_node(objs, sv);
740
741 uint first_ind = spobj->first_index(sfpt->jvms());
742 for (uint i = 0; i < spobj->n_fields(); i++) {
743 Node* fld_node = sfpt->in(first_ind+i);
744 (void)FillLocArray(sv->field_values()->length(), sfpt, fld_node, sv->field_values(), objs);
745 }
746 }
747 array->append(sv);
748 return;
749 } else if (local->is_SafePointScalarMerge()) {
750 SafePointScalarMergeNode* smerge = local->as_SafePointScalarMerge();
751 ObjectMergeValue* mv = (ObjectMergeValue*) sv_for_node_id(objs, smerge->_idx);
752
753 if (mv == nullptr) {
754 GrowableArray<ScopeValue*> deps;
755
756 int merge_pointer_idx = smerge->merge_pointer_idx(sfpt->jvms());
757 (void)FillLocArray(0, sfpt, sfpt->in(merge_pointer_idx), &deps, objs);
758 assert(deps.length() == 1, "missing value");
759
760 int selector_idx = smerge->selector_idx(sfpt->jvms());
761 (void)FillLocArray(1, nullptr, sfpt->in(selector_idx), &deps, nullptr);
967 if (!n->is_SafePointScalarObject()) {
968 continue;
969 }
970
971 ObjectValue* other = sv_for_node_id(objs, n->_idx);
972 if (ov == other) {
973 return true;
974 }
975 }
976 return false;
977 }
978
979 //--------------------------Process_OopMap_Node--------------------------------
980 void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) {
981 // Handle special safepoint nodes for synchronization
982 MachSafePointNode *sfn = mach->as_MachSafePoint();
983 MachCallNode *mcall;
984
985 int safepoint_pc_offset = current_offset;
986 bool return_oop = false;
987 bool has_ea_local_in_scope = sfn->_has_ea_local_in_scope;
988 bool arg_escape = false;
989
990 // Add the safepoint in the DebugInfoRecorder
991 if( !mach->is_MachCall() ) {
992 mcall = nullptr;
993 C->debug_info()->add_safepoint(safepoint_pc_offset, sfn->_oop_map);
994 } else {
995 mcall = mach->as_MachCall();
996
997 if (mcall->is_MachCallJava()) {
998 arg_escape = mcall->as_MachCallJava()->_arg_escape;
999 }
1000
1001 // Check if a call returns an object.
1002 if (mcall->returns_pointer()) {
1003 return_oop = true;
1004 }
1005 safepoint_pc_offset += mcall->ret_addr_offset();
1006 C->debug_info()->add_safepoint(safepoint_pc_offset, mcall->_oop_map);
1007 }
1008
1009 // Loop over the JVMState list to add scope information
1010 // Do not skip safepoints with a null method, they need monitor info
1011 JVMState* youngest_jvms = sfn->jvms();
1012 int max_depth = youngest_jvms->depth();
1013
1014 // Allocate the object pool for scalar-replaced objects -- the map from
1015 // small-integer keys (which can be recorded in the local and ostack
1016 // arrays) to descriptions of the object state.
1017 GrowableArray<ScopeValue*> *objs = new GrowableArray<ScopeValue*>();
1018
1019 // Visit scopes from oldest to youngest.
1020 for (int depth = 1; depth <= max_depth; depth++) {
1021 JVMState* jvms = youngest_jvms->of_depth(depth);
1022 int idx;
1023 ciMethod* method = jvms->has_method() ? jvms->method() : nullptr;
1024 // Safepoints that do not have method() set only provide oop-map and monitor info
1053 // Build the growable array of ScopeValues for exp stack
1054 GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
1055
1056 // Loop over monitors and insert into array
1057 for (idx = 0; idx < num_mon; idx++) {
1058 // Grab the node that defines this monitor
1059 Node* box_node = sfn->monitor_box(jvms, idx);
1060 Node* obj_node = sfn->monitor_obj(jvms, idx);
1061
1062 // Create ScopeValue for object
1063 ScopeValue *scval = nullptr;
1064
1065 if (obj_node->is_SafePointScalarObject()) {
1066 SafePointScalarObjectNode* spobj = obj_node->as_SafePointScalarObject();
1067 scval = PhaseOutput::sv_for_node_id(objs, spobj->_idx);
1068 if (scval == nullptr) {
1069 const Type *t = spobj->bottom_type();
1070 ciKlass* cik = t->is_oopptr()->exact_klass();
1071 assert(cik->is_instance_klass() ||
1072 cik->is_array_klass(), "Not supported allocation.");
1073 ObjectValue* sv = new ObjectValue(spobj->_idx,
1074 new ConstantOopWriteValue(cik->java_mirror()->constant_encoding()));
1075 PhaseOutput::set_sv_for_object_node(objs, sv);
1076
1077 uint first_ind = spobj->first_index(youngest_jvms);
1078 for (uint i = 0; i < spobj->n_fields(); i++) {
1079 Node* fld_node = sfn->in(first_ind+i);
1080 (void)FillLocArray(sv->field_values()->length(), sfn, fld_node, sv->field_values(), objs);
1081 }
1082 scval = sv;
1083 }
1084 } else if (obj_node->is_SafePointScalarMerge()) {
1085 SafePointScalarMergeNode* smerge = obj_node->as_SafePointScalarMerge();
1086 ObjectMergeValue* mv = (ObjectMergeValue*) sv_for_node_id(objs, smerge->_idx);
1087
1088 if (mv == nullptr) {
1089 GrowableArray<ScopeValue*> deps;
1090
1091 int merge_pointer_idx = smerge->merge_pointer_idx(youngest_jvms);
1092 FillLocArray(0, sfn, sfn->in(merge_pointer_idx), &deps, objs);
1093 assert(deps.length() == 1, "missing value");
1094
1161 DebugToken *locvals = C->debug_info()->create_scope_values(locarray);
1162 DebugToken *expvals = C->debug_info()->create_scope_values(exparray);
1163 DebugToken *monvals = C->debug_info()->create_monitor_values(monarray);
1164
1165 // Make method available for all Safepoints
1166 ciMethod* scope_method = method ? method : C->method();
1167 // Describe the scope here
1168 assert(jvms->bci() >= InvocationEntryBci && jvms->bci() <= 0x10000, "must be a valid or entry BCI");
1169 assert(!jvms->should_reexecute() || depth == max_depth, "reexecute allowed only for the youngest");
1170 // Now we can describe the scope.
1171 methodHandle null_mh;
1172 bool rethrow_exception = false;
1173 C->debug_info()->describe_scope(
1174 safepoint_pc_offset,
1175 null_mh,
1176 scope_method,
1177 jvms->bci(),
1178 jvms->should_reexecute(),
1179 rethrow_exception,
1180 return_oop,
1181 has_ea_local_in_scope,
1182 arg_escape,
1183 locvals,
1184 expvals,
1185 monvals
1186 );
1187 } // End jvms loop
1188
1189 // Mark the end of the scope set.
1190 C->debug_info()->end_safepoint(safepoint_pc_offset);
1191 }
1192
1193
1194
1195 // A simplified version of Process_OopMap_Node, to handle non-safepoints.
1196 class NonSafepointEmitter {
1197 Compile* C;
1198 JVMState* _pending_jvms;
1199 int _pending_offset;
1200
1269 methodHandle null_mh;
1270 debug_info->describe_scope(pc_offset, null_mh, method, jvms->bci(), jvms->should_reexecute());
1271 }
1272
1273 // Mark the end of the scope set.
1274 debug_info->end_non_safepoint(pc_offset);
1275 }
1276
1277 //------------------------------init_buffer------------------------------------
1278 void PhaseOutput::estimate_buffer_size(int& const_req) {
1279
1280 // Set the initially allocated size
1281 const_req = initial_const_capacity;
1282
1283 // The extra spacing after the code is necessary on some platforms.
1284 // Sometimes we need to patch in a jump after the last instruction,
1285 // if the nmethod has been deoptimized. (See 4932387, 4894843.)
1286
1287 // Compute the byte offset where we can store the deopt pc.
1288 if (C->fixed_slots() != 0) {
1289 _orig_pc_slot_offset_in_bytes = C->regalloc()->reg2offset(OptoReg::stack2reg(_orig_pc_slot));
1290 }
1291
1292 // Compute prolog code size
1293 _frame_slots = OptoReg::reg2stack(C->matcher()->_old_SP) + C->regalloc()->_framesize;
1294 assert(_frame_slots >= 0 && _frame_slots < 1000000, "sanity check");
1295
1296 if (C->has_mach_constant_base_node()) {
1297 uint add_size = 0;
1298 // Fill the constant table.
1299 // Note: This must happen before shorten_branches.
1300 for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1301 Block* b = C->cfg()->get_block(i);
1302
1303 for (uint j = 0; j < b->number_of_nodes(); j++) {
1304 Node* n = b->get_node(j);
1305
1306 // If the node is a MachConstantNode evaluate the constant
1307 // value section.
1308 if (n->is_MachConstant()) {
1309 MachConstantNode* machcon = n->as_MachConstant();
1514 MachNode *nop = new MachNopNode(nops_cnt);
1515 block->insert_node(nop, j++);
1516 last_inst++;
1517 C->cfg()->map_node_to_block(nop, block);
1518 // Ensure enough space.
1519 masm->code()->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1520 if ((masm->code()->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1521 C->record_failure("CodeCache is full");
1522 return;
1523 }
1524 nop->emit(masm, C->regalloc());
1525 masm->code()->flush_bundle(true);
1526 current_offset = masm->offset();
1527 }
1528
1529 bool observe_safepoint = is_sfn;
1530 // Remember the start of the last call in a basic block
1531 if (is_mcall) {
1532 MachCallNode *mcall = mach->as_MachCall();
1533
1534 // This destination address is NOT PC-relative
1535 mcall->method_set((intptr_t)mcall->entry_point());
1536
1537 // Save the return address
1538 call_returns[block->_pre_order] = current_offset + mcall->ret_addr_offset();
1539
1540 observe_safepoint = mcall->guaranteed_safepoint();
1541 }
1542
1543 // sfn will be valid whenever mcall is valid now because of inheritance
1544 if (observe_safepoint) {
1545 // Handle special safepoint nodes for synchronization
1546 if (!is_mcall) {
1547 MachSafePointNode *sfn = mach->as_MachSafePoint();
1548 // !!!!! Stubs only need an oopmap right now, so bail out
1549 if (sfn->jvms()->method() == nullptr) {
1550 // Write the oopmap directly to the code blob??!!
1551 continue;
1552 }
1553 } // End synchronization
1554
1555 non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1653 if ((node_offsets != nullptr) && (n->_idx < node_offset_limit)) {
1654 node_offsets[n->_idx] = masm->offset();
1655 }
1656 #endif
1657 assert(!C->failing_internal() || C->failure_is_artificial(), "Should not reach here if failing.");
1658
1659 // "Normal" instruction case
1660 DEBUG_ONLY(uint instr_offset = masm->offset());
1661 n->emit(masm, C->regalloc());
1662 current_offset = masm->offset();
1663
1664 // Above we only verified that there is enough space in the instruction section.
1665 // However, the instruction may emit stubs that cause code buffer expansion.
1666 // Bail out here if expansion failed due to a lack of code cache space.
1667 if (C->failing()) {
1668 return;
1669 }
1670
1671 assert(!is_mcall || (call_returns[block->_pre_order] <= (uint)current_offset),
1672 "ret_addr_offset() not within emitted code");
1673
1674 #ifdef ASSERT
1675 uint n_size = n->size(C->regalloc());
1676 if (n_size < (current_offset-instr_offset)) {
1677 MachNode* mach = n->as_Mach();
1678 n->dump();
1679 mach->dump_format(C->regalloc(), tty);
1680 tty->print_cr(" n_size (%d), current_offset (%d), instr_offset (%d)", n_size, current_offset, instr_offset);
1681 Disassembler::decode(masm->code()->insts_begin() + instr_offset, masm->code()->insts_begin() + current_offset + 1, tty);
1682 tty->print_cr(" ------------------- ");
1683 BufferBlob* blob = this->scratch_buffer_blob();
1684 address blob_begin = blob->content_begin();
1685 Disassembler::decode(blob_begin, blob_begin + n_size + 1, tty);
1686 assert(false, "wrong size of mach node");
1687 }
1688 #endif
1689 non_safepoints.observe_instruction(n, current_offset);
1690
1691 // mcall is last "call" that can be a safepoint
1692 // record it so we can see if a poll will directly follow it
1693 // in which case we'll need a pad to make the PcDesc sites unique
2909 anti_do_use( b, n, _regalloc->get_reg_second(def) );
2910 }
2911 }
2912 // Do not allow defs of new derived values to float above GC
2913 // points unless the base is definitely available at the GC point.
2914
2915 Node *m = b->get_node(i);
2916
2917 if (last_safept_node != end_node &&
2918 m != last_safept_node) {
2919 bool need_safept_prec = false;
2920 // Add precedence edge from following safepoint to use of derived pointer
2921 for (uint k = 1; k < m->req(); k++) {
2922 const Type *t = m->in(k)->bottom_type();
2923 if (t->isa_oop_ptr() &&
2924 t->is_ptr()->offset() != 0) {
2925 need_safept_prec = true;
2926 break;
2927 }
2928 }
2929 // A CheckCastPP whose input is still RawPtr must stay above the following safepoint.
2930 // Otherwise post-regalloc block-local scheduling can leave a live raw oop at the safepoint.
2931 if (!need_safept_prec && m->is_Mach() &&
2932 m->as_Mach()->ideal_Opcode() == Op_CheckCastPP) {
2933 Node* def = m->in(1);
2934 if (def != nullptr && def->bottom_type()->base() == Type::RawPtr) {
2935 need_safept_prec = true;
2936 }
2937 }
2938 if (need_safept_prec) {
2939 last_safept_node->add_prec(m);
2940 }
2941 }
2942
2943 if( n->jvms() ) { // Precedence edge from derived to safept
2944 // Check if last_safept_node was moved by pinch-point insertion in anti_do_use()
2945 if( b->get_node(last_safept) != last_safept_node ) {
2946 last_safept = b->find_node(last_safept_node);
2947 }
2948 for( uint j=last_safept; j > i; j-- ) {
3069 }
3070 #endif
3071
3072 //-----------------------init_scratch_buffer_blob------------------------------
3073 // Construct a temporary BufferBlob and cache it for this compile.
3074 void PhaseOutput::init_scratch_buffer_blob(int const_size) {
3075 // If there is already a scratch buffer blob allocated and the
3076 // constant section is big enough, use it. Otherwise free the
3077 // current and allocate a new one.
3078 BufferBlob* blob = scratch_buffer_blob();
3079 if ((blob != nullptr) && (const_size <= _scratch_const_size)) {
3080 // Use the current blob.
3081 } else {
3082 if (blob != nullptr) {
3083 BufferBlob::free(blob);
3084 }
3085
3086 ResourceMark rm;
3087 _scratch_const_size = const_size;
3088 int size = C2Compiler::initial_code_buffer_size(const_size);
3089 blob = BufferBlob::create("Compile::scratch_buffer", size);
3090 // Record the buffer blob for next time.
3091 set_scratch_buffer_blob(blob);
3092 // Have we run out of code space?
3093 if (scratch_buffer_blob() == nullptr) {
3094 // Let CompilerBroker disable further compilations.
3095 C->record_failure("Not enough space for scratch buffer in CodeCache");
3096 return;
3097 }
3098 }
3099
3100 // Initialize the relocation buffers
3101 relocInfo* locs_buf = (relocInfo*) blob->content_end() - MAX_locs_size;
3102 set_scratch_locs_memory(locs_buf);
3103 }
3104
3105
3106 //-----------------------scratch_emit_size-------------------------------------
3107 // Helper function that computes size by emitting code
3108 uint PhaseOutput::scratch_emit_size(const Node* n) {
3139 buf.insts()->set_scratch_emit();
3140 buf.stubs()->set_scratch_emit();
3141
3142 // Do the emission.
3143
3144 Label fakeL; // Fake label for branch instructions.
3145 Label* saveL = nullptr;
3146 uint save_bnum = 0;
3147 bool is_branch = n->is_MachBranch();
3148 C2_MacroAssembler masm(&buf);
3149 masm.bind(fakeL);
3150 if (is_branch) {
3151 n->as_MachBranch()->save_label(&saveL, &save_bnum);
3152 n->as_MachBranch()->label_set(&fakeL, 0);
3153 }
3154 n->emit(&masm, C->regalloc());
3155
3156 // Emitting into the scratch buffer should not fail
3157 assert(!C->failing_internal() || C->failure_is_artificial(), "Must not have pending failure. Reason is: %s", C->failure_reason());
3158
3159 if (is_branch) // Restore label.
3160 n->as_MachBranch()->label_set(saveL, save_bnum);
3161
3162 // End scratch_emit_size section.
3163 set_in_scratch_emit_size(false);
3164
3165 return buf.insts_size();
3166 }
3167
3168 void PhaseOutput::install() {
3169 if (!C->should_install_code()) {
3170 return;
3171 } else if (C->stub_function() != nullptr) {
3172 install_stub(C->stub_name());
3173 } else {
3174 install_code(C->method(),
3175 C->entry_bci(),
3176 CompileBroker::compiler2(),
3177 C->has_unsafe_access(),
3178 SharedRuntime::is_wide_vector(C->max_vector_size()));
3179 }
3180 }
3181
3182 void PhaseOutput::install_code(ciMethod* target,
3183 int entry_bci,
3184 AbstractCompiler* compiler,
3185 bool has_unsafe_access,
3186 bool has_wide_vectors) {
3187 // Check if we want to skip execution of all compiled code.
3188 {
3189 #ifndef PRODUCT
3190 if (OptoNoExecute) {
3191 C->record_method_not_compilable("+OptoNoExecute"); // Flag as failed
3192 return;
3193 }
3194 #endif
3195 Compile::TracePhase tp(_t_registerMethod);
3196
3197 if (C->is_osr_compilation()) {
3198 _code_offsets.set_value(CodeOffsets::Verified_Entry, 0);
3199 _code_offsets.set_value(CodeOffsets::OSR_Entry, _first_block_size);
3200 } else {
3201 if (!target->is_static()) {
3202 // The UEP of an nmethod ensures that the VEP is padded. However, the padding of the UEP is placed
3203 // before the inline cache check, so we don't have to execute any nop instructions when dispatching
3204 // through the UEP, yet we can ensure that the VEP is aligned appropriately.
3205 _code_offsets.set_value(CodeOffsets::Entry, _first_block_size - MacroAssembler::ic_check_size());
3206 }
3207 _code_offsets.set_value(CodeOffsets::Verified_Entry, _first_block_size);
3208 _code_offsets.set_value(CodeOffsets::OSR_Entry, 0);
3209 }
3210
3211 C->env()->register_method(target,
3212 entry_bci,
3213 &_code_offsets,
3214 _orig_pc_slot_offset_in_bytes,
3215 code_buffer(),
3216 frame_size_in_words(),
3217 oop_map_set(),
3218 &_handler_table,
3219 inc_table(),
3220 compiler,
3221 has_unsafe_access,
3222 SharedRuntime::is_wide_vector(C->max_vector_size()),
3223 C->has_monitors(),
3224 C->has_scoped_access(),
3225 0);
3226
3227 if (C->log() != nullptr) { // Print code cache state into compiler log
3228 C->log()->code_cache_state();
3229 }
3230 }
3231 }
3232 void PhaseOutput::install_stub(const char* stub_name) {
3233 // Entry point will be accessed using stub_entry_point();
3234 if (code_buffer() == nullptr) {
3235 Matcher::soft_match_failure();
3236 } else {
3237 if (PrintAssembly && (WizardMode || Verbose))
3238 tty->print_cr("### Stub::%s", stub_name);
3239
3240 if (!C->failing()) {
3241 assert(C->fixed_slots() == 0, "no fixed slots used for runtime stubs");
3242
3243 // Make the NMethod
3244 // For now we mark the frame as never safe for profile stackwalking
3245 RuntimeStub *rs = RuntimeStub::new_runtime_stub(stub_name,
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.inline.hpp"
26 #include "code/aotCodeCache.hpp"
27 #include "code/compiledIC.hpp"
28 #include "code/debugInfo.hpp"
29 #include "code/debugInfoRec.hpp"
30 #include "compiler/compileBroker.hpp"
31 #include "compiler/compilerDirectives.hpp"
32 #include "compiler/disassembler.hpp"
33 #include "compiler/oopMap.hpp"
34 #include "gc/shared/barrierSet.hpp"
35 #include "gc/shared/c2/barrierSetC2.hpp"
36 #include "gc/shared/gc_globals.hpp"
37 #include "memory/allocation.hpp"
38 #include "opto/ad.hpp"
39 #include "opto/block.hpp"
40 #include "opto/c2_MacroAssembler.hpp"
41 #include "opto/c2compiler.hpp"
42 #include "opto/callnode.hpp"
43 #include "opto/cfgnode.hpp"
44 #include "opto/locknode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/node.hpp"
47 #include "opto/optoreg.hpp"
48 #include "opto/output.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/type.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "utilities/macros.hpp"
53 #include "utilities/powerOfTwo.hpp"
54 #include "utilities/xmlstream.hpp"
55
56 #ifndef PRODUCT
206 #endif
207
208 };
209
210 PhaseOutput::PhaseOutput()
211 : Phase(Phase::Output),
212 _code_buffer("Compile::Fill_buffer"),
213 _first_block_size(0),
214 _handler_table(),
215 _inc_table(),
216 _stub_list(),
217 _oop_map_set(nullptr),
218 _scratch_buffer_blob(nullptr),
219 _scratch_locs_memory(nullptr),
220 _scratch_const_size(-1),
221 _in_scratch_emit_size(false),
222 _frame_slots(0),
223 _code_offsets(),
224 _node_bundling_limit(0),
225 _node_bundling_base(nullptr),
226 _orig_pc_slot_offset_in_bytes(0),
227 _buf_sizes(),
228 _block(nullptr),
229 _index(0) {
230 C->set_output(this);
231
232 }
233
234 PhaseOutput::~PhaseOutput() {
235 C->set_output(nullptr);
236 if (_scratch_buffer_blob != nullptr) {
237 BufferBlob::free(_scratch_buffer_blob);
238 }
239 }
240
241 void PhaseOutput::perform_mach_node_analysis() {
242 // Late barrier analysis must be done after schedule and bundle
243 // Otherwise liveness based spilling will fail
244 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
245 bs->late_barrier_analysis();
246
247 pd_perform_mach_node_analysis();
248
249 C->print_method(CompilerPhaseType::PHASE_MACH_ANALYSIS, 3);
250 }
251
252 // Convert Nodes to instruction bits and pass off to the VM
253 void PhaseOutput::Output() {
254 // RootNode goes
255 assert( C->cfg()->get_root_block()->number_of_nodes() == 0, "" );
256
257 // The number of new nodes (mostly MachNop) is proportional to
258 // the number of java calls and inner loops which are aligned.
259 if ( C->check_node_count((NodeLimitFudgeFactor + C->java_calls()*3 +
260 C->inner_loops()*(OptoLoopAlignment-1)),
261 "out of nodes before code generation" ) ) {
262 return;
263 }
264 // Make sure I can find the Start Node
265 Block *entry = C->cfg()->get_block(1);
266 Block *broot = C->cfg()->get_root_block();
267
268 const StartNode *start = entry->head()->as_Start();
269
270 // Replace StartNode with prolog
271 Label verified_entry;
272 MachPrologNode* prolog = new MachPrologNode(&verified_entry);
273 entry->map_node(prolog, 0);
274 C->cfg()->map_node_to_block(prolog, entry);
275 C->cfg()->unmap_node_from_block(start); // start is no longer in any block
276
277 // Virtual methods need an unverified entry point
278 if (C->is_osr_compilation()) {
279 if (PoisonOSREntry) {
280 // TODO: Should use a ShouldNotReachHereNode...
281 C->cfg()->insert( broot, 0, new MachBreakpointNode() );
282 }
283 } else {
284 if (C->method()) {
285 if (C->method()->has_scalarized_args()) {
286 // Add entry point to unpack all inline type arguments
287 C->cfg()->insert(broot, 0, new MachVEPNode(&verified_entry, /* verified */ true, /* receiver_only */ false));
288 if (!C->method()->is_static()) {
289 // Add verified/unverified entry points to only unpack inline type receiver at interface calls
290 C->cfg()->insert(broot, 0, new MachVEPNode(&verified_entry, /* verified */ false, /* receiver_only */ false));
291 C->cfg()->insert(broot, 0, new MachVEPNode(&verified_entry, /* verified */ true, /* receiver_only */ true));
292 C->cfg()->insert(broot, 0, new MachVEPNode(&verified_entry, /* verified */ false, /* receiver_only */ true));
293 }
294 } else if (!C->method()->is_static()) {
295 // Insert unvalidated entry point
296 C->cfg()->insert(broot, 0, new MachUEPNode());
297 }
298 }
299 }
300
301 // Break before main entry point
302 if ((C->method() && C->directive()->BreakAtExecuteOption) ||
303 (OptoBreakpoint && C->is_method_compilation()) ||
304 (OptoBreakpointOSR && C->is_osr_compilation()) ||
305 (OptoBreakpointC2R && !C->method()) ) {
306 // checking for C->method() means that OptoBreakpoint does not apply to
307 // runtime stubs or frame converters
308 C->cfg()->insert( entry, 1, new MachBreakpointNode() );
309 }
310
311 // Insert epilogs before every return
312 for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
313 Block* block = C->cfg()->get_block(i);
314 if (!block->is_connector() && block->non_connector_successor(0) == C->cfg()->get_root_block()) { // Found a program exit point?
315 Node* m = block->end();
316 if (m->is_Mach() && m->as_Mach()->ideal_Opcode() != Op_Halt) {
317 MachEpilogNode* epilog = new MachEpilogNode(m->as_Mach()->ideal_Opcode() == Op_Return);
318 block->add_inst(epilog);
319 C->cfg()->map_node_to_block(epilog, block);
320 }
321 }
322 }
323
324 // Keeper of sizing aspects
325 _buf_sizes = BufferSizingData();
326
327 // Initialize code buffer
328 estimate_buffer_size(_buf_sizes._const);
329 if (C->failing()) return;
330
331 // Pre-compute the length of blocks and replace
332 // long branches with short if machine supports it.
333 // Must be done before ScheduleAndBundle due to SPARC delay slots
334 uint* blk_starts = NEW_RESOURCE_ARRAY(uint, C->cfg()->number_of_blocks() + 1);
335 blk_starts[0] = 0;
336 shorten_branches(blk_starts);
337
338 if (!C->is_osr_compilation() && C->has_scalarized_args()) {
339 // Compute the offsets of the entry points required by the inline type calling convention
340 if (!C->method()->is_static()) {
341 // We have entries at the beginning of the method, implemented by the first 4 nodes.
342 // Entry (unverified) @ offset 0
343 // Verified_Inline_Entry_RO
344 // Inline_Entry (unverified)
345 // Verified_Inline_Entry
346 uint offset = 0;
347 _code_offsets.set_value(CodeOffsets::Entry, offset);
348
349 offset += ((MachVEPNode*)broot->get_node(0))->size(C->regalloc());
350 _code_offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, offset);
351
352 offset += ((MachVEPNode*)broot->get_node(1))->size(C->regalloc());
353 _code_offsets.set_value(CodeOffsets::Inline_Entry, offset);
354
355 offset += ((MachVEPNode*)broot->get_node(2))->size(C->regalloc());
356 _code_offsets.set_value(CodeOffsets::Verified_Inline_Entry, offset);
357 } else {
358 _code_offsets.set_value(CodeOffsets::Entry, CodeOffsets::no_such_entry_point); // will be patched later
359 _code_offsets.set_value(CodeOffsets::Verified_Inline_Entry, 0);
360 }
361 }
362
363 ScheduleAndBundle();
364 if (C->failing()) {
365 return;
366 }
367
368 perform_mach_node_analysis();
369
370 // Complete sizing of codebuffer
371 CodeBuffer* cb = init_buffer();
372 if (cb == nullptr || C->failing()) {
373 return;
374 }
375
376 BuildOopMaps();
377
378 if (C->failing()) {
379 return;
380 }
381
382 C2_MacroAssembler masm(cb);
383 fill_buffer(&masm, blk_starts);
384 if (C->failing()) {
385 // If we bailed out during matching, not all nodes were visited and the
386 // label might be in inconsistent state (used but not bound). Reset it.
387 verified_entry.reset();
388 }
389 }
390
391 bool PhaseOutput::need_stack_bang(int frame_size_in_bytes) const {
392 // Determine if we need to generate a stack overflow check.
393 // Do it if the method is not a stub function and
394 // has java calls or has frame size > vm_page_size/8.
395 // The debug VM checks that deoptimization doesn't trigger an
396 // unexpected stack overflow (compiled method stack banging should
397 // guarantee it doesn't happen) so we always need the stack bang in
398 // a debug VM.
399 return (C->stub_function() == nullptr &&
400 (C->has_java_calls() || frame_size_in_bytes > (int)(os::vm_page_size())>>3
401 DEBUG_ONLY(|| true)));
402 }
403
404 // Compute the size of first NumberOfLoopInstrToAlign instructions at the top
405 // of a loop. When aligning a loop we need to provide enough instructions
406 // in cpu's fetch buffer to feed decoders. The loop alignment could be
407 // avoided if we have enough instructions in fetch buffer at the head of a loop.
408 // By default, the size is set to 999999 by Block's constructor so that
500 // Sum all instruction sizes to compute block size
501 uint last_inst = block->number_of_nodes();
502 uint blk_size = 0;
503 for (uint j = 0; j < last_inst; j++) {
504 _index = j;
505 Node* nj = block->get_node(_index);
506 // Handle machine instruction nodes
507 if (nj->is_Mach()) {
508 MachNode* mach = nj->as_Mach();
509 blk_size += (mach->alignment_required() - 1) * relocInfo::addr_unit(); // assume worst case padding
510 reloc_size += mach->reloc();
511 if (mach->is_MachCall()) {
512 // add size information for trampoline stub
513 // class CallStubImpl is platform-specific and defined in the *.ad files.
514 stub_size += CallStubImpl::size_call_trampoline();
515 reloc_size += CallStubImpl::reloc_call_trampoline();
516
517 MachCallNode *mcall = mach->as_MachCall();
518 // This destination address is NOT PC-relative
519
520 if (mcall->entry_point() != nullptr) {
521 mcall->method_set((intptr_t)mcall->entry_point());
522 }
523
524 if (mcall->is_MachCallJava() && mcall->as_MachCallJava()->_method) {
525 stub_size += CompiledDirectCall::to_interp_stub_size();
526 reloc_size += CompiledDirectCall::reloc_to_interp_stub();
527 }
528 } else if (mach->is_MachSafePoint()) {
529 // If call/safepoint are adjacent, account for possible
530 // nop to disambiguate the two safepoints.
531 // ScheduleAndBundle() can rearrange nodes in a block,
532 // check for all offsets inside this block.
533 if (last_call_adr >= blk_starts[i]) {
534 blk_size += nop_size;
535 }
536 }
537 if (mach->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
538 // Nop is inserted between "avoid back to back" instructions.
539 // ScheduleAndBundle() can rearrange nodes in a block,
540 // check for all offsets inside this block.
541 if (last_avoid_back_to_back_adr >= blk_starts[i]) {
542 blk_size += nop_size;
757 // New functionality:
758 // Assert if the local is not top. In product mode let the new node
759 // override the old entry.
760 assert(local == C->top(), "LocArray collision");
761 if (local == C->top()) {
762 return;
763 }
764 array->pop();
765 }
766 const Type *t = local->bottom_type();
767
768 // Is it a safepoint scalar object node?
769 if (local->is_SafePointScalarObject()) {
770 SafePointScalarObjectNode* spobj = local->as_SafePointScalarObject();
771
772 ObjectValue* sv = sv_for_node_id(objs, spobj->_idx);
773 if (sv == nullptr) {
774 ciKlass* cik = t->is_oopptr()->exact_klass();
775 assert(cik->is_instance_klass() ||
776 cik->is_array_klass(), "Not supported allocation.");
777 uint first_ind = spobj->first_index(sfpt->jvms());
778 // Nullable, scalarized inline types have a null_marker input
779 // that needs to be checked before using the field values.
780 ScopeValue* properties = nullptr;
781 if (cik->is_inlinetype()) {
782 Node* null_marker_node = sfpt->in(first_ind++);
783 assert(null_marker_node != nullptr, "null_marker node not found");
784 if (!null_marker_node->is_top()) {
785 const TypeInt* null_marker_type = null_marker_node->bottom_type()->is_int();
786 if (null_marker_node->is_Con()) {
787 properties = new ConstantIntValue(null_marker_type->get_con());
788 } else {
789 OptoReg::Name null_marker_reg = C->regalloc()->get_reg_first(null_marker_node);
790 properties = new_loc_value(C->regalloc(), null_marker_reg, Location::normal);
791 }
792 }
793 }
794 if (cik->is_array_klass() && !cik->is_type_array_klass()) {
795 ciArrayKlass* ciak = cik->as_array_klass();
796 const bool is_element_inline = ciak->element_klass()->is_inlinetype();
797
798 const ArrayProperties props = ArrayProperties::Default()
799 .with_null_restricted(is_element_inline && ciak->is_elem_null_free())
800 .with_non_atomic(is_element_inline && !ciak->is_elem_atomic());
801
802 properties = new ConstantIntValue((jint)props.value());
803 }
804 sv = new ObjectValue(spobj->_idx,
805 new ConstantOopWriteValue(cik->java_mirror()->constant_encoding()), true, properties);
806 set_sv_for_object_node(objs, sv);
807
808 for (uint i = 0; i < spobj->n_fields(); i++) {
809 Node* fld_node = sfpt->in(first_ind+i);
810 (void)FillLocArray(sv->field_values()->length(), sfpt, fld_node, sv->field_values(), objs);
811 }
812 }
813 array->append(sv);
814 return;
815 } else if (local->is_SafePointScalarMerge()) {
816 SafePointScalarMergeNode* smerge = local->as_SafePointScalarMerge();
817 ObjectMergeValue* mv = (ObjectMergeValue*) sv_for_node_id(objs, smerge->_idx);
818
819 if (mv == nullptr) {
820 GrowableArray<ScopeValue*> deps;
821
822 int merge_pointer_idx = smerge->merge_pointer_idx(sfpt->jvms());
823 (void)FillLocArray(0, sfpt, sfpt->in(merge_pointer_idx), &deps, objs);
824 assert(deps.length() == 1, "missing value");
825
826 int selector_idx = smerge->selector_idx(sfpt->jvms());
827 (void)FillLocArray(1, nullptr, sfpt->in(selector_idx), &deps, nullptr);
1033 if (!n->is_SafePointScalarObject()) {
1034 continue;
1035 }
1036
1037 ObjectValue* other = sv_for_node_id(objs, n->_idx);
1038 if (ov == other) {
1039 return true;
1040 }
1041 }
1042 return false;
1043 }
1044
1045 //--------------------------Process_OopMap_Node--------------------------------
1046 void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) {
1047 // Handle special safepoint nodes for synchronization
1048 MachSafePointNode *sfn = mach->as_MachSafePoint();
1049 MachCallNode *mcall;
1050
1051 int safepoint_pc_offset = current_offset;
1052 bool return_oop = false;
1053 bool return_scalarized = false;
1054 bool has_ea_local_in_scope = sfn->_has_ea_local_in_scope;
1055 bool arg_escape = false;
1056
1057 // Add the safepoint in the DebugInfoRecorder
1058 if( !mach->is_MachCall() ) {
1059 mcall = nullptr;
1060 C->debug_info()->add_safepoint(safepoint_pc_offset, sfn->_oop_map);
1061 } else {
1062 mcall = mach->as_MachCall();
1063
1064 if (mcall->is_MachCallJava()) {
1065 arg_escape = mcall->as_MachCallJava()->_arg_escape;
1066 }
1067
1068 // Check if a call returns an object.
1069 if (mcall->returns_pointer() || mcall->returns_scalarized()) {
1070 return_oop = true;
1071 }
1072 if (mcall->returns_scalarized()) {
1073 return_scalarized = true;
1074 }
1075 safepoint_pc_offset += mcall->ret_addr_offset();
1076 C->debug_info()->add_safepoint(safepoint_pc_offset, mcall->_oop_map);
1077 }
1078
1079 // Loop over the JVMState list to add scope information
1080 // Do not skip safepoints with a null method, they need monitor info
1081 JVMState* youngest_jvms = sfn->jvms();
1082 int max_depth = youngest_jvms->depth();
1083
1084 // Allocate the object pool for scalar-replaced objects -- the map from
1085 // small-integer keys (which can be recorded in the local and ostack
1086 // arrays) to descriptions of the object state.
1087 GrowableArray<ScopeValue*> *objs = new GrowableArray<ScopeValue*>();
1088
1089 // Visit scopes from oldest to youngest.
1090 for (int depth = 1; depth <= max_depth; depth++) {
1091 JVMState* jvms = youngest_jvms->of_depth(depth);
1092 int idx;
1093 ciMethod* method = jvms->has_method() ? jvms->method() : nullptr;
1094 // Safepoints that do not have method() set only provide oop-map and monitor info
1123 // Build the growable array of ScopeValues for exp stack
1124 GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
1125
1126 // Loop over monitors and insert into array
1127 for (idx = 0; idx < num_mon; idx++) {
1128 // Grab the node that defines this monitor
1129 Node* box_node = sfn->monitor_box(jvms, idx);
1130 Node* obj_node = sfn->monitor_obj(jvms, idx);
1131
1132 // Create ScopeValue for object
1133 ScopeValue *scval = nullptr;
1134
1135 if (obj_node->is_SafePointScalarObject()) {
1136 SafePointScalarObjectNode* spobj = obj_node->as_SafePointScalarObject();
1137 scval = PhaseOutput::sv_for_node_id(objs, spobj->_idx);
1138 if (scval == nullptr) {
1139 const Type *t = spobj->bottom_type();
1140 ciKlass* cik = t->is_oopptr()->exact_klass();
1141 assert(cik->is_instance_klass() ||
1142 cik->is_array_klass(), "Not supported allocation.");
1143 assert(!cik->is_inlinetype(), "Synchronization on value object?");
1144 ScopeValue* properties = nullptr;
1145 if (cik->is_array_klass() && !cik->is_type_array_klass()) {
1146 ciArrayKlass* ciak = cik->as_array_klass();
1147 const bool is_element_inline = ciak->element_klass()->is_inlinetype();
1148
1149 const ArrayProperties props = ArrayProperties::Default()
1150 .with_null_restricted(is_element_inline && ciak->is_elem_null_free())
1151 .with_non_atomic(is_element_inline && !ciak->is_elem_atomic());
1152
1153 properties = new ConstantIntValue((jint)props.value());
1154 }
1155 ObjectValue* sv = new ObjectValue(spobj->_idx,
1156 new ConstantOopWriteValue(cik->java_mirror()->constant_encoding()), true, properties);
1157 PhaseOutput::set_sv_for_object_node(objs, sv);
1158
1159 uint first_ind = spobj->first_index(youngest_jvms);
1160 for (uint i = 0; i < spobj->n_fields(); i++) {
1161 Node* fld_node = sfn->in(first_ind+i);
1162 (void)FillLocArray(sv->field_values()->length(), sfn, fld_node, sv->field_values(), objs);
1163 }
1164 scval = sv;
1165 }
1166 } else if (obj_node->is_SafePointScalarMerge()) {
1167 SafePointScalarMergeNode* smerge = obj_node->as_SafePointScalarMerge();
1168 ObjectMergeValue* mv = (ObjectMergeValue*) sv_for_node_id(objs, smerge->_idx);
1169
1170 if (mv == nullptr) {
1171 GrowableArray<ScopeValue*> deps;
1172
1173 int merge_pointer_idx = smerge->merge_pointer_idx(youngest_jvms);
1174 FillLocArray(0, sfn, sfn->in(merge_pointer_idx), &deps, objs);
1175 assert(deps.length() == 1, "missing value");
1176
1243 DebugToken *locvals = C->debug_info()->create_scope_values(locarray);
1244 DebugToken *expvals = C->debug_info()->create_scope_values(exparray);
1245 DebugToken *monvals = C->debug_info()->create_monitor_values(monarray);
1246
1247 // Make method available for all Safepoints
1248 ciMethod* scope_method = method ? method : C->method();
1249 // Describe the scope here
1250 assert(jvms->bci() >= InvocationEntryBci && jvms->bci() <= 0x10000, "must be a valid or entry BCI");
1251 assert(!jvms->should_reexecute() || depth == max_depth, "reexecute allowed only for the youngest");
1252 // Now we can describe the scope.
1253 methodHandle null_mh;
1254 bool rethrow_exception = false;
1255 C->debug_info()->describe_scope(
1256 safepoint_pc_offset,
1257 null_mh,
1258 scope_method,
1259 jvms->bci(),
1260 jvms->should_reexecute(),
1261 rethrow_exception,
1262 return_oop,
1263 return_scalarized,
1264 has_ea_local_in_scope,
1265 arg_escape,
1266 locvals,
1267 expvals,
1268 monvals
1269 );
1270 } // End jvms loop
1271
1272 // Mark the end of the scope set.
1273 C->debug_info()->end_safepoint(safepoint_pc_offset);
1274 }
1275
1276
1277
1278 // A simplified version of Process_OopMap_Node, to handle non-safepoints.
1279 class NonSafepointEmitter {
1280 Compile* C;
1281 JVMState* _pending_jvms;
1282 int _pending_offset;
1283
1352 methodHandle null_mh;
1353 debug_info->describe_scope(pc_offset, null_mh, method, jvms->bci(), jvms->should_reexecute());
1354 }
1355
1356 // Mark the end of the scope set.
1357 debug_info->end_non_safepoint(pc_offset);
1358 }
1359
1360 //------------------------------init_buffer------------------------------------
1361 void PhaseOutput::estimate_buffer_size(int& const_req) {
1362
1363 // Set the initially allocated size
1364 const_req = initial_const_capacity;
1365
1366 // The extra spacing after the code is necessary on some platforms.
1367 // Sometimes we need to patch in a jump after the last instruction,
1368 // if the nmethod has been deoptimized. (See 4932387, 4894843.)
1369
1370 // Compute the byte offset where we can store the deopt pc.
1371 if (C->fixed_slots() != 0) {
1372 // Skip other fixed slots
1373 int current_slot = C->fixed_slots();
1374 if (C->needs_stack_repair()) {
1375 current_slot -= VMRegImpl::slots_per_word;
1376 }
1377 if (C->needs_nm_slot()) {
1378 current_slot -= VMRegImpl::slots_per_word;
1379 }
1380 int orig_pc_slot = current_slot - VMRegImpl::slots_per_word;
1381 _orig_pc_slot_offset_in_bytes = C->regalloc()->reg2offset(OptoReg::stack2reg(orig_pc_slot));
1382 }
1383
1384 // Compute prolog code size
1385 _frame_slots = OptoReg::reg2stack(C->matcher()->_old_SP) + C->regalloc()->_framesize;
1386 assert(_frame_slots >= 0 && _frame_slots < 1000000, "sanity check");
1387
1388 if (C->has_mach_constant_base_node()) {
1389 uint add_size = 0;
1390 // Fill the constant table.
1391 // Note: This must happen before shorten_branches.
1392 for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1393 Block* b = C->cfg()->get_block(i);
1394
1395 for (uint j = 0; j < b->number_of_nodes(); j++) {
1396 Node* n = b->get_node(j);
1397
1398 // If the node is a MachConstantNode evaluate the constant
1399 // value section.
1400 if (n->is_MachConstant()) {
1401 MachConstantNode* machcon = n->as_MachConstant();
1606 MachNode *nop = new MachNopNode(nops_cnt);
1607 block->insert_node(nop, j++);
1608 last_inst++;
1609 C->cfg()->map_node_to_block(nop, block);
1610 // Ensure enough space.
1611 masm->code()->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1612 if ((masm->code()->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1613 C->record_failure("CodeCache is full");
1614 return;
1615 }
1616 nop->emit(masm, C->regalloc());
1617 masm->code()->flush_bundle(true);
1618 current_offset = masm->offset();
1619 }
1620
1621 bool observe_safepoint = is_sfn;
1622 // Remember the start of the last call in a basic block
1623 if (is_mcall) {
1624 MachCallNode *mcall = mach->as_MachCall();
1625
1626 if (mcall->entry_point() != nullptr) {
1627 // This destination address is NOT PC-relative
1628 mcall->method_set((intptr_t)mcall->entry_point());
1629 }
1630
1631 // Save the return address
1632 call_returns[block->_pre_order] = current_offset + mcall->ret_addr_offset();
1633
1634 observe_safepoint = mcall->guaranteed_safepoint();
1635 }
1636
1637 // sfn will be valid whenever mcall is valid now because of inheritance
1638 if (observe_safepoint) {
1639 // Handle special safepoint nodes for synchronization
1640 if (!is_mcall) {
1641 MachSafePointNode *sfn = mach->as_MachSafePoint();
1642 // !!!!! Stubs only need an oopmap right now, so bail out
1643 if (sfn->jvms()->method() == nullptr) {
1644 // Write the oopmap directly to the code blob??!!
1645 continue;
1646 }
1647 } // End synchronization
1648
1649 non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1747 if ((node_offsets != nullptr) && (n->_idx < node_offset_limit)) {
1748 node_offsets[n->_idx] = masm->offset();
1749 }
1750 #endif
1751 assert(!C->failing_internal() || C->failure_is_artificial(), "Should not reach here if failing.");
1752
1753 // "Normal" instruction case
1754 DEBUG_ONLY(uint instr_offset = masm->offset());
1755 n->emit(masm, C->regalloc());
1756 current_offset = masm->offset();
1757
1758 // Above we only verified that there is enough space in the instruction section.
1759 // However, the instruction may emit stubs that cause code buffer expansion.
1760 // Bail out here if expansion failed due to a lack of code cache space.
1761 if (C->failing()) {
1762 return;
1763 }
1764
1765 assert(!is_mcall || (call_returns[block->_pre_order] <= (uint)current_offset),
1766 "ret_addr_offset() not within emitted code");
1767 #ifdef ASSERT
1768 uint n_size = n->size(C->regalloc());
1769 if (n_size < (current_offset-instr_offset)) {
1770 MachNode* mach = n->as_Mach();
1771 n->dump();
1772 mach->dump_format(C->regalloc(), tty);
1773 tty->print_cr(" n_size (%d), current_offset (%d), instr_offset (%d)", n_size, current_offset, instr_offset);
1774 Disassembler::decode(masm->code()->insts_begin() + instr_offset, masm->code()->insts_begin() + current_offset + 1, tty);
1775 tty->print_cr(" ------------------- ");
1776 BufferBlob* blob = this->scratch_buffer_blob();
1777 address blob_begin = blob->content_begin();
1778 Disassembler::decode(blob_begin, blob_begin + n_size + 1, tty);
1779 assert(false, "wrong size of mach node");
1780 }
1781 #endif
1782 non_safepoints.observe_instruction(n, current_offset);
1783
1784 // mcall is last "call" that can be a safepoint
1785 // record it so we can see if a poll will directly follow it
1786 // in which case we'll need a pad to make the PcDesc sites unique
3002 anti_do_use( b, n, _regalloc->get_reg_second(def) );
3003 }
3004 }
3005 // Do not allow defs of new derived values to float above GC
3006 // points unless the base is definitely available at the GC point.
3007
3008 Node *m = b->get_node(i);
3009
3010 if (last_safept_node != end_node &&
3011 m != last_safept_node) {
3012 bool need_safept_prec = false;
3013 // Add precedence edge from following safepoint to use of derived pointer
3014 for (uint k = 1; k < m->req(); k++) {
3015 const Type *t = m->in(k)->bottom_type();
3016 if (t->isa_oop_ptr() &&
3017 t->is_ptr()->offset() != 0) {
3018 need_safept_prec = true;
3019 break;
3020 }
3021 }
3022
3023 // A CheckCastPP whose input is still RawPtr must stay above the following safepoint.
3024 // Otherwise post-regalloc block-local scheduling can leave a live raw oop at the safepoint.
3025 if (!need_safept_prec && m->is_Mach() &&
3026 m->as_Mach()->ideal_Opcode() == Op_CheckCastPP) {
3027 Node* def = m->in(1);
3028 if (def != nullptr && def->bottom_type()->base() == Type::RawPtr) {
3029 need_safept_prec = true;
3030 }
3031 }
3032 if (need_safept_prec) {
3033 last_safept_node->add_prec(m);
3034 }
3035 }
3036
3037 if( n->jvms() ) { // Precedence edge from derived to safept
3038 // Check if last_safept_node was moved by pinch-point insertion in anti_do_use()
3039 if( b->get_node(last_safept) != last_safept_node ) {
3040 last_safept = b->find_node(last_safept_node);
3041 }
3042 for( uint j=last_safept; j > i; j-- ) {
3163 }
3164 #endif
3165
3166 //-----------------------init_scratch_buffer_blob------------------------------
3167 // Construct a temporary BufferBlob and cache it for this compile.
3168 void PhaseOutput::init_scratch_buffer_blob(int const_size) {
3169 // If there is already a scratch buffer blob allocated and the
3170 // constant section is big enough, use it. Otherwise free the
3171 // current and allocate a new one.
3172 BufferBlob* blob = scratch_buffer_blob();
3173 if ((blob != nullptr) && (const_size <= _scratch_const_size)) {
3174 // Use the current blob.
3175 } else {
3176 if (blob != nullptr) {
3177 BufferBlob::free(blob);
3178 }
3179
3180 ResourceMark rm;
3181 _scratch_const_size = const_size;
3182 int size = C2Compiler::initial_code_buffer_size(const_size);
3183 if (C->has_scalarized_args()) {
3184 // Inline type entry points (MachVEPNodes) require lots of space for GC barriers and oop verification
3185 // when loading object fields from the buffered argument. Increase scratch buffer size accordingly.
3186 ciMethod* method = C->method();
3187 int barrier_size = UseZGC ? 200 : (7 DEBUG_ONLY(+ 37));
3188 int arg_num = 0;
3189 if (!method->is_static()) {
3190 if (method->is_scalarized_arg(arg_num)) {
3191 size += method->holder()->as_inline_klass()->oop_count() * barrier_size;
3192 }
3193 arg_num++;
3194 }
3195 for (ciSignatureStream str(method->signature()); !str.at_return_type(); str.next()) {
3196 if (method->is_scalarized_arg(arg_num)) {
3197 size += str.type()->as_inline_klass()->oop_count() * barrier_size;
3198 }
3199 arg_num++;
3200 }
3201 }
3202 blob = BufferBlob::create("Compile::scratch_buffer", size);
3203 // Record the buffer blob for next time.
3204 set_scratch_buffer_blob(blob);
3205 // Have we run out of code space?
3206 if (scratch_buffer_blob() == nullptr) {
3207 // Let CompilerBroker disable further compilations.
3208 C->record_failure("Not enough space for scratch buffer in CodeCache");
3209 return;
3210 }
3211 }
3212
3213 // Initialize the relocation buffers
3214 relocInfo* locs_buf = (relocInfo*) blob->content_end() - MAX_locs_size;
3215 set_scratch_locs_memory(locs_buf);
3216 }
3217
3218
3219 //-----------------------scratch_emit_size-------------------------------------
3220 // Helper function that computes size by emitting code
3221 uint PhaseOutput::scratch_emit_size(const Node* n) {
3252 buf.insts()->set_scratch_emit();
3253 buf.stubs()->set_scratch_emit();
3254
3255 // Do the emission.
3256
3257 Label fakeL; // Fake label for branch instructions.
3258 Label* saveL = nullptr;
3259 uint save_bnum = 0;
3260 bool is_branch = n->is_MachBranch();
3261 C2_MacroAssembler masm(&buf);
3262 masm.bind(fakeL);
3263 if (is_branch) {
3264 n->as_MachBranch()->save_label(&saveL, &save_bnum);
3265 n->as_MachBranch()->label_set(&fakeL, 0);
3266 }
3267 n->emit(&masm, C->regalloc());
3268
3269 // Emitting into the scratch buffer should not fail
3270 assert(!C->failing_internal() || C->failure_is_artificial(), "Must not have pending failure. Reason is: %s", C->failure_reason());
3271
3272 // Restore label.
3273 if (is_branch) {
3274 n->as_MachBranch()->label_set(saveL, save_bnum);
3275 }
3276
3277 // End scratch_emit_size section.
3278 set_in_scratch_emit_size(false);
3279
3280 return buf.insts_size();
3281 }
3282
3283 void PhaseOutput::install() {
3284 if (!C->should_install_code()) {
3285 return;
3286 } else if (C->stub_function() != nullptr) {
3287 install_stub(C->stub_name());
3288 } else {
3289 install_code(C->method(),
3290 C->entry_bci(),
3291 CompileBroker::compiler2(),
3292 C->has_unsafe_access(),
3293 SharedRuntime::is_wide_vector(C->max_vector_size()));
3294 }
3295 }
3296
3297 void PhaseOutput::install_code(ciMethod* target,
3298 int entry_bci,
3299 AbstractCompiler* compiler,
3300 bool has_unsafe_access,
3301 bool has_wide_vectors) {
3302 // Check if we want to skip execution of all compiled code.
3303 {
3304 #ifndef PRODUCT
3305 if (OptoNoExecute) {
3306 C->record_method_not_compilable("+OptoNoExecute"); // Flag as failed
3307 return;
3308 }
3309 #endif
3310 Compile::TracePhase tp(_t_registerMethod);
3311
3312 if (C->is_osr_compilation()) {
3313 _code_offsets.set_value(CodeOffsets::Verified_Entry, 0);
3314 _code_offsets.set_value(CodeOffsets::OSR_Entry, _first_block_size);
3315 } else {
3316 _code_offsets.set_value(CodeOffsets::Verified_Entry, _first_block_size);
3317 if (_code_offsets.value(CodeOffsets::Verified_Inline_Entry) == CodeOffsets::no_such_entry_point) {
3318 _code_offsets.set_value(CodeOffsets::Verified_Inline_Entry, _first_block_size);
3319 }
3320 if (_code_offsets.value(CodeOffsets::Verified_Inline_Entry_RO) == CodeOffsets::no_such_entry_point) {
3321 _code_offsets.set_value(CodeOffsets::Verified_Inline_Entry_RO, _first_block_size);
3322 }
3323 if (_code_offsets.value(CodeOffsets::Entry) == CodeOffsets::no_such_entry_point) {
3324 _code_offsets.set_value(CodeOffsets::Entry, _first_block_size);
3325 }
3326 _code_offsets.set_value(CodeOffsets::OSR_Entry, 0);
3327 }
3328
3329 C->env()->register_method(target,
3330 entry_bci,
3331 &_code_offsets,
3332 _orig_pc_slot_offset_in_bytes,
3333 code_buffer(),
3334 frame_size_in_words(),
3335 _oop_map_set,
3336 &_handler_table,
3337 inc_table(),
3338 compiler,
3339 has_unsafe_access,
3340 SharedRuntime::is_wide_vector(C->max_vector_size()),
3341 C->has_monitors(),
3342 C->has_scoped_access(),
3343 0);
3344
3345 if (C->log() != nullptr) { // Print code cache state into compiler log
3346 C->log()->code_cache_state();
3347 }
3348 }
3349 }
3350 void PhaseOutput::install_stub(const char* stub_name) {
3351 // Entry point will be accessed using stub_entry_point();
3352 if (code_buffer() == nullptr) {
3353 Matcher::soft_match_failure();
3354 } else {
3355 if (PrintAssembly && (WizardMode || Verbose))
3356 tty->print_cr("### Stub::%s", stub_name);
3357
3358 if (!C->failing()) {
3359 assert(C->fixed_slots() == 0, "no fixed slots used for runtime stubs");
3360
3361 // Make the NMethod
3362 // For now we mark the frame as never safe for profile stackwalking
3363 RuntimeStub *rs = RuntimeStub::new_runtime_stub(stub_name,
|