1 /*
2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "ci/ciReplay.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "code/aotCodeCache.hpp"
30 #include "code/exceptionHandlerTable.hpp"
31 #include "code/nmethod.hpp"
32 #include "compiler/compilationFailureInfo.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/compileLog.hpp"
36 #include "compiler/compiler_globals.hpp"
37 #include "compiler/compilerDefinitions.hpp"
38 #include "compiler/compilerOracle.hpp"
39 #include "compiler/disassembler.hpp"
40 #include "compiler/oopMap.hpp"
41 #include "gc/shared/barrierSet.hpp"
42 #include "gc/shared/c2/barrierSetC2.hpp"
43 #include "jfr/jfrEvents.hpp"
44 #include "jvm_io.h"
45 #include "memory/allocation.hpp"
46 #include "memory/arena.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "opto/addnode.hpp"
49 #include "opto/block.hpp"
50 #include "opto/c2compiler.hpp"
51 #include "opto/callGenerator.hpp"
52 #include "opto/callnode.hpp"
53 #include "opto/castnode.hpp"
54 #include "opto/cfgnode.hpp"
55 #include "opto/chaitin.hpp"
56 #include "opto/compile.hpp"
57 #include "opto/connode.hpp"
58 #include "opto/convertnode.hpp"
59 #include "opto/divnode.hpp"
60 #include "opto/escape.hpp"
61 #include "opto/idealGraphPrinter.hpp"
62 #include "opto/locknode.hpp"
63 #include "opto/loopnode.hpp"
64 #include "opto/machnode.hpp"
65 #include "opto/macro.hpp"
66 #include "opto/matcher.hpp"
67 #include "opto/mathexactnode.hpp"
68 #include "opto/memnode.hpp"
69 #include "opto/mulnode.hpp"
70 #include "opto/narrowptrnode.hpp"
71 #include "opto/node.hpp"
72 #include "opto/opaquenode.hpp"
73 #include "opto/opcodes.hpp"
74 #include "opto/output.hpp"
75 #include "opto/parse.hpp"
76 #include "opto/phaseX.hpp"
77 #include "opto/rootnode.hpp"
78 #include "opto/runtime.hpp"
79 #include "opto/stringopts.hpp"
80 #include "opto/type.hpp"
81 #include "opto/vector.hpp"
82 #include "opto/vectornode.hpp"
83 #include "runtime/globals_extension.hpp"
84 #include "runtime/sharedRuntime.hpp"
85 #include "runtime/signature.hpp"
86 #include "runtime/stubRoutines.hpp"
87 #include "runtime/timer.hpp"
88 #include "utilities/align.hpp"
89 #include "utilities/copy.hpp"
90 #include "utilities/hashTable.hpp"
91 #include "utilities/macros.hpp"
92
93 // -------------------- Compile::mach_constant_base_node -----------------------
94 // Constant table base node singleton.
95 MachConstantBaseNode* Compile::mach_constant_base_node() {
96 if (_mach_constant_base_node == nullptr) {
97 _mach_constant_base_node = new MachConstantBaseNode();
98 _mach_constant_base_node->add_req(C->root());
99 }
100 return _mach_constant_base_node;
101 }
102
388 // as dead to be conservative about the dead node count at any
389 // given time.
390 if (!dead->is_Con()) {
391 record_dead_node(dead->_idx);
392 }
393 if (dead->is_macro()) {
394 remove_macro_node(dead);
395 }
396 if (dead->is_expensive()) {
397 remove_expensive_node(dead);
398 }
399 if (dead->is_OpaqueTemplateAssertionPredicate()) {
400 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
401 }
402 if (dead->is_ParsePredicate()) {
403 remove_parse_predicate(dead->as_ParsePredicate());
404 }
405 if (dead->for_post_loop_opts_igvn()) {
406 remove_from_post_loop_opts_igvn(dead);
407 }
408 if (dead->for_merge_stores_igvn()) {
409 remove_from_merge_stores_igvn(dead);
410 }
411 if (dead->is_Call()) {
412 remove_useless_late_inlines( &_late_inlines, dead);
413 remove_useless_late_inlines( &_string_late_inlines, dead);
414 remove_useless_late_inlines( &_boxing_late_inlines, dead);
415 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
416
417 if (dead->is_CallStaticJava()) {
418 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
419 }
420 }
421 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
422 bs->unregister_potential_barrier_node(dead);
423 }
424
425 // Disconnect all useless nodes by disconnecting those at the boundary.
426 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
427 uint next = 0;
435 // Use raw traversal of out edges since this code removes out edges
436 int max = n->outcnt();
437 for (int j = 0; j < max; ++j) {
438 Node* child = n->raw_out(j);
439 if (!useful.member(child)) {
440 assert(!child->is_top() || child != top(),
441 "If top is cached in Compile object it is in useful list");
442 // Only need to remove this out-edge to the useless node
443 n->raw_del_out(j);
444 --j;
445 --max;
446 if (child->is_data_proj_of_pure_function(n)) {
447 worklist.push(n);
448 }
449 }
450 }
451 if (n->outcnt() == 1 && n->has_special_unique_user()) {
452 assert(useful.member(n->unique_out()), "do not push a useless node");
453 worklist.push(n->unique_out());
454 }
455 }
456
457 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
458 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
459 // Remove useless Template Assertion Predicate opaque nodes
460 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
461 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
462 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
463 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
464 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
465 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
466 #ifdef ASSERT
467 if (_modified_nodes != nullptr) {
468 _modified_nodes->remove_useless_nodes(useful.member_set());
469 }
470 #endif
471
472 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
473 bs->eliminate_useless_gc_barriers(useful, this);
474 // clean up the late inline lists
475 remove_useless_late_inlines( &_late_inlines, useful);
476 remove_useless_late_inlines( &_string_late_inlines, useful);
477 remove_useless_late_inlines( &_boxing_late_inlines, useful);
478 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
479 DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
480 }
481
482 // ============================================================================
629
630
631 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
632 Options options, DirectiveSet* directive)
633 : Phase(Compiler),
634 _compile_id(ci_env->compile_id()),
635 _options(options),
636 _method(target),
637 _entry_bci(osr_bci),
638 _ilt(nullptr),
639 _stub_function(nullptr),
640 _stub_name(nullptr),
641 _stub_id(StubId::NO_STUBID),
642 _stub_entry_point(nullptr),
643 _max_node_limit(MaxNodeLimit),
644 _post_loop_opts_phase(false),
645 _merge_stores_phase(false),
646 _allow_macro_nodes(true),
647 _inlining_progress(false),
648 _inlining_incrementally(false),
649 _do_cleanup(false),
650 _has_reserved_stack_access(target->has_reserved_stack_access()),
651 #ifndef PRODUCT
652 _igv_idx(0),
653 _trace_opto_output(directive->TraceOptoOutputOption),
654 #endif
655 _clinit_barrier_on_entry(false),
656 _stress_seed(0),
657 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
658 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
659 _env(ci_env),
660 _directive(directive),
661 _log(ci_env->log()),
662 _first_failure_details(nullptr),
663 _intrinsics(comp_arena(), 0, 0, nullptr),
664 _macro_nodes(comp_arena(), 8, 0, nullptr),
665 _parse_predicates(comp_arena(), 8, 0, nullptr),
666 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
667 _expensive_nodes(comp_arena(), 8, 0, nullptr),
668 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
669 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
670 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
671 _coarsened_locks(comp_arena(), 8, 0, nullptr),
672 _congraph(nullptr),
673 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
674 _unique(0),
675 _dead_node_count(0),
676 _dead_node_list(comp_arena()),
677 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
678 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
679 _node_arena(&_node_arena_one),
680 _mach_constant_base_node(nullptr),
681 _Compile_types(mtCompiler, Arena::Tag::tag_type),
682 _initial_gvn(nullptr),
683 _igvn_worklist(nullptr),
684 _types(nullptr),
685 _node_hash(nullptr),
686 _late_inlines(comp_arena(), 2, 0, nullptr),
687 _string_late_inlines(comp_arena(), 2, 0, nullptr),
688 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
757 #define MINIMUM_NODE_HASH 1023
758
759 // GVN that will be run immediately on new nodes
760 uint estimated_size = method()->code_size()*4+64;
761 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
762 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
763 _types = new (comp_arena()) Type_Array(comp_arena());
764 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
765 PhaseGVN gvn;
766 set_initial_gvn(&gvn);
767
768 { // Scope for timing the parser
769 TracePhase tp(_t_parser);
770
771 // Put top into the hash table ASAP.
772 initial_gvn()->transform(top());
773
774 // Set up tf(), start(), and find a CallGenerator.
775 CallGenerator* cg = nullptr;
776 if (is_osr_compilation()) {
777 const TypeTuple *domain = StartOSRNode::osr_domain();
778 const TypeTuple *range = TypeTuple::make_range(method()->signature());
779 init_tf(TypeFunc::make(domain, range));
780 StartNode* s = new StartOSRNode(root(), domain);
781 initial_gvn()->set_type_bottom(s);
782 verify_start(s);
783 cg = CallGenerator::for_osr(method(), entry_bci());
784 } else {
785 // Normal case.
786 init_tf(TypeFunc::make(method()));
787 StartNode* s = new StartNode(root(), tf()->domain());
788 initial_gvn()->set_type_bottom(s);
789 verify_start(s);
790 float past_uses = method()->interpreter_invocation_count();
791 float expected_uses = past_uses;
792 cg = CallGenerator::for_inline(method(), expected_uses);
793 }
794 if (failing()) return;
795 if (cg == nullptr) {
796 const char* reason = InlineTree::check_can_parse(method());
797 assert(reason != nullptr, "expect reason for parse failure");
798 stringStream ss;
799 ss.print("cannot parse method: %s", reason);
800 record_method_not_compilable(ss.as_string());
801 return;
802 }
803
804 gvn.set_type(root(), root()->bottom_type());
805
806 JVMState* jvms = build_start_state(start(), tf());
807 if ((jvms = cg->generate(jvms)) == nullptr) {
868 print_ideal_ir("print_ideal");
869 }
870 #endif
871
872 #ifdef ASSERT
873 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
874 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
875 #endif
876
877 // Dump compilation data to replay it.
878 if (directive->DumpReplayOption) {
879 env()->dump_replay_data(_compile_id);
880 }
881 if (directive->DumpInlineOption && (ilt() != nullptr)) {
882 env()->dump_inline_data(_compile_id);
883 }
884
885 // Now that we know the size of all the monitors we can add a fixed slot
886 // for the original deopt pc.
887 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
888 set_fixed_slots(next_slot);
889
890 // Compute when to use implicit null checks. Used by matching trap based
891 // nodes and NullCheck optimization.
892 set_allowed_deopt_reasons();
893
894 // Now generate code
895 Code_Gen();
896 }
897
898 //------------------------------Compile----------------------------------------
899 // Compile a runtime stub
900 Compile::Compile(ciEnv* ci_env,
901 TypeFunc_generator generator,
902 address stub_function,
903 const char* stub_name,
904 StubId stub_id,
905 int is_fancy_jump,
906 bool pass_tls,
907 bool return_pc,
908 DirectiveSet* directive)
909 : Phase(Compiler),
910 _compile_id(0),
911 _options(Options::for_runtime_stub()),
912 _method(nullptr),
913 _entry_bci(InvocationEntryBci),
914 _stub_function(stub_function),
915 _stub_name(stub_name),
916 _stub_id(stub_id),
917 _stub_entry_point(nullptr),
918 _max_node_limit(MaxNodeLimit),
919 _post_loop_opts_phase(false),
920 _merge_stores_phase(false),
921 _allow_macro_nodes(true),
922 _inlining_progress(false),
923 _inlining_incrementally(false),
924 _has_reserved_stack_access(false),
925 #ifndef PRODUCT
926 _igv_idx(0),
927 _trace_opto_output(directive->TraceOptoOutputOption),
928 #endif
929 _clinit_barrier_on_entry(false),
930 _stress_seed(0),
931 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
932 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
933 _env(ci_env),
934 _directive(directive),
935 _log(ci_env->log()),
936 _first_failure_details(nullptr),
937 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
938 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
939 _congraph(nullptr),
940 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
941 _unique(0),
942 _dead_node_count(0),
943 _dead_node_list(comp_arena()),
944 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
1059 _fixed_slots = 0;
1060 set_has_split_ifs(false);
1061 set_has_loops(false); // first approximation
1062 set_has_stringbuilder(false);
1063 set_has_boxed_value(false);
1064 _trap_can_recompile = false; // no traps emitted yet
1065 _major_progress = true; // start out assuming good things will happen
1066 set_has_unsafe_access(false);
1067 set_max_vector_size(0);
1068 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1069 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1070 set_decompile_count(0);
1071
1072 #ifndef PRODUCT
1073 _phase_counter = 0;
1074 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1075 #endif
1076
1077 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1078 _loop_opts_cnt = LoopOptsCount;
1079 set_do_inlining(Inline);
1080 set_max_inline_size(MaxInlineSize);
1081 set_freq_inline_size(FreqInlineSize);
1082 set_do_scheduling(OptoScheduling);
1083
1084 set_do_vector_loop(false);
1085 set_has_monitors(false);
1086 set_has_scoped_access(false);
1087
1088 if (AllowVectorizeOnDemand) {
1089 if (has_method() && _directive->VectorizeOption) {
1090 set_do_vector_loop(true);
1091 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1092 } else if (has_method() && method()->name() != nullptr &&
1093 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1094 set_do_vector_loop(true);
1095 }
1096 }
1097 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1098 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1339 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1340 assert(do_aliasing(), "Aliasing should be enabled");
1341 int offset = tj->offset();
1342 TypePtr::PTR ptr = tj->ptr();
1343
1344 // Known instance (scalarizable allocation) alias only with itself.
1345 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1346 tj->is_oopptr()->is_known_instance();
1347
1348 // Process weird unsafe references.
1349 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1350 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1351 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1352 tj = TypeOopPtr::BOTTOM;
1353 ptr = tj->ptr();
1354 offset = tj->offset();
1355 }
1356
1357 // Array pointers need some flattening
1358 const TypeAryPtr* ta = tj->isa_aryptr();
1359 if (ta && ta->is_stable()) {
1360 // Erase stability property for alias analysis.
1361 tj = ta = ta->cast_to_stable(false);
1362 }
1363 if( ta && is_known_inst ) {
1364 if ( offset != Type::OffsetBot &&
1365 offset > arrayOopDesc::length_offset_in_bytes() ) {
1366 offset = Type::OffsetBot; // Flatten constant access into array body only
1367 tj = ta = ta->
1368 remove_speculative()->
1369 cast_to_ptr_type(ptr)->
1370 with_offset(offset);
1371 }
1372 } else if (ta) {
1373 // For arrays indexed by constant indices, we flatten the alias
1374 // space to include all of the array body. Only the header, klass
1375 // and array length can be accessed un-aliased.
1376 if( offset != Type::OffsetBot ) {
1377 if( ta->const_oop() ) { // MethodData* or Method*
1378 offset = Type::OffsetBot; // Flatten constant access into array body
1379 tj = ta = ta->
1380 remove_speculative()->
1381 cast_to_ptr_type(ptr)->
1382 cast_to_exactness(false)->
1383 with_offset(offset);
1384 } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1385 // range is OK as-is.
1386 tj = ta = TypeAryPtr::RANGE;
1387 } else if( offset == oopDesc::klass_offset_in_bytes() ) {
1388 tj = TypeInstPtr::KLASS; // all klass loads look alike
1389 ta = TypeAryPtr::RANGE; // generic ignored junk
1390 ptr = TypePtr::BotPTR;
1391 } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1392 tj = TypeInstPtr::MARK;
1393 ta = TypeAryPtr::RANGE; // generic ignored junk
1394 ptr = TypePtr::BotPTR;
1395 } else { // Random constant offset into array body
1396 offset = Type::OffsetBot; // Flatten constant access into array body
1397 tj = ta = ta->
1398 remove_speculative()->
1399 cast_to_ptr_type(ptr)->
1400 cast_to_exactness(false)->
1401 with_offset(offset);
1402 }
1403 }
1404 // Arrays of fixed size alias with arrays of unknown size.
1405 if (ta->size() != TypeInt::POS) {
1406 const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1407 tj = ta = ta->
1408 remove_speculative()->
1409 cast_to_ptr_type(ptr)->
1410 with_ary(tary)->
1411 cast_to_exactness(false);
1412 }
1413 // Arrays of known objects become arrays of unknown objects.
1414 if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1415 const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1416 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1417 }
1418 if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1419 const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1420 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1421 }
1422 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1423 // cannot be distinguished by bytecode alone.
1424 if (ta->elem() == TypeInt::BOOL) {
1425 const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1426 ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1427 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,offset);
1428 }
1429 // During the 2nd round of IterGVN, NotNull castings are removed.
1430 // Make sure the Bottom and NotNull variants alias the same.
1431 // Also, make sure exact and non-exact variants alias the same.
1432 if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1433 tj = ta = ta->
1434 remove_speculative()->
1435 cast_to_ptr_type(TypePtr::BotPTR)->
1436 cast_to_exactness(false)->
1437 with_offset(offset);
1438 }
1439 }
1440
1441 // Oop pointers need some flattening
1442 const TypeInstPtr *to = tj->isa_instptr();
1443 if (to && to != TypeOopPtr::BOTTOM) {
1444 ciInstanceKlass* ik = to->instance_klass();
1445 if( ptr == TypePtr::Constant ) {
1446 if (ik != ciEnv::current()->Class_klass() ||
1447 offset < ik->layout_helper_size_in_bytes()) {
1448 // No constant oop pointers (such as Strings); they alias with
1449 // unknown strings.
1450 assert(!is_known_inst, "not scalarizable allocation");
1451 tj = to = to->
1452 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1453 remove_speculative()->
1454 cast_to_ptr_type(TypePtr::BotPTR)->
1455 cast_to_exactness(false);
1456 }
1457 } else if( is_known_inst ) {
1458 tj = to; // Keep NotNull and klass_is_exact for instance type
1459 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1460 // During the 2nd round of IterGVN, NotNull castings are removed.
1461 // Make sure the Bottom and NotNull variants alias the same.
1462 // Also, make sure exact and non-exact variants alias the same.
1463 tj = to = to->
1464 remove_speculative()->
1465 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1466 cast_to_ptr_type(TypePtr::BotPTR)->
1467 cast_to_exactness(false);
1468 }
1469 if (to->speculative() != nullptr) {
1470 tj = to = to->remove_speculative();
1471 }
1472 // Canonicalize the holder of this field
1473 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1474 // First handle header references such as a LoadKlassNode, even if the
1475 // object's klass is unloaded at compile time (4965979).
1476 if (!is_known_inst) { // Do it only for non-instance types
1477 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, offset);
1478 }
1479 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1480 // Static fields are in the space above the normal instance
1481 // fields in the java.lang.Class instance.
1482 if (ik != ciEnv::current()->Class_klass()) {
1483 to = nullptr;
1484 tj = TypeOopPtr::BOTTOM;
1485 offset = tj->offset();
1486 }
1487 } else {
1488 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1489 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1490 assert(tj->offset() == offset, "no change to offset expected");
1491 bool xk = to->klass_is_exact();
1492 int instance_id = to->instance_id();
1493
1494 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1495 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1496 // its interfaces are included.
1497 if (xk && ik->equals(canonical_holder)) {
1498 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id), "exact type should be canonical type");
1499 } else {
1500 assert(xk || !is_known_inst, "Known instance should be exact type");
1501 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id);
1502 }
1503 }
1504 }
1505
1506 // Klass pointers to object array klasses need some flattening
1507 const TypeKlassPtr *tk = tj->isa_klassptr();
1508 if( tk ) {
1509 // If we are referencing a field within a Klass, we need
1510 // to assume the worst case of an Object. Both exact and
1511 // inexact types must flatten to the same alias class so
1512 // use NotNull as the PTR.
1513 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1514 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1515 env()->Object_klass(),
1516 offset);
1517 }
1518
1519 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1520 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1521 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1522 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), offset);
1523 } else {
1524 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, offset);
1525 }
1526 }
1527
1528 // Check for precise loads from the primary supertype array and force them
1529 // to the supertype cache alias index. Check for generic array loads from
1530 // the primary supertype array and also force them to the supertype cache
1531 // alias index. Since the same load can reach both, we need to merge
1532 // these 2 disparate memories into the same alias class. Since the
1533 // primary supertype array is read-only, there's no chance of confusion
1534 // where we bypass an array load and an array store.
1535 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1536 if (offset == Type::OffsetBot ||
1537 (offset >= primary_supers_offset &&
1538 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1539 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1540 offset = in_bytes(Klass::secondary_super_cache_offset());
1541 tj = tk = tk->with_offset(offset);
1542 }
1543 }
1544
1545 // Flatten all Raw pointers together.
1546 if (tj->base() == Type::RawPtr)
1547 tj = TypeRawPtr::BOTTOM;
1637 intptr_t key = (intptr_t) adr_type;
1638 key ^= key >> logAliasCacheSize;
1639 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1640 }
1641
1642
1643 //-----------------------------grow_alias_types--------------------------------
1644 void Compile::grow_alias_types() {
1645 const int old_ats = _max_alias_types; // how many before?
1646 const int new_ats = old_ats; // how many more?
1647 const int grow_ats = old_ats+new_ats; // how many now?
1648 _max_alias_types = grow_ats;
1649 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1650 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1651 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1652 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1653 }
1654
1655
1656 //--------------------------------find_alias_type------------------------------
1657 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field) {
1658 if (!do_aliasing()) {
1659 return alias_type(AliasIdxBot);
1660 }
1661
1662 AliasCacheEntry* ace = probe_alias_cache(adr_type);
1663 if (ace->_adr_type == adr_type) {
1664 return alias_type(ace->_index);
1665 }
1666
1667 // Handle special cases.
1668 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1669 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1670
1671 // Do it the slow way.
1672 const TypePtr* flat = flatten_alias_type(adr_type);
1673
1674 #ifdef ASSERT
1675 {
1676 ResourceMark rm;
1677 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1678 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1679 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1680 Type::str(adr_type));
1681 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1682 const TypeOopPtr* foop = flat->is_oopptr();
1683 // Scalarizable allocations have exact klass always.
1684 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1694 if (alias_type(i)->adr_type() == flat) {
1695 idx = i;
1696 break;
1697 }
1698 }
1699
1700 if (idx == AliasIdxTop) {
1701 if (no_create) return nullptr;
1702 // Grow the array if necessary.
1703 if (_num_alias_types == _max_alias_types) grow_alias_types();
1704 // Add a new alias type.
1705 idx = _num_alias_types++;
1706 _alias_types[idx]->Init(idx, flat);
1707 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1708 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1709 if (flat->isa_instptr()) {
1710 if (flat->offset() == java_lang_Class::klass_offset()
1711 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1712 alias_type(idx)->set_rewritable(false);
1713 }
1714 if (flat->isa_aryptr()) {
1715 #ifdef ASSERT
1716 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1717 // (T_BYTE has the weakest alignment and size restrictions...)
1718 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1719 #endif
1720 if (flat->offset() == TypePtr::OffsetBot) {
1721 alias_type(idx)->set_element(flat->is_aryptr()->elem());
1722 }
1723 }
1724 if (flat->isa_klassptr()) {
1725 if (UseCompactObjectHeaders) {
1726 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1727 alias_type(idx)->set_rewritable(false);
1728 }
1729 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1730 alias_type(idx)->set_rewritable(false);
1731 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1732 alias_type(idx)->set_rewritable(false);
1733 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1734 alias_type(idx)->set_rewritable(false);
1735 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1736 alias_type(idx)->set_rewritable(false);
1737 }
1738
1739 if (flat->isa_instklassptr()) {
1740 if (flat->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
1741 alias_type(idx)->set_rewritable(false);
1742 }
1743 }
1744 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1745 // but the base pointer type is not distinctive enough to identify
1746 // references into JavaThread.)
1747
1748 // Check for final fields.
1749 const TypeInstPtr* tinst = flat->isa_instptr();
1750 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1751 ciField* field;
1752 if (tinst->const_oop() != nullptr &&
1753 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1754 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1755 // static field
1756 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1757 field = k->get_field_by_offset(tinst->offset(), true);
1758 } else {
1759 ciInstanceKlass *k = tinst->instance_klass();
1760 field = k->get_field_by_offset(tinst->offset(), false);
1761 }
1762 assert(field == nullptr ||
1763 original_field == nullptr ||
1764 (field->holder() == original_field->holder() &&
1765 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1766 field->is_static() == original_field->is_static()), "wrong field?");
1767 // Set field() and is_rewritable() attributes.
1768 if (field != nullptr) alias_type(idx)->set_field(field);
1769 }
1770 }
1771
1772 // Fill the cache for next time.
1773 ace->_adr_type = adr_type;
1774 ace->_index = idx;
1775 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1776
1777 // Might as well try to fill the cache for the flattened version, too.
1778 AliasCacheEntry* face = probe_alias_cache(flat);
1779 if (face->_adr_type == nullptr) {
1780 face->_adr_type = flat;
1781 face->_index = idx;
1782 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1783 }
1784
1785 return alias_type(idx);
1786 }
1787
1788
1789 Compile::AliasType* Compile::alias_type(ciField* field) {
1790 const TypeOopPtr* t;
1791 if (field->is_static())
1792 t = TypeInstPtr::make(field->holder()->java_mirror());
1793 else
1794 t = TypeOopPtr::make_from_klass_raw(field->holder());
1795 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1796 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1797 return atp;
1798 }
1799
1800
1801 //------------------------------have_alias_type--------------------------------
1802 bool Compile::have_alias_type(const TypePtr* adr_type) {
1884 assert(!C->major_progress(), "not cleared");
1885
1886 if (_for_post_loop_igvn.length() > 0) {
1887 while (_for_post_loop_igvn.length() > 0) {
1888 Node* n = _for_post_loop_igvn.pop();
1889 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1890 igvn._worklist.push(n);
1891 }
1892 igvn.optimize();
1893 if (failing()) return;
1894 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1895 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1896
1897 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1898 if (C->major_progress()) {
1899 C->clear_major_progress(); // ensure that major progress is now clear
1900 }
1901 }
1902 }
1903
1904 void Compile::record_for_merge_stores_igvn(Node* n) {
1905 if (!n->for_merge_stores_igvn()) {
1906 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
1907 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1908 _for_merge_stores_igvn.append(n);
1909 }
1910 }
1911
1912 void Compile::remove_from_merge_stores_igvn(Node* n) {
1913 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1914 _for_merge_stores_igvn.remove(n);
1915 }
1916
1917 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
1918 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
1919 // the stores, and we merge the wrong sequence of stores.
1920 // Example:
1921 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
1922 // Apply MergeStores:
1923 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
2002 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2003 Bytecodes::Code c = iter.cur_bc();
2004 Node* lhs = nullptr;
2005 Node* rhs = nullptr;
2006 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2007 lhs = unc->peek_operand(0);
2008 rhs = unc->peek_operand(1);
2009 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2010 lhs = unc->peek_operand(0);
2011 }
2012
2013 ResourceMark rm;
2014 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2015 assert(live_locals.is_valid(), "broken liveness info");
2016 int len = (int)live_locals.size();
2017
2018 for (int i = 0; i < len; i++) {
2019 Node* local = unc->local(jvms, i);
2020 // kill local using the liveness of next_bci.
2021 // give up when the local looks like an operand to secure reexecution.
2022 if (!live_locals.at(i) && !local->is_top() && local != lhs && local!= rhs) {
2023 uint idx = jvms->locoff() + i;
2024 #ifdef ASSERT
2025 if (PrintOpto && Verbose) {
2026 tty->print("[unstable_if] kill local#%d: ", idx);
2027 local->dump();
2028 tty->cr();
2029 }
2030 #endif
2031 igvn.replace_input_of(unc, idx, top());
2032 modified = true;
2033 }
2034 }
2035 }
2036
2037 // keep the mondified trap for late query
2038 if (modified) {
2039 trap->set_modified();
2040 } else {
2041 _unstable_if_traps.delete_at(i);
2042 }
2043 }
2044 igvn.optimize();
2045 }
2046
2047 // StringOpts and late inlining of string methods
2048 void Compile::inline_string_calls(bool parse_time) {
2049 {
2050 // remove useless nodes to make the usage analysis simpler
2051 ResourceMark rm;
2052 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2053 }
2054
2055 {
2056 ResourceMark rm;
2057 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2229
2230 if (_string_late_inlines.length() > 0) {
2231 assert(has_stringbuilder(), "inconsistent");
2232
2233 inline_string_calls(false);
2234
2235 if (failing()) return;
2236
2237 inline_incrementally_cleanup(igvn);
2238 }
2239
2240 set_inlining_incrementally(false);
2241 }
2242
2243 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2244 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2245 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2246 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2247 // as if "inlining_incrementally() == true" were set.
2248 assert(inlining_incrementally() == false, "not allowed");
2249 assert(_modified_nodes == nullptr, "not allowed");
2250 assert(_late_inlines.length() > 0, "sanity");
2251
2252 while (_late_inlines.length() > 0) {
2253 igvn_worklist()->ensure_empty(); // should be done with igvn
2254
2255 while (inline_incrementally_one()) {
2256 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2257 }
2258 if (failing()) return;
2259
2260 inline_incrementally_cleanup(igvn);
2261 }
2262 }
2263
2264 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2265 if (_loop_opts_cnt > 0) {
2266 while (major_progress() && (_loop_opts_cnt > 0)) {
2267 TracePhase tp(_t_idealLoop);
2268 PhaseIdealLoop::optimize(igvn, mode);
2269 _loop_opts_cnt--;
2270 if (failing()) return false;
2271 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2272 }
2273 }
2274 return true;
2275 }
2276
2277 // Remove edges from "root" to each SafePoint at a backward branch.
2278 // They were inserted during parsing (see add_safepoint()) to make
2279 // infinite loops without calls or exceptions visible to root, i.e.,
2280 // useful.
2281 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2385 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2386 }
2387 assert(!has_vbox_nodes(), "sanity");
2388
2389 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2390 Compile::TracePhase tp(_t_renumberLive);
2391 igvn_worklist()->ensure_empty(); // should be done with igvn
2392 {
2393 ResourceMark rm;
2394 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2395 }
2396 igvn.reset();
2397 igvn.optimize();
2398 if (failing()) return;
2399 }
2400
2401 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2402 // safepoints
2403 remove_root_to_sfpts_edges(igvn);
2404
2405 if (failing()) return;
2406
2407 if (has_loops()) {
2408 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2409 }
2410
2411 // Perform escape analysis
2412 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2413 if (has_loops()) {
2414 // Cleanup graph (remove dead nodes).
2415 TracePhase tp(_t_idealLoop);
2416 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2417 if (failing()) return;
2418 }
2419 bool progress;
2420 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2421 do {
2422 ConnectionGraph::do_analysis(this, &igvn);
2423
2424 if (failing()) return;
2425
2426 int mcount = macro_count(); // Record number of allocations and locks before IGVN
2427
2428 // Optimize out fields loads from scalar replaceable allocations.
2429 igvn.optimize();
2430 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2431
2432 if (failing()) return;
2433
2434 if (congraph() != nullptr && macro_count() > 0) {
2435 TracePhase tp(_t_macroEliminate);
2436 PhaseMacroExpand mexp(igvn);
2437 mexp.eliminate_macro_nodes();
2438 if (failing()) return;
2439 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
2440
2441 igvn.set_delay_transform(false);
2442 igvn.optimize();
2443 if (failing()) return;
2444
2445 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2446 }
2447
2448 ConnectionGraph::verify_ram_nodes(this, root());
2449 if (failing()) return;
2450
2451 progress = do_iterative_escape_analysis() &&
2452 (macro_count() < mcount) &&
2453 ConnectionGraph::has_candidates(this);
2454 // Try again if candidates exist and made progress
2455 // by removing some allocations and/or locks.
2456 } while (progress);
2457 }
2458
2459 // Loop transforms on the ideal graph. Range Check Elimination,
2460 // peeling, unrolling, etc.
2461
2462 // Set loop opts counter
2463 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2464 {
2465 TracePhase tp(_t_idealLoop);
2466 PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
2467 _loop_opts_cnt--;
2468 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
2469 if (failing()) return;
2470 }
2471 // Loop opts pass if partial peeling occurred in previous pass
2472 if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
2473 TracePhase tp(_t_idealLoop);
2474 PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
2475 _loop_opts_cnt--;
2476 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
2477 if (failing()) return;
2478 }
2515 // Loop transforms on the ideal graph. Range Check Elimination,
2516 // peeling, unrolling, etc.
2517 if (!optimize_loops(igvn, LoopOptsDefault)) {
2518 return;
2519 }
2520
2521 if (failing()) return;
2522
2523 C->clear_major_progress(); // ensure that major progress is now clear
2524
2525 process_for_post_loop_opts_igvn(igvn);
2526
2527 process_for_merge_stores_igvn(igvn);
2528
2529 if (failing()) return;
2530
2531 #ifdef ASSERT
2532 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
2533 #endif
2534
2535 {
2536 TracePhase tp(_t_macroExpand);
2537 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
2538 PhaseMacroExpand mex(igvn);
2539 // Do not allow new macro nodes once we start to eliminate and expand
2540 C->reset_allow_macro_nodes();
2541 // Last attempt to eliminate macro nodes before expand
2542 mex.eliminate_macro_nodes();
2543 if (failing()) {
2544 return;
2545 }
2546 mex.eliminate_opaque_looplimit_macro_nodes();
2547 if (failing()) {
2548 return;
2549 }
2550 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
2551 if (mex.expand_macro_nodes()) {
2552 assert(failing(), "must bail out w/ explicit message");
2553 return;
2554 }
2555 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
2556 }
2557
2558 {
2559 TracePhase tp(_t_barrierExpand);
2560 if (bs->expand_barriers(this, igvn)) {
2561 assert(failing(), "must bail out w/ explicit message");
2562 return;
2563 }
2564 print_method(PHASE_BARRIER_EXPANSION, 2);
2565 }
2566
2567 if (C->max_vector_size() > 0) {
2568 C->optimize_logic_cones(igvn);
2569 igvn.optimize();
2570 if (failing()) return;
2571 }
2572
2573 DEBUG_ONLY( _modified_nodes = nullptr; )
2574
2575 assert(igvn._worklist.size() == 0, "not empty");
2576
2577 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
2578
2579 if (_late_inlines.length() > 0) {
2580 // More opportunities to optimize virtual and MH calls.
2581 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
2582 process_late_inline_calls_no_inline(igvn);
2583 if (failing()) return;
2584 }
2585 } // (End scope of igvn; run destructor if necessary for asserts.)
2586
2587 check_no_dead_use();
2588
2589 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
2590 // to remove hashes to unlock nodes for modifications.
2591 C->node_hash()->clear();
2592
2593 // A method with only infinite loops has no edges entering loops from root
2594 {
2595 TracePhase tp(_t_graphReshaping);
2596 if (final_graph_reshaping()) {
2597 assert(failing(), "must bail out w/ explicit message");
2598 return;
2599 }
2600 }
2601
2602 print_method(PHASE_OPTIMIZE_FINISHED, 2);
2603 DEBUG_ONLY(set_phase_optimize_finished();)
2604 }
3310 case Op_CmpD3:
3311 case Op_StoreD:
3312 case Op_LoadD:
3313 case Op_LoadD_unaligned:
3314 frc.inc_double_count();
3315 break;
3316 case Op_Opaque1: // Remove Opaque Nodes before matching
3317 n->subsume_by(n->in(1), this);
3318 break;
3319 case Op_CallLeafPure: {
3320 // If the pure call is not supported, then lower to a CallLeaf.
3321 if (!Matcher::match_rule_supported(Op_CallLeafPure)) {
3322 CallNode* call = n->as_Call();
3323 CallNode* new_call = new CallLeafNode(call->tf(), call->entry_point(),
3324 call->_name, TypeRawPtr::BOTTOM);
3325 new_call->init_req(TypeFunc::Control, call->in(TypeFunc::Control));
3326 new_call->init_req(TypeFunc::I_O, C->top());
3327 new_call->init_req(TypeFunc::Memory, C->top());
3328 new_call->init_req(TypeFunc::ReturnAdr, C->top());
3329 new_call->init_req(TypeFunc::FramePtr, C->top());
3330 for (unsigned int i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) {
3331 new_call->init_req(i, call->in(i));
3332 }
3333 n->subsume_by(new_call, this);
3334 }
3335 frc.inc_call_count();
3336 break;
3337 }
3338 case Op_CallStaticJava:
3339 case Op_CallJava:
3340 case Op_CallDynamicJava:
3341 frc.inc_java_call_count(); // Count java call site;
3342 case Op_CallRuntime:
3343 case Op_CallLeaf:
3344 case Op_CallLeafVector:
3345 case Op_CallLeafNoFP: {
3346 assert (n->is_Call(), "");
3347 CallNode *call = n->as_Call();
3348 // Count call sites where the FP mode bit would have to be flipped.
3349 // Do not count uncommon runtime calls:
3350 // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3356 int nop = n->Opcode();
3357 // Clone shared simple arguments to uncommon calls, item (1).
3358 if (n->outcnt() > 1 &&
3359 !n->is_Proj() &&
3360 nop != Op_CreateEx &&
3361 nop != Op_CheckCastPP &&
3362 nop != Op_DecodeN &&
3363 nop != Op_DecodeNKlass &&
3364 !n->is_Mem() &&
3365 !n->is_Phi()) {
3366 Node *x = n->clone();
3367 call->set_req(TypeFunc::Parms, x);
3368 }
3369 }
3370 break;
3371 }
3372 case Op_StoreB:
3373 case Op_StoreC:
3374 case Op_StoreI:
3375 case Op_StoreL:
3376 case Op_CompareAndSwapB:
3377 case Op_CompareAndSwapS:
3378 case Op_CompareAndSwapI:
3379 case Op_CompareAndSwapL:
3380 case Op_CompareAndSwapP:
3381 case Op_CompareAndSwapN:
3382 case Op_WeakCompareAndSwapB:
3383 case Op_WeakCompareAndSwapS:
3384 case Op_WeakCompareAndSwapI:
3385 case Op_WeakCompareAndSwapL:
3386 case Op_WeakCompareAndSwapP:
3387 case Op_WeakCompareAndSwapN:
3388 case Op_CompareAndExchangeB:
3389 case Op_CompareAndExchangeS:
3390 case Op_CompareAndExchangeI:
3391 case Op_CompareAndExchangeL:
3392 case Op_CompareAndExchangeP:
3393 case Op_CompareAndExchangeN:
3394 case Op_GetAndAddS:
3395 case Op_GetAndAddB:
3905 k->subsume_by(m, this);
3906 }
3907 }
3908 }
3909 break;
3910 }
3911 case Op_CmpUL: {
3912 if (!Matcher::has_match_rule(Op_CmpUL)) {
3913 // No support for unsigned long comparisons
3914 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
3915 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
3916 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
3917 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
3918 Node* andl = new AndLNode(orl, remove_sign_mask);
3919 Node* cmp = new CmpLNode(andl, n->in(2));
3920 n->subsume_by(cmp, this);
3921 }
3922 break;
3923 }
3924 #ifdef ASSERT
3925 case Op_ConNKlass: {
3926 const TypePtr* tp = n->as_Type()->type()->make_ptr();
3927 ciKlass* klass = tp->is_klassptr()->exact_klass();
3928 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
3929 break;
3930 }
3931 #endif
3932 default:
3933 assert(!n->is_Call(), "");
3934 assert(!n->is_Mem(), "");
3935 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
3936 break;
3937 }
3938 }
3939
3940 //------------------------------final_graph_reshaping_walk---------------------
3941 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
3942 // requires that the walk visits a node's inputs before visiting the node.
3943 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3944 Unique_Node_List sfpt;
4280 }
4281 }
4282
4283 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4284 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4285 }
4286
4287 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4288 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4289 }
4290
4291 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4292 if (holder->is_initialized()) {
4293 return false;
4294 }
4295 if (holder->is_being_initialized()) {
4296 if (accessing_method->holder() == holder) {
4297 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4298 // <init>, or a static method. In all those cases, there was an initialization
4299 // barrier on the holder klass passed.
4300 if (accessing_method->is_static_initializer() ||
4301 accessing_method->is_object_initializer() ||
4302 accessing_method->is_static()) {
4303 return false;
4304 }
4305 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4306 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4307 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4308 // child class can become fully initialized while its parent class is still being initialized.
4309 if (accessing_method->is_static_initializer()) {
4310 return false;
4311 }
4312 }
4313 ciMethod* root = method(); // the root method of compilation
4314 if (root != accessing_method) {
4315 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4316 }
4317 }
4318 return true;
4319 }
4320
4321 #ifndef PRODUCT
4322 //------------------------------verify_bidirectional_edges---------------------
4323 // For each input edge to a node (ie - for each Use-Def edge), verify that
4324 // there is a corresponding Def-Use edge.
4325 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4326 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4327 uint stack_size = live_nodes() >> 4;
4328 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4329 if (root_and_safepoints != nullptr) {
4359 if (in != nullptr && !in->is_top()) {
4360 // Count instances of `next`
4361 int cnt = 0;
4362 for (uint idx = 0; idx < in->_outcnt; idx++) {
4363 if (in->_out[idx] == n) {
4364 cnt++;
4365 }
4366 }
4367 assert(cnt > 0, "Failed to find Def-Use edge.");
4368 // Check for duplicate edges
4369 // walk the input array downcounting the input edges to n
4370 for (uint j = 0; j < length; j++) {
4371 if (n->in(j) == in) {
4372 cnt--;
4373 }
4374 }
4375 assert(cnt == 0, "Mismatched edge count.");
4376 } else if (in == nullptr) {
4377 assert(i == 0 || i >= n->req() ||
4378 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4379 (n->is_Unlock() && i == (n->req() - 1)) ||
4380 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4381 "only region, phi, arraycopy, unlock or membar nodes have null data edges");
4382 } else {
4383 assert(in->is_top(), "sanity");
4384 // Nothing to check.
4385 }
4386 }
4387 }
4388 }
4389
4390 //------------------------------verify_graph_edges---------------------------
4391 // Walk the Graph and verify that there is a one-to-one correspondence
4392 // between Use-Def edges and Def-Use edges in the graph.
4393 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4394 if (VerifyGraphEdges) {
4395 Unique_Node_List visited;
4396
4397 // Call graph walk to check edges
4398 verify_bidirectional_edges(visited, root_and_safepoints);
4399 if (no_dead_code) {
4400 // Now make sure that no visited node is used by an unvisited node.
4401 bool dead_nodes = false;
4512 // (1) subklass is already limited to a subtype of superklass => always ok
4513 // (2) subklass does not overlap with superklass => always fail
4514 // (3) superklass has NO subtypes and we can check with a simple compare.
4515 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
4516 if (skip) {
4517 return SSC_full_test; // Let caller generate the general case.
4518 }
4519
4520 if (subk->is_java_subtype_of(superk)) {
4521 return SSC_always_true; // (0) and (1) this test cannot fail
4522 }
4523
4524 if (!subk->maybe_java_subtype_of(superk)) {
4525 return SSC_always_false; // (2) true path dead; no dynamic test needed
4526 }
4527
4528 const Type* superelem = superk;
4529 if (superk->isa_aryklassptr()) {
4530 int ignored;
4531 superelem = superk->is_aryklassptr()->base_element_type(ignored);
4532 }
4533
4534 if (superelem->isa_instklassptr()) {
4535 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
4536 if (!ik->has_subklass()) {
4537 if (!ik->is_final()) {
4538 // Add a dependency if there is a chance of a later subclass.
4539 dependencies()->assert_leaf_type(ik);
4540 }
4541 if (!superk->maybe_java_subtype_of(subk)) {
4542 return SSC_always_false;
4543 }
4544 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4545 }
4546 } else {
4547 // A primitive array type has no subtypes.
4548 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4549 }
4550
4551 return SSC_full_test;
5350 } else {
5351 _debug_network_printer->update_compiled_method(C->method());
5352 }
5353 tty->print_cr("Method printed over network stream to IGV");
5354 _debug_network_printer->print(name, C->root(), visible_nodes, fr);
5355 }
5356 #endif // !PRODUCT
5357
5358 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5359 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5360 return value;
5361 }
5362 Node* result = nullptr;
5363 if (bt == T_BYTE) {
5364 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5365 result = new RShiftINode(result, phase->intcon(24));
5366 } else if (bt == T_BOOLEAN) {
5367 result = new AndINode(value, phase->intcon(0xFF));
5368 } else if (bt == T_CHAR) {
5369 result = new AndINode(value,phase->intcon(0xFFFF));
5370 } else {
5371 assert(bt == T_SHORT, "unexpected narrow type");
5372 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5373 result = new RShiftINode(result, phase->intcon(16));
5374 }
5375 if (transform_res) {
5376 result = phase->transform(result);
5377 }
5378 return result;
5379 }
5380
5381 void Compile::record_method_not_compilable_oom() {
5382 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5383 }
5384
5385 #ifndef PRODUCT
5386 // Collects all the control inputs from nodes on the worklist and from their data dependencies
5387 static void find_candidate_control_inputs(Unique_Node_List& worklist, Unique_Node_List& candidates) {
5388 // Follow non-control edges until we reach CFG nodes
5389 for (uint i = 0; i < worklist.size(); i++) {
|
1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "ci/ciFlatArray.hpp"
28 #include "ci/ciInlineKlass.hpp"
29 #include "ci/ciReplay.hpp"
30 #include "classfile/javaClasses.hpp"
31 #include "code/aotCodeCache.hpp"
32 #include "code/exceptionHandlerTable.hpp"
33 #include "code/nmethod.hpp"
34 #include "compiler/compilationFailureInfo.hpp"
35 #include "compiler/compilationMemoryStatistic.hpp"
36 #include "compiler/compileBroker.hpp"
37 #include "compiler/compileLog.hpp"
38 #include "compiler/compiler_globals.hpp"
39 #include "compiler/compilerDefinitions.hpp"
40 #include "compiler/compilerOracle.hpp"
41 #include "compiler/disassembler.hpp"
42 #include "compiler/oopMap.hpp"
43 #include "gc/shared/barrierSet.hpp"
44 #include "gc/shared/c2/barrierSetC2.hpp"
45 #include "jfr/jfrEvents.hpp"
46 #include "jvm_io.h"
47 #include "memory/allocation.hpp"
48 #include "memory/arena.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "opto/addnode.hpp"
51 #include "opto/block.hpp"
52 #include "opto/c2compiler.hpp"
53 #include "opto/callGenerator.hpp"
54 #include "opto/callnode.hpp"
55 #include "opto/castnode.hpp"
56 #include "opto/cfgnode.hpp"
57 #include "opto/chaitin.hpp"
58 #include "opto/compile.hpp"
59 #include "opto/connode.hpp"
60 #include "opto/convertnode.hpp"
61 #include "opto/divnode.hpp"
62 #include "opto/escape.hpp"
63 #include "opto/idealGraphPrinter.hpp"
64 #include "opto/inlinetypenode.hpp"
65 #include "opto/locknode.hpp"
66 #include "opto/loopnode.hpp"
67 #include "opto/machnode.hpp"
68 #include "opto/macro.hpp"
69 #include "opto/matcher.hpp"
70 #include "opto/mathexactnode.hpp"
71 #include "opto/memnode.hpp"
72 #include "opto/movenode.hpp"
73 #include "opto/mulnode.hpp"
74 #include "opto/multnode.hpp"
75 #include "opto/narrowptrnode.hpp"
76 #include "opto/node.hpp"
77 #include "opto/opaquenode.hpp"
78 #include "opto/opcodes.hpp"
79 #include "opto/output.hpp"
80 #include "opto/parse.hpp"
81 #include "opto/phaseX.hpp"
82 #include "opto/rootnode.hpp"
83 #include "opto/runtime.hpp"
84 #include "opto/stringopts.hpp"
85 #include "opto/type.hpp"
86 #include "opto/vector.hpp"
87 #include "opto/vectornode.hpp"
88 #include "runtime/arguments.hpp"
89 #include "runtime/globals_extension.hpp"
90 #include "runtime/sharedRuntime.hpp"
91 #include "runtime/signature.hpp"
92 #include "runtime/stubRoutines.hpp"
93 #include "runtime/timer.hpp"
94 #include "utilities/align.hpp"
95 #include "utilities/copy.hpp"
96 #include "utilities/hashTable.hpp"
97 #include "utilities/macros.hpp"
98
99 // -------------------- Compile::mach_constant_base_node -----------------------
100 // Constant table base node singleton.
101 MachConstantBaseNode* Compile::mach_constant_base_node() {
102 if (_mach_constant_base_node == nullptr) {
103 _mach_constant_base_node = new MachConstantBaseNode();
104 _mach_constant_base_node->add_req(C->root());
105 }
106 return _mach_constant_base_node;
107 }
108
394 // as dead to be conservative about the dead node count at any
395 // given time.
396 if (!dead->is_Con()) {
397 record_dead_node(dead->_idx);
398 }
399 if (dead->is_macro()) {
400 remove_macro_node(dead);
401 }
402 if (dead->is_expensive()) {
403 remove_expensive_node(dead);
404 }
405 if (dead->is_OpaqueTemplateAssertionPredicate()) {
406 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
407 }
408 if (dead->is_ParsePredicate()) {
409 remove_parse_predicate(dead->as_ParsePredicate());
410 }
411 if (dead->for_post_loop_opts_igvn()) {
412 remove_from_post_loop_opts_igvn(dead);
413 }
414 if (dead->is_InlineType()) {
415 remove_inline_type(dead);
416 }
417 if (dead->is_LoadFlat() || dead->is_StoreFlat()) {
418 remove_flat_access(dead);
419 }
420 if (dead->for_merge_stores_igvn()) {
421 remove_from_merge_stores_igvn(dead);
422 }
423 if (dead->is_Call()) {
424 remove_useless_late_inlines( &_late_inlines, dead);
425 remove_useless_late_inlines( &_string_late_inlines, dead);
426 remove_useless_late_inlines( &_boxing_late_inlines, dead);
427 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
428
429 if (dead->is_CallStaticJava()) {
430 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
431 }
432 }
433 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
434 bs->unregister_potential_barrier_node(dead);
435 }
436
437 // Disconnect all useless nodes by disconnecting those at the boundary.
438 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
439 uint next = 0;
447 // Use raw traversal of out edges since this code removes out edges
448 int max = n->outcnt();
449 for (int j = 0; j < max; ++j) {
450 Node* child = n->raw_out(j);
451 if (!useful.member(child)) {
452 assert(!child->is_top() || child != top(),
453 "If top is cached in Compile object it is in useful list");
454 // Only need to remove this out-edge to the useless node
455 n->raw_del_out(j);
456 --j;
457 --max;
458 if (child->is_data_proj_of_pure_function(n)) {
459 worklist.push(n);
460 }
461 }
462 }
463 if (n->outcnt() == 1 && n->has_special_unique_user()) {
464 assert(useful.member(n->unique_out()), "do not push a useless node");
465 worklist.push(n->unique_out());
466 }
467 if (n->outcnt() == 0) {
468 worklist.push(n);
469 }
470 }
471
472 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
473 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
474 // Remove useless Template Assertion Predicate opaque nodes
475 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
476 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
477 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
478 remove_useless_nodes(_inline_type_nodes, useful); // remove useless inline type nodes
479 remove_useless_nodes(_flat_access_nodes, useful); // remove useless flat access nodes
480 #ifdef ASSERT
481 if (_modified_nodes != nullptr) {
482 _modified_nodes->remove_useless_nodes(useful.member_set());
483 }
484 #endif
485 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
486 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
487 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
488 #ifdef ASSERT
489 if (_modified_nodes != nullptr) {
490 _modified_nodes->remove_useless_nodes(useful.member_set());
491 }
492 #endif
493
494 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
495 bs->eliminate_useless_gc_barriers(useful, this);
496 // clean up the late inline lists
497 remove_useless_late_inlines( &_late_inlines, useful);
498 remove_useless_late_inlines( &_string_late_inlines, useful);
499 remove_useless_late_inlines( &_boxing_late_inlines, useful);
500 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
501 DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
502 }
503
504 // ============================================================================
651
652
653 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
654 Options options, DirectiveSet* directive)
655 : Phase(Compiler),
656 _compile_id(ci_env->compile_id()),
657 _options(options),
658 _method(target),
659 _entry_bci(osr_bci),
660 _ilt(nullptr),
661 _stub_function(nullptr),
662 _stub_name(nullptr),
663 _stub_id(StubId::NO_STUBID),
664 _stub_entry_point(nullptr),
665 _max_node_limit(MaxNodeLimit),
666 _post_loop_opts_phase(false),
667 _merge_stores_phase(false),
668 _allow_macro_nodes(true),
669 _inlining_progress(false),
670 _inlining_incrementally(false),
671 _strength_reduction(false),
672 _do_cleanup(false),
673 _has_reserved_stack_access(target->has_reserved_stack_access()),
674 _has_circular_inline_type(false),
675 #ifndef PRODUCT
676 _igv_idx(0),
677 _trace_opto_output(directive->TraceOptoOutputOption),
678 #endif
679 _clinit_barrier_on_entry(false),
680 _stress_seed(0),
681 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
682 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
683 _env(ci_env),
684 _directive(directive),
685 _log(ci_env->log()),
686 _first_failure_details(nullptr),
687 _intrinsics(comp_arena(), 0, 0, nullptr),
688 _macro_nodes(comp_arena(), 8, 0, nullptr),
689 _parse_predicates(comp_arena(), 8, 0, nullptr),
690 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
691 _expensive_nodes(comp_arena(), 8, 0, nullptr),
692 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
693 _inline_type_nodes (comp_arena(), 8, 0, nullptr),
694 _flat_access_nodes(comp_arena(), 8, 0, nullptr),
695 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
696 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
697 _coarsened_locks(comp_arena(), 8, 0, nullptr),
698 _congraph(nullptr),
699 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
700 _unique(0),
701 _dead_node_count(0),
702 _dead_node_list(comp_arena()),
703 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
704 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
705 _node_arena(&_node_arena_one),
706 _mach_constant_base_node(nullptr),
707 _Compile_types(mtCompiler, Arena::Tag::tag_type),
708 _initial_gvn(nullptr),
709 _igvn_worklist(nullptr),
710 _types(nullptr),
711 _node_hash(nullptr),
712 _late_inlines(comp_arena(), 2, 0, nullptr),
713 _string_late_inlines(comp_arena(), 2, 0, nullptr),
714 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
783 #define MINIMUM_NODE_HASH 1023
784
785 // GVN that will be run immediately on new nodes
786 uint estimated_size = method()->code_size()*4+64;
787 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
788 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
789 _types = new (comp_arena()) Type_Array(comp_arena());
790 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
791 PhaseGVN gvn;
792 set_initial_gvn(&gvn);
793
794 { // Scope for timing the parser
795 TracePhase tp(_t_parser);
796
797 // Put top into the hash table ASAP.
798 initial_gvn()->transform(top());
799
800 // Set up tf(), start(), and find a CallGenerator.
801 CallGenerator* cg = nullptr;
802 if (is_osr_compilation()) {
803 init_tf(TypeFunc::make(method(), /* is_osr_compilation = */ true));
804 StartNode* s = new StartOSRNode(root(), tf()->domain_sig());
805 initial_gvn()->set_type_bottom(s);
806 verify_start(s);
807 cg = CallGenerator::for_osr(method(), entry_bci());
808 } else {
809 // Normal case.
810 init_tf(TypeFunc::make(method()));
811 StartNode* s = new StartNode(root(), tf()->domain_cc());
812 initial_gvn()->set_type_bottom(s);
813 verify_start(s);
814 float past_uses = method()->interpreter_invocation_count();
815 float expected_uses = past_uses;
816 cg = CallGenerator::for_inline(method(), expected_uses);
817 }
818 if (failing()) return;
819 if (cg == nullptr) {
820 const char* reason = InlineTree::check_can_parse(method());
821 assert(reason != nullptr, "expect reason for parse failure");
822 stringStream ss;
823 ss.print("cannot parse method: %s", reason);
824 record_method_not_compilable(ss.as_string());
825 return;
826 }
827
828 gvn.set_type(root(), root()->bottom_type());
829
830 JVMState* jvms = build_start_state(start(), tf());
831 if ((jvms = cg->generate(jvms)) == nullptr) {
892 print_ideal_ir("print_ideal");
893 }
894 #endif
895
896 #ifdef ASSERT
897 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
898 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
899 #endif
900
901 // Dump compilation data to replay it.
902 if (directive->DumpReplayOption) {
903 env()->dump_replay_data(_compile_id);
904 }
905 if (directive->DumpInlineOption && (ilt() != nullptr)) {
906 env()->dump_inline_data(_compile_id);
907 }
908
909 // Now that we know the size of all the monitors we can add a fixed slot
910 // for the original deopt pc.
911 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
912 if (needs_stack_repair()) {
913 // One extra slot for the special stack increment value
914 next_slot += 2;
915 }
916 // TODO 8284443 Only reserve extra slot if needed
917 if (InlineTypeReturnedAsFields) {
918 // One extra slot to hold the null marker for a nullable
919 // inline type return if we run out of registers.
920 next_slot += 2;
921 }
922 set_fixed_slots(next_slot);
923
924 // Compute when to use implicit null checks. Used by matching trap based
925 // nodes and NullCheck optimization.
926 set_allowed_deopt_reasons();
927
928 // Now generate code
929 Code_Gen();
930 }
931
932 //------------------------------Compile----------------------------------------
933 // Compile a runtime stub
934 Compile::Compile(ciEnv* ci_env,
935 TypeFunc_generator generator,
936 address stub_function,
937 const char* stub_name,
938 StubId stub_id,
939 int is_fancy_jump,
940 bool pass_tls,
941 bool return_pc,
942 DirectiveSet* directive)
943 : Phase(Compiler),
944 _compile_id(0),
945 _options(Options::for_runtime_stub()),
946 _method(nullptr),
947 _entry_bci(InvocationEntryBci),
948 _stub_function(stub_function),
949 _stub_name(stub_name),
950 _stub_id(stub_id),
951 _stub_entry_point(nullptr),
952 _max_node_limit(MaxNodeLimit),
953 _post_loop_opts_phase(false),
954 _merge_stores_phase(false),
955 _allow_macro_nodes(true),
956 _inlining_progress(false),
957 _inlining_incrementally(false),
958 _has_reserved_stack_access(false),
959 _has_circular_inline_type(false),
960 #ifndef PRODUCT
961 _igv_idx(0),
962 _trace_opto_output(directive->TraceOptoOutputOption),
963 #endif
964 _clinit_barrier_on_entry(false),
965 _stress_seed(0),
966 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
967 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
968 _env(ci_env),
969 _directive(directive),
970 _log(ci_env->log()),
971 _first_failure_details(nullptr),
972 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
973 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
974 _congraph(nullptr),
975 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
976 _unique(0),
977 _dead_node_count(0),
978 _dead_node_list(comp_arena()),
979 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
1094 _fixed_slots = 0;
1095 set_has_split_ifs(false);
1096 set_has_loops(false); // first approximation
1097 set_has_stringbuilder(false);
1098 set_has_boxed_value(false);
1099 _trap_can_recompile = false; // no traps emitted yet
1100 _major_progress = true; // start out assuming good things will happen
1101 set_has_unsafe_access(false);
1102 set_max_vector_size(0);
1103 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1104 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1105 set_decompile_count(0);
1106
1107 #ifndef PRODUCT
1108 _phase_counter = 0;
1109 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1110 #endif
1111
1112 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1113 _loop_opts_cnt = LoopOptsCount;
1114 _has_flat_accesses = false;
1115 _flat_accesses_share_alias = true;
1116 _scalarize_in_safepoints = false;
1117
1118 set_do_inlining(Inline);
1119 set_max_inline_size(MaxInlineSize);
1120 set_freq_inline_size(FreqInlineSize);
1121 set_do_scheduling(OptoScheduling);
1122
1123 set_do_vector_loop(false);
1124 set_has_monitors(false);
1125 set_has_scoped_access(false);
1126
1127 if (AllowVectorizeOnDemand) {
1128 if (has_method() && _directive->VectorizeOption) {
1129 set_do_vector_loop(true);
1130 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1131 } else if (has_method() && method()->name() != nullptr &&
1132 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1133 set_do_vector_loop(true);
1134 }
1135 }
1136 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1137 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1378 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1379 assert(do_aliasing(), "Aliasing should be enabled");
1380 int offset = tj->offset();
1381 TypePtr::PTR ptr = tj->ptr();
1382
1383 // Known instance (scalarizable allocation) alias only with itself.
1384 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1385 tj->is_oopptr()->is_known_instance();
1386
1387 // Process weird unsafe references.
1388 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1389 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1390 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1391 tj = TypeOopPtr::BOTTOM;
1392 ptr = tj->ptr();
1393 offset = tj->offset();
1394 }
1395
1396 // Array pointers need some flattening
1397 const TypeAryPtr* ta = tj->isa_aryptr();
1398 if( ta && is_known_inst ) {
1399 if ( offset != Type::OffsetBot &&
1400 offset > arrayOopDesc::length_offset_in_bytes() ) {
1401 offset = Type::OffsetBot; // Flatten constant access into array body only
1402 tj = ta = ta->
1403 remove_speculative()->
1404 cast_to_ptr_type(ptr)->
1405 with_offset(offset);
1406 }
1407 } else if (ta) {
1408 // Common slices
1409 if (offset == arrayOopDesc::length_offset_in_bytes()) {
1410 return TypeAryPtr::RANGE;
1411 } else if (offset == oopDesc::klass_offset_in_bytes()) {
1412 return TypeInstPtr::KLASS;
1413 } else if (offset == oopDesc::mark_offset_in_bytes()) {
1414 return TypeInstPtr::MARK;
1415 }
1416
1417 // Remove size and stability
1418 const TypeAry* normalized_ary = TypeAry::make(ta->elem(), TypeInt::POS, false, ta->is_flat(), ta->is_not_flat(), ta->is_not_null_free(), ta->is_atomic());
1419 // Remove ptr, const_oop, and offset
1420 if (ta->elem() == Type::BOTTOM) {
1421 // Bottom array (meet of int[] and byte[] for example), accesses to it will be done with
1422 // Unsafe. This should alias with all arrays. For now just leave it as it is (this is
1423 // incorrect, see JDK-8331133).
1424 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, false, Type::Offset::bottom);
1425 } else if (ta->elem()->make_oopptr() != nullptr) {
1426 // Object arrays, keep field_offset
1427 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, ta->klass_is_exact(), Type::Offset::bottom, Type::Offset(ta->field_offset()));
1428 } else {
1429 // Primitive arrays
1430 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, ta->exact_klass(), true, Type::Offset::bottom);
1431 }
1432
1433 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1434 // cannot be distinguished by bytecode alone.
1435 if (ta->elem() == TypeInt::BOOL) {
1436 tj = ta = TypeAryPtr::BYTES;
1437 }
1438
1439 // All arrays of references share the same slice
1440 if (!ta->is_flat() && ta->elem()->make_oopptr() != nullptr) {
1441 const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, TypeInt::POS, false, false, true, true, true);
1442 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, tary, nullptr, false, Type::Offset::bottom);
1443 }
1444
1445 if (ta->is_flat()) {
1446 if (_flat_accesses_share_alias) {
1447 // Initially all flattened array accesses share a single slice
1448 tj = ta = TypeAryPtr::INLINES;
1449 } else {
1450 // Flat accesses are always exact
1451 tj = ta = ta->cast_to_exactness(true);
1452 }
1453 }
1454 }
1455
1456 // Oop pointers need some flattening
1457 const TypeInstPtr *to = tj->isa_instptr();
1458 if (to && to != TypeOopPtr::BOTTOM) {
1459 ciInstanceKlass* ik = to->instance_klass();
1460 tj = to = to->cast_to_maybe_flat_in_array(); // flatten to maybe flat in array
1461 if( ptr == TypePtr::Constant ) {
1462 if (ik != ciEnv::current()->Class_klass() ||
1463 offset < ik->layout_helper_size_in_bytes()) {
1464 // No constant oop pointers (such as Strings); they alias with
1465 // unknown strings.
1466 assert(!is_known_inst, "not scalarizable allocation");
1467 tj = to = to->
1468 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1469 remove_speculative()->
1470 cast_to_ptr_type(TypePtr::BotPTR)->
1471 cast_to_exactness(false);
1472 }
1473 } else if( is_known_inst ) {
1474 tj = to; // Keep NotNull and klass_is_exact for instance type
1475 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1476 // During the 2nd round of IterGVN, NotNull castings are removed.
1477 // Make sure the Bottom and NotNull variants alias the same.
1478 // Also, make sure exact and non-exact variants alias the same.
1479 tj = to = to->
1480 remove_speculative()->
1481 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1482 cast_to_ptr_type(TypePtr::BotPTR)->
1483 cast_to_exactness(false);
1484 }
1485 if (to->speculative() != nullptr) {
1486 tj = to = to->remove_speculative();
1487 }
1488 // Canonicalize the holder of this field
1489 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1490 // First handle header references such as a LoadKlassNode, even if the
1491 // object's klass is unloaded at compile time (4965979).
1492 if (!is_known_inst) { // Do it only for non-instance types
1493 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, Type::Offset(offset));
1494 }
1495 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1496 // Static fields are in the space above the normal instance
1497 // fields in the java.lang.Class instance.
1498 if (ik != ciEnv::current()->Class_klass()) {
1499 to = nullptr;
1500 tj = TypeOopPtr::BOTTOM;
1501 offset = tj->offset();
1502 }
1503 } else {
1504 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1505 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1506 assert(tj->offset() == offset, "no change to offset expected");
1507 bool xk = to->klass_is_exact();
1508 int instance_id = to->instance_id();
1509
1510 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1511 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1512 // its interfaces are included.
1513 if (xk && ik->equals(canonical_holder)) {
1514 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id,
1515 TypePtr::MaybeFlat), "exact type should be canonical type");
1516 } else {
1517 assert(xk || !is_known_inst, "Known instance should be exact type");
1518 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id,
1519 TypePtr::MaybeFlat);
1520 }
1521 }
1522 }
1523
1524 // Klass pointers to object array klasses need some flattening
1525 const TypeKlassPtr *tk = tj->isa_klassptr();
1526 if( tk ) {
1527 // If we are referencing a field within a Klass, we need
1528 // to assume the worst case of an Object. Both exact and
1529 // inexact types must flatten to the same alias class so
1530 // use NotNull as the PTR.
1531 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1532 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1533 env()->Object_klass(),
1534 Type::Offset(offset),
1535 TypePtr::MaybeFlat);
1536 }
1537
1538 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1539 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1540 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1541 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), Type::Offset(offset), TypePtr::MaybeFlat);
1542 } else {
1543 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, Type::Offset(offset), tk->is_not_flat(), tk->is_not_null_free(), tk->is_flat(), tk->is_null_free(), tk->is_atomic(), tk->is_aryklassptr()->is_refined_type());
1544 }
1545 }
1546 // Check for precise loads from the primary supertype array and force them
1547 // to the supertype cache alias index. Check for generic array loads from
1548 // the primary supertype array and also force them to the supertype cache
1549 // alias index. Since the same load can reach both, we need to merge
1550 // these 2 disparate memories into the same alias class. Since the
1551 // primary supertype array is read-only, there's no chance of confusion
1552 // where we bypass an array load and an array store.
1553 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1554 if (offset == Type::OffsetBot ||
1555 (offset >= primary_supers_offset &&
1556 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1557 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1558 offset = in_bytes(Klass::secondary_super_cache_offset());
1559 tj = tk = tk->with_offset(offset);
1560 }
1561 }
1562
1563 // Flatten all Raw pointers together.
1564 if (tj->base() == Type::RawPtr)
1565 tj = TypeRawPtr::BOTTOM;
1655 intptr_t key = (intptr_t) adr_type;
1656 key ^= key >> logAliasCacheSize;
1657 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1658 }
1659
1660
1661 //-----------------------------grow_alias_types--------------------------------
1662 void Compile::grow_alias_types() {
1663 const int old_ats = _max_alias_types; // how many before?
1664 const int new_ats = old_ats; // how many more?
1665 const int grow_ats = old_ats+new_ats; // how many now?
1666 _max_alias_types = grow_ats;
1667 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1668 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1669 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1670 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1671 }
1672
1673
1674 //--------------------------------find_alias_type------------------------------
1675 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field, bool uncached) {
1676 if (!do_aliasing()) {
1677 return alias_type(AliasIdxBot);
1678 }
1679
1680 AliasCacheEntry* ace = nullptr;
1681 if (!uncached) {
1682 ace = probe_alias_cache(adr_type);
1683 if (ace->_adr_type == adr_type) {
1684 return alias_type(ace->_index);
1685 }
1686 }
1687
1688 // Handle special cases.
1689 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1690 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1691
1692 // Do it the slow way.
1693 const TypePtr* flat = flatten_alias_type(adr_type);
1694
1695 #ifdef ASSERT
1696 {
1697 ResourceMark rm;
1698 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1699 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1700 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1701 Type::str(adr_type));
1702 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1703 const TypeOopPtr* foop = flat->is_oopptr();
1704 // Scalarizable allocations have exact klass always.
1705 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1715 if (alias_type(i)->adr_type() == flat) {
1716 idx = i;
1717 break;
1718 }
1719 }
1720
1721 if (idx == AliasIdxTop) {
1722 if (no_create) return nullptr;
1723 // Grow the array if necessary.
1724 if (_num_alias_types == _max_alias_types) grow_alias_types();
1725 // Add a new alias type.
1726 idx = _num_alias_types++;
1727 _alias_types[idx]->Init(idx, flat);
1728 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1729 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1730 if (flat->isa_instptr()) {
1731 if (flat->offset() == java_lang_Class::klass_offset()
1732 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1733 alias_type(idx)->set_rewritable(false);
1734 }
1735 ciField* field = nullptr;
1736 if (flat->isa_aryptr()) {
1737 #ifdef ASSERT
1738 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1739 // (T_BYTE has the weakest alignment and size restrictions...)
1740 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1741 #endif
1742 const Type* elemtype = flat->is_aryptr()->elem();
1743 if (flat->offset() == TypePtr::OffsetBot) {
1744 alias_type(idx)->set_element(elemtype);
1745 }
1746 int field_offset = flat->is_aryptr()->field_offset().get();
1747 if (flat->is_flat() &&
1748 field_offset != Type::OffsetBot) {
1749 ciInlineKlass* vk = elemtype->inline_klass();
1750 field_offset += vk->payload_offset();
1751 field = vk->get_field_by_offset(field_offset, false);
1752 }
1753 }
1754 if (flat->isa_klassptr()) {
1755 if (UseCompactObjectHeaders) {
1756 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1757 alias_type(idx)->set_rewritable(false);
1758 }
1759 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1760 alias_type(idx)->set_rewritable(false);
1761 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1762 alias_type(idx)->set_rewritable(false);
1763 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1764 alias_type(idx)->set_rewritable(false);
1765 if (flat->offset() == in_bytes(Klass::layout_helper_offset()))
1766 alias_type(idx)->set_rewritable(false);
1767 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1768 alias_type(idx)->set_rewritable(false);
1769 }
1770
1771 if (flat->isa_instklassptr()) {
1772 if (flat->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
1773 alias_type(idx)->set_rewritable(false);
1774 }
1775 }
1776 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1777 // but the base pointer type is not distinctive enough to identify
1778 // references into JavaThread.)
1779
1780 // Check for final fields.
1781 const TypeInstPtr* tinst = flat->isa_instptr();
1782 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1783 if (tinst->const_oop() != nullptr &&
1784 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1785 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1786 // static field
1787 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1788 field = k->get_field_by_offset(tinst->offset(), true);
1789 } else if (tinst->is_inlinetypeptr()) {
1790 // Inline type field
1791 ciInlineKlass* vk = tinst->inline_klass();
1792 field = vk->get_field_by_offset(tinst->offset(), false);
1793 } else {
1794 ciInstanceKlass *k = tinst->instance_klass();
1795 field = k->get_field_by_offset(tinst->offset(), false);
1796 }
1797 }
1798 assert(field == nullptr ||
1799 original_field == nullptr ||
1800 (field->holder() == original_field->holder() &&
1801 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1802 field->is_static() == original_field->is_static()), "wrong field?");
1803 // Set field() and is_rewritable() attributes.
1804 if (field != nullptr) {
1805 alias_type(idx)->set_field(field);
1806 if (flat->isa_aryptr()) {
1807 // Fields of flat arrays are rewritable although they are declared final
1808 assert(flat->is_flat(), "must be a flat array");
1809 alias_type(idx)->set_rewritable(true);
1810 }
1811 }
1812 }
1813
1814 // Fill the cache for next time.
1815 if (!uncached) {
1816 ace->_adr_type = adr_type;
1817 ace->_index = idx;
1818 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1819
1820 // Might as well try to fill the cache for the flattened version, too.
1821 AliasCacheEntry* face = probe_alias_cache(flat);
1822 if (face->_adr_type == nullptr) {
1823 face->_adr_type = flat;
1824 face->_index = idx;
1825 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1826 }
1827 }
1828
1829 return alias_type(idx);
1830 }
1831
1832
1833 Compile::AliasType* Compile::alias_type(ciField* field) {
1834 const TypeOopPtr* t;
1835 if (field->is_static())
1836 t = TypeInstPtr::make(field->holder()->java_mirror());
1837 else
1838 t = TypeOopPtr::make_from_klass_raw(field->holder());
1839 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1840 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1841 return atp;
1842 }
1843
1844
1845 //------------------------------have_alias_type--------------------------------
1846 bool Compile::have_alias_type(const TypePtr* adr_type) {
1928 assert(!C->major_progress(), "not cleared");
1929
1930 if (_for_post_loop_igvn.length() > 0) {
1931 while (_for_post_loop_igvn.length() > 0) {
1932 Node* n = _for_post_loop_igvn.pop();
1933 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1934 igvn._worklist.push(n);
1935 }
1936 igvn.optimize();
1937 if (failing()) return;
1938 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1939 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1940
1941 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1942 if (C->major_progress()) {
1943 C->clear_major_progress(); // ensure that major progress is now clear
1944 }
1945 }
1946 }
1947
1948 void Compile::add_inline_type(Node* n) {
1949 assert(n->is_InlineType(), "unexpected node");
1950 _inline_type_nodes.push(n);
1951 }
1952
1953 void Compile::remove_inline_type(Node* n) {
1954 assert(n->is_InlineType(), "unexpected node");
1955 if (_inline_type_nodes.contains(n)) {
1956 _inline_type_nodes.remove(n);
1957 }
1958 }
1959
1960 // Does the return value keep otherwise useless inline type allocations alive?
1961 static bool return_val_keeps_allocations_alive(Node* ret_val) {
1962 ResourceMark rm;
1963 Unique_Node_List wq;
1964 wq.push(ret_val);
1965 bool some_allocations = false;
1966 for (uint i = 0; i < wq.size(); i++) {
1967 Node* n = wq.at(i);
1968 if (n->outcnt() > 1) {
1969 // Some other use for the allocation
1970 return false;
1971 } else if (n->is_InlineType()) {
1972 wq.push(n->in(1));
1973 } else if (n->is_Phi()) {
1974 for (uint j = 1; j < n->req(); j++) {
1975 wq.push(n->in(j));
1976 }
1977 } else if (n->is_CheckCastPP() &&
1978 n->in(1)->is_Proj() &&
1979 n->in(1)->in(0)->is_Allocate()) {
1980 some_allocations = true;
1981 } else if (n->is_CheckCastPP()) {
1982 wq.push(n->in(1));
1983 }
1984 }
1985 return some_allocations;
1986 }
1987
1988 void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
1989 // Make sure that the return value does not keep an otherwise unused allocation alive
1990 if (tf()->returns_inline_type_as_fields()) {
1991 Node* ret = nullptr;
1992 for (uint i = 1; i < root()->req(); i++) {
1993 Node* in = root()->in(i);
1994 if (in->Opcode() == Op_Return) {
1995 assert(ret == nullptr, "only one return");
1996 ret = in;
1997 }
1998 }
1999 if (ret != nullptr) {
2000 Node* ret_val = ret->in(TypeFunc::Parms);
2001 if (igvn.type(ret_val)->isa_oopptr() &&
2002 return_val_keeps_allocations_alive(ret_val)) {
2003 igvn.replace_input_of(ret, TypeFunc::Parms, InlineTypeNode::tagged_klass(igvn.type(ret_val)->inline_klass(), igvn));
2004 assert(ret_val->outcnt() == 0, "should be dead now");
2005 igvn.remove_dead_node(ret_val);
2006 }
2007 }
2008 }
2009 if (_inline_type_nodes.length() == 0) {
2010 // keep the graph canonical
2011 igvn.optimize();
2012 return;
2013 }
2014 // Scalarize inline types in safepoint debug info.
2015 // Delay this until all inlining is over to avoid getting inconsistent debug info.
2016 set_scalarize_in_safepoints(true);
2017 for (int i = _inline_type_nodes.length()-1; i >= 0; i--) {
2018 InlineTypeNode* vt = _inline_type_nodes.at(i)->as_InlineType();
2019 vt->make_scalar_in_safepoints(&igvn);
2020 igvn.record_for_igvn(vt);
2021 }
2022 if (remove) {
2023 // Remove inline type nodes by replacing them with their oop input
2024 while (_inline_type_nodes.length() > 0) {
2025 InlineTypeNode* vt = _inline_type_nodes.pop()->as_InlineType();
2026 if (vt->outcnt() == 0) {
2027 igvn.remove_dead_node(vt);
2028 continue;
2029 }
2030 for (DUIterator i = vt->outs(); vt->has_out(i); i++) {
2031 DEBUG_ONLY(bool must_be_buffered = false);
2032 Node* u = vt->out(i);
2033 // Check if any users are blackholes. If so, rewrite them to use either the
2034 // allocated buffer, or individual components, instead of the inline type node
2035 // that goes away.
2036 if (u->is_Blackhole()) {
2037 BlackholeNode* bh = u->as_Blackhole();
2038
2039 // Unlink the old input
2040 int idx = bh->find_edge(vt);
2041 assert(idx != -1, "The edge should be there");
2042 bh->del_req(idx);
2043 --i;
2044
2045 if (vt->is_allocated(&igvn)) {
2046 // Already has the allocated instance, blackhole that
2047 bh->add_req(vt->get_oop());
2048 } else {
2049 // Not allocated yet, blackhole the components
2050 for (uint c = 0; c < vt->field_count(); c++) {
2051 bh->add_req(vt->field_value(c));
2052 }
2053 }
2054
2055 // Node modified, record for IGVN
2056 igvn.record_for_igvn(bh);
2057 }
2058 #ifdef ASSERT
2059 // Verify that inline type is buffered when replacing by oop
2060 else if (u->is_InlineType()) {
2061 // InlineType uses don't need buffering because they are about to be replaced as well
2062 } else if (u->is_Phi()) {
2063 // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through
2064 } else {
2065 must_be_buffered = true;
2066 }
2067 if (must_be_buffered && !vt->is_allocated(&igvn)) {
2068 vt->dump(0);
2069 u->dump(0);
2070 assert(false, "Should have been buffered");
2071 }
2072 #endif
2073 }
2074 igvn.replace_node(vt, vt->get_oop());
2075 }
2076 }
2077 igvn.optimize();
2078 }
2079
2080 void Compile::add_flat_access(Node* n) {
2081 assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2082 assert(!_flat_access_nodes.contains(n), "duplicate insertion");
2083 _flat_access_nodes.push(n);
2084 }
2085
2086 void Compile::remove_flat_access(Node* n) {
2087 assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2088 _flat_access_nodes.remove_if_existing(n);
2089 }
2090
2091 void Compile::process_flat_accesses(PhaseIterGVN& igvn) {
2092 assert(igvn._worklist.size() == 0, "should be empty");
2093 igvn.set_delay_transform(true);
2094 for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) {
2095 Node* n = _flat_access_nodes.at(i);
2096 assert(n != nullptr, "unexpected nullptr");
2097 if (n->is_LoadFlat()) {
2098 LoadFlatNode* loadn = n->as_LoadFlat();
2099 // Expending a flat load atomically means that we get a chunk of memory spanning multiple fields
2100 // that we chop with bitwise operations. That is too subtle for some optimizations, especially
2101 // constant folding when fields are constant. If we can get a constant object from which we are
2102 // flat-loading, we can simply replace the loads at compilation-time by the field of the constant
2103 // object.
2104 ciInstance* loaded_from = nullptr;
2105 if (FoldStableValues) {
2106 const TypeOopPtr* base_type = igvn.type(loadn->base())->is_oopptr();
2107 ciObject* oop = base_type->const_oop();
2108 int off = igvn.type(loadn->ptr())->isa_ptr()->offset();
2109
2110 if (oop != nullptr && oop->is_instance()) {
2111 ciInstance* holder = oop->as_instance();
2112 ciKlass* klass = holder->klass();
2113 ciInstanceKlass* iklass = klass->as_instance_klass();
2114 ciField* field = iklass->get_non_flat_field_by_offset(off);
2115
2116 if (field->is_stable()) {
2117 ciConstant fv = holder->field_value(field);
2118 if (is_reference_type(fv.basic_type()) && fv.as_object()->is_instance()) {
2119 // The field value is an object, not null. We can use stability.
2120 loaded_from = fv.as_object()->as_instance();
2121 }
2122 }
2123 } else if (oop != nullptr && oop->is_array() && off != Type::OffsetBot) {
2124 ciArray* array = oop->as_array();
2125 ciConstant elt = array->element_value_by_offset(off);
2126 const TypeAryPtr* aryptr = base_type->is_aryptr();
2127 if (aryptr->is_stable() && aryptr->is_atomic() && is_reference_type(elt.basic_type()) && elt.as_object()->is_instance()) {
2128 loaded_from = elt.as_object()->as_instance();
2129 }
2130 }
2131 }
2132
2133 if (loaded_from != nullptr) {
2134 loadn->expand_constant(igvn, loaded_from);
2135 } else {
2136 loadn->expand_atomic(igvn);
2137 }
2138 } else {
2139 n->as_StoreFlat()->expand_atomic(igvn);
2140 }
2141 }
2142 _flat_access_nodes.clear_and_deallocate();
2143 igvn.set_delay_transform(false);
2144 igvn.optimize();
2145 }
2146
2147 void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) {
2148 DEBUG_ONLY(igvn.verify_empty_worklist(nullptr));
2149 if (!_has_flat_accesses) {
2150 return;
2151 }
2152 // Initially, all flat array accesses share the same slice to
2153 // keep dependencies with Object[] array accesses (that could be
2154 // to a flat array) correct. We're done with parsing so we
2155 // now know all flat array accesses in this compile
2156 // unit. Let's move flat array accesses to their own slice,
2157 // one per element field. This should help memory access
2158 // optimizations.
2159 ResourceMark rm;
2160 Unique_Node_List wq;
2161 wq.push(root());
2162
2163 Node_List mergememnodes;
2164 Node_List memnodes;
2165
2166 // Alias index currently shared by all flat memory accesses
2167 int index = get_alias_index(TypeAryPtr::INLINES);
2168
2169 // Find MergeMem nodes and flat array accesses
2170 for (uint i = 0; i < wq.size(); i++) {
2171 Node* n = wq.at(i);
2172 if (n->is_Mem()) {
2173 const TypePtr* adr_type = nullptr;
2174 adr_type = get_adr_type(get_alias_index(n->adr_type()));
2175 if (adr_type == TypeAryPtr::INLINES) {
2176 memnodes.push(n);
2177 }
2178 } else if (n->is_MergeMem()) {
2179 MergeMemNode* mm = n->as_MergeMem();
2180 if (mm->memory_at(index) != mm->base_memory()) {
2181 mergememnodes.push(n);
2182 }
2183 }
2184 for (uint j = 0; j < n->req(); j++) {
2185 Node* m = n->in(j);
2186 if (m != nullptr) {
2187 wq.push(m);
2188 }
2189 }
2190 }
2191
2192 _flat_accesses_share_alias = false;
2193
2194 // We are going to change the slice for the flat array
2195 // accesses so we need to clear the cache entries that refer to
2196 // them.
2197 for (uint i = 0; i < AliasCacheSize; i++) {
2198 AliasCacheEntry* ace = &_alias_cache[i];
2199 if (ace->_adr_type != nullptr &&
2200 ace->_adr_type->is_flat()) {
2201 ace->_adr_type = nullptr;
2202 ace->_index = (i != 0) ? 0 : AliasIdxTop; // Make sure the nullptr adr_type resolves to AliasIdxTop
2203 }
2204 }
2205
2206 #ifdef ASSERT
2207 for (uint i = 0; i < memnodes.size(); i++) {
2208 Node* m = memnodes.at(i);
2209 const TypePtr* adr_type = m->adr_type();
2210 m->as_Mem()->set_adr_type(adr_type);
2211 }
2212 #endif // ASSERT
2213
2214 int start_alias = num_alias_types(); // Start of new aliases
2215 Node_Stack stack(0);
2216 #ifdef ASSERT
2217 VectorSet seen(Thread::current()->resource_area());
2218 #endif
2219 // Now let's fix the memory graph so each flat array access
2220 // is moved to the right slice. Start from the MergeMem nodes.
2221 uint last = unique();
2222 for (uint i = 0; i < mergememnodes.size(); i++) {
2223 MergeMemNode* current = mergememnodes.at(i)->as_MergeMem();
2224 if (current->outcnt() == 0) {
2225 // This node is killed by a previous iteration
2226 continue;
2227 }
2228
2229 Node* n = current->memory_at(index);
2230 MergeMemNode* mm = nullptr;
2231 do {
2232 // Follow memory edges through memory accesses, phis and
2233 // narrow membars and push nodes on the stack. Once we hit
2234 // bottom memory, we pop element off the stack one at a
2235 // time, in reverse order, and move them to the right slice
2236 // by changing their memory edges.
2237 if ((n->is_Phi() && n->adr_type() != TypePtr::BOTTOM) || n->is_Mem() ||
2238 (n->adr_type() == TypeAryPtr::INLINES && !n->is_NarrowMemProj())) {
2239 assert(!seen.test_set(n->_idx), "");
2240 // Uses (a load for instance) will need to be moved to the
2241 // right slice as well and will get a new memory state
2242 // that we don't know yet. The use could also be the
2243 // backedge of a loop. We put a place holder node between
2244 // the memory node and its uses. We replace that place
2245 // holder with the correct memory state once we know it,
2246 // i.e. when nodes are popped off the stack. Using the
2247 // place holder make the logic work in the presence of
2248 // loops.
2249 if (n->outcnt() > 1) {
2250 Node* place_holder = nullptr;
2251 assert(!n->has_out_with(Op_Node), "");
2252 for (DUIterator k = n->outs(); n->has_out(k); k++) {
2253 Node* u = n->out(k);
2254 if (u != current && u->_idx < last) {
2255 bool success = false;
2256 for (uint l = 0; l < u->req(); l++) {
2257 if (!stack.is_empty() && u == stack.node() && l == stack.index()) {
2258 continue;
2259 }
2260 Node* in = u->in(l);
2261 if (in == n) {
2262 if (place_holder == nullptr) {
2263 place_holder = new Node(1);
2264 place_holder->init_req(0, n);
2265 }
2266 igvn.replace_input_of(u, l, place_holder);
2267 success = true;
2268 }
2269 }
2270 if (success) {
2271 --k;
2272 }
2273 }
2274 }
2275 }
2276 if (n->is_Phi()) {
2277 stack.push(n, 1);
2278 n = n->in(1);
2279 } else if (n->is_Mem()) {
2280 stack.push(n, n->req());
2281 n = n->in(MemNode::Memory);
2282 } else {
2283 assert(n->is_Proj() && n->in(0)->Opcode() == Op_MemBarCPUOrder, "");
2284 stack.push(n, n->req());
2285 n = n->in(0)->in(TypeFunc::Memory);
2286 }
2287 } else {
2288 assert(n->adr_type() == TypePtr::BOTTOM || (n->Opcode() == Op_Node && n->_idx >= last) || n->is_NarrowMemProj(), "");
2289 // Build a new MergeMem node to carry the new memory state
2290 // as we build it. IGVN should fold extraneous MergeMem
2291 // nodes.
2292 if (n->is_NarrowMemProj()) {
2293 // We need 1 NarrowMemProj for each slice of this array
2294 InitializeNode* init = n->in(0)->as_Initialize();
2295 AllocateNode* alloc = init->allocation();
2296 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2297 const TypeAryKlassPtr* klass_type = klass_node->bottom_type()->isa_aryklassptr();
2298 assert(klass_type != nullptr, "must be an array");
2299 assert(klass_type->klass_is_exact(), "must be an exact klass");
2300 ciArrayKlass* klass = klass_type->exact_klass()->as_array_klass();
2301 assert(klass->is_flat_array_klass(), "must be a flat array");
2302 ciInlineKlass* elem_klass = klass->element_klass()->as_inline_klass();
2303 const TypeAryPtr* oop_type = klass_type->as_instance_type()->is_aryptr();
2304 assert(oop_type->klass_is_exact(), "must be an exact klass");
2305
2306 Node* base = alloc->in(TypeFunc::Memory);
2307 assert(base->bottom_type() == Type::MEMORY, "the memory input of AllocateNode must be a memory");
2308 assert(base->adr_type() == TypePtr::BOTTOM, "the memory input of AllocateNode must be a bottom memory");
2309 // Must create a MergeMem with base as the base memory, do not clone if base is a
2310 // MergeMem because it may not be processed yet
2311 mm = MergeMemNode::make(nullptr);
2312 mm->set_base_memory(base);
2313 for (int j = 0; j < elem_klass->nof_nonstatic_fields(); j++) {
2314 int field_offset = elem_klass->nonstatic_field_at(j)->offset_in_bytes() - elem_klass->payload_offset();
2315 const TypeAryPtr* field_ptr = oop_type->with_offset(Type::OffsetBot)->with_field_offset(field_offset);
2316 int field_alias_idx = get_alias_index(field_ptr);
2317 assert(field_ptr == get_adr_type(field_alias_idx), "must match");
2318 Node* new_proj = new NarrowMemProjNode(init, field_ptr);
2319 igvn.register_new_node_with_optimizer(new_proj);
2320 mm->set_memory_at(field_alias_idx, new_proj);
2321 }
2322 if (!klass->is_elem_null_free()) {
2323 int nm_offset = elem_klass->null_marker_offset_in_payload();
2324 const TypeAryPtr* nm_ptr = oop_type->with_offset(Type::OffsetBot)->with_field_offset(nm_offset);
2325 int nm_alias_idx = get_alias_index(nm_ptr);
2326 assert(nm_ptr == get_adr_type(nm_alias_idx), "must match");
2327 Node* new_proj = new NarrowMemProjNode(init, nm_ptr);
2328 igvn.register_new_node_with_optimizer(new_proj);
2329 mm->set_memory_at(nm_alias_idx, new_proj);
2330 }
2331
2332 // Replace all uses of the old NarrowMemProj with the correct state
2333 MergeMemNode* new_n = MergeMemNode::make(mm);
2334 igvn.register_new_node_with_optimizer(new_n);
2335 igvn.replace_node(n, new_n);
2336 } else {
2337 // Must create a MergeMem with n as the base memory, do not clone if n is a MergeMem
2338 // because it may not be processed yet
2339 mm = MergeMemNode::make(nullptr);
2340 mm->set_base_memory(n);
2341 }
2342
2343 igvn.register_new_node_with_optimizer(mm);
2344 while (stack.size() > 0) {
2345 Node* m = stack.node();
2346 uint idx = stack.index();
2347 if (m->is_Mem()) {
2348 // Move memory node to its new slice
2349 const TypePtr* adr_type = m->adr_type();
2350 int alias = get_alias_index(adr_type);
2351 Node* prev = mm->memory_at(alias);
2352 igvn.replace_input_of(m, MemNode::Memory, prev);
2353 mm->set_memory_at(alias, m);
2354 } else if (m->is_Phi()) {
2355 // We need as many new phis as there are new aliases
2356 Node* new_phi_in = MergeMemNode::make(mm);
2357 igvn.register_new_node_with_optimizer(new_phi_in);
2358 igvn.replace_input_of(m, idx, new_phi_in);
2359 if (idx == m->req()-1) {
2360 Node* r = m->in(0);
2361 for (int j = start_alias; j < num_alias_types(); j++) {
2362 const TypePtr* adr_type = get_adr_type(j);
2363 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2364 continue;
2365 }
2366 Node* phi = new PhiNode(r, Type::MEMORY, get_adr_type(j));
2367 igvn.register_new_node_with_optimizer(phi);
2368 for (uint k = 1; k < m->req(); k++) {
2369 phi->init_req(k, m->in(k)->as_MergeMem()->memory_at(j));
2370 }
2371 mm->set_memory_at(j, phi);
2372 }
2373 Node* base_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2374 igvn.register_new_node_with_optimizer(base_phi);
2375 for (uint k = 1; k < m->req(); k++) {
2376 base_phi->init_req(k, m->in(k)->as_MergeMem()->base_memory());
2377 }
2378 mm->set_base_memory(base_phi);
2379 }
2380 } else {
2381 // This is a MemBarCPUOrder node from
2382 // Parse::array_load()/Parse::array_store(), in the
2383 // branch that handles flat arrays hidden under
2384 // an Object[] array. We also need one new membar per
2385 // new alias to keep the unknown access that the
2386 // membars protect properly ordered with accesses to
2387 // known flat array.
2388 assert(m->is_Proj(), "projection expected");
2389 Node* ctrl = m->in(0)->in(TypeFunc::Control);
2390 igvn.replace_input_of(m->in(0), TypeFunc::Control, top());
2391 for (int j = start_alias; j < num_alias_types(); j++) {
2392 const TypePtr* adr_type = get_adr_type(j);
2393 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2394 continue;
2395 }
2396 MemBarNode* mb = new MemBarCPUOrderNode(this, j, nullptr);
2397 igvn.register_new_node_with_optimizer(mb);
2398 Node* mem = mm->memory_at(j);
2399 mb->init_req(TypeFunc::Control, ctrl);
2400 mb->init_req(TypeFunc::Memory, mem);
2401 ctrl = new ProjNode(mb, TypeFunc::Control);
2402 igvn.register_new_node_with_optimizer(ctrl);
2403 mem = new ProjNode(mb, TypeFunc::Memory);
2404 igvn.register_new_node_with_optimizer(mem);
2405 mm->set_memory_at(j, mem);
2406 }
2407 igvn.replace_node(m->in(0)->as_Multi()->proj_out(TypeFunc::Control), ctrl);
2408 }
2409 if (idx < m->req()-1) {
2410 idx += 1;
2411 stack.set_index(idx);
2412 n = m->in(idx);
2413 break;
2414 }
2415 // Take care of place holder nodes
2416 if (m->has_out_with(Op_Node)) {
2417 Node* place_holder = m->find_out_with(Op_Node);
2418 if (place_holder != nullptr) {
2419 Node* mm_clone = mm->clone();
2420 igvn.register_new_node_with_optimizer(mm_clone);
2421 Node* hook = new Node(1);
2422 hook->init_req(0, mm);
2423 igvn.replace_node(place_holder, mm_clone);
2424 hook->destruct(&igvn);
2425 }
2426 assert(!m->has_out_with(Op_Node), "place holder should be gone now");
2427 }
2428 stack.pop();
2429 }
2430 }
2431 } while(stack.size() > 0);
2432 // Fix the memory state at the MergeMem we started from
2433 igvn.rehash_node_delayed(current);
2434 for (int j = start_alias; j < num_alias_types(); j++) {
2435 const TypePtr* adr_type = get_adr_type(j);
2436 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2437 continue;
2438 }
2439 current->set_memory_at(j, mm);
2440 }
2441 current->set_memory_at(index, current->base_memory());
2442 }
2443 igvn.optimize();
2444
2445 #ifdef ASSERT
2446 wq.clear();
2447 wq.push(root());
2448 for (uint i = 0; i < wq.size(); i++) {
2449 Node* n = wq.at(i);
2450 assert(n->adr_type() != TypeAryPtr::INLINES, "should have been removed from the graph");
2451 for (uint j = 0; j < n->req(); j++) {
2452 Node* m = n->in(j);
2453 if (m != nullptr) {
2454 wq.push(m);
2455 }
2456 }
2457 }
2458 #endif
2459
2460 print_method(PHASE_SPLIT_INLINES_ARRAY, 2);
2461 }
2462
2463 void Compile::record_for_merge_stores_igvn(Node* n) {
2464 if (!n->for_merge_stores_igvn()) {
2465 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
2466 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2467 _for_merge_stores_igvn.append(n);
2468 }
2469 }
2470
2471 void Compile::remove_from_merge_stores_igvn(Node* n) {
2472 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2473 _for_merge_stores_igvn.remove(n);
2474 }
2475
2476 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
2477 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
2478 // the stores, and we merge the wrong sequence of stores.
2479 // Example:
2480 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
2481 // Apply MergeStores:
2482 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
2561 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2562 Bytecodes::Code c = iter.cur_bc();
2563 Node* lhs = nullptr;
2564 Node* rhs = nullptr;
2565 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2566 lhs = unc->peek_operand(0);
2567 rhs = unc->peek_operand(1);
2568 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2569 lhs = unc->peek_operand(0);
2570 }
2571
2572 ResourceMark rm;
2573 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2574 assert(live_locals.is_valid(), "broken liveness info");
2575 int len = (int)live_locals.size();
2576
2577 for (int i = 0; i < len; i++) {
2578 Node* local = unc->local(jvms, i);
2579 // kill local using the liveness of next_bci.
2580 // give up when the local looks like an operand to secure reexecution.
2581 if (!live_locals.at(i) && !local->is_top() && local != lhs && local != rhs) {
2582 uint idx = jvms->locoff() + i;
2583 #ifdef ASSERT
2584 if (PrintOpto && Verbose) {
2585 tty->print("[unstable_if] kill local#%d: ", idx);
2586 local->dump();
2587 tty->cr();
2588 }
2589 #endif
2590 igvn.replace_input_of(unc, idx, top());
2591 modified = true;
2592 }
2593 }
2594 }
2595
2596 // keep the modified trap for late query
2597 if (modified) {
2598 trap->set_modified();
2599 } else {
2600 _unstable_if_traps.delete_at(i);
2601 }
2602 }
2603 igvn.optimize();
2604 }
2605
2606 // StringOpts and late inlining of string methods
2607 void Compile::inline_string_calls(bool parse_time) {
2608 {
2609 // remove useless nodes to make the usage analysis simpler
2610 ResourceMark rm;
2611 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2612 }
2613
2614 {
2615 ResourceMark rm;
2616 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2788
2789 if (_string_late_inlines.length() > 0) {
2790 assert(has_stringbuilder(), "inconsistent");
2791
2792 inline_string_calls(false);
2793
2794 if (failing()) return;
2795
2796 inline_incrementally_cleanup(igvn);
2797 }
2798
2799 set_inlining_incrementally(false);
2800 }
2801
2802 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2803 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2804 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2805 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2806 // as if "inlining_incrementally() == true" were set.
2807 assert(inlining_incrementally() == false, "not allowed");
2808 set_strength_reduction(true);
2809 #ifdef ASSERT
2810 Unique_Node_List* modified_nodes = _modified_nodes;
2811 _modified_nodes = nullptr;
2812 #endif
2813 assert(_late_inlines.length() > 0, "sanity");
2814
2815 while (_late_inlines.length() > 0) {
2816 igvn_worklist()->ensure_empty(); // should be done with igvn
2817
2818 while (inline_incrementally_one()) {
2819 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2820 }
2821 if (failing()) return;
2822
2823 inline_incrementally_cleanup(igvn);
2824 }
2825 DEBUG_ONLY( _modified_nodes = modified_nodes; )
2826 set_strength_reduction(false);
2827 }
2828
2829 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2830 if (_loop_opts_cnt > 0) {
2831 while (major_progress() && (_loop_opts_cnt > 0)) {
2832 TracePhase tp(_t_idealLoop);
2833 PhaseIdealLoop::optimize(igvn, mode);
2834 _loop_opts_cnt--;
2835 if (failing()) return false;
2836 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2837 }
2838 }
2839 return true;
2840 }
2841
2842 // Remove edges from "root" to each SafePoint at a backward branch.
2843 // They were inserted during parsing (see add_safepoint()) to make
2844 // infinite loops without calls or exceptions visible to root, i.e.,
2845 // useful.
2846 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2950 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2951 }
2952 assert(!has_vbox_nodes(), "sanity");
2953
2954 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2955 Compile::TracePhase tp(_t_renumberLive);
2956 igvn_worklist()->ensure_empty(); // should be done with igvn
2957 {
2958 ResourceMark rm;
2959 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2960 }
2961 igvn.reset();
2962 igvn.optimize();
2963 if (failing()) return;
2964 }
2965
2966 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2967 // safepoints
2968 remove_root_to_sfpts_edges(igvn);
2969
2970 // Process inline type nodes now that all inlining is over
2971 process_inline_types(igvn);
2972
2973 adjust_flat_array_access_aliases(igvn);
2974
2975 if (failing()) return;
2976
2977 if (C->macro_count() > 0) {
2978 // Eliminate some macro nodes before EA to reduce analysis pressure
2979 PhaseMacroExpand mexp(igvn);
2980 mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
2981 if (failing()) {
2982 return;
2983 }
2984 igvn.set_delay_transform(false);
2985 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2986 }
2987
2988 if (has_loops()) {
2989 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2990 }
2991
2992 // Perform escape analysis
2993 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2994 if (has_loops()) {
2995 // Cleanup graph (remove dead nodes).
2996 TracePhase tp(_t_idealLoop);
2997 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2998 if (failing()) {
2999 return;
3000 }
3001 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
3002 if (C->macro_count() > 0) {
3003 // Eliminate some macro nodes before EA to reduce analysis pressure
3004 PhaseMacroExpand mexp(igvn);
3005 mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
3006 if (failing()) {
3007 return;
3008 }
3009 igvn.set_delay_transform(false);
3010 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
3011 }
3012 }
3013
3014 bool progress;
3015 do {
3016 ConnectionGraph::do_analysis(this, &igvn);
3017
3018 if (failing()) return;
3019
3020 int mcount = macro_count(); // Record number of allocations and locks before IGVN
3021
3022 // Optimize out fields loads from scalar replaceable allocations.
3023 igvn.optimize();
3024 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
3025
3026 if (failing()) return;
3027
3028 if (congraph() != nullptr && macro_count() > 0) {
3029 TracePhase tp(_t_macroEliminate);
3030 PhaseMacroExpand mexp(igvn);
3031 mexp.eliminate_macro_nodes();
3032 if (failing()) {
3033 return;
3034 }
3035 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
3036
3037 igvn.set_delay_transform(false);
3038 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
3039 }
3040
3041 ConnectionGraph::verify_ram_nodes(this, root());
3042 if (failing()) return;
3043
3044 progress = do_iterative_escape_analysis() &&
3045 (macro_count() < mcount) &&
3046 ConnectionGraph::has_candidates(this);
3047 // Try again if candidates exist and made progress
3048 // by removing some allocations and/or locks.
3049 } while (progress);
3050 }
3051
3052 process_flat_accesses(igvn);
3053 if (failing()) {
3054 return;
3055 }
3056
3057 // Loop transforms on the ideal graph. Range Check Elimination,
3058 // peeling, unrolling, etc.
3059
3060 // Set loop opts counter
3061 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
3062 {
3063 TracePhase tp(_t_idealLoop);
3064 PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
3065 _loop_opts_cnt--;
3066 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
3067 if (failing()) return;
3068 }
3069 // Loop opts pass if partial peeling occurred in previous pass
3070 if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
3071 TracePhase tp(_t_idealLoop);
3072 PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
3073 _loop_opts_cnt--;
3074 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
3075 if (failing()) return;
3076 }
3113 // Loop transforms on the ideal graph. Range Check Elimination,
3114 // peeling, unrolling, etc.
3115 if (!optimize_loops(igvn, LoopOptsDefault)) {
3116 return;
3117 }
3118
3119 if (failing()) return;
3120
3121 C->clear_major_progress(); // ensure that major progress is now clear
3122
3123 process_for_post_loop_opts_igvn(igvn);
3124
3125 process_for_merge_stores_igvn(igvn);
3126
3127 if (failing()) return;
3128
3129 #ifdef ASSERT
3130 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
3131 #endif
3132
3133 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
3134
3135 if (_late_inlines.length() > 0) {
3136 // More opportunities to optimize virtual and MH calls.
3137 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
3138 process_late_inline_calls_no_inline(igvn);
3139 }
3140
3141 {
3142 TracePhase tp(_t_macroExpand);
3143 PhaseMacroExpand mex(igvn);
3144 // Last attempt to eliminate macro nodes.
3145 mex.eliminate_macro_nodes();
3146 if (failing()) {
3147 return;
3148 }
3149
3150 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
3151 // Do not allow new macro nodes once we start to eliminate and expand
3152 C->reset_allow_macro_nodes();
3153 // Last attempt to eliminate macro nodes before expand
3154 mex.eliminate_macro_nodes();
3155 if (failing()) {
3156 return;
3157 }
3158 mex.eliminate_opaque_looplimit_macro_nodes();
3159 if (failing()) {
3160 return;
3161 }
3162 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
3163 if (mex.expand_macro_nodes()) {
3164 assert(failing(), "must bail out w/ explicit message");
3165 return;
3166 }
3167 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
3168 }
3169
3170 // Process inline type nodes again and remove them. From here
3171 // on we don't need to keep track of field values anymore.
3172 process_inline_types(igvn, /* remove= */ true);
3173
3174 {
3175 TracePhase tp(_t_barrierExpand);
3176 if (bs->expand_barriers(this, igvn)) {
3177 assert(failing(), "must bail out w/ explicit message");
3178 return;
3179 }
3180 print_method(PHASE_BARRIER_EXPANSION, 2);
3181 }
3182
3183 if (C->max_vector_size() > 0) {
3184 C->optimize_logic_cones(igvn);
3185 igvn.optimize();
3186 if (failing()) return;
3187 }
3188
3189 DEBUG_ONLY( _modified_nodes = nullptr; )
3190 DEBUG_ONLY( _late_inlines.clear(); )
3191
3192 assert(igvn._worklist.size() == 0, "not empty");
3193 } // (End scope of igvn; run destructor if necessary for asserts.)
3194
3195 check_no_dead_use();
3196
3197 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
3198 // to remove hashes to unlock nodes for modifications.
3199 C->node_hash()->clear();
3200
3201 // A method with only infinite loops has no edges entering loops from root
3202 {
3203 TracePhase tp(_t_graphReshaping);
3204 if (final_graph_reshaping()) {
3205 assert(failing(), "must bail out w/ explicit message");
3206 return;
3207 }
3208 }
3209
3210 print_method(PHASE_OPTIMIZE_FINISHED, 2);
3211 DEBUG_ONLY(set_phase_optimize_finished();)
3212 }
3918 case Op_CmpD3:
3919 case Op_StoreD:
3920 case Op_LoadD:
3921 case Op_LoadD_unaligned:
3922 frc.inc_double_count();
3923 break;
3924 case Op_Opaque1: // Remove Opaque Nodes before matching
3925 n->subsume_by(n->in(1), this);
3926 break;
3927 case Op_CallLeafPure: {
3928 // If the pure call is not supported, then lower to a CallLeaf.
3929 if (!Matcher::match_rule_supported(Op_CallLeafPure)) {
3930 CallNode* call = n->as_Call();
3931 CallNode* new_call = new CallLeafNode(call->tf(), call->entry_point(),
3932 call->_name, TypeRawPtr::BOTTOM);
3933 new_call->init_req(TypeFunc::Control, call->in(TypeFunc::Control));
3934 new_call->init_req(TypeFunc::I_O, C->top());
3935 new_call->init_req(TypeFunc::Memory, C->top());
3936 new_call->init_req(TypeFunc::ReturnAdr, C->top());
3937 new_call->init_req(TypeFunc::FramePtr, C->top());
3938 for (unsigned int i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
3939 new_call->init_req(i, call->in(i));
3940 }
3941 n->subsume_by(new_call, this);
3942 }
3943 frc.inc_call_count();
3944 break;
3945 }
3946 case Op_CallStaticJava:
3947 case Op_CallJava:
3948 case Op_CallDynamicJava:
3949 frc.inc_java_call_count(); // Count java call site;
3950 case Op_CallRuntime:
3951 case Op_CallLeaf:
3952 case Op_CallLeafVector:
3953 case Op_CallLeafNoFP: {
3954 assert (n->is_Call(), "");
3955 CallNode *call = n->as_Call();
3956 // Count call sites where the FP mode bit would have to be flipped.
3957 // Do not count uncommon runtime calls:
3958 // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3964 int nop = n->Opcode();
3965 // Clone shared simple arguments to uncommon calls, item (1).
3966 if (n->outcnt() > 1 &&
3967 !n->is_Proj() &&
3968 nop != Op_CreateEx &&
3969 nop != Op_CheckCastPP &&
3970 nop != Op_DecodeN &&
3971 nop != Op_DecodeNKlass &&
3972 !n->is_Mem() &&
3973 !n->is_Phi()) {
3974 Node *x = n->clone();
3975 call->set_req(TypeFunc::Parms, x);
3976 }
3977 }
3978 break;
3979 }
3980 case Op_StoreB:
3981 case Op_StoreC:
3982 case Op_StoreI:
3983 case Op_StoreL:
3984 case Op_StoreLSpecial:
3985 case Op_CompareAndSwapB:
3986 case Op_CompareAndSwapS:
3987 case Op_CompareAndSwapI:
3988 case Op_CompareAndSwapL:
3989 case Op_CompareAndSwapP:
3990 case Op_CompareAndSwapN:
3991 case Op_WeakCompareAndSwapB:
3992 case Op_WeakCompareAndSwapS:
3993 case Op_WeakCompareAndSwapI:
3994 case Op_WeakCompareAndSwapL:
3995 case Op_WeakCompareAndSwapP:
3996 case Op_WeakCompareAndSwapN:
3997 case Op_CompareAndExchangeB:
3998 case Op_CompareAndExchangeS:
3999 case Op_CompareAndExchangeI:
4000 case Op_CompareAndExchangeL:
4001 case Op_CompareAndExchangeP:
4002 case Op_CompareAndExchangeN:
4003 case Op_GetAndAddS:
4004 case Op_GetAndAddB:
4514 k->subsume_by(m, this);
4515 }
4516 }
4517 }
4518 break;
4519 }
4520 case Op_CmpUL: {
4521 if (!Matcher::has_match_rule(Op_CmpUL)) {
4522 // No support for unsigned long comparisons
4523 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
4524 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
4525 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
4526 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
4527 Node* andl = new AndLNode(orl, remove_sign_mask);
4528 Node* cmp = new CmpLNode(andl, n->in(2));
4529 n->subsume_by(cmp, this);
4530 }
4531 break;
4532 }
4533 #ifdef ASSERT
4534 case Op_InlineType: {
4535 n->dump(-1);
4536 assert(false, "inline type node was not removed");
4537 break;
4538 }
4539 case Op_ConNKlass: {
4540 const TypePtr* tp = n->as_Type()->type()->make_ptr();
4541 ciKlass* klass = tp->is_klassptr()->exact_klass();
4542 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
4543 break;
4544 }
4545 #endif
4546 default:
4547 assert(!n->is_Call(), "");
4548 assert(!n->is_Mem(), "");
4549 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
4550 break;
4551 }
4552 }
4553
4554 //------------------------------final_graph_reshaping_walk---------------------
4555 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
4556 // requires that the walk visits a node's inputs before visiting the node.
4557 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
4558 Unique_Node_List sfpt;
4894 }
4895 }
4896
4897 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4898 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4899 }
4900
4901 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4902 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4903 }
4904
4905 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4906 if (holder->is_initialized()) {
4907 return false;
4908 }
4909 if (holder->is_being_initialized()) {
4910 if (accessing_method->holder() == holder) {
4911 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4912 // <init>, or a static method. In all those cases, there was an initialization
4913 // barrier on the holder klass passed.
4914 if (accessing_method->is_class_initializer() ||
4915 accessing_method->is_object_constructor() ||
4916 accessing_method->is_static()) {
4917 return false;
4918 }
4919 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4920 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4921 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4922 // child class can become fully initialized while its parent class is still being initialized.
4923 if (accessing_method->is_class_initializer()) {
4924 return false;
4925 }
4926 }
4927 ciMethod* root = method(); // the root method of compilation
4928 if (root != accessing_method) {
4929 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4930 }
4931 }
4932 return true;
4933 }
4934
4935 #ifndef PRODUCT
4936 //------------------------------verify_bidirectional_edges---------------------
4937 // For each input edge to a node (ie - for each Use-Def edge), verify that
4938 // there is a corresponding Def-Use edge.
4939 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4940 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4941 uint stack_size = live_nodes() >> 4;
4942 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4943 if (root_and_safepoints != nullptr) {
4973 if (in != nullptr && !in->is_top()) {
4974 // Count instances of `next`
4975 int cnt = 0;
4976 for (uint idx = 0; idx < in->_outcnt; idx++) {
4977 if (in->_out[idx] == n) {
4978 cnt++;
4979 }
4980 }
4981 assert(cnt > 0, "Failed to find Def-Use edge.");
4982 // Check for duplicate edges
4983 // walk the input array downcounting the input edges to n
4984 for (uint j = 0; j < length; j++) {
4985 if (n->in(j) == in) {
4986 cnt--;
4987 }
4988 }
4989 assert(cnt == 0, "Mismatched edge count.");
4990 } else if (in == nullptr) {
4991 assert(i == 0 || i >= n->req() ||
4992 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4993 (n->is_Allocate() && i >= AllocateNode::InlineType) ||
4994 (n->is_Unlock() && i == (n->req() - 1)) ||
4995 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4996 "only region, phi, arraycopy, allocate, unlock or membar nodes have null data edges");
4997 } else {
4998 assert(in->is_top(), "sanity");
4999 // Nothing to check.
5000 }
5001 }
5002 }
5003 }
5004
5005 //------------------------------verify_graph_edges---------------------------
5006 // Walk the Graph and verify that there is a one-to-one correspondence
5007 // between Use-Def edges and Def-Use edges in the graph.
5008 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
5009 if (VerifyGraphEdges) {
5010 Unique_Node_List visited;
5011
5012 // Call graph walk to check edges
5013 verify_bidirectional_edges(visited, root_and_safepoints);
5014 if (no_dead_code) {
5015 // Now make sure that no visited node is used by an unvisited node.
5016 bool dead_nodes = false;
5127 // (1) subklass is already limited to a subtype of superklass => always ok
5128 // (2) subklass does not overlap with superklass => always fail
5129 // (3) superklass has NO subtypes and we can check with a simple compare.
5130 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
5131 if (skip) {
5132 return SSC_full_test; // Let caller generate the general case.
5133 }
5134
5135 if (subk->is_java_subtype_of(superk)) {
5136 return SSC_always_true; // (0) and (1) this test cannot fail
5137 }
5138
5139 if (!subk->maybe_java_subtype_of(superk)) {
5140 return SSC_always_false; // (2) true path dead; no dynamic test needed
5141 }
5142
5143 const Type* superelem = superk;
5144 if (superk->isa_aryklassptr()) {
5145 int ignored;
5146 superelem = superk->is_aryklassptr()->base_element_type(ignored);
5147
5148 // Do not fold the subtype check to an array klass pointer comparison for null-able inline type arrays
5149 // because null-free [LMyValue <: null-able [LMyValue but the klasses are different. Perform a full test.
5150 if (!superk->is_aryklassptr()->is_null_free() && superk->is_aryklassptr()->elem()->isa_instklassptr() &&
5151 superk->is_aryklassptr()->elem()->is_instklassptr()->instance_klass()->is_inlinetype()) {
5152 return SSC_full_test;
5153 }
5154 }
5155
5156 if (superelem->isa_instklassptr()) {
5157 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
5158 if (!ik->has_subklass()) {
5159 if (!ik->is_final()) {
5160 // Add a dependency if there is a chance of a later subclass.
5161 dependencies()->assert_leaf_type(ik);
5162 }
5163 if (!superk->maybe_java_subtype_of(subk)) {
5164 return SSC_always_false;
5165 }
5166 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5167 }
5168 } else {
5169 // A primitive array type has no subtypes.
5170 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5171 }
5172
5173 return SSC_full_test;
5972 } else {
5973 _debug_network_printer->update_compiled_method(C->method());
5974 }
5975 tty->print_cr("Method printed over network stream to IGV");
5976 _debug_network_printer->print(name, C->root(), visible_nodes, fr);
5977 }
5978 #endif // !PRODUCT
5979
5980 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5981 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5982 return value;
5983 }
5984 Node* result = nullptr;
5985 if (bt == T_BYTE) {
5986 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5987 result = new RShiftINode(result, phase->intcon(24));
5988 } else if (bt == T_BOOLEAN) {
5989 result = new AndINode(value, phase->intcon(0xFF));
5990 } else if (bt == T_CHAR) {
5991 result = new AndINode(value,phase->intcon(0xFFFF));
5992 } else if (bt == T_FLOAT) {
5993 result = new MoveI2FNode(value);
5994 } else {
5995 assert(bt == T_SHORT, "unexpected narrow type");
5996 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5997 result = new RShiftINode(result, phase->intcon(16));
5998 }
5999 if (transform_res) {
6000 result = phase->transform(result);
6001 }
6002 return result;
6003 }
6004
6005 void Compile::record_method_not_compilable_oom() {
6006 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
6007 }
6008
6009 #ifndef PRODUCT
6010 // Collects all the control inputs from nodes on the worklist and from their data dependencies
6011 static void find_candidate_control_inputs(Unique_Node_List& worklist, Unique_Node_List& candidates) {
6012 // Follow non-control edges until we reach CFG nodes
6013 for (uint i = 0; i < worklist.size(); i++) {
|