1 /*
2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "ci/ciReplay.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "code/aotCodeCache.hpp"
30 #include "code/exceptionHandlerTable.hpp"
31 #include "code/nmethod.hpp"
32 #include "compiler/compilationFailureInfo.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/compileLog.hpp"
36 #include "compiler/compiler_globals.hpp"
37 #include "compiler/compilerDefinitions.hpp"
38 #include "compiler/compilerOracle.hpp"
39 #include "compiler/disassembler.hpp"
40 #include "compiler/oopMap.hpp"
41 #include "gc/shared/barrierSet.hpp"
42 #include "gc/shared/c2/barrierSetC2.hpp"
43 #include "jfr/jfrEvents.hpp"
44 #include "jvm_io.h"
45 #include "memory/allocation.hpp"
46 #include "memory/arena.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "opto/addnode.hpp"
49 #include "opto/block.hpp"
50 #include "opto/c2compiler.hpp"
51 #include "opto/callGenerator.hpp"
52 #include "opto/callnode.hpp"
53 #include "opto/castnode.hpp"
54 #include "opto/cfgnode.hpp"
55 #include "opto/chaitin.hpp"
56 #include "opto/compile.hpp"
57 #include "opto/connode.hpp"
58 #include "opto/convertnode.hpp"
59 #include "opto/divnode.hpp"
60 #include "opto/escape.hpp"
61 #include "opto/idealGraphPrinter.hpp"
62 #include "opto/locknode.hpp"
63 #include "opto/loopnode.hpp"
64 #include "opto/machnode.hpp"
65 #include "opto/macro.hpp"
66 #include "opto/matcher.hpp"
67 #include "opto/mathexactnode.hpp"
68 #include "opto/memnode.hpp"
69 #include "opto/mulnode.hpp"
70 #include "opto/narrowptrnode.hpp"
71 #include "opto/node.hpp"
72 #include "opto/opaquenode.hpp"
73 #include "opto/opcodes.hpp"
74 #include "opto/output.hpp"
75 #include "opto/parse.hpp"
76 #include "opto/phaseX.hpp"
77 #include "opto/rootnode.hpp"
78 #include "opto/runtime.hpp"
79 #include "opto/stringopts.hpp"
80 #include "opto/type.hpp"
81 #include "opto/vector.hpp"
82 #include "opto/vectornode.hpp"
83 #include "runtime/globals_extension.hpp"
84 #include "runtime/sharedRuntime.hpp"
85 #include "runtime/signature.hpp"
86 #include "runtime/stubRoutines.hpp"
87 #include "runtime/timer.hpp"
88 #include "utilities/align.hpp"
89 #include "utilities/copy.hpp"
90 #include "utilities/hashTable.hpp"
91 #include "utilities/macros.hpp"
92
93 // -------------------- Compile::mach_constant_base_node -----------------------
94 // Constant table base node singleton.
95 MachConstantBaseNode* Compile::mach_constant_base_node() {
96 if (_mach_constant_base_node == nullptr) {
97 _mach_constant_base_node = new MachConstantBaseNode();
98 _mach_constant_base_node->add_req(C->root());
99 }
100 return _mach_constant_base_node;
101 }
102
388 // as dead to be conservative about the dead node count at any
389 // given time.
390 if (!dead->is_Con()) {
391 record_dead_node(dead->_idx);
392 }
393 if (dead->is_macro()) {
394 remove_macro_node(dead);
395 }
396 if (dead->is_expensive()) {
397 remove_expensive_node(dead);
398 }
399 if (dead->is_OpaqueTemplateAssertionPredicate()) {
400 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
401 }
402 if (dead->is_ParsePredicate()) {
403 remove_parse_predicate(dead->as_ParsePredicate());
404 }
405 if (dead->for_post_loop_opts_igvn()) {
406 remove_from_post_loop_opts_igvn(dead);
407 }
408 if (dead->for_merge_stores_igvn()) {
409 remove_from_merge_stores_igvn(dead);
410 }
411 if (dead->is_Call()) {
412 remove_useless_late_inlines( &_late_inlines, dead);
413 remove_useless_late_inlines( &_string_late_inlines, dead);
414 remove_useless_late_inlines( &_boxing_late_inlines, dead);
415 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
416
417 if (dead->is_CallStaticJava()) {
418 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
419 }
420 }
421 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
422 bs->unregister_potential_barrier_node(dead);
423 }
424
425 // Disconnect all useless nodes by disconnecting those at the boundary.
426 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
427 uint next = 0;
435 // Use raw traversal of out edges since this code removes out edges
436 int max = n->outcnt();
437 for (int j = 0; j < max; ++j) {
438 Node* child = n->raw_out(j);
439 if (!useful.member(child)) {
440 assert(!child->is_top() || child != top(),
441 "If top is cached in Compile object it is in useful list");
442 // Only need to remove this out-edge to the useless node
443 n->raw_del_out(j);
444 --j;
445 --max;
446 if (child->is_data_proj_of_pure_function(n)) {
447 worklist.push(n);
448 }
449 }
450 }
451 if (n->outcnt() == 1 && n->has_special_unique_user()) {
452 assert(useful.member(n->unique_out()), "do not push a useless node");
453 worklist.push(n->unique_out());
454 }
455 }
456
457 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
458 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
459 // Remove useless Template Assertion Predicate opaque nodes
460 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
461 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
462 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
463 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
464 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
465 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
466 #ifdef ASSERT
467 if (_modified_nodes != nullptr) {
468 _modified_nodes->remove_useless_nodes(useful.member_set());
469 }
470 #endif
471
472 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
473 bs->eliminate_useless_gc_barriers(useful, this);
474 // clean up the late inline lists
475 remove_useless_late_inlines( &_late_inlines, useful);
476 remove_useless_late_inlines( &_string_late_inlines, useful);
477 remove_useless_late_inlines( &_boxing_late_inlines, useful);
478 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
479 DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
480 }
481
482 // ============================================================================
629
630
631 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
632 Options options, DirectiveSet* directive)
633 : Phase(Compiler),
634 _compile_id(ci_env->compile_id()),
635 _options(options),
636 _method(target),
637 _entry_bci(osr_bci),
638 _ilt(nullptr),
639 _stub_function(nullptr),
640 _stub_name(nullptr),
641 _stub_id(StubId::NO_STUBID),
642 _stub_entry_point(nullptr),
643 _max_node_limit(MaxNodeLimit),
644 _post_loop_opts_phase(false),
645 _merge_stores_phase(false),
646 _allow_macro_nodes(true),
647 _inlining_progress(false),
648 _inlining_incrementally(false),
649 _do_cleanup(false),
650 _has_reserved_stack_access(target->has_reserved_stack_access()),
651 #ifndef PRODUCT
652 _igv_idx(0),
653 _trace_opto_output(directive->TraceOptoOutputOption),
654 #endif
655 _clinit_barrier_on_entry(false),
656 _stress_seed(0),
657 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
658 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
659 _env(ci_env),
660 _directive(directive),
661 _log(ci_env->log()),
662 _first_failure_details(nullptr),
663 _intrinsics(comp_arena(), 0, 0, nullptr),
664 _macro_nodes(comp_arena(), 8, 0, nullptr),
665 _parse_predicates(comp_arena(), 8, 0, nullptr),
666 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
667 _expensive_nodes(comp_arena(), 8, 0, nullptr),
668 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
669 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
670 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
671 _coarsened_locks(comp_arena(), 8, 0, nullptr),
672 _congraph(nullptr),
673 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
674 _unique(0),
675 _dead_node_count(0),
676 _dead_node_list(comp_arena()),
677 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
678 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
679 _node_arena(&_node_arena_one),
680 _mach_constant_base_node(nullptr),
681 _Compile_types(mtCompiler, Arena::Tag::tag_type),
682 _initial_gvn(nullptr),
683 _igvn_worklist(nullptr),
684 _types(nullptr),
685 _node_hash(nullptr),
686 _late_inlines(comp_arena(), 2, 0, nullptr),
687 _string_late_inlines(comp_arena(), 2, 0, nullptr),
688 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
757 #define MINIMUM_NODE_HASH 1023
758
759 // GVN that will be run immediately on new nodes
760 uint estimated_size = method()->code_size()*4+64;
761 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
762 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
763 _types = new (comp_arena()) Type_Array(comp_arena());
764 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
765 PhaseGVN gvn;
766 set_initial_gvn(&gvn);
767
768 { // Scope for timing the parser
769 TracePhase tp(_t_parser);
770
771 // Put top into the hash table ASAP.
772 initial_gvn()->transform(top());
773
774 // Set up tf(), start(), and find a CallGenerator.
775 CallGenerator* cg = nullptr;
776 if (is_osr_compilation()) {
777 const TypeTuple *domain = StartOSRNode::osr_domain();
778 const TypeTuple *range = TypeTuple::make_range(method()->signature());
779 init_tf(TypeFunc::make(domain, range));
780 StartNode* s = new StartOSRNode(root(), domain);
781 initial_gvn()->set_type_bottom(s);
782 verify_start(s);
783 cg = CallGenerator::for_osr(method(), entry_bci());
784 } else {
785 // Normal case.
786 init_tf(TypeFunc::make(method()));
787 StartNode* s = new StartNode(root(), tf()->domain());
788 initial_gvn()->set_type_bottom(s);
789 verify_start(s);
790 float past_uses = method()->interpreter_invocation_count();
791 float expected_uses = past_uses;
792 cg = CallGenerator::for_inline(method(), expected_uses);
793 }
794 if (failing()) return;
795 if (cg == nullptr) {
796 const char* reason = InlineTree::check_can_parse(method());
797 assert(reason != nullptr, "expect reason for parse failure");
798 stringStream ss;
799 ss.print("cannot parse method: %s", reason);
800 record_method_not_compilable(ss.as_string());
801 return;
802 }
803
804 gvn.set_type(root(), root()->bottom_type());
805
806 JVMState* jvms = build_start_state(start(), tf());
807 if ((jvms = cg->generate(jvms)) == nullptr) {
868 print_ideal_ir("print_ideal");
869 }
870 #endif
871
872 #ifdef ASSERT
873 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
874 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
875 #endif
876
877 // Dump compilation data to replay it.
878 if (directive->DumpReplayOption) {
879 env()->dump_replay_data(_compile_id);
880 }
881 if (directive->DumpInlineOption && (ilt() != nullptr)) {
882 env()->dump_inline_data(_compile_id);
883 }
884
885 // Now that we know the size of all the monitors we can add a fixed slot
886 // for the original deopt pc.
887 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
888 set_fixed_slots(next_slot);
889
890 // Compute when to use implicit null checks. Used by matching trap based
891 // nodes and NullCheck optimization.
892 set_allowed_deopt_reasons();
893
894 // Now generate code
895 Code_Gen();
896 }
897
898 //------------------------------Compile----------------------------------------
899 // Compile a runtime stub
900 Compile::Compile(ciEnv* ci_env,
901 TypeFunc_generator generator,
902 address stub_function,
903 const char* stub_name,
904 StubId stub_id,
905 int is_fancy_jump,
906 bool pass_tls,
907 bool return_pc,
908 DirectiveSet* directive)
909 : Phase(Compiler),
910 _compile_id(0),
911 _options(Options::for_runtime_stub()),
912 _method(nullptr),
913 _entry_bci(InvocationEntryBci),
914 _stub_function(stub_function),
915 _stub_name(stub_name),
916 _stub_id(stub_id),
917 _stub_entry_point(nullptr),
918 _max_node_limit(MaxNodeLimit),
919 _post_loop_opts_phase(false),
920 _merge_stores_phase(false),
921 _allow_macro_nodes(true),
922 _inlining_progress(false),
923 _inlining_incrementally(false),
924 _has_reserved_stack_access(false),
925 #ifndef PRODUCT
926 _igv_idx(0),
927 _trace_opto_output(directive->TraceOptoOutputOption),
928 #endif
929 _clinit_barrier_on_entry(false),
930 _stress_seed(0),
931 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
932 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
933 _env(ci_env),
934 _directive(directive),
935 _log(ci_env->log()),
936 _first_failure_details(nullptr),
937 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
938 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
939 _congraph(nullptr),
940 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
941 _unique(0),
942 _dead_node_count(0),
943 _dead_node_list(comp_arena()),
944 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
1059 _fixed_slots = 0;
1060 set_has_split_ifs(false);
1061 set_has_loops(false); // first approximation
1062 set_has_stringbuilder(false);
1063 set_has_boxed_value(false);
1064 _trap_can_recompile = false; // no traps emitted yet
1065 _major_progress = true; // start out assuming good things will happen
1066 set_has_unsafe_access(false);
1067 set_max_vector_size(0);
1068 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1069 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1070 set_decompile_count(0);
1071
1072 #ifndef PRODUCT
1073 _phase_counter = 0;
1074 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1075 #endif
1076
1077 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1078 _loop_opts_cnt = LoopOptsCount;
1079 set_do_inlining(Inline);
1080 set_max_inline_size(MaxInlineSize);
1081 set_freq_inline_size(FreqInlineSize);
1082 set_do_scheduling(OptoScheduling);
1083
1084 set_do_vector_loop(false);
1085 set_has_monitors(false);
1086 set_has_scoped_access(false);
1087
1088 if (AllowVectorizeOnDemand) {
1089 if (has_method() && _directive->VectorizeOption) {
1090 set_do_vector_loop(true);
1091 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1092 } else if (has_method() && method()->name() != nullptr &&
1093 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1094 set_do_vector_loop(true);
1095 }
1096 }
1097 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1098 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1339 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1340 assert(do_aliasing(), "Aliasing should be enabled");
1341 int offset = tj->offset();
1342 TypePtr::PTR ptr = tj->ptr();
1343
1344 // Known instance (scalarizable allocation) alias only with itself.
1345 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1346 tj->is_oopptr()->is_known_instance();
1347
1348 // Process weird unsafe references.
1349 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1350 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1351 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1352 tj = TypeOopPtr::BOTTOM;
1353 ptr = tj->ptr();
1354 offset = tj->offset();
1355 }
1356
1357 // Array pointers need some flattening
1358 const TypeAryPtr* ta = tj->isa_aryptr();
1359 if (ta && ta->is_stable()) {
1360 // Erase stability property for alias analysis.
1361 tj = ta = ta->cast_to_stable(false);
1362 }
1363 if( ta && is_known_inst ) {
1364 if ( offset != Type::OffsetBot &&
1365 offset > arrayOopDesc::length_offset_in_bytes() ) {
1366 offset = Type::OffsetBot; // Flatten constant access into array body only
1367 tj = ta = ta->
1368 remove_speculative()->
1369 cast_to_ptr_type(ptr)->
1370 with_offset(offset);
1371 }
1372 } else if (ta) {
1373 // For arrays indexed by constant indices, we flatten the alias
1374 // space to include all of the array body. Only the header, klass
1375 // and array length can be accessed un-aliased.
1376 if( offset != Type::OffsetBot ) {
1377 if( ta->const_oop() ) { // MethodData* or Method*
1378 offset = Type::OffsetBot; // Flatten constant access into array body
1379 tj = ta = ta->
1380 remove_speculative()->
1381 cast_to_ptr_type(ptr)->
1382 cast_to_exactness(false)->
1383 with_offset(offset);
1384 } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1385 // range is OK as-is.
1386 tj = ta = TypeAryPtr::RANGE;
1387 } else if( offset == oopDesc::klass_offset_in_bytes() ) {
1388 tj = TypeInstPtr::KLASS; // all klass loads look alike
1389 ta = TypeAryPtr::RANGE; // generic ignored junk
1390 ptr = TypePtr::BotPTR;
1391 } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1392 tj = TypeInstPtr::MARK;
1393 ta = TypeAryPtr::RANGE; // generic ignored junk
1394 ptr = TypePtr::BotPTR;
1395 } else { // Random constant offset into array body
1396 offset = Type::OffsetBot; // Flatten constant access into array body
1397 tj = ta = ta->
1398 remove_speculative()->
1399 cast_to_ptr_type(ptr)->
1400 cast_to_exactness(false)->
1401 with_offset(offset);
1402 }
1403 }
1404 // Arrays of fixed size alias with arrays of unknown size.
1405 if (ta->size() != TypeInt::POS) {
1406 const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1407 tj = ta = ta->
1408 remove_speculative()->
1409 cast_to_ptr_type(ptr)->
1410 with_ary(tary)->
1411 cast_to_exactness(false);
1412 }
1413 // Arrays of known objects become arrays of unknown objects.
1414 if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1415 const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1416 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1417 }
1418 if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1419 const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1420 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1421 }
1422 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1423 // cannot be distinguished by bytecode alone.
1424 if (ta->elem() == TypeInt::BOOL) {
1425 const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1426 ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1427 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,offset);
1428 }
1429 // During the 2nd round of IterGVN, NotNull castings are removed.
1430 // Make sure the Bottom and NotNull variants alias the same.
1431 // Also, make sure exact and non-exact variants alias the same.
1432 if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1433 tj = ta = ta->
1434 remove_speculative()->
1435 cast_to_ptr_type(TypePtr::BotPTR)->
1436 cast_to_exactness(false)->
1437 with_offset(offset);
1438 }
1439 }
1440
1441 // Oop pointers need some flattening
1442 const TypeInstPtr *to = tj->isa_instptr();
1443 if (to && to != TypeOopPtr::BOTTOM) {
1444 ciInstanceKlass* ik = to->instance_klass();
1445 if( ptr == TypePtr::Constant ) {
1446 if (ik != ciEnv::current()->Class_klass() ||
1447 offset < ik->layout_helper_size_in_bytes()) {
1448 // No constant oop pointers (such as Strings); they alias with
1449 // unknown strings.
1450 assert(!is_known_inst, "not scalarizable allocation");
1451 tj = to = to->
1452 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1453 remove_speculative()->
1454 cast_to_ptr_type(TypePtr::BotPTR)->
1455 cast_to_exactness(false);
1456 }
1457 } else if( is_known_inst ) {
1458 tj = to; // Keep NotNull and klass_is_exact for instance type
1459 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1460 // During the 2nd round of IterGVN, NotNull castings are removed.
1461 // Make sure the Bottom and NotNull variants alias the same.
1462 // Also, make sure exact and non-exact variants alias the same.
1463 tj = to = to->
1464 remove_speculative()->
1465 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1466 cast_to_ptr_type(TypePtr::BotPTR)->
1467 cast_to_exactness(false);
1468 }
1469 if (to->speculative() != nullptr) {
1470 tj = to = to->remove_speculative();
1471 }
1472 // Canonicalize the holder of this field
1473 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1474 // First handle header references such as a LoadKlassNode, even if the
1475 // object's klass is unloaded at compile time (4965979).
1476 if (!is_known_inst) { // Do it only for non-instance types
1477 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, offset);
1478 }
1479 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1480 // Static fields are in the space above the normal instance
1481 // fields in the java.lang.Class instance.
1482 if (ik != ciEnv::current()->Class_klass()) {
1483 to = nullptr;
1484 tj = TypeOopPtr::BOTTOM;
1485 offset = tj->offset();
1486 }
1487 } else {
1488 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1489 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1490 assert(tj->offset() == offset, "no change to offset expected");
1491 bool xk = to->klass_is_exact();
1492 int instance_id = to->instance_id();
1493
1494 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1495 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1496 // its interfaces are included.
1497 if (xk && ik->equals(canonical_holder)) {
1498 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id), "exact type should be canonical type");
1499 } else {
1500 assert(xk || !is_known_inst, "Known instance should be exact type");
1501 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id);
1502 }
1503 }
1504 }
1505
1506 // Klass pointers to object array klasses need some flattening
1507 const TypeKlassPtr *tk = tj->isa_klassptr();
1508 if( tk ) {
1509 // If we are referencing a field within a Klass, we need
1510 // to assume the worst case of an Object. Both exact and
1511 // inexact types must flatten to the same alias class so
1512 // use NotNull as the PTR.
1513 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1514 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1515 env()->Object_klass(),
1516 offset);
1517 }
1518
1519 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1520 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1521 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1522 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), offset);
1523 } else {
1524 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, offset);
1525 }
1526 }
1527
1528 // Check for precise loads from the primary supertype array and force them
1529 // to the supertype cache alias index. Check for generic array loads from
1530 // the primary supertype array and also force them to the supertype cache
1531 // alias index. Since the same load can reach both, we need to merge
1532 // these 2 disparate memories into the same alias class. Since the
1533 // primary supertype array is read-only, there's no chance of confusion
1534 // where we bypass an array load and an array store.
1535 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1536 if (offset == Type::OffsetBot ||
1537 (offset >= primary_supers_offset &&
1538 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1539 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1540 offset = in_bytes(Klass::secondary_super_cache_offset());
1541 tj = tk = tk->with_offset(offset);
1542 }
1543 }
1544
1545 // Flatten all Raw pointers together.
1546 if (tj->base() == Type::RawPtr)
1547 tj = TypeRawPtr::BOTTOM;
1637 intptr_t key = (intptr_t) adr_type;
1638 key ^= key >> logAliasCacheSize;
1639 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1640 }
1641
1642
1643 //-----------------------------grow_alias_types--------------------------------
1644 void Compile::grow_alias_types() {
1645 const int old_ats = _max_alias_types; // how many before?
1646 const int new_ats = old_ats; // how many more?
1647 const int grow_ats = old_ats+new_ats; // how many now?
1648 _max_alias_types = grow_ats;
1649 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1650 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1651 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1652 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1653 }
1654
1655
1656 //--------------------------------find_alias_type------------------------------
1657 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field) {
1658 if (!do_aliasing()) {
1659 return alias_type(AliasIdxBot);
1660 }
1661
1662 AliasCacheEntry* ace = probe_alias_cache(adr_type);
1663 if (ace->_adr_type == adr_type) {
1664 return alias_type(ace->_index);
1665 }
1666
1667 // Handle special cases.
1668 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1669 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1670
1671 // Do it the slow way.
1672 const TypePtr* flat = flatten_alias_type(adr_type);
1673
1674 #ifdef ASSERT
1675 {
1676 ResourceMark rm;
1677 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1678 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1679 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1680 Type::str(adr_type));
1681 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1682 const TypeOopPtr* foop = flat->is_oopptr();
1683 // Scalarizable allocations have exact klass always.
1684 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1694 if (alias_type(i)->adr_type() == flat) {
1695 idx = i;
1696 break;
1697 }
1698 }
1699
1700 if (idx == AliasIdxTop) {
1701 if (no_create) return nullptr;
1702 // Grow the array if necessary.
1703 if (_num_alias_types == _max_alias_types) grow_alias_types();
1704 // Add a new alias type.
1705 idx = _num_alias_types++;
1706 _alias_types[idx]->Init(idx, flat);
1707 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1708 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1709 if (flat->isa_instptr()) {
1710 if (flat->offset() == java_lang_Class::klass_offset()
1711 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1712 alias_type(idx)->set_rewritable(false);
1713 }
1714 if (flat->isa_aryptr()) {
1715 #ifdef ASSERT
1716 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1717 // (T_BYTE has the weakest alignment and size restrictions...)
1718 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1719 #endif
1720 if (flat->offset() == TypePtr::OffsetBot) {
1721 alias_type(idx)->set_element(flat->is_aryptr()->elem());
1722 }
1723 }
1724 if (flat->isa_klassptr()) {
1725 if (UseCompactObjectHeaders) {
1726 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1727 alias_type(idx)->set_rewritable(false);
1728 }
1729 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1730 alias_type(idx)->set_rewritable(false);
1731 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1732 alias_type(idx)->set_rewritable(false);
1733 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1734 alias_type(idx)->set_rewritable(false);
1735 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1736 alias_type(idx)->set_rewritable(false);
1737 }
1738
1739 if (flat->isa_instklassptr()) {
1740 if (flat->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
1741 alias_type(idx)->set_rewritable(false);
1742 }
1743 }
1744 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1745 // but the base pointer type is not distinctive enough to identify
1746 // references into JavaThread.)
1747
1748 // Check for final fields.
1749 const TypeInstPtr* tinst = flat->isa_instptr();
1750 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1751 ciField* field;
1752 if (tinst->const_oop() != nullptr &&
1753 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1754 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1755 // static field
1756 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1757 field = k->get_field_by_offset(tinst->offset(), true);
1758 } else {
1759 ciInstanceKlass *k = tinst->instance_klass();
1760 field = k->get_field_by_offset(tinst->offset(), false);
1761 }
1762 assert(field == nullptr ||
1763 original_field == nullptr ||
1764 (field->holder() == original_field->holder() &&
1765 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1766 field->is_static() == original_field->is_static()), "wrong field?");
1767 // Set field() and is_rewritable() attributes.
1768 if (field != nullptr) alias_type(idx)->set_field(field);
1769 }
1770 }
1771
1772 // Fill the cache for next time.
1773 ace->_adr_type = adr_type;
1774 ace->_index = idx;
1775 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1776
1777 // Might as well try to fill the cache for the flattened version, too.
1778 AliasCacheEntry* face = probe_alias_cache(flat);
1779 if (face->_adr_type == nullptr) {
1780 face->_adr_type = flat;
1781 face->_index = idx;
1782 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1783 }
1784
1785 return alias_type(idx);
1786 }
1787
1788
1789 Compile::AliasType* Compile::alias_type(ciField* field) {
1790 const TypeOopPtr* t;
1791 if (field->is_static())
1792 t = TypeInstPtr::make(field->holder()->java_mirror());
1793 else
1794 t = TypeOopPtr::make_from_klass_raw(field->holder());
1795 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1796 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1797 return atp;
1798 }
1799
1800
1801 //------------------------------have_alias_type--------------------------------
1802 bool Compile::have_alias_type(const TypePtr* adr_type) {
1884 assert(!C->major_progress(), "not cleared");
1885
1886 if (_for_post_loop_igvn.length() > 0) {
1887 while (_for_post_loop_igvn.length() > 0) {
1888 Node* n = _for_post_loop_igvn.pop();
1889 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1890 igvn._worklist.push(n);
1891 }
1892 igvn.optimize();
1893 if (failing()) return;
1894 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1895 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1896
1897 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1898 if (C->major_progress()) {
1899 C->clear_major_progress(); // ensure that major progress is now clear
1900 }
1901 }
1902 }
1903
1904 void Compile::record_for_merge_stores_igvn(Node* n) {
1905 if (!n->for_merge_stores_igvn()) {
1906 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
1907 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1908 _for_merge_stores_igvn.append(n);
1909 }
1910 }
1911
1912 void Compile::remove_from_merge_stores_igvn(Node* n) {
1913 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1914 _for_merge_stores_igvn.remove(n);
1915 }
1916
1917 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
1918 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
1919 // the stores, and we merge the wrong sequence of stores.
1920 // Example:
1921 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
1922 // Apply MergeStores:
1923 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
2002 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2003 Bytecodes::Code c = iter.cur_bc();
2004 Node* lhs = nullptr;
2005 Node* rhs = nullptr;
2006 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2007 lhs = unc->peek_operand(0);
2008 rhs = unc->peek_operand(1);
2009 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2010 lhs = unc->peek_operand(0);
2011 }
2012
2013 ResourceMark rm;
2014 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2015 assert(live_locals.is_valid(), "broken liveness info");
2016 int len = (int)live_locals.size();
2017
2018 for (int i = 0; i < len; i++) {
2019 Node* local = unc->local(jvms, i);
2020 // kill local using the liveness of next_bci.
2021 // give up when the local looks like an operand to secure reexecution.
2022 if (!live_locals.at(i) && !local->is_top() && local != lhs && local!= rhs) {
2023 uint idx = jvms->locoff() + i;
2024 #ifdef ASSERT
2025 if (PrintOpto && Verbose) {
2026 tty->print("[unstable_if] kill local#%d: ", idx);
2027 local->dump();
2028 tty->cr();
2029 }
2030 #endif
2031 igvn.replace_input_of(unc, idx, top());
2032 modified = true;
2033 }
2034 }
2035 }
2036
2037 // keep the mondified trap for late query
2038 if (modified) {
2039 trap->set_modified();
2040 } else {
2041 _unstable_if_traps.delete_at(i);
2042 }
2043 }
2044 igvn.optimize();
2045 }
2046
2047 // StringOpts and late inlining of string methods
2048 void Compile::inline_string_calls(bool parse_time) {
2049 {
2050 // remove useless nodes to make the usage analysis simpler
2051 ResourceMark rm;
2052 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2053 }
2054
2055 {
2056 ResourceMark rm;
2057 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2229
2230 if (_string_late_inlines.length() > 0) {
2231 assert(has_stringbuilder(), "inconsistent");
2232
2233 inline_string_calls(false);
2234
2235 if (failing()) return;
2236
2237 inline_incrementally_cleanup(igvn);
2238 }
2239
2240 set_inlining_incrementally(false);
2241 }
2242
2243 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2244 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2245 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2246 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2247 // as if "inlining_incrementally() == true" were set.
2248 assert(inlining_incrementally() == false, "not allowed");
2249 assert(_modified_nodes == nullptr, "not allowed");
2250 assert(_late_inlines.length() > 0, "sanity");
2251
2252 while (_late_inlines.length() > 0) {
2253 igvn_worklist()->ensure_empty(); // should be done with igvn
2254
2255 while (inline_incrementally_one()) {
2256 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2257 }
2258 if (failing()) return;
2259
2260 inline_incrementally_cleanup(igvn);
2261 }
2262 }
2263
2264 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2265 if (_loop_opts_cnt > 0) {
2266 while (major_progress() && (_loop_opts_cnt > 0)) {
2267 TracePhase tp(_t_idealLoop);
2268 PhaseIdealLoop::optimize(igvn, mode);
2269 _loop_opts_cnt--;
2270 if (failing()) return false;
2271 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2272 }
2273 }
2274 return true;
2275 }
2276
2277 // Remove edges from "root" to each SafePoint at a backward branch.
2278 // They were inserted during parsing (see add_safepoint()) to make
2279 // infinite loops without calls or exceptions visible to root, i.e.,
2280 // useful.
2281 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2385 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2386 }
2387 assert(!has_vbox_nodes(), "sanity");
2388
2389 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2390 Compile::TracePhase tp(_t_renumberLive);
2391 igvn_worklist()->ensure_empty(); // should be done with igvn
2392 {
2393 ResourceMark rm;
2394 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2395 }
2396 igvn.reset();
2397 igvn.optimize();
2398 if (failing()) return;
2399 }
2400
2401 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2402 // safepoints
2403 remove_root_to_sfpts_edges(igvn);
2404
2405 if (failing()) return;
2406
2407 if (has_loops()) {
2408 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2409 }
2410
2411 // Perform escape analysis
2412 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2413 if (has_loops()) {
2414 // Cleanup graph (remove dead nodes).
2415 TracePhase tp(_t_idealLoop);
2416 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2417 if (failing()) return;
2418 }
2419 bool progress;
2420 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2421 do {
2422 ConnectionGraph::do_analysis(this, &igvn);
2423
2424 if (failing()) return;
2425
2426 int mcount = macro_count(); // Record number of allocations and locks before IGVN
2427
2428 // Optimize out fields loads from scalar replaceable allocations.
2429 igvn.optimize();
2430 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2431
2432 if (failing()) return;
2433
2434 if (congraph() != nullptr && macro_count() > 0) {
2435 TracePhase tp(_t_macroEliminate);
2436 PhaseMacroExpand mexp(igvn);
2437 mexp.eliminate_macro_nodes();
2438 if (failing()) return;
2439 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
2440
2441 igvn.set_delay_transform(false);
2442 igvn.optimize();
2443 if (failing()) return;
2444
2445 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2446 }
2447
2448 ConnectionGraph::verify_ram_nodes(this, root());
2449 if (failing()) return;
2450
2451 progress = do_iterative_escape_analysis() &&
2452 (macro_count() < mcount) &&
2453 ConnectionGraph::has_candidates(this);
2454 // Try again if candidates exist and made progress
2455 // by removing some allocations and/or locks.
2456 } while (progress);
2457 }
2458
2459 // Loop transforms on the ideal graph. Range Check Elimination,
2460 // peeling, unrolling, etc.
2461
2462 // Set loop opts counter
2463 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2464 {
2465 TracePhase tp(_t_idealLoop);
2466 PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
2467 _loop_opts_cnt--;
2468 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
2469 if (failing()) return;
2470 }
2471 // Loop opts pass if partial peeling occurred in previous pass
2472 if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
2473 TracePhase tp(_t_idealLoop);
2474 PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
2475 _loop_opts_cnt--;
2476 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
2477 if (failing()) return;
2478 }
2515 // Loop transforms on the ideal graph. Range Check Elimination,
2516 // peeling, unrolling, etc.
2517 if (!optimize_loops(igvn, LoopOptsDefault)) {
2518 return;
2519 }
2520
2521 if (failing()) return;
2522
2523 C->clear_major_progress(); // ensure that major progress is now clear
2524
2525 process_for_post_loop_opts_igvn(igvn);
2526
2527 process_for_merge_stores_igvn(igvn);
2528
2529 if (failing()) return;
2530
2531 #ifdef ASSERT
2532 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
2533 #endif
2534
2535 {
2536 TracePhase tp(_t_macroExpand);
2537 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
2538 PhaseMacroExpand mex(igvn);
2539 // Do not allow new macro nodes once we start to eliminate and expand
2540 C->reset_allow_macro_nodes();
2541 // Last attempt to eliminate macro nodes before expand
2542 mex.eliminate_macro_nodes();
2543 if (failing()) {
2544 return;
2545 }
2546 mex.eliminate_opaque_looplimit_macro_nodes();
2547 if (failing()) {
2548 return;
2549 }
2550 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
2551 if (mex.expand_macro_nodes()) {
2552 assert(failing(), "must bail out w/ explicit message");
2553 return;
2554 }
2555 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
2556 }
2557
2558 {
2559 TracePhase tp(_t_barrierExpand);
2560 if (bs->expand_barriers(this, igvn)) {
2561 assert(failing(), "must bail out w/ explicit message");
2562 return;
2563 }
2564 print_method(PHASE_BARRIER_EXPANSION, 2);
2565 }
2566
2567 if (C->max_vector_size() > 0) {
2568 C->optimize_logic_cones(igvn);
2569 igvn.optimize();
2570 if (failing()) return;
2571 }
2572
2573 DEBUG_ONLY( _modified_nodes = nullptr; )
2574
2575 assert(igvn._worklist.size() == 0, "not empty");
2576
2577 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
2578
2579 if (_late_inlines.length() > 0) {
2580 // More opportunities to optimize virtual and MH calls.
2581 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
2582 process_late_inline_calls_no_inline(igvn);
2583 if (failing()) return;
2584 }
2585 } // (End scope of igvn; run destructor if necessary for asserts.)
2586
2587 check_no_dead_use();
2588
2589 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
2590 // to remove hashes to unlock nodes for modifications.
2591 C->node_hash()->clear();
2592
2593 // A method with only infinite loops has no edges entering loops from root
2594 {
2595 TracePhase tp(_t_graphReshaping);
2596 if (final_graph_reshaping()) {
2597 assert(failing(), "must bail out w/ explicit message");
2598 return;
2599 }
2600 }
2601
2602 print_method(PHASE_OPTIMIZE_FINISHED, 2);
2603 DEBUG_ONLY(set_phase_optimize_finished();)
2604 }
3310 case Op_CmpD3:
3311 case Op_StoreD:
3312 case Op_LoadD:
3313 case Op_LoadD_unaligned:
3314 frc.inc_double_count();
3315 break;
3316 case Op_Opaque1: // Remove Opaque Nodes before matching
3317 n->subsume_by(n->in(1), this);
3318 break;
3319 case Op_CallLeafPure: {
3320 // If the pure call is not supported, then lower to a CallLeaf.
3321 if (!Matcher::match_rule_supported(Op_CallLeafPure)) {
3322 CallNode* call = n->as_Call();
3323 CallNode* new_call = new CallLeafNode(call->tf(), call->entry_point(),
3324 call->_name, TypeRawPtr::BOTTOM);
3325 new_call->init_req(TypeFunc::Control, call->in(TypeFunc::Control));
3326 new_call->init_req(TypeFunc::I_O, C->top());
3327 new_call->init_req(TypeFunc::Memory, C->top());
3328 new_call->init_req(TypeFunc::ReturnAdr, C->top());
3329 new_call->init_req(TypeFunc::FramePtr, C->top());
3330 for (unsigned int i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) {
3331 new_call->init_req(i, call->in(i));
3332 }
3333 n->subsume_by(new_call, this);
3334 }
3335 frc.inc_call_count();
3336 break;
3337 }
3338 case Op_CallStaticJava:
3339 case Op_CallJava:
3340 case Op_CallDynamicJava:
3341 frc.inc_java_call_count(); // Count java call site;
3342 case Op_CallRuntime:
3343 case Op_CallLeaf:
3344 case Op_CallLeafVector:
3345 case Op_CallLeafNoFP: {
3346 assert (n->is_Call(), "");
3347 CallNode *call = n->as_Call();
3348 // Count call sites where the FP mode bit would have to be flipped.
3349 // Do not count uncommon runtime calls:
3350 // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3356 int nop = n->Opcode();
3357 // Clone shared simple arguments to uncommon calls, item (1).
3358 if (n->outcnt() > 1 &&
3359 !n->is_Proj() &&
3360 nop != Op_CreateEx &&
3361 nop != Op_CheckCastPP &&
3362 nop != Op_DecodeN &&
3363 nop != Op_DecodeNKlass &&
3364 !n->is_Mem() &&
3365 !n->is_Phi()) {
3366 Node *x = n->clone();
3367 call->set_req(TypeFunc::Parms, x);
3368 }
3369 }
3370 break;
3371 }
3372 case Op_StoreB:
3373 case Op_StoreC:
3374 case Op_StoreI:
3375 case Op_StoreL:
3376 case Op_CompareAndSwapB:
3377 case Op_CompareAndSwapS:
3378 case Op_CompareAndSwapI:
3379 case Op_CompareAndSwapL:
3380 case Op_CompareAndSwapP:
3381 case Op_CompareAndSwapN:
3382 case Op_WeakCompareAndSwapB:
3383 case Op_WeakCompareAndSwapS:
3384 case Op_WeakCompareAndSwapI:
3385 case Op_WeakCompareAndSwapL:
3386 case Op_WeakCompareAndSwapP:
3387 case Op_WeakCompareAndSwapN:
3388 case Op_CompareAndExchangeB:
3389 case Op_CompareAndExchangeS:
3390 case Op_CompareAndExchangeI:
3391 case Op_CompareAndExchangeL:
3392 case Op_CompareAndExchangeP:
3393 case Op_CompareAndExchangeN:
3394 case Op_GetAndAddS:
3395 case Op_GetAndAddB:
3905 k->subsume_by(m, this);
3906 }
3907 }
3908 }
3909 break;
3910 }
3911 case Op_CmpUL: {
3912 if (!Matcher::has_match_rule(Op_CmpUL)) {
3913 // No support for unsigned long comparisons
3914 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
3915 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
3916 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
3917 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
3918 Node* andl = new AndLNode(orl, remove_sign_mask);
3919 Node* cmp = new CmpLNode(andl, n->in(2));
3920 n->subsume_by(cmp, this);
3921 }
3922 break;
3923 }
3924 #ifdef ASSERT
3925 case Op_ConNKlass: {
3926 const TypePtr* tp = n->as_Type()->type()->make_ptr();
3927 ciKlass* klass = tp->is_klassptr()->exact_klass();
3928 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
3929 break;
3930 }
3931 #endif
3932 default:
3933 assert(!n->is_Call(), "");
3934 assert(!n->is_Mem(), "");
3935 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
3936 break;
3937 }
3938 }
3939
3940 //------------------------------final_graph_reshaping_walk---------------------
3941 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
3942 // requires that the walk visits a node's inputs before visiting the node.
3943 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3944 Unique_Node_List sfpt;
4280 }
4281 }
4282
4283 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4284 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4285 }
4286
4287 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4288 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4289 }
4290
4291 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4292 if (holder->is_initialized()) {
4293 return false;
4294 }
4295 if (holder->is_being_initialized()) {
4296 if (accessing_method->holder() == holder) {
4297 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4298 // <init>, or a static method. In all those cases, there was an initialization
4299 // barrier on the holder klass passed.
4300 if (accessing_method->is_static_initializer() ||
4301 accessing_method->is_object_initializer() ||
4302 accessing_method->is_static()) {
4303 return false;
4304 }
4305 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4306 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4307 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4308 // child class can become fully initialized while its parent class is still being initialized.
4309 if (accessing_method->is_static_initializer()) {
4310 return false;
4311 }
4312 }
4313 ciMethod* root = method(); // the root method of compilation
4314 if (root != accessing_method) {
4315 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4316 }
4317 }
4318 return true;
4319 }
4320
4321 #ifndef PRODUCT
4322 //------------------------------verify_bidirectional_edges---------------------
4323 // For each input edge to a node (ie - for each Use-Def edge), verify that
4324 // there is a corresponding Def-Use edge.
4325 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4326 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4327 uint stack_size = live_nodes() >> 4;
4328 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4329 if (root_and_safepoints != nullptr) {
4359 if (in != nullptr && !in->is_top()) {
4360 // Count instances of `next`
4361 int cnt = 0;
4362 for (uint idx = 0; idx < in->_outcnt; idx++) {
4363 if (in->_out[idx] == n) {
4364 cnt++;
4365 }
4366 }
4367 assert(cnt > 0, "Failed to find Def-Use edge.");
4368 // Check for duplicate edges
4369 // walk the input array downcounting the input edges to n
4370 for (uint j = 0; j < length; j++) {
4371 if (n->in(j) == in) {
4372 cnt--;
4373 }
4374 }
4375 assert(cnt == 0, "Mismatched edge count.");
4376 } else if (in == nullptr) {
4377 assert(i == 0 || i >= n->req() ||
4378 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4379 (n->is_Unlock() && i == (n->req() - 1)) ||
4380 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4381 "only region, phi, arraycopy, unlock or membar nodes have null data edges");
4382 } else {
4383 assert(in->is_top(), "sanity");
4384 // Nothing to check.
4385 }
4386 }
4387 }
4388 }
4389
4390 //------------------------------verify_graph_edges---------------------------
4391 // Walk the Graph and verify that there is a one-to-one correspondence
4392 // between Use-Def edges and Def-Use edges in the graph.
4393 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4394 if (VerifyGraphEdges) {
4395 Unique_Node_List visited;
4396
4397 // Call graph walk to check edges
4398 verify_bidirectional_edges(visited, root_and_safepoints);
4399 if (no_dead_code) {
4400 // Now make sure that no visited node is used by an unvisited node.
4401 bool dead_nodes = false;
4512 // (1) subklass is already limited to a subtype of superklass => always ok
4513 // (2) subklass does not overlap with superklass => always fail
4514 // (3) superklass has NO subtypes and we can check with a simple compare.
4515 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
4516 if (skip) {
4517 return SSC_full_test; // Let caller generate the general case.
4518 }
4519
4520 if (subk->is_java_subtype_of(superk)) {
4521 return SSC_always_true; // (0) and (1) this test cannot fail
4522 }
4523
4524 if (!subk->maybe_java_subtype_of(superk)) {
4525 return SSC_always_false; // (2) true path dead; no dynamic test needed
4526 }
4527
4528 const Type* superelem = superk;
4529 if (superk->isa_aryklassptr()) {
4530 int ignored;
4531 superelem = superk->is_aryklassptr()->base_element_type(ignored);
4532 }
4533
4534 if (superelem->isa_instklassptr()) {
4535 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
4536 if (!ik->has_subklass()) {
4537 if (!ik->is_final()) {
4538 // Add a dependency if there is a chance of a later subclass.
4539 dependencies()->assert_leaf_type(ik);
4540 }
4541 if (!superk->maybe_java_subtype_of(subk)) {
4542 return SSC_always_false;
4543 }
4544 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4545 }
4546 } else {
4547 // A primitive array type has no subtypes.
4548 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4549 }
4550
4551 return SSC_full_test;
5350 } else {
5351 _debug_network_printer->update_compiled_method(C->method());
5352 }
5353 tty->print_cr("Method printed over network stream to IGV");
5354 _debug_network_printer->print(name, C->root(), visible_nodes, fr);
5355 }
5356 #endif // !PRODUCT
5357
5358 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5359 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5360 return value;
5361 }
5362 Node* result = nullptr;
5363 if (bt == T_BYTE) {
5364 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5365 result = new RShiftINode(result, phase->intcon(24));
5366 } else if (bt == T_BOOLEAN) {
5367 result = new AndINode(value, phase->intcon(0xFF));
5368 } else if (bt == T_CHAR) {
5369 result = new AndINode(value,phase->intcon(0xFFFF));
5370 } else {
5371 assert(bt == T_SHORT, "unexpected narrow type");
5372 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5373 result = new RShiftINode(result, phase->intcon(16));
5374 }
5375 if (transform_res) {
5376 result = phase->transform(result);
5377 }
5378 return result;
5379 }
5380
5381 void Compile::record_method_not_compilable_oom() {
5382 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5383 }
5384
5385 #ifndef PRODUCT
5386 // Collects all the control inputs from nodes on the worklist and from their data dependencies
5387 static void find_candidate_control_inputs(Unique_Node_List& worklist, Unique_Node_List& candidates) {
5388 // Follow non-control edges until we reach CFG nodes
5389 for (uint i = 0; i < worklist.size(); i++) {
|
1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "ci/ciFlatArray.hpp"
28 #include "ci/ciInlineKlass.hpp"
29 #include "ci/ciReplay.hpp"
30 #include "classfile/javaClasses.hpp"
31 #include "code/aotCodeCache.hpp"
32 #include "code/exceptionHandlerTable.hpp"
33 #include "code/nmethod.hpp"
34 #include "compiler/compilationFailureInfo.hpp"
35 #include "compiler/compilationMemoryStatistic.hpp"
36 #include "compiler/compileBroker.hpp"
37 #include "compiler/compileLog.hpp"
38 #include "compiler/compiler_globals.hpp"
39 #include "compiler/compilerDefinitions.hpp"
40 #include "compiler/compilerOracle.hpp"
41 #include "compiler/disassembler.hpp"
42 #include "compiler/oopMap.hpp"
43 #include "gc/shared/barrierSet.hpp"
44 #include "gc/shared/c2/barrierSetC2.hpp"
45 #include "jfr/jfrEvents.hpp"
46 #include "jvm_io.h"
47 #include "memory/allocation.hpp"
48 #include "memory/arena.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "opto/addnode.hpp"
51 #include "opto/block.hpp"
52 #include "opto/c2compiler.hpp"
53 #include "opto/callGenerator.hpp"
54 #include "opto/callnode.hpp"
55 #include "opto/castnode.hpp"
56 #include "opto/cfgnode.hpp"
57 #include "opto/chaitin.hpp"
58 #include "opto/compile.hpp"
59 #include "opto/connode.hpp"
60 #include "opto/convertnode.hpp"
61 #include "opto/divnode.hpp"
62 #include "opto/escape.hpp"
63 #include "opto/idealGraphPrinter.hpp"
64 #include "opto/inlinetypenode.hpp"
65 #include "opto/locknode.hpp"
66 #include "opto/loopnode.hpp"
67 #include "opto/machnode.hpp"
68 #include "opto/macro.hpp"
69 #include "opto/matcher.hpp"
70 #include "opto/mathexactnode.hpp"
71 #include "opto/memnode.hpp"
72 #include "opto/movenode.hpp"
73 #include "opto/mulnode.hpp"
74 #include "opto/multnode.hpp"
75 #include "opto/narrowptrnode.hpp"
76 #include "opto/node.hpp"
77 #include "opto/opaquenode.hpp"
78 #include "opto/opcodes.hpp"
79 #include "opto/output.hpp"
80 #include "opto/parse.hpp"
81 #include "opto/phaseX.hpp"
82 #include "opto/rootnode.hpp"
83 #include "opto/runtime.hpp"
84 #include "opto/stringopts.hpp"
85 #include "opto/type.hpp"
86 #include "opto/vector.hpp"
87 #include "opto/vectornode.hpp"
88 #include "runtime/arguments.hpp"
89 #include "runtime/globals_extension.hpp"
90 #include "runtime/sharedRuntime.hpp"
91 #include "runtime/signature.hpp"
92 #include "runtime/stubRoutines.hpp"
93 #include "runtime/timer.hpp"
94 #include "utilities/align.hpp"
95 #include "utilities/copy.hpp"
96 #include "utilities/hashTable.hpp"
97 #include "utilities/macros.hpp"
98
99 // -------------------- Compile::mach_constant_base_node -----------------------
100 // Constant table base node singleton.
101 MachConstantBaseNode* Compile::mach_constant_base_node() {
102 if (_mach_constant_base_node == nullptr) {
103 _mach_constant_base_node = new MachConstantBaseNode();
104 _mach_constant_base_node->add_req(C->root());
105 }
106 return _mach_constant_base_node;
107 }
108
394 // as dead to be conservative about the dead node count at any
395 // given time.
396 if (!dead->is_Con()) {
397 record_dead_node(dead->_idx);
398 }
399 if (dead->is_macro()) {
400 remove_macro_node(dead);
401 }
402 if (dead->is_expensive()) {
403 remove_expensive_node(dead);
404 }
405 if (dead->is_OpaqueTemplateAssertionPredicate()) {
406 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
407 }
408 if (dead->is_ParsePredicate()) {
409 remove_parse_predicate(dead->as_ParsePredicate());
410 }
411 if (dead->for_post_loop_opts_igvn()) {
412 remove_from_post_loop_opts_igvn(dead);
413 }
414 if (dead->is_InlineType()) {
415 remove_inline_type(dead);
416 }
417 if (dead->is_LoadFlat() || dead->is_StoreFlat()) {
418 remove_flat_access(dead);
419 }
420 if (dead->for_merge_stores_igvn()) {
421 remove_from_merge_stores_igvn(dead);
422 }
423 if (dead->is_Call()) {
424 remove_useless_late_inlines( &_late_inlines, dead);
425 remove_useless_late_inlines( &_string_late_inlines, dead);
426 remove_useless_late_inlines( &_boxing_late_inlines, dead);
427 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
428
429 if (dead->is_CallStaticJava()) {
430 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
431 }
432 }
433 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
434 bs->unregister_potential_barrier_node(dead);
435 }
436
437 // Disconnect all useless nodes by disconnecting those at the boundary.
438 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
439 uint next = 0;
447 // Use raw traversal of out edges since this code removes out edges
448 int max = n->outcnt();
449 for (int j = 0; j < max; ++j) {
450 Node* child = n->raw_out(j);
451 if (!useful.member(child)) {
452 assert(!child->is_top() || child != top(),
453 "If top is cached in Compile object it is in useful list");
454 // Only need to remove this out-edge to the useless node
455 n->raw_del_out(j);
456 --j;
457 --max;
458 if (child->is_data_proj_of_pure_function(n)) {
459 worklist.push(n);
460 }
461 }
462 }
463 if (n->outcnt() == 1 && n->has_special_unique_user()) {
464 assert(useful.member(n->unique_out()), "do not push a useless node");
465 worklist.push(n->unique_out());
466 }
467 if (n->outcnt() == 0) {
468 worklist.push(n);
469 }
470 }
471
472 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
473 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
474 // Remove useless Template Assertion Predicate opaque nodes
475 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
476 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
477 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
478 remove_useless_nodes(_inline_type_nodes, useful); // remove useless inline type nodes
479 remove_useless_nodes(_flat_access_nodes, useful); // remove useless flat access nodes
480 #ifdef ASSERT
481 if (_modified_nodes != nullptr) {
482 _modified_nodes->remove_useless_nodes(useful.member_set());
483 }
484 #endif
485 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
486 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
487 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
488 #ifdef ASSERT
489 if (_modified_nodes != nullptr) {
490 _modified_nodes->remove_useless_nodes(useful.member_set());
491 }
492 #endif
493
494 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
495 bs->eliminate_useless_gc_barriers(useful, this);
496 // clean up the late inline lists
497 remove_useless_late_inlines( &_late_inlines, useful);
498 remove_useless_late_inlines( &_string_late_inlines, useful);
499 remove_useless_late_inlines( &_boxing_late_inlines, useful);
500 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
501 DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
502 }
503
504 // ============================================================================
651
652
653 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
654 Options options, DirectiveSet* directive)
655 : Phase(Compiler),
656 _compile_id(ci_env->compile_id()),
657 _options(options),
658 _method(target),
659 _entry_bci(osr_bci),
660 _ilt(nullptr),
661 _stub_function(nullptr),
662 _stub_name(nullptr),
663 _stub_id(StubId::NO_STUBID),
664 _stub_entry_point(nullptr),
665 _max_node_limit(MaxNodeLimit),
666 _post_loop_opts_phase(false),
667 _merge_stores_phase(false),
668 _allow_macro_nodes(true),
669 _inlining_progress(false),
670 _inlining_incrementally(false),
671 _strength_reduction(false),
672 _do_cleanup(false),
673 _has_reserved_stack_access(target->has_reserved_stack_access()),
674 _has_circular_inline_type(false),
675 #ifndef PRODUCT
676 _igv_idx(0),
677 _trace_opto_output(directive->TraceOptoOutputOption),
678 #endif
679 _clinit_barrier_on_entry(false),
680 _stress_seed(0),
681 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
682 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
683 _env(ci_env),
684 _directive(directive),
685 _log(ci_env->log()),
686 _first_failure_details(nullptr),
687 _intrinsics(comp_arena(), 0, 0, nullptr),
688 _macro_nodes(comp_arena(), 8, 0, nullptr),
689 _parse_predicates(comp_arena(), 8, 0, nullptr),
690 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
691 _expensive_nodes(comp_arena(), 8, 0, nullptr),
692 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
693 _inline_type_nodes (comp_arena(), 8, 0, nullptr),
694 _flat_access_nodes(comp_arena(), 8, 0, nullptr),
695 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
696 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
697 _coarsened_locks(comp_arena(), 8, 0, nullptr),
698 _congraph(nullptr),
699 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
700 _unique(0),
701 _dead_node_count(0),
702 _dead_node_list(comp_arena()),
703 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
704 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
705 _node_arena(&_node_arena_one),
706 _mach_constant_base_node(nullptr),
707 _Compile_types(mtCompiler, Arena::Tag::tag_type),
708 _initial_gvn(nullptr),
709 _igvn_worklist(nullptr),
710 _types(nullptr),
711 _node_hash(nullptr),
712 _late_inlines(comp_arena(), 2, 0, nullptr),
713 _string_late_inlines(comp_arena(), 2, 0, nullptr),
714 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
783 #define MINIMUM_NODE_HASH 1023
784
785 // GVN that will be run immediately on new nodes
786 uint estimated_size = method()->code_size()*4+64;
787 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
788 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
789 _types = new (comp_arena()) Type_Array(comp_arena());
790 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
791 PhaseGVN gvn;
792 set_initial_gvn(&gvn);
793
794 { // Scope for timing the parser
795 TracePhase tp(_t_parser);
796
797 // Put top into the hash table ASAP.
798 initial_gvn()->transform(top());
799
800 // Set up tf(), start(), and find a CallGenerator.
801 CallGenerator* cg = nullptr;
802 if (is_osr_compilation()) {
803 init_tf(TypeFunc::make(method(), /* is_osr_compilation = */ true));
804 StartNode* s = new StartOSRNode(root(), tf()->domain_sig());
805 initial_gvn()->set_type_bottom(s);
806 verify_start(s);
807 cg = CallGenerator::for_osr(method(), entry_bci());
808 } else {
809 // Normal case.
810 init_tf(TypeFunc::make(method()));
811 StartNode* s = new StartNode(root(), tf()->domain_cc());
812 initial_gvn()->set_type_bottom(s);
813 verify_start(s);
814 float past_uses = method()->interpreter_invocation_count();
815 float expected_uses = past_uses;
816 cg = CallGenerator::for_inline(method(), expected_uses);
817 }
818 if (failing()) return;
819 if (cg == nullptr) {
820 const char* reason = InlineTree::check_can_parse(method());
821 assert(reason != nullptr, "expect reason for parse failure");
822 stringStream ss;
823 ss.print("cannot parse method: %s", reason);
824 record_method_not_compilable(ss.as_string());
825 return;
826 }
827
828 gvn.set_type(root(), root()->bottom_type());
829
830 JVMState* jvms = build_start_state(start(), tf());
831 if ((jvms = cg->generate(jvms)) == nullptr) {
892 print_ideal_ir("print_ideal");
893 }
894 #endif
895
896 #ifdef ASSERT
897 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
898 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
899 #endif
900
901 // Dump compilation data to replay it.
902 if (directive->DumpReplayOption) {
903 env()->dump_replay_data(_compile_id);
904 }
905 if (directive->DumpInlineOption && (ilt() != nullptr)) {
906 env()->dump_inline_data(_compile_id);
907 }
908
909 // Now that we know the size of all the monitors we can add a fixed slot
910 // for the original deopt pc.
911 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
912 if (needs_stack_repair()) {
913 // One extra slot for the special stack increment value
914 next_slot += 2;
915 }
916 // TODO 8284443 Only reserve extra slot if needed
917 if (InlineTypeReturnedAsFields) {
918 // One extra slot to hold the null marker for a nullable
919 // inline type return if we run out of registers.
920 next_slot += 2;
921 }
922 set_fixed_slots(next_slot);
923
924 // Compute when to use implicit null checks. Used by matching trap based
925 // nodes and NullCheck optimization.
926 set_allowed_deopt_reasons();
927
928 // Now generate code
929 Code_Gen();
930 }
931
932 //------------------------------Compile----------------------------------------
933 // Compile a runtime stub
934 Compile::Compile(ciEnv* ci_env,
935 TypeFunc_generator generator,
936 address stub_function,
937 const char* stub_name,
938 StubId stub_id,
939 int is_fancy_jump,
940 bool pass_tls,
941 bool return_pc,
942 DirectiveSet* directive)
943 : Phase(Compiler),
944 _compile_id(0),
945 _options(Options::for_runtime_stub()),
946 _method(nullptr),
947 _entry_bci(InvocationEntryBci),
948 _stub_function(stub_function),
949 _stub_name(stub_name),
950 _stub_id(stub_id),
951 _stub_entry_point(nullptr),
952 _max_node_limit(MaxNodeLimit),
953 _post_loop_opts_phase(false),
954 _merge_stores_phase(false),
955 _allow_macro_nodes(true),
956 _inlining_progress(false),
957 _inlining_incrementally(false),
958 _has_reserved_stack_access(false),
959 _has_circular_inline_type(false),
960 #ifndef PRODUCT
961 _igv_idx(0),
962 _trace_opto_output(directive->TraceOptoOutputOption),
963 #endif
964 _clinit_barrier_on_entry(false),
965 _stress_seed(0),
966 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
967 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
968 _env(ci_env),
969 _directive(directive),
970 _log(ci_env->log()),
971 _first_failure_details(nullptr),
972 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
973 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
974 _congraph(nullptr),
975 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
976 _unique(0),
977 _dead_node_count(0),
978 _dead_node_list(comp_arena()),
979 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
1094 _fixed_slots = 0;
1095 set_has_split_ifs(false);
1096 set_has_loops(false); // first approximation
1097 set_has_stringbuilder(false);
1098 set_has_boxed_value(false);
1099 _trap_can_recompile = false; // no traps emitted yet
1100 _major_progress = true; // start out assuming good things will happen
1101 set_has_unsafe_access(false);
1102 set_max_vector_size(0);
1103 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1104 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1105 set_decompile_count(0);
1106
1107 #ifndef PRODUCT
1108 _phase_counter = 0;
1109 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1110 #endif
1111
1112 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1113 _loop_opts_cnt = LoopOptsCount;
1114 _has_flat_accesses = false;
1115 _flat_accesses_share_alias = true;
1116 _scalarize_in_safepoints = false;
1117
1118 set_do_inlining(Inline);
1119 set_max_inline_size(MaxInlineSize);
1120 set_freq_inline_size(FreqInlineSize);
1121 set_do_scheduling(OptoScheduling);
1122
1123 set_do_vector_loop(false);
1124 set_has_monitors(false);
1125 set_has_scoped_access(false);
1126
1127 if (AllowVectorizeOnDemand) {
1128 if (has_method() && _directive->VectorizeOption) {
1129 set_do_vector_loop(true);
1130 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1131 } else if (has_method() && method()->name() != nullptr &&
1132 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1133 set_do_vector_loop(true);
1134 }
1135 }
1136 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1137 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1378 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1379 assert(do_aliasing(), "Aliasing should be enabled");
1380 int offset = tj->offset();
1381 TypePtr::PTR ptr = tj->ptr();
1382
1383 // Known instance (scalarizable allocation) alias only with itself.
1384 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1385 tj->is_oopptr()->is_known_instance();
1386
1387 // Process weird unsafe references.
1388 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1389 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1390 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1391 tj = TypeOopPtr::BOTTOM;
1392 ptr = tj->ptr();
1393 offset = tj->offset();
1394 }
1395
1396 // Array pointers need some flattening
1397 const TypeAryPtr* ta = tj->isa_aryptr();
1398 if( ta && is_known_inst ) {
1399 if ( offset != Type::OffsetBot &&
1400 offset > arrayOopDesc::length_offset_in_bytes() ) {
1401 offset = Type::OffsetBot; // Flatten constant access into array body only
1402 tj = ta = ta->
1403 remove_speculative()->
1404 cast_to_ptr_type(ptr)->
1405 with_offset(offset);
1406 }
1407 } else if (ta) {
1408 // Common slices
1409 if (offset == arrayOopDesc::length_offset_in_bytes()) {
1410 return TypeAryPtr::RANGE;
1411 } else if (offset == oopDesc::klass_offset_in_bytes()) {
1412 return TypeInstPtr::KLASS;
1413 } else if (offset == oopDesc::mark_offset_in_bytes()) {
1414 return TypeInstPtr::MARK;
1415 }
1416
1417 // Remove size and stability
1418 const TypeAry* normalized_ary = TypeAry::make(ta->elem(), TypeInt::POS, false, ta->is_flat(), ta->is_not_flat(), ta->is_not_null_free(), ta->is_atomic());
1419 // Remove ptr, const_oop, and offset
1420 if (ta->elem() == Type::BOTTOM) {
1421 // Bottom array (meet of int[] and byte[] for example), accesses to it will be done with
1422 // Unsafe. This should alias with all arrays. For now just leave it as it is (this is
1423 // incorrect, see JDK-8331133).
1424 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, false, Type::Offset::bottom);
1425 } else if (ta->elem()->make_oopptr() != nullptr) {
1426 // Object arrays, keep field_offset
1427 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, nullptr, ta->klass_is_exact(), Type::Offset::bottom, Type::Offset(ta->field_offset()));
1428 } else {
1429 // Primitive arrays
1430 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, normalized_ary, ta->exact_klass(), true, Type::Offset::bottom);
1431 }
1432
1433 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1434 // cannot be distinguished by bytecode alone.
1435 if (ta->elem() == TypeInt::BOOL) {
1436 tj = ta = TypeAryPtr::BYTES;
1437 }
1438
1439 // All arrays of references share the same slice
1440 if (!ta->is_flat() && ta->elem()->make_oopptr() != nullptr) {
1441 const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, TypeInt::POS, false, false, true, true, true);
1442 tj = ta = TypeAryPtr::make(TypePtr::BotPTR, nullptr, tary, nullptr, false, Type::Offset::bottom);
1443 }
1444
1445 if (ta->is_flat()) {
1446 if (_flat_accesses_share_alias) {
1447 // Initially all flattened array accesses share a single slice
1448 tj = ta = TypeAryPtr::INLINES;
1449 } else {
1450 // Flat accesses are always exact
1451 tj = ta = ta->cast_to_exactness(true);
1452 }
1453 }
1454 }
1455
1456 // Oop pointers need some flattening
1457 const TypeInstPtr *to = tj->isa_instptr();
1458 if (to && to != TypeOopPtr::BOTTOM) {
1459 ciInstanceKlass* ik = to->instance_klass();
1460 tj = to = to->cast_to_maybe_flat_in_array(); // flatten to maybe flat in array
1461 if( ptr == TypePtr::Constant ) {
1462 if (ik != ciEnv::current()->Class_klass() ||
1463 offset < ik->layout_helper_size_in_bytes()) {
1464 // No constant oop pointers (such as Strings); they alias with
1465 // unknown strings.
1466 assert(!is_known_inst, "not scalarizable allocation");
1467 tj = to = to->
1468 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1469 remove_speculative()->
1470 cast_to_ptr_type(TypePtr::BotPTR)->
1471 cast_to_exactness(false);
1472 }
1473 } else if( is_known_inst ) {
1474 tj = to; // Keep NotNull and klass_is_exact for instance type
1475 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1476 // During the 2nd round of IterGVN, NotNull castings are removed.
1477 // Make sure the Bottom and NotNull variants alias the same.
1478 // Also, make sure exact and non-exact variants alias the same.
1479 tj = to = to->
1480 remove_speculative()->
1481 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1482 cast_to_ptr_type(TypePtr::BotPTR)->
1483 cast_to_exactness(false);
1484 }
1485 if (to->speculative() != nullptr) {
1486 tj = to = to->remove_speculative();
1487 }
1488 // Canonicalize the holder of this field
1489 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1490 // First handle header references such as a LoadKlassNode, even if the
1491 // object's klass is unloaded at compile time (4965979).
1492 if (!is_known_inst) { // Do it only for non-instance types
1493 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, Type::Offset(offset));
1494 }
1495 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1496 // Static fields are in the space above the normal instance
1497 // fields in the java.lang.Class instance.
1498 if (ik != ciEnv::current()->Class_klass()) {
1499 to = nullptr;
1500 tj = TypeOopPtr::BOTTOM;
1501 offset = tj->offset();
1502 }
1503 } else {
1504 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1505 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1506 assert(tj->offset() == offset, "no change to offset expected");
1507 bool xk = to->klass_is_exact();
1508 int instance_id = to->instance_id();
1509
1510 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1511 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1512 // its interfaces are included.
1513 if (xk && ik->equals(canonical_holder)) {
1514 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id,
1515 TypePtr::MaybeFlat), "exact type should be canonical type");
1516 } else {
1517 assert(xk || !is_known_inst, "Known instance should be exact type");
1518 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id,
1519 TypePtr::MaybeFlat);
1520 }
1521 }
1522 }
1523
1524 // Klass pointers to object array klasses need some flattening
1525 const TypeKlassPtr *tk = tj->isa_klassptr();
1526 if( tk ) {
1527 // If we are referencing a field within a Klass, we need
1528 // to assume the worst case of an Object. Both exact and
1529 // inexact types must flatten to the same alias class so
1530 // use NotNull as the PTR.
1531 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1532 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1533 env()->Object_klass(),
1534 Type::Offset(offset),
1535 TypePtr::MaybeFlat);
1536 }
1537
1538 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1539 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1540 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1541 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), Type::Offset(offset), TypePtr::MaybeFlat);
1542 } else {
1543 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, Type::Offset(offset), tk->is_not_flat(), tk->is_not_null_free(), tk->is_flat(), tk->is_null_free(), tk->is_atomic(), tk->is_aryklassptr()->is_refined_type());
1544 }
1545 }
1546 // Check for precise loads from the primary supertype array and force them
1547 // to the supertype cache alias index. Check for generic array loads from
1548 // the primary supertype array and also force them to the supertype cache
1549 // alias index. Since the same load can reach both, we need to merge
1550 // these 2 disparate memories into the same alias class. Since the
1551 // primary supertype array is read-only, there's no chance of confusion
1552 // where we bypass an array load and an array store.
1553 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1554 if (offset == Type::OffsetBot ||
1555 (offset >= primary_supers_offset &&
1556 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1557 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1558 offset = in_bytes(Klass::secondary_super_cache_offset());
1559 tj = tk = tk->with_offset(offset);
1560 }
1561 }
1562
1563 // Flatten all Raw pointers together.
1564 if (tj->base() == Type::RawPtr)
1565 tj = TypeRawPtr::BOTTOM;
1655 intptr_t key = (intptr_t) adr_type;
1656 key ^= key >> logAliasCacheSize;
1657 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1658 }
1659
1660
1661 //-----------------------------grow_alias_types--------------------------------
1662 void Compile::grow_alias_types() {
1663 const int old_ats = _max_alias_types; // how many before?
1664 const int new_ats = old_ats; // how many more?
1665 const int grow_ats = old_ats+new_ats; // how many now?
1666 _max_alias_types = grow_ats;
1667 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1668 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1669 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1670 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1671 }
1672
1673
1674 //--------------------------------find_alias_type------------------------------
1675 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field, bool uncached) {
1676 if (!do_aliasing()) {
1677 return alias_type(AliasIdxBot);
1678 }
1679
1680 AliasCacheEntry* ace = nullptr;
1681 if (!uncached) {
1682 ace = probe_alias_cache(adr_type);
1683 if (ace->_adr_type == adr_type) {
1684 return alias_type(ace->_index);
1685 }
1686 }
1687
1688 // Handle special cases.
1689 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1690 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1691
1692 // Do it the slow way.
1693 const TypePtr* flat = flatten_alias_type(adr_type);
1694
1695 #ifdef ASSERT
1696 {
1697 ResourceMark rm;
1698 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1699 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1700 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1701 Type::str(adr_type));
1702 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1703 const TypeOopPtr* foop = flat->is_oopptr();
1704 // Scalarizable allocations have exact klass always.
1705 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1715 if (alias_type(i)->adr_type() == flat) {
1716 idx = i;
1717 break;
1718 }
1719 }
1720
1721 if (idx == AliasIdxTop) {
1722 if (no_create) return nullptr;
1723 // Grow the array if necessary.
1724 if (_num_alias_types == _max_alias_types) grow_alias_types();
1725 // Add a new alias type.
1726 idx = _num_alias_types++;
1727 _alias_types[idx]->Init(idx, flat);
1728 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1729 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1730 if (flat->isa_instptr()) {
1731 if (flat->offset() == java_lang_Class::klass_offset()
1732 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1733 alias_type(idx)->set_rewritable(false);
1734 }
1735 ciField* field = nullptr;
1736 if (flat->isa_aryptr()) {
1737 #ifdef ASSERT
1738 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1739 // (T_BYTE has the weakest alignment and size restrictions...)
1740 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1741 #endif
1742 const Type* elemtype = flat->is_aryptr()->elem();
1743 if (flat->offset() == TypePtr::OffsetBot) {
1744 alias_type(idx)->set_element(elemtype);
1745 }
1746 int field_offset = flat->is_aryptr()->field_offset().get();
1747 if (flat->is_flat() &&
1748 field_offset != Type::OffsetBot) {
1749 ciInlineKlass* vk = elemtype->inline_klass();
1750 field_offset += vk->payload_offset();
1751 field = vk->get_field_by_offset(field_offset, false);
1752 }
1753 }
1754 if (flat->isa_klassptr()) {
1755 if (UseCompactObjectHeaders) {
1756 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1757 alias_type(idx)->set_rewritable(false);
1758 }
1759 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1760 alias_type(idx)->set_rewritable(false);
1761 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1762 alias_type(idx)->set_rewritable(false);
1763 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1764 alias_type(idx)->set_rewritable(false);
1765 if (flat->offset() == in_bytes(Klass::layout_helper_offset()))
1766 alias_type(idx)->set_rewritable(false);
1767 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1768 alias_type(idx)->set_rewritable(false);
1769 }
1770
1771 if (flat->isa_instklassptr()) {
1772 if (flat->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
1773 alias_type(idx)->set_rewritable(false);
1774 }
1775 }
1776 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1777 // but the base pointer type is not distinctive enough to identify
1778 // references into JavaThread.)
1779
1780 // Check for final fields.
1781 const TypeInstPtr* tinst = flat->isa_instptr();
1782 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1783 if (tinst->const_oop() != nullptr &&
1784 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1785 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1786 // static field
1787 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1788 field = k->get_field_by_offset(tinst->offset(), true);
1789 } else if (tinst->is_inlinetypeptr()) {
1790 // Inline type field
1791 ciInlineKlass* vk = tinst->inline_klass();
1792 field = vk->get_field_by_offset(tinst->offset(), false);
1793 } else {
1794 ciInstanceKlass *k = tinst->instance_klass();
1795 field = k->get_field_by_offset(tinst->offset(), false);
1796 }
1797 }
1798 assert(field == nullptr ||
1799 original_field == nullptr ||
1800 (field->holder() == original_field->holder() &&
1801 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1802 field->is_static() == original_field->is_static()), "wrong field?");
1803 // Set field() and is_rewritable() attributes.
1804 if (field != nullptr) {
1805 alias_type(idx)->set_field(field);
1806 if (flat->isa_aryptr()) {
1807 // Fields of flat arrays are rewritable although they are declared final
1808 assert(flat->is_flat(), "must be a flat array");
1809 alias_type(idx)->set_rewritable(true);
1810 }
1811 }
1812 }
1813
1814 // Fill the cache for next time.
1815 if (!uncached) {
1816 ace->_adr_type = adr_type;
1817 ace->_index = idx;
1818 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1819
1820 // Might as well try to fill the cache for the flattened version, too.
1821 AliasCacheEntry* face = probe_alias_cache(flat);
1822 if (face->_adr_type == nullptr) {
1823 face->_adr_type = flat;
1824 face->_index = idx;
1825 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1826 }
1827 }
1828
1829 return alias_type(idx);
1830 }
1831
1832
1833 Compile::AliasType* Compile::alias_type(ciField* field) {
1834 const TypeOopPtr* t;
1835 if (field->is_static())
1836 t = TypeInstPtr::make(field->holder()->java_mirror());
1837 else
1838 t = TypeOopPtr::make_from_klass_raw(field->holder());
1839 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1840 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1841 return atp;
1842 }
1843
1844
1845 //------------------------------have_alias_type--------------------------------
1846 bool Compile::have_alias_type(const TypePtr* adr_type) {
1928 assert(!C->major_progress(), "not cleared");
1929
1930 if (_for_post_loop_igvn.length() > 0) {
1931 while (_for_post_loop_igvn.length() > 0) {
1932 Node* n = _for_post_loop_igvn.pop();
1933 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1934 igvn._worklist.push(n);
1935 }
1936 igvn.optimize();
1937 if (failing()) return;
1938 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1939 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1940
1941 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1942 if (C->major_progress()) {
1943 C->clear_major_progress(); // ensure that major progress is now clear
1944 }
1945 }
1946 }
1947
1948 void Compile::add_inline_type(Node* n) {
1949 assert(n->is_InlineType(), "unexpected node");
1950 _inline_type_nodes.push(n);
1951 }
1952
1953 void Compile::remove_inline_type(Node* n) {
1954 assert(n->is_InlineType(), "unexpected node");
1955 if (_inline_type_nodes.contains(n)) {
1956 _inline_type_nodes.remove(n);
1957 }
1958 }
1959
1960 // Does the return value keep otherwise useless inline type allocations alive?
1961 static bool return_val_keeps_allocations_alive(Node* ret_val) {
1962 ResourceMark rm;
1963 Unique_Node_List wq;
1964 wq.push(ret_val);
1965 bool some_allocations = false;
1966 for (uint i = 0; i < wq.size(); i++) {
1967 Node* n = wq.at(i);
1968 if (n->outcnt() > 1) {
1969 // Some other use for the allocation
1970 return false;
1971 } else if (n->is_InlineType()) {
1972 wq.push(n->in(1));
1973 } else if (n->is_Phi()) {
1974 for (uint j = 1; j < n->req(); j++) {
1975 wq.push(n->in(j));
1976 }
1977 } else if (n->is_CheckCastPP() &&
1978 n->in(1)->is_Proj() &&
1979 n->in(1)->in(0)->is_Allocate()) {
1980 some_allocations = true;
1981 } else if (n->is_CheckCastPP()) {
1982 wq.push(n->in(1));
1983 }
1984 }
1985 return some_allocations;
1986 }
1987
1988 void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
1989 // Make sure that the return value does not keep an otherwise unused allocation alive
1990 if (tf()->returns_inline_type_as_fields()) {
1991 Node* ret = nullptr;
1992 for (uint i = 1; i < root()->req(); i++) {
1993 Node* in = root()->in(i);
1994 if (in->Opcode() == Op_Return) {
1995 assert(ret == nullptr, "only one return");
1996 ret = in;
1997 }
1998 }
1999 if (ret != nullptr) {
2000 Node* ret_val = ret->in(TypeFunc::Parms);
2001 if (igvn.type(ret_val)->isa_oopptr() &&
2002 return_val_keeps_allocations_alive(ret_val)) {
2003 igvn.replace_input_of(ret, TypeFunc::Parms, InlineTypeNode::tagged_klass(igvn.type(ret_val)->inline_klass(), igvn));
2004 assert(ret_val->outcnt() == 0, "should be dead now");
2005 igvn.remove_dead_node(ret_val);
2006 }
2007 }
2008 }
2009 if (_inline_type_nodes.length() == 0) {
2010 // keep the graph canonical
2011 igvn.optimize();
2012 return;
2013 }
2014 // Scalarize inline types in safepoint debug info.
2015 // Delay this until all inlining is over to avoid getting inconsistent debug info.
2016 set_scalarize_in_safepoints(true);
2017 for (int i = _inline_type_nodes.length()-1; i >= 0; i--) {
2018 InlineTypeNode* vt = _inline_type_nodes.at(i)->as_InlineType();
2019 vt->make_scalar_in_safepoints(&igvn);
2020 igvn.record_for_igvn(vt);
2021 }
2022 if (remove) {
2023 // Remove inline type nodes by replacing them with their oop input
2024 while (_inline_type_nodes.length() > 0) {
2025 InlineTypeNode* vt = _inline_type_nodes.pop()->as_InlineType();
2026 if (vt->outcnt() == 0) {
2027 igvn.remove_dead_node(vt);
2028 continue;
2029 }
2030 for (DUIterator i = vt->outs(); vt->has_out(i); i++) {
2031 DEBUG_ONLY(bool must_be_buffered = false);
2032 Node* u = vt->out(i);
2033 // Check if any users are blackholes. If so, rewrite them to use either the
2034 // allocated buffer, or individual components, instead of the inline type node
2035 // that goes away.
2036 if (u->is_Blackhole()) {
2037 BlackholeNode* bh = u->as_Blackhole();
2038
2039 // Unlink the old input
2040 int idx = bh->find_edge(vt);
2041 assert(idx != -1, "The edge should be there");
2042 bh->del_req(idx);
2043 --i;
2044
2045 if (vt->is_allocated(&igvn)) {
2046 // Already has the allocated instance, blackhole that
2047 bh->add_req(vt->get_oop());
2048 } else {
2049 // Not allocated yet, blackhole the components
2050 for (uint c = 0; c < vt->field_count(); c++) {
2051 bh->add_req(vt->field_value(c));
2052 }
2053 }
2054
2055 // Node modified, record for IGVN
2056 igvn.record_for_igvn(bh);
2057 }
2058 #ifdef ASSERT
2059 // Verify that inline type is buffered when replacing by oop
2060 else if (u->is_InlineType()) {
2061 // InlineType uses don't need buffering because they are about to be replaced as well
2062 } else if (u->is_Phi()) {
2063 // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through
2064 } else {
2065 must_be_buffered = true;
2066 }
2067 if (must_be_buffered && !vt->is_allocated(&igvn)) {
2068 vt->dump(0);
2069 u->dump(0);
2070 assert(false, "Should have been buffered");
2071 }
2072 #endif
2073 }
2074 igvn.replace_node(vt, vt->get_oop());
2075 }
2076 }
2077 igvn.optimize();
2078 }
2079
2080 void Compile::add_flat_access(Node* n) {
2081 assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2082 assert(!_flat_access_nodes.contains(n), "duplicate insertion");
2083 _flat_access_nodes.push(n);
2084 }
2085
2086 void Compile::remove_flat_access(Node* n) {
2087 assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2088 _flat_access_nodes.remove_if_existing(n);
2089 }
2090
2091 void Compile::process_flat_accesses(PhaseIterGVN& igvn) {
2092 assert(igvn._worklist.size() == 0, "should be empty");
2093 igvn.set_delay_transform(true);
2094 for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) {
2095 Node* n = _flat_access_nodes.at(i);
2096 assert(n != nullptr, "unexpected nullptr");
2097 if (n->is_LoadFlat()) {
2098 LoadFlatNode* loadn = n->as_LoadFlat();
2099 // Expending a flat load atomically means that we get a chunk of memory spanning multiple fields
2100 // that we chop with bitwise operations. That is too subtle for some optimizations, especially
2101 // constant folding when fields are constant. But if the flattened field being accessed is read-only
2102 // then no concurrent writes can happen and non-atomic loads are fine, allowing better optimizations.
2103 // A way for fields to be read-only is to be stable and already initialized. Here, we check if the
2104 // field being accessed is stable, and if the null marker of the field/array element is non-zero.
2105 // If so, we know that the stable value was initialized away from the default value (null), and
2106 // that we can assume it's read-only, so can the load can be performed non-atomically.
2107 bool non_atomic_is_fine = false;
2108 if (FoldStableValues) {
2109 const TypeOopPtr* base_type = igvn.type(loadn->base())->isa_oopptr();
2110 ciObject* oop = base_type->const_oop();
2111 ciInstance* holder = oop != nullptr && oop->is_instance() ? oop->as_instance() : nullptr;
2112 ciArray* array = oop != nullptr && oop->is_array() ? oop->as_array() : nullptr;
2113 int off = igvn.type(loadn->ptr())->isa_ptr()->offset();
2114
2115 if (holder != nullptr) {
2116 ciKlass* klass = holder->klass();
2117 ciInstanceKlass* iklass = klass->as_instance_klass();
2118 const ciField* field = iklass->get_non_flat_field_by_offset(off);
2119 ciField* nm_field = iklass->get_field_by_offset(field->null_marker_offset(), false);
2120 ciConstant cst = nm_field != nullptr ? holder->field_value(nm_field) : ciConstant() /* invalid */;
2121 non_atomic_is_fine = field->is_stable() && cst.is_valid() && cst.as_boolean();
2122 } else if (array != nullptr) {
2123 const TypeAryPtr* aryptr = base_type->is_aryptr();
2124 ciConstant elt = ((ciFlatArray*)array)->null_marker_of_element_by_offset(off);
2125 non_atomic_is_fine = aryptr->is_stable() && elt.is_valid() && !elt.is_null_or_zero();
2126 }
2127 }
2128
2129 if (non_atomic_is_fine) {
2130 loadn->expand_non_atomic(igvn);
2131 } else {
2132 loadn->expand_atomic(igvn);
2133 }
2134 } else {
2135 n->as_StoreFlat()->expand_atomic(igvn);
2136 }
2137 }
2138 _flat_access_nodes.clear_and_deallocate();
2139 igvn.set_delay_transform(false);
2140 igvn.optimize();
2141 }
2142
2143 void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) {
2144 DEBUG_ONLY(igvn.verify_empty_worklist(nullptr));
2145 if (!_has_flat_accesses) {
2146 return;
2147 }
2148 // Initially, all flat array accesses share the same slice to
2149 // keep dependencies with Object[] array accesses (that could be
2150 // to a flat array) correct. We're done with parsing so we
2151 // now know all flat array accesses in this compile
2152 // unit. Let's move flat array accesses to their own slice,
2153 // one per element field. This should help memory access
2154 // optimizations.
2155 ResourceMark rm;
2156 Unique_Node_List wq;
2157 wq.push(root());
2158
2159 Node_List mergememnodes;
2160 Node_List memnodes;
2161
2162 // Alias index currently shared by all flat memory accesses
2163 int index = get_alias_index(TypeAryPtr::INLINES);
2164
2165 // Find MergeMem nodes and flat array accesses
2166 for (uint i = 0; i < wq.size(); i++) {
2167 Node* n = wq.at(i);
2168 if (n->is_Mem()) {
2169 const TypePtr* adr_type = nullptr;
2170 adr_type = get_adr_type(get_alias_index(n->adr_type()));
2171 if (adr_type == TypeAryPtr::INLINES) {
2172 memnodes.push(n);
2173 }
2174 } else if (n->is_MergeMem()) {
2175 MergeMemNode* mm = n->as_MergeMem();
2176 if (mm->memory_at(index) != mm->base_memory()) {
2177 mergememnodes.push(n);
2178 }
2179 }
2180 for (uint j = 0; j < n->req(); j++) {
2181 Node* m = n->in(j);
2182 if (m != nullptr) {
2183 wq.push(m);
2184 }
2185 }
2186 }
2187
2188 _flat_accesses_share_alias = false;
2189
2190 // We are going to change the slice for the flat array
2191 // accesses so we need to clear the cache entries that refer to
2192 // them.
2193 for (uint i = 0; i < AliasCacheSize; i++) {
2194 AliasCacheEntry* ace = &_alias_cache[i];
2195 if (ace->_adr_type != nullptr &&
2196 ace->_adr_type->is_flat()) {
2197 ace->_adr_type = nullptr;
2198 ace->_index = (i != 0) ? 0 : AliasIdxTop; // Make sure the nullptr adr_type resolves to AliasIdxTop
2199 }
2200 }
2201
2202 #ifdef ASSERT
2203 for (uint i = 0; i < memnodes.size(); i++) {
2204 Node* m = memnodes.at(i);
2205 const TypePtr* adr_type = m->adr_type();
2206 m->as_Mem()->set_adr_type(adr_type);
2207 }
2208 #endif // ASSERT
2209
2210 int start_alias = num_alias_types(); // Start of new aliases
2211 Node_Stack stack(0);
2212 #ifdef ASSERT
2213 VectorSet seen(Thread::current()->resource_area());
2214 #endif
2215 // Now let's fix the memory graph so each flat array access
2216 // is moved to the right slice. Start from the MergeMem nodes.
2217 uint last = unique();
2218 for (uint i = 0; i < mergememnodes.size(); i++) {
2219 MergeMemNode* current = mergememnodes.at(i)->as_MergeMem();
2220 if (current->outcnt() == 0) {
2221 // This node is killed by a previous iteration
2222 continue;
2223 }
2224
2225 Node* n = current->memory_at(index);
2226 MergeMemNode* mm = nullptr;
2227 do {
2228 // Follow memory edges through memory accesses, phis and
2229 // narrow membars and push nodes on the stack. Once we hit
2230 // bottom memory, we pop element off the stack one at a
2231 // time, in reverse order, and move them to the right slice
2232 // by changing their memory edges.
2233 if ((n->is_Phi() && n->adr_type() != TypePtr::BOTTOM) || n->is_Mem() ||
2234 (n->adr_type() == TypeAryPtr::INLINES && !n->is_NarrowMemProj())) {
2235 assert(!seen.test_set(n->_idx), "");
2236 // Uses (a load for instance) will need to be moved to the
2237 // right slice as well and will get a new memory state
2238 // that we don't know yet. The use could also be the
2239 // backedge of a loop. We put a place holder node between
2240 // the memory node and its uses. We replace that place
2241 // holder with the correct memory state once we know it,
2242 // i.e. when nodes are popped off the stack. Using the
2243 // place holder make the logic work in the presence of
2244 // loops.
2245 if (n->outcnt() > 1) {
2246 Node* place_holder = nullptr;
2247 assert(!n->has_out_with(Op_Node), "");
2248 for (DUIterator k = n->outs(); n->has_out(k); k++) {
2249 Node* u = n->out(k);
2250 if (u != current && u->_idx < last) {
2251 bool success = false;
2252 for (uint l = 0; l < u->req(); l++) {
2253 if (!stack.is_empty() && u == stack.node() && l == stack.index()) {
2254 continue;
2255 }
2256 Node* in = u->in(l);
2257 if (in == n) {
2258 if (place_holder == nullptr) {
2259 place_holder = new Node(1);
2260 place_holder->init_req(0, n);
2261 }
2262 igvn.replace_input_of(u, l, place_holder);
2263 success = true;
2264 }
2265 }
2266 if (success) {
2267 --k;
2268 }
2269 }
2270 }
2271 }
2272 if (n->is_Phi()) {
2273 stack.push(n, 1);
2274 n = n->in(1);
2275 } else if (n->is_Mem()) {
2276 stack.push(n, n->req());
2277 n = n->in(MemNode::Memory);
2278 } else {
2279 assert(n->is_Proj() && n->in(0)->Opcode() == Op_MemBarCPUOrder, "");
2280 stack.push(n, n->req());
2281 n = n->in(0)->in(TypeFunc::Memory);
2282 }
2283 } else {
2284 assert(n->adr_type() == TypePtr::BOTTOM || (n->Opcode() == Op_Node && n->_idx >= last) || n->is_NarrowMemProj(), "");
2285 // Build a new MergeMem node to carry the new memory state
2286 // as we build it. IGVN should fold extraneous MergeMem
2287 // nodes.
2288 if (n->is_NarrowMemProj()) {
2289 // We need 1 NarrowMemProj for each slice of this array
2290 InitializeNode* init = n->in(0)->as_Initialize();
2291 AllocateNode* alloc = init->allocation();
2292 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2293 const TypeAryKlassPtr* klass_type = klass_node->bottom_type()->isa_aryklassptr();
2294 assert(klass_type != nullptr, "must be an array");
2295 assert(klass_type->klass_is_exact(), "must be an exact klass");
2296 ciArrayKlass* klass = klass_type->exact_klass()->as_array_klass();
2297 assert(klass->is_flat_array_klass(), "must be a flat array");
2298 ciInlineKlass* elem_klass = klass->element_klass()->as_inline_klass();
2299 const TypeAryPtr* oop_type = klass_type->as_instance_type()->is_aryptr();
2300 assert(oop_type->klass_is_exact(), "must be an exact klass");
2301
2302 Node* base = alloc->in(TypeFunc::Memory);
2303 assert(base->bottom_type() == Type::MEMORY, "the memory input of AllocateNode must be a memory");
2304 assert(base->adr_type() == TypePtr::BOTTOM, "the memory input of AllocateNode must be a bottom memory");
2305 // Must create a MergeMem with base as the base memory, do not clone if base is a
2306 // MergeMem because it may not be processed yet
2307 mm = MergeMemNode::make(nullptr);
2308 mm->set_base_memory(base);
2309 for (int j = 0; j < elem_klass->nof_nonstatic_fields(); j++) {
2310 int field_offset = elem_klass->nonstatic_field_at(j)->offset_in_bytes() - elem_klass->payload_offset();
2311 const TypeAryPtr* field_ptr = oop_type->with_offset(Type::OffsetBot)->with_field_offset(field_offset);
2312 int field_alias_idx = get_alias_index(field_ptr);
2313 assert(field_ptr == get_adr_type(field_alias_idx), "must match");
2314 Node* new_proj = new NarrowMemProjNode(init, field_ptr);
2315 igvn.register_new_node_with_optimizer(new_proj);
2316 mm->set_memory_at(field_alias_idx, new_proj);
2317 }
2318 if (!klass->is_elem_null_free()) {
2319 int nm_offset = elem_klass->null_marker_offset_in_payload();
2320 const TypeAryPtr* nm_ptr = oop_type->with_offset(Type::OffsetBot)->with_field_offset(nm_offset);
2321 int nm_alias_idx = get_alias_index(nm_ptr);
2322 assert(nm_ptr == get_adr_type(nm_alias_idx), "must match");
2323 Node* new_proj = new NarrowMemProjNode(init, nm_ptr);
2324 igvn.register_new_node_with_optimizer(new_proj);
2325 mm->set_memory_at(nm_alias_idx, new_proj);
2326 }
2327
2328 // Replace all uses of the old NarrowMemProj with the correct state
2329 MergeMemNode* new_n = MergeMemNode::make(mm);
2330 igvn.register_new_node_with_optimizer(new_n);
2331 igvn.replace_node(n, new_n);
2332 } else {
2333 // Must create a MergeMem with n as the base memory, do not clone if n is a MergeMem
2334 // because it may not be processed yet
2335 mm = MergeMemNode::make(nullptr);
2336 mm->set_base_memory(n);
2337 }
2338
2339 igvn.register_new_node_with_optimizer(mm);
2340 while (stack.size() > 0) {
2341 Node* m = stack.node();
2342 uint idx = stack.index();
2343 if (m->is_Mem()) {
2344 // Move memory node to its new slice
2345 const TypePtr* adr_type = m->adr_type();
2346 int alias = get_alias_index(adr_type);
2347 Node* prev = mm->memory_at(alias);
2348 igvn.replace_input_of(m, MemNode::Memory, prev);
2349 mm->set_memory_at(alias, m);
2350 } else if (m->is_Phi()) {
2351 // We need as many new phis as there are new aliases
2352 Node* new_phi_in = MergeMemNode::make(mm);
2353 igvn.register_new_node_with_optimizer(new_phi_in);
2354 igvn.replace_input_of(m, idx, new_phi_in);
2355 if (idx == m->req()-1) {
2356 Node* r = m->in(0);
2357 for (int j = start_alias; j < num_alias_types(); j++) {
2358 const TypePtr* adr_type = get_adr_type(j);
2359 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2360 continue;
2361 }
2362 Node* phi = new PhiNode(r, Type::MEMORY, get_adr_type(j));
2363 igvn.register_new_node_with_optimizer(phi);
2364 for (uint k = 1; k < m->req(); k++) {
2365 phi->init_req(k, m->in(k)->as_MergeMem()->memory_at(j));
2366 }
2367 mm->set_memory_at(j, phi);
2368 }
2369 Node* base_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2370 igvn.register_new_node_with_optimizer(base_phi);
2371 for (uint k = 1; k < m->req(); k++) {
2372 base_phi->init_req(k, m->in(k)->as_MergeMem()->base_memory());
2373 }
2374 mm->set_base_memory(base_phi);
2375 }
2376 } else {
2377 // This is a MemBarCPUOrder node from
2378 // Parse::array_load()/Parse::array_store(), in the
2379 // branch that handles flat arrays hidden under
2380 // an Object[] array. We also need one new membar per
2381 // new alias to keep the unknown access that the
2382 // membars protect properly ordered with accesses to
2383 // known flat array.
2384 assert(m->is_Proj(), "projection expected");
2385 Node* ctrl = m->in(0)->in(TypeFunc::Control);
2386 igvn.replace_input_of(m->in(0), TypeFunc::Control, top());
2387 for (int j = start_alias; j < num_alias_types(); j++) {
2388 const TypePtr* adr_type = get_adr_type(j);
2389 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2390 continue;
2391 }
2392 MemBarNode* mb = new MemBarCPUOrderNode(this, j, nullptr);
2393 igvn.register_new_node_with_optimizer(mb);
2394 Node* mem = mm->memory_at(j);
2395 mb->init_req(TypeFunc::Control, ctrl);
2396 mb->init_req(TypeFunc::Memory, mem);
2397 ctrl = new ProjNode(mb, TypeFunc::Control);
2398 igvn.register_new_node_with_optimizer(ctrl);
2399 mem = new ProjNode(mb, TypeFunc::Memory);
2400 igvn.register_new_node_with_optimizer(mem);
2401 mm->set_memory_at(j, mem);
2402 }
2403 igvn.replace_node(m->in(0)->as_Multi()->proj_out(TypeFunc::Control), ctrl);
2404 }
2405 if (idx < m->req()-1) {
2406 idx += 1;
2407 stack.set_index(idx);
2408 n = m->in(idx);
2409 break;
2410 }
2411 // Take care of place holder nodes
2412 if (m->has_out_with(Op_Node)) {
2413 Node* place_holder = m->find_out_with(Op_Node);
2414 if (place_holder != nullptr) {
2415 Node* mm_clone = mm->clone();
2416 igvn.register_new_node_with_optimizer(mm_clone);
2417 Node* hook = new Node(1);
2418 hook->init_req(0, mm);
2419 igvn.replace_node(place_holder, mm_clone);
2420 hook->destruct(&igvn);
2421 }
2422 assert(!m->has_out_with(Op_Node), "place holder should be gone now");
2423 }
2424 stack.pop();
2425 }
2426 }
2427 } while(stack.size() > 0);
2428 // Fix the memory state at the MergeMem we started from
2429 igvn.rehash_node_delayed(current);
2430 for (int j = start_alias; j < num_alias_types(); j++) {
2431 const TypePtr* adr_type = get_adr_type(j);
2432 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2433 continue;
2434 }
2435 current->set_memory_at(j, mm);
2436 }
2437 current->set_memory_at(index, current->base_memory());
2438 }
2439 igvn.optimize();
2440
2441 #ifdef ASSERT
2442 wq.clear();
2443 wq.push(root());
2444 for (uint i = 0; i < wq.size(); i++) {
2445 Node* n = wq.at(i);
2446 assert(n->adr_type() != TypeAryPtr::INLINES, "should have been removed from the graph");
2447 for (uint j = 0; j < n->req(); j++) {
2448 Node* m = n->in(j);
2449 if (m != nullptr) {
2450 wq.push(m);
2451 }
2452 }
2453 }
2454 #endif
2455
2456 print_method(PHASE_SPLIT_INLINES_ARRAY, 2);
2457 }
2458
2459 void Compile::record_for_merge_stores_igvn(Node* n) {
2460 if (!n->for_merge_stores_igvn()) {
2461 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
2462 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2463 _for_merge_stores_igvn.append(n);
2464 }
2465 }
2466
2467 void Compile::remove_from_merge_stores_igvn(Node* n) {
2468 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2469 _for_merge_stores_igvn.remove(n);
2470 }
2471
2472 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
2473 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
2474 // the stores, and we merge the wrong sequence of stores.
2475 // Example:
2476 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
2477 // Apply MergeStores:
2478 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
2557 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2558 Bytecodes::Code c = iter.cur_bc();
2559 Node* lhs = nullptr;
2560 Node* rhs = nullptr;
2561 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2562 lhs = unc->peek_operand(0);
2563 rhs = unc->peek_operand(1);
2564 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2565 lhs = unc->peek_operand(0);
2566 }
2567
2568 ResourceMark rm;
2569 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2570 assert(live_locals.is_valid(), "broken liveness info");
2571 int len = (int)live_locals.size();
2572
2573 for (int i = 0; i < len; i++) {
2574 Node* local = unc->local(jvms, i);
2575 // kill local using the liveness of next_bci.
2576 // give up when the local looks like an operand to secure reexecution.
2577 if (!live_locals.at(i) && !local->is_top() && local != lhs && local != rhs) {
2578 uint idx = jvms->locoff() + i;
2579 #ifdef ASSERT
2580 if (PrintOpto && Verbose) {
2581 tty->print("[unstable_if] kill local#%d: ", idx);
2582 local->dump();
2583 tty->cr();
2584 }
2585 #endif
2586 igvn.replace_input_of(unc, idx, top());
2587 modified = true;
2588 }
2589 }
2590 }
2591
2592 // keep the modified trap for late query
2593 if (modified) {
2594 trap->set_modified();
2595 } else {
2596 _unstable_if_traps.delete_at(i);
2597 }
2598 }
2599 igvn.optimize();
2600 }
2601
2602 // StringOpts and late inlining of string methods
2603 void Compile::inline_string_calls(bool parse_time) {
2604 {
2605 // remove useless nodes to make the usage analysis simpler
2606 ResourceMark rm;
2607 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2608 }
2609
2610 {
2611 ResourceMark rm;
2612 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2784
2785 if (_string_late_inlines.length() > 0) {
2786 assert(has_stringbuilder(), "inconsistent");
2787
2788 inline_string_calls(false);
2789
2790 if (failing()) return;
2791
2792 inline_incrementally_cleanup(igvn);
2793 }
2794
2795 set_inlining_incrementally(false);
2796 }
2797
2798 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2799 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2800 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2801 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2802 // as if "inlining_incrementally() == true" were set.
2803 assert(inlining_incrementally() == false, "not allowed");
2804 set_strength_reduction(true);
2805 #ifdef ASSERT
2806 Unique_Node_List* modified_nodes = _modified_nodes;
2807 _modified_nodes = nullptr;
2808 #endif
2809 assert(_late_inlines.length() > 0, "sanity");
2810
2811 while (_late_inlines.length() > 0) {
2812 igvn_worklist()->ensure_empty(); // should be done with igvn
2813
2814 while (inline_incrementally_one()) {
2815 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2816 }
2817 if (failing()) return;
2818
2819 inline_incrementally_cleanup(igvn);
2820 }
2821 DEBUG_ONLY( _modified_nodes = modified_nodes; )
2822 set_strength_reduction(false);
2823 }
2824
2825 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2826 if (_loop_opts_cnt > 0) {
2827 while (major_progress() && (_loop_opts_cnt > 0)) {
2828 TracePhase tp(_t_idealLoop);
2829 PhaseIdealLoop::optimize(igvn, mode);
2830 _loop_opts_cnt--;
2831 if (failing()) return false;
2832 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2833 }
2834 }
2835 return true;
2836 }
2837
2838 // Remove edges from "root" to each SafePoint at a backward branch.
2839 // They were inserted during parsing (see add_safepoint()) to make
2840 // infinite loops without calls or exceptions visible to root, i.e.,
2841 // useful.
2842 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2946 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2947 }
2948 assert(!has_vbox_nodes(), "sanity");
2949
2950 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2951 Compile::TracePhase tp(_t_renumberLive);
2952 igvn_worklist()->ensure_empty(); // should be done with igvn
2953 {
2954 ResourceMark rm;
2955 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2956 }
2957 igvn.reset();
2958 igvn.optimize();
2959 if (failing()) return;
2960 }
2961
2962 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2963 // safepoints
2964 remove_root_to_sfpts_edges(igvn);
2965
2966 // Process inline type nodes now that all inlining is over
2967 process_inline_types(igvn);
2968
2969 adjust_flat_array_access_aliases(igvn);
2970
2971 if (failing()) return;
2972
2973 if (C->macro_count() > 0) {
2974 // Eliminate some macro nodes before EA to reduce analysis pressure
2975 PhaseMacroExpand mexp(igvn);
2976 mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
2977 if (failing()) {
2978 return;
2979 }
2980 igvn.set_delay_transform(false);
2981 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2982 }
2983
2984 if (has_loops()) {
2985 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2986 }
2987
2988 // Perform escape analysis
2989 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2990 if (has_loops()) {
2991 // Cleanup graph (remove dead nodes).
2992 TracePhase tp(_t_idealLoop);
2993 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2994 if (failing()) {
2995 return;
2996 }
2997 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2998 if (C->macro_count() > 0) {
2999 // Eliminate some macro nodes before EA to reduce analysis pressure
3000 PhaseMacroExpand mexp(igvn);
3001 mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
3002 if (failing()) {
3003 return;
3004 }
3005 igvn.set_delay_transform(false);
3006 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
3007 }
3008 }
3009
3010 bool progress;
3011 do {
3012 ConnectionGraph::do_analysis(this, &igvn);
3013
3014 if (failing()) return;
3015
3016 int mcount = macro_count(); // Record number of allocations and locks before IGVN
3017
3018 // Optimize out fields loads from scalar replaceable allocations.
3019 igvn.optimize();
3020 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
3021
3022 if (failing()) return;
3023
3024 if (congraph() != nullptr && macro_count() > 0) {
3025 TracePhase tp(_t_macroEliminate);
3026 PhaseMacroExpand mexp(igvn);
3027 mexp.eliminate_macro_nodes();
3028 if (failing()) {
3029 return;
3030 }
3031 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
3032
3033 igvn.set_delay_transform(false);
3034 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
3035 }
3036
3037 ConnectionGraph::verify_ram_nodes(this, root());
3038 if (failing()) return;
3039
3040 progress = do_iterative_escape_analysis() &&
3041 (macro_count() < mcount) &&
3042 ConnectionGraph::has_candidates(this);
3043 // Try again if candidates exist and made progress
3044 // by removing some allocations and/or locks.
3045 } while (progress);
3046 }
3047
3048 process_flat_accesses(igvn);
3049 if (failing()) {
3050 return;
3051 }
3052
3053 // Loop transforms on the ideal graph. Range Check Elimination,
3054 // peeling, unrolling, etc.
3055
3056 // Set loop opts counter
3057 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
3058 {
3059 TracePhase tp(_t_idealLoop);
3060 PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
3061 _loop_opts_cnt--;
3062 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
3063 if (failing()) return;
3064 }
3065 // Loop opts pass if partial peeling occurred in previous pass
3066 if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
3067 TracePhase tp(_t_idealLoop);
3068 PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
3069 _loop_opts_cnt--;
3070 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
3071 if (failing()) return;
3072 }
3109 // Loop transforms on the ideal graph. Range Check Elimination,
3110 // peeling, unrolling, etc.
3111 if (!optimize_loops(igvn, LoopOptsDefault)) {
3112 return;
3113 }
3114
3115 if (failing()) return;
3116
3117 C->clear_major_progress(); // ensure that major progress is now clear
3118
3119 process_for_post_loop_opts_igvn(igvn);
3120
3121 process_for_merge_stores_igvn(igvn);
3122
3123 if (failing()) return;
3124
3125 #ifdef ASSERT
3126 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
3127 #endif
3128
3129 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
3130
3131 if (_late_inlines.length() > 0) {
3132 // More opportunities to optimize virtual and MH calls.
3133 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
3134 process_late_inline_calls_no_inline(igvn);
3135 }
3136
3137 {
3138 TracePhase tp(_t_macroExpand);
3139 PhaseMacroExpand mex(igvn);
3140 // Last attempt to eliminate macro nodes.
3141 mex.eliminate_macro_nodes();
3142 if (failing()) {
3143 return;
3144 }
3145
3146 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
3147 // Do not allow new macro nodes once we start to eliminate and expand
3148 C->reset_allow_macro_nodes();
3149 // Last attempt to eliminate macro nodes before expand
3150 mex.eliminate_macro_nodes();
3151 if (failing()) {
3152 return;
3153 }
3154 mex.eliminate_opaque_looplimit_macro_nodes();
3155 if (failing()) {
3156 return;
3157 }
3158 print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
3159 if (mex.expand_macro_nodes()) {
3160 assert(failing(), "must bail out w/ explicit message");
3161 return;
3162 }
3163 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
3164 }
3165
3166 // Process inline type nodes again and remove them. From here
3167 // on we don't need to keep track of field values anymore.
3168 process_inline_types(igvn, /* remove= */ true);
3169
3170 {
3171 TracePhase tp(_t_barrierExpand);
3172 if (bs->expand_barriers(this, igvn)) {
3173 assert(failing(), "must bail out w/ explicit message");
3174 return;
3175 }
3176 print_method(PHASE_BARRIER_EXPANSION, 2);
3177 }
3178
3179 if (C->max_vector_size() > 0) {
3180 C->optimize_logic_cones(igvn);
3181 igvn.optimize();
3182 if (failing()) return;
3183 }
3184
3185 DEBUG_ONLY( _modified_nodes = nullptr; )
3186 DEBUG_ONLY( _late_inlines.clear(); )
3187
3188 assert(igvn._worklist.size() == 0, "not empty");
3189 } // (End scope of igvn; run destructor if necessary for asserts.)
3190
3191 check_no_dead_use();
3192
3193 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
3194 // to remove hashes to unlock nodes for modifications.
3195 C->node_hash()->clear();
3196
3197 // A method with only infinite loops has no edges entering loops from root
3198 {
3199 TracePhase tp(_t_graphReshaping);
3200 if (final_graph_reshaping()) {
3201 assert(failing(), "must bail out w/ explicit message");
3202 return;
3203 }
3204 }
3205
3206 print_method(PHASE_OPTIMIZE_FINISHED, 2);
3207 DEBUG_ONLY(set_phase_optimize_finished();)
3208 }
3914 case Op_CmpD3:
3915 case Op_StoreD:
3916 case Op_LoadD:
3917 case Op_LoadD_unaligned:
3918 frc.inc_double_count();
3919 break;
3920 case Op_Opaque1: // Remove Opaque Nodes before matching
3921 n->subsume_by(n->in(1), this);
3922 break;
3923 case Op_CallLeafPure: {
3924 // If the pure call is not supported, then lower to a CallLeaf.
3925 if (!Matcher::match_rule_supported(Op_CallLeafPure)) {
3926 CallNode* call = n->as_Call();
3927 CallNode* new_call = new CallLeafNode(call->tf(), call->entry_point(),
3928 call->_name, TypeRawPtr::BOTTOM);
3929 new_call->init_req(TypeFunc::Control, call->in(TypeFunc::Control));
3930 new_call->init_req(TypeFunc::I_O, C->top());
3931 new_call->init_req(TypeFunc::Memory, C->top());
3932 new_call->init_req(TypeFunc::ReturnAdr, C->top());
3933 new_call->init_req(TypeFunc::FramePtr, C->top());
3934 for (unsigned int i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
3935 new_call->init_req(i, call->in(i));
3936 }
3937 n->subsume_by(new_call, this);
3938 }
3939 frc.inc_call_count();
3940 break;
3941 }
3942 case Op_CallStaticJava:
3943 case Op_CallJava:
3944 case Op_CallDynamicJava:
3945 frc.inc_java_call_count(); // Count java call site;
3946 case Op_CallRuntime:
3947 case Op_CallLeaf:
3948 case Op_CallLeafVector:
3949 case Op_CallLeafNoFP: {
3950 assert (n->is_Call(), "");
3951 CallNode *call = n->as_Call();
3952 // Count call sites where the FP mode bit would have to be flipped.
3953 // Do not count uncommon runtime calls:
3954 // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3960 int nop = n->Opcode();
3961 // Clone shared simple arguments to uncommon calls, item (1).
3962 if (n->outcnt() > 1 &&
3963 !n->is_Proj() &&
3964 nop != Op_CreateEx &&
3965 nop != Op_CheckCastPP &&
3966 nop != Op_DecodeN &&
3967 nop != Op_DecodeNKlass &&
3968 !n->is_Mem() &&
3969 !n->is_Phi()) {
3970 Node *x = n->clone();
3971 call->set_req(TypeFunc::Parms, x);
3972 }
3973 }
3974 break;
3975 }
3976 case Op_StoreB:
3977 case Op_StoreC:
3978 case Op_StoreI:
3979 case Op_StoreL:
3980 case Op_StoreLSpecial:
3981 case Op_CompareAndSwapB:
3982 case Op_CompareAndSwapS:
3983 case Op_CompareAndSwapI:
3984 case Op_CompareAndSwapL:
3985 case Op_CompareAndSwapP:
3986 case Op_CompareAndSwapN:
3987 case Op_WeakCompareAndSwapB:
3988 case Op_WeakCompareAndSwapS:
3989 case Op_WeakCompareAndSwapI:
3990 case Op_WeakCompareAndSwapL:
3991 case Op_WeakCompareAndSwapP:
3992 case Op_WeakCompareAndSwapN:
3993 case Op_CompareAndExchangeB:
3994 case Op_CompareAndExchangeS:
3995 case Op_CompareAndExchangeI:
3996 case Op_CompareAndExchangeL:
3997 case Op_CompareAndExchangeP:
3998 case Op_CompareAndExchangeN:
3999 case Op_GetAndAddS:
4000 case Op_GetAndAddB:
4510 k->subsume_by(m, this);
4511 }
4512 }
4513 }
4514 break;
4515 }
4516 case Op_CmpUL: {
4517 if (!Matcher::has_match_rule(Op_CmpUL)) {
4518 // No support for unsigned long comparisons
4519 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
4520 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
4521 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
4522 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
4523 Node* andl = new AndLNode(orl, remove_sign_mask);
4524 Node* cmp = new CmpLNode(andl, n->in(2));
4525 n->subsume_by(cmp, this);
4526 }
4527 break;
4528 }
4529 #ifdef ASSERT
4530 case Op_InlineType: {
4531 n->dump(-1);
4532 assert(false, "inline type node was not removed");
4533 break;
4534 }
4535 case Op_ConNKlass: {
4536 const TypePtr* tp = n->as_Type()->type()->make_ptr();
4537 ciKlass* klass = tp->is_klassptr()->exact_klass();
4538 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
4539 break;
4540 }
4541 #endif
4542 default:
4543 assert(!n->is_Call(), "");
4544 assert(!n->is_Mem(), "");
4545 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
4546 break;
4547 }
4548 }
4549
4550 //------------------------------final_graph_reshaping_walk---------------------
4551 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
4552 // requires that the walk visits a node's inputs before visiting the node.
4553 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
4554 Unique_Node_List sfpt;
4890 }
4891 }
4892
4893 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4894 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4895 }
4896
4897 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4898 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4899 }
4900
4901 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4902 if (holder->is_initialized()) {
4903 return false;
4904 }
4905 if (holder->is_being_initialized()) {
4906 if (accessing_method->holder() == holder) {
4907 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4908 // <init>, or a static method. In all those cases, there was an initialization
4909 // barrier on the holder klass passed.
4910 if (accessing_method->is_class_initializer() ||
4911 accessing_method->is_object_constructor() ||
4912 accessing_method->is_static()) {
4913 return false;
4914 }
4915 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4916 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4917 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4918 // child class can become fully initialized while its parent class is still being initialized.
4919 if (accessing_method->is_class_initializer()) {
4920 return false;
4921 }
4922 }
4923 ciMethod* root = method(); // the root method of compilation
4924 if (root != accessing_method) {
4925 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4926 }
4927 }
4928 return true;
4929 }
4930
4931 #ifndef PRODUCT
4932 //------------------------------verify_bidirectional_edges---------------------
4933 // For each input edge to a node (ie - for each Use-Def edge), verify that
4934 // there is a corresponding Def-Use edge.
4935 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4936 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4937 uint stack_size = live_nodes() >> 4;
4938 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4939 if (root_and_safepoints != nullptr) {
4969 if (in != nullptr && !in->is_top()) {
4970 // Count instances of `next`
4971 int cnt = 0;
4972 for (uint idx = 0; idx < in->_outcnt; idx++) {
4973 if (in->_out[idx] == n) {
4974 cnt++;
4975 }
4976 }
4977 assert(cnt > 0, "Failed to find Def-Use edge.");
4978 // Check for duplicate edges
4979 // walk the input array downcounting the input edges to n
4980 for (uint j = 0; j < length; j++) {
4981 if (n->in(j) == in) {
4982 cnt--;
4983 }
4984 }
4985 assert(cnt == 0, "Mismatched edge count.");
4986 } else if (in == nullptr) {
4987 assert(i == 0 || i >= n->req() ||
4988 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4989 (n->is_Allocate() && i >= AllocateNode::InlineType) ||
4990 (n->is_Unlock() && i == (n->req() - 1)) ||
4991 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4992 "only region, phi, arraycopy, allocate, unlock or membar nodes have null data edges");
4993 } else {
4994 assert(in->is_top(), "sanity");
4995 // Nothing to check.
4996 }
4997 }
4998 }
4999 }
5000
5001 //------------------------------verify_graph_edges---------------------------
5002 // Walk the Graph and verify that there is a one-to-one correspondence
5003 // between Use-Def edges and Def-Use edges in the graph.
5004 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
5005 if (VerifyGraphEdges) {
5006 Unique_Node_List visited;
5007
5008 // Call graph walk to check edges
5009 verify_bidirectional_edges(visited, root_and_safepoints);
5010 if (no_dead_code) {
5011 // Now make sure that no visited node is used by an unvisited node.
5012 bool dead_nodes = false;
5123 // (1) subklass is already limited to a subtype of superklass => always ok
5124 // (2) subklass does not overlap with superklass => always fail
5125 // (3) superklass has NO subtypes and we can check with a simple compare.
5126 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
5127 if (skip) {
5128 return SSC_full_test; // Let caller generate the general case.
5129 }
5130
5131 if (subk->is_java_subtype_of(superk)) {
5132 return SSC_always_true; // (0) and (1) this test cannot fail
5133 }
5134
5135 if (!subk->maybe_java_subtype_of(superk)) {
5136 return SSC_always_false; // (2) true path dead; no dynamic test needed
5137 }
5138
5139 const Type* superelem = superk;
5140 if (superk->isa_aryklassptr()) {
5141 int ignored;
5142 superelem = superk->is_aryklassptr()->base_element_type(ignored);
5143
5144 // Do not fold the subtype check to an array klass pointer comparison for null-able inline type arrays
5145 // because null-free [LMyValue <: null-able [LMyValue but the klasses are different. Perform a full test.
5146 if (!superk->is_aryklassptr()->is_null_free() && superk->is_aryklassptr()->elem()->isa_instklassptr() &&
5147 superk->is_aryklassptr()->elem()->is_instklassptr()->instance_klass()->is_inlinetype()) {
5148 return SSC_full_test;
5149 }
5150 }
5151
5152 if (superelem->isa_instklassptr()) {
5153 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
5154 if (!ik->has_subklass()) {
5155 if (!ik->is_final()) {
5156 // Add a dependency if there is a chance of a later subclass.
5157 dependencies()->assert_leaf_type(ik);
5158 }
5159 if (!superk->maybe_java_subtype_of(subk)) {
5160 return SSC_always_false;
5161 }
5162 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5163 }
5164 } else {
5165 // A primitive array type has no subtypes.
5166 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5167 }
5168
5169 return SSC_full_test;
5968 } else {
5969 _debug_network_printer->update_compiled_method(C->method());
5970 }
5971 tty->print_cr("Method printed over network stream to IGV");
5972 _debug_network_printer->print(name, C->root(), visible_nodes, fr);
5973 }
5974 #endif // !PRODUCT
5975
5976 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5977 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5978 return value;
5979 }
5980 Node* result = nullptr;
5981 if (bt == T_BYTE) {
5982 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5983 result = new RShiftINode(result, phase->intcon(24));
5984 } else if (bt == T_BOOLEAN) {
5985 result = new AndINode(value, phase->intcon(0xFF));
5986 } else if (bt == T_CHAR) {
5987 result = new AndINode(value,phase->intcon(0xFFFF));
5988 } else if (bt == T_FLOAT) {
5989 result = new MoveI2FNode(value);
5990 } else {
5991 assert(bt == T_SHORT, "unexpected narrow type");
5992 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5993 result = new RShiftINode(result, phase->intcon(16));
5994 }
5995 if (transform_res) {
5996 result = phase->transform(result);
5997 }
5998 return result;
5999 }
6000
6001 void Compile::record_method_not_compilable_oom() {
6002 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
6003 }
6004
6005 #ifndef PRODUCT
6006 // Collects all the control inputs from nodes on the worklist and from their data dependencies
6007 static void find_candidate_control_inputs(Unique_Node_List& worklist, Unique_Node_List& candidates) {
6008 // Follow non-control edges until we reach CFG nodes
6009 for (uint i = 0; i < worklist.size(); i++) {
|