40 #include "gc/shared/c2/barrierSetC2.hpp"
41 #include "jfr/jfrEvents.hpp"
42 #include "jvm_io.h"
43 #include "memory/allocation.hpp"
44 #include "memory/arena.hpp"
45 #include "memory/resourceArea.hpp"
46 #include "opto/addnode.hpp"
47 #include "opto/block.hpp"
48 #include "opto/c2compiler.hpp"
49 #include "opto/callGenerator.hpp"
50 #include "opto/callnode.hpp"
51 #include "opto/castnode.hpp"
52 #include "opto/cfgnode.hpp"
53 #include "opto/chaitin.hpp"
54 #include "opto/compile.hpp"
55 #include "opto/connode.hpp"
56 #include "opto/convertnode.hpp"
57 #include "opto/divnode.hpp"
58 #include "opto/escape.hpp"
59 #include "opto/idealGraphPrinter.hpp"
60 #include "opto/locknode.hpp"
61 #include "opto/loopnode.hpp"
62 #include "opto/machnode.hpp"
63 #include "opto/macro.hpp"
64 #include "opto/matcher.hpp"
65 #include "opto/mathexactnode.hpp"
66 #include "opto/memnode.hpp"
67 #include "opto/mulnode.hpp"
68 #include "opto/narrowptrnode.hpp"
69 #include "opto/node.hpp"
70 #include "opto/opaquenode.hpp"
71 #include "opto/opcodes.hpp"
72 #include "opto/output.hpp"
73 #include "opto/parse.hpp"
74 #include "opto/phaseX.hpp"
75 #include "opto/rootnode.hpp"
76 #include "opto/runtime.hpp"
77 #include "opto/stringopts.hpp"
78 #include "opto/type.hpp"
79 #include "opto/vector.hpp"
80 #include "opto/vectornode.hpp"
81 #include "runtime/globals_extension.hpp"
82 #include "runtime/sharedRuntime.hpp"
83 #include "runtime/signature.hpp"
84 #include "runtime/stubRoutines.hpp"
85 #include "runtime/timer.hpp"
86 #include "utilities/align.hpp"
386 // as dead to be conservative about the dead node count at any
387 // given time.
388 if (!dead->is_Con()) {
389 record_dead_node(dead->_idx);
390 }
391 if (dead->is_macro()) {
392 remove_macro_node(dead);
393 }
394 if (dead->is_expensive()) {
395 remove_expensive_node(dead);
396 }
397 if (dead->is_OpaqueTemplateAssertionPredicate()) {
398 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
399 }
400 if (dead->is_ParsePredicate()) {
401 remove_parse_predicate(dead->as_ParsePredicate());
402 }
403 if (dead->for_post_loop_opts_igvn()) {
404 remove_from_post_loop_opts_igvn(dead);
405 }
406 if (dead->for_merge_stores_igvn()) {
407 remove_from_merge_stores_igvn(dead);
408 }
409 if (dead->is_Call()) {
410 remove_useless_late_inlines( &_late_inlines, dead);
411 remove_useless_late_inlines( &_string_late_inlines, dead);
412 remove_useless_late_inlines( &_boxing_late_inlines, dead);
413 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
414
415 if (dead->is_CallStaticJava()) {
416 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
417 }
418 }
419 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
420 bs->unregister_potential_barrier_node(dead);
421 }
422
423 // Disconnect all useless nodes by disconnecting those at the boundary.
424 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
425 uint next = 0;
433 // Use raw traversal of out edges since this code removes out edges
434 int max = n->outcnt();
435 for (int j = 0; j < max; ++j) {
436 Node* child = n->raw_out(j);
437 if (!useful.member(child)) {
438 assert(!child->is_top() || child != top(),
439 "If top is cached in Compile object it is in useful list");
440 // Only need to remove this out-edge to the useless node
441 n->raw_del_out(j);
442 --j;
443 --max;
444 if (child->is_data_proj_of_pure_function(n)) {
445 worklist.push(n);
446 }
447 }
448 }
449 if (n->outcnt() == 1 && n->has_special_unique_user()) {
450 assert(useful.member(n->unique_out()), "do not push a useless node");
451 worklist.push(n->unique_out());
452 }
453 }
454
455 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
456 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
457 // Remove useless Template Assertion Predicate opaque nodes
458 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
459 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
460 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
461 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
462 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
463 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
464 #ifdef ASSERT
465 if (_modified_nodes != nullptr) {
466 _modified_nodes->remove_useless_nodes(useful.member_set());
467 }
468 #endif
469
470 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
471 bs->eliminate_useless_gc_barriers(useful, this);
472 // clean up the late inline lists
473 remove_useless_late_inlines( &_late_inlines, useful);
474 remove_useless_late_inlines( &_string_late_inlines, useful);
475 remove_useless_late_inlines( &_boxing_late_inlines, useful);
476 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
477 debug_only(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
478 }
479
480 // ============================================================================
622
623 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
624 Options options, DirectiveSet* directive)
625 : Phase(Compiler),
626 _compile_id(ci_env->compile_id()),
627 _options(options),
628 _method(target),
629 _entry_bci(osr_bci),
630 _ilt(nullptr),
631 _stub_function(nullptr),
632 _stub_name(nullptr),
633 _stub_entry_point(nullptr),
634 _max_node_limit(MaxNodeLimit),
635 _post_loop_opts_phase(false),
636 _merge_stores_phase(false),
637 _allow_macro_nodes(true),
638 _inlining_progress(false),
639 _inlining_incrementally(false),
640 _do_cleanup(false),
641 _has_reserved_stack_access(target->has_reserved_stack_access()),
642 #ifndef PRODUCT
643 _igv_idx(0),
644 _trace_opto_output(directive->TraceOptoOutputOption),
645 #endif
646 _has_method_handle_invokes(false),
647 _clinit_barrier_on_entry(false),
648 _stress_seed(0),
649 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
650 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
651 _env(ci_env),
652 _directive(directive),
653 _log(ci_env->log()),
654 _first_failure_details(nullptr),
655 _intrinsics(comp_arena(), 0, 0, nullptr),
656 _macro_nodes(comp_arena(), 8, 0, nullptr),
657 _parse_predicates(comp_arena(), 8, 0, nullptr),
658 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
659 _expensive_nodes(comp_arena(), 8, 0, nullptr),
660 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
661 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
662 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
663 _coarsened_locks(comp_arena(), 8, 0, nullptr),
664 _congraph(nullptr),
665 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
666 _unique(0),
667 _dead_node_count(0),
668 _dead_node_list(comp_arena()),
669 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
670 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
671 _node_arena(&_node_arena_one),
672 _mach_constant_base_node(nullptr),
673 _Compile_types(mtCompiler, Arena::Tag::tag_type),
674 _initial_gvn(nullptr),
675 _igvn_worklist(nullptr),
676 _types(nullptr),
677 _node_hash(nullptr),
678 _late_inlines(comp_arena(), 2, 0, nullptr),
679 _string_late_inlines(comp_arena(), 2, 0, nullptr),
680 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
745 #define MINIMUM_NODE_HASH 1023
746
747 // GVN that will be run immediately on new nodes
748 uint estimated_size = method()->code_size()*4+64;
749 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
750 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
751 _types = new (comp_arena()) Type_Array(comp_arena());
752 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
753 PhaseGVN gvn;
754 set_initial_gvn(&gvn);
755
756 { // Scope for timing the parser
757 TracePhase tp(_t_parser);
758
759 // Put top into the hash table ASAP.
760 initial_gvn()->transform(top());
761
762 // Set up tf(), start(), and find a CallGenerator.
763 CallGenerator* cg = nullptr;
764 if (is_osr_compilation()) {
765 const TypeTuple *domain = StartOSRNode::osr_domain();
766 const TypeTuple *range = TypeTuple::make_range(method()->signature());
767 init_tf(TypeFunc::make(domain, range));
768 StartNode* s = new StartOSRNode(root(), domain);
769 initial_gvn()->set_type_bottom(s);
770 verify_start(s);
771 cg = CallGenerator::for_osr(method(), entry_bci());
772 } else {
773 // Normal case.
774 init_tf(TypeFunc::make(method()));
775 StartNode* s = new StartNode(root(), tf()->domain());
776 initial_gvn()->set_type_bottom(s);
777 verify_start(s);
778 if (method()->intrinsic_id() == vmIntrinsics::_Reference_get) {
779 // With java.lang.ref.reference.get() we must go through the
780 // intrinsic - even when get() is the root
781 // method of the compile - so that, if necessary, the value in
782 // the referent field of the reference object gets recorded by
783 // the pre-barrier code.
784 cg = find_intrinsic(method(), false);
785 }
786 if (cg == nullptr) {
787 float past_uses = method()->interpreter_invocation_count();
788 float expected_uses = past_uses;
789 cg = CallGenerator::for_inline(method(), expected_uses);
790 }
791 }
792 if (failing()) return;
793 if (cg == nullptr) {
794 const char* reason = InlineTree::check_can_parse(method());
795 assert(reason != nullptr, "expect reason for parse failure");
866 print_ideal_ir("print_ideal");
867 }
868 #endif
869
870 #ifdef ASSERT
871 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
872 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
873 #endif
874
875 // Dump compilation data to replay it.
876 if (directive->DumpReplayOption) {
877 env()->dump_replay_data(_compile_id);
878 }
879 if (directive->DumpInlineOption && (ilt() != nullptr)) {
880 env()->dump_inline_data(_compile_id);
881 }
882
883 // Now that we know the size of all the monitors we can add a fixed slot
884 // for the original deopt pc.
885 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
886 set_fixed_slots(next_slot);
887
888 // Compute when to use implicit null checks. Used by matching trap based
889 // nodes and NullCheck optimization.
890 set_allowed_deopt_reasons();
891
892 // Now generate code
893 Code_Gen();
894 }
895
896 //------------------------------Compile----------------------------------------
897 // Compile a runtime stub
898 Compile::Compile(ciEnv* ci_env,
899 TypeFunc_generator generator,
900 address stub_function,
901 const char* stub_name,
902 int is_fancy_jump,
903 bool pass_tls,
904 bool return_pc,
905 DirectiveSet* directive)
906 : Phase(Compiler),
907 _compile_id(0),
908 _options(Options::for_runtime_stub()),
909 _method(nullptr),
910 _entry_bci(InvocationEntryBci),
911 _stub_function(stub_function),
912 _stub_name(stub_name),
913 _stub_entry_point(nullptr),
914 _max_node_limit(MaxNodeLimit),
915 _post_loop_opts_phase(false),
916 _merge_stores_phase(false),
917 _allow_macro_nodes(true),
918 _inlining_progress(false),
919 _inlining_incrementally(false),
920 _has_reserved_stack_access(false),
921 #ifndef PRODUCT
922 _igv_idx(0),
923 _trace_opto_output(directive->TraceOptoOutputOption),
924 #endif
925 _has_method_handle_invokes(false),
926 _clinit_barrier_on_entry(false),
927 _stress_seed(0),
928 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
929 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
930 _env(ci_env),
931 _directive(directive),
932 _log(ci_env->log()),
933 _first_failure_details(nullptr),
934 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
935 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
936 _congraph(nullptr),
937 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
938 _unique(0),
939 _dead_node_count(0),
940 _dead_node_list(comp_arena()),
1042
1043 _fixed_slots = 0;
1044 set_has_split_ifs(false);
1045 set_has_loops(false); // first approximation
1046 set_has_stringbuilder(false);
1047 set_has_boxed_value(false);
1048 _trap_can_recompile = false; // no traps emitted yet
1049 _major_progress = true; // start out assuming good things will happen
1050 set_has_unsafe_access(false);
1051 set_max_vector_size(0);
1052 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1053 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1054 set_decompile_count(0);
1055
1056 #ifndef PRODUCT
1057 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1058 #endif
1059
1060 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1061 _loop_opts_cnt = LoopOptsCount;
1062 set_do_inlining(Inline);
1063 set_max_inline_size(MaxInlineSize);
1064 set_freq_inline_size(FreqInlineSize);
1065 set_do_scheduling(OptoScheduling);
1066
1067 set_do_vector_loop(false);
1068 set_has_monitors(false);
1069 set_has_scoped_access(false);
1070
1071 if (AllowVectorizeOnDemand) {
1072 if (has_method() && _directive->VectorizeOption) {
1073 set_do_vector_loop(true);
1074 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1075 } else if (has_method() && method()->name() != nullptr &&
1076 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1077 set_do_vector_loop(true);
1078 }
1079 }
1080 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1081 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1326
1327 // Known instance (scalarizable allocation) alias only with itself.
1328 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1329 tj->is_oopptr()->is_known_instance();
1330
1331 // Process weird unsafe references.
1332 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1333 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1334 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1335 tj = TypeOopPtr::BOTTOM;
1336 ptr = tj->ptr();
1337 offset = tj->offset();
1338 }
1339
1340 // Array pointers need some flattening
1341 const TypeAryPtr* ta = tj->isa_aryptr();
1342 if (ta && ta->is_stable()) {
1343 // Erase stability property for alias analysis.
1344 tj = ta = ta->cast_to_stable(false);
1345 }
1346 if( ta && is_known_inst ) {
1347 if ( offset != Type::OffsetBot &&
1348 offset > arrayOopDesc::length_offset_in_bytes() ) {
1349 offset = Type::OffsetBot; // Flatten constant access into array body only
1350 tj = ta = ta->
1351 remove_speculative()->
1352 cast_to_ptr_type(ptr)->
1353 with_offset(offset);
1354 }
1355 } else if (ta) {
1356 // For arrays indexed by constant indices, we flatten the alias
1357 // space to include all of the array body. Only the header, klass
1358 // and array length can be accessed un-aliased.
1359 if( offset != Type::OffsetBot ) {
1360 if( ta->const_oop() ) { // MethodData* or Method*
1361 offset = Type::OffsetBot; // Flatten constant access into array body
1362 tj = ta = ta->
1363 remove_speculative()->
1364 cast_to_ptr_type(ptr)->
1365 cast_to_exactness(false)->
1366 with_offset(offset);
1367 } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1368 // range is OK as-is.
1369 tj = ta = TypeAryPtr::RANGE;
1370 } else if( offset == oopDesc::klass_offset_in_bytes() ) {
1371 tj = TypeInstPtr::KLASS; // all klass loads look alike
1372 ta = TypeAryPtr::RANGE; // generic ignored junk
1373 ptr = TypePtr::BotPTR;
1374 } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1375 tj = TypeInstPtr::MARK;
1376 ta = TypeAryPtr::RANGE; // generic ignored junk
1377 ptr = TypePtr::BotPTR;
1378 } else { // Random constant offset into array body
1379 offset = Type::OffsetBot; // Flatten constant access into array body
1380 tj = ta = ta->
1381 remove_speculative()->
1382 cast_to_ptr_type(ptr)->
1383 cast_to_exactness(false)->
1384 with_offset(offset);
1385 }
1386 }
1387 // Arrays of fixed size alias with arrays of unknown size.
1388 if (ta->size() != TypeInt::POS) {
1389 const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1390 tj = ta = ta->
1391 remove_speculative()->
1392 cast_to_ptr_type(ptr)->
1393 with_ary(tary)->
1394 cast_to_exactness(false);
1395 }
1396 // Arrays of known objects become arrays of unknown objects.
1397 if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1398 const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1399 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1400 }
1401 if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1402 const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1403 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1404 }
1405 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1406 // cannot be distinguished by bytecode alone.
1407 if (ta->elem() == TypeInt::BOOL) {
1408 const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1409 ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1410 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,offset);
1411 }
1412 // During the 2nd round of IterGVN, NotNull castings are removed.
1413 // Make sure the Bottom and NotNull variants alias the same.
1414 // Also, make sure exact and non-exact variants alias the same.
1415 if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1416 tj = ta = ta->
1417 remove_speculative()->
1418 cast_to_ptr_type(TypePtr::BotPTR)->
1419 cast_to_exactness(false)->
1420 with_offset(offset);
1421 }
1422 }
1423
1424 // Oop pointers need some flattening
1425 const TypeInstPtr *to = tj->isa_instptr();
1426 if (to && to != TypeOopPtr::BOTTOM) {
1427 ciInstanceKlass* ik = to->instance_klass();
1428 if( ptr == TypePtr::Constant ) {
1429 if (ik != ciEnv::current()->Class_klass() ||
1430 offset < ik->layout_helper_size_in_bytes()) {
1440 } else if( is_known_inst ) {
1441 tj = to; // Keep NotNull and klass_is_exact for instance type
1442 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1443 // During the 2nd round of IterGVN, NotNull castings are removed.
1444 // Make sure the Bottom and NotNull variants alias the same.
1445 // Also, make sure exact and non-exact variants alias the same.
1446 tj = to = to->
1447 remove_speculative()->
1448 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1449 cast_to_ptr_type(TypePtr::BotPTR)->
1450 cast_to_exactness(false);
1451 }
1452 if (to->speculative() != nullptr) {
1453 tj = to = to->remove_speculative();
1454 }
1455 // Canonicalize the holder of this field
1456 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1457 // First handle header references such as a LoadKlassNode, even if the
1458 // object's klass is unloaded at compile time (4965979).
1459 if (!is_known_inst) { // Do it only for non-instance types
1460 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, offset);
1461 }
1462 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1463 // Static fields are in the space above the normal instance
1464 // fields in the java.lang.Class instance.
1465 if (ik != ciEnv::current()->Class_klass()) {
1466 to = nullptr;
1467 tj = TypeOopPtr::BOTTOM;
1468 offset = tj->offset();
1469 }
1470 } else {
1471 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1472 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1473 assert(tj->offset() == offset, "no change to offset expected");
1474 bool xk = to->klass_is_exact();
1475 int instance_id = to->instance_id();
1476
1477 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1478 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1479 // its interfaces are included.
1480 if (xk && ik->equals(canonical_holder)) {
1481 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id), "exact type should be canonical type");
1482 } else {
1483 assert(xk || !is_known_inst, "Known instance should be exact type");
1484 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id);
1485 }
1486 }
1487 }
1488
1489 // Klass pointers to object array klasses need some flattening
1490 const TypeKlassPtr *tk = tj->isa_klassptr();
1491 if( tk ) {
1492 // If we are referencing a field within a Klass, we need
1493 // to assume the worst case of an Object. Both exact and
1494 // inexact types must flatten to the same alias class so
1495 // use NotNull as the PTR.
1496 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1497 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1498 env()->Object_klass(),
1499 offset);
1500 }
1501
1502 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1503 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1504 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1505 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), offset);
1506 } else {
1507 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, offset);
1508 }
1509 }
1510
1511 // Check for precise loads from the primary supertype array and force them
1512 // to the supertype cache alias index. Check for generic array loads from
1513 // the primary supertype array and also force them to the supertype cache
1514 // alias index. Since the same load can reach both, we need to merge
1515 // these 2 disparate memories into the same alias class. Since the
1516 // primary supertype array is read-only, there's no chance of confusion
1517 // where we bypass an array load and an array store.
1518 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1519 if (offset == Type::OffsetBot ||
1520 (offset >= primary_supers_offset &&
1521 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1522 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1523 offset = in_bytes(Klass::secondary_super_cache_offset());
1524 tj = tk = tk->with_offset(offset);
1525 }
1526 }
1527
1528 // Flatten all Raw pointers together.
1529 if (tj->base() == Type::RawPtr)
1530 tj = TypeRawPtr::BOTTOM;
1620 intptr_t key = (intptr_t) adr_type;
1621 key ^= key >> logAliasCacheSize;
1622 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1623 }
1624
1625
1626 //-----------------------------grow_alias_types--------------------------------
1627 void Compile::grow_alias_types() {
1628 const int old_ats = _max_alias_types; // how many before?
1629 const int new_ats = old_ats; // how many more?
1630 const int grow_ats = old_ats+new_ats; // how many now?
1631 _max_alias_types = grow_ats;
1632 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1633 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1634 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1635 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1636 }
1637
1638
1639 //--------------------------------find_alias_type------------------------------
1640 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field) {
1641 if (!do_aliasing()) {
1642 return alias_type(AliasIdxBot);
1643 }
1644
1645 AliasCacheEntry* ace = probe_alias_cache(adr_type);
1646 if (ace->_adr_type == adr_type) {
1647 return alias_type(ace->_index);
1648 }
1649
1650 // Handle special cases.
1651 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1652 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1653
1654 // Do it the slow way.
1655 const TypePtr* flat = flatten_alias_type(adr_type);
1656
1657 #ifdef ASSERT
1658 {
1659 ResourceMark rm;
1660 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1661 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1662 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1663 Type::str(adr_type));
1664 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1665 const TypeOopPtr* foop = flat->is_oopptr();
1666 // Scalarizable allocations have exact klass always.
1667 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1677 if (alias_type(i)->adr_type() == flat) {
1678 idx = i;
1679 break;
1680 }
1681 }
1682
1683 if (idx == AliasIdxTop) {
1684 if (no_create) return nullptr;
1685 // Grow the array if necessary.
1686 if (_num_alias_types == _max_alias_types) grow_alias_types();
1687 // Add a new alias type.
1688 idx = _num_alias_types++;
1689 _alias_types[idx]->Init(idx, flat);
1690 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1691 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1692 if (flat->isa_instptr()) {
1693 if (flat->offset() == java_lang_Class::klass_offset()
1694 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1695 alias_type(idx)->set_rewritable(false);
1696 }
1697 if (flat->isa_aryptr()) {
1698 #ifdef ASSERT
1699 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1700 // (T_BYTE has the weakest alignment and size restrictions...)
1701 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1702 #endif
1703 if (flat->offset() == TypePtr::OffsetBot) {
1704 alias_type(idx)->set_element(flat->is_aryptr()->elem());
1705 }
1706 }
1707 if (flat->isa_klassptr()) {
1708 if (UseCompactObjectHeaders) {
1709 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1710 alias_type(idx)->set_rewritable(false);
1711 }
1712 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1713 alias_type(idx)->set_rewritable(false);
1714 if (flat->offset() == in_bytes(Klass::access_flags_offset()))
1715 alias_type(idx)->set_rewritable(false);
1716 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1717 alias_type(idx)->set_rewritable(false);
1718 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1719 alias_type(idx)->set_rewritable(false);
1720 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1721 alias_type(idx)->set_rewritable(false);
1722 }
1723 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1724 // but the base pointer type is not distinctive enough to identify
1725 // references into JavaThread.)
1726
1727 // Check for final fields.
1728 const TypeInstPtr* tinst = flat->isa_instptr();
1729 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1730 ciField* field;
1731 if (tinst->const_oop() != nullptr &&
1732 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1733 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1734 // static field
1735 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1736 field = k->get_field_by_offset(tinst->offset(), true);
1737 } else {
1738 ciInstanceKlass *k = tinst->instance_klass();
1739 field = k->get_field_by_offset(tinst->offset(), false);
1740 }
1741 assert(field == nullptr ||
1742 original_field == nullptr ||
1743 (field->holder() == original_field->holder() &&
1744 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1745 field->is_static() == original_field->is_static()), "wrong field?");
1746 // Set field() and is_rewritable() attributes.
1747 if (field != nullptr) alias_type(idx)->set_field(field);
1748 }
1749 }
1750
1751 // Fill the cache for next time.
1752 ace->_adr_type = adr_type;
1753 ace->_index = idx;
1754 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1755
1756 // Might as well try to fill the cache for the flattened version, too.
1757 AliasCacheEntry* face = probe_alias_cache(flat);
1758 if (face->_adr_type == nullptr) {
1759 face->_adr_type = flat;
1760 face->_index = idx;
1761 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1762 }
1763
1764 return alias_type(idx);
1765 }
1766
1767
1768 Compile::AliasType* Compile::alias_type(ciField* field) {
1769 const TypeOopPtr* t;
1770 if (field->is_static())
1771 t = TypeInstPtr::make(field->holder()->java_mirror());
1772 else
1773 t = TypeOopPtr::make_from_klass_raw(field->holder());
1774 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1775 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1776 return atp;
1777 }
1778
1779
1780 //------------------------------have_alias_type--------------------------------
1781 bool Compile::have_alias_type(const TypePtr* adr_type) {
1863 assert(!C->major_progress(), "not cleared");
1864
1865 if (_for_post_loop_igvn.length() > 0) {
1866 while (_for_post_loop_igvn.length() > 0) {
1867 Node* n = _for_post_loop_igvn.pop();
1868 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1869 igvn._worklist.push(n);
1870 }
1871 igvn.optimize();
1872 if (failing()) return;
1873 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1874 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1875
1876 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1877 if (C->major_progress()) {
1878 C->clear_major_progress(); // ensure that major progress is now clear
1879 }
1880 }
1881 }
1882
1883 void Compile::record_for_merge_stores_igvn(Node* n) {
1884 if (!n->for_merge_stores_igvn()) {
1885 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
1886 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1887 _for_merge_stores_igvn.append(n);
1888 }
1889 }
1890
1891 void Compile::remove_from_merge_stores_igvn(Node* n) {
1892 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1893 _for_merge_stores_igvn.remove(n);
1894 }
1895
1896 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
1897 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
1898 // the stores, and we merge the wrong sequence of stores.
1899 // Example:
1900 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
1901 // Apply MergeStores:
1902 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
1981 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
1982 Bytecodes::Code c = iter.cur_bc();
1983 Node* lhs = nullptr;
1984 Node* rhs = nullptr;
1985 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
1986 lhs = unc->peek_operand(0);
1987 rhs = unc->peek_operand(1);
1988 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
1989 lhs = unc->peek_operand(0);
1990 }
1991
1992 ResourceMark rm;
1993 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
1994 assert(live_locals.is_valid(), "broken liveness info");
1995 int len = (int)live_locals.size();
1996
1997 for (int i = 0; i < len; i++) {
1998 Node* local = unc->local(jvms, i);
1999 // kill local using the liveness of next_bci.
2000 // give up when the local looks like an operand to secure reexecution.
2001 if (!live_locals.at(i) && !local->is_top() && local != lhs && local!= rhs) {
2002 uint idx = jvms->locoff() + i;
2003 #ifdef ASSERT
2004 if (PrintOpto && Verbose) {
2005 tty->print("[unstable_if] kill local#%d: ", idx);
2006 local->dump();
2007 tty->cr();
2008 }
2009 #endif
2010 igvn.replace_input_of(unc, idx, top());
2011 modified = true;
2012 }
2013 }
2014 }
2015
2016 // keep the mondified trap for late query
2017 if (modified) {
2018 trap->set_modified();
2019 } else {
2020 _unstable_if_traps.delete_at(i);
2021 }
2022 }
2023 igvn.optimize();
2024 }
2025
2026 // StringOpts and late inlining of string methods
2027 void Compile::inline_string_calls(bool parse_time) {
2028 {
2029 // remove useless nodes to make the usage analysis simpler
2030 ResourceMark rm;
2031 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2032 }
2033
2034 {
2035 ResourceMark rm;
2036 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2202
2203 if (_string_late_inlines.length() > 0) {
2204 assert(has_stringbuilder(), "inconsistent");
2205
2206 inline_string_calls(false);
2207
2208 if (failing()) return;
2209
2210 inline_incrementally_cleanup(igvn);
2211 }
2212
2213 set_inlining_incrementally(false);
2214 }
2215
2216 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2217 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2218 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2219 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2220 // as if "inlining_incrementally() == true" were set.
2221 assert(inlining_incrementally() == false, "not allowed");
2222 assert(_modified_nodes == nullptr, "not allowed");
2223 assert(_late_inlines.length() > 0, "sanity");
2224
2225 while (_late_inlines.length() > 0) {
2226 igvn_worklist()->ensure_empty(); // should be done with igvn
2227
2228 while (inline_incrementally_one()) {
2229 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2230 }
2231 if (failing()) return;
2232
2233 inline_incrementally_cleanup(igvn);
2234 }
2235 }
2236
2237 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2238 if (_loop_opts_cnt > 0) {
2239 while (major_progress() && (_loop_opts_cnt > 0)) {
2240 TracePhase tp(_t_idealLoop);
2241 PhaseIdealLoop::optimize(igvn, mode);
2242 _loop_opts_cnt--;
2243 if (failing()) return false;
2244 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2245 }
2246 }
2247 return true;
2248 }
2249
2250 // Remove edges from "root" to each SafePoint at a backward branch.
2251 // They were inserted during parsing (see add_safepoint()) to make
2252 // infinite loops without calls or exceptions visible to root, i.e.,
2253 // useful.
2254 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2359 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2360 }
2361 assert(!has_vbox_nodes(), "sanity");
2362
2363 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2364 Compile::TracePhase tp(_t_renumberLive);
2365 igvn_worklist()->ensure_empty(); // should be done with igvn
2366 {
2367 ResourceMark rm;
2368 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2369 }
2370 igvn.reset_from_gvn(initial_gvn());
2371 igvn.optimize();
2372 if (failing()) return;
2373 }
2374
2375 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2376 // safepoints
2377 remove_root_to_sfpts_edges(igvn);
2378
2379 if (failing()) return;
2380
2381 if (has_loops()) {
2382 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2383 }
2384
2385 // Perform escape analysis
2386 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2387 if (has_loops()) {
2388 // Cleanup graph (remove dead nodes).
2389 TracePhase tp(_t_idealLoop);
2390 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2391 if (failing()) return;
2392 }
2393 bool progress;
2394 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2395 do {
2396 ConnectionGraph::do_analysis(this, &igvn);
2397
2398 if (failing()) return;
2399
2400 int mcount = macro_count(); // Record number of allocations and locks before IGVN
2401
2402 // Optimize out fields loads from scalar replaceable allocations.
2403 igvn.optimize();
2404 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2405
2406 if (failing()) return;
2407
2408 if (congraph() != nullptr && macro_count() > 0) {
2409 TracePhase tp(_t_macroEliminate);
2410 PhaseMacroExpand mexp(igvn);
2411 mexp.eliminate_macro_nodes();
2412 if (failing()) return;
2413
2414 igvn.set_delay_transform(false);
2415 igvn.optimize();
2416 if (failing()) return;
2417
2418 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2419 }
2420
2421 ConnectionGraph::verify_ram_nodes(this, root());
2422 if (failing()) return;
2423
2424 progress = do_iterative_escape_analysis() &&
2425 (macro_count() < mcount) &&
2426 ConnectionGraph::has_candidates(this);
2427 // Try again if candidates exist and made progress
2428 // by removing some allocations and/or locks.
2429 } while (progress);
2430 }
2431
2432 // Loop transforms on the ideal graph. Range Check Elimination,
2433 // peeling, unrolling, etc.
2434
2435 // Set loop opts counter
2436 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2437 {
2488 // Loop transforms on the ideal graph. Range Check Elimination,
2489 // peeling, unrolling, etc.
2490 if (!optimize_loops(igvn, LoopOptsDefault)) {
2491 return;
2492 }
2493
2494 if (failing()) return;
2495
2496 C->clear_major_progress(); // ensure that major progress is now clear
2497
2498 process_for_post_loop_opts_igvn(igvn);
2499
2500 process_for_merge_stores_igvn(igvn);
2501
2502 if (failing()) return;
2503
2504 #ifdef ASSERT
2505 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
2506 #endif
2507
2508 {
2509 TracePhase tp(_t_macroExpand);
2510 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
2511 PhaseMacroExpand mex(igvn);
2512 if (mex.expand_macro_nodes()) {
2513 assert(failing(), "must bail out w/ explicit message");
2514 return;
2515 }
2516 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
2517 }
2518
2519 {
2520 TracePhase tp(_t_barrierExpand);
2521 if (bs->expand_barriers(this, igvn)) {
2522 assert(failing(), "must bail out w/ explicit message");
2523 return;
2524 }
2525 print_method(PHASE_BARRIER_EXPANSION, 2);
2526 }
2527
2528 if (C->max_vector_size() > 0) {
2529 C->optimize_logic_cones(igvn);
2530 igvn.optimize();
2531 if (failing()) return;
2532 }
2533
2534 DEBUG_ONLY( _modified_nodes = nullptr; )
2535
2536 assert(igvn._worklist.size() == 0, "not empty");
2537
2538 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
2539
2540 if (_late_inlines.length() > 0) {
2541 // More opportunities to optimize virtual and MH calls.
2542 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
2543 process_late_inline_calls_no_inline(igvn);
2544 if (failing()) return;
2545 }
2546 } // (End scope of igvn; run destructor if necessary for asserts.)
2547
2548 check_no_dead_use();
2549
2550 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
2551 // to remove hashes to unlock nodes for modifications.
2552 C->node_hash()->clear();
2553
2554 // A method with only infinite loops has no edges entering loops from root
2555 {
2556 TracePhase tp(_t_graphReshaping);
2557 if (final_graph_reshaping()) {
2558 assert(failing(), "must bail out w/ explicit message");
2559 return;
2560 }
2561 }
2562
2563 print_method(PHASE_OPTIMIZE_FINISHED, 2);
2564 DEBUG_ONLY(set_phase_optimize_finished();)
2565 }
3298 int nop = n->Opcode();
3299 // Clone shared simple arguments to uncommon calls, item (1).
3300 if (n->outcnt() > 1 &&
3301 !n->is_Proj() &&
3302 nop != Op_CreateEx &&
3303 nop != Op_CheckCastPP &&
3304 nop != Op_DecodeN &&
3305 nop != Op_DecodeNKlass &&
3306 !n->is_Mem() &&
3307 !n->is_Phi()) {
3308 Node *x = n->clone();
3309 call->set_req(TypeFunc::Parms, x);
3310 }
3311 }
3312 break;
3313 }
3314 case Op_StoreB:
3315 case Op_StoreC:
3316 case Op_StoreI:
3317 case Op_StoreL:
3318 case Op_CompareAndSwapB:
3319 case Op_CompareAndSwapS:
3320 case Op_CompareAndSwapI:
3321 case Op_CompareAndSwapL:
3322 case Op_CompareAndSwapP:
3323 case Op_CompareAndSwapN:
3324 case Op_WeakCompareAndSwapB:
3325 case Op_WeakCompareAndSwapS:
3326 case Op_WeakCompareAndSwapI:
3327 case Op_WeakCompareAndSwapL:
3328 case Op_WeakCompareAndSwapP:
3329 case Op_WeakCompareAndSwapN:
3330 case Op_CompareAndExchangeB:
3331 case Op_CompareAndExchangeS:
3332 case Op_CompareAndExchangeI:
3333 case Op_CompareAndExchangeL:
3334 case Op_CompareAndExchangeP:
3335 case Op_CompareAndExchangeN:
3336 case Op_GetAndAddS:
3337 case Op_GetAndAddB:
3841 k->subsume_by(m, this);
3842 }
3843 }
3844 }
3845 break;
3846 }
3847 case Op_CmpUL: {
3848 if (!Matcher::has_match_rule(Op_CmpUL)) {
3849 // No support for unsigned long comparisons
3850 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
3851 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
3852 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
3853 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
3854 Node* andl = new AndLNode(orl, remove_sign_mask);
3855 Node* cmp = new CmpLNode(andl, n->in(2));
3856 n->subsume_by(cmp, this);
3857 }
3858 break;
3859 }
3860 #ifdef ASSERT
3861 case Op_ConNKlass: {
3862 const TypePtr* tp = n->as_Type()->type()->make_ptr();
3863 ciKlass* klass = tp->is_klassptr()->exact_klass();
3864 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
3865 break;
3866 }
3867 #endif
3868 default:
3869 assert(!n->is_Call(), "");
3870 assert(!n->is_Mem(), "");
3871 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
3872 break;
3873 }
3874 }
3875
3876 //------------------------------final_graph_reshaping_walk---------------------
3877 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
3878 // requires that the walk visits a node's inputs before visiting the node.
3879 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3880 Unique_Node_List sfpt;
4216 }
4217 }
4218
4219 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4220 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4221 }
4222
4223 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4224 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4225 }
4226
4227 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4228 if (holder->is_initialized()) {
4229 return false;
4230 }
4231 if (holder->is_being_initialized()) {
4232 if (accessing_method->holder() == holder) {
4233 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4234 // <init>, or a static method. In all those cases, there was an initialization
4235 // barrier on the holder klass passed.
4236 if (accessing_method->is_static_initializer() ||
4237 accessing_method->is_object_initializer() ||
4238 accessing_method->is_static()) {
4239 return false;
4240 }
4241 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4242 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4243 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4244 // child class can become fully initialized while its parent class is still being initialized.
4245 if (accessing_method->is_static_initializer()) {
4246 return false;
4247 }
4248 }
4249 ciMethod* root = method(); // the root method of compilation
4250 if (root != accessing_method) {
4251 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4252 }
4253 }
4254 return true;
4255 }
4256
4257 #ifndef PRODUCT
4258 //------------------------------verify_bidirectional_edges---------------------
4259 // For each input edge to a node (ie - for each Use-Def edge), verify that
4260 // there is a corresponding Def-Use edge.
4261 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4262 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4263 uint stack_size = live_nodes() >> 4;
4264 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4265 if (root_and_safepoints != nullptr) {
4295 if (in != nullptr && !in->is_top()) {
4296 // Count instances of `next`
4297 int cnt = 0;
4298 for (uint idx = 0; idx < in->_outcnt; idx++) {
4299 if (in->_out[idx] == n) {
4300 cnt++;
4301 }
4302 }
4303 assert(cnt > 0, "Failed to find Def-Use edge.");
4304 // Check for duplicate edges
4305 // walk the input array downcounting the input edges to n
4306 for (uint j = 0; j < length; j++) {
4307 if (n->in(j) == in) {
4308 cnt--;
4309 }
4310 }
4311 assert(cnt == 0, "Mismatched edge count.");
4312 } else if (in == nullptr) {
4313 assert(i == 0 || i >= n->req() ||
4314 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4315 (n->is_Unlock() && i == (n->req() - 1)) ||
4316 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4317 "only region, phi, arraycopy, unlock or membar nodes have null data edges");
4318 } else {
4319 assert(in->is_top(), "sanity");
4320 // Nothing to check.
4321 }
4322 }
4323 }
4324 }
4325
4326 //------------------------------verify_graph_edges---------------------------
4327 // Walk the Graph and verify that there is a one-to-one correspondence
4328 // between Use-Def edges and Def-Use edges in the graph.
4329 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4330 if (VerifyGraphEdges) {
4331 Unique_Node_List visited;
4332
4333 // Call graph walk to check edges
4334 verify_bidirectional_edges(visited, root_and_safepoints);
4335 if (no_dead_code) {
4336 // Now make sure that no visited node is used by an unvisited node.
4337 bool dead_nodes = false;
4448 // (1) subklass is already limited to a subtype of superklass => always ok
4449 // (2) subklass does not overlap with superklass => always fail
4450 // (3) superklass has NO subtypes and we can check with a simple compare.
4451 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
4452 if (skip) {
4453 return SSC_full_test; // Let caller generate the general case.
4454 }
4455
4456 if (subk->is_java_subtype_of(superk)) {
4457 return SSC_always_true; // (0) and (1) this test cannot fail
4458 }
4459
4460 if (!subk->maybe_java_subtype_of(superk)) {
4461 return SSC_always_false; // (2) true path dead; no dynamic test needed
4462 }
4463
4464 const Type* superelem = superk;
4465 if (superk->isa_aryklassptr()) {
4466 int ignored;
4467 superelem = superk->is_aryklassptr()->base_element_type(ignored);
4468 }
4469
4470 if (superelem->isa_instklassptr()) {
4471 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
4472 if (!ik->has_subklass()) {
4473 if (!ik->is_final()) {
4474 // Add a dependency if there is a chance of a later subclass.
4475 dependencies()->assert_leaf_type(ik);
4476 }
4477 if (!superk->maybe_java_subtype_of(subk)) {
4478 return SSC_always_false;
4479 }
4480 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4481 }
4482 } else {
4483 // A primitive array type has no subtypes.
4484 return SSC_easy_test; // (3) caller can do a simple ptr comparison
4485 }
4486
4487 return SSC_full_test;
4929 const Type* t = igvn.type_or_null(n);
4930 assert((t == nullptr) || (t == t->remove_speculative()), "no more speculative types");
4931 if (n->is_Type()) {
4932 t = n->as_Type()->type();
4933 assert(t == t->remove_speculative(), "no more speculative types");
4934 }
4935 // Iterate over outs - endless loops is unreachable from below
4936 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4937 Node *m = n->fast_out(i);
4938 if (not_a_node(m)) {
4939 continue;
4940 }
4941 worklist.push(m);
4942 }
4943 }
4944 igvn.check_no_speculative_types();
4945 #endif
4946 }
4947 }
4948
4949 // Auxiliary methods to support randomized stressing/fuzzing.
4950
4951 void Compile::initialize_stress_seed(const DirectiveSet* directive) {
4952 if (FLAG_IS_DEFAULT(StressSeed) || (FLAG_IS_ERGO(StressSeed) && directive->RepeatCompilationOption)) {
4953 _stress_seed = static_cast<uint>(Ticks::now().nanoseconds());
4954 FLAG_SET_ERGO(StressSeed, _stress_seed);
4955 } else {
4956 _stress_seed = StressSeed;
4957 }
4958 if (_log != nullptr) {
4959 _log->elem("stress_test seed='%u'", _stress_seed);
4960 }
4961 }
4962
4963 int Compile::random() {
4964 _stress_seed = os::next_random(_stress_seed);
4965 return static_cast<int>(_stress_seed);
4966 }
4967
4968 // This method can be called the arbitrary number of times, with current count
5274 } else {
5275 _debug_network_printer->update_compiled_method(C->method());
5276 }
5277 tty->print_cr("Method printed over network stream to IGV");
5278 _debug_network_printer->print(name, C->root(), visible_nodes);
5279 }
5280 #endif
5281
5282 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5283 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5284 return value;
5285 }
5286 Node* result = nullptr;
5287 if (bt == T_BYTE) {
5288 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5289 result = new RShiftINode(result, phase->intcon(24));
5290 } else if (bt == T_BOOLEAN) {
5291 result = new AndINode(value, phase->intcon(0xFF));
5292 } else if (bt == T_CHAR) {
5293 result = new AndINode(value,phase->intcon(0xFFFF));
5294 } else {
5295 assert(bt == T_SHORT, "unexpected narrow type");
5296 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5297 result = new RShiftINode(result, phase->intcon(16));
5298 }
5299 if (transform_res) {
5300 result = phase->transform(result);
5301 }
5302 return result;
5303 }
5304
5305 void Compile::record_method_not_compilable_oom() {
5306 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5307 }
|
40 #include "gc/shared/c2/barrierSetC2.hpp"
41 #include "jfr/jfrEvents.hpp"
42 #include "jvm_io.h"
43 #include "memory/allocation.hpp"
44 #include "memory/arena.hpp"
45 #include "memory/resourceArea.hpp"
46 #include "opto/addnode.hpp"
47 #include "opto/block.hpp"
48 #include "opto/c2compiler.hpp"
49 #include "opto/callGenerator.hpp"
50 #include "opto/callnode.hpp"
51 #include "opto/castnode.hpp"
52 #include "opto/cfgnode.hpp"
53 #include "opto/chaitin.hpp"
54 #include "opto/compile.hpp"
55 #include "opto/connode.hpp"
56 #include "opto/convertnode.hpp"
57 #include "opto/divnode.hpp"
58 #include "opto/escape.hpp"
59 #include "opto/idealGraphPrinter.hpp"
60 #include "opto/inlinetypenode.hpp"
61 #include "opto/locknode.hpp"
62 #include "opto/loopnode.hpp"
63 #include "opto/machnode.hpp"
64 #include "opto/macro.hpp"
65 #include "opto/matcher.hpp"
66 #include "opto/mathexactnode.hpp"
67 #include "opto/memnode.hpp"
68 #include "opto/movenode.hpp"
69 #include "opto/mulnode.hpp"
70 #include "opto/narrowptrnode.hpp"
71 #include "opto/node.hpp"
72 #include "opto/opaquenode.hpp"
73 #include "opto/opcodes.hpp"
74 #include "opto/output.hpp"
75 #include "opto/parse.hpp"
76 #include "opto/phaseX.hpp"
77 #include "opto/rootnode.hpp"
78 #include "opto/runtime.hpp"
79 #include "opto/stringopts.hpp"
80 #include "opto/type.hpp"
81 #include "opto/vector.hpp"
82 #include "opto/vectornode.hpp"
83 #include "runtime/globals_extension.hpp"
84 #include "runtime/sharedRuntime.hpp"
85 #include "runtime/signature.hpp"
86 #include "runtime/stubRoutines.hpp"
87 #include "runtime/timer.hpp"
88 #include "utilities/align.hpp"
388 // as dead to be conservative about the dead node count at any
389 // given time.
390 if (!dead->is_Con()) {
391 record_dead_node(dead->_idx);
392 }
393 if (dead->is_macro()) {
394 remove_macro_node(dead);
395 }
396 if (dead->is_expensive()) {
397 remove_expensive_node(dead);
398 }
399 if (dead->is_OpaqueTemplateAssertionPredicate()) {
400 remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
401 }
402 if (dead->is_ParsePredicate()) {
403 remove_parse_predicate(dead->as_ParsePredicate());
404 }
405 if (dead->for_post_loop_opts_igvn()) {
406 remove_from_post_loop_opts_igvn(dead);
407 }
408 if (dead->is_InlineType()) {
409 remove_inline_type(dead);
410 }
411 if (dead->for_merge_stores_igvn()) {
412 remove_from_merge_stores_igvn(dead);
413 }
414 if (dead->is_Call()) {
415 remove_useless_late_inlines( &_late_inlines, dead);
416 remove_useless_late_inlines( &_string_late_inlines, dead);
417 remove_useless_late_inlines( &_boxing_late_inlines, dead);
418 remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
419
420 if (dead->is_CallStaticJava()) {
421 remove_unstable_if_trap(dead->as_CallStaticJava(), false);
422 }
423 }
424 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
425 bs->unregister_potential_barrier_node(dead);
426 }
427
428 // Disconnect all useless nodes by disconnecting those at the boundary.
429 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
430 uint next = 0;
438 // Use raw traversal of out edges since this code removes out edges
439 int max = n->outcnt();
440 for (int j = 0; j < max; ++j) {
441 Node* child = n->raw_out(j);
442 if (!useful.member(child)) {
443 assert(!child->is_top() || child != top(),
444 "If top is cached in Compile object it is in useful list");
445 // Only need to remove this out-edge to the useless node
446 n->raw_del_out(j);
447 --j;
448 --max;
449 if (child->is_data_proj_of_pure_function(n)) {
450 worklist.push(n);
451 }
452 }
453 }
454 if (n->outcnt() == 1 && n->has_special_unique_user()) {
455 assert(useful.member(n->unique_out()), "do not push a useless node");
456 worklist.push(n->unique_out());
457 }
458 if (n->outcnt() == 0) {
459 worklist.push(n);
460 }
461 }
462
463 remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
464 remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
465 // Remove useless Template Assertion Predicate opaque nodes
466 remove_useless_nodes(_template_assertion_predicate_opaques, useful);
467 remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
468 remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
469 remove_useless_nodes(_inline_type_nodes, useful); // remove useless inline type nodes
470 #ifdef ASSERT
471 if (_modified_nodes != nullptr) {
472 _modified_nodes->remove_useless_nodes(useful.member_set());
473 }
474 #endif
475 remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
476 remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
477 remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
478 #ifdef ASSERT
479 if (_modified_nodes != nullptr) {
480 _modified_nodes->remove_useless_nodes(useful.member_set());
481 }
482 #endif
483
484 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
485 bs->eliminate_useless_gc_barriers(useful, this);
486 // clean up the late inline lists
487 remove_useless_late_inlines( &_late_inlines, useful);
488 remove_useless_late_inlines( &_string_late_inlines, useful);
489 remove_useless_late_inlines( &_boxing_late_inlines, useful);
490 remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
491 debug_only(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
492 }
493
494 // ============================================================================
636
637 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
638 Options options, DirectiveSet* directive)
639 : Phase(Compiler),
640 _compile_id(ci_env->compile_id()),
641 _options(options),
642 _method(target),
643 _entry_bci(osr_bci),
644 _ilt(nullptr),
645 _stub_function(nullptr),
646 _stub_name(nullptr),
647 _stub_entry_point(nullptr),
648 _max_node_limit(MaxNodeLimit),
649 _post_loop_opts_phase(false),
650 _merge_stores_phase(false),
651 _allow_macro_nodes(true),
652 _inlining_progress(false),
653 _inlining_incrementally(false),
654 _do_cleanup(false),
655 _has_reserved_stack_access(target->has_reserved_stack_access()),
656 _has_circular_inline_type(false),
657 #ifndef PRODUCT
658 _igv_idx(0),
659 _trace_opto_output(directive->TraceOptoOutputOption),
660 #endif
661 _has_method_handle_invokes(false),
662 _clinit_barrier_on_entry(false),
663 _stress_seed(0),
664 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
665 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
666 _env(ci_env),
667 _directive(directive),
668 _log(ci_env->log()),
669 _first_failure_details(nullptr),
670 _intrinsics(comp_arena(), 0, 0, nullptr),
671 _macro_nodes(comp_arena(), 8, 0, nullptr),
672 _parse_predicates(comp_arena(), 8, 0, nullptr),
673 _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
674 _expensive_nodes(comp_arena(), 8, 0, nullptr),
675 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
676 _inline_type_nodes (comp_arena(), 8, 0, nullptr),
677 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
678 _unstable_if_traps(comp_arena(), 8, 0, nullptr),
679 _coarsened_locks(comp_arena(), 8, 0, nullptr),
680 _congraph(nullptr),
681 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
682 _unique(0),
683 _dead_node_count(0),
684 _dead_node_list(comp_arena()),
685 _node_arena_one(mtCompiler, Arena::Tag::tag_node),
686 _node_arena_two(mtCompiler, Arena::Tag::tag_node),
687 _node_arena(&_node_arena_one),
688 _mach_constant_base_node(nullptr),
689 _Compile_types(mtCompiler, Arena::Tag::tag_type),
690 _initial_gvn(nullptr),
691 _igvn_worklist(nullptr),
692 _types(nullptr),
693 _node_hash(nullptr),
694 _late_inlines(comp_arena(), 2, 0, nullptr),
695 _string_late_inlines(comp_arena(), 2, 0, nullptr),
696 _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
761 #define MINIMUM_NODE_HASH 1023
762
763 // GVN that will be run immediately on new nodes
764 uint estimated_size = method()->code_size()*4+64;
765 estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
766 _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
767 _types = new (comp_arena()) Type_Array(comp_arena());
768 _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
769 PhaseGVN gvn;
770 set_initial_gvn(&gvn);
771
772 { // Scope for timing the parser
773 TracePhase tp(_t_parser);
774
775 // Put top into the hash table ASAP.
776 initial_gvn()->transform(top());
777
778 // Set up tf(), start(), and find a CallGenerator.
779 CallGenerator* cg = nullptr;
780 if (is_osr_compilation()) {
781 init_tf(TypeFunc::make(method(), /* is_osr_compilation = */ true));
782 StartNode* s = new StartOSRNode(root(), tf()->domain_sig());
783 initial_gvn()->set_type_bottom(s);
784 verify_start(s);
785 cg = CallGenerator::for_osr(method(), entry_bci());
786 } else {
787 // Normal case.
788 init_tf(TypeFunc::make(method()));
789 StartNode* s = new StartNode(root(), tf()->domain_cc());
790 initial_gvn()->set_type_bottom(s);
791 verify_start(s);
792 if (method()->intrinsic_id() == vmIntrinsics::_Reference_get) {
793 // With java.lang.ref.reference.get() we must go through the
794 // intrinsic - even when get() is the root
795 // method of the compile - so that, if necessary, the value in
796 // the referent field of the reference object gets recorded by
797 // the pre-barrier code.
798 cg = find_intrinsic(method(), false);
799 }
800 if (cg == nullptr) {
801 float past_uses = method()->interpreter_invocation_count();
802 float expected_uses = past_uses;
803 cg = CallGenerator::for_inline(method(), expected_uses);
804 }
805 }
806 if (failing()) return;
807 if (cg == nullptr) {
808 const char* reason = InlineTree::check_can_parse(method());
809 assert(reason != nullptr, "expect reason for parse failure");
880 print_ideal_ir("print_ideal");
881 }
882 #endif
883
884 #ifdef ASSERT
885 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
886 bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
887 #endif
888
889 // Dump compilation data to replay it.
890 if (directive->DumpReplayOption) {
891 env()->dump_replay_data(_compile_id);
892 }
893 if (directive->DumpInlineOption && (ilt() != nullptr)) {
894 env()->dump_inline_data(_compile_id);
895 }
896
897 // Now that we know the size of all the monitors we can add a fixed slot
898 // for the original deopt pc.
899 int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
900 if (needs_stack_repair()) {
901 // One extra slot for the special stack increment value
902 next_slot += 2;
903 }
904 // TODO 8284443 Only reserve extra slot if needed
905 if (InlineTypeReturnedAsFields) {
906 // One extra slot to hold the null marker for a nullable
907 // inline type return if we run out of registers.
908 next_slot += 2;
909 }
910 set_fixed_slots(next_slot);
911
912 // Compute when to use implicit null checks. Used by matching trap based
913 // nodes and NullCheck optimization.
914 set_allowed_deopt_reasons();
915
916 // Now generate code
917 Code_Gen();
918 }
919
920 //------------------------------Compile----------------------------------------
921 // Compile a runtime stub
922 Compile::Compile(ciEnv* ci_env,
923 TypeFunc_generator generator,
924 address stub_function,
925 const char *stub_name,
926 int is_fancy_jump,
927 bool pass_tls,
928 bool return_pc,
929 DirectiveSet* directive)
930 : Phase(Compiler),
931 _compile_id(0),
932 _options(Options::for_runtime_stub()),
933 _method(nullptr),
934 _entry_bci(InvocationEntryBci),
935 _stub_function(stub_function),
936 _stub_name(stub_name),
937 _stub_entry_point(nullptr),
938 _max_node_limit(MaxNodeLimit),
939 _post_loop_opts_phase(false),
940 _merge_stores_phase(false),
941 _allow_macro_nodes(true),
942 _inlining_progress(false),
943 _inlining_incrementally(false),
944 _has_reserved_stack_access(false),
945 _has_circular_inline_type(false),
946 #ifndef PRODUCT
947 _igv_idx(0),
948 _trace_opto_output(directive->TraceOptoOutputOption),
949 #endif
950 _has_method_handle_invokes(false),
951 _clinit_barrier_on_entry(false),
952 _stress_seed(0),
953 _comp_arena(mtCompiler, Arena::Tag::tag_comp),
954 _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
955 _env(ci_env),
956 _directive(directive),
957 _log(ci_env->log()),
958 _first_failure_details(nullptr),
959 _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
960 _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
961 _congraph(nullptr),
962 NOT_PRODUCT(_igv_printer(nullptr) COMMA)
963 _unique(0),
964 _dead_node_count(0),
965 _dead_node_list(comp_arena()),
1067
1068 _fixed_slots = 0;
1069 set_has_split_ifs(false);
1070 set_has_loops(false); // first approximation
1071 set_has_stringbuilder(false);
1072 set_has_boxed_value(false);
1073 _trap_can_recompile = false; // no traps emitted yet
1074 _major_progress = true; // start out assuming good things will happen
1075 set_has_unsafe_access(false);
1076 set_max_vector_size(0);
1077 set_clear_upper_avx(false); //false as default for clear upper bits of ymm registers
1078 Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1079 set_decompile_count(0);
1080
1081 #ifndef PRODUCT
1082 Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1083 #endif
1084
1085 set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1086 _loop_opts_cnt = LoopOptsCount;
1087 _has_flat_accesses = false;
1088 _flat_accesses_share_alias = true;
1089 _scalarize_in_safepoints = false;
1090
1091 set_do_inlining(Inline);
1092 set_max_inline_size(MaxInlineSize);
1093 set_freq_inline_size(FreqInlineSize);
1094 set_do_scheduling(OptoScheduling);
1095
1096 set_do_vector_loop(false);
1097 set_has_monitors(false);
1098 set_has_scoped_access(false);
1099
1100 if (AllowVectorizeOnDemand) {
1101 if (has_method() && _directive->VectorizeOption) {
1102 set_do_vector_loop(true);
1103 NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n", method()->name()->as_quoted_ascii());})
1104 } else if (has_method() && method()->name() != nullptr &&
1105 method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1106 set_do_vector_loop(true);
1107 }
1108 }
1109 set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1110 NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n", method()->name()->as_quoted_ascii());})
1355
1356 // Known instance (scalarizable allocation) alias only with itself.
1357 bool is_known_inst = tj->isa_oopptr() != nullptr &&
1358 tj->is_oopptr()->is_known_instance();
1359
1360 // Process weird unsafe references.
1361 if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1362 assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1363 assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1364 tj = TypeOopPtr::BOTTOM;
1365 ptr = tj->ptr();
1366 offset = tj->offset();
1367 }
1368
1369 // Array pointers need some flattening
1370 const TypeAryPtr* ta = tj->isa_aryptr();
1371 if (ta && ta->is_stable()) {
1372 // Erase stability property for alias analysis.
1373 tj = ta = ta->cast_to_stable(false);
1374 }
1375 if (ta && ta->is_not_flat()) {
1376 // Erase not flat property for alias analysis.
1377 tj = ta = ta->cast_to_not_flat(false);
1378 }
1379 if (ta && ta->is_not_null_free()) {
1380 // Erase not null free property for alias analysis.
1381 tj = ta = ta->cast_to_not_null_free(false);
1382 }
1383
1384 if( ta && is_known_inst ) {
1385 if ( offset != Type::OffsetBot &&
1386 offset > arrayOopDesc::length_offset_in_bytes() ) {
1387 offset = Type::OffsetBot; // Flatten constant access into array body only
1388 tj = ta = ta->
1389 remove_speculative()->
1390 cast_to_ptr_type(ptr)->
1391 with_offset(offset);
1392 }
1393 } else if (ta) {
1394 // For arrays indexed by constant indices, we flatten the alias
1395 // space to include all of the array body. Only the header, klass
1396 // and array length can be accessed un-aliased.
1397 // For flat inline type array, each field has its own slice so
1398 // we must include the field offset.
1399 if( offset != Type::OffsetBot ) {
1400 if( ta->const_oop() ) { // MethodData* or Method*
1401 offset = Type::OffsetBot; // Flatten constant access into array body
1402 tj = ta = ta->
1403 remove_speculative()->
1404 cast_to_ptr_type(ptr)->
1405 cast_to_exactness(false)->
1406 with_offset(offset);
1407 } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1408 // range is OK as-is.
1409 tj = ta = TypeAryPtr::RANGE;
1410 } else if( offset == oopDesc::klass_offset_in_bytes() ) {
1411 tj = TypeInstPtr::KLASS; // all klass loads look alike
1412 ta = TypeAryPtr::RANGE; // generic ignored junk
1413 ptr = TypePtr::BotPTR;
1414 } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1415 tj = TypeInstPtr::MARK;
1416 ta = TypeAryPtr::RANGE; // generic ignored junk
1417 ptr = TypePtr::BotPTR;
1418 } else { // Random constant offset into array body
1419 offset = Type::OffsetBot; // Flatten constant access into array body
1420 tj = ta = ta->
1421 remove_speculative()->
1422 cast_to_ptr_type(ptr)->
1423 cast_to_exactness(false)->
1424 with_offset(offset);
1425 }
1426 }
1427 // Arrays of fixed size alias with arrays of unknown size.
1428 if (ta->size() != TypeInt::POS) {
1429 const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1430 tj = ta = ta->
1431 remove_speculative()->
1432 cast_to_ptr_type(ptr)->
1433 with_ary(tary)->
1434 cast_to_exactness(false);
1435 }
1436 // Arrays of known objects become arrays of unknown objects.
1437 if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1438 const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1439 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
1440 }
1441 if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1442 const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1443 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
1444 }
1445 // Initially all flattened array accesses share a single slice
1446 if (ta->is_flat() && ta->elem() != TypeInstPtr::BOTTOM && _flat_accesses_share_alias) {
1447 const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size(), /* stable= */ false, /* flat= */ true);
1448 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), Type::Offset(Type::OffsetBot));
1449 }
1450 // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1451 // cannot be distinguished by bytecode alone.
1452 if (ta->elem() == TypeInt::BOOL) {
1453 const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1454 ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1455 tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,Type::Offset(offset), ta->field_offset());
1456 }
1457 // During the 2nd round of IterGVN, NotNull castings are removed.
1458 // Make sure the Bottom and NotNull variants alias the same.
1459 // Also, make sure exact and non-exact variants alias the same.
1460 if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1461 tj = ta = ta->
1462 remove_speculative()->
1463 cast_to_ptr_type(TypePtr::BotPTR)->
1464 cast_to_exactness(false)->
1465 with_offset(offset);
1466 }
1467 }
1468
1469 // Oop pointers need some flattening
1470 const TypeInstPtr *to = tj->isa_instptr();
1471 if (to && to != TypeOopPtr::BOTTOM) {
1472 ciInstanceKlass* ik = to->instance_klass();
1473 if( ptr == TypePtr::Constant ) {
1474 if (ik != ciEnv::current()->Class_klass() ||
1475 offset < ik->layout_helper_size_in_bytes()) {
1485 } else if( is_known_inst ) {
1486 tj = to; // Keep NotNull and klass_is_exact for instance type
1487 } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1488 // During the 2nd round of IterGVN, NotNull castings are removed.
1489 // Make sure the Bottom and NotNull variants alias the same.
1490 // Also, make sure exact and non-exact variants alias the same.
1491 tj = to = to->
1492 remove_speculative()->
1493 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1494 cast_to_ptr_type(TypePtr::BotPTR)->
1495 cast_to_exactness(false);
1496 }
1497 if (to->speculative() != nullptr) {
1498 tj = to = to->remove_speculative();
1499 }
1500 // Canonicalize the holder of this field
1501 if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1502 // First handle header references such as a LoadKlassNode, even if the
1503 // object's klass is unloaded at compile time (4965979).
1504 if (!is_known_inst) { // Do it only for non-instance types
1505 tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, Type::Offset(offset));
1506 }
1507 } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1508 // Static fields are in the space above the normal instance
1509 // fields in the java.lang.Class instance.
1510 if (ik != ciEnv::current()->Class_klass()) {
1511 to = nullptr;
1512 tj = TypeOopPtr::BOTTOM;
1513 offset = tj->offset();
1514 }
1515 } else {
1516 ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1517 assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1518 assert(tj->offset() == offset, "no change to offset expected");
1519 bool xk = to->klass_is_exact();
1520 int instance_id = to->instance_id();
1521
1522 // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1523 // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1524 // its interfaces are included.
1525 if (xk && ik->equals(canonical_holder)) {
1526 assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id), "exact type should be canonical type");
1527 } else {
1528 assert(xk || !is_known_inst, "Known instance should be exact type");
1529 tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id);
1530 }
1531 }
1532 }
1533
1534 // Klass pointers to object array klasses need some flattening
1535 const TypeKlassPtr *tk = tj->isa_klassptr();
1536 if( tk ) {
1537 // If we are referencing a field within a Klass, we need
1538 // to assume the worst case of an Object. Both exact and
1539 // inexact types must flatten to the same alias class so
1540 // use NotNull as the PTR.
1541 if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1542 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1543 env()->Object_klass(),
1544 Type::Offset(offset));
1545 }
1546
1547 if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1548 ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1549 if (!k || !k->is_loaded()) { // Only fails for some -Xcomp runs
1550 tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), Type::Offset(offset));
1551 } else {
1552 tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, Type::Offset(offset), tk->is_not_flat(), tk->is_not_null_free(), tk->is_flat(), tk->is_null_free());
1553 }
1554 }
1555 // Check for precise loads from the primary supertype array and force them
1556 // to the supertype cache alias index. Check for generic array loads from
1557 // the primary supertype array and also force them to the supertype cache
1558 // alias index. Since the same load can reach both, we need to merge
1559 // these 2 disparate memories into the same alias class. Since the
1560 // primary supertype array is read-only, there's no chance of confusion
1561 // where we bypass an array load and an array store.
1562 int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1563 if (offset == Type::OffsetBot ||
1564 (offset >= primary_supers_offset &&
1565 offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1566 offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1567 offset = in_bytes(Klass::secondary_super_cache_offset());
1568 tj = tk = tk->with_offset(offset);
1569 }
1570 }
1571
1572 // Flatten all Raw pointers together.
1573 if (tj->base() == Type::RawPtr)
1574 tj = TypeRawPtr::BOTTOM;
1664 intptr_t key = (intptr_t) adr_type;
1665 key ^= key >> logAliasCacheSize;
1666 return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1667 }
1668
1669
1670 //-----------------------------grow_alias_types--------------------------------
1671 void Compile::grow_alias_types() {
1672 const int old_ats = _max_alias_types; // how many before?
1673 const int new_ats = old_ats; // how many more?
1674 const int grow_ats = old_ats+new_ats; // how many now?
1675 _max_alias_types = grow_ats;
1676 _alias_types = REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1677 AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1678 Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1679 for (int i = 0; i < new_ats; i++) _alias_types[old_ats+i] = &ats[i];
1680 }
1681
1682
1683 //--------------------------------find_alias_type------------------------------
1684 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field, bool uncached) {
1685 if (!do_aliasing()) {
1686 return alias_type(AliasIdxBot);
1687 }
1688
1689 AliasCacheEntry* ace = nullptr;
1690 if (!uncached) {
1691 ace = probe_alias_cache(adr_type);
1692 if (ace->_adr_type == adr_type) {
1693 return alias_type(ace->_index);
1694 }
1695 }
1696
1697 // Handle special cases.
1698 if (adr_type == nullptr) return alias_type(AliasIdxTop);
1699 if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot);
1700
1701 // Do it the slow way.
1702 const TypePtr* flat = flatten_alias_type(adr_type);
1703
1704 #ifdef ASSERT
1705 {
1706 ResourceMark rm;
1707 assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1708 Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1709 assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1710 Type::str(adr_type));
1711 if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1712 const TypeOopPtr* foop = flat->is_oopptr();
1713 // Scalarizable allocations have exact klass always.
1714 bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1724 if (alias_type(i)->adr_type() == flat) {
1725 idx = i;
1726 break;
1727 }
1728 }
1729
1730 if (idx == AliasIdxTop) {
1731 if (no_create) return nullptr;
1732 // Grow the array if necessary.
1733 if (_num_alias_types == _max_alias_types) grow_alias_types();
1734 // Add a new alias type.
1735 idx = _num_alias_types++;
1736 _alias_types[idx]->Init(idx, flat);
1737 if (flat == TypeInstPtr::KLASS) alias_type(idx)->set_rewritable(false);
1738 if (flat == TypeAryPtr::RANGE) alias_type(idx)->set_rewritable(false);
1739 if (flat->isa_instptr()) {
1740 if (flat->offset() == java_lang_Class::klass_offset()
1741 && flat->is_instptr()->instance_klass() == env()->Class_klass())
1742 alias_type(idx)->set_rewritable(false);
1743 }
1744 ciField* field = nullptr;
1745 if (flat->isa_aryptr()) {
1746 #ifdef ASSERT
1747 const int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1748 // (T_BYTE has the weakest alignment and size restrictions...)
1749 assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1750 #endif
1751 const Type* elemtype = flat->is_aryptr()->elem();
1752 if (flat->offset() == TypePtr::OffsetBot) {
1753 alias_type(idx)->set_element(elemtype);
1754 }
1755 int field_offset = flat->is_aryptr()->field_offset().get();
1756 if (flat->is_flat() &&
1757 field_offset != Type::OffsetBot) {
1758 ciInlineKlass* vk = elemtype->inline_klass();
1759 field_offset += vk->payload_offset();
1760 field = vk->get_field_by_offset(field_offset, false);
1761 }
1762 }
1763 if (flat->isa_klassptr()) {
1764 if (UseCompactObjectHeaders) {
1765 if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1766 alias_type(idx)->set_rewritable(false);
1767 }
1768 if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1769 alias_type(idx)->set_rewritable(false);
1770 if (flat->offset() == in_bytes(Klass::access_flags_offset()))
1771 alias_type(idx)->set_rewritable(false);
1772 if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1773 alias_type(idx)->set_rewritable(false);
1774 if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1775 alias_type(idx)->set_rewritable(false);
1776 if (flat->offset() == in_bytes(Klass::layout_helper_offset()))
1777 alias_type(idx)->set_rewritable(false);
1778 if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1779 alias_type(idx)->set_rewritable(false);
1780 }
1781 // %%% (We would like to finalize JavaThread::threadObj_offset(),
1782 // but the base pointer type is not distinctive enough to identify
1783 // references into JavaThread.)
1784
1785 // Check for final fields.
1786 const TypeInstPtr* tinst = flat->isa_instptr();
1787 if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1788 if (tinst->const_oop() != nullptr &&
1789 tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1790 tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1791 // static field
1792 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1793 field = k->get_field_by_offset(tinst->offset(), true);
1794 } else if (tinst->is_inlinetypeptr()) {
1795 // Inline type field
1796 ciInlineKlass* vk = tinst->inline_klass();
1797 field = vk->get_field_by_offset(tinst->offset(), false);
1798 } else {
1799 ciInstanceKlass *k = tinst->instance_klass();
1800 field = k->get_field_by_offset(tinst->offset(), false);
1801 }
1802 }
1803 assert(field == nullptr ||
1804 original_field == nullptr ||
1805 (field->holder() == original_field->holder() &&
1806 field->offset_in_bytes() == original_field->offset_in_bytes() &&
1807 field->is_static() == original_field->is_static()), "wrong field?");
1808 // Set field() and is_rewritable() attributes.
1809 if (field != nullptr) {
1810 alias_type(idx)->set_field(field);
1811 if (flat->isa_aryptr()) {
1812 // Fields of flat arrays are rewritable although they are declared final
1813 assert(flat->is_flat(), "must be a flat array");
1814 alias_type(idx)->set_rewritable(true);
1815 }
1816 }
1817 }
1818
1819 // Fill the cache for next time.
1820 if (!uncached) {
1821 ace->_adr_type = adr_type;
1822 ace->_index = idx;
1823 assert(alias_type(adr_type) == alias_type(idx), "type must be installed");
1824
1825 // Might as well try to fill the cache for the flattened version, too.
1826 AliasCacheEntry* face = probe_alias_cache(flat);
1827 if (face->_adr_type == nullptr) {
1828 face->_adr_type = flat;
1829 face->_index = idx;
1830 assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1831 }
1832 }
1833
1834 return alias_type(idx);
1835 }
1836
1837
1838 Compile::AliasType* Compile::alias_type(ciField* field) {
1839 const TypeOopPtr* t;
1840 if (field->is_static())
1841 t = TypeInstPtr::make(field->holder()->java_mirror());
1842 else
1843 t = TypeOopPtr::make_from_klass_raw(field->holder());
1844 AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1845 assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1846 return atp;
1847 }
1848
1849
1850 //------------------------------have_alias_type--------------------------------
1851 bool Compile::have_alias_type(const TypePtr* adr_type) {
1933 assert(!C->major_progress(), "not cleared");
1934
1935 if (_for_post_loop_igvn.length() > 0) {
1936 while (_for_post_loop_igvn.length() > 0) {
1937 Node* n = _for_post_loop_igvn.pop();
1938 n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1939 igvn._worklist.push(n);
1940 }
1941 igvn.optimize();
1942 if (failing()) return;
1943 assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1944 assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1945
1946 // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1947 if (C->major_progress()) {
1948 C->clear_major_progress(); // ensure that major progress is now clear
1949 }
1950 }
1951 }
1952
1953 void Compile::add_inline_type(Node* n) {
1954 assert(n->is_InlineType(), "unexpected node");
1955 _inline_type_nodes.push(n);
1956 }
1957
1958 void Compile::remove_inline_type(Node* n) {
1959 assert(n->is_InlineType(), "unexpected node");
1960 if (_inline_type_nodes.contains(n)) {
1961 _inline_type_nodes.remove(n);
1962 }
1963 }
1964
1965 // Does the return value keep otherwise useless inline type allocations alive?
1966 static bool return_val_keeps_allocations_alive(Node* ret_val) {
1967 ResourceMark rm;
1968 Unique_Node_List wq;
1969 wq.push(ret_val);
1970 bool some_allocations = false;
1971 for (uint i = 0; i < wq.size(); i++) {
1972 Node* n = wq.at(i);
1973 if (n->outcnt() > 1) {
1974 // Some other use for the allocation
1975 return false;
1976 } else if (n->is_InlineType()) {
1977 wq.push(n->in(1));
1978 } else if (n->is_Phi()) {
1979 for (uint j = 1; j < n->req(); j++) {
1980 wq.push(n->in(j));
1981 }
1982 } else if (n->is_CheckCastPP() &&
1983 n->in(1)->is_Proj() &&
1984 n->in(1)->in(0)->is_Allocate()) {
1985 some_allocations = true;
1986 } else if (n->is_CheckCastPP()) {
1987 wq.push(n->in(1));
1988 }
1989 }
1990 return some_allocations;
1991 }
1992
1993 void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
1994 // Make sure that the return value does not keep an otherwise unused allocation alive
1995 if (tf()->returns_inline_type_as_fields()) {
1996 Node* ret = nullptr;
1997 for (uint i = 1; i < root()->req(); i++) {
1998 Node* in = root()->in(i);
1999 if (in->Opcode() == Op_Return) {
2000 assert(ret == nullptr, "only one return");
2001 ret = in;
2002 }
2003 }
2004 if (ret != nullptr) {
2005 Node* ret_val = ret->in(TypeFunc::Parms);
2006 if (igvn.type(ret_val)->isa_oopptr() &&
2007 return_val_keeps_allocations_alive(ret_val)) {
2008 igvn.replace_input_of(ret, TypeFunc::Parms, InlineTypeNode::tagged_klass(igvn.type(ret_val)->inline_klass(), igvn));
2009 assert(ret_val->outcnt() == 0, "should be dead now");
2010 igvn.remove_dead_node(ret_val);
2011 }
2012 }
2013 }
2014 if (_inline_type_nodes.length() == 0) {
2015 return;
2016 }
2017 // Scalarize inline types in safepoint debug info.
2018 // Delay this until all inlining is over to avoid getting inconsistent debug info.
2019 set_scalarize_in_safepoints(true);
2020 for (int i = _inline_type_nodes.length()-1; i >= 0; i--) {
2021 InlineTypeNode* vt = _inline_type_nodes.at(i)->as_InlineType();
2022 vt->make_scalar_in_safepoints(&igvn);
2023 igvn.record_for_igvn(vt);
2024 }
2025 if (remove) {
2026 // Remove inline type nodes by replacing them with their oop input
2027 while (_inline_type_nodes.length() > 0) {
2028 InlineTypeNode* vt = _inline_type_nodes.pop()->as_InlineType();
2029 if (vt->outcnt() == 0) {
2030 igvn.remove_dead_node(vt);
2031 continue;
2032 }
2033 for (DUIterator i = vt->outs(); vt->has_out(i); i++) {
2034 DEBUG_ONLY(bool must_be_buffered = false);
2035 Node* u = vt->out(i);
2036 // Check if any users are blackholes. If so, rewrite them to use either the
2037 // allocated buffer, or individual components, instead of the inline type node
2038 // that goes away.
2039 if (u->is_Blackhole()) {
2040 BlackholeNode* bh = u->as_Blackhole();
2041
2042 // Unlink the old input
2043 int idx = bh->find_edge(vt);
2044 assert(idx != -1, "The edge should be there");
2045 bh->del_req(idx);
2046 --i;
2047
2048 if (vt->is_allocated(&igvn)) {
2049 // Already has the allocated instance, blackhole that
2050 bh->add_req(vt->get_oop());
2051 } else {
2052 // Not allocated yet, blackhole the components
2053 for (uint c = 0; c < vt->field_count(); c++) {
2054 bh->add_req(vt->field_value(c));
2055 }
2056 }
2057
2058 // Node modified, record for IGVN
2059 igvn.record_for_igvn(bh);
2060 }
2061 #ifdef ASSERT
2062 // Verify that inline type is buffered when replacing by oop
2063 else if (u->is_InlineType()) {
2064 // InlineType uses don't need buffering because they are about to be replaced as well
2065 } else if (u->is_Phi()) {
2066 // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through
2067 } else {
2068 must_be_buffered = true;
2069 }
2070 if (must_be_buffered && !vt->is_allocated(&igvn)) {
2071 vt->dump(0);
2072 u->dump(0);
2073 assert(false, "Should have been buffered");
2074 }
2075 #endif
2076 }
2077 igvn.replace_node(vt, vt->get_oop());
2078 }
2079 }
2080 igvn.optimize();
2081 }
2082
2083 void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) {
2084 if (!_has_flat_accesses) {
2085 return;
2086 }
2087 // Initially, all flat array accesses share the same slice to
2088 // keep dependencies with Object[] array accesses (that could be
2089 // to a flat array) correct. We're done with parsing so we
2090 // now know all flat array accesses in this compile
2091 // unit. Let's move flat array accesses to their own slice,
2092 // one per element field. This should help memory access
2093 // optimizations.
2094 ResourceMark rm;
2095 Unique_Node_List wq;
2096 wq.push(root());
2097
2098 Node_List mergememnodes;
2099 Node_List memnodes;
2100
2101 // Alias index currently shared by all flat memory accesses
2102 int index = get_alias_index(TypeAryPtr::INLINES);
2103
2104 // Find MergeMem nodes and flat array accesses
2105 for (uint i = 0; i < wq.size(); i++) {
2106 Node* n = wq.at(i);
2107 if (n->is_Mem()) {
2108 const TypePtr* adr_type = nullptr;
2109 adr_type = get_adr_type(get_alias_index(n->adr_type()));
2110 if (adr_type == TypeAryPtr::INLINES) {
2111 memnodes.push(n);
2112 }
2113 } else if (n->is_MergeMem()) {
2114 MergeMemNode* mm = n->as_MergeMem();
2115 if (mm->memory_at(index) != mm->base_memory()) {
2116 mergememnodes.push(n);
2117 }
2118 }
2119 for (uint j = 0; j < n->req(); j++) {
2120 Node* m = n->in(j);
2121 if (m != nullptr) {
2122 wq.push(m);
2123 }
2124 }
2125 }
2126
2127 if (memnodes.size() > 0) {
2128 _flat_accesses_share_alias = false;
2129
2130 // We are going to change the slice for the flat array
2131 // accesses so we need to clear the cache entries that refer to
2132 // them.
2133 for (uint i = 0; i < AliasCacheSize; i++) {
2134 AliasCacheEntry* ace = &_alias_cache[i];
2135 if (ace->_adr_type != nullptr &&
2136 ace->_adr_type->is_flat()) {
2137 ace->_adr_type = nullptr;
2138 ace->_index = (i != 0) ? 0 : AliasIdxTop; // Make sure the nullptr adr_type resolves to AliasIdxTop
2139 }
2140 }
2141
2142 // Find what aliases we are going to add
2143 int start_alias = num_alias_types()-1;
2144 int stop_alias = 0;
2145
2146 for (uint i = 0; i < memnodes.size(); i++) {
2147 Node* m = memnodes.at(i);
2148 const TypePtr* adr_type = nullptr;
2149 adr_type = m->adr_type();
2150 #ifdef ASSERT
2151 m->as_Mem()->set_adr_type(adr_type);
2152 #endif
2153 int idx = get_alias_index(adr_type);
2154 start_alias = MIN2(start_alias, idx);
2155 stop_alias = MAX2(stop_alias, idx);
2156 }
2157
2158 assert(stop_alias >= start_alias, "should have expanded aliases");
2159
2160 Node_Stack stack(0);
2161 #ifdef ASSERT
2162 VectorSet seen(Thread::current()->resource_area());
2163 #endif
2164 // Now let's fix the memory graph so each flat array access
2165 // is moved to the right slice. Start from the MergeMem nodes.
2166 uint last = unique();
2167 for (uint i = 0; i < mergememnodes.size(); i++) {
2168 MergeMemNode* current = mergememnodes.at(i)->as_MergeMem();
2169 Node* n = current->memory_at(index);
2170 MergeMemNode* mm = nullptr;
2171 do {
2172 // Follow memory edges through memory accesses, phis and
2173 // narrow membars and push nodes on the stack. Once we hit
2174 // bottom memory, we pop element off the stack one at a
2175 // time, in reverse order, and move them to the right slice
2176 // by changing their memory edges.
2177 if ((n->is_Phi() && n->adr_type() != TypePtr::BOTTOM) || n->is_Mem() || n->adr_type() == TypeAryPtr::INLINES) {
2178 assert(!seen.test_set(n->_idx), "");
2179 // Uses (a load for instance) will need to be moved to the
2180 // right slice as well and will get a new memory state
2181 // that we don't know yet. The use could also be the
2182 // backedge of a loop. We put a place holder node between
2183 // the memory node and its uses. We replace that place
2184 // holder with the correct memory state once we know it,
2185 // i.e. when nodes are popped off the stack. Using the
2186 // place holder make the logic work in the presence of
2187 // loops.
2188 if (n->outcnt() > 1) {
2189 Node* place_holder = nullptr;
2190 assert(!n->has_out_with(Op_Node), "");
2191 for (DUIterator k = n->outs(); n->has_out(k); k++) {
2192 Node* u = n->out(k);
2193 if (u != current && u->_idx < last) {
2194 bool success = false;
2195 for (uint l = 0; l < u->req(); l++) {
2196 if (!stack.is_empty() && u == stack.node() && l == stack.index()) {
2197 continue;
2198 }
2199 Node* in = u->in(l);
2200 if (in == n) {
2201 if (place_holder == nullptr) {
2202 place_holder = new Node(1);
2203 place_holder->init_req(0, n);
2204 }
2205 igvn.replace_input_of(u, l, place_holder);
2206 success = true;
2207 }
2208 }
2209 if (success) {
2210 --k;
2211 }
2212 }
2213 }
2214 }
2215 if (n->is_Phi()) {
2216 stack.push(n, 1);
2217 n = n->in(1);
2218 } else if (n->is_Mem()) {
2219 stack.push(n, n->req());
2220 n = n->in(MemNode::Memory);
2221 } else {
2222 assert(n->is_Proj() && n->in(0)->Opcode() == Op_MemBarCPUOrder, "");
2223 stack.push(n, n->req());
2224 n = n->in(0)->in(TypeFunc::Memory);
2225 }
2226 } else {
2227 assert(n->adr_type() == TypePtr::BOTTOM || (n->Opcode() == Op_Node && n->_idx >= last) || (n->is_Proj() && n->in(0)->is_Initialize()), "");
2228 // Build a new MergeMem node to carry the new memory state
2229 // as we build it. IGVN should fold extraneous MergeMem
2230 // nodes.
2231 mm = MergeMemNode::make(n);
2232 igvn.register_new_node_with_optimizer(mm);
2233 while (stack.size() > 0) {
2234 Node* m = stack.node();
2235 uint idx = stack.index();
2236 if (m->is_Mem()) {
2237 // Move memory node to its new slice
2238 const TypePtr* adr_type = m->adr_type();
2239 int alias = get_alias_index(adr_type);
2240 Node* prev = mm->memory_at(alias);
2241 igvn.replace_input_of(m, MemNode::Memory, prev);
2242 mm->set_memory_at(alias, m);
2243 } else if (m->is_Phi()) {
2244 // We need as many new phis as there are new aliases
2245 igvn.replace_input_of(m, idx, mm);
2246 if (idx == m->req()-1) {
2247 Node* r = m->in(0);
2248 for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2249 const TypePtr* adr_type = get_adr_type(j);
2250 if (!adr_type->isa_aryptr() || !adr_type->is_flat() || j == (uint)index) {
2251 continue;
2252 }
2253 Node* phi = new PhiNode(r, Type::MEMORY, get_adr_type(j));
2254 igvn.register_new_node_with_optimizer(phi);
2255 for (uint k = 1; k < m->req(); k++) {
2256 phi->init_req(k, m->in(k)->as_MergeMem()->memory_at(j));
2257 }
2258 mm->set_memory_at(j, phi);
2259 }
2260 Node* base_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2261 igvn.register_new_node_with_optimizer(base_phi);
2262 for (uint k = 1; k < m->req(); k++) {
2263 base_phi->init_req(k, m->in(k)->as_MergeMem()->base_memory());
2264 }
2265 mm->set_base_memory(base_phi);
2266 }
2267 } else {
2268 // This is a MemBarCPUOrder node from
2269 // Parse::array_load()/Parse::array_store(), in the
2270 // branch that handles flat arrays hidden under
2271 // an Object[] array. We also need one new membar per
2272 // new alias to keep the unknown access that the
2273 // membars protect properly ordered with accesses to
2274 // known flat array.
2275 assert(m->is_Proj(), "projection expected");
2276 Node* ctrl = m->in(0)->in(TypeFunc::Control);
2277 igvn.replace_input_of(m->in(0), TypeFunc::Control, top());
2278 for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2279 const TypePtr* adr_type = get_adr_type(j);
2280 if (!adr_type->isa_aryptr() || !adr_type->is_flat() || j == (uint)index) {
2281 continue;
2282 }
2283 MemBarNode* mb = new MemBarCPUOrderNode(this, j, nullptr);
2284 igvn.register_new_node_with_optimizer(mb);
2285 Node* mem = mm->memory_at(j);
2286 mb->init_req(TypeFunc::Control, ctrl);
2287 mb->init_req(TypeFunc::Memory, mem);
2288 ctrl = new ProjNode(mb, TypeFunc::Control);
2289 igvn.register_new_node_with_optimizer(ctrl);
2290 mem = new ProjNode(mb, TypeFunc::Memory);
2291 igvn.register_new_node_with_optimizer(mem);
2292 mm->set_memory_at(j, mem);
2293 }
2294 igvn.replace_node(m->in(0)->as_Multi()->proj_out(TypeFunc::Control), ctrl);
2295 }
2296 if (idx < m->req()-1) {
2297 idx += 1;
2298 stack.set_index(idx);
2299 n = m->in(idx);
2300 break;
2301 }
2302 // Take care of place holder nodes
2303 if (m->has_out_with(Op_Node)) {
2304 Node* place_holder = m->find_out_with(Op_Node);
2305 if (place_holder != nullptr) {
2306 Node* mm_clone = mm->clone();
2307 igvn.register_new_node_with_optimizer(mm_clone);
2308 Node* hook = new Node(1);
2309 hook->init_req(0, mm);
2310 igvn.replace_node(place_holder, mm_clone);
2311 hook->destruct(&igvn);
2312 }
2313 assert(!m->has_out_with(Op_Node), "place holder should be gone now");
2314 }
2315 stack.pop();
2316 }
2317 }
2318 } while(stack.size() > 0);
2319 // Fix the memory state at the MergeMem we started from
2320 igvn.rehash_node_delayed(current);
2321 for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2322 const TypePtr* adr_type = get_adr_type(j);
2323 if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2324 continue;
2325 }
2326 current->set_memory_at(j, mm);
2327 }
2328 current->set_memory_at(index, current->base_memory());
2329 }
2330 igvn.optimize();
2331 }
2332 print_method(PHASE_SPLIT_INLINES_ARRAY, 2);
2333 #ifdef ASSERT
2334 if (!_flat_accesses_share_alias) {
2335 wq.clear();
2336 wq.push(root());
2337 for (uint i = 0; i < wq.size(); i++) {
2338 Node* n = wq.at(i);
2339 assert(n->adr_type() != TypeAryPtr::INLINES, "should have been removed from the graph");
2340 for (uint j = 0; j < n->req(); j++) {
2341 Node* m = n->in(j);
2342 if (m != nullptr) {
2343 wq.push(m);
2344 }
2345 }
2346 }
2347 }
2348 #endif
2349 }
2350
2351 void Compile::record_for_merge_stores_igvn(Node* n) {
2352 if (!n->for_merge_stores_igvn()) {
2353 assert(!_for_merge_stores_igvn.contains(n), "duplicate");
2354 n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2355 _for_merge_stores_igvn.append(n);
2356 }
2357 }
2358
2359 void Compile::remove_from_merge_stores_igvn(Node* n) {
2360 n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2361 _for_merge_stores_igvn.remove(n);
2362 }
2363
2364 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
2365 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
2366 // the stores, and we merge the wrong sequence of stores.
2367 // Example:
2368 // StoreI RangeCheck StoreI StoreI RangeCheck StoreI
2369 // Apply MergeStores:
2370 // StoreI RangeCheck [ StoreL ] RangeCheck StoreI
2449 assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2450 Bytecodes::Code c = iter.cur_bc();
2451 Node* lhs = nullptr;
2452 Node* rhs = nullptr;
2453 if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2454 lhs = unc->peek_operand(0);
2455 rhs = unc->peek_operand(1);
2456 } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2457 lhs = unc->peek_operand(0);
2458 }
2459
2460 ResourceMark rm;
2461 const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2462 assert(live_locals.is_valid(), "broken liveness info");
2463 int len = (int)live_locals.size();
2464
2465 for (int i = 0; i < len; i++) {
2466 Node* local = unc->local(jvms, i);
2467 // kill local using the liveness of next_bci.
2468 // give up when the local looks like an operand to secure reexecution.
2469 if (!live_locals.at(i) && !local->is_top() && local != lhs && local != rhs) {
2470 uint idx = jvms->locoff() + i;
2471 #ifdef ASSERT
2472 if (PrintOpto && Verbose) {
2473 tty->print("[unstable_if] kill local#%d: ", idx);
2474 local->dump();
2475 tty->cr();
2476 }
2477 #endif
2478 igvn.replace_input_of(unc, idx, top());
2479 modified = true;
2480 }
2481 }
2482 }
2483
2484 // keep the modified trap for late query
2485 if (modified) {
2486 trap->set_modified();
2487 } else {
2488 _unstable_if_traps.delete_at(i);
2489 }
2490 }
2491 igvn.optimize();
2492 }
2493
2494 // StringOpts and late inlining of string methods
2495 void Compile::inline_string_calls(bool parse_time) {
2496 {
2497 // remove useless nodes to make the usage analysis simpler
2498 ResourceMark rm;
2499 PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2500 }
2501
2502 {
2503 ResourceMark rm;
2504 print_method(PHASE_BEFORE_STRINGOPTS, 3);
2670
2671 if (_string_late_inlines.length() > 0) {
2672 assert(has_stringbuilder(), "inconsistent");
2673
2674 inline_string_calls(false);
2675
2676 if (failing()) return;
2677
2678 inline_incrementally_cleanup(igvn);
2679 }
2680
2681 set_inlining_incrementally(false);
2682 }
2683
2684 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2685 // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2686 // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2687 // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2688 // as if "inlining_incrementally() == true" were set.
2689 assert(inlining_incrementally() == false, "not allowed");
2690 #ifdef ASSERT
2691 Unique_Node_List* modified_nodes = _modified_nodes;
2692 _modified_nodes = nullptr;
2693 #endif
2694 assert(_late_inlines.length() > 0, "sanity");
2695
2696 while (_late_inlines.length() > 0) {
2697 igvn_worklist()->ensure_empty(); // should be done with igvn
2698
2699 while (inline_incrementally_one()) {
2700 assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2701 }
2702 if (failing()) return;
2703
2704 inline_incrementally_cleanup(igvn);
2705 }
2706 DEBUG_ONLY( _modified_nodes = modified_nodes; )
2707 }
2708
2709 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2710 if (_loop_opts_cnt > 0) {
2711 while (major_progress() && (_loop_opts_cnt > 0)) {
2712 TracePhase tp(_t_idealLoop);
2713 PhaseIdealLoop::optimize(igvn, mode);
2714 _loop_opts_cnt--;
2715 if (failing()) return false;
2716 if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2717 }
2718 }
2719 return true;
2720 }
2721
2722 // Remove edges from "root" to each SafePoint at a backward branch.
2723 // They were inserted during parsing (see add_safepoint()) to make
2724 // infinite loops without calls or exceptions visible to root, i.e.,
2725 // useful.
2726 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2831 print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2832 }
2833 assert(!has_vbox_nodes(), "sanity");
2834
2835 if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2836 Compile::TracePhase tp(_t_renumberLive);
2837 igvn_worklist()->ensure_empty(); // should be done with igvn
2838 {
2839 ResourceMark rm;
2840 PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2841 }
2842 igvn.reset_from_gvn(initial_gvn());
2843 igvn.optimize();
2844 if (failing()) return;
2845 }
2846
2847 // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2848 // safepoints
2849 remove_root_to_sfpts_edges(igvn);
2850
2851 // Process inline type nodes now that all inlining is over
2852 process_inline_types(igvn);
2853
2854 adjust_flat_array_access_aliases(igvn);
2855
2856 if (failing()) return;
2857
2858 {
2859 // Eliminate some macro nodes before EA to reduce analysis pressure
2860 PhaseMacroExpand mexp(igvn);
2861 mexp.eliminate_macro_nodes();
2862 if (failing()) {
2863 return;
2864 }
2865 igvn.set_delay_transform(false);
2866 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2867 }
2868
2869 if (has_loops()) {
2870 print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2871 }
2872
2873 // Perform escape analysis
2874 if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2875 if (has_loops()) {
2876 // Cleanup graph (remove dead nodes).
2877 TracePhase tp(_t_idealLoop);
2878 PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2879 if (failing()) {
2880 return;
2881 }
2882 print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2883
2884 // Eliminate some macro nodes before EA to reduce analysis pressure
2885 PhaseMacroExpand mexp(igvn);
2886 mexp.eliminate_macro_nodes();
2887 if (failing()) {
2888 return;
2889 }
2890 igvn.set_delay_transform(false);
2891 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2892 }
2893
2894 bool progress;
2895 do {
2896 ConnectionGraph::do_analysis(this, &igvn);
2897
2898 if (failing()) return;
2899
2900 int mcount = macro_count(); // Record number of allocations and locks before IGVN
2901
2902 // Optimize out fields loads from scalar replaceable allocations.
2903 igvn.optimize();
2904 print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2905
2906 if (failing()) return;
2907
2908 if (congraph() != nullptr && macro_count() > 0) {
2909 TracePhase tp(_t_macroEliminate);
2910 PhaseMacroExpand mexp(igvn);
2911 mexp.eliminate_macro_nodes();
2912 if (failing()) {
2913 return;
2914 }
2915 igvn.set_delay_transform(false);
2916 print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2917 }
2918
2919 ConnectionGraph::verify_ram_nodes(this, root());
2920 if (failing()) return;
2921
2922 progress = do_iterative_escape_analysis() &&
2923 (macro_count() < mcount) &&
2924 ConnectionGraph::has_candidates(this);
2925 // Try again if candidates exist and made progress
2926 // by removing some allocations and/or locks.
2927 } while (progress);
2928 }
2929
2930 // Loop transforms on the ideal graph. Range Check Elimination,
2931 // peeling, unrolling, etc.
2932
2933 // Set loop opts counter
2934 if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2935 {
2986 // Loop transforms on the ideal graph. Range Check Elimination,
2987 // peeling, unrolling, etc.
2988 if (!optimize_loops(igvn, LoopOptsDefault)) {
2989 return;
2990 }
2991
2992 if (failing()) return;
2993
2994 C->clear_major_progress(); // ensure that major progress is now clear
2995
2996 process_for_post_loop_opts_igvn(igvn);
2997
2998 process_for_merge_stores_igvn(igvn);
2999
3000 if (failing()) return;
3001
3002 #ifdef ASSERT
3003 bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
3004 #endif
3005
3006 assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
3007
3008 if (_late_inlines.length() > 0) {
3009 // More opportunities to optimize virtual and MH calls.
3010 // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
3011 process_late_inline_calls_no_inline(igvn);
3012 }
3013
3014 {
3015 TracePhase tp(_t_macroExpand);
3016 PhaseMacroExpand mex(igvn);
3017 // Last attempt to eliminate macro nodes.
3018 mex.eliminate_macro_nodes();
3019 if (failing()) {
3020 return;
3021 }
3022
3023 print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
3024 if (mex.expand_macro_nodes()) {
3025 assert(failing(), "must bail out w/ explicit message");
3026 return;
3027 }
3028 print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
3029 }
3030
3031 // Process inline type nodes again and remove them. From here
3032 // on we don't need to keep track of field values anymore.
3033 process_inline_types(igvn, /* remove= */ true);
3034
3035 {
3036 TracePhase tp(_t_barrierExpand);
3037 if (bs->expand_barriers(this, igvn)) {
3038 assert(failing(), "must bail out w/ explicit message");
3039 return;
3040 }
3041 print_method(PHASE_BARRIER_EXPANSION, 2);
3042 }
3043
3044 if (C->max_vector_size() > 0) {
3045 C->optimize_logic_cones(igvn);
3046 igvn.optimize();
3047 if (failing()) return;
3048 }
3049
3050 DEBUG_ONLY( _modified_nodes = nullptr; )
3051 DEBUG_ONLY( _late_inlines.clear(); )
3052
3053 assert(igvn._worklist.size() == 0, "not empty");
3054 } // (End scope of igvn; run destructor if necessary for asserts.)
3055
3056 check_no_dead_use();
3057
3058 // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
3059 // to remove hashes to unlock nodes for modifications.
3060 C->node_hash()->clear();
3061
3062 // A method with only infinite loops has no edges entering loops from root
3063 {
3064 TracePhase tp(_t_graphReshaping);
3065 if (final_graph_reshaping()) {
3066 assert(failing(), "must bail out w/ explicit message");
3067 return;
3068 }
3069 }
3070
3071 print_method(PHASE_OPTIMIZE_FINISHED, 2);
3072 DEBUG_ONLY(set_phase_optimize_finished();)
3073 }
3806 int nop = n->Opcode();
3807 // Clone shared simple arguments to uncommon calls, item (1).
3808 if (n->outcnt() > 1 &&
3809 !n->is_Proj() &&
3810 nop != Op_CreateEx &&
3811 nop != Op_CheckCastPP &&
3812 nop != Op_DecodeN &&
3813 nop != Op_DecodeNKlass &&
3814 !n->is_Mem() &&
3815 !n->is_Phi()) {
3816 Node *x = n->clone();
3817 call->set_req(TypeFunc::Parms, x);
3818 }
3819 }
3820 break;
3821 }
3822 case Op_StoreB:
3823 case Op_StoreC:
3824 case Op_StoreI:
3825 case Op_StoreL:
3826 case Op_StoreLSpecial:
3827 case Op_CompareAndSwapB:
3828 case Op_CompareAndSwapS:
3829 case Op_CompareAndSwapI:
3830 case Op_CompareAndSwapL:
3831 case Op_CompareAndSwapP:
3832 case Op_CompareAndSwapN:
3833 case Op_WeakCompareAndSwapB:
3834 case Op_WeakCompareAndSwapS:
3835 case Op_WeakCompareAndSwapI:
3836 case Op_WeakCompareAndSwapL:
3837 case Op_WeakCompareAndSwapP:
3838 case Op_WeakCompareAndSwapN:
3839 case Op_CompareAndExchangeB:
3840 case Op_CompareAndExchangeS:
3841 case Op_CompareAndExchangeI:
3842 case Op_CompareAndExchangeL:
3843 case Op_CompareAndExchangeP:
3844 case Op_CompareAndExchangeN:
3845 case Op_GetAndAddS:
3846 case Op_GetAndAddB:
4350 k->subsume_by(m, this);
4351 }
4352 }
4353 }
4354 break;
4355 }
4356 case Op_CmpUL: {
4357 if (!Matcher::has_match_rule(Op_CmpUL)) {
4358 // No support for unsigned long comparisons
4359 ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
4360 Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
4361 Node* orl = new OrLNode(n->in(1), sign_bit_mask);
4362 ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
4363 Node* andl = new AndLNode(orl, remove_sign_mask);
4364 Node* cmp = new CmpLNode(andl, n->in(2));
4365 n->subsume_by(cmp, this);
4366 }
4367 break;
4368 }
4369 #ifdef ASSERT
4370 case Op_InlineType: {
4371 n->dump(-1);
4372 assert(false, "inline type node was not removed");
4373 break;
4374 }
4375 case Op_ConNKlass: {
4376 const TypePtr* tp = n->as_Type()->type()->make_ptr();
4377 ciKlass* klass = tp->is_klassptr()->exact_klass();
4378 assert(klass->is_in_encoding_range(), "klass cannot be compressed");
4379 break;
4380 }
4381 #endif
4382 default:
4383 assert(!n->is_Call(), "");
4384 assert(!n->is_Mem(), "");
4385 assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
4386 break;
4387 }
4388 }
4389
4390 //------------------------------final_graph_reshaping_walk---------------------
4391 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
4392 // requires that the walk visits a node's inputs before visiting the node.
4393 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
4394 Unique_Node_List sfpt;
4730 }
4731 }
4732
4733 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4734 return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4735 }
4736
4737 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4738 return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4739 }
4740
4741 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4742 if (holder->is_initialized()) {
4743 return false;
4744 }
4745 if (holder->is_being_initialized()) {
4746 if (accessing_method->holder() == holder) {
4747 // Access inside a class. The barrier can be elided when access happens in <clinit>,
4748 // <init>, or a static method. In all those cases, there was an initialization
4749 // barrier on the holder klass passed.
4750 if (accessing_method->is_class_initializer() ||
4751 accessing_method->is_object_constructor() ||
4752 accessing_method->is_static()) {
4753 return false;
4754 }
4755 } else if (accessing_method->holder()->is_subclass_of(holder)) {
4756 // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4757 // In case of <init> or a static method, the barrier is on the subclass is not enough:
4758 // child class can become fully initialized while its parent class is still being initialized.
4759 if (accessing_method->is_class_initializer()) {
4760 return false;
4761 }
4762 }
4763 ciMethod* root = method(); // the root method of compilation
4764 if (root != accessing_method) {
4765 return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4766 }
4767 }
4768 return true;
4769 }
4770
4771 #ifndef PRODUCT
4772 //------------------------------verify_bidirectional_edges---------------------
4773 // For each input edge to a node (ie - for each Use-Def edge), verify that
4774 // there is a corresponding Def-Use edge.
4775 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4776 // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4777 uint stack_size = live_nodes() >> 4;
4778 Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4779 if (root_and_safepoints != nullptr) {
4809 if (in != nullptr && !in->is_top()) {
4810 // Count instances of `next`
4811 int cnt = 0;
4812 for (uint idx = 0; idx < in->_outcnt; idx++) {
4813 if (in->_out[idx] == n) {
4814 cnt++;
4815 }
4816 }
4817 assert(cnt > 0, "Failed to find Def-Use edge.");
4818 // Check for duplicate edges
4819 // walk the input array downcounting the input edges to n
4820 for (uint j = 0; j < length; j++) {
4821 if (n->in(j) == in) {
4822 cnt--;
4823 }
4824 }
4825 assert(cnt == 0, "Mismatched edge count.");
4826 } else if (in == nullptr) {
4827 assert(i == 0 || i >= n->req() ||
4828 n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4829 (n->is_Allocate() && i >= AllocateNode::InlineType) ||
4830 (n->is_Unlock() && i == (n->req() - 1)) ||
4831 (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4832 "only region, phi, arraycopy, allocate, unlock or membar nodes have null data edges");
4833 } else {
4834 assert(in->is_top(), "sanity");
4835 // Nothing to check.
4836 }
4837 }
4838 }
4839 }
4840
4841 //------------------------------verify_graph_edges---------------------------
4842 // Walk the Graph and verify that there is a one-to-one correspondence
4843 // between Use-Def edges and Def-Use edges in the graph.
4844 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4845 if (VerifyGraphEdges) {
4846 Unique_Node_List visited;
4847
4848 // Call graph walk to check edges
4849 verify_bidirectional_edges(visited, root_and_safepoints);
4850 if (no_dead_code) {
4851 // Now make sure that no visited node is used by an unvisited node.
4852 bool dead_nodes = false;
4963 // (1) subklass is already limited to a subtype of superklass => always ok
4964 // (2) subklass does not overlap with superklass => always fail
4965 // (3) superklass has NO subtypes and we can check with a simple compare.
4966 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
4967 if (skip) {
4968 return SSC_full_test; // Let caller generate the general case.
4969 }
4970
4971 if (subk->is_java_subtype_of(superk)) {
4972 return SSC_always_true; // (0) and (1) this test cannot fail
4973 }
4974
4975 if (!subk->maybe_java_subtype_of(superk)) {
4976 return SSC_always_false; // (2) true path dead; no dynamic test needed
4977 }
4978
4979 const Type* superelem = superk;
4980 if (superk->isa_aryklassptr()) {
4981 int ignored;
4982 superelem = superk->is_aryklassptr()->base_element_type(ignored);
4983
4984 // Do not fold the subtype check to an array klass pointer comparison for null-able inline type arrays
4985 // because null-free [LMyValue <: null-able [LMyValue but the klasses are different. Perform a full test.
4986 if (!superk->is_aryklassptr()->is_null_free() && superk->is_aryklassptr()->elem()->isa_instklassptr() &&
4987 superk->is_aryklassptr()->elem()->is_instklassptr()->instance_klass()->is_inlinetype()) {
4988 return SSC_full_test;
4989 }
4990 }
4991
4992 if (superelem->isa_instklassptr()) {
4993 ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
4994 if (!ik->has_subklass()) {
4995 if (!ik->is_final()) {
4996 // Add a dependency if there is a chance of a later subclass.
4997 dependencies()->assert_leaf_type(ik);
4998 }
4999 if (!superk->maybe_java_subtype_of(subk)) {
5000 return SSC_always_false;
5001 }
5002 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5003 }
5004 } else {
5005 // A primitive array type has no subtypes.
5006 return SSC_easy_test; // (3) caller can do a simple ptr comparison
5007 }
5008
5009 return SSC_full_test;
5451 const Type* t = igvn.type_or_null(n);
5452 assert((t == nullptr) || (t == t->remove_speculative()), "no more speculative types");
5453 if (n->is_Type()) {
5454 t = n->as_Type()->type();
5455 assert(t == t->remove_speculative(), "no more speculative types");
5456 }
5457 // Iterate over outs - endless loops is unreachable from below
5458 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
5459 Node *m = n->fast_out(i);
5460 if (not_a_node(m)) {
5461 continue;
5462 }
5463 worklist.push(m);
5464 }
5465 }
5466 igvn.check_no_speculative_types();
5467 #endif
5468 }
5469 }
5470
5471 Node* Compile::optimize_acmp(PhaseGVN* phase, Node* a, Node* b) {
5472 const TypeInstPtr* ta = phase->type(a)->isa_instptr();
5473 const TypeInstPtr* tb = phase->type(b)->isa_instptr();
5474 if (!EnableValhalla || ta == nullptr || tb == nullptr ||
5475 ta->is_zero_type() || tb->is_zero_type() ||
5476 !ta->can_be_inline_type() || !tb->can_be_inline_type()) {
5477 // Use old acmp if one operand is null or not an inline type
5478 return new CmpPNode(a, b);
5479 } else if (ta->is_inlinetypeptr() || tb->is_inlinetypeptr()) {
5480 // We know that one operand is an inline type. Therefore,
5481 // new acmp will only return true if both operands are nullptr.
5482 // Check if both operands are null by or'ing the oops.
5483 a = phase->transform(new CastP2XNode(nullptr, a));
5484 b = phase->transform(new CastP2XNode(nullptr, b));
5485 a = phase->transform(new OrXNode(a, b));
5486 return new CmpXNode(a, phase->MakeConX(0));
5487 }
5488 // Use new acmp
5489 return nullptr;
5490 }
5491
5492 // Auxiliary methods to support randomized stressing/fuzzing.
5493
5494 void Compile::initialize_stress_seed(const DirectiveSet* directive) {
5495 if (FLAG_IS_DEFAULT(StressSeed) || (FLAG_IS_ERGO(StressSeed) && directive->RepeatCompilationOption)) {
5496 _stress_seed = static_cast<uint>(Ticks::now().nanoseconds());
5497 FLAG_SET_ERGO(StressSeed, _stress_seed);
5498 } else {
5499 _stress_seed = StressSeed;
5500 }
5501 if (_log != nullptr) {
5502 _log->elem("stress_test seed='%u'", _stress_seed);
5503 }
5504 }
5505
5506 int Compile::random() {
5507 _stress_seed = os::next_random(_stress_seed);
5508 return static_cast<int>(_stress_seed);
5509 }
5510
5511 // This method can be called the arbitrary number of times, with current count
5817 } else {
5818 _debug_network_printer->update_compiled_method(C->method());
5819 }
5820 tty->print_cr("Method printed over network stream to IGV");
5821 _debug_network_printer->print(name, C->root(), visible_nodes);
5822 }
5823 #endif
5824
5825 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5826 if (type != nullptr && phase->type(value)->higher_equal(type)) {
5827 return value;
5828 }
5829 Node* result = nullptr;
5830 if (bt == T_BYTE) {
5831 result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5832 result = new RShiftINode(result, phase->intcon(24));
5833 } else if (bt == T_BOOLEAN) {
5834 result = new AndINode(value, phase->intcon(0xFF));
5835 } else if (bt == T_CHAR) {
5836 result = new AndINode(value,phase->intcon(0xFFFF));
5837 } else if (bt == T_FLOAT) {
5838 result = new MoveI2FNode(value);
5839 } else {
5840 assert(bt == T_SHORT, "unexpected narrow type");
5841 result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5842 result = new RShiftINode(result, phase->intcon(16));
5843 }
5844 if (transform_res) {
5845 result = phase->transform(result);
5846 }
5847 return result;
5848 }
5849
5850 void Compile::record_method_not_compilable_oom() {
5851 record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5852 }
|