29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if ((state->iu_barriers_count() +
50 state->load_reference_barriers_count()) > 0) {
51 assert(C->post_loop_opts_phase(), "no loop opts allowed");
52 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
53 C->clear_major_progress();
54 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
55 if (C->failing()) return false;
56
57 C->set_major_progress();
58 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
59 return false;
60 }
61 C->clear_major_progress();
62 C->process_for_post_loop_opts_igvn(igvn);
63 if (C->failing()) return false;
64
65 C->set_post_loop_opts_phase(); // now for real!
66 }
67 return true;
68 }
69
70 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
71 if (!UseShenandoahGC) {
72 return false;
73 }
74 assert(iff->is_If(), "bad input");
75 if (iff->Opcode() != Op_If) {
76 return false;
77 }
78 Node* bol = iff->in(1);
79 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
80 return false;
81 }
169 continue;
170 } else if (in->is_Con()) {
171 if (trace) {
172 tty->print("Found constant");
173 in->dump();
174 }
175 } else if (in->Opcode() == Op_Parm) {
176 if (trace) {
177 tty->print("Found argument");
178 }
179 } else if (in->Opcode() == Op_CreateEx) {
180 if (trace) {
181 tty->print("Found create-exception");
182 }
183 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
184 if (trace) {
185 tty->print("Found raw LoadP (OSR argument?)");
186 }
187 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
188 if (t == ShenandoahOopStore) {
189 uint i = 0;
190 for (; i < phis.size(); i++) {
191 Node* n = phis.node_at(i);
192 if (n->Opcode() == Op_ShenandoahIUBarrier) {
193 break;
194 }
195 }
196 if (i == phis.size()) {
197 return false;
198 }
199 }
200 barriers_used.push(in);
201 if (trace) {tty->print("Found barrier"); in->dump();}
202 } else if (in->Opcode() == Op_ShenandoahIUBarrier) {
203 if (t != ShenandoahOopStore) {
204 in = in->in(1);
205 continue;
206 }
207 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
208 phis.push(in, in->req());
209 in = in->in(1);
210 continue;
211 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
212 if (trace) {
213 tty->print("Found alloc");
214 in->in(0)->dump();
215 }
216 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
217 if (trace) {
218 tty->print("Found Java call");
219 }
220 } else if (in->is_Phi()) {
221 if (!visited.test_set(in->_idx)) {
222 if (trace) {tty->print("Pushed phi:"); in->dump();}
223 phis.push(in, 2);
224 in = in->in(1);
225 continue;
226 }
227 if (trace) {tty->print("Already seen phi:"); in->dump();}
228 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
229 if (!visited.test_set(in->_idx)) {
230 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
309 if (trace) {tty->print("Verifying"); n->dump();}
310 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
311 Node* adr = n->in(MemNode::Address);
312 bool verify = true;
313
314 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
315 adr = adr->in(AddPNode::Address);
316 if (adr->is_AddP()) {
317 assert(adr->in(AddPNode::Base)->is_top(), "");
318 adr = adr->in(AddPNode::Address);
319 if (adr->Opcode() == Op_LoadP &&
320 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
321 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
322 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
323 if (trace) {tty->print_cr("SATB prebarrier");}
324 verify = false;
325 }
326 }
327 }
328
329 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
330 report_verify_failure("Shenandoah verification: Store should have barriers", n);
331 }
332 }
333 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
334 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
335 }
336 } else if (n->Opcode() == Op_CmpP) {
337 const bool trace = false;
338
339 Node* in1 = n->in(1);
340 Node* in2 = n->in(2);
341 if (in1->bottom_type()->isa_oopptr()) {
342 if (trace) {tty->print("Verifying"); n->dump();}
343
344 bool mark_inputs = false;
345 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
346 (in1->is_Con() || in2->is_Con())) {
347 if (trace) {tty->print_cr("Comparison against a constant");}
348 mark_inputs = true;
349 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
351 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
352 mark_inputs = true;
353 } else {
354 assert(in2->bottom_type()->isa_oopptr(), "");
355
356 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
357 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
358 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
359 }
360 }
361 if (verify_no_useless_barrier &&
362 mark_inputs &&
363 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
364 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
365 phis.clear();
366 visited.reset();
367 }
368 }
369 } else if (n->is_LoadStore()) {
370 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
371 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
372 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
373 }
374
375 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
376 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
377 }
378 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
379 CallNode* call = n->as_Call();
380
381 static struct {
382 const char* name;
383 struct {
384 int pos;
385 verify_type t;
386 } args[6];
387 } calls[] = {
388 "aescrypt_encryptBlock",
389 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
390 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
391 "aescrypt_decryptBlock",
392 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
394 "multiplyToLen",
395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
397 "squareToLen",
398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
400 "montgomery_multiply",
401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
402 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
403 "montgomery_square",
404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
405 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
406 "mulAdd",
407 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
410 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
412 "updateBytesCRC32",
413 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
414 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
415 "updateBytesAdler32",
416 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
418 "updateBytesCRC32C",
419 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
421 "counterMode_AESCrypt",
422 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
423 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
424 "cipherBlockChaining_encryptAESCrypt",
425 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
426 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
427 "cipherBlockChaining_decryptAESCrypt",
428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
429 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
430 "shenandoah_clone_barrier",
431 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
432 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
433 "ghash_processBlocks",
434 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
435 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
436 "sha1_implCompress",
437 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
439 "sha256_implCompress",
440 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
442 "sha512_implCompress",
443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
445 "sha1_implCompressMB",
446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
448 "sha256_implCompressMB",
449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
503 }
504 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
505 if (call->in(j)->bottom_type()->make_ptr() &&
506 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
507 uint k = 0;
508 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
509 if (k == args_len) {
510 fatal("arg %d for call %s not covered", j, call->_name);
511 }
512 }
513 }
514 } else {
515 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
516 if (call->in(j)->bottom_type()->make_ptr() &&
517 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
518 fatal("%s not covered", call->_name);
519 }
520 }
521 }
522 }
523 } else if (n->Opcode() == Op_ShenandoahIUBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
524 // skip
525 } else if (n->is_AddP()
526 || n->is_Phi()
527 || n->is_ConstraintCast()
528 || n->Opcode() == Op_Return
529 || n->Opcode() == Op_CMoveP
530 || n->Opcode() == Op_CMoveN
531 || n->Opcode() == Op_Rethrow
532 || n->is_MemBar()
533 || n->Opcode() == Op_Conv2B
534 || n->Opcode() == Op_SafePoint
535 || n->is_CallJava()
536 || n->Opcode() == Op_Unlock
537 || n->Opcode() == Op_EncodeP
538 || n->Opcode() == Op_DecodeN) {
539 // nothing to do
540 } else {
541 static struct {
542 int opcode;
543 struct {
644 return maybe_load->is_Load() && phase->C->can_alias(store->adr_type(), phase->C->get_alias_index(maybe_load->adr_type())) &&
645 phase->ctrl_or_self(maybe_load) == control;
646 }
647
648 void ShenandoahBarrierC2Support::maybe_push_anti_dependent_loads(PhaseIdealLoop* phase, Node* maybe_store, Node* control, Unique_Node_List &wq) {
649 if (!maybe_store->is_Store() && !maybe_store->is_LoadStore()) {
650 return;
651 }
652 Node* mem = maybe_store->in(MemNode::Memory);
653 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
654 Node* u = mem->fast_out(i);
655 if (is_anti_dependent_load_at_control(phase, u, maybe_store, control)) {
656 wq.push(u);
657 }
658 }
659 }
660
661 void ShenandoahBarrierC2Support::push_data_inputs_at_control(PhaseIdealLoop* phase, Node* n, Node* ctrl, Unique_Node_List &wq) {
662 for (uint i = 0; i < n->req(); i++) {
663 Node* in = n->in(i);
664 if (in != nullptr && (ShenandoahIUBarrier ? (phase->ctrl_or_self(in) == ctrl) : (phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl))) {
665 wq.push(in);
666 }
667 }
668 }
669
670 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) {
671 // That both nodes have the same control is not sufficient to prove
672 // domination, verify that there's no path from d to n
673 ResourceMark rm;
674 Unique_Node_List wq;
675 wq.push(d);
676 for (uint next = 0; next < wq.size(); next++) {
677 Node *m = wq.at(next);
678 if (m == n) {
679 return false;
680 }
681 if (m->is_Phi() && m->in(0)->is_Loop()) {
682 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
683 } else {
684 // Take anti-dependencies into account
1100 assert(region != nullptr, "");
1101 Node* phi = new PhiNode(region, n->bottom_type());
1102 for (uint j = 1; j < region->req(); j++) {
1103 Node* in = region->in(j);
1104 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1105 phi->init_req(j, n);
1106 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1107 phi->init_req(j, n_clone);
1108 } else {
1109 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1110 }
1111 }
1112 phase->register_new_node(phi, region);
1113 return phi;
1114 }
1115
1116 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1117 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1118
1119 Unique_Node_List uses;
1120 for (int i = 0; i < state->iu_barriers_count(); i++) {
1121 Node* barrier = state->iu_barrier(i);
1122 Node* ctrl = phase->get_ctrl(barrier);
1123 IdealLoopTree* loop = phase->get_loop(ctrl);
1124 Node* head = loop->head();
1125 if (head->is_OuterStripMinedLoop()) {
1126 // Expanding a barrier here will break loop strip mining
1127 // verification. Transform the loop so the loop nest doesn't
1128 // appear as strip mined.
1129 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop();
1130 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1131 }
1132 }
1133
1134 Node_Stack stack(0);
1135 Node_List clones;
1136 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1137 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1138
1139 Node* ctrl = phase->get_ctrl(lrb);
1140 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1141
1142 CallStaticJavaNode* unc = nullptr;
1143 Node* unc_ctrl = nullptr;
1144 Node* uncasted_val = val;
1145
1146 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1147 Node* u = lrb->fast_out(i);
1148 if (u->Opcode() == Op_CastPP &&
1149 u->in(0) != nullptr &&
1150 phase->is_dominator(u->in(0), ctrl)) {
1151 const Type* u_t = phase->igvn().type(u);
1152
1153 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1455
1456 fix_ctrl(lrb, region, fixer, uses, nodes_above_barriers, last, phase);
1457
1458 ctrl = orig_ctrl;
1459
1460 phase->igvn().replace_node(lrb, out_val);
1461
1462 follow_barrier_uses(out_val, ctrl, uses, phase);
1463
1464 for(uint next = 0; next < uses.size(); next++ ) {
1465 Node *n = uses.at(next);
1466 assert(phase->get_ctrl(n) == ctrl, "bad control");
1467 assert(n != raw_mem, "should leave input raw mem above the barrier");
1468 phase->set_ctrl(n, region);
1469 follow_barrier_uses(n, ctrl, uses, phase);
1470 }
1471 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1472 }
1473 // Done expanding load-reference-barriers.
1474 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1475
1476 for (int i = state->iu_barriers_count() - 1; i >= 0; i--) {
1477 Node* barrier = state->iu_barrier(i);
1478 Node* pre_val = barrier->in(1);
1479
1480 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
1481 ShouldNotReachHere();
1482 continue;
1483 }
1484
1485 Node* ctrl = phase->get_ctrl(barrier);
1486
1487 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1488 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
1489 ctrl = ctrl->in(0)->in(0);
1490 phase->set_ctrl(barrier, ctrl);
1491 } else if (ctrl->is_CallRuntime()) {
1492 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
1493 ctrl = ctrl->in(0);
1494 phase->set_ctrl(barrier, ctrl);
1495 }
1496
1497 Node* init_ctrl = ctrl;
1498 IdealLoopTree* loop = phase->get_loop(ctrl);
1499 Node* raw_mem = fixer.find_mem(ctrl, barrier);
1500 Node* init_raw_mem = raw_mem;
1501 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr);
1502 Node* heap_stable_ctrl = nullptr;
1503 Node* null_ctrl = nullptr;
1504 uint last = phase->C->unique();
1505
1506 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1507 Node* region = new RegionNode(PATH_LIMIT);
1508 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1509
1510 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1511 Node* region2 = new RegionNode(PATH_LIMIT2);
1512 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1513
1514 // Stable path.
1515 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
1516 region->init_req(_heap_stable, heap_stable_ctrl);
1517 phi->init_req(_heap_stable, raw_mem);
1518
1519 // Null path
1520 Node* reg2_ctrl = nullptr;
1521 test_null(ctrl, pre_val, null_ctrl, phase);
1522 if (null_ctrl != nullptr) {
1523 reg2_ctrl = null_ctrl->in(0);
1524 region2->init_req(_null_path, null_ctrl);
1525 phi2->init_req(_null_path, raw_mem);
1526 } else {
1527 region2->del_req(_null_path);
1528 phi2->del_req(_null_path);
1529 }
1530
1531 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1532 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1533 Node* thread = new ThreadLocalNode();
1534 phase->register_new_node(thread, ctrl);
1535 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
1536 phase->register_new_node(buffer_adr, ctrl);
1537 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
1538 phase->register_new_node(index_adr, ctrl);
1539
1540 BasicType index_bt = TypeX_X->basic_type();
1541 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
1542 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1543 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
1544 phase->register_new_node(index, ctrl);
1545 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
1546 phase->register_new_node(index_cmp, ctrl);
1547 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
1548 phase->register_new_node(index_test, ctrl);
1549 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1550 if (reg2_ctrl == nullptr) reg2_ctrl = queue_full_iff;
1551 phase->register_control(queue_full_iff, loop, ctrl);
1552 Node* not_full = new IfTrueNode(queue_full_iff);
1553 phase->register_control(not_full, loop, queue_full_iff);
1554 Node* full = new IfFalseNode(queue_full_iff);
1555 phase->register_control(full, loop, queue_full_iff);
1556
1557 ctrl = not_full;
1558
1559 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
1560 phase->register_new_node(next_index, ctrl);
1561
1562 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
1563 phase->register_new_node(buffer, ctrl);
1564 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
1565 phase->register_new_node(log_addr, ctrl);
1566 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
1567 phase->register_new_node(log_store, ctrl);
1568 // update the index
1569 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
1570 phase->register_new_node(index_update, ctrl);
1571
1572 // Fast-path case
1573 region2->init_req(_fast_path, ctrl);
1574 phi2->init_req(_fast_path, index_update);
1575
1576 ctrl = full;
1577
1578 Node* base = find_bottom_mem(ctrl, phase);
1579
1580 MergeMemNode* mm = MergeMemNode::make(base);
1581 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1582 phase->register_new_node(mm, ctrl);
1583
1584 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
1585 call->init_req(TypeFunc::Control, ctrl);
1586 call->init_req(TypeFunc::I_O, phase->C->top());
1587 call->init_req(TypeFunc::Memory, mm);
1588 call->init_req(TypeFunc::FramePtr, phase->C->top());
1589 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1590 call->init_req(TypeFunc::Parms, pre_val);
1591 call->init_req(TypeFunc::Parms+1, thread);
1592 phase->register_control(call, loop, ctrl);
1593
1594 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
1595 phase->register_control(ctrl_proj, loop, call);
1596 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
1597 phase->register_new_node(mem_proj, call);
1598
1599 // Slow-path case
1600 region2->init_req(_slow_path, ctrl_proj);
1601 phi2->init_req(_slow_path, mem_proj);
1602
1603 phase->register_control(region2, loop, reg2_ctrl);
1604 phase->register_new_node(phi2, region2);
1605
1606 region->init_req(_heap_unstable, region2);
1607 phi->init_req(_heap_unstable, phi2);
1608
1609 phase->register_control(region, loop, heap_stable_ctrl->in(0));
1610 phase->register_new_node(phi, region);
1611
1612 fix_ctrl(barrier, region, fixer, uses, nodes_above_barriers, last, phase);
1613 for(uint next = 0; next < uses.size(); next++ ) {
1614 Node *n = uses.at(next);
1615 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1616 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1617 phase->set_ctrl(n, region);
1618 follow_barrier_uses(n, init_ctrl, uses, phase);
1619 }
1620 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1621
1622 phase->igvn().replace_node(barrier, pre_val);
1623 }
1624 assert(state->iu_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
1625
1626 }
1627
1628 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1629 if (visited.test_set(in->_idx)) {
1630 return nullptr;
1631 }
1632 switch (in->Opcode()) {
1633 case Op_Proj:
1634 return get_load_addr(phase, visited, in->in(0));
1635 case Op_CastPP:
1636 case Op_CheckCastPP:
1637 case Op_DecodeN:
1638 case Op_EncodeP:
1639 return get_load_addr(phase, visited, in->in(1));
1640 case Op_LoadN:
1641 case Op_LoadP:
1642 return in->in(MemNode::Address);
1643 case Op_CompareAndExchangeN:
1644 case Op_CompareAndExchangeP:
1645 case Op_GetAndSetN:
1658 if (t == nullptr && f != nullptr) return f;
1659 if (t != nullptr && t == f) return t;
1660 // Ambiguity.
1661 return phase->igvn().zerocon(T_OBJECT);
1662 }
1663 case Op_Phi: {
1664 Node* addr = nullptr;
1665 for (uint i = 1; i < in->req(); i++) {
1666 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1667 if (addr == nullptr) {
1668 addr = addr1;
1669 }
1670 if (addr != addr1) {
1671 return phase->igvn().zerocon(T_OBJECT);
1672 }
1673 }
1674 return addr;
1675 }
1676 case Op_ShenandoahLoadReferenceBarrier:
1677 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1678 case Op_ShenandoahIUBarrier:
1679 return get_load_addr(phase, visited, in->in(1));
1680 case Op_CallDynamicJava:
1681 case Op_CallLeaf:
1682 case Op_CallStaticJava:
1683 case Op_ConN:
1684 case Op_ConP:
1685 case Op_Parm:
1686 case Op_CreateEx:
1687 return phase->igvn().zerocon(T_OBJECT);
1688 default:
1689 #ifdef ASSERT
1690 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1691 #endif
1692 return phase->igvn().zerocon(T_OBJECT);
1693 }
1694
1695 }
1696
1697 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1698 IdealLoopTree *loop = phase->get_loop(iff);
1699 Node* loop_head = loop->_head;
1700 Node* entry_c = loop_head->in(LoopNode::EntryControl);
1701
1702 Node* bol = iff->in(1);
1703 Node* cmp = bol->in(1);
1704 Node* andi = cmp->in(1);
1705 Node* load = andi->in(1);
1706
1707 assert(is_gc_state_load(load), "broken");
1708 if (!phase->is_dominator(load->in(0), entry_c)) {
1709 Node* mem_ctrl = nullptr;
1710 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
1711 load = load->clone();
1712 load->set_req(MemNode::Memory, mem);
1713 load->set_req(0, entry_c);
1714 phase->register_new_node(load, entry_c);
1715 andi = andi->clone();
1716 andi->set_req(1, load);
1717 phase->register_new_node(andi, entry_c);
1718 cmp = cmp->clone();
1719 cmp->set_req(1, andi);
1720 phase->register_new_node(cmp, entry_c);
1721 bol = bol->clone();
1722 bol->set_req(1, cmp);
1723 phase->register_new_node(bol, entry_c);
1724
1725 phase->igvn().replace_input_of(iff, 1, bol);
1726 }
1727 }
1728
1729 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
1730 if (!n->is_If() || n->is_CountedLoopEnd()) {
1731 return false;
1732 }
1733 Node* region = n->in(0);
1734
1735 if (!region->is_Region()) {
1736 return false;
1737 }
1738 Node* dom = phase->idom(region);
1739 if (!dom->is_If()) {
1740 return false;
1741 }
1742
1743 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
1744 return false;
1745 }
1746
1747 IfNode* dom_if = dom->as_If();
1748 Node* proj_true = dom_if->proj_out(1);
1749 Node* proj_false = dom_if->proj_out(0);
1750
1751 for (uint i = 1; i < region->req(); i++) {
1752 if (phase->is_dominator(proj_true, region->in(i))) {
1753 continue;
1754 }
1755 if (phase->is_dominator(proj_false, region->in(i))) {
1756 continue;
1757 }
1758 return false;
1759 }
1760
1761 return true;
1762 }
1763
1764 bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) {
1765 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1766 Node* n = region->fast_out(i);
1767 if (n->is_LoadStore()) {
1768 // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support
1769 // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes
1770 // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get
1771 // expanded around a LoadStore node.
1772 return false;
1773 }
1774 }
1775 return true;
1776 }
1777
1778
1779 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
1780 assert(is_heap_stable_test(n), "no other tests");
1781 if (identical_backtoback_ifs(n, phase)) {
1782 Node* n_ctrl = n->in(0);
1783 if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) {
1784 IfNode* dom_if = phase->idom(n_ctrl)->as_If();
1785 if (is_heap_stable_test(n)) {
1786 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
1787 assert(is_gc_state_load(gc_state_load), "broken");
1788 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
1789 assert(is_gc_state_load(dom_gc_state_load), "broken");
1790 if (gc_state_load != dom_gc_state_load) {
1791 phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
1792 }
1793 }
1794 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
1795 Node* proj_true = dom_if->proj_out(1);
1796 Node* proj_false = dom_if->proj_out(0);
1797 Node* con_true = phase->igvn().makecon(TypeInt::ONE);
1798 Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
1799
1800 for (uint i = 1; i < n_ctrl->req(); i++) {
1801 if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
1802 bolphi->init_req(i, con_true);
1803 } else {
1804 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
1805 bolphi->init_req(i, con_false);
1806 }
1807 }
1808 phase->register_new_node(bolphi, n_ctrl);
1809 phase->igvn().replace_input_of(n, 1, bolphi);
1810 phase->do_split_if(n);
1811 }
1812 }
1813 }
1814
1815 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
1816 // Find first invariant test that doesn't exit the loop
1817 LoopNode *head = loop->_head->as_Loop();
1818 IfNode* unswitch_iff = nullptr;
1819 Node* n = head->in(LoopNode::LoopBackControl);
1820 int loop_has_sfpts = -1;
1821 while (n != head) {
1822 Node* n_dom = phase->idom(n);
1823 if (n->is_Region()) {
1824 if (n_dom->is_If()) {
1825 IfNode* iff = n_dom->as_If();
1826 if (iff->in(1)->is_Bool()) {
1827 BoolNode* bol = iff->in(1)->as_Bool();
1828 if (bol->in(1)->is_Cmp()) {
1829 // If condition is invariant and not a loop exit,
1830 // then found reason to unswitch.
1831 if (is_heap_stable_test(iff) &&
1832 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
1833 assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
1834 if (loop_has_sfpts == -1) {
1835 for(uint i = 0; i < loop->_body.size(); i++) {
1836 Node *m = loop->_body[i];
1837 if (m->is_SafePoint() && !m->is_CallLeaf()) {
1838 loop_has_sfpts = 1;
1839 break;
1840 }
1841 }
1842 if (loop_has_sfpts == -1) {
1843 loop_has_sfpts = 0;
1844 }
1845 }
1846 if (!loop_has_sfpts) {
1847 unswitch_iff = iff;
1848 }
1849 }
1850 }
1851 }
1852 }
1853 }
1854 n = n_dom;
1855 }
1856 return unswitch_iff;
1857 }
1858
1859
1860 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
1861 Node_List heap_stable_tests;
1862 stack.push(phase->C->start(), 0);
1863 do {
1864 Node* n = stack.node();
1865 uint i = stack.index();
1866
1867 if (i < n->outcnt()) {
1868 Node* u = n->raw_out(i);
1869 stack.set_index(i+1);
1870 if (!visited.test_set(u->_idx)) {
1871 stack.push(u, 0);
1872 }
1873 } else {
1874 stack.pop();
1875 if (n->is_If() && is_heap_stable_test(n)) {
1876 heap_stable_tests.push(n);
1877 }
1878 }
1879 } while (stack.size() > 0);
1880
1881 for (uint i = 0; i < heap_stable_tests.size(); i++) {
1882 Node* n = heap_stable_tests.at(i);
1883 assert(is_heap_stable_test(n), "only evacuation test");
1884 merge_back_to_back_tests(n, phase);
1885 }
1886
1887 if (!phase->C->major_progress()) {
1888 VectorSet seen;
1889 for (uint i = 0; i < heap_stable_tests.size(); i++) {
1890 Node* n = heap_stable_tests.at(i);
1891 IdealLoopTree* loop = phase->get_loop(n);
1892 if (loop != phase->ltree_root() &&
1893 loop->_child == nullptr &&
1894 !loop->_irreducible) {
1895 Node* head = loop->_head;
1896 if (head->is_Loop() &&
1897 (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
1898 !seen.test_set(head->_idx)) {
1899 IfNode* iff = find_unswitching_candidate(loop, phase);
1900 if (iff != nullptr) {
1901 Node* bol = iff->in(1);
1902 if (head->as_Loop()->is_strip_mined()) {
1903 head->as_Loop()->verify_strip_mined(0);
1904 }
1905 move_gc_state_test_out_of_loop(iff, phase);
1906
1907 AutoNodeBudget node_budget(phase);
1908
1909 if (loop->policy_unswitching(phase)) {
1910 if (head->as_Loop()->is_strip_mined()) {
1911 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1912 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1913 }
1914 phase->do_unswitching(loop, old_new);
1915 } else {
1916 // Not proceeding with unswitching. Move load back in
1917 // the loop.
1918 phase->igvn().replace_input_of(iff, 1, bol);
1919 }
1920 }
1921 }
1922 }
1923 }
1924 }
1925 }
1926
1927 ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(nullptr, val) {
1928 ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
1929 }
1930
1931 const Type* ShenandoahIUBarrierNode::bottom_type() const {
1932 if (in(1) == nullptr || in(1)->is_top()) {
1933 return Type::TOP;
1934 }
1935 const Type* t = in(1)->bottom_type();
1936 if (t == TypePtr::NULL_PTR) {
1937 return t;
1938 }
1939 return t->is_oopptr();
1940 }
1941
1942 const Type* ShenandoahIUBarrierNode::Value(PhaseGVN* phase) const {
1943 if (in(1) == nullptr) {
1944 return Type::TOP;
1945 }
1946 const Type* t = phase->type(in(1));
1947 if (t == Type::TOP) {
1948 return Type::TOP;
1949 }
1950 if (t == TypePtr::NULL_PTR) {
1951 return t;
1952 }
1953 return t->is_oopptr();
1954 }
1955
1956 int ShenandoahIUBarrierNode::needed(Node* n) {
1957 if (n == nullptr ||
1958 n->is_Allocate() ||
1959 n->Opcode() == Op_ShenandoahIUBarrier ||
1960 n->bottom_type() == TypePtr::NULL_PTR ||
1961 (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr)) {
1962 return NotNeeded;
1963 }
1964 if (n->is_Phi() ||
1965 n->is_CMove()) {
1966 return MaybeNeeded;
1967 }
1968 return Needed;
1969 }
1970
1971 Node* ShenandoahIUBarrierNode::next(Node* n) {
1972 for (;;) {
1973 if (n == nullptr) {
1974 return n;
1975 } else if (n->bottom_type() == TypePtr::NULL_PTR) {
1976 return n;
1977 } else if (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr) {
1978 return n;
1979 } else if (n->is_ConstraintCast() ||
1980 n->Opcode() == Op_DecodeN ||
1981 n->Opcode() == Op_EncodeP) {
1982 n = n->in(1);
1983 } else if (n->is_Proj()) {
1984 n = n->in(0);
1985 } else {
1986 return n;
1987 }
1988 }
1989 ShouldNotReachHere();
1990 return nullptr;
1991 }
1992
1993 Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) {
1994 PhaseIterGVN* igvn = phase->is_IterGVN();
1995
1996 Node* n = next(in(1));
1997
1998 int cont = needed(n);
1999
2000 if (cont == NotNeeded) {
2001 return in(1);
2002 } else if (cont == MaybeNeeded) {
2003 if (igvn == nullptr) {
2004 phase->record_for_igvn(this);
2005 return this;
2006 } else {
2007 ResourceMark rm;
2008 Unique_Node_List wq;
2009 uint wq_i = 0;
2010
2011 for (;;) {
2012 if (n->is_Phi()) {
2013 for (uint i = 1; i < n->req(); i++) {
2014 Node* m = n->in(i);
2015 if (m != nullptr) {
2016 wq.push(m);
2017 }
2018 }
2019 } else {
2020 assert(n->is_CMove(), "nothing else here");
2021 Node* m = n->in(CMoveNode::IfFalse);
2022 wq.push(m);
2023 m = n->in(CMoveNode::IfTrue);
2024 wq.push(m);
2025 }
2026 Node* orig_n = nullptr;
2027 do {
2028 if (wq_i >= wq.size()) {
2029 return in(1);
2030 }
2031 n = wq.at(wq_i);
2032 wq_i++;
2033 orig_n = n;
2034 n = next(n);
2035 cont = needed(n);
2036 if (cont == Needed) {
2037 return this;
2038 }
2039 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
2040 }
2041 }
2042 }
2043
2044 return this;
2045 }
2046
2047 #ifdef ASSERT
2048 static bool has_never_branch(Node* root) {
2049 for (uint i = 1; i < root->req(); i++) {
2050 Node* in = root->in(i);
2051 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
2052 return true;
2053 }
2054 }
2055 return false;
2056 }
2057 #endif
2058
2059 void MemoryGraphFixer::collect_memory_nodes() {
2060 Node_Stack stack(0);
2061 VectorSet visited;
2062 Node_List regions;
2063
2064 // Walk the raw memory graph and create a mapping from CFG node to
2065 // memory node. Exclude phis for now.
2066 stack.push(_phase->C->root(), 1);
2183 int iteration = 0;
2184 Node_List dead_phis;
2185 while (progress) {
2186 progress = false;
2187 iteration++;
2188 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2189 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2190
2191 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2192 Node* c = rpo_list.at(i);
2193
2194 Node* prev_mem = _memory_nodes[c->_idx];
2195 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2196 Node* prev_region = regions[c->_idx];
2197 Node* unique = nullptr;
2198 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2199 Node* m = _memory_nodes[c->in(j)->_idx];
2200 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2201 if (m != nullptr) {
2202 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2203 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2204 // continue
2205 } else if (unique == nullptr) {
2206 unique = m;
2207 } else if (m == unique) {
2208 // continue
2209 } else {
2210 unique = NodeSentinel;
2211 }
2212 }
2213 }
2214 assert(unique != nullptr, "empty phi???");
2215 if (unique != NodeSentinel) {
2216 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
2217 dead_phis.push(prev_region);
2218 }
2219 regions.map(c->_idx, unique);
2220 } else {
2221 Node* phi = nullptr;
2222 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
2223 phi = prev_region;
3010 case Op_CheckCastPP:
3011 case Op_CastPP:
3012 return needs_barrier_impl(phase, n->in(1), visited);
3013 case Op_Proj:
3014 return needs_barrier_impl(phase, n->in(0), visited);
3015 case Op_ShenandoahLoadReferenceBarrier:
3016 // tty->print_cr("optimize barrier on barrier");
3017 return false;
3018 case Op_Parm:
3019 // tty->print_cr("optimize barrier on input arg");
3020 return false;
3021 case Op_DecodeN:
3022 case Op_EncodeP:
3023 return needs_barrier_impl(phase, n->in(1), visited);
3024 case Op_LoadN:
3025 return true;
3026 case Op_CMoveN:
3027 case Op_CMoveP:
3028 return needs_barrier_impl(phase, n->in(2), visited) ||
3029 needs_barrier_impl(phase, n->in(3), visited);
3030 case Op_ShenandoahIUBarrier:
3031 return needs_barrier_impl(phase, n->in(1), visited);
3032 case Op_CreateEx:
3033 return false;
3034 default:
3035 break;
3036 }
3037 #ifdef ASSERT
3038 tty->print("need barrier on?: ");
3039 tty->print_cr("ins:");
3040 n->dump(2);
3041 tty->print_cr("outs:");
3042 n->dump(-2);
3043 ShouldNotReachHere();
3044 #endif
3045 return true;
3046 }
|
29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if (state->load_reference_barriers_count() > 0) {
50 assert(C->post_loop_opts_phase(), "no loop opts allowed");
51 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
52 C->clear_major_progress();
53 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
54 if (C->failing()) return false;
55 C->process_for_post_loop_opts_igvn(igvn);
56 if (C->failing()) return false;
57
58 C->set_post_loop_opts_phase(); // now for real!
59 }
60 return true;
61 }
62
63 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
64 if (!UseShenandoahGC) {
65 return false;
66 }
67 assert(iff->is_If(), "bad input");
68 if (iff->Opcode() != Op_If) {
69 return false;
70 }
71 Node* bol = iff->in(1);
72 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
73 return false;
74 }
162 continue;
163 } else if (in->is_Con()) {
164 if (trace) {
165 tty->print("Found constant");
166 in->dump();
167 }
168 } else if (in->Opcode() == Op_Parm) {
169 if (trace) {
170 tty->print("Found argument");
171 }
172 } else if (in->Opcode() == Op_CreateEx) {
173 if (trace) {
174 tty->print("Found create-exception");
175 }
176 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
177 if (trace) {
178 tty->print("Found raw LoadP (OSR argument?)");
179 }
180 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
181 if (t == ShenandoahOopStore) {
182 return false;
183 }
184 barriers_used.push(in);
185 if (trace) {tty->print("Found barrier"); in->dump();}
186 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
187 if (trace) {
188 tty->print("Found alloc");
189 in->in(0)->dump();
190 }
191 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
192 if (trace) {
193 tty->print("Found Java call");
194 }
195 } else if (in->is_Phi()) {
196 if (!visited.test_set(in->_idx)) {
197 if (trace) {tty->print("Pushed phi:"); in->dump();}
198 phis.push(in, 2);
199 in = in->in(1);
200 continue;
201 }
202 if (trace) {tty->print("Already seen phi:"); in->dump();}
203 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
204 if (!visited.test_set(in->_idx)) {
205 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
284 if (trace) {tty->print("Verifying"); n->dump();}
285 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
286 Node* adr = n->in(MemNode::Address);
287 bool verify = true;
288
289 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
290 adr = adr->in(AddPNode::Address);
291 if (adr->is_AddP()) {
292 assert(adr->in(AddPNode::Base)->is_top(), "");
293 adr = adr->in(AddPNode::Address);
294 if (adr->Opcode() == Op_LoadP &&
295 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
296 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
297 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
298 if (trace) {tty->print_cr("SATB prebarrier");}
299 verify = false;
300 }
301 }
302 }
303
304 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
305 report_verify_failure("Shenandoah verification: Store should have barriers", n);
306 }
307 }
308 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
309 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
310 }
311 } else if (n->Opcode() == Op_CmpP) {
312 const bool trace = false;
313
314 Node* in1 = n->in(1);
315 Node* in2 = n->in(2);
316 if (in1->bottom_type()->isa_oopptr()) {
317 if (trace) {tty->print("Verifying"); n->dump();}
318
319 bool mark_inputs = false;
320 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
321 (in1->is_Con() || in2->is_Con())) {
322 if (trace) {tty->print_cr("Comparison against a constant");}
323 mark_inputs = true;
324 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
326 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
327 mark_inputs = true;
328 } else {
329 assert(in2->bottom_type()->isa_oopptr(), "");
330
331 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
332 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
333 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
334 }
335 }
336 if (verify_no_useless_barrier &&
337 mark_inputs &&
338 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
339 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
340 phis.clear();
341 visited.reset();
342 }
343 }
344 } else if (n->is_LoadStore()) {
345 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
346 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
347 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
348 }
349
350 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
351 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
352 }
353 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
354 CallNode* call = n->as_Call();
355
356 static struct {
357 const char* name;
358 struct {
359 int pos;
360 verify_type t;
361 } args[6];
362 } calls[] = {
363 "array_partition_stub",
364 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone },
365 { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
366 "arraysort_stub",
367 { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone },
368 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
369 "aescrypt_encryptBlock",
370 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
371 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
372 "aescrypt_decryptBlock",
373 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
374 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
375 "multiplyToLen",
376 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
377 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
378 "squareToLen",
379 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
380 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
381 "montgomery_multiply",
382 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
383 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
384 "montgomery_square",
385 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
386 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
387 "mulAdd",
388 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
391 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
392 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
393 "updateBytesCRC32",
394 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
395 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
396 "updateBytesAdler32",
397 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
398 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
399 "updateBytesCRC32C",
400 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
401 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
402 "counterMode_AESCrypt",
403 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
404 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
405 "cipherBlockChaining_encryptAESCrypt",
406 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
407 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
408 "cipherBlockChaining_decryptAESCrypt",
409 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
410 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
411 "shenandoah_clone",
412 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
413 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
414 "ghash_processBlocks",
415 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
416 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
417 "sha1_implCompress",
418 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
419 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
420 "sha256_implCompress",
421 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
422 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
423 "sha512_implCompress",
424 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
425 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
426 "sha1_implCompressMB",
427 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
428 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
429 "sha256_implCompressMB",
430 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
484 }
485 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
486 if (call->in(j)->bottom_type()->make_ptr() &&
487 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
488 uint k = 0;
489 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
490 if (k == args_len) {
491 fatal("arg %d for call %s not covered", j, call->_name);
492 }
493 }
494 }
495 } else {
496 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
497 if (call->in(j)->bottom_type()->make_ptr() &&
498 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
499 fatal("%s not covered", call->_name);
500 }
501 }
502 }
503 }
504 } else if (n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
505 // skip
506 } else if (n->is_AddP()
507 || n->is_Phi()
508 || n->is_ConstraintCast()
509 || n->Opcode() == Op_Return
510 || n->Opcode() == Op_CMoveP
511 || n->Opcode() == Op_CMoveN
512 || n->Opcode() == Op_Rethrow
513 || n->is_MemBar()
514 || n->Opcode() == Op_Conv2B
515 || n->Opcode() == Op_SafePoint
516 || n->is_CallJava()
517 || n->Opcode() == Op_Unlock
518 || n->Opcode() == Op_EncodeP
519 || n->Opcode() == Op_DecodeN) {
520 // nothing to do
521 } else {
522 static struct {
523 int opcode;
524 struct {
625 return maybe_load->is_Load() && phase->C->can_alias(store->adr_type(), phase->C->get_alias_index(maybe_load->adr_type())) &&
626 phase->ctrl_or_self(maybe_load) == control;
627 }
628
629 void ShenandoahBarrierC2Support::maybe_push_anti_dependent_loads(PhaseIdealLoop* phase, Node* maybe_store, Node* control, Unique_Node_List &wq) {
630 if (!maybe_store->is_Store() && !maybe_store->is_LoadStore()) {
631 return;
632 }
633 Node* mem = maybe_store->in(MemNode::Memory);
634 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
635 Node* u = mem->fast_out(i);
636 if (is_anti_dependent_load_at_control(phase, u, maybe_store, control)) {
637 wq.push(u);
638 }
639 }
640 }
641
642 void ShenandoahBarrierC2Support::push_data_inputs_at_control(PhaseIdealLoop* phase, Node* n, Node* ctrl, Unique_Node_List &wq) {
643 for (uint i = 0; i < n->req(); i++) {
644 Node* in = n->in(i);
645 if (in != nullptr && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
646 wq.push(in);
647 }
648 }
649 }
650
651 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) {
652 // That both nodes have the same control is not sufficient to prove
653 // domination, verify that there's no path from d to n
654 ResourceMark rm;
655 Unique_Node_List wq;
656 wq.push(d);
657 for (uint next = 0; next < wq.size(); next++) {
658 Node *m = wq.at(next);
659 if (m == n) {
660 return false;
661 }
662 if (m->is_Phi() && m->in(0)->is_Loop()) {
663 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
664 } else {
665 // Take anti-dependencies into account
1081 assert(region != nullptr, "");
1082 Node* phi = new PhiNode(region, n->bottom_type());
1083 for (uint j = 1; j < region->req(); j++) {
1084 Node* in = region->in(j);
1085 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1086 phi->init_req(j, n);
1087 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1088 phi->init_req(j, n_clone);
1089 } else {
1090 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1091 }
1092 }
1093 phase->register_new_node(phi, region);
1094 return phi;
1095 }
1096
1097 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1098 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1099
1100 Unique_Node_List uses;
1101 Node_Stack stack(0);
1102 Node_List clones;
1103 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1104 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1105
1106 Node* ctrl = phase->get_ctrl(lrb);
1107 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1108
1109 CallStaticJavaNode* unc = nullptr;
1110 Node* unc_ctrl = nullptr;
1111 Node* uncasted_val = val;
1112
1113 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1114 Node* u = lrb->fast_out(i);
1115 if (u->Opcode() == Op_CastPP &&
1116 u->in(0) != nullptr &&
1117 phase->is_dominator(u->in(0), ctrl)) {
1118 const Type* u_t = phase->igvn().type(u);
1119
1120 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1422
1423 fix_ctrl(lrb, region, fixer, uses, nodes_above_barriers, last, phase);
1424
1425 ctrl = orig_ctrl;
1426
1427 phase->igvn().replace_node(lrb, out_val);
1428
1429 follow_barrier_uses(out_val, ctrl, uses, phase);
1430
1431 for(uint next = 0; next < uses.size(); next++ ) {
1432 Node *n = uses.at(next);
1433 assert(phase->get_ctrl(n) == ctrl, "bad control");
1434 assert(n != raw_mem, "should leave input raw mem above the barrier");
1435 phase->set_ctrl(n, region);
1436 follow_barrier_uses(n, ctrl, uses, phase);
1437 }
1438 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1439 }
1440 // Done expanding load-reference-barriers.
1441 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1442 }
1443
1444 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1445 if (visited.test_set(in->_idx)) {
1446 return nullptr;
1447 }
1448 switch (in->Opcode()) {
1449 case Op_Proj:
1450 return get_load_addr(phase, visited, in->in(0));
1451 case Op_CastPP:
1452 case Op_CheckCastPP:
1453 case Op_DecodeN:
1454 case Op_EncodeP:
1455 return get_load_addr(phase, visited, in->in(1));
1456 case Op_LoadN:
1457 case Op_LoadP:
1458 return in->in(MemNode::Address);
1459 case Op_CompareAndExchangeN:
1460 case Op_CompareAndExchangeP:
1461 case Op_GetAndSetN:
1474 if (t == nullptr && f != nullptr) return f;
1475 if (t != nullptr && t == f) return t;
1476 // Ambiguity.
1477 return phase->igvn().zerocon(T_OBJECT);
1478 }
1479 case Op_Phi: {
1480 Node* addr = nullptr;
1481 for (uint i = 1; i < in->req(); i++) {
1482 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1483 if (addr == nullptr) {
1484 addr = addr1;
1485 }
1486 if (addr != addr1) {
1487 return phase->igvn().zerocon(T_OBJECT);
1488 }
1489 }
1490 return addr;
1491 }
1492 case Op_ShenandoahLoadReferenceBarrier:
1493 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1494 case Op_CallDynamicJava:
1495 case Op_CallLeaf:
1496 case Op_CallStaticJava:
1497 case Op_ConN:
1498 case Op_ConP:
1499 case Op_Parm:
1500 case Op_CreateEx:
1501 return phase->igvn().zerocon(T_OBJECT);
1502 default:
1503 #ifdef ASSERT
1504 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1505 #endif
1506 return phase->igvn().zerocon(T_OBJECT);
1507 }
1508
1509 }
1510
1511 #ifdef ASSERT
1512 static bool has_never_branch(Node* root) {
1513 for (uint i = 1; i < root->req(); i++) {
1514 Node* in = root->in(i);
1515 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
1516 return true;
1517 }
1518 }
1519 return false;
1520 }
1521 #endif
1522
1523 void MemoryGraphFixer::collect_memory_nodes() {
1524 Node_Stack stack(0);
1525 VectorSet visited;
1526 Node_List regions;
1527
1528 // Walk the raw memory graph and create a mapping from CFG node to
1529 // memory node. Exclude phis for now.
1530 stack.push(_phase->C->root(), 1);
1647 int iteration = 0;
1648 Node_List dead_phis;
1649 while (progress) {
1650 progress = false;
1651 iteration++;
1652 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1653 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
1654
1655 for (int i = rpo_list.size() - 1; i >= 0; i--) {
1656 Node* c = rpo_list.at(i);
1657
1658 Node* prev_mem = _memory_nodes[c->_idx];
1659 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
1660 Node* prev_region = regions[c->_idx];
1661 Node* unique = nullptr;
1662 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
1663 Node* m = _memory_nodes[c->in(j)->_idx];
1664 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
1665 if (m != nullptr) {
1666 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
1667 assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1668 // continue
1669 } else if (unique == nullptr) {
1670 unique = m;
1671 } else if (m == unique) {
1672 // continue
1673 } else {
1674 unique = NodeSentinel;
1675 }
1676 }
1677 }
1678 assert(unique != nullptr, "empty phi???");
1679 if (unique != NodeSentinel) {
1680 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
1681 dead_phis.push(prev_region);
1682 }
1683 regions.map(c->_idx, unique);
1684 } else {
1685 Node* phi = nullptr;
1686 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
1687 phi = prev_region;
2474 case Op_CheckCastPP:
2475 case Op_CastPP:
2476 return needs_barrier_impl(phase, n->in(1), visited);
2477 case Op_Proj:
2478 return needs_barrier_impl(phase, n->in(0), visited);
2479 case Op_ShenandoahLoadReferenceBarrier:
2480 // tty->print_cr("optimize barrier on barrier");
2481 return false;
2482 case Op_Parm:
2483 // tty->print_cr("optimize barrier on input arg");
2484 return false;
2485 case Op_DecodeN:
2486 case Op_EncodeP:
2487 return needs_barrier_impl(phase, n->in(1), visited);
2488 case Op_LoadN:
2489 return true;
2490 case Op_CMoveN:
2491 case Op_CMoveP:
2492 return needs_barrier_impl(phase, n->in(2), visited) ||
2493 needs_barrier_impl(phase, n->in(3), visited);
2494 case Op_CreateEx:
2495 return false;
2496 default:
2497 break;
2498 }
2499 #ifdef ASSERT
2500 tty->print("need barrier on?: ");
2501 tty->print_cr("ins:");
2502 n->dump(2);
2503 tty->print_cr("outs:");
2504 n->dump(-2);
2505 ShouldNotReachHere();
2506 #endif
2507 return true;
2508 }
|