29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if ((state->iu_barriers_count() +
50 state->load_reference_barriers_count()) > 0) {
51 assert(C->post_loop_opts_phase(), "no loop opts allowed");
52 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
53 C->clear_major_progress();
54 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
55 if (C->failing()) return false;
56
57 C->set_major_progress();
58 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
59 return false;
60 }
61 C->clear_major_progress();
62 C->process_for_post_loop_opts_igvn(igvn);
63 if (C->failing()) return false;
64
65 C->set_post_loop_opts_phase(); // now for real!
66 }
67 return true;
68 }
69
70 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
71 if (!UseShenandoahGC) {
72 return false;
73 }
74 assert(iff->is_If(), "bad input");
75 if (iff->Opcode() != Op_If) {
76 return false;
77 }
78 Node* bol = iff->in(1);
79 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
80 return false;
81 }
169 continue;
170 } else if (in->is_Con()) {
171 if (trace) {
172 tty->print("Found constant");
173 in->dump();
174 }
175 } else if (in->Opcode() == Op_Parm) {
176 if (trace) {
177 tty->print("Found argument");
178 }
179 } else if (in->Opcode() == Op_CreateEx) {
180 if (trace) {
181 tty->print("Found create-exception");
182 }
183 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
184 if (trace) {
185 tty->print("Found raw LoadP (OSR argument?)");
186 }
187 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
188 if (t == ShenandoahOopStore) {
189 uint i = 0;
190 for (; i < phis.size(); i++) {
191 Node* n = phis.node_at(i);
192 if (n->Opcode() == Op_ShenandoahIUBarrier) {
193 break;
194 }
195 }
196 if (i == phis.size()) {
197 return false;
198 }
199 }
200 barriers_used.push(in);
201 if (trace) {tty->print("Found barrier"); in->dump();}
202 } else if (in->Opcode() == Op_ShenandoahIUBarrier) {
203 if (t != ShenandoahOopStore) {
204 in = in->in(1);
205 continue;
206 }
207 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
208 phis.push(in, in->req());
209 in = in->in(1);
210 continue;
211 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
212 if (trace) {
213 tty->print("Found alloc");
214 in->in(0)->dump();
215 }
216 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
217 if (trace) {
218 tty->print("Found Java call");
219 }
220 } else if (in->is_Phi()) {
221 if (!visited.test_set(in->_idx)) {
222 if (trace) {tty->print("Pushed phi:"); in->dump();}
223 phis.push(in, 2);
224 in = in->in(1);
225 continue;
226 }
227 if (trace) {tty->print("Already seen phi:"); in->dump();}
228 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
229 if (!visited.test_set(in->_idx)) {
230 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
309 if (trace) {tty->print("Verifying"); n->dump();}
310 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
311 Node* adr = n->in(MemNode::Address);
312 bool verify = true;
313
314 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
315 adr = adr->in(AddPNode::Address);
316 if (adr->is_AddP()) {
317 assert(adr->in(AddPNode::Base)->is_top(), "");
318 adr = adr->in(AddPNode::Address);
319 if (adr->Opcode() == Op_LoadP &&
320 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
321 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
322 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
323 if (trace) {tty->print_cr("SATB prebarrier");}
324 verify = false;
325 }
326 }
327 }
328
329 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
330 report_verify_failure("Shenandoah verification: Store should have barriers", n);
331 }
332 }
333 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
334 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
335 }
336 } else if (n->Opcode() == Op_CmpP) {
337 const bool trace = false;
338
339 Node* in1 = n->in(1);
340 Node* in2 = n->in(2);
341 if (in1->bottom_type()->isa_oopptr()) {
342 if (trace) {tty->print("Verifying"); n->dump();}
343
344 bool mark_inputs = false;
345 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
346 (in1->is_Con() || in2->is_Con())) {
347 if (trace) {tty->print_cr("Comparison against a constant");}
348 mark_inputs = true;
349 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
351 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
352 mark_inputs = true;
353 } else {
354 assert(in2->bottom_type()->isa_oopptr(), "");
355
356 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
357 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
358 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
359 }
360 }
361 if (verify_no_useless_barrier &&
362 mark_inputs &&
363 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
364 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
365 phis.clear();
366 visited.reset();
367 }
368 }
369 } else if (n->is_LoadStore()) {
370 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
371 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
372 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
373 }
374
375 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
376 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
377 }
378 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
379 CallNode* call = n->as_Call();
380
381 static struct {
382 const char* name;
383 struct {
384 int pos;
385 verify_type t;
386 } args[6];
387 } calls[] = {
388 "aescrypt_encryptBlock",
389 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
390 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
391 "aescrypt_decryptBlock",
392 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
394 "multiplyToLen",
395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
397 "squareToLen",
398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
400 "montgomery_multiply",
401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
402 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
403 "montgomery_square",
404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
405 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
406 "mulAdd",
407 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
410 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
412 "updateBytesCRC32",
413 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
414 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
415 "updateBytesAdler32",
416 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
418 "updateBytesCRC32C",
419 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
421 "counterMode_AESCrypt",
422 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
423 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
424 "cipherBlockChaining_encryptAESCrypt",
425 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
426 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
427 "cipherBlockChaining_decryptAESCrypt",
428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
429 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
430 "shenandoah_clone_barrier",
431 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
432 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
433 "ghash_processBlocks",
434 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
435 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
436 "sha1_implCompress",
437 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
439 "sha256_implCompress",
440 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
442 "sha512_implCompress",
443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
445 "sha1_implCompressMB",
446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
448 "sha256_implCompressMB",
449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
503 }
504 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
505 if (call->in(j)->bottom_type()->make_ptr() &&
506 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
507 uint k = 0;
508 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
509 if (k == args_len) {
510 fatal("arg %d for call %s not covered", j, call->_name);
511 }
512 }
513 }
514 } else {
515 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
516 if (call->in(j)->bottom_type()->make_ptr() &&
517 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
518 fatal("%s not covered", call->_name);
519 }
520 }
521 }
522 }
523 } else if (n->Opcode() == Op_ShenandoahIUBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
524 // skip
525 } else if (n->is_AddP()
526 || n->is_Phi()
527 || n->is_ConstraintCast()
528 || n->Opcode() == Op_Return
529 || n->Opcode() == Op_CMoveP
530 || n->Opcode() == Op_CMoveN
531 || n->Opcode() == Op_Rethrow
532 || n->is_MemBar()
533 || n->Opcode() == Op_Conv2B
534 || n->Opcode() == Op_SafePoint
535 || n->is_CallJava()
536 || n->Opcode() == Op_Unlock
537 || n->Opcode() == Op_EncodeP
538 || n->Opcode() == Op_DecodeN) {
539 // nothing to do
540 } else {
541 static struct {
542 int opcode;
543 struct {
1085 assert(region != nullptr, "");
1086 Node* phi = new PhiNode(region, n->bottom_type());
1087 for (uint j = 1; j < region->req(); j++) {
1088 Node* in = region->in(j);
1089 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1090 phi->init_req(j, n);
1091 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1092 phi->init_req(j, n_clone);
1093 } else {
1094 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1095 }
1096 }
1097 phase->register_new_node(phi, region);
1098 return phi;
1099 }
1100
1101 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1102 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1103
1104 Unique_Node_List uses;
1105 for (int i = 0; i < state->iu_barriers_count(); i++) {
1106 Node* barrier = state->iu_barrier(i);
1107 Node* ctrl = phase->get_ctrl(barrier);
1108 IdealLoopTree* loop = phase->get_loop(ctrl);
1109 Node* head = loop->head();
1110 if (head->is_OuterStripMinedLoop()) {
1111 // Expanding a barrier here will break loop strip mining
1112 // verification. Transform the loop so the loop nest doesn't
1113 // appear as strip mined.
1114 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop();
1115 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1116 }
1117 }
1118
1119 Node_Stack stack(0);
1120 Node_List clones;
1121 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1122 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1123
1124 Node* ctrl = phase->get_ctrl(lrb);
1125 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1126
1127 CallStaticJavaNode* unc = nullptr;
1128 Node* unc_ctrl = nullptr;
1129 Node* uncasted_val = val;
1130
1131 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1132 Node* u = lrb->fast_out(i);
1133 if (u->Opcode() == Op_CastPP &&
1134 u->in(0) != nullptr &&
1135 phase->is_dominator(u->in(0), ctrl)) {
1136 const Type* u_t = phase->igvn().type(u);
1137
1138 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1431
1432 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1433
1434 ctrl = orig_ctrl;
1435
1436 phase->igvn().replace_node(lrb, out_val);
1437
1438 follow_barrier_uses(out_val, ctrl, uses, phase);
1439
1440 for(uint next = 0; next < uses.size(); next++ ) {
1441 Node *n = uses.at(next);
1442 assert(phase->get_ctrl(n) == ctrl, "bad control");
1443 assert(n != raw_mem, "should leave input raw mem above the barrier");
1444 phase->set_ctrl(n, region);
1445 follow_barrier_uses(n, ctrl, uses, phase);
1446 }
1447 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1448 }
1449 // Done expanding load-reference-barriers.
1450 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1451
1452 for (int i = state->iu_barriers_count() - 1; i >= 0; i--) {
1453 Node* barrier = state->iu_barrier(i);
1454 Node* pre_val = barrier->in(1);
1455
1456 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
1457 ShouldNotReachHere();
1458 continue;
1459 }
1460
1461 Node* ctrl = phase->get_ctrl(barrier);
1462
1463 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1464 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
1465 ctrl = ctrl->in(0)->in(0);
1466 phase->set_ctrl(barrier, ctrl);
1467 } else if (ctrl->is_CallRuntime()) {
1468 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
1469 ctrl = ctrl->in(0);
1470 phase->set_ctrl(barrier, ctrl);
1471 }
1472
1473 Node* init_ctrl = ctrl;
1474 IdealLoopTree* loop = phase->get_loop(ctrl);
1475 Node* raw_mem = fixer.find_mem(ctrl, barrier);
1476 Node* init_raw_mem = raw_mem;
1477 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr);
1478 Node* heap_stable_ctrl = nullptr;
1479 Node* null_ctrl = nullptr;
1480 uint last = phase->C->unique();
1481
1482 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1483 Node* region = new RegionNode(PATH_LIMIT);
1484 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1485
1486 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1487 Node* region2 = new RegionNode(PATH_LIMIT2);
1488 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1489
1490 // Stable path.
1491 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
1492 region->init_req(_heap_stable, heap_stable_ctrl);
1493 phi->init_req(_heap_stable, raw_mem);
1494
1495 // Null path
1496 Node* reg2_ctrl = nullptr;
1497 test_null(ctrl, pre_val, null_ctrl, phase);
1498 if (null_ctrl != nullptr) {
1499 reg2_ctrl = null_ctrl->in(0);
1500 region2->init_req(_null_path, null_ctrl);
1501 phi2->init_req(_null_path, raw_mem);
1502 } else {
1503 region2->del_req(_null_path);
1504 phi2->del_req(_null_path);
1505 }
1506
1507 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1508 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1509 Node* thread = new ThreadLocalNode();
1510 phase->register_new_node(thread, ctrl);
1511 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
1512 phase->register_new_node(buffer_adr, ctrl);
1513 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
1514 phase->register_new_node(index_adr, ctrl);
1515
1516 BasicType index_bt = TypeX_X->basic_type();
1517 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
1518 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1519 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
1520 phase->register_new_node(index, ctrl);
1521 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
1522 phase->register_new_node(index_cmp, ctrl);
1523 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
1524 phase->register_new_node(index_test, ctrl);
1525 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1526 if (reg2_ctrl == nullptr) reg2_ctrl = queue_full_iff;
1527 phase->register_control(queue_full_iff, loop, ctrl);
1528 Node* not_full = new IfTrueNode(queue_full_iff);
1529 phase->register_control(not_full, loop, queue_full_iff);
1530 Node* full = new IfFalseNode(queue_full_iff);
1531 phase->register_control(full, loop, queue_full_iff);
1532
1533 ctrl = not_full;
1534
1535 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
1536 phase->register_new_node(next_index, ctrl);
1537
1538 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
1539 phase->register_new_node(buffer, ctrl);
1540 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
1541 phase->register_new_node(log_addr, ctrl);
1542 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
1543 phase->register_new_node(log_store, ctrl);
1544 // update the index
1545 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
1546 phase->register_new_node(index_update, ctrl);
1547
1548 // Fast-path case
1549 region2->init_req(_fast_path, ctrl);
1550 phi2->init_req(_fast_path, index_update);
1551
1552 ctrl = full;
1553
1554 Node* base = find_bottom_mem(ctrl, phase);
1555
1556 MergeMemNode* mm = MergeMemNode::make(base);
1557 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1558 phase->register_new_node(mm, ctrl);
1559
1560 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
1561 call->init_req(TypeFunc::Control, ctrl);
1562 call->init_req(TypeFunc::I_O, phase->C->top());
1563 call->init_req(TypeFunc::Memory, mm);
1564 call->init_req(TypeFunc::FramePtr, phase->C->top());
1565 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1566 call->init_req(TypeFunc::Parms, pre_val);
1567 call->init_req(TypeFunc::Parms+1, thread);
1568 phase->register_control(call, loop, ctrl);
1569
1570 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
1571 phase->register_control(ctrl_proj, loop, call);
1572 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
1573 phase->register_new_node(mem_proj, call);
1574
1575 // Slow-path case
1576 region2->init_req(_slow_path, ctrl_proj);
1577 phi2->init_req(_slow_path, mem_proj);
1578
1579 phase->register_control(region2, loop, reg2_ctrl);
1580 phase->register_new_node(phi2, region2);
1581
1582 region->init_req(_heap_unstable, region2);
1583 phi->init_req(_heap_unstable, phi2);
1584
1585 phase->register_control(region, loop, heap_stable_ctrl->in(0));
1586 phase->register_new_node(phi, region);
1587
1588 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
1589 for(uint next = 0; next < uses.size(); next++ ) {
1590 Node *n = uses.at(next);
1591 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1592 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1593 phase->set_ctrl(n, region);
1594 follow_barrier_uses(n, init_ctrl, uses, phase);
1595 }
1596 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1597
1598 phase->igvn().replace_node(barrier, pre_val);
1599 }
1600 assert(state->iu_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
1601
1602 }
1603
1604 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1605 if (visited.test_set(in->_idx)) {
1606 return nullptr;
1607 }
1608 switch (in->Opcode()) {
1609 case Op_Proj:
1610 return get_load_addr(phase, visited, in->in(0));
1611 case Op_CastPP:
1612 case Op_CheckCastPP:
1613 case Op_DecodeN:
1614 case Op_EncodeP:
1615 return get_load_addr(phase, visited, in->in(1));
1616 case Op_LoadN:
1617 case Op_LoadP:
1618 return in->in(MemNode::Address);
1619 case Op_CompareAndExchangeN:
1620 case Op_CompareAndExchangeP:
1621 case Op_GetAndSetN:
1634 if (t == nullptr && f != nullptr) return f;
1635 if (t != nullptr && t == f) return t;
1636 // Ambiguity.
1637 return phase->igvn().zerocon(T_OBJECT);
1638 }
1639 case Op_Phi: {
1640 Node* addr = nullptr;
1641 for (uint i = 1; i < in->req(); i++) {
1642 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1643 if (addr == nullptr) {
1644 addr = addr1;
1645 }
1646 if (addr != addr1) {
1647 return phase->igvn().zerocon(T_OBJECT);
1648 }
1649 }
1650 return addr;
1651 }
1652 case Op_ShenandoahLoadReferenceBarrier:
1653 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1654 case Op_ShenandoahIUBarrier:
1655 return get_load_addr(phase, visited, in->in(1));
1656 case Op_CallDynamicJava:
1657 case Op_CallLeaf:
1658 case Op_CallStaticJava:
1659 case Op_ConN:
1660 case Op_ConP:
1661 case Op_Parm:
1662 case Op_CreateEx:
1663 return phase->igvn().zerocon(T_OBJECT);
1664 default:
1665 #ifdef ASSERT
1666 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1667 #endif
1668 return phase->igvn().zerocon(T_OBJECT);
1669 }
1670
1671 }
1672
1673 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1674 IdealLoopTree *loop = phase->get_loop(iff);
1675 Node* loop_head = loop->_head;
1676 Node* entry_c = loop_head->in(LoopNode::EntryControl);
1677
1678 Node* bol = iff->in(1);
1679 Node* cmp = bol->in(1);
1680 Node* andi = cmp->in(1);
1681 Node* load = andi->in(1);
1682
1683 assert(is_gc_state_load(load), "broken");
1684 if (!phase->is_dominator(load->in(0), entry_c)) {
1685 Node* mem_ctrl = nullptr;
1686 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
1687 load = load->clone();
1688 load->set_req(MemNode::Memory, mem);
1689 load->set_req(0, entry_c);
1690 phase->register_new_node(load, entry_c);
1691 andi = andi->clone();
1692 andi->set_req(1, load);
1693 phase->register_new_node(andi, entry_c);
1694 cmp = cmp->clone();
1695 cmp->set_req(1, andi);
1696 phase->register_new_node(cmp, entry_c);
1697 bol = bol->clone();
1698 bol->set_req(1, cmp);
1699 phase->register_new_node(bol, entry_c);
1700
1701 phase->igvn().replace_input_of(iff, 1, bol);
1702 }
1703 }
1704
1705 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
1706 if (!n->is_If() || n->is_CountedLoopEnd()) {
1707 return false;
1708 }
1709 Node* region = n->in(0);
1710
1711 if (!region->is_Region()) {
1712 return false;
1713 }
1714 Node* dom = phase->idom(region);
1715 if (!dom->is_If()) {
1716 return false;
1717 }
1718
1719 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
1720 return false;
1721 }
1722
1723 IfNode* dom_if = dom->as_If();
1724 Node* proj_true = dom_if->proj_out(1);
1725 Node* proj_false = dom_if->proj_out(0);
1726
1727 for (uint i = 1; i < region->req(); i++) {
1728 if (phase->is_dominator(proj_true, region->in(i))) {
1729 continue;
1730 }
1731 if (phase->is_dominator(proj_false, region->in(i))) {
1732 continue;
1733 }
1734 return false;
1735 }
1736
1737 return true;
1738 }
1739
1740 bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) {
1741 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1742 Node* n = region->fast_out(i);
1743 if (n->is_LoadStore()) {
1744 // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support
1745 // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes
1746 // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get
1747 // expanded around a LoadStore node.
1748 return false;
1749 }
1750 }
1751 return true;
1752 }
1753
1754
1755 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
1756 assert(is_heap_stable_test(n), "no other tests");
1757 if (identical_backtoback_ifs(n, phase)) {
1758 Node* n_ctrl = n->in(0);
1759 if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) {
1760 IfNode* dom_if = phase->idom(n_ctrl)->as_If();
1761 if (is_heap_stable_test(n)) {
1762 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
1763 assert(is_gc_state_load(gc_state_load), "broken");
1764 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
1765 assert(is_gc_state_load(dom_gc_state_load), "broken");
1766 if (gc_state_load != dom_gc_state_load) {
1767 phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
1768 }
1769 }
1770 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
1771 Node* proj_true = dom_if->proj_out(1);
1772 Node* proj_false = dom_if->proj_out(0);
1773 Node* con_true = phase->igvn().makecon(TypeInt::ONE);
1774 Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
1775
1776 for (uint i = 1; i < n_ctrl->req(); i++) {
1777 if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
1778 bolphi->init_req(i, con_true);
1779 } else {
1780 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
1781 bolphi->init_req(i, con_false);
1782 }
1783 }
1784 phase->register_new_node(bolphi, n_ctrl);
1785 phase->igvn().replace_input_of(n, 1, bolphi);
1786 phase->do_split_if(n);
1787 }
1788 }
1789 }
1790
1791 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
1792 // Find first invariant test that doesn't exit the loop
1793 LoopNode *head = loop->_head->as_Loop();
1794 IfNode* unswitch_iff = nullptr;
1795 Node* n = head->in(LoopNode::LoopBackControl);
1796 int loop_has_sfpts = -1;
1797 while (n != head) {
1798 Node* n_dom = phase->idom(n);
1799 if (n->is_Region()) {
1800 if (n_dom->is_If()) {
1801 IfNode* iff = n_dom->as_If();
1802 if (iff->in(1)->is_Bool()) {
1803 BoolNode* bol = iff->in(1)->as_Bool();
1804 if (bol->in(1)->is_Cmp()) {
1805 // If condition is invariant and not a loop exit,
1806 // then found reason to unswitch.
1807 if (is_heap_stable_test(iff) &&
1808 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
1809 assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
1810 if (loop_has_sfpts == -1) {
1811 for(uint i = 0; i < loop->_body.size(); i++) {
1812 Node *m = loop->_body[i];
1813 if (m->is_SafePoint() && !m->is_CallLeaf()) {
1814 loop_has_sfpts = 1;
1815 break;
1816 }
1817 }
1818 if (loop_has_sfpts == -1) {
1819 loop_has_sfpts = 0;
1820 }
1821 }
1822 if (!loop_has_sfpts) {
1823 unswitch_iff = iff;
1824 }
1825 }
1826 }
1827 }
1828 }
1829 }
1830 n = n_dom;
1831 }
1832 return unswitch_iff;
1833 }
1834
1835
1836 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
1837 Node_List heap_stable_tests;
1838 stack.push(phase->C->start(), 0);
1839 do {
1840 Node* n = stack.node();
1841 uint i = stack.index();
1842
1843 if (i < n->outcnt()) {
1844 Node* u = n->raw_out(i);
1845 stack.set_index(i+1);
1846 if (!visited.test_set(u->_idx)) {
1847 stack.push(u, 0);
1848 }
1849 } else {
1850 stack.pop();
1851 if (n->is_If() && is_heap_stable_test(n)) {
1852 heap_stable_tests.push(n);
1853 }
1854 }
1855 } while (stack.size() > 0);
1856
1857 for (uint i = 0; i < heap_stable_tests.size(); i++) {
1858 Node* n = heap_stable_tests.at(i);
1859 assert(is_heap_stable_test(n), "only evacuation test");
1860 merge_back_to_back_tests(n, phase);
1861 }
1862
1863 if (!phase->C->major_progress()) {
1864 VectorSet seen;
1865 for (uint i = 0; i < heap_stable_tests.size(); i++) {
1866 Node* n = heap_stable_tests.at(i);
1867 IdealLoopTree* loop = phase->get_loop(n);
1868 if (loop != phase->ltree_root() &&
1869 loop->_child == nullptr &&
1870 !loop->_irreducible) {
1871 Node* head = loop->_head;
1872 if (head->is_Loop() &&
1873 (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
1874 !seen.test_set(head->_idx)) {
1875 IfNode* iff = find_unswitching_candidate(loop, phase);
1876 if (iff != nullptr) {
1877 Node* bol = iff->in(1);
1878 if (head->as_Loop()->is_strip_mined()) {
1879 head->as_Loop()->verify_strip_mined(0);
1880 }
1881 move_gc_state_test_out_of_loop(iff, phase);
1882
1883 AutoNodeBudget node_budget(phase);
1884
1885 if (loop->policy_unswitching(phase)) {
1886 if (head->as_Loop()->is_strip_mined()) {
1887 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1888 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1889 }
1890 phase->do_unswitching(loop, old_new);
1891 } else {
1892 // Not proceeding with unswitching. Move load back in
1893 // the loop.
1894 phase->igvn().replace_input_of(iff, 1, bol);
1895 }
1896 }
1897 }
1898 }
1899 }
1900 }
1901 }
1902
1903 ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(nullptr, val) {
1904 ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
1905 }
1906
1907 const Type* ShenandoahIUBarrierNode::bottom_type() const {
1908 if (in(1) == nullptr || in(1)->is_top()) {
1909 return Type::TOP;
1910 }
1911 const Type* t = in(1)->bottom_type();
1912 if (t == TypePtr::NULL_PTR) {
1913 return t;
1914 }
1915 return t->is_oopptr();
1916 }
1917
1918 const Type* ShenandoahIUBarrierNode::Value(PhaseGVN* phase) const {
1919 if (in(1) == nullptr) {
1920 return Type::TOP;
1921 }
1922 const Type* t = phase->type(in(1));
1923 if (t == Type::TOP) {
1924 return Type::TOP;
1925 }
1926 if (t == TypePtr::NULL_PTR) {
1927 return t;
1928 }
1929 return t->is_oopptr();
1930 }
1931
1932 int ShenandoahIUBarrierNode::needed(Node* n) {
1933 if (n == nullptr ||
1934 n->is_Allocate() ||
1935 n->Opcode() == Op_ShenandoahIUBarrier ||
1936 n->bottom_type() == TypePtr::NULL_PTR ||
1937 (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr)) {
1938 return NotNeeded;
1939 }
1940 if (n->is_Phi() ||
1941 n->is_CMove()) {
1942 return MaybeNeeded;
1943 }
1944 return Needed;
1945 }
1946
1947 Node* ShenandoahIUBarrierNode::next(Node* n) {
1948 for (;;) {
1949 if (n == nullptr) {
1950 return n;
1951 } else if (n->bottom_type() == TypePtr::NULL_PTR) {
1952 return n;
1953 } else if (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr) {
1954 return n;
1955 } else if (n->is_ConstraintCast() ||
1956 n->Opcode() == Op_DecodeN ||
1957 n->Opcode() == Op_EncodeP) {
1958 n = n->in(1);
1959 } else if (n->is_Proj()) {
1960 n = n->in(0);
1961 } else {
1962 return n;
1963 }
1964 }
1965 ShouldNotReachHere();
1966 return nullptr;
1967 }
1968
1969 Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) {
1970 PhaseIterGVN* igvn = phase->is_IterGVN();
1971
1972 Node* n = next(in(1));
1973
1974 int cont = needed(n);
1975
1976 if (cont == NotNeeded) {
1977 return in(1);
1978 } else if (cont == MaybeNeeded) {
1979 if (igvn == nullptr) {
1980 phase->record_for_igvn(this);
1981 return this;
1982 } else {
1983 ResourceMark rm;
1984 Unique_Node_List wq;
1985 uint wq_i = 0;
1986
1987 for (;;) {
1988 if (n->is_Phi()) {
1989 for (uint i = 1; i < n->req(); i++) {
1990 Node* m = n->in(i);
1991 if (m != nullptr) {
1992 wq.push(m);
1993 }
1994 }
1995 } else {
1996 assert(n->is_CMove(), "nothing else here");
1997 Node* m = n->in(CMoveNode::IfFalse);
1998 wq.push(m);
1999 m = n->in(CMoveNode::IfTrue);
2000 wq.push(m);
2001 }
2002 Node* orig_n = nullptr;
2003 do {
2004 if (wq_i >= wq.size()) {
2005 return in(1);
2006 }
2007 n = wq.at(wq_i);
2008 wq_i++;
2009 orig_n = n;
2010 n = next(n);
2011 cont = needed(n);
2012 if (cont == Needed) {
2013 return this;
2014 }
2015 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
2016 }
2017 }
2018 }
2019
2020 return this;
2021 }
2022
2023 #ifdef ASSERT
2024 static bool has_never_branch(Node* root) {
2025 for (uint i = 1; i < root->req(); i++) {
2026 Node* in = root->in(i);
2027 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
2028 return true;
2029 }
2030 }
2031 return false;
2032 }
2033 #endif
2034
2035 void MemoryGraphFixer::collect_memory_nodes() {
2036 Node_Stack stack(0);
2037 VectorSet visited;
2038 Node_List regions;
2039
2040 // Walk the raw memory graph and create a mapping from CFG node to
2041 // memory node. Exclude phis for now.
2042 stack.push(_phase->C->root(), 1);
2159 int iteration = 0;
2160 Node_List dead_phis;
2161 while (progress) {
2162 progress = false;
2163 iteration++;
2164 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2165 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2166
2167 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2168 Node* c = rpo_list.at(i);
2169
2170 Node* prev_mem = _memory_nodes[c->_idx];
2171 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2172 Node* prev_region = regions[c->_idx];
2173 Node* unique = nullptr;
2174 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2175 Node* m = _memory_nodes[c->in(j)->_idx];
2176 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2177 if (m != nullptr) {
2178 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2179 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2180 // continue
2181 } else if (unique == nullptr) {
2182 unique = m;
2183 } else if (m == unique) {
2184 // continue
2185 } else {
2186 unique = NodeSentinel;
2187 }
2188 }
2189 }
2190 assert(unique != nullptr, "empty phi???");
2191 if (unique != NodeSentinel) {
2192 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
2193 dead_phis.push(prev_region);
2194 }
2195 regions.map(c->_idx, unique);
2196 } else {
2197 Node* phi = nullptr;
2198 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
2199 phi = prev_region;
2986 case Op_CheckCastPP:
2987 case Op_CastPP:
2988 return needs_barrier_impl(phase, n->in(1), visited);
2989 case Op_Proj:
2990 return needs_barrier_impl(phase, n->in(0), visited);
2991 case Op_ShenandoahLoadReferenceBarrier:
2992 // tty->print_cr("optimize barrier on barrier");
2993 return false;
2994 case Op_Parm:
2995 // tty->print_cr("optimize barrier on input arg");
2996 return false;
2997 case Op_DecodeN:
2998 case Op_EncodeP:
2999 return needs_barrier_impl(phase, n->in(1), visited);
3000 case Op_LoadN:
3001 return true;
3002 case Op_CMoveN:
3003 case Op_CMoveP:
3004 return needs_barrier_impl(phase, n->in(2), visited) ||
3005 needs_barrier_impl(phase, n->in(3), visited);
3006 case Op_ShenandoahIUBarrier:
3007 return needs_barrier_impl(phase, n->in(1), visited);
3008 case Op_CreateEx:
3009 return false;
3010 default:
3011 break;
3012 }
3013 #ifdef ASSERT
3014 tty->print("need barrier on?: ");
3015 tty->print_cr("ins:");
3016 n->dump(2);
3017 tty->print_cr("outs:");
3018 n->dump(-2);
3019 ShouldNotReachHere();
3020 #endif
3021 return true;
3022 }
|
29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if (state->load_reference_barriers_count() > 0) {
50 assert(C->post_loop_opts_phase(), "no loop opts allowed");
51 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
52 C->clear_major_progress();
53 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
54 if (C->failing()) return false;
55 C->process_for_post_loop_opts_igvn(igvn);
56 if (C->failing()) return false;
57
58 C->set_post_loop_opts_phase(); // now for real!
59 }
60 return true;
61 }
62
63 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
64 if (!UseShenandoahGC) {
65 return false;
66 }
67 assert(iff->is_If(), "bad input");
68 if (iff->Opcode() != Op_If) {
69 return false;
70 }
71 Node* bol = iff->in(1);
72 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
73 return false;
74 }
162 continue;
163 } else if (in->is_Con()) {
164 if (trace) {
165 tty->print("Found constant");
166 in->dump();
167 }
168 } else if (in->Opcode() == Op_Parm) {
169 if (trace) {
170 tty->print("Found argument");
171 }
172 } else if (in->Opcode() == Op_CreateEx) {
173 if (trace) {
174 tty->print("Found create-exception");
175 }
176 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
177 if (trace) {
178 tty->print("Found raw LoadP (OSR argument?)");
179 }
180 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
181 if (t == ShenandoahOopStore) {
182 return false;
183 }
184 barriers_used.push(in);
185 if (trace) {tty->print("Found barrier"); in->dump();}
186 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
187 if (trace) {
188 tty->print("Found alloc");
189 in->in(0)->dump();
190 }
191 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
192 if (trace) {
193 tty->print("Found Java call");
194 }
195 } else if (in->is_Phi()) {
196 if (!visited.test_set(in->_idx)) {
197 if (trace) {tty->print("Pushed phi:"); in->dump();}
198 phis.push(in, 2);
199 in = in->in(1);
200 continue;
201 }
202 if (trace) {tty->print("Already seen phi:"); in->dump();}
203 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
204 if (!visited.test_set(in->_idx)) {
205 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
284 if (trace) {tty->print("Verifying"); n->dump();}
285 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
286 Node* adr = n->in(MemNode::Address);
287 bool verify = true;
288
289 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
290 adr = adr->in(AddPNode::Address);
291 if (adr->is_AddP()) {
292 assert(adr->in(AddPNode::Base)->is_top(), "");
293 adr = adr->in(AddPNode::Address);
294 if (adr->Opcode() == Op_LoadP &&
295 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
296 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
297 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
298 if (trace) {tty->print_cr("SATB prebarrier");}
299 verify = false;
300 }
301 }
302 }
303
304 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
305 report_verify_failure("Shenandoah verification: Store should have barriers", n);
306 }
307 }
308 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
309 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
310 }
311 } else if (n->Opcode() == Op_CmpP) {
312 const bool trace = false;
313
314 Node* in1 = n->in(1);
315 Node* in2 = n->in(2);
316 if (in1->bottom_type()->isa_oopptr()) {
317 if (trace) {tty->print("Verifying"); n->dump();}
318
319 bool mark_inputs = false;
320 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
321 (in1->is_Con() || in2->is_Con())) {
322 if (trace) {tty->print_cr("Comparison against a constant");}
323 mark_inputs = true;
324 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
326 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
327 mark_inputs = true;
328 } else {
329 assert(in2->bottom_type()->isa_oopptr(), "");
330
331 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
332 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
333 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
334 }
335 }
336 if (verify_no_useless_barrier &&
337 mark_inputs &&
338 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
339 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
340 phis.clear();
341 visited.reset();
342 }
343 }
344 } else if (n->is_LoadStore()) {
345 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
346 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
347 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
348 }
349
350 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
351 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
352 }
353 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
354 CallNode* call = n->as_Call();
355
356 static struct {
357 const char* name;
358 struct {
359 int pos;
360 verify_type t;
361 } args[6];
362 } calls[] = {
363 "array_partition_stub",
364 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone },
365 { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
366 "arraysort_stub",
367 { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone },
368 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
369 "aescrypt_encryptBlock",
370 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
371 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
372 "aescrypt_decryptBlock",
373 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
374 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
375 "multiplyToLen",
376 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
377 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
378 "squareToLen",
379 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
380 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
381 "montgomery_multiply",
382 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
383 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
384 "montgomery_square",
385 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
386 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
387 "mulAdd",
388 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
391 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
392 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
393 "updateBytesCRC32",
394 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
395 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
396 "updateBytesAdler32",
397 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
398 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
399 "updateBytesCRC32C",
400 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
401 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
402 "counterMode_AESCrypt",
403 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
404 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
405 "cipherBlockChaining_encryptAESCrypt",
406 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
407 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
408 "cipherBlockChaining_decryptAESCrypt",
409 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
410 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
411 "shenandoah_clone",
412 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
413 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
414 "ghash_processBlocks",
415 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
416 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
417 "sha1_implCompress",
418 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
419 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
420 "sha256_implCompress",
421 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
422 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
423 "sha512_implCompress",
424 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
425 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
426 "sha1_implCompressMB",
427 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
428 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
429 "sha256_implCompressMB",
430 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
484 }
485 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
486 if (call->in(j)->bottom_type()->make_ptr() &&
487 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
488 uint k = 0;
489 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
490 if (k == args_len) {
491 fatal("arg %d for call %s not covered", j, call->_name);
492 }
493 }
494 }
495 } else {
496 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
497 if (call->in(j)->bottom_type()->make_ptr() &&
498 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
499 fatal("%s not covered", call->_name);
500 }
501 }
502 }
503 }
504 } else if (n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
505 // skip
506 } else if (n->is_AddP()
507 || n->is_Phi()
508 || n->is_ConstraintCast()
509 || n->Opcode() == Op_Return
510 || n->Opcode() == Op_CMoveP
511 || n->Opcode() == Op_CMoveN
512 || n->Opcode() == Op_Rethrow
513 || n->is_MemBar()
514 || n->Opcode() == Op_Conv2B
515 || n->Opcode() == Op_SafePoint
516 || n->is_CallJava()
517 || n->Opcode() == Op_Unlock
518 || n->Opcode() == Op_EncodeP
519 || n->Opcode() == Op_DecodeN) {
520 // nothing to do
521 } else {
522 static struct {
523 int opcode;
524 struct {
1066 assert(region != nullptr, "");
1067 Node* phi = new PhiNode(region, n->bottom_type());
1068 for (uint j = 1; j < region->req(); j++) {
1069 Node* in = region->in(j);
1070 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1071 phi->init_req(j, n);
1072 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1073 phi->init_req(j, n_clone);
1074 } else {
1075 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1076 }
1077 }
1078 phase->register_new_node(phi, region);
1079 return phi;
1080 }
1081
1082 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1083 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1084
1085 Unique_Node_List uses;
1086 Node_Stack stack(0);
1087 Node_List clones;
1088 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1089 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1090
1091 Node* ctrl = phase->get_ctrl(lrb);
1092 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1093
1094 CallStaticJavaNode* unc = nullptr;
1095 Node* unc_ctrl = nullptr;
1096 Node* uncasted_val = val;
1097
1098 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1099 Node* u = lrb->fast_out(i);
1100 if (u->Opcode() == Op_CastPP &&
1101 u->in(0) != nullptr &&
1102 phase->is_dominator(u->in(0), ctrl)) {
1103 const Type* u_t = phase->igvn().type(u);
1104
1105 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1398
1399 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1400
1401 ctrl = orig_ctrl;
1402
1403 phase->igvn().replace_node(lrb, out_val);
1404
1405 follow_barrier_uses(out_val, ctrl, uses, phase);
1406
1407 for(uint next = 0; next < uses.size(); next++ ) {
1408 Node *n = uses.at(next);
1409 assert(phase->get_ctrl(n) == ctrl, "bad control");
1410 assert(n != raw_mem, "should leave input raw mem above the barrier");
1411 phase->set_ctrl(n, region);
1412 follow_barrier_uses(n, ctrl, uses, phase);
1413 }
1414 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1415 }
1416 // Done expanding load-reference-barriers.
1417 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1418 }
1419
1420 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1421 if (visited.test_set(in->_idx)) {
1422 return nullptr;
1423 }
1424 switch (in->Opcode()) {
1425 case Op_Proj:
1426 return get_load_addr(phase, visited, in->in(0));
1427 case Op_CastPP:
1428 case Op_CheckCastPP:
1429 case Op_DecodeN:
1430 case Op_EncodeP:
1431 return get_load_addr(phase, visited, in->in(1));
1432 case Op_LoadN:
1433 case Op_LoadP:
1434 return in->in(MemNode::Address);
1435 case Op_CompareAndExchangeN:
1436 case Op_CompareAndExchangeP:
1437 case Op_GetAndSetN:
1450 if (t == nullptr && f != nullptr) return f;
1451 if (t != nullptr && t == f) return t;
1452 // Ambiguity.
1453 return phase->igvn().zerocon(T_OBJECT);
1454 }
1455 case Op_Phi: {
1456 Node* addr = nullptr;
1457 for (uint i = 1; i < in->req(); i++) {
1458 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1459 if (addr == nullptr) {
1460 addr = addr1;
1461 }
1462 if (addr != addr1) {
1463 return phase->igvn().zerocon(T_OBJECT);
1464 }
1465 }
1466 return addr;
1467 }
1468 case Op_ShenandoahLoadReferenceBarrier:
1469 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1470 case Op_CallDynamicJava:
1471 case Op_CallLeaf:
1472 case Op_CallStaticJava:
1473 case Op_ConN:
1474 case Op_ConP:
1475 case Op_Parm:
1476 case Op_CreateEx:
1477 return phase->igvn().zerocon(T_OBJECT);
1478 default:
1479 #ifdef ASSERT
1480 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1481 #endif
1482 return phase->igvn().zerocon(T_OBJECT);
1483 }
1484
1485 }
1486
1487 #ifdef ASSERT
1488 static bool has_never_branch(Node* root) {
1489 for (uint i = 1; i < root->req(); i++) {
1490 Node* in = root->in(i);
1491 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
1492 return true;
1493 }
1494 }
1495 return false;
1496 }
1497 #endif
1498
1499 void MemoryGraphFixer::collect_memory_nodes() {
1500 Node_Stack stack(0);
1501 VectorSet visited;
1502 Node_List regions;
1503
1504 // Walk the raw memory graph and create a mapping from CFG node to
1505 // memory node. Exclude phis for now.
1506 stack.push(_phase->C->root(), 1);
1623 int iteration = 0;
1624 Node_List dead_phis;
1625 while (progress) {
1626 progress = false;
1627 iteration++;
1628 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1629 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
1630
1631 for (int i = rpo_list.size() - 1; i >= 0; i--) {
1632 Node* c = rpo_list.at(i);
1633
1634 Node* prev_mem = _memory_nodes[c->_idx];
1635 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
1636 Node* prev_region = regions[c->_idx];
1637 Node* unique = nullptr;
1638 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
1639 Node* m = _memory_nodes[c->in(j)->_idx];
1640 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
1641 if (m != nullptr) {
1642 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
1643 assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1644 // continue
1645 } else if (unique == nullptr) {
1646 unique = m;
1647 } else if (m == unique) {
1648 // continue
1649 } else {
1650 unique = NodeSentinel;
1651 }
1652 }
1653 }
1654 assert(unique != nullptr, "empty phi???");
1655 if (unique != NodeSentinel) {
1656 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
1657 dead_phis.push(prev_region);
1658 }
1659 regions.map(c->_idx, unique);
1660 } else {
1661 Node* phi = nullptr;
1662 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
1663 phi = prev_region;
2450 case Op_CheckCastPP:
2451 case Op_CastPP:
2452 return needs_barrier_impl(phase, n->in(1), visited);
2453 case Op_Proj:
2454 return needs_barrier_impl(phase, n->in(0), visited);
2455 case Op_ShenandoahLoadReferenceBarrier:
2456 // tty->print_cr("optimize barrier on barrier");
2457 return false;
2458 case Op_Parm:
2459 // tty->print_cr("optimize barrier on input arg");
2460 return false;
2461 case Op_DecodeN:
2462 case Op_EncodeP:
2463 return needs_barrier_impl(phase, n->in(1), visited);
2464 case Op_LoadN:
2465 return true;
2466 case Op_CMoveN:
2467 case Op_CMoveP:
2468 return needs_barrier_impl(phase, n->in(2), visited) ||
2469 needs_barrier_impl(phase, n->in(3), visited);
2470 case Op_CreateEx:
2471 return false;
2472 default:
2473 break;
2474 }
2475 #ifdef ASSERT
2476 tty->print("need barrier on?: ");
2477 tty->print_cr("ins:");
2478 n->dump(2);
2479 tty->print_cr("outs:");
2480 n->dump(-2);
2481 ShouldNotReachHere();
2482 #endif
2483 return true;
2484 }
|