29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if ((state->iu_barriers_count() +
50 state->load_reference_barriers_count()) > 0) {
51 assert(C->post_loop_opts_phase(), "no loop opts allowed");
52 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
53 C->clear_major_progress();
54 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
55 if (C->failing()) return false;
56
57 C->set_major_progress();
58 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
59 return false;
60 }
61 C->clear_major_progress();
62 C->process_for_post_loop_opts_igvn(igvn);
63 if (C->failing()) return false;
64
65 C->set_post_loop_opts_phase(); // now for real!
66 }
67 return true;
68 }
69
70 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
169 continue;
170 } else if (in->is_Con()) {
171 if (trace) {
172 tty->print("Found constant");
173 in->dump();
174 }
175 } else if (in->Opcode() == Op_Parm) {
176 if (trace) {
177 tty->print("Found argument");
178 }
179 } else if (in->Opcode() == Op_CreateEx) {
180 if (trace) {
181 tty->print("Found create-exception");
182 }
183 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
184 if (trace) {
185 tty->print("Found raw LoadP (OSR argument?)");
186 }
187 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
188 if (t == ShenandoahOopStore) {
189 uint i = 0;
190 for (; i < phis.size(); i++) {
191 Node* n = phis.node_at(i);
192 if (n->Opcode() == Op_ShenandoahIUBarrier) {
193 break;
194 }
195 }
196 if (i == phis.size()) {
197 return false;
198 }
199 }
200 barriers_used.push(in);
201 if (trace) {tty->print("Found barrier"); in->dump();}
202 } else if (in->Opcode() == Op_ShenandoahIUBarrier) {
203 if (t != ShenandoahOopStore) {
204 in = in->in(1);
205 continue;
206 }
207 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
208 phis.push(in, in->req());
209 in = in->in(1);
210 continue;
211 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
212 if (trace) {
213 tty->print("Found alloc");
214 in->in(0)->dump();
215 }
216 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
217 if (trace) {
218 tty->print("Found Java call");
219 }
220 } else if (in->is_Phi()) {
221 if (!visited.test_set(in->_idx)) {
222 if (trace) {tty->print("Pushed phi:"); in->dump();}
223 phis.push(in, 2);
224 in = in->in(1);
225 continue;
226 }
227 if (trace) {tty->print("Already seen phi:"); in->dump();}
228 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
229 if (!visited.test_set(in->_idx)) {
230 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
309 if (trace) {tty->print("Verifying"); n->dump();}
310 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
311 Node* adr = n->in(MemNode::Address);
312 bool verify = true;
313
314 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
315 adr = adr->in(AddPNode::Address);
316 if (adr->is_AddP()) {
317 assert(adr->in(AddPNode::Base)->is_top(), "");
318 adr = adr->in(AddPNode::Address);
319 if (adr->Opcode() == Op_LoadP &&
320 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
321 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
322 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
323 if (trace) {tty->print_cr("SATB prebarrier");}
324 verify = false;
325 }
326 }
327 }
328
329 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
330 report_verify_failure("Shenandoah verification: Store should have barriers", n);
331 }
332 }
333 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
334 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
335 }
336 } else if (n->Opcode() == Op_CmpP) {
337 const bool trace = false;
338
339 Node* in1 = n->in(1);
340 Node* in2 = n->in(2);
341 if (in1->bottom_type()->isa_oopptr()) {
342 if (trace) {tty->print("Verifying"); n->dump();}
343
344 bool mark_inputs = false;
345 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
346 (in1->is_Con() || in2->is_Con())) {
347 if (trace) {tty->print_cr("Comparison against a constant");}
348 mark_inputs = true;
349 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
351 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
352 mark_inputs = true;
353 } else {
354 assert(in2->bottom_type()->isa_oopptr(), "");
355
356 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
357 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
358 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
359 }
360 }
361 if (verify_no_useless_barrier &&
362 mark_inputs &&
363 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
364 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
365 phis.clear();
366 visited.reset();
367 }
368 }
369 } else if (n->is_LoadStore()) {
370 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
371 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
372 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
373 }
374
375 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
376 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
377 }
378 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
379 CallNode* call = n->as_Call();
380
381 static struct {
382 const char* name;
383 struct {
384 int pos;
385 verify_type t;
386 } args[6];
387 } calls[] = {
388 "aescrypt_encryptBlock",
389 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
390 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
391 "aescrypt_decryptBlock",
392 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
394 "multiplyToLen",
395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
397 "squareToLen",
398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
400 "montgomery_multiply",
401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
402 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
403 "montgomery_square",
404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
405 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
406 "mulAdd",
407 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
410 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
412 "updateBytesCRC32",
413 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
414 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
415 "updateBytesAdler32",
416 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
418 "updateBytesCRC32C",
419 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
421 "counterMode_AESCrypt",
422 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
423 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
424 "cipherBlockChaining_encryptAESCrypt",
425 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
426 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
427 "cipherBlockChaining_decryptAESCrypt",
428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
429 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
430 "shenandoah_clone_barrier",
431 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
432 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
433 "ghash_processBlocks",
434 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
435 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
436 "sha1_implCompress",
437 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
439 "sha256_implCompress",
440 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
442 "sha512_implCompress",
443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
445 "sha1_implCompressMB",
446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
448 "sha256_implCompressMB",
449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
503 }
504 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
505 if (call->in(j)->bottom_type()->make_ptr() &&
506 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
507 uint k = 0;
508 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
509 if (k == args_len) {
510 fatal("arg %d for call %s not covered", j, call->_name);
511 }
512 }
513 }
514 } else {
515 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
516 if (call->in(j)->bottom_type()->make_ptr() &&
517 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
518 fatal("%s not covered", call->_name);
519 }
520 }
521 }
522 }
523 } else if (n->Opcode() == Op_ShenandoahIUBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
524 // skip
525 } else if (n->is_AddP()
526 || n->is_Phi()
527 || n->is_ConstraintCast()
528 || n->Opcode() == Op_Return
529 || n->Opcode() == Op_CMoveP
530 || n->Opcode() == Op_CMoveN
531 || n->Opcode() == Op_Rethrow
532 || n->is_MemBar()
533 || n->Opcode() == Op_Conv2B
534 || n->Opcode() == Op_SafePoint
535 || n->is_CallJava()
536 || n->Opcode() == Op_Unlock
537 || n->Opcode() == Op_EncodeP
538 || n->Opcode() == Op_DecodeN) {
539 // nothing to do
540 } else {
541 static struct {
542 int opcode;
543 struct {
1085 assert(region != nullptr, "");
1086 Node* phi = new PhiNode(region, n->bottom_type());
1087 for (uint j = 1; j < region->req(); j++) {
1088 Node* in = region->in(j);
1089 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1090 phi->init_req(j, n);
1091 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1092 phi->init_req(j, n_clone);
1093 } else {
1094 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1095 }
1096 }
1097 phase->register_new_node(phi, region);
1098 return phi;
1099 }
1100
1101 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1102 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1103
1104 Unique_Node_List uses;
1105 for (int i = 0; i < state->iu_barriers_count(); i++) {
1106 Node* barrier = state->iu_barrier(i);
1107 Node* ctrl = phase->get_ctrl(barrier);
1108 IdealLoopTree* loop = phase->get_loop(ctrl);
1109 Node* head = loop->head();
1110 if (head->is_OuterStripMinedLoop()) {
1111 // Expanding a barrier here will break loop strip mining
1112 // verification. Transform the loop so the loop nest doesn't
1113 // appear as strip mined.
1114 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop();
1115 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1116 }
1117 }
1118
1119 Node_Stack stack(0);
1120 Node_List clones;
1121 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1122 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1123
1124 Node* ctrl = phase->get_ctrl(lrb);
1125 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1126
1127 CallStaticJavaNode* unc = nullptr;
1128 Node* unc_ctrl = nullptr;
1129 Node* uncasted_val = val;
1130
1131 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1132 Node* u = lrb->fast_out(i);
1133 if (u->Opcode() == Op_CastPP &&
1134 u->in(0) != nullptr &&
1135 phase->is_dominator(u->in(0), ctrl)) {
1136 const Type* u_t = phase->igvn().type(u);
1137
1138 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1423
1424 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1425
1426 ctrl = orig_ctrl;
1427
1428 phase->igvn().replace_node(lrb, out_val);
1429
1430 follow_barrier_uses(out_val, ctrl, uses, phase);
1431
1432 for(uint next = 0; next < uses.size(); next++ ) {
1433 Node *n = uses.at(next);
1434 assert(phase->get_ctrl(n) == ctrl, "bad control");
1435 assert(n != raw_mem, "should leave input raw mem above the barrier");
1436 phase->set_ctrl(n, region);
1437 follow_barrier_uses(n, ctrl, uses, phase);
1438 }
1439 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1440 }
1441 // Done expanding load-reference-barriers.
1442 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1443
1444 for (int i = state->iu_barriers_count() - 1; i >= 0; i--) {
1445 Node* barrier = state->iu_barrier(i);
1446 Node* pre_val = barrier->in(1);
1447
1448 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
1449 ShouldNotReachHere();
1450 continue;
1451 }
1452
1453 Node* ctrl = phase->get_ctrl(barrier);
1454
1455 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1456 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
1457 ctrl = ctrl->in(0)->in(0);
1458 phase->set_ctrl(barrier, ctrl);
1459 } else if (ctrl->is_CallRuntime()) {
1460 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
1461 ctrl = ctrl->in(0);
1462 phase->set_ctrl(barrier, ctrl);
1463 }
1464
1465 Node* init_ctrl = ctrl;
1466 IdealLoopTree* loop = phase->get_loop(ctrl);
1467 Node* raw_mem = fixer.find_mem(ctrl, barrier);
1468 Node* init_raw_mem = raw_mem;
1469 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr);
1470 Node* heap_stable_ctrl = nullptr;
1471 Node* null_ctrl = nullptr;
1472 uint last = phase->C->unique();
1473
1474 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1475 Node* region = new RegionNode(PATH_LIMIT);
1476 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1477
1478 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1479 Node* region2 = new RegionNode(PATH_LIMIT2);
1480 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1481
1482 // Stable path.
1483 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
1484 region->init_req(_heap_stable, heap_stable_ctrl);
1485 phi->init_req(_heap_stable, raw_mem);
1486
1487 // Null path
1488 Node* reg2_ctrl = nullptr;
1489 test_null(ctrl, pre_val, null_ctrl, phase);
1490 if (null_ctrl != nullptr) {
1491 reg2_ctrl = null_ctrl->in(0);
1492 region2->init_req(_null_path, null_ctrl);
1493 phi2->init_req(_null_path, raw_mem);
1494 } else {
1495 region2->del_req(_null_path);
1496 phi2->del_req(_null_path);
1497 }
1498
1499 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1500 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1501 Node* thread = new ThreadLocalNode();
1502 phase->register_new_node(thread, ctrl);
1503 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
1504 phase->register_new_node(buffer_adr, ctrl);
1505 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
1506 phase->register_new_node(index_adr, ctrl);
1507
1508 BasicType index_bt = TypeX_X->basic_type();
1509 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
1510 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1511 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
1512 phase->register_new_node(index, ctrl);
1513 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
1514 phase->register_new_node(index_cmp, ctrl);
1515 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
1516 phase->register_new_node(index_test, ctrl);
1517 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1518 if (reg2_ctrl == nullptr) reg2_ctrl = queue_full_iff;
1519 phase->register_control(queue_full_iff, loop, ctrl);
1520 Node* not_full = new IfTrueNode(queue_full_iff);
1521 phase->register_control(not_full, loop, queue_full_iff);
1522 Node* full = new IfFalseNode(queue_full_iff);
1523 phase->register_control(full, loop, queue_full_iff);
1524
1525 ctrl = not_full;
1526
1527 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
1528 phase->register_new_node(next_index, ctrl);
1529
1530 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
1531 phase->register_new_node(buffer, ctrl);
1532 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
1533 phase->register_new_node(log_addr, ctrl);
1534 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
1535 phase->register_new_node(log_store, ctrl);
1536 // update the index
1537 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
1538 phase->register_new_node(index_update, ctrl);
1539
1540 // Fast-path case
1541 region2->init_req(_fast_path, ctrl);
1542 phi2->init_req(_fast_path, index_update);
1543
1544 ctrl = full;
1545
1546 Node* base = find_bottom_mem(ctrl, phase);
1547
1548 MergeMemNode* mm = MergeMemNode::make(base);
1549 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1550 phase->register_new_node(mm, ctrl);
1551
1552 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
1553 call->init_req(TypeFunc::Control, ctrl);
1554 call->init_req(TypeFunc::I_O, phase->C->top());
1555 call->init_req(TypeFunc::Memory, mm);
1556 call->init_req(TypeFunc::FramePtr, phase->C->top());
1557 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1558 call->init_req(TypeFunc::Parms, pre_val);
1559 call->init_req(TypeFunc::Parms+1, thread);
1560 phase->register_control(call, loop, ctrl);
1561
1562 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
1563 phase->register_control(ctrl_proj, loop, call);
1564 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
1565 phase->register_new_node(mem_proj, call);
1566
1567 // Slow-path case
1568 region2->init_req(_slow_path, ctrl_proj);
1569 phi2->init_req(_slow_path, mem_proj);
1570
1571 phase->register_control(region2, loop, reg2_ctrl);
1572 phase->register_new_node(phi2, region2);
1573
1574 region->init_req(_heap_unstable, region2);
1575 phi->init_req(_heap_unstable, phi2);
1576
1577 phase->register_control(region, loop, heap_stable_ctrl->in(0));
1578 phase->register_new_node(phi, region);
1579
1580 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
1581 for(uint next = 0; next < uses.size(); next++ ) {
1582 Node *n = uses.at(next);
1583 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1584 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1585 phase->set_ctrl(n, region);
1586 follow_barrier_uses(n, init_ctrl, uses, phase);
1587 }
1588 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1589
1590 phase->igvn().replace_node(barrier, pre_val);
1591 }
1592 assert(state->iu_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
1593
1594 }
1595
1596 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1597 if (visited.test_set(in->_idx)) {
1598 return nullptr;
1599 }
1600 switch (in->Opcode()) {
1601 case Op_Proj:
1602 return get_load_addr(phase, visited, in->in(0));
1603 case Op_CastPP:
1604 case Op_CheckCastPP:
1605 case Op_DecodeN:
1606 case Op_EncodeP:
1607 return get_load_addr(phase, visited, in->in(1));
1608 case Op_LoadN:
1609 case Op_LoadP:
1610 return in->in(MemNode::Address);
1611 case Op_CompareAndExchangeN:
1612 case Op_CompareAndExchangeP:
1613 case Op_GetAndSetN:
1626 if (t == nullptr && f != nullptr) return f;
1627 if (t != nullptr && t == f) return t;
1628 // Ambiguity.
1629 return phase->igvn().zerocon(T_OBJECT);
1630 }
1631 case Op_Phi: {
1632 Node* addr = nullptr;
1633 for (uint i = 1; i < in->req(); i++) {
1634 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1635 if (addr == nullptr) {
1636 addr = addr1;
1637 }
1638 if (addr != addr1) {
1639 return phase->igvn().zerocon(T_OBJECT);
1640 }
1641 }
1642 return addr;
1643 }
1644 case Op_ShenandoahLoadReferenceBarrier:
1645 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1646 case Op_ShenandoahIUBarrier:
1647 return get_load_addr(phase, visited, in->in(1));
1648 case Op_CallDynamicJava:
1649 case Op_CallLeaf:
1650 case Op_CallStaticJava:
1651 case Op_ConN:
1652 case Op_ConP:
1653 case Op_Parm:
1654 case Op_CreateEx:
1655 return phase->igvn().zerocon(T_OBJECT);
1656 default:
1657 #ifdef ASSERT
1658 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1659 #endif
1660 return phase->igvn().zerocon(T_OBJECT);
1661 }
1662
1663 }
1664
1665 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1666 IdealLoopTree *loop = phase->get_loop(iff);
1667 Node* loop_head = loop->_head;
1875 AutoNodeBudget node_budget(phase);
1876
1877 if (loop->policy_unswitching(phase)) {
1878 if (head->as_Loop()->is_strip_mined()) {
1879 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1880 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1881 }
1882 phase->do_unswitching(loop, old_new);
1883 } else {
1884 // Not proceeding with unswitching. Move load back in
1885 // the loop.
1886 phase->igvn().replace_input_of(iff, 1, bol);
1887 }
1888 }
1889 }
1890 }
1891 }
1892 }
1893 }
1894
1895 ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(nullptr, val) {
1896 ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
1897 }
1898
1899 const Type* ShenandoahIUBarrierNode::bottom_type() const {
1900 if (in(1) == nullptr || in(1)->is_top()) {
1901 return Type::TOP;
1902 }
1903 const Type* t = in(1)->bottom_type();
1904 if (t == TypePtr::NULL_PTR) {
1905 return t;
1906 }
1907 return t->is_oopptr();
1908 }
1909
1910 const Type* ShenandoahIUBarrierNode::Value(PhaseGVN* phase) const {
1911 if (in(1) == nullptr) {
1912 return Type::TOP;
1913 }
1914 const Type* t = phase->type(in(1));
1915 if (t == Type::TOP) {
1916 return Type::TOP;
1917 }
1918 if (t == TypePtr::NULL_PTR) {
1919 return t;
1920 }
1921 return t->is_oopptr();
1922 }
1923
1924 int ShenandoahIUBarrierNode::needed(Node* n) {
1925 if (n == nullptr ||
1926 n->is_Allocate() ||
1927 n->Opcode() == Op_ShenandoahIUBarrier ||
1928 n->bottom_type() == TypePtr::NULL_PTR ||
1929 (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr)) {
1930 return NotNeeded;
1931 }
1932 if (n->is_Phi() ||
1933 n->is_CMove()) {
1934 return MaybeNeeded;
1935 }
1936 return Needed;
1937 }
1938
1939 Node* ShenandoahIUBarrierNode::next(Node* n) {
1940 for (;;) {
1941 if (n == nullptr) {
1942 return n;
1943 } else if (n->bottom_type() == TypePtr::NULL_PTR) {
1944 return n;
1945 } else if (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr) {
1946 return n;
1947 } else if (n->is_ConstraintCast() ||
1948 n->Opcode() == Op_DecodeN ||
1949 n->Opcode() == Op_EncodeP) {
1950 n = n->in(1);
1951 } else if (n->is_Proj()) {
1952 n = n->in(0);
1953 } else {
1954 return n;
1955 }
1956 }
1957 ShouldNotReachHere();
1958 return nullptr;
1959 }
1960
1961 Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) {
1962 PhaseIterGVN* igvn = phase->is_IterGVN();
1963
1964 Node* n = next(in(1));
1965
1966 int cont = needed(n);
1967
1968 if (cont == NotNeeded) {
1969 return in(1);
1970 } else if (cont == MaybeNeeded) {
1971 if (igvn == nullptr) {
1972 phase->record_for_igvn(this);
1973 return this;
1974 } else {
1975 ResourceMark rm;
1976 Unique_Node_List wq;
1977 uint wq_i = 0;
1978
1979 for (;;) {
1980 if (n->is_Phi()) {
1981 for (uint i = 1; i < n->req(); i++) {
1982 Node* m = n->in(i);
1983 if (m != nullptr) {
1984 wq.push(m);
1985 }
1986 }
1987 } else {
1988 assert(n->is_CMove(), "nothing else here");
1989 Node* m = n->in(CMoveNode::IfFalse);
1990 wq.push(m);
1991 m = n->in(CMoveNode::IfTrue);
1992 wq.push(m);
1993 }
1994 Node* orig_n = nullptr;
1995 do {
1996 if (wq_i >= wq.size()) {
1997 return in(1);
1998 }
1999 n = wq.at(wq_i);
2000 wq_i++;
2001 orig_n = n;
2002 n = next(n);
2003 cont = needed(n);
2004 if (cont == Needed) {
2005 return this;
2006 }
2007 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
2008 }
2009 }
2010 }
2011
2012 return this;
2013 }
2014
2015 #ifdef ASSERT
2016 static bool has_never_branch(Node* root) {
2017 for (uint i = 1; i < root->req(); i++) {
2018 Node* in = root->in(i);
2019 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
2020 return true;
2021 }
2022 }
2023 return false;
2024 }
2025 #endif
2026
2027 void MemoryGraphFixer::collect_memory_nodes() {
2028 Node_Stack stack(0);
2029 VectorSet visited;
2030 Node_List regions;
2031
2032 // Walk the raw memory graph and create a mapping from CFG node to
2033 // memory node. Exclude phis for now.
2034 stack.push(_phase->C->root(), 1);
2151 int iteration = 0;
2152 Node_List dead_phis;
2153 while (progress) {
2154 progress = false;
2155 iteration++;
2156 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2157 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2158
2159 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2160 Node* c = rpo_list.at(i);
2161
2162 Node* prev_mem = _memory_nodes[c->_idx];
2163 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2164 Node* prev_region = regions[c->_idx];
2165 Node* unique = nullptr;
2166 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2167 Node* m = _memory_nodes[c->in(j)->_idx];
2168 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2169 if (m != nullptr) {
2170 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2171 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2172 // continue
2173 } else if (unique == nullptr) {
2174 unique = m;
2175 } else if (m == unique) {
2176 // continue
2177 } else {
2178 unique = NodeSentinel;
2179 }
2180 }
2181 }
2182 assert(unique != nullptr, "empty phi???");
2183 if (unique != NodeSentinel) {
2184 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
2185 dead_phis.push(prev_region);
2186 }
2187 regions.map(c->_idx, unique);
2188 } else {
2189 Node* phi = nullptr;
2190 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
2191 phi = prev_region;
2978 case Op_CheckCastPP:
2979 case Op_CastPP:
2980 return needs_barrier_impl(phase, n->in(1), visited);
2981 case Op_Proj:
2982 return needs_barrier_impl(phase, n->in(0), visited);
2983 case Op_ShenandoahLoadReferenceBarrier:
2984 // tty->print_cr("optimize barrier on barrier");
2985 return false;
2986 case Op_Parm:
2987 // tty->print_cr("optimize barrier on input arg");
2988 return false;
2989 case Op_DecodeN:
2990 case Op_EncodeP:
2991 return needs_barrier_impl(phase, n->in(1), visited);
2992 case Op_LoadN:
2993 return true;
2994 case Op_CMoveN:
2995 case Op_CMoveP:
2996 return needs_barrier_impl(phase, n->in(2), visited) ||
2997 needs_barrier_impl(phase, n->in(3), visited);
2998 case Op_ShenandoahIUBarrier:
2999 return needs_barrier_impl(phase, n->in(1), visited);
3000 case Op_CreateEx:
3001 return false;
3002 default:
3003 break;
3004 }
3005 #ifdef ASSERT
3006 tty->print("need barrier on?: ");
3007 tty->print_cr("ins:");
3008 n->dump(2);
3009 tty->print_cr("outs:");
3010 n->dump(-2);
3011 ShouldNotReachHere();
3012 #endif
3013 return true;
3014 }
|
29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahForwarding.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/block.hpp"
39 #include "opto/callnode.hpp"
40 #include "opto/castnode.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46
47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
49 if (state->load_reference_barriers_count() > 0) {
50 assert(C->post_loop_opts_phase(), "no loop opts allowed");
51 C->reset_post_loop_opts_phase(); // ... but we know what we are doing
52 C->clear_major_progress();
53 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
54 if (C->failing()) return false;
55
56 C->set_major_progress();
57 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
58 return false;
59 }
60 C->clear_major_progress();
61 C->process_for_post_loop_opts_igvn(igvn);
62 if (C->failing()) return false;
63
64 C->set_post_loop_opts_phase(); // now for real!
65 }
66 return true;
67 }
68
69 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
168 continue;
169 } else if (in->is_Con()) {
170 if (trace) {
171 tty->print("Found constant");
172 in->dump();
173 }
174 } else if (in->Opcode() == Op_Parm) {
175 if (trace) {
176 tty->print("Found argument");
177 }
178 } else if (in->Opcode() == Op_CreateEx) {
179 if (trace) {
180 tty->print("Found create-exception");
181 }
182 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
183 if (trace) {
184 tty->print("Found raw LoadP (OSR argument?)");
185 }
186 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
187 if (t == ShenandoahOopStore) {
188 return false;
189 }
190 barriers_used.push(in);
191 if (trace) {tty->print("Found barrier"); in->dump();}
192 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
193 if (trace) {
194 tty->print("Found alloc");
195 in->in(0)->dump();
196 }
197 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
198 if (trace) {
199 tty->print("Found Java call");
200 }
201 } else if (in->is_Phi()) {
202 if (!visited.test_set(in->_idx)) {
203 if (trace) {tty->print("Pushed phi:"); in->dump();}
204 phis.push(in, 2);
205 in = in->in(1);
206 continue;
207 }
208 if (trace) {tty->print("Already seen phi:"); in->dump();}
209 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
210 if (!visited.test_set(in->_idx)) {
211 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
290 if (trace) {tty->print("Verifying"); n->dump();}
291 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
292 Node* adr = n->in(MemNode::Address);
293 bool verify = true;
294
295 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
296 adr = adr->in(AddPNode::Address);
297 if (adr->is_AddP()) {
298 assert(adr->in(AddPNode::Base)->is_top(), "");
299 adr = adr->in(AddPNode::Address);
300 if (adr->Opcode() == Op_LoadP &&
301 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
302 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
303 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
304 if (trace) {tty->print_cr("SATB prebarrier");}
305 verify = false;
306 }
307 }
308 }
309
310 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
311 report_verify_failure("Shenandoah verification: Store should have barriers", n);
312 }
313 }
314 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
315 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
316 }
317 } else if (n->Opcode() == Op_CmpP) {
318 const bool trace = false;
319
320 Node* in1 = n->in(1);
321 Node* in2 = n->in(2);
322 if (in1->bottom_type()->isa_oopptr()) {
323 if (trace) {tty->print("Verifying"); n->dump();}
324
325 bool mark_inputs = false;
326 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
327 (in1->is_Con() || in2->is_Con())) {
328 if (trace) {tty->print_cr("Comparison against a constant");}
329 mark_inputs = true;
330 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
332 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
333 mark_inputs = true;
334 } else {
335 assert(in2->bottom_type()->isa_oopptr(), "");
336
337 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
338 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
339 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
340 }
341 }
342 if (verify_no_useless_barrier &&
343 mark_inputs &&
344 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
345 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
346 phis.clear();
347 visited.reset();
348 }
349 }
350 } else if (n->is_LoadStore()) {
351 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
352 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
353 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
354 }
355
356 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
357 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
358 }
359 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
360 CallNode* call = n->as_Call();
361
362 static struct {
363 const char* name;
364 struct {
365 int pos;
366 verify_type t;
367 } args[6];
368 } calls[] = {
369 "array_partition_stub",
370 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone },
371 { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
372 "arraysort_stub",
373 { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone },
374 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
375 "aescrypt_encryptBlock",
376 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
377 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
378 "aescrypt_decryptBlock",
379 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
380 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
381 "multiplyToLen",
382 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
383 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
384 "squareToLen",
385 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
386 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
387 "montgomery_multiply",
388 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
389 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
390 "montgomery_square",
391 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
392 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
393 "mulAdd",
394 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
397 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
398 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
399 "updateBytesCRC32",
400 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
401 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
402 "updateBytesAdler32",
403 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
404 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
405 "updateBytesCRC32C",
406 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
407 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
408 "counterMode_AESCrypt",
409 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
410 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
411 "cipherBlockChaining_encryptAESCrypt",
412 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
413 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
414 "cipherBlockChaining_decryptAESCrypt",
415 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
416 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
417 "shenandoah_clone",
418 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
419 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
420 "ghash_processBlocks",
421 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
422 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
423 "sha1_implCompress",
424 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
425 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
426 "sha256_implCompress",
427 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
428 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
429 "sha512_implCompress",
430 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
432 "sha1_implCompressMB",
433 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
434 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
435 "sha256_implCompressMB",
436 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
490 }
491 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
492 if (call->in(j)->bottom_type()->make_ptr() &&
493 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
494 uint k = 0;
495 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
496 if (k == args_len) {
497 fatal("arg %d for call %s not covered", j, call->_name);
498 }
499 }
500 }
501 } else {
502 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
503 if (call->in(j)->bottom_type()->make_ptr() &&
504 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
505 fatal("%s not covered", call->_name);
506 }
507 }
508 }
509 }
510 } else if (n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
511 // skip
512 } else if (n->is_AddP()
513 || n->is_Phi()
514 || n->is_ConstraintCast()
515 || n->Opcode() == Op_Return
516 || n->Opcode() == Op_CMoveP
517 || n->Opcode() == Op_CMoveN
518 || n->Opcode() == Op_Rethrow
519 || n->is_MemBar()
520 || n->Opcode() == Op_Conv2B
521 || n->Opcode() == Op_SafePoint
522 || n->is_CallJava()
523 || n->Opcode() == Op_Unlock
524 || n->Opcode() == Op_EncodeP
525 || n->Opcode() == Op_DecodeN) {
526 // nothing to do
527 } else {
528 static struct {
529 int opcode;
530 struct {
1072 assert(region != nullptr, "");
1073 Node* phi = new PhiNode(region, n->bottom_type());
1074 for (uint j = 1; j < region->req(); j++) {
1075 Node* in = region->in(j);
1076 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1077 phi->init_req(j, n);
1078 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1079 phi->init_req(j, n_clone);
1080 } else {
1081 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1082 }
1083 }
1084 phase->register_new_node(phi, region);
1085 return phi;
1086 }
1087
1088 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1089 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1090
1091 Unique_Node_List uses;
1092 Node_Stack stack(0);
1093 Node_List clones;
1094 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1095 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1096
1097 Node* ctrl = phase->get_ctrl(lrb);
1098 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1099
1100 CallStaticJavaNode* unc = nullptr;
1101 Node* unc_ctrl = nullptr;
1102 Node* uncasted_val = val;
1103
1104 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1105 Node* u = lrb->fast_out(i);
1106 if (u->Opcode() == Op_CastPP &&
1107 u->in(0) != nullptr &&
1108 phase->is_dominator(u->in(0), ctrl)) {
1109 const Type* u_t = phase->igvn().type(u);
1110
1111 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1396
1397 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1398
1399 ctrl = orig_ctrl;
1400
1401 phase->igvn().replace_node(lrb, out_val);
1402
1403 follow_barrier_uses(out_val, ctrl, uses, phase);
1404
1405 for(uint next = 0; next < uses.size(); next++ ) {
1406 Node *n = uses.at(next);
1407 assert(phase->get_ctrl(n) == ctrl, "bad control");
1408 assert(n != raw_mem, "should leave input raw mem above the barrier");
1409 phase->set_ctrl(n, region);
1410 follow_barrier_uses(n, ctrl, uses, phase);
1411 }
1412 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
1413 }
1414 // Done expanding load-reference-barriers.
1415 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1416 }
1417
1418 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1419 if (visited.test_set(in->_idx)) {
1420 return nullptr;
1421 }
1422 switch (in->Opcode()) {
1423 case Op_Proj:
1424 return get_load_addr(phase, visited, in->in(0));
1425 case Op_CastPP:
1426 case Op_CheckCastPP:
1427 case Op_DecodeN:
1428 case Op_EncodeP:
1429 return get_load_addr(phase, visited, in->in(1));
1430 case Op_LoadN:
1431 case Op_LoadP:
1432 return in->in(MemNode::Address);
1433 case Op_CompareAndExchangeN:
1434 case Op_CompareAndExchangeP:
1435 case Op_GetAndSetN:
1448 if (t == nullptr && f != nullptr) return f;
1449 if (t != nullptr && t == f) return t;
1450 // Ambiguity.
1451 return phase->igvn().zerocon(T_OBJECT);
1452 }
1453 case Op_Phi: {
1454 Node* addr = nullptr;
1455 for (uint i = 1; i < in->req(); i++) {
1456 Node* addr1 = get_load_addr(phase, visited, in->in(i));
1457 if (addr == nullptr) {
1458 addr = addr1;
1459 }
1460 if (addr != addr1) {
1461 return phase->igvn().zerocon(T_OBJECT);
1462 }
1463 }
1464 return addr;
1465 }
1466 case Op_ShenandoahLoadReferenceBarrier:
1467 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1468 case Op_CallDynamicJava:
1469 case Op_CallLeaf:
1470 case Op_CallStaticJava:
1471 case Op_ConN:
1472 case Op_ConP:
1473 case Op_Parm:
1474 case Op_CreateEx:
1475 return phase->igvn().zerocon(T_OBJECT);
1476 default:
1477 #ifdef ASSERT
1478 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1479 #endif
1480 return phase->igvn().zerocon(T_OBJECT);
1481 }
1482
1483 }
1484
1485 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1486 IdealLoopTree *loop = phase->get_loop(iff);
1487 Node* loop_head = loop->_head;
1695 AutoNodeBudget node_budget(phase);
1696
1697 if (loop->policy_unswitching(phase)) {
1698 if (head->as_Loop()->is_strip_mined()) {
1699 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1700 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1701 }
1702 phase->do_unswitching(loop, old_new);
1703 } else {
1704 // Not proceeding with unswitching. Move load back in
1705 // the loop.
1706 phase->igvn().replace_input_of(iff, 1, bol);
1707 }
1708 }
1709 }
1710 }
1711 }
1712 }
1713 }
1714
1715 #ifdef ASSERT
1716 static bool has_never_branch(Node* root) {
1717 for (uint i = 1; i < root->req(); i++) {
1718 Node* in = root->in(i);
1719 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
1720 return true;
1721 }
1722 }
1723 return false;
1724 }
1725 #endif
1726
1727 void MemoryGraphFixer::collect_memory_nodes() {
1728 Node_Stack stack(0);
1729 VectorSet visited;
1730 Node_List regions;
1731
1732 // Walk the raw memory graph and create a mapping from CFG node to
1733 // memory node. Exclude phis for now.
1734 stack.push(_phase->C->root(), 1);
1851 int iteration = 0;
1852 Node_List dead_phis;
1853 while (progress) {
1854 progress = false;
1855 iteration++;
1856 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1857 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
1858
1859 for (int i = rpo_list.size() - 1; i >= 0; i--) {
1860 Node* c = rpo_list.at(i);
1861
1862 Node* prev_mem = _memory_nodes[c->_idx];
1863 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
1864 Node* prev_region = regions[c->_idx];
1865 Node* unique = nullptr;
1866 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
1867 Node* m = _memory_nodes[c->in(j)->_idx];
1868 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
1869 if (m != nullptr) {
1870 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
1871 assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
1872 // continue
1873 } else if (unique == nullptr) {
1874 unique = m;
1875 } else if (m == unique) {
1876 // continue
1877 } else {
1878 unique = NodeSentinel;
1879 }
1880 }
1881 }
1882 assert(unique != nullptr, "empty phi???");
1883 if (unique != NodeSentinel) {
1884 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
1885 dead_phis.push(prev_region);
1886 }
1887 regions.map(c->_idx, unique);
1888 } else {
1889 Node* phi = nullptr;
1890 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
1891 phi = prev_region;
2678 case Op_CheckCastPP:
2679 case Op_CastPP:
2680 return needs_barrier_impl(phase, n->in(1), visited);
2681 case Op_Proj:
2682 return needs_barrier_impl(phase, n->in(0), visited);
2683 case Op_ShenandoahLoadReferenceBarrier:
2684 // tty->print_cr("optimize barrier on barrier");
2685 return false;
2686 case Op_Parm:
2687 // tty->print_cr("optimize barrier on input arg");
2688 return false;
2689 case Op_DecodeN:
2690 case Op_EncodeP:
2691 return needs_barrier_impl(phase, n->in(1), visited);
2692 case Op_LoadN:
2693 return true;
2694 case Op_CMoveN:
2695 case Op_CMoveP:
2696 return needs_barrier_impl(phase, n->in(2), visited) ||
2697 needs_barrier_impl(phase, n->in(3), visited);
2698 case Op_CreateEx:
2699 return false;
2700 default:
2701 break;
2702 }
2703 #ifdef ASSERT
2704 tty->print("need barrier on?: ");
2705 tty->print_cr("ins:");
2706 n->dump(2);
2707 tty->print_cr("outs:");
2708 n->dump(-2);
2709 ShouldNotReachHere();
2710 #endif
2711 return true;
2712 }
|