368 }
369 } else if (n->is_LoadStore()) {
370 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
371 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
372 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
373 }
374
375 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
376 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
377 }
378 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
379 CallNode* call = n->as_Call();
380
381 static struct {
382 const char* name;
383 struct {
384 int pos;
385 verify_type t;
386 } args[6];
387 } calls[] = {
388 "aescrypt_encryptBlock",
389 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
390 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
391 "aescrypt_decryptBlock",
392 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
394 "multiplyToLen",
395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
397 "squareToLen",
398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
400 "montgomery_multiply",
401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
402 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
403 "montgomery_square",
404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
405 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
406 "mulAdd",
407 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
410 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
412 "updateBytesCRC32",
413 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
414 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
415 "updateBytesAdler32",
416 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
418 "updateBytesCRC32C",
419 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
421 "counterMode_AESCrypt",
422 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
423 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
424 "cipherBlockChaining_encryptAESCrypt",
425 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
426 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
427 "cipherBlockChaining_decryptAESCrypt",
428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
429 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
430 "shenandoah_clone_barrier",
431 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
432 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
433 "ghash_processBlocks",
434 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
435 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
436 "sha1_implCompress",
437 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
439 "sha256_implCompress",
440 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
442 "sha512_implCompress",
443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
445 "sha1_implCompressMB",
446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
448 "sha256_implCompressMB",
449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
2151 int iteration = 0;
2152 Node_List dead_phis;
2153 while (progress) {
2154 progress = false;
2155 iteration++;
2156 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2157 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2158
2159 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2160 Node* c = rpo_list.at(i);
2161
2162 Node* prev_mem = _memory_nodes[c->_idx];
2163 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2164 Node* prev_region = regions[c->_idx];
2165 Node* unique = nullptr;
2166 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2167 Node* m = _memory_nodes[c->in(j)->_idx];
2168 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2169 if (m != nullptr) {
2170 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2171 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2172 // continue
2173 } else if (unique == nullptr) {
2174 unique = m;
2175 } else if (m == unique) {
2176 // continue
2177 } else {
2178 unique = NodeSentinel;
2179 }
2180 }
2181 }
2182 assert(unique != nullptr, "empty phi???");
2183 if (unique != NodeSentinel) {
2184 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
2185 dead_phis.push(prev_region);
2186 }
2187 regions.map(c->_idx, unique);
2188 } else {
2189 Node* phi = nullptr;
2190 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
2191 phi = prev_region;
|
368 }
369 } else if (n->is_LoadStore()) {
370 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
371 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
372 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
373 }
374
375 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
376 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
377 }
378 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
379 CallNode* call = n->as_Call();
380
381 static struct {
382 const char* name;
383 struct {
384 int pos;
385 verify_type t;
386 } args[6];
387 } calls[] = {
388 "array_partition_stub",
389 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone },
390 { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
391 "arraysort_stub",
392 { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone },
393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
394 "aescrypt_encryptBlock",
395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
397 "aescrypt_decryptBlock",
398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
400 "multiplyToLen",
401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
402 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
403 "squareToLen",
404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
405 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
406 "montgomery_multiply",
407 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
408 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
409 "montgomery_square",
410 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
412 "mulAdd",
413 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
416 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
418 "updateBytesCRC32",
419 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
421 "updateBytesAdler32",
422 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
423 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
424 "updateBytesCRC32C",
425 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
426 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
427 "counterMode_AESCrypt",
428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
429 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
430 "cipherBlockChaining_encryptAESCrypt",
431 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
432 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
433 "cipherBlockChaining_decryptAESCrypt",
434 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
435 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
436 "shenandoah_clone",
437 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
439 "ghash_processBlocks",
440 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
442 "sha1_implCompress",
443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
445 "sha256_implCompress",
446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
448 "sha512_implCompress",
449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
451 "sha1_implCompressMB",
452 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
453 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
454 "sha256_implCompressMB",
455 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
456 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
2157 int iteration = 0;
2158 Node_List dead_phis;
2159 while (progress) {
2160 progress = false;
2161 iteration++;
2162 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2163 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2164
2165 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2166 Node* c = rpo_list.at(i);
2167
2168 Node* prev_mem = _memory_nodes[c->_idx];
2169 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2170 Node* prev_region = regions[c->_idx];
2171 Node* unique = nullptr;
2172 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2173 Node* m = _memory_nodes[c->in(j)->_idx];
2174 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2175 if (m != nullptr) {
2176 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2177 assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2178 // continue
2179 } else if (unique == nullptr) {
2180 unique = m;
2181 } else if (m == unique) {
2182 // continue
2183 } else {
2184 unique = NodeSentinel;
2185 }
2186 }
2187 }
2188 assert(unique != nullptr, "empty phi???");
2189 if (unique != NodeSentinel) {
2190 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) {
2191 dead_phis.push(prev_region);
2192 }
2193 regions.map(c->_idx, unique);
2194 } else {
2195 Node* phi = nullptr;
2196 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
2197 phi = prev_region;
|