1 /* 2 * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 28 #include "classfile/javaClasses.hpp" 29 #include "gc/shenandoah/c2/shenandoahSupport.hpp" 30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 32 #include "gc/shenandoah/shenandoahForwarding.hpp" 33 #include "gc/shenandoah/shenandoahHeap.hpp" 34 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 35 #include "gc/shenandoah/shenandoahRuntime.hpp" 36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 37 #include "opto/arraycopynode.hpp" 38 #include "opto/block.hpp" 39 #include "opto/callnode.hpp" 40 #include "opto/castnode.hpp" 41 #include "opto/movenode.hpp" 42 #include "opto/phaseX.hpp" 43 #include "opto/rootnode.hpp" 44 #include "opto/runtime.hpp" 45 #include "opto/subnode.hpp" 46 47 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { 48 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 49 if (state->load_reference_barriers_count() > 0) { 50 assert(C->post_loop_opts_phase(), "no loop opts allowed"); 51 C->reset_post_loop_opts_phase(); // ... but we know what we are doing 52 C->clear_major_progress(); 53 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand); 54 if (C->failing()) return false; 55 C->process_for_post_loop_opts_igvn(igvn); 56 if (C->failing()) return false; 57 58 C->set_post_loop_opts_phase(); // now for real! 59 } 60 return true; 61 } 62 63 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) { 64 if (!UseShenandoahGC) { 65 return false; 66 } 67 assert(iff->is_If(), "bad input"); 68 if (iff->Opcode() != Op_If) { 69 return false; 70 } 71 Node* bol = iff->in(1); 72 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 73 return false; 74 } 75 Node* cmp = bol->in(1); 76 if (cmp->Opcode() != Op_CmpI) { 77 return false; 78 } 79 Node* in1 = cmp->in(1); 80 Node* in2 = cmp->in(2); 81 if (in2->find_int_con(-1) != 0) { 82 return false; 83 } 84 if (in1->Opcode() != Op_AndI) { 85 return false; 86 } 87 in2 = in1->in(2); 88 if (in2->find_int_con(-1) != mask) { 89 return false; 90 } 91 in1 = in1->in(1); 92 93 return is_gc_state_load(in1); 94 } 95 96 bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { 97 return is_gc_state_test(iff, ShenandoahHeap::HAS_FORWARDED); 98 } 99 100 bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { 101 if (!UseShenandoahGC) { 102 return false; 103 } 104 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) { 105 return false; 106 } 107 Node* addp = n->in(MemNode::Address); 108 if (!addp->is_AddP()) { 109 return false; 110 } 111 Node* base = addp->in(AddPNode::Address); 112 Node* off = addp->in(AddPNode::Offset); 113 if (base->Opcode() != Op_ThreadLocal) { 114 return false; 115 } 116 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 117 return false; 118 } 119 return true; 120 } 121 122 bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { 123 assert(phase->is_dominator(stop, start), "bad inputs"); 124 ResourceMark rm; 125 Unique_Node_List wq; 126 wq.push(start); 127 for (uint next = 0; next < wq.size(); next++) { 128 Node *m = wq.at(next); 129 if (m == stop) { 130 continue; 131 } 132 if (m->is_SafePoint() && !m->is_CallLeaf()) { 133 return true; 134 } 135 if (m->is_Region()) { 136 for (uint i = 1; i < m->req(); i++) { 137 wq.push(m->in(i)); 138 } 139 } else { 140 wq.push(m->in(0)); 141 } 142 } 143 return false; 144 } 145 146 #ifdef ASSERT 147 bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 148 assert(phis.size() == 0, ""); 149 150 while (true) { 151 if (in->bottom_type() == TypePtr::NULL_PTR) { 152 if (trace) {tty->print_cr("null");} 153 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) { 154 if (trace) {tty->print_cr("Non oop");} 155 } else { 156 if (in->is_ConstraintCast()) { 157 in = in->in(1); 158 continue; 159 } else if (in->is_AddP()) { 160 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 161 in = in->in(AddPNode::Address); 162 continue; 163 } else if (in->is_Con()) { 164 if (trace) { 165 tty->print("Found constant"); 166 in->dump(); 167 } 168 } else if (in->Opcode() == Op_Parm) { 169 if (trace) { 170 tty->print("Found argument"); 171 } 172 } else if (in->Opcode() == Op_CreateEx) { 173 if (trace) { 174 tty->print("Found create-exception"); 175 } 176 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { 177 if (trace) { 178 tty->print("Found raw LoadP (OSR argument?)"); 179 } 180 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 181 if (t == ShenandoahOopStore) { 182 return false; 183 } 184 barriers_used.push(in); 185 if (trace) {tty->print("Found barrier"); in->dump();} 186 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 187 if (trace) { 188 tty->print("Found alloc"); 189 in->in(0)->dump(); 190 } 191 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { 192 if (trace) { 193 tty->print("Found Java call"); 194 } 195 } else if (in->is_Phi()) { 196 if (!visited.test_set(in->_idx)) { 197 if (trace) {tty->print("Pushed phi:"); in->dump();} 198 phis.push(in, 2); 199 in = in->in(1); 200 continue; 201 } 202 if (trace) {tty->print("Already seen phi:"); in->dump();} 203 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 204 if (!visited.test_set(in->_idx)) { 205 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 206 phis.push(in, CMoveNode::IfTrue); 207 in = in->in(CMoveNode::IfFalse); 208 continue; 209 } 210 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 211 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 212 in = in->in(1); 213 continue; 214 } else { 215 return false; 216 } 217 } 218 bool cont = false; 219 while (phis.is_nonempty()) { 220 uint idx = phis.index(); 221 Node* phi = phis.node(); 222 if (idx >= phi->req()) { 223 if (trace) {tty->print("Popped phi:"); phi->dump();} 224 phis.pop(); 225 continue; 226 } 227 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 228 in = phi->in(idx); 229 phis.set_index(idx+1); 230 cont = true; 231 break; 232 } 233 if (!cont) { 234 break; 235 } 236 } 237 return true; 238 } 239 240 void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { 241 if (n1 != nullptr) { 242 n1->dump(+10); 243 } 244 if (n2 != nullptr) { 245 n2->dump(+10); 246 } 247 fatal("%s", msg); 248 } 249 250 void ShenandoahBarrierC2Support::verify(RootNode* root) { 251 ResourceMark rm; 252 Unique_Node_List wq; 253 GrowableArray<Node*> barriers; 254 Unique_Node_List barriers_used; 255 Node_Stack phis(0); 256 VectorSet visited; 257 const bool trace = false; 258 const bool verify_no_useless_barrier = false; 259 260 wq.push(root); 261 for (uint next = 0; next < wq.size(); next++) { 262 Node *n = wq.at(next); 263 if (n->is_Load()) { 264 const bool trace = false; 265 if (trace) {tty->print("Verifying"); n->dump();} 266 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 267 if (trace) {tty->print_cr("Load range/klass");} 268 } else { 269 const TypePtr* adr_type = n->as_Load()->adr_type(); 270 271 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 272 if (trace) {tty->print_cr("Mark load");} 273 } else if (adr_type->isa_instptr() && 274 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 275 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset()) { 276 if (trace) {tty->print_cr("Reference.get()");} 277 } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 278 report_verify_failure("Shenandoah verification: Load should have barriers", n); 279 } 280 } 281 } else if (n->is_Store()) { 282 const bool trace = false; 283 284 if (trace) {tty->print("Verifying"); n->dump();} 285 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 286 Node* adr = n->in(MemNode::Address); 287 bool verify = true; 288 289 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 290 adr = adr->in(AddPNode::Address); 291 if (adr->is_AddP()) { 292 assert(adr->in(AddPNode::Base)->is_top(), ""); 293 adr = adr->in(AddPNode::Address); 294 if (adr->Opcode() == Op_LoadP && 295 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 296 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 297 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 298 if (trace) {tty->print_cr("SATB prebarrier");} 299 verify = false; 300 } 301 } 302 } 303 304 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 305 report_verify_failure("Shenandoah verification: Store should have barriers", n); 306 } 307 } 308 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 309 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 310 } 311 } else if (n->Opcode() == Op_CmpP) { 312 const bool trace = false; 313 314 Node* in1 = n->in(1); 315 Node* in2 = n->in(2); 316 if (in1->bottom_type()->isa_oopptr()) { 317 if (trace) {tty->print("Verifying"); n->dump();} 318 319 bool mark_inputs = false; 320 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 321 (in1->is_Con() || in2->is_Con())) { 322 if (trace) {tty->print_cr("Comparison against a constant");} 323 mark_inputs = true; 324 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 325 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 326 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 327 mark_inputs = true; 328 } else { 329 assert(in2->bottom_type()->isa_oopptr(), ""); 330 331 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 332 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 333 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 334 } 335 } 336 if (verify_no_useless_barrier && 337 mark_inputs && 338 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 339 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 340 phis.clear(); 341 visited.reset(); 342 } 343 } 344 } else if (n->is_LoadStore()) { 345 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && 346 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 347 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 348 } 349 350 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 351 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 352 } 353 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 354 CallNode* call = n->as_Call(); 355 356 static struct { 357 const char* name; 358 struct { 359 int pos; 360 verify_type t; 361 } args[6]; 362 } calls[] = { 363 "array_partition_stub", 364 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone }, 365 { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } }, 366 "arraysort_stub", 367 { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone }, 368 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 369 "aescrypt_encryptBlock", 370 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 371 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 372 "aescrypt_decryptBlock", 373 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 374 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 375 "multiplyToLen", 376 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 377 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 378 "squareToLen", 379 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 380 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 381 "montgomery_multiply", 382 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 383 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 384 "montgomery_square", 385 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 386 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 387 "mulAdd", 388 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 389 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 390 "vectorizedMismatch", 391 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 392 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 393 "updateBytesCRC32", 394 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 395 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 396 "updateBytesAdler32", 397 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 398 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 399 "updateBytesCRC32C", 400 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 401 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 402 "counterMode_AESCrypt", 403 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 404 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 405 "cipherBlockChaining_encryptAESCrypt", 406 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 407 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 408 "cipherBlockChaining_decryptAESCrypt", 409 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 410 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 411 "shenandoah_clone", 412 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 413 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 414 "ghash_processBlocks", 415 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 416 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 417 "sha1_implCompress", 418 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 419 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 420 "sha256_implCompress", 421 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 422 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 423 "sha512_implCompress", 424 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 425 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 426 "sha1_implCompressMB", 427 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 428 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 429 "sha256_implCompressMB", 430 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 432 "sha512_implCompressMB", 433 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 434 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 435 "encodeBlock", 436 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 438 "decodeBlock", 439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 441 }; 442 443 if (call->is_call_to_arraycopystub()) { 444 Node* dest = nullptr; 445 const TypeTuple* args = n->as_Call()->_tf->domain(); 446 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 447 if (args->field_at(i)->isa_ptr()) { 448 j++; 449 if (j == 2) { 450 dest = n->in(i); 451 break; 452 } 453 } 454 } 455 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 456 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 457 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 458 } 459 } else if (strlen(call->_name) > 5 && 460 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 461 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 462 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 463 } 464 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { 465 // skip 466 } else { 467 const int calls_len = sizeof(calls) / sizeof(calls[0]); 468 int i = 0; 469 for (; i < calls_len; i++) { 470 if (!strcmp(calls[i].name, call->_name)) { 471 break; 472 } 473 } 474 if (i != calls_len) { 475 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 476 for (uint j = 0; j < args_len; j++) { 477 int pos = calls[i].args[j].pos; 478 if (pos == -1) { 479 break; 480 } 481 if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 482 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 483 } 484 } 485 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 486 if (call->in(j)->bottom_type()->make_ptr() && 487 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 488 uint k = 0; 489 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 490 if (k == args_len) { 491 fatal("arg %d for call %s not covered", j, call->_name); 492 } 493 } 494 } 495 } else { 496 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 497 if (call->in(j)->bottom_type()->make_ptr() && 498 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 499 fatal("%s not covered", call->_name); 500 } 501 } 502 } 503 } 504 } else if (n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 505 // skip 506 } else if (n->is_AddP() 507 || n->is_Phi() 508 || n->is_ConstraintCast() 509 || n->Opcode() == Op_Return 510 || n->Opcode() == Op_CMoveP 511 || n->Opcode() == Op_CMoveN 512 || n->Opcode() == Op_Rethrow 513 || n->is_MemBar() 514 || n->Opcode() == Op_Conv2B 515 || n->Opcode() == Op_SafePoint 516 || n->is_CallJava() 517 || n->Opcode() == Op_Unlock 518 || n->Opcode() == Op_EncodeP 519 || n->Opcode() == Op_DecodeN) { 520 // nothing to do 521 } else { 522 static struct { 523 int opcode; 524 struct { 525 int pos; 526 verify_type t; 527 } inputs[2]; 528 } others[] = { 529 Op_FastLock, 530 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 531 Op_Lock, 532 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 533 Op_ArrayCopy, 534 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 535 Op_StrCompressedCopy, 536 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 537 Op_StrInflatedCopy, 538 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 539 Op_AryEq, 540 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 541 Op_StrIndexOf, 542 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 543 Op_StrComp, 544 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 545 Op_StrEquals, 546 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 547 Op_VectorizedHashCode, 548 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 549 Op_EncodeISOArray, 550 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 551 Op_CountPositives, 552 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 553 Op_CastP2X, 554 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 555 Op_StrIndexOfChar, 556 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 557 }; 558 559 const int others_len = sizeof(others) / sizeof(others[0]); 560 int i = 0; 561 for (; i < others_len; i++) { 562 if (others[i].opcode == n->Opcode()) { 563 break; 564 } 565 } 566 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 567 if (i != others_len) { 568 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 569 for (uint j = 0; j < inputs_len; j++) { 570 int pos = others[i].inputs[j].pos; 571 if (pos == -1) { 572 break; 573 } 574 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 575 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 576 } 577 } 578 for (uint j = 1; j < stop; j++) { 579 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() && 580 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 581 uint k = 0; 582 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 583 if (k == inputs_len) { 584 fatal("arg %d for node %s not covered", j, n->Name()); 585 } 586 } 587 } 588 } else { 589 for (uint j = 1; j < stop; j++) { 590 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() && 591 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 592 fatal("%s not covered", n->Name()); 593 } 594 } 595 } 596 } 597 598 if (n->is_SafePoint()) { 599 SafePointNode* sfpt = n->as_SafePoint(); 600 if (verify_no_useless_barrier && sfpt->jvms() != nullptr) { 601 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 602 if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 603 phis.clear(); 604 visited.reset(); 605 } 606 } 607 } 608 } 609 } 610 611 if (verify_no_useless_barrier) { 612 for (int i = 0; i < barriers.length(); i++) { 613 Node* n = barriers.at(i); 614 if (!barriers_used.member(n)) { 615 tty->print("XXX useless barrier"); n->dump(-2); 616 ShouldNotReachHere(); 617 } 618 } 619 } 620 } 621 #endif 622 623 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { 624 // That both nodes have the same control is not sufficient to prove 625 // domination, verify that there's no path from d to n 626 ResourceMark rm; 627 Unique_Node_List wq; 628 wq.push(d); 629 for (uint next = 0; next < wq.size(); next++) { 630 Node *m = wq.at(next); 631 if (m == n) { 632 return false; 633 } 634 if (m->is_Phi() && m->in(0)->is_Loop()) { 635 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 636 } else { 637 if (m->is_Store() || m->is_LoadStore()) { 638 // Take anti-dependencies into account 639 Node* mem = m->in(MemNode::Memory); 640 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 641 Node* u = mem->fast_out(i); 642 if (u->is_Load() && phase->C->can_alias(m->adr_type(), phase->C->get_alias_index(u->adr_type())) && 643 phase->ctrl_or_self(u) == c) { 644 wq.push(u); 645 } 646 } 647 } 648 for (uint i = 0; i < m->req(); i++) { 649 if (m->in(i) != nullptr && phase->ctrl_or_self(m->in(i)) == c) { 650 wq.push(m->in(i)); 651 } 652 } 653 } 654 } 655 return true; 656 } 657 658 bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 659 if (d_c != n_c) { 660 return phase->is_dominator(d_c, n_c); 661 } 662 return is_dominator_same_ctrl(d_c, d, n, phase); 663 } 664 665 Node* next_mem(Node* mem, int alias) { 666 Node* res = nullptr; 667 if (mem->is_Proj()) { 668 res = mem->in(0); 669 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 670 res = mem->in(TypeFunc::Memory); 671 } else if (mem->is_Phi()) { 672 res = mem->in(1); 673 } else if (mem->is_MergeMem()) { 674 res = mem->as_MergeMem()->memory_at(alias); 675 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 676 assert(alias == Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 677 res = mem->in(MemNode::Memory); 678 } else { 679 #ifdef ASSERT 680 mem->dump(); 681 #endif 682 ShouldNotReachHere(); 683 } 684 return res; 685 } 686 687 Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 688 Node* iffproj = nullptr; 689 while (c != dom) { 690 Node* next = phase->idom(c); 691 assert(next->unique_ctrl_out_or_null() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 692 if (c->is_Region()) { 693 ResourceMark rm; 694 Unique_Node_List wq; 695 wq.push(c); 696 for (uint i = 0; i < wq.size(); i++) { 697 Node *n = wq.at(i); 698 if (n == next) { 699 continue; 700 } 701 if (n->is_Region()) { 702 for (uint j = 1; j < n->req(); j++) { 703 wq.push(n->in(j)); 704 } 705 } else { 706 wq.push(n->in(0)); 707 } 708 } 709 for (uint i = 0; i < wq.size(); i++) { 710 Node *n = wq.at(i); 711 assert(n->is_CFG(), ""); 712 if (n->is_Multi()) { 713 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 714 Node* u = n->fast_out(j); 715 if (u->is_CFG()) { 716 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 717 return NodeSentinel; 718 } 719 } 720 } 721 } 722 } 723 } else if (c->is_Proj()) { 724 if (c->is_IfProj()) { 725 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != nullptr) { 726 // continue; 727 } else { 728 if (!allow_one_proj) { 729 return NodeSentinel; 730 } 731 if (iffproj == nullptr) { 732 iffproj = c; 733 } else { 734 return NodeSentinel; 735 } 736 } 737 } else if (c->Opcode() == Op_JumpProj) { 738 return NodeSentinel; // unsupported 739 } else if (c->Opcode() == Op_CatchProj) { 740 return NodeSentinel; // unsupported 741 } else if (c->Opcode() == Op_CProj && next->is_NeverBranch()) { 742 return NodeSentinel; // unsupported 743 } else { 744 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 745 } 746 } 747 c = next; 748 } 749 return iffproj; 750 } 751 752 Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 753 ResourceMark rm; 754 VectorSet wq; 755 wq.set(mem->_idx); 756 mem_ctrl = phase->ctrl_or_self(mem); 757 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 758 mem = next_mem(mem, alias); 759 if (wq.test_set(mem->_idx)) { 760 return nullptr; 761 } 762 mem_ctrl = phase->ctrl_or_self(mem); 763 } 764 if (mem->is_MergeMem()) { 765 mem = mem->as_MergeMem()->memory_at(alias); 766 mem_ctrl = phase->ctrl_or_self(mem); 767 } 768 return mem; 769 } 770 771 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 772 Node* mem = nullptr; 773 Node* c = ctrl; 774 do { 775 if (c->is_Region()) { 776 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) { 777 Node* u = c->fast_out(i); 778 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 779 if (u->adr_type() == TypePtr::BOTTOM) { 780 mem = u; 781 } 782 } 783 } 784 } else { 785 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) { 786 CallProjections projs; 787 c->as_Call()->extract_projections(&projs, true, false); 788 if (projs.fallthrough_memproj != nullptr) { 789 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 790 if (projs.catchall_memproj == nullptr) { 791 mem = projs.fallthrough_memproj; 792 } else { 793 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 794 mem = projs.fallthrough_memproj; 795 } else { 796 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 797 mem = projs.catchall_memproj; 798 } 799 } 800 } 801 } else { 802 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 803 if (proj != nullptr && 804 proj->adr_type() == TypePtr::BOTTOM) { 805 mem = proj; 806 } 807 } 808 } else { 809 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 810 Node* u = c->fast_out(i); 811 if (u->is_Proj() && 812 u->bottom_type() == Type::MEMORY && 813 u->adr_type() == TypePtr::BOTTOM) { 814 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 815 assert(mem == nullptr, "only one proj"); 816 mem = u; 817 } 818 } 819 assert(!c->is_Call() || c->as_Call()->adr_type() != nullptr || mem == nullptr, "no mem projection expected"); 820 } 821 } 822 c = phase->idom(c); 823 } while (mem == nullptr); 824 return mem; 825 } 826 827 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 828 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 829 Node* u = n->fast_out(i); 830 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 831 uses.push(u); 832 } 833 } 834 } 835 836 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) { 837 OuterStripMinedLoopEndNode* le = inner->outer_loop_end(); 838 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 839 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 840 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 841 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 842 phase->lazy_replace(outer, new_outer); 843 phase->lazy_replace(le, new_le); 844 inner->clear_strip_mined(); 845 } 846 847 void ShenandoahBarrierC2Support::test_gc_state(Node*& ctrl, Node* raw_mem, Node*& test_fail_ctrl, 848 PhaseIdealLoop* phase, int flags) { 849 PhaseIterGVN& igvn = phase->igvn(); 850 Node* old_ctrl = ctrl; 851 852 Node* thread = new ThreadLocalNode(); 853 Node* gc_state_offset = igvn.MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 854 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, gc_state_offset); 855 Node* gc_state = new LoadBNode(old_ctrl, raw_mem, gc_state_addr, 856 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), 857 TypeInt::BYTE, MemNode::unordered); 858 Node* gc_state_and = new AndINode(gc_state, igvn.intcon(flags)); 859 Node* gc_state_cmp = new CmpINode(gc_state_and, igvn.zerocon(T_INT)); 860 Node* gc_state_bool = new BoolNode(gc_state_cmp, BoolTest::ne); 861 862 IfNode* gc_state_iff = new IfNode(old_ctrl, gc_state_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 863 ctrl = new IfTrueNode(gc_state_iff); 864 test_fail_ctrl = new IfFalseNode(gc_state_iff); 865 866 IdealLoopTree* loop = phase->get_loop(old_ctrl); 867 phase->register_control(gc_state_iff, loop, old_ctrl); 868 phase->register_control(ctrl, loop, gc_state_iff); 869 phase->register_control(test_fail_ctrl, loop, gc_state_iff); 870 871 phase->register_new_node(thread, old_ctrl); 872 phase->register_new_node(gc_state_addr, old_ctrl); 873 phase->register_new_node(gc_state, old_ctrl); 874 phase->register_new_node(gc_state_and, old_ctrl); 875 phase->register_new_node(gc_state_cmp, old_ctrl); 876 phase->register_new_node(gc_state_bool, old_ctrl); 877 878 phase->set_ctrl(gc_state_offset, phase->C->root()); 879 880 assert(is_gc_state_test(gc_state_iff, flags), "Should match the shape"); 881 } 882 883 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { 884 Node* old_ctrl = ctrl; 885 PhaseIterGVN& igvn = phase->igvn(); 886 887 const Type* val_t = igvn.type(val); 888 if (val_t->meet(TypePtr::NULL_PTR) == val_t) { 889 Node* null_cmp = new CmpPNode(val, igvn.zerocon(T_OBJECT)); 890 Node* null_test = new BoolNode(null_cmp, BoolTest::ne); 891 892 IfNode* null_iff = new IfNode(old_ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 893 ctrl = new IfTrueNode(null_iff); 894 null_ctrl = new IfFalseNode(null_iff); 895 896 IdealLoopTree* loop = phase->get_loop(old_ctrl); 897 phase->register_control(null_iff, loop, old_ctrl); 898 phase->register_control(ctrl, loop, null_iff); 899 phase->register_control(null_ctrl, loop, null_iff); 900 901 phase->register_new_node(null_cmp, old_ctrl); 902 phase->register_new_node(null_test, old_ctrl); 903 } 904 } 905 906 void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { 907 Node* old_ctrl = ctrl; 908 PhaseIterGVN& igvn = phase->igvn(); 909 910 Node* raw_val = new CastP2XNode(old_ctrl, val); 911 Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 912 913 // Figure out the target cset address with raw pointer math. 914 // This avoids matching AddP+LoadB that would emit inefficient code. 915 // See JDK-8245465. 916 Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 917 Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr); 918 Node* cset_load_addr = new AddXNode(cset_addr, cset_idx); 919 Node* cset_load_ptr = new CastX2PNode(cset_load_addr); 920 921 Node* cset_load = new LoadBNode(old_ctrl, raw_mem, cset_load_ptr, 922 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), 923 TypeInt::BYTE, MemNode::unordered); 924 Node* cset_cmp = new CmpINode(cset_load, igvn.zerocon(T_INT)); 925 Node* cset_bool = new BoolNode(cset_cmp, BoolTest::ne); 926 927 IfNode* cset_iff = new IfNode(old_ctrl, cset_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 928 ctrl = new IfTrueNode(cset_iff); 929 not_cset_ctrl = new IfFalseNode(cset_iff); 930 931 IdealLoopTree *loop = phase->get_loop(old_ctrl); 932 phase->register_control(cset_iff, loop, old_ctrl); 933 phase->register_control(ctrl, loop, cset_iff); 934 phase->register_control(not_cset_ctrl, loop, cset_iff); 935 936 phase->set_ctrl(cset_addr_ptr, phase->C->root()); 937 938 phase->register_new_node(raw_val, old_ctrl); 939 phase->register_new_node(cset_idx, old_ctrl); 940 phase->register_new_node(cset_addr, old_ctrl); 941 phase->register_new_node(cset_load_addr, old_ctrl); 942 phase->register_new_node(cset_load_ptr, old_ctrl); 943 phase->register_new_node(cset_load, old_ctrl); 944 phase->register_new_node(cset_cmp, old_ctrl); 945 phase->register_new_node(cset_bool, old_ctrl); 946 } 947 948 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, 949 DecoratorSet decorators, PhaseIdealLoop* phase) { 950 IdealLoopTree*loop = phase->get_loop(ctrl); 951 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); 952 953 address calladdr = nullptr; 954 const char* name = nullptr; 955 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 956 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 957 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 958 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 959 bool is_narrow = UseCompressedOops && !is_native; 960 if (is_strong) { 961 if (is_narrow) { 962 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); 963 name = "load_reference_barrier_strong_narrow"; 964 } else { 965 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); 966 name = "load_reference_barrier_strong"; 967 } 968 } else if (is_weak) { 969 if (is_narrow) { 970 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); 971 name = "load_reference_barrier_weak_narrow"; 972 } else { 973 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); 974 name = "load_reference_barrier_weak"; 975 } 976 } else { 977 assert(is_phantom, "only remaining strength"); 978 if (is_narrow) { 979 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow); 980 name = "load_reference_barrier_phantom_narrow"; 981 } else { 982 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom); 983 name = "load_reference_barrier_phantom"; 984 } 985 } 986 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); 987 988 call->init_req(TypeFunc::Control, ctrl); 989 call->init_req(TypeFunc::I_O, phase->C->top()); 990 call->init_req(TypeFunc::Memory, phase->C->top()); 991 call->init_req(TypeFunc::FramePtr, phase->C->top()); 992 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 993 call->init_req(TypeFunc::Parms, val); 994 call->init_req(TypeFunc::Parms+1, load_addr); 995 phase->register_control(call, loop, ctrl); 996 ctrl = new ProjNode(call, TypeFunc::Control); 997 phase->register_control(ctrl, loop, call); 998 val = new ProjNode(call, TypeFunc::Parms); 999 phase->register_new_node(val, call); 1000 val = new CheckCastPPNode(ctrl, val, obj_type); 1001 phase->register_new_node(val, ctrl); 1002 } 1003 1004 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { 1005 Node* ctrl = phase->get_ctrl(barrier); 1006 Node* init_raw_mem = fixer.find_mem(ctrl, barrier); 1007 1008 // Update the control of all nodes that should be after the 1009 // barrier control flow 1010 uses.clear(); 1011 // Every node that is control dependent on the barrier's input 1012 // control will be after the expanded barrier. The raw memory (if 1013 // its memory is control dependent on the barrier's input control) 1014 // must stay above the barrier. 1015 uses_to_ignore.clear(); 1016 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 1017 uses_to_ignore.push(init_raw_mem); 1018 } 1019 for (uint next = 0; next < uses_to_ignore.size(); next++) { 1020 Node *n = uses_to_ignore.at(next); 1021 for (uint i = 0; i < n->req(); i++) { 1022 Node* in = n->in(i); 1023 if (in != nullptr && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 1024 uses_to_ignore.push(in); 1025 } 1026 } 1027 } 1028 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 1029 Node* u = ctrl->fast_out(i); 1030 if (u->_idx < last && 1031 u != barrier && 1032 !u->depends_only_on_test() && // preserve dependency on test 1033 !uses_to_ignore.member(u) && 1034 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 1035 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 1036 Node* old_c = phase->ctrl_or_self(u); 1037 Node* c = old_c; 1038 if (c != ctrl || 1039 is_dominator_same_ctrl(old_c, barrier, u, phase) || 1040 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) { 1041 phase->igvn().rehash_node_delayed(u); 1042 int nb = u->replace_edge(ctrl, region, &phase->igvn()); 1043 if (u->is_CFG()) { 1044 if (phase->idom(u) == ctrl) { 1045 phase->set_idom(u, region, phase->dom_depth(region)); 1046 } 1047 } else if (phase->get_ctrl(u) == ctrl) { 1048 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 1049 uses.push(u); 1050 } 1051 assert(nb == 1, "more than 1 ctrl input?"); 1052 --i, imax -= nb; 1053 } 1054 } 1055 } 1056 } 1057 1058 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { 1059 Node* region = nullptr; 1060 while (c != ctrl) { 1061 if (c->is_Region()) { 1062 region = c; 1063 } 1064 c = phase->idom(c); 1065 } 1066 assert(region != nullptr, ""); 1067 Node* phi = new PhiNode(region, n->bottom_type()); 1068 for (uint j = 1; j < region->req(); j++) { 1069 Node* in = region->in(j); 1070 if (phase->is_dominator(projs.fallthrough_catchproj, in)) { 1071 phi->init_req(j, n); 1072 } else if (phase->is_dominator(projs.catchall_catchproj, in)) { 1073 phi->init_req(j, n_clone); 1074 } else { 1075 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); 1076 } 1077 } 1078 phase->register_new_node(phi, region); 1079 return phi; 1080 } 1081 1082 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { 1083 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 1084 1085 Unique_Node_List uses; 1086 Node_Stack stack(0); 1087 Node_List clones; 1088 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1089 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1090 1091 Node* ctrl = phase->get_ctrl(lrb); 1092 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1093 1094 CallStaticJavaNode* unc = nullptr; 1095 Node* unc_ctrl = nullptr; 1096 Node* uncasted_val = val; 1097 1098 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { 1099 Node* u = lrb->fast_out(i); 1100 if (u->Opcode() == Op_CastPP && 1101 u->in(0) != nullptr && 1102 phase->is_dominator(u->in(0), ctrl)) { 1103 const Type* u_t = phase->igvn().type(u); 1104 1105 if (u_t->meet(TypePtr::NULL_PTR) != u_t && 1106 u->in(0)->Opcode() == Op_IfTrue && 1107 u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 1108 u->in(0)->in(0)->is_If() && 1109 u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 1110 u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 1111 u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 1112 u->in(0)->in(0)->in(1)->in(1)->in(1) == val && 1113 u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 1114 IdealLoopTree* loop = phase->get_loop(ctrl); 1115 IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); 1116 1117 if (!unc_loop->is_member(loop)) { 1118 continue; 1119 } 1120 1121 Node* branch = no_branches(ctrl, u->in(0), false, phase); 1122 assert(branch == nullptr || branch == NodeSentinel, "was not looking for a branch"); 1123 if (branch == NodeSentinel) { 1124 continue; 1125 } 1126 1127 Node* iff = u->in(0)->in(0); 1128 Node* bol = iff->in(1)->clone(); 1129 Node* cmp = bol->in(1)->clone(); 1130 cmp->set_req(1, lrb); 1131 bol->set_req(1, cmp); 1132 phase->igvn().replace_input_of(iff, 1, bol); 1133 phase->set_ctrl(lrb, iff->in(0)); 1134 phase->register_new_node(cmp, iff->in(0)); 1135 phase->register_new_node(bol, iff->in(0)); 1136 break; 1137 } 1138 } 1139 } 1140 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { 1141 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); 1142 if (call->entry_point() == OptoRuntime::rethrow_stub()) { 1143 // The rethrow call may have too many projections to be 1144 // properly handled here. Given there's no reason for a 1145 // barrier to depend on the call, move it above the call 1146 stack.push(lrb, 0); 1147 do { 1148 Node* n = stack.node(); 1149 uint idx = stack.index(); 1150 if (idx < n->req()) { 1151 Node* in = n->in(idx); 1152 stack.set_index(idx+1); 1153 if (in != nullptr) { 1154 if (phase->has_ctrl(in)) { 1155 if (phase->is_dominator(call, phase->get_ctrl(in))) { 1156 #ifdef ASSERT 1157 for (uint i = 0; i < stack.size(); i++) { 1158 assert(stack.node_at(i) != in, "node shouldn't have been seen yet"); 1159 } 1160 #endif 1161 stack.push(in, 0); 1162 } 1163 } else { 1164 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call"); 1165 } 1166 } 1167 } else { 1168 phase->set_ctrl(n, call->in(0)); 1169 stack.pop(); 1170 } 1171 } while(stack.size() > 0); 1172 continue; 1173 } 1174 CallProjections projs; 1175 call->extract_projections(&projs, false, false); 1176 1177 #ifdef ASSERT 1178 VectorSet cloned; 1179 #endif 1180 Node* lrb_clone = lrb->clone(); 1181 phase->register_new_node(lrb_clone, projs.catchall_catchproj); 1182 phase->set_ctrl(lrb, projs.fallthrough_catchproj); 1183 1184 stack.push(lrb, 0); 1185 clones.push(lrb_clone); 1186 1187 do { 1188 assert(stack.size() == clones.size(), ""); 1189 Node* n = stack.node(); 1190 #ifdef ASSERT 1191 if (n->is_Load()) { 1192 Node* mem = n->in(MemNode::Memory); 1193 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { 1194 Node* u = mem->fast_out(j); 1195 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); 1196 } 1197 } 1198 #endif 1199 uint idx = stack.index(); 1200 Node* n_clone = clones.at(clones.size()-1); 1201 if (idx < n->outcnt()) { 1202 Node* u = n->raw_out(idx); 1203 Node* c = phase->ctrl_or_self(u); 1204 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) { 1205 stack.set_index(idx+1); 1206 assert(!u->is_CFG(), ""); 1207 stack.push(u, 0); 1208 assert(!cloned.test_set(u->_idx), "only one clone"); 1209 Node* u_clone = u->clone(); 1210 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn()); 1211 assert(nb > 0, "should have replaced some uses"); 1212 phase->register_new_node(u_clone, projs.catchall_catchproj); 1213 clones.push(u_clone); 1214 phase->set_ctrl(u, projs.fallthrough_catchproj); 1215 } else { 1216 bool replaced = false; 1217 if (u->is_Phi()) { 1218 for (uint k = 1; k < u->req(); k++) { 1219 if (u->in(k) == n) { 1220 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { 1221 phase->igvn().replace_input_of(u, k, n_clone); 1222 replaced = true; 1223 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { 1224 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); 1225 replaced = true; 1226 } 1227 } 1228 } 1229 } else { 1230 if (phase->is_dominator(projs.catchall_catchproj, c)) { 1231 phase->igvn().rehash_node_delayed(u); 1232 int nb = u->replace_edge(n, n_clone, &phase->igvn()); 1233 assert(nb > 0, "should have replaced some uses"); 1234 replaced = true; 1235 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { 1236 if (u->is_If()) { 1237 // Can't break If/Bool/Cmp chain 1238 assert(n->is_Bool(), "unexpected If shape"); 1239 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape"); 1240 assert(n_clone->is_Bool(), "unexpected clone"); 1241 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone"); 1242 Node* bol_clone = n->clone(); 1243 Node* cmp_clone = stack.node_at(stack.size()-2)->clone(); 1244 bol_clone->set_req(1, cmp_clone); 1245 1246 Node* nn = stack.node_at(stack.size()-3); 1247 Node* nn_clone = clones.at(clones.size()-3); 1248 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch"); 1249 1250 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase), 1251 &phase->igvn()); 1252 assert(nb > 0, "should have replaced some uses"); 1253 1254 phase->register_new_node(bol_clone, u->in(0)); 1255 phase->register_new_node(cmp_clone, u->in(0)); 1256 1257 phase->igvn().replace_input_of(u, 1, bol_clone); 1258 1259 } else { 1260 phase->igvn().rehash_node_delayed(u); 1261 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase), &phase->igvn()); 1262 assert(nb > 0, "should have replaced some uses"); 1263 } 1264 replaced = true; 1265 } 1266 } 1267 if (!replaced) { 1268 stack.set_index(idx+1); 1269 } 1270 } 1271 } else { 1272 stack.pop(); 1273 clones.pop(); 1274 } 1275 } while (stack.size() > 0); 1276 assert(stack.size() == 0 && clones.size() == 0, ""); 1277 } 1278 } 1279 1280 for (int i = 0; i < state->load_reference_barriers_count(); i++) { 1281 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1282 Node* ctrl = phase->get_ctrl(lrb); 1283 IdealLoopTree* loop = phase->get_loop(ctrl); 1284 Node* head = loop->head(); 1285 if (head->is_OuterStripMinedLoop()) { 1286 // Expanding a barrier here will break loop strip mining 1287 // verification. Transform the loop so the loop nest doesn't 1288 // appear as strip mined. 1289 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop(); 1290 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1291 } 1292 if (head->is_BaseCountedLoop() && ctrl->is_IfProj() && ctrl->in(0)->is_BaseCountedLoopEnd() && 1293 head->as_BaseCountedLoop()->loopexit() == ctrl->in(0)) { 1294 Node* entry = head->in(LoopNode::EntryControl); 1295 Node* backedge = head->in(LoopNode::LoopBackControl); 1296 Node* new_head = new LoopNode(entry, backedge); 1297 phase->register_control(new_head, phase->get_loop(entry), entry); 1298 phase->lazy_replace(head, new_head); 1299 } 1300 } 1301 1302 // Expand load-reference-barriers 1303 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); 1304 Unique_Node_List uses_to_ignore; 1305 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1306 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1307 uint last = phase->C->unique(); 1308 Node* ctrl = phase->get_ctrl(lrb); 1309 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1310 1311 Node* orig_ctrl = ctrl; 1312 1313 Node* raw_mem = fixer.find_mem(ctrl, lrb); 1314 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr); 1315 1316 IdealLoopTree *loop = phase->get_loop(ctrl); 1317 1318 Node* heap_stable_ctrl = nullptr; 1319 Node* null_ctrl = nullptr; 1320 1321 assert(val->bottom_type()->make_oopptr(), "need oop"); 1322 assert(val->bottom_type()->make_oopptr()->const_oop() == nullptr, "expect non-constant"); 1323 1324 enum { _heap_stable = 1, _evac_path, _not_cset, PATH_LIMIT }; 1325 Node* region = new RegionNode(PATH_LIMIT); 1326 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()); 1327 1328 // Stable path. 1329 int flags = ShenandoahHeap::HAS_FORWARDED; 1330 if (!ShenandoahBarrierSet::is_strong_access(lrb->decorators())) { 1331 flags |= ShenandoahHeap::WEAK_ROOTS; 1332 } 1333 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, flags); 1334 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); 1335 1336 // Heap stable case 1337 region->init_req(_heap_stable, heap_stable_ctrl); 1338 val_phi->init_req(_heap_stable, val); 1339 1340 // Test for in-cset, unless it's a native-LRB. Native LRBs need to return null 1341 // even for non-cset objects to prevent resurrection of such objects. 1342 // Wires !in_cset(obj) to slot 2 of region and phis 1343 Node* not_cset_ctrl = nullptr; 1344 if (ShenandoahBarrierSet::is_strong_access(lrb->decorators())) { 1345 test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase); 1346 } 1347 if (not_cset_ctrl != nullptr) { 1348 region->init_req(_not_cset, not_cset_ctrl); 1349 val_phi->init_req(_not_cset, val); 1350 } else { 1351 region->del_req(_not_cset); 1352 val_phi->del_req(_not_cset); 1353 } 1354 1355 // Resolve object when orig-value is in cset. 1356 // Make the unconditional resolve for fwdptr. 1357 1358 // Call lrb-stub and wire up that path in slots 4 1359 Node* result_mem = nullptr; 1360 1361 Node* addr; 1362 { 1363 VectorSet visited; 1364 addr = get_load_addr(phase, visited, lrb); 1365 } 1366 if (addr->Opcode() == Op_AddP) { 1367 Node* orig_base = addr->in(AddPNode::Base); 1368 Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), ConstraintCastNode::StrongDependency); 1369 phase->register_new_node(base, ctrl); 1370 if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { 1371 // Field access 1372 addr = addr->clone(); 1373 addr->set_req(AddPNode::Base, base); 1374 addr->set_req(AddPNode::Address, base); 1375 phase->register_new_node(addr, ctrl); 1376 } else { 1377 Node* addr2 = addr->in(AddPNode::Address); 1378 if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && 1379 addr2->in(AddPNode::Base) == orig_base) { 1380 addr2 = addr2->clone(); 1381 addr2->set_req(AddPNode::Base, base); 1382 addr2->set_req(AddPNode::Address, base); 1383 phase->register_new_node(addr2, ctrl); 1384 addr = addr->clone(); 1385 addr->set_req(AddPNode::Base, base); 1386 addr->set_req(AddPNode::Address, addr2); 1387 phase->register_new_node(addr, ctrl); 1388 } 1389 } 1390 } 1391 call_lrb_stub(ctrl, val, addr, lrb->decorators(), phase); 1392 region->init_req(_evac_path, ctrl); 1393 val_phi->init_req(_evac_path, val); 1394 1395 phase->register_control(region, loop, heap_stable_iff); 1396 Node* out_val = val_phi; 1397 phase->register_new_node(val_phi, region); 1398 1399 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); 1400 1401 ctrl = orig_ctrl; 1402 1403 phase->igvn().replace_node(lrb, out_val); 1404 1405 follow_barrier_uses(out_val, ctrl, uses, phase); 1406 1407 for(uint next = 0; next < uses.size(); next++ ) { 1408 Node *n = uses.at(next); 1409 assert(phase->get_ctrl(n) == ctrl, "bad control"); 1410 assert(n != raw_mem, "should leave input raw mem above the barrier"); 1411 phase->set_ctrl(n, region); 1412 follow_barrier_uses(n, ctrl, uses, phase); 1413 } 1414 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl); 1415 } 1416 // Done expanding load-reference-barriers. 1417 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); 1418 } 1419 1420 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { 1421 if (visited.test_set(in->_idx)) { 1422 return nullptr; 1423 } 1424 switch (in->Opcode()) { 1425 case Op_Proj: 1426 return get_load_addr(phase, visited, in->in(0)); 1427 case Op_CastPP: 1428 case Op_CheckCastPP: 1429 case Op_DecodeN: 1430 case Op_EncodeP: 1431 return get_load_addr(phase, visited, in->in(1)); 1432 case Op_LoadN: 1433 case Op_LoadP: 1434 return in->in(MemNode::Address); 1435 case Op_CompareAndExchangeN: 1436 case Op_CompareAndExchangeP: 1437 case Op_GetAndSetN: 1438 case Op_GetAndSetP: 1439 case Op_ShenandoahCompareAndExchangeP: 1440 case Op_ShenandoahCompareAndExchangeN: 1441 // Those instructions would just have stored a different 1442 // value into the field. No use to attempt to fix it at this point. 1443 return phase->igvn().zerocon(T_OBJECT); 1444 case Op_CMoveP: 1445 case Op_CMoveN: { 1446 Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); 1447 Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); 1448 // Handle unambiguous cases: single address reported on both branches. 1449 if (t != nullptr && f == nullptr) return t; 1450 if (t == nullptr && f != nullptr) return f; 1451 if (t != nullptr && t == f) return t; 1452 // Ambiguity. 1453 return phase->igvn().zerocon(T_OBJECT); 1454 } 1455 case Op_Phi: { 1456 Node* addr = nullptr; 1457 for (uint i = 1; i < in->req(); i++) { 1458 Node* addr1 = get_load_addr(phase, visited, in->in(i)); 1459 if (addr == nullptr) { 1460 addr = addr1; 1461 } 1462 if (addr != addr1) { 1463 return phase->igvn().zerocon(T_OBJECT); 1464 } 1465 } 1466 return addr; 1467 } 1468 case Op_ShenandoahLoadReferenceBarrier: 1469 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); 1470 case Op_CallDynamicJava: 1471 case Op_CallLeaf: 1472 case Op_CallStaticJava: 1473 case Op_ConN: 1474 case Op_ConP: 1475 case Op_Parm: 1476 case Op_CreateEx: 1477 return phase->igvn().zerocon(T_OBJECT); 1478 default: 1479 #ifdef ASSERT 1480 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); 1481 #endif 1482 return phase->igvn().zerocon(T_OBJECT); 1483 } 1484 1485 } 1486 1487 #ifdef ASSERT 1488 static bool has_never_branch(Node* root) { 1489 for (uint i = 1; i < root->req(); i++) { 1490 Node* in = root->in(i); 1491 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) { 1492 return true; 1493 } 1494 } 1495 return false; 1496 } 1497 #endif 1498 1499 void MemoryGraphFixer::collect_memory_nodes() { 1500 Node_Stack stack(0); 1501 VectorSet visited; 1502 Node_List regions; 1503 1504 // Walk the raw memory graph and create a mapping from CFG node to 1505 // memory node. Exclude phis for now. 1506 stack.push(_phase->C->root(), 1); 1507 do { 1508 Node* n = stack.node(); 1509 int opc = n->Opcode(); 1510 uint i = stack.index(); 1511 if (i < n->req()) { 1512 Node* mem = nullptr; 1513 if (opc == Op_Root) { 1514 Node* in = n->in(i); 1515 int in_opc = in->Opcode(); 1516 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 1517 mem = in->in(TypeFunc::Memory); 1518 } else if (in_opc == Op_Halt) { 1519 if (in->in(0)->is_Region()) { 1520 Node* r = in->in(0); 1521 for (uint j = 1; j < r->req(); j++) { 1522 assert(!r->in(j)->is_NeverBranch(), ""); 1523 } 1524 } else { 1525 Node* proj = in->in(0); 1526 assert(proj->is_Proj(), ""); 1527 Node* in = proj->in(0); 1528 assert(in->is_CallStaticJava() || in->is_NeverBranch() || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 1529 if (in->is_CallStaticJava()) { 1530 mem = in->in(TypeFunc::Memory); 1531 } else if (in->Opcode() == Op_Catch) { 1532 Node* call = in->in(0)->in(0); 1533 assert(call->is_Call(), ""); 1534 mem = call->in(TypeFunc::Memory); 1535 } else if (in->is_NeverBranch()) { 1536 mem = collect_memory_for_infinite_loop(in); 1537 } 1538 } 1539 } else { 1540 #ifdef ASSERT 1541 n->dump(); 1542 in->dump(); 1543 #endif 1544 ShouldNotReachHere(); 1545 } 1546 } else { 1547 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 1548 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, ""); 1549 mem = n->in(i); 1550 } 1551 i++; 1552 stack.set_index(i); 1553 if (mem == nullptr) { 1554 continue; 1555 } 1556 for (;;) { 1557 if (visited.test_set(mem->_idx) || mem->is_Start()) { 1558 break; 1559 } 1560 if (mem->is_Phi()) { 1561 stack.push(mem, 2); 1562 mem = mem->in(1); 1563 } else if (mem->is_Proj()) { 1564 stack.push(mem, mem->req()); 1565 mem = mem->in(0); 1566 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1567 mem = mem->in(TypeFunc::Memory); 1568 } else if (mem->is_MergeMem()) { 1569 MergeMemNode* mm = mem->as_MergeMem(); 1570 mem = mm->memory_at(_alias); 1571 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1572 assert(_alias == Compile::AliasIdxRaw, ""); 1573 stack.push(mem, mem->req()); 1574 mem = mem->in(MemNode::Memory); 1575 } else { 1576 #ifdef ASSERT 1577 mem->dump(); 1578 #endif 1579 ShouldNotReachHere(); 1580 } 1581 } 1582 } else { 1583 if (n->is_Phi()) { 1584 // Nothing 1585 } else if (!n->is_Root()) { 1586 Node* c = get_ctrl(n); 1587 _memory_nodes.map(c->_idx, n); 1588 } 1589 stack.pop(); 1590 } 1591 } while(stack.is_nonempty()); 1592 1593 // Iterate over CFG nodes in rpo and propagate memory state to 1594 // compute memory state at regions, creating new phis if needed. 1595 Node_List rpo_list; 1596 visited.clear(); 1597 _phase->rpo(_phase->C->root(), stack, visited, rpo_list); 1598 Node* root = rpo_list.pop(); 1599 assert(root == _phase->C->root(), ""); 1600 1601 const bool trace = false; 1602 #ifdef ASSERT 1603 if (trace) { 1604 for (int i = rpo_list.size() - 1; i >= 0; i--) { 1605 Node* c = rpo_list.at(i); 1606 if (_memory_nodes[c->_idx] != nullptr) { 1607 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); 1608 } 1609 } 1610 } 1611 #endif 1612 uint last = _phase->C->unique(); 1613 1614 #ifdef ASSERT 1615 uint16_t max_depth = 0; 1616 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) { 1617 IdealLoopTree* lpt = iter.current(); 1618 max_depth = MAX2(max_depth, lpt->_nest); 1619 } 1620 #endif 1621 1622 bool progress = true; 1623 int iteration = 0; 1624 Node_List dead_phis; 1625 while (progress) { 1626 progress = false; 1627 iteration++; 1628 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 1629 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 1630 1631 for (int i = rpo_list.size() - 1; i >= 0; i--) { 1632 Node* c = rpo_list.at(i); 1633 1634 Node* prev_mem = _memory_nodes[c->_idx]; 1635 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 1636 Node* prev_region = regions[c->_idx]; 1637 Node* unique = nullptr; 1638 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 1639 Node* m = _memory_nodes[c->in(j)->_idx]; 1640 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state"); 1641 if (m != nullptr) { 1642 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 1643 assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 1644 // continue 1645 } else if (unique == nullptr) { 1646 unique = m; 1647 } else if (m == unique) { 1648 // continue 1649 } else { 1650 unique = NodeSentinel; 1651 } 1652 } 1653 } 1654 assert(unique != nullptr, "empty phi???"); 1655 if (unique != NodeSentinel) { 1656 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) { 1657 dead_phis.push(prev_region); 1658 } 1659 regions.map(c->_idx, unique); 1660 } else { 1661 Node* phi = nullptr; 1662 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 1663 phi = prev_region; 1664 for (uint k = 1; k < c->req(); k++) { 1665 Node* m = _memory_nodes[c->in(k)->_idx]; 1666 assert(m != nullptr, "expect memory state"); 1667 phi->set_req(k, m); 1668 } 1669 } else { 1670 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == nullptr; j++) { 1671 Node* u = c->fast_out(j); 1672 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 1673 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) { 1674 phi = u; 1675 for (uint k = 1; k < c->req() && phi != nullptr; k++) { 1676 Node* m = _memory_nodes[c->in(k)->_idx]; 1677 assert(m != nullptr, "expect memory state"); 1678 if (u->in(k) != m) { 1679 phi = NodeSentinel; 1680 } 1681 } 1682 } 1683 } 1684 if (phi == NodeSentinel) { 1685 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); 1686 for (uint k = 1; k < c->req(); k++) { 1687 Node* m = _memory_nodes[c->in(k)->_idx]; 1688 assert(m != nullptr, "expect memory state"); 1689 phi->init_req(k, m); 1690 } 1691 } 1692 } 1693 if (phi != nullptr) { 1694 regions.map(c->_idx, phi); 1695 } else { 1696 assert(c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 1697 } 1698 } 1699 Node* current_region = regions[c->_idx]; 1700 if (current_region != prev_region) { 1701 progress = true; 1702 if (prev_region == prev_mem) { 1703 _memory_nodes.map(c->_idx, current_region); 1704 } 1705 } 1706 } else if (prev_mem == nullptr || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { 1707 Node* m = _memory_nodes[_phase->idom(c)->_idx]; 1708 assert(m != nullptr || c->Opcode() == Op_Halt, "expect memory state"); 1709 if (m != prev_mem) { 1710 _memory_nodes.map(c->_idx, m); 1711 progress = true; 1712 } 1713 } 1714 #ifdef ASSERT 1715 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); } 1716 #endif 1717 } 1718 } 1719 1720 // Replace existing phi with computed memory state for that region 1721 // if different (could be a new phi or a dominating memory node if 1722 // that phi was found to be useless). 1723 while (dead_phis.size() > 0) { 1724 Node* n = dead_phis.pop(); 1725 n->replace_by(_phase->C->top()); 1726 n->destruct(&_phase->igvn()); 1727 } 1728 for (int i = rpo_list.size() - 1; i >= 0; i--) { 1729 Node* c = rpo_list.at(i); 1730 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 1731 Node* n = regions[c->_idx]; 1732 assert(n != nullptr || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 1733 if (n != nullptr && n->is_Phi() && n->_idx >= last && n->in(0) == c) { 1734 _phase->register_new_node(n, c); 1735 } 1736 } 1737 } 1738 for (int i = rpo_list.size() - 1; i >= 0; i--) { 1739 Node* c = rpo_list.at(i); 1740 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 1741 Node* n = regions[c->_idx]; 1742 assert(n != nullptr || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 1743 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 1744 Node* u = c->fast_out(i); 1745 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 1746 u != n) { 1747 assert(c->unique_ctrl_out()->Opcode() != Op_Halt, "expected memory state"); 1748 if (u->adr_type() == TypePtr::BOTTOM) { 1749 fix_memory_uses(u, n, n, c); 1750 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 1751 _phase->lazy_replace(u, n); 1752 --i; --imax; 1753 } 1754 } 1755 } 1756 } 1757 } 1758 } 1759 1760 Node* MemoryGraphFixer::collect_memory_for_infinite_loop(const Node* in) { 1761 Node* mem = nullptr; 1762 Node* head = in->in(0); 1763 assert(head->is_Region(), "unexpected infinite loop graph shape"); 1764 1765 Node* phi_mem = nullptr; 1766 for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) { 1767 Node* u = head->fast_out(j); 1768 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 1769 if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 1770 assert(phi_mem == nullptr || phi_mem->adr_type() == TypePtr::BOTTOM, ""); 1771 phi_mem = u; 1772 } else if (u->adr_type() == TypePtr::BOTTOM) { 1773 assert(phi_mem == nullptr || _phase->C->get_alias_index(phi_mem->adr_type()) == _alias, ""); 1774 if (phi_mem == nullptr) { 1775 phi_mem = u; 1776 } 1777 } 1778 } 1779 } 1780 if (phi_mem == nullptr) { 1781 ResourceMark rm; 1782 Node_Stack stack(0); 1783 stack.push(head, 1); 1784 do { 1785 Node* n = stack.node(); 1786 uint i = stack.index(); 1787 if (i >= n->req()) { 1788 stack.pop(); 1789 } else { 1790 stack.set_index(i + 1); 1791 Node* c = n->in(i); 1792 assert(c != head, "should have found a safepoint on the way"); 1793 if (stack.size() != 1 || _phase->is_dominator(head, c)) { 1794 for (;;) { 1795 if (c->is_Region()) { 1796 stack.push(c, 1); 1797 break; 1798 } else if (c->is_SafePoint() && !c->is_CallLeaf()) { 1799 Node* m = c->in(TypeFunc::Memory); 1800 if (m->is_MergeMem()) { 1801 m = m->as_MergeMem()->memory_at(_alias); 1802 } 1803 assert(mem == nullptr || mem == m, "several memory states"); 1804 mem = m; 1805 break; 1806 } else { 1807 assert(c != c->in(0), ""); 1808 c = c->in(0); 1809 } 1810 } 1811 } 1812 } 1813 } while (stack.size() > 0); 1814 assert(mem != nullptr, "should have found safepoint"); 1815 } else { 1816 mem = phi_mem; 1817 } 1818 return mem; 1819 } 1820 1821 Node* MemoryGraphFixer::get_ctrl(Node* n) const { 1822 Node* c = _phase->get_ctrl(n); 1823 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) { 1824 assert(c == n->in(0), ""); 1825 CallNode* call = c->as_Call(); 1826 CallProjections projs; 1827 call->extract_projections(&projs, true, false); 1828 if (projs.catchall_memproj != nullptr) { 1829 if (projs.fallthrough_memproj == n) { 1830 c = projs.fallthrough_catchproj; 1831 } else { 1832 assert(projs.catchall_memproj == n, ""); 1833 c = projs.catchall_catchproj; 1834 } 1835 } 1836 } 1837 return c; 1838 } 1839 1840 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const { 1841 if (_phase->has_ctrl(n)) 1842 return get_ctrl(n); 1843 else { 1844 assert (n->is_CFG(), "must be a CFG node"); 1845 return n; 1846 } 1847 } 1848 1849 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const { 1850 return m != nullptr && get_ctrl(m) == c; 1851 } 1852 1853 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const { 1854 assert(n == nullptr || _phase->ctrl_or_self(n) == ctrl, ""); 1855 assert(!ctrl->is_Call() || ctrl == n, "projection expected"); 1856 #ifdef ASSERT 1857 if ((ctrl->is_Proj() && ctrl->in(0)->is_Call()) || 1858 (ctrl->is_Catch() && ctrl->in(0)->in(0)->is_Call())) { 1859 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_Call() : ctrl->in(0)->in(0)->as_Call(); 1860 int mems = 0; 1861 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) { 1862 Node* u = call->fast_out(i); 1863 if (u->bottom_type() == Type::MEMORY) { 1864 mems++; 1865 } 1866 } 1867 assert(mems <= 1, "No node right after call if multiple mem projections"); 1868 } 1869 #endif 1870 Node* mem = _memory_nodes[ctrl->_idx]; 1871 Node* c = ctrl; 1872 while (!mem_is_valid(mem, c) && 1873 (!c->is_CatchProj() || mem == nullptr || c->in(0)->in(0)->in(0) != get_ctrl(mem))) { 1874 c = _phase->idom(c); 1875 mem = _memory_nodes[c->_idx]; 1876 } 1877 if (n != nullptr && mem_is_valid(mem, c)) { 1878 while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { 1879 mem = next_mem(mem, _alias); 1880 } 1881 if (mem->is_MergeMem()) { 1882 mem = mem->as_MergeMem()->memory_at(_alias); 1883 } 1884 if (!mem_is_valid(mem, c)) { 1885 do { 1886 c = _phase->idom(c); 1887 mem = _memory_nodes[c->_idx]; 1888 } while (!mem_is_valid(mem, c) && 1889 (!c->is_CatchProj() || mem == nullptr || c->in(0)->in(0)->in(0) != get_ctrl(mem))); 1890 } 1891 } 1892 assert(mem->bottom_type() == Type::MEMORY, ""); 1893 return mem; 1894 } 1895 1896 bool MemoryGraphFixer::has_mem_phi(Node* region) const { 1897 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 1898 Node* use = region->fast_out(i); 1899 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 1900 (_phase->C->get_alias_index(use->adr_type()) == _alias)) { 1901 return true; 1902 } 1903 } 1904 return false; 1905 } 1906 1907 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { 1908 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); 1909 const bool trace = false; 1910 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 1911 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); }); 1912 GrowableArray<Node*> phis; 1913 if (mem_for_ctrl != mem) { 1914 Node* old = mem_for_ctrl; 1915 Node* prev = nullptr; 1916 while (old != mem) { 1917 prev = old; 1918 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { 1919 assert(_alias == Compile::AliasIdxRaw, ""); 1920 old = old->in(MemNode::Memory); 1921 } else if (old->Opcode() == Op_SCMemProj) { 1922 assert(_alias == Compile::AliasIdxRaw, ""); 1923 old = old->in(0); 1924 } else { 1925 ShouldNotReachHere(); 1926 } 1927 } 1928 assert(prev != nullptr, ""); 1929 if (new_ctrl != ctrl) { 1930 _memory_nodes.map(ctrl->_idx, mem); 1931 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 1932 } 1933 uint input = (uint)MemNode::Memory; 1934 _phase->igvn().replace_input_of(prev, input, new_mem); 1935 } else { 1936 uses.clear(); 1937 _memory_nodes.map(new_ctrl->_idx, new_mem); 1938 uses.push(new_ctrl); 1939 for(uint next = 0; next < uses.size(); next++ ) { 1940 Node *n = uses.at(next); 1941 assert(n->is_CFG(), ""); 1942 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 1943 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 1944 Node* u = n->fast_out(i); 1945 if (!u->is_Root() && u->is_CFG() && u != n) { 1946 Node* m = _memory_nodes[u->_idx]; 1947 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) && 1948 !has_mem_phi(u) && 1949 u->unique_ctrl_out()->Opcode() != Op_Halt) { 1950 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 1951 DEBUG_ONLY(if (trace && m != nullptr) { tty->print("ZZZ mem"); m->dump(); }); 1952 1953 if (!mem_is_valid(m, u) || !m->is_Phi()) { 1954 bool push = true; 1955 bool create_phi = true; 1956 if (_phase->is_dominator(new_ctrl, u)) { 1957 create_phi = false; 1958 } 1959 if (create_phi) { 1960 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias)); 1961 _phase->register_new_node(phi, u); 1962 phis.push(phi); 1963 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 1964 if (!mem_is_valid(m, u)) { 1965 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 1966 _memory_nodes.map(u->_idx, phi); 1967 } else { 1968 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 1969 for (;;) { 1970 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); 1971 Node* next = nullptr; 1972 if (m->is_Proj()) { 1973 next = m->in(0); 1974 } else { 1975 assert(m->is_Mem() || m->is_LoadStore(), ""); 1976 assert(_alias == Compile::AliasIdxRaw, ""); 1977 next = m->in(MemNode::Memory); 1978 } 1979 if (_phase->get_ctrl(next) != u) { 1980 break; 1981 } 1982 if (next->is_MergeMem()) { 1983 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, ""); 1984 break; 1985 } 1986 if (next->is_Phi()) { 1987 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 1988 break; 1989 } 1990 m = next; 1991 } 1992 1993 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 1994 assert(m->is_Mem() || m->is_LoadStore(), ""); 1995 uint input = (uint)MemNode::Memory; 1996 _phase->igvn().replace_input_of(m, input, phi); 1997 push = false; 1998 } 1999 } else { 2000 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 2001 } 2002 if (push) { 2003 uses.push(u); 2004 } 2005 } 2006 } else if (!mem_is_valid(m, u) && 2007 !(u->Opcode() == Op_CProj && u->in(0)->is_NeverBranch() && u->as_Proj()->_con == 1)) { 2008 uses.push(u); 2009 } 2010 } 2011 } 2012 } 2013 for (int i = 0; i < phis.length(); i++) { 2014 Node* n = phis.at(i); 2015 Node* r = n->in(0); 2016 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 2017 for (uint j = 1; j < n->req(); j++) { 2018 Node* m = find_mem(r->in(j), nullptr); 2019 _phase->igvn().replace_input_of(n, j, m); 2020 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 2021 } 2022 } 2023 } 2024 uint last = _phase->C->unique(); 2025 MergeMemNode* mm = nullptr; 2026 int alias = _alias; 2027 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); }); 2028 // Process loads first to not miss an anti-dependency: if the memory 2029 // edge of a store is updated before a load is processed then an 2030 // anti-dependency may be missed. 2031 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2032 Node* u = mem->out(i); 2033 if (u->_idx < last && u->is_Load() && _phase->C->get_alias_index(u->adr_type()) == alias) { 2034 Node* m = find_mem(_phase->get_ctrl(u), u); 2035 if (m != mem) { 2036 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2037 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2038 --i; 2039 } 2040 } 2041 } 2042 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2043 Node* u = mem->out(i); 2044 if (u->_idx < last) { 2045 if (u->is_Mem()) { 2046 if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2047 Node* m = find_mem(_phase->get_ctrl(u), u); 2048 if (m != mem) { 2049 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2050 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2051 --i; 2052 } 2053 } 2054 } else if (u->is_MergeMem()) { 2055 MergeMemNode* u_mm = u->as_MergeMem(); 2056 if (u_mm->memory_at(alias) == mem) { 2057 MergeMemNode* newmm = nullptr; 2058 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2059 Node* uu = u->fast_out(j); 2060 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2061 if (uu->is_Phi()) { 2062 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 2063 Node* region = uu->in(0); 2064 int nb = 0; 2065 for (uint k = 1; k < uu->req(); k++) { 2066 if (uu->in(k) == u) { 2067 Node* m = find_mem(region->in(k), nullptr); 2068 if (m != mem) { 2069 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 2070 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2071 if (newmm != u) { 2072 _phase->igvn().replace_input_of(uu, k, newmm); 2073 nb++; 2074 --jmax; 2075 } 2076 } 2077 } 2078 } 2079 if (nb > 0) { 2080 --j; 2081 } 2082 } else { 2083 Node* m = find_mem(_phase->ctrl_or_self(uu), uu); 2084 if (m != mem) { 2085 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 2086 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2087 if (newmm != u) { 2088 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2089 --j, --jmax; 2090 } 2091 } 2092 } 2093 } 2094 } 2095 } else if (u->is_Phi()) { 2096 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2097 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) { 2098 Node* region = u->in(0); 2099 bool replaced = false; 2100 for (uint j = 1; j < u->req(); j++) { 2101 if (u->in(j) == mem) { 2102 Node* m = find_mem(region->in(j), nullptr); 2103 Node* nnew = m; 2104 if (m != mem) { 2105 if (u->adr_type() == TypePtr::BOTTOM) { 2106 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m)); 2107 nnew = mm; 2108 } 2109 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 2110 _phase->igvn().replace_input_of(u, j, nnew); 2111 replaced = true; 2112 } 2113 } 2114 } 2115 if (replaced) { 2116 --i; 2117 } 2118 } 2119 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2120 u->adr_type() == nullptr) { 2121 assert(u->adr_type() != nullptr || 2122 u->Opcode() == Op_Rethrow || 2123 u->Opcode() == Op_Return || 2124 u->Opcode() == Op_SafePoint || 2125 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2126 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2127 u->Opcode() == Op_CallLeaf, ""); 2128 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2129 if (m != mem) { 2130 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m)); 2131 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2132 --i; 2133 } 2134 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2135 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2136 if (m != mem) { 2137 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2138 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2139 --i; 2140 } 2141 } else if (u->adr_type() != TypePtr::BOTTOM && 2142 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) { 2143 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2144 assert(m != mem, ""); 2145 // u is on the wrong slice... 2146 assert(u->is_ClearArray(), ""); 2147 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2148 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2149 --i; 2150 } 2151 } 2152 } 2153 #ifdef ASSERT 2154 assert(new_mem->outcnt() > 0, ""); 2155 for (int i = 0; i < phis.length(); i++) { 2156 Node* n = phis.at(i); 2157 assert(n->outcnt() > 0, "new phi must have uses now"); 2158 } 2159 #endif 2160 } 2161 2162 void MemoryGraphFixer::record_new_ctrl(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl) { 2163 if (mem_for_ctrl != mem && new_ctrl != ctrl) { 2164 _memory_nodes.map(ctrl->_idx, mem); 2165 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 2166 } 2167 } 2168 2169 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { 2170 MergeMemNode* mm = MergeMemNode::make(mem); 2171 mm->set_memory_at(_alias, rep_proj); 2172 _phase->register_new_node(mm, rep_ctrl); 2173 return mm; 2174 } 2175 2176 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const { 2177 MergeMemNode* newmm = nullptr; 2178 MergeMemNode* u_mm = u->as_MergeMem(); 2179 Node* c = _phase->get_ctrl(u); 2180 if (_phase->is_dominator(c, rep_ctrl)) { 2181 c = rep_ctrl; 2182 } else { 2183 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 2184 } 2185 if (u->outcnt() == 1) { 2186 if (u->req() > (uint)_alias && u->in(_alias) == mem) { 2187 _phase->igvn().replace_input_of(u, _alias, rep_proj); 2188 --i; 2189 } else { 2190 _phase->igvn().rehash_node_delayed(u); 2191 u_mm->set_memory_at(_alias, rep_proj); 2192 } 2193 newmm = u_mm; 2194 _phase->set_ctrl_and_loop(u, c); 2195 } else { 2196 // can't simply clone u and then change one of its input because 2197 // it adds and then removes an edge which messes with the 2198 // DUIterator 2199 newmm = MergeMemNode::make(u_mm->base_memory()); 2200 for (uint j = 0; j < u->req(); j++) { 2201 if (j < newmm->req()) { 2202 if (j == (uint)_alias) { 2203 newmm->set_req(j, rep_proj); 2204 } else if (newmm->in(j) != u->in(j)) { 2205 newmm->set_req(j, u->in(j)); 2206 } 2207 } else if (j == (uint)_alias) { 2208 newmm->add_req(rep_proj); 2209 } else { 2210 newmm->add_req(u->in(j)); 2211 } 2212 } 2213 if ((uint)_alias >= u->req()) { 2214 newmm->set_memory_at(_alias, rep_proj); 2215 } 2216 _phase->register_new_node(newmm, c); 2217 } 2218 return newmm; 2219 } 2220 2221 bool MemoryGraphFixer::should_process_phi(Node* phi) const { 2222 if (phi->adr_type() == TypePtr::BOTTOM) { 2223 Node* region = phi->in(0); 2224 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 2225 Node* uu = region->fast_out(j); 2226 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) { 2227 return false; 2228 } 2229 } 2230 return true; 2231 } 2232 return _phase->C->get_alias_index(phi->adr_type()) == _alias; 2233 } 2234 2235 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const { 2236 uint last = _phase-> C->unique(); 2237 MergeMemNode* mm = nullptr; 2238 assert(mem->bottom_type() == Type::MEMORY, ""); 2239 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2240 Node* u = mem->out(i); 2241 if (u != replacement && u->_idx < last) { 2242 if (u->is_MergeMem()) { 2243 MergeMemNode* u_mm = u->as_MergeMem(); 2244 if (u_mm->memory_at(_alias) == mem) { 2245 MergeMemNode* newmm = nullptr; 2246 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2247 Node* uu = u->fast_out(j); 2248 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2249 if (uu->is_Phi()) { 2250 if (should_process_phi(uu)) { 2251 Node* region = uu->in(0); 2252 int nb = 0; 2253 for (uint k = 1; k < uu->req(); k++) { 2254 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) { 2255 if (newmm == nullptr) { 2256 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2257 } 2258 if (newmm != u) { 2259 _phase->igvn().replace_input_of(uu, k, newmm); 2260 nb++; 2261 --jmax; 2262 } 2263 } 2264 } 2265 if (nb > 0) { 2266 --j; 2267 } 2268 } 2269 } else { 2270 if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { 2271 if (newmm == nullptr) { 2272 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2273 } 2274 if (newmm != u) { 2275 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2276 --j, --jmax; 2277 } 2278 } 2279 } 2280 } 2281 } 2282 } else if (u->is_Phi()) { 2283 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2284 Node* region = u->in(0); 2285 if (should_process_phi(u)) { 2286 bool replaced = false; 2287 for (uint j = 1; j < u->req(); j++) { 2288 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) { 2289 Node* nnew = rep_proj; 2290 if (u->adr_type() == TypePtr::BOTTOM) { 2291 if (mm == nullptr) { 2292 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2293 } 2294 nnew = mm; 2295 } 2296 _phase->igvn().replace_input_of(u, j, nnew); 2297 replaced = true; 2298 } 2299 } 2300 if (replaced) { 2301 --i; 2302 } 2303 2304 } 2305 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2306 u->adr_type() == nullptr) { 2307 assert(u->adr_type() != nullptr || 2308 u->Opcode() == Op_Rethrow || 2309 u->Opcode() == Op_Return || 2310 u->Opcode() == Op_SafePoint || 2311 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2312 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2313 u->Opcode() == Op_CallLeaf, "%s", u->Name()); 2314 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2315 if (mm == nullptr) { 2316 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2317 } 2318 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2319 --i; 2320 } 2321 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2322 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2323 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 2324 --i; 2325 } 2326 } 2327 } 2328 } 2329 } 2330 2331 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, DecoratorSet decorators) 2332 : Node(ctrl, obj), _decorators(decorators) { 2333 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); 2334 } 2335 2336 DecoratorSet ShenandoahLoadReferenceBarrierNode::decorators() const { 2337 return _decorators; 2338 } 2339 2340 uint ShenandoahLoadReferenceBarrierNode::size_of() const { 2341 return sizeof(*this); 2342 } 2343 2344 static DecoratorSet mask_decorators(DecoratorSet decorators) { 2345 return decorators & (ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF | IN_NATIVE); 2346 } 2347 2348 uint ShenandoahLoadReferenceBarrierNode::hash() const { 2349 uint hash = Node::hash(); 2350 hash += mask_decorators(_decorators); 2351 return hash; 2352 } 2353 2354 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { 2355 return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && 2356 mask_decorators(_decorators) == mask_decorators(((const ShenandoahLoadReferenceBarrierNode&)n)._decorators); 2357 } 2358 2359 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { 2360 if (in(ValueIn) == nullptr || in(ValueIn)->is_top()) { 2361 return Type::TOP; 2362 } 2363 const Type* t = in(ValueIn)->bottom_type(); 2364 if (t == TypePtr::NULL_PTR) { 2365 return t; 2366 } 2367 2368 if (ShenandoahBarrierSet::is_strong_access(decorators())) { 2369 return t; 2370 } 2371 2372 return t->meet(TypePtr::NULL_PTR); 2373 } 2374 2375 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { 2376 // Either input is TOP ==> the result is TOP 2377 const Type *t2 = phase->type(in(ValueIn)); 2378 if( t2 == Type::TOP ) return Type::TOP; 2379 2380 if (t2 == TypePtr::NULL_PTR) { 2381 return t2; 2382 } 2383 2384 if (ShenandoahBarrierSet::is_strong_access(decorators())) { 2385 return t2; 2386 } 2387 2388 return t2->meet(TypePtr::NULL_PTR); 2389 } 2390 2391 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { 2392 Node* value = in(ValueIn); 2393 if (!needs_barrier(phase, value)) { 2394 return value; 2395 } 2396 return this; 2397 } 2398 2399 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { 2400 Unique_Node_List visited; 2401 return needs_barrier_impl(phase, n, visited); 2402 } 2403 2404 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { 2405 if (n == nullptr) return false; 2406 if (visited.member(n)) { 2407 return false; // Been there. 2408 } 2409 visited.push(n); 2410 2411 if (n->is_Allocate()) { 2412 // tty->print_cr("optimize barrier on alloc"); 2413 return false; 2414 } 2415 if (n->is_Call()) { 2416 // tty->print_cr("optimize barrier on call"); 2417 return false; 2418 } 2419 2420 const Type* type = phase->type(n); 2421 if (type == Type::TOP) { 2422 return false; 2423 } 2424 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 2425 // tty->print_cr("optimize barrier on null"); 2426 return false; 2427 } 2428 if (type->make_oopptr() && type->make_oopptr()->const_oop() != nullptr) { 2429 // tty->print_cr("optimize barrier on constant"); 2430 return false; 2431 } 2432 2433 switch (n->Opcode()) { 2434 case Op_AddP: 2435 return true; // TODO: Can refine? 2436 case Op_LoadP: 2437 case Op_ShenandoahCompareAndExchangeN: 2438 case Op_ShenandoahCompareAndExchangeP: 2439 case Op_CompareAndExchangeN: 2440 case Op_CompareAndExchangeP: 2441 case Op_GetAndSetN: 2442 case Op_GetAndSetP: 2443 return true; 2444 case Op_Phi: { 2445 for (uint i = 1; i < n->req(); i++) { 2446 if (needs_barrier_impl(phase, n->in(i), visited)) return true; 2447 } 2448 return false; 2449 } 2450 case Op_CheckCastPP: 2451 case Op_CastPP: 2452 return needs_barrier_impl(phase, n->in(1), visited); 2453 case Op_Proj: 2454 return needs_barrier_impl(phase, n->in(0), visited); 2455 case Op_ShenandoahLoadReferenceBarrier: 2456 // tty->print_cr("optimize barrier on barrier"); 2457 return false; 2458 case Op_Parm: 2459 // tty->print_cr("optimize barrier on input arg"); 2460 return false; 2461 case Op_DecodeN: 2462 case Op_EncodeP: 2463 return needs_barrier_impl(phase, n->in(1), visited); 2464 case Op_LoadN: 2465 return true; 2466 case Op_CMoveN: 2467 case Op_CMoveP: 2468 return needs_barrier_impl(phase, n->in(2), visited) || 2469 needs_barrier_impl(phase, n->in(3), visited); 2470 case Op_CreateEx: 2471 return false; 2472 default: 2473 break; 2474 } 2475 #ifdef ASSERT 2476 tty->print("need barrier on?: "); 2477 tty->print_cr("ins:"); 2478 n->dump(2); 2479 tty->print_cr("outs:"); 2480 n->dump(-2); 2481 ShouldNotReachHere(); 2482 #endif 2483 return true; 2484 }