1 /* 2 * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 29 #include "classfile/javaClasses.hpp" 30 #include "gc/shenandoah/c2/shenandoahSupport.hpp" 31 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 33 #include "gc/shenandoah/shenandoahForwarding.hpp" 34 #include "gc/shenandoah/shenandoahHeap.hpp" 35 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 36 #include "gc/shenandoah/shenandoahRuntime.hpp" 37 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 38 #include "opto/arraycopynode.hpp" 39 #include "opto/block.hpp" 40 #include "opto/callnode.hpp" 41 #include "opto/castnode.hpp" 42 #include "opto/movenode.hpp" 43 #include "opto/phaseX.hpp" 44 #include "opto/rootnode.hpp" 45 #include "opto/runtime.hpp" 46 #include "opto/subnode.hpp" 47 48 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { 49 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 50 if ((state->iu_barriers_count() + 51 state->load_reference_barriers_count()) > 0) { 52 assert(C->post_loop_opts_phase(), "no loop opts allowed"); 53 C->reset_post_loop_opts_phase(); // ... but we know what we are doing 54 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; 55 C->clear_major_progress(); 56 PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand); 57 if (C->failing()) return false; 58 PhaseIdealLoop::verify(igvn); 59 if (attempt_more_loopopts) { 60 C->set_major_progress(); 61 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { 62 return false; 63 } 64 C->clear_major_progress(); 65 66 C->process_for_post_loop_opts_igvn(igvn); 67 } 68 C->set_post_loop_opts_phase(); // now for real! 69 } 70 return true; 71 } 72 73 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) { 74 if (!UseShenandoahGC) { 75 return false; 76 } 77 assert(iff->is_If(), "bad input"); 78 if (iff->Opcode() != Op_If) { 79 return false; 80 } 81 Node* bol = iff->in(1); 82 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 83 return false; 84 } 85 Node* cmp = bol->in(1); 86 if (cmp->Opcode() != Op_CmpI) { 87 return false; 88 } 89 Node* in1 = cmp->in(1); 90 Node* in2 = cmp->in(2); 91 if (in2->find_int_con(-1) != 0) { 92 return false; 93 } 94 if (in1->Opcode() != Op_AndI) { 95 return false; 96 } 97 in2 = in1->in(2); 98 if (in2->find_int_con(-1) != mask) { 99 return false; 100 } 101 in1 = in1->in(1); 102 103 return is_gc_state_load(in1); 104 } 105 106 bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { 107 return is_gc_state_test(iff, ShenandoahHeap::HAS_FORWARDED); 108 } 109 110 bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { 111 if (!UseShenandoahGC) { 112 return false; 113 } 114 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) { 115 return false; 116 } 117 Node* addp = n->in(MemNode::Address); 118 if (!addp->is_AddP()) { 119 return false; 120 } 121 Node* base = addp->in(AddPNode::Address); 122 Node* off = addp->in(AddPNode::Offset); 123 if (base->Opcode() != Op_ThreadLocal) { 124 return false; 125 } 126 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 127 return false; 128 } 129 return true; 130 } 131 132 bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { 133 assert(phase->is_dominator(stop, start), "bad inputs"); 134 ResourceMark rm; 135 Unique_Node_List wq; 136 wq.push(start); 137 for (uint next = 0; next < wq.size(); next++) { 138 Node *m = wq.at(next); 139 if (m == stop) { 140 continue; 141 } 142 if (m->is_SafePoint() && !m->is_CallLeaf()) { 143 return true; 144 } 145 if (m->is_Region()) { 146 for (uint i = 1; i < m->req(); i++) { 147 wq.push(m->in(i)); 148 } 149 } else { 150 wq.push(m->in(0)); 151 } 152 } 153 return false; 154 } 155 156 #ifdef ASSERT 157 bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 158 assert(phis.size() == 0, ""); 159 160 while (true) { 161 if (in->bottom_type() == TypePtr::NULL_PTR) { 162 if (trace) {tty->print_cr("null");} 163 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) { 164 if (trace) {tty->print_cr("Non oop");} 165 } else { 166 if (in->is_ConstraintCast()) { 167 in = in->in(1); 168 continue; 169 } else if (in->is_AddP()) { 170 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 171 in = in->in(AddPNode::Address); 172 continue; 173 } else if (in->is_Con()) { 174 if (trace) { 175 tty->print("Found constant"); 176 in->dump(); 177 } 178 } else if (in->Opcode() == Op_Parm) { 179 if (trace) { 180 tty->print("Found argument"); 181 } 182 } else if (in->Opcode() == Op_CreateEx) { 183 if (trace) { 184 tty->print("Found create-exception"); 185 } 186 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { 187 if (trace) { 188 tty->print("Found raw LoadP (OSR argument?)"); 189 } 190 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 191 if (t == ShenandoahOopStore) { 192 uint i = 0; 193 for (; i < phis.size(); i++) { 194 Node* n = phis.node_at(i); 195 if (n->Opcode() == Op_ShenandoahIUBarrier) { 196 break; 197 } 198 } 199 if (i == phis.size()) { 200 return false; 201 } 202 } 203 barriers_used.push(in); 204 if (trace) {tty->print("Found barrier"); in->dump();} 205 } else if (in->Opcode() == Op_ShenandoahIUBarrier) { 206 if (t != ShenandoahOopStore) { 207 in = in->in(1); 208 continue; 209 } 210 if (trace) {tty->print("Found enqueue barrier"); in->dump();} 211 phis.push(in, in->req()); 212 in = in->in(1); 213 continue; 214 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 215 if (trace) { 216 tty->print("Found alloc"); 217 in->in(0)->dump(); 218 } 219 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { 220 if (trace) { 221 tty->print("Found Java call"); 222 } 223 } else if (in->is_Phi()) { 224 if (!visited.test_set(in->_idx)) { 225 if (trace) {tty->print("Pushed phi:"); in->dump();} 226 phis.push(in, 2); 227 in = in->in(1); 228 continue; 229 } 230 if (trace) {tty->print("Already seen phi:"); in->dump();} 231 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 232 if (!visited.test_set(in->_idx)) { 233 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 234 phis.push(in, CMoveNode::IfTrue); 235 in = in->in(CMoveNode::IfFalse); 236 continue; 237 } 238 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 239 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 240 in = in->in(1); 241 continue; 242 } else { 243 return false; 244 } 245 } 246 bool cont = false; 247 while (phis.is_nonempty()) { 248 uint idx = phis.index(); 249 Node* phi = phis.node(); 250 if (idx >= phi->req()) { 251 if (trace) {tty->print("Popped phi:"); phi->dump();} 252 phis.pop(); 253 continue; 254 } 255 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 256 in = phi->in(idx); 257 phis.set_index(idx+1); 258 cont = true; 259 break; 260 } 261 if (!cont) { 262 break; 263 } 264 } 265 return true; 266 } 267 268 void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { 269 if (n1 != nullptr) { 270 n1->dump(+10); 271 } 272 if (n2 != nullptr) { 273 n2->dump(+10); 274 } 275 fatal("%s", msg); 276 } 277 278 void ShenandoahBarrierC2Support::verify(RootNode* root) { 279 ResourceMark rm; 280 Unique_Node_List wq; 281 GrowableArray<Node*> barriers; 282 Unique_Node_List barriers_used; 283 Node_Stack phis(0); 284 VectorSet visited; 285 const bool trace = false; 286 const bool verify_no_useless_barrier = false; 287 288 wq.push(root); 289 for (uint next = 0; next < wq.size(); next++) { 290 Node *n = wq.at(next); 291 if (n->is_Load()) { 292 const bool trace = false; 293 if (trace) {tty->print("Verifying"); n->dump();} 294 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 295 if (trace) {tty->print_cr("Load range/klass");} 296 } else { 297 const TypePtr* adr_type = n->as_Load()->adr_type(); 298 299 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 300 if (trace) {tty->print_cr("Mark load");} 301 } else if (adr_type->isa_instptr() && 302 adr_type->is_instptr()->instance_klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 303 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset()) { 304 if (trace) {tty->print_cr("Reference.get()");} 305 } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 306 report_verify_failure("Shenandoah verification: Load should have barriers", n); 307 } 308 } 309 } else if (n->is_Store()) { 310 const bool trace = false; 311 312 if (trace) {tty->print("Verifying"); n->dump();} 313 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 314 Node* adr = n->in(MemNode::Address); 315 bool verify = true; 316 317 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 318 adr = adr->in(AddPNode::Address); 319 if (adr->is_AddP()) { 320 assert(adr->in(AddPNode::Base)->is_top(), ""); 321 adr = adr->in(AddPNode::Address); 322 if (adr->Opcode() == Op_LoadP && 323 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 324 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 325 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 326 if (trace) {tty->print_cr("SATB prebarrier");} 327 verify = false; 328 } 329 } 330 } 331 332 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 333 report_verify_failure("Shenandoah verification: Store should have barriers", n); 334 } 335 } 336 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 337 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 338 } 339 } else if (n->Opcode() == Op_CmpP) { 340 const bool trace = false; 341 342 Node* in1 = n->in(1); 343 Node* in2 = n->in(2); 344 if (in1->bottom_type()->isa_oopptr()) { 345 if (trace) {tty->print("Verifying"); n->dump();} 346 347 bool mark_inputs = false; 348 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 349 (in1->is_Con() || in2->is_Con())) { 350 if (trace) {tty->print_cr("Comparison against a constant");} 351 mark_inputs = true; 352 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 353 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 354 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 355 mark_inputs = true; 356 } else { 357 assert(in2->bottom_type()->isa_oopptr(), ""); 358 359 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 360 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 361 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 362 } 363 } 364 if (verify_no_useless_barrier && 365 mark_inputs && 366 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 367 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 368 phis.clear(); 369 visited.reset(); 370 } 371 } 372 } else if (n->is_LoadStore()) { 373 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && 374 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 375 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 376 } 377 378 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 379 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 380 } 381 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 382 CallNode* call = n->as_Call(); 383 384 static struct { 385 const char* name; 386 struct { 387 int pos; 388 verify_type t; 389 } args[6]; 390 } calls[] = { 391 "aescrypt_encryptBlock", 392 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 393 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 394 "aescrypt_decryptBlock", 395 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 396 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 397 "multiplyToLen", 398 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 399 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 400 "squareToLen", 401 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 402 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 403 "montgomery_multiply", 404 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 405 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 406 "montgomery_square", 407 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 408 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 409 "mulAdd", 410 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 411 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 412 "vectorizedMismatch", 413 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 414 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 415 "updateBytesCRC32", 416 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 417 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 418 "updateBytesAdler32", 419 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 420 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 421 "updateBytesCRC32C", 422 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 423 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 424 "counterMode_AESCrypt", 425 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 426 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 427 "cipherBlockChaining_encryptAESCrypt", 428 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 429 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 430 "cipherBlockChaining_decryptAESCrypt", 431 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 432 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 433 "shenandoah_clone_barrier", 434 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 435 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 436 "ghash_processBlocks", 437 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 438 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 439 "sha1_implCompress", 440 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 441 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 442 "sha256_implCompress", 443 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 444 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 445 "sha512_implCompress", 446 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 447 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 448 "sha1_implCompressMB", 449 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 451 "sha256_implCompressMB", 452 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 453 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 454 "sha512_implCompressMB", 455 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 456 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 457 "encodeBlock", 458 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 459 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 460 "decodeBlock", 461 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 462 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 463 }; 464 465 if (call->is_call_to_arraycopystub()) { 466 Node* dest = nullptr; 467 const TypeTuple* args = n->as_Call()->_tf->domain(); 468 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 469 if (args->field_at(i)->isa_ptr()) { 470 j++; 471 if (j == 2) { 472 dest = n->in(i); 473 break; 474 } 475 } 476 } 477 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 478 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 479 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 480 } 481 } else if (strlen(call->_name) > 5 && 482 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 483 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 484 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 485 } 486 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { 487 // skip 488 } else { 489 const int calls_len = sizeof(calls) / sizeof(calls[0]); 490 int i = 0; 491 for (; i < calls_len; i++) { 492 if (!strcmp(calls[i].name, call->_name)) { 493 break; 494 } 495 } 496 if (i != calls_len) { 497 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 498 for (uint j = 0; j < args_len; j++) { 499 int pos = calls[i].args[j].pos; 500 if (pos == -1) { 501 break; 502 } 503 if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 504 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 505 } 506 } 507 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 508 if (call->in(j)->bottom_type()->make_ptr() && 509 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 510 uint k = 0; 511 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 512 if (k == args_len) { 513 fatal("arg %d for call %s not covered", j, call->_name); 514 } 515 } 516 } 517 } else { 518 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 519 if (call->in(j)->bottom_type()->make_ptr() && 520 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 521 fatal("%s not covered", call->_name); 522 } 523 } 524 } 525 } 526 } else if (n->Opcode() == Op_ShenandoahIUBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 527 // skip 528 } else if (n->is_AddP() 529 || n->is_Phi() 530 || n->is_ConstraintCast() 531 || n->Opcode() == Op_Return 532 || n->Opcode() == Op_CMoveP 533 || n->Opcode() == Op_CMoveN 534 || n->Opcode() == Op_Rethrow 535 || n->is_MemBar() 536 || n->Opcode() == Op_Conv2B 537 || n->Opcode() == Op_SafePoint 538 || n->is_CallJava() 539 || n->Opcode() == Op_Unlock 540 || n->Opcode() == Op_EncodeP 541 || n->Opcode() == Op_DecodeN) { 542 // nothing to do 543 } else { 544 static struct { 545 int opcode; 546 struct { 547 int pos; 548 verify_type t; 549 } inputs[2]; 550 } others[] = { 551 Op_FastLock, 552 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 553 Op_Lock, 554 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 555 Op_ArrayCopy, 556 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 557 Op_StrCompressedCopy, 558 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 559 Op_StrInflatedCopy, 560 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 561 Op_AryEq, 562 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 563 Op_StrIndexOf, 564 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 565 Op_StrComp, 566 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 567 Op_StrEquals, 568 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 569 Op_VectorizedHashCode, 570 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 571 Op_EncodeISOArray, 572 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 573 Op_CountPositives, 574 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 575 Op_CastP2X, 576 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 577 Op_StrIndexOfChar, 578 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 579 }; 580 581 const int others_len = sizeof(others) / sizeof(others[0]); 582 int i = 0; 583 for (; i < others_len; i++) { 584 if (others[i].opcode == n->Opcode()) { 585 break; 586 } 587 } 588 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 589 if (i != others_len) { 590 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 591 for (uint j = 0; j < inputs_len; j++) { 592 int pos = others[i].inputs[j].pos; 593 if (pos == -1) { 594 break; 595 } 596 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 597 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 598 } 599 } 600 for (uint j = 1; j < stop; j++) { 601 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() && 602 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 603 uint k = 0; 604 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 605 if (k == inputs_len) { 606 fatal("arg %d for node %s not covered", j, n->Name()); 607 } 608 } 609 } 610 } else { 611 for (uint j = 1; j < stop; j++) { 612 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() && 613 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 614 fatal("%s not covered", n->Name()); 615 } 616 } 617 } 618 } 619 620 if (n->is_SafePoint()) { 621 SafePointNode* sfpt = n->as_SafePoint(); 622 if (verify_no_useless_barrier && sfpt->jvms() != nullptr) { 623 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 624 if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 625 phis.clear(); 626 visited.reset(); 627 } 628 } 629 } 630 } 631 } 632 633 if (verify_no_useless_barrier) { 634 for (int i = 0; i < barriers.length(); i++) { 635 Node* n = barriers.at(i); 636 if (!barriers_used.member(n)) { 637 tty->print("XXX useless barrier"); n->dump(-2); 638 ShouldNotReachHere(); 639 } 640 } 641 } 642 } 643 #endif 644 645 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { 646 // That both nodes have the same control is not sufficient to prove 647 // domination, verify that there's no path from d to n 648 ResourceMark rm; 649 Unique_Node_List wq; 650 wq.push(d); 651 for (uint next = 0; next < wq.size(); next++) { 652 Node *m = wq.at(next); 653 if (m == n) { 654 return false; 655 } 656 if (m->is_Phi() && m->in(0)->is_Loop()) { 657 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 658 } else { 659 if (m->is_Store() || m->is_LoadStore()) { 660 // Take anti-dependencies into account 661 Node* mem = m->in(MemNode::Memory); 662 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 663 Node* u = mem->fast_out(i); 664 if (u->is_Load() && phase->C->can_alias(m->adr_type(), phase->C->get_alias_index(u->adr_type())) && 665 phase->ctrl_or_self(u) == c) { 666 wq.push(u); 667 } 668 } 669 } 670 for (uint i = 0; i < m->req(); i++) { 671 if (m->in(i) != nullptr && phase->ctrl_or_self(m->in(i)) == c) { 672 wq.push(m->in(i)); 673 } 674 } 675 } 676 } 677 return true; 678 } 679 680 bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 681 if (d_c != n_c) { 682 return phase->is_dominator(d_c, n_c); 683 } 684 return is_dominator_same_ctrl(d_c, d, n, phase); 685 } 686 687 Node* next_mem(Node* mem, int alias) { 688 Node* res = nullptr; 689 if (mem->is_Proj()) { 690 res = mem->in(0); 691 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 692 res = mem->in(TypeFunc::Memory); 693 } else if (mem->is_Phi()) { 694 res = mem->in(1); 695 } else if (mem->is_MergeMem()) { 696 res = mem->as_MergeMem()->memory_at(alias); 697 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 698 assert(alias == Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 699 res = mem->in(MemNode::Memory); 700 } else { 701 #ifdef ASSERT 702 mem->dump(); 703 #endif 704 ShouldNotReachHere(); 705 } 706 return res; 707 } 708 709 Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 710 Node* iffproj = nullptr; 711 while (c != dom) { 712 Node* next = phase->idom(c); 713 assert(next->unique_ctrl_out_or_null() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 714 if (c->is_Region()) { 715 ResourceMark rm; 716 Unique_Node_List wq; 717 wq.push(c); 718 for (uint i = 0; i < wq.size(); i++) { 719 Node *n = wq.at(i); 720 if (n == next) { 721 continue; 722 } 723 if (n->is_Region()) { 724 for (uint j = 1; j < n->req(); j++) { 725 wq.push(n->in(j)); 726 } 727 } else { 728 wq.push(n->in(0)); 729 } 730 } 731 for (uint i = 0; i < wq.size(); i++) { 732 Node *n = wq.at(i); 733 assert(n->is_CFG(), ""); 734 if (n->is_Multi()) { 735 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 736 Node* u = n->fast_out(j); 737 if (u->is_CFG()) { 738 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj()) { 739 return NodeSentinel; 740 } 741 } 742 } 743 } 744 } 745 } else if (c->is_Proj()) { 746 if (c->is_IfProj()) { 747 if (c->as_Proj()->is_uncommon_trap_if_pattern() != nullptr) { 748 // continue; 749 } else { 750 if (!allow_one_proj) { 751 return NodeSentinel; 752 } 753 if (iffproj == nullptr) { 754 iffproj = c; 755 } else { 756 return NodeSentinel; 757 } 758 } 759 } else if (c->Opcode() == Op_JumpProj) { 760 return NodeSentinel; // unsupported 761 } else if (c->Opcode() == Op_CatchProj) { 762 return NodeSentinel; // unsupported 763 } else if (c->Opcode() == Op_CProj && next->is_NeverBranch()) { 764 return NodeSentinel; // unsupported 765 } else { 766 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 767 } 768 } 769 c = next; 770 } 771 return iffproj; 772 } 773 774 Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 775 ResourceMark rm; 776 VectorSet wq; 777 wq.set(mem->_idx); 778 mem_ctrl = phase->ctrl_or_self(mem); 779 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 780 mem = next_mem(mem, alias); 781 if (wq.test_set(mem->_idx)) { 782 return nullptr; 783 } 784 mem_ctrl = phase->ctrl_or_self(mem); 785 } 786 if (mem->is_MergeMem()) { 787 mem = mem->as_MergeMem()->memory_at(alias); 788 mem_ctrl = phase->ctrl_or_self(mem); 789 } 790 return mem; 791 } 792 793 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 794 Node* mem = nullptr; 795 Node* c = ctrl; 796 do { 797 if (c->is_Region()) { 798 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) { 799 Node* u = c->fast_out(i); 800 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 801 if (u->adr_type() == TypePtr::BOTTOM) { 802 mem = u; 803 } 804 } 805 } 806 } else { 807 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) { 808 CallProjections projs; 809 c->as_Call()->extract_projections(&projs, true, false); 810 if (projs.fallthrough_memproj != nullptr) { 811 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 812 if (projs.catchall_memproj == nullptr) { 813 mem = projs.fallthrough_memproj; 814 } else { 815 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 816 mem = projs.fallthrough_memproj; 817 } else { 818 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 819 mem = projs.catchall_memproj; 820 } 821 } 822 } 823 } else { 824 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 825 if (proj != nullptr && 826 proj->adr_type() == TypePtr::BOTTOM) { 827 mem = proj; 828 } 829 } 830 } else { 831 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 832 Node* u = c->fast_out(i); 833 if (u->is_Proj() && 834 u->bottom_type() == Type::MEMORY && 835 u->adr_type() == TypePtr::BOTTOM) { 836 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 837 assert(mem == nullptr, "only one proj"); 838 mem = u; 839 } 840 } 841 assert(!c->is_Call() || c->as_Call()->adr_type() != nullptr || mem == nullptr, "no mem projection expected"); 842 } 843 } 844 c = phase->idom(c); 845 } while (mem == nullptr); 846 return mem; 847 } 848 849 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 850 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 851 Node* u = n->fast_out(i); 852 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 853 uses.push(u); 854 } 855 } 856 } 857 858 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) { 859 OuterStripMinedLoopEndNode* le = inner->outer_loop_end(); 860 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 861 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 862 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 863 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 864 phase->lazy_replace(outer, new_outer); 865 phase->lazy_replace(le, new_le); 866 inner->clear_strip_mined(); 867 } 868 869 void ShenandoahBarrierC2Support::test_gc_state(Node*& ctrl, Node* raw_mem, Node*& test_fail_ctrl, 870 PhaseIdealLoop* phase, int flags) { 871 PhaseIterGVN& igvn = phase->igvn(); 872 Node* old_ctrl = ctrl; 873 874 Node* thread = new ThreadLocalNode(); 875 Node* gc_state_offset = igvn.MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 876 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, gc_state_offset); 877 Node* gc_state = new LoadBNode(old_ctrl, raw_mem, gc_state_addr, 878 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), 879 TypeInt::BYTE, MemNode::unordered); 880 Node* gc_state_and = new AndINode(gc_state, igvn.intcon(flags)); 881 Node* gc_state_cmp = new CmpINode(gc_state_and, igvn.zerocon(T_INT)); 882 Node* gc_state_bool = new BoolNode(gc_state_cmp, BoolTest::ne); 883 884 IfNode* gc_state_iff = new IfNode(old_ctrl, gc_state_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 885 ctrl = new IfTrueNode(gc_state_iff); 886 test_fail_ctrl = new IfFalseNode(gc_state_iff); 887 888 IdealLoopTree* loop = phase->get_loop(old_ctrl); 889 phase->register_control(gc_state_iff, loop, old_ctrl); 890 phase->register_control(ctrl, loop, gc_state_iff); 891 phase->register_control(test_fail_ctrl, loop, gc_state_iff); 892 893 phase->register_new_node(thread, old_ctrl); 894 phase->register_new_node(gc_state_addr, old_ctrl); 895 phase->register_new_node(gc_state, old_ctrl); 896 phase->register_new_node(gc_state_and, old_ctrl); 897 phase->register_new_node(gc_state_cmp, old_ctrl); 898 phase->register_new_node(gc_state_bool, old_ctrl); 899 900 phase->set_ctrl(gc_state_offset, phase->C->root()); 901 902 assert(is_gc_state_test(gc_state_iff, flags), "Should match the shape"); 903 } 904 905 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { 906 Node* old_ctrl = ctrl; 907 PhaseIterGVN& igvn = phase->igvn(); 908 909 const Type* val_t = igvn.type(val); 910 if (val_t->meet(TypePtr::NULL_PTR) == val_t) { 911 Node* null_cmp = new CmpPNode(val, igvn.zerocon(T_OBJECT)); 912 Node* null_test = new BoolNode(null_cmp, BoolTest::ne); 913 914 IfNode* null_iff = new IfNode(old_ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 915 ctrl = new IfTrueNode(null_iff); 916 null_ctrl = new IfFalseNode(null_iff); 917 918 IdealLoopTree* loop = phase->get_loop(old_ctrl); 919 phase->register_control(null_iff, loop, old_ctrl); 920 phase->register_control(ctrl, loop, null_iff); 921 phase->register_control(null_ctrl, loop, null_iff); 922 923 phase->register_new_node(null_cmp, old_ctrl); 924 phase->register_new_node(null_test, old_ctrl); 925 } 926 } 927 928 void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { 929 Node* old_ctrl = ctrl; 930 PhaseIterGVN& igvn = phase->igvn(); 931 932 Node* raw_val = new CastP2XNode(old_ctrl, val); 933 Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 934 935 // Figure out the target cset address with raw pointer math. 936 // This avoids matching AddP+LoadB that would emit inefficient code. 937 // See JDK-8245465. 938 Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 939 Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr); 940 Node* cset_load_addr = new AddXNode(cset_addr, cset_idx); 941 Node* cset_load_ptr = new CastX2PNode(cset_load_addr); 942 943 Node* cset_load = new LoadBNode(old_ctrl, raw_mem, cset_load_ptr, 944 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), 945 TypeInt::BYTE, MemNode::unordered); 946 Node* cset_cmp = new CmpINode(cset_load, igvn.zerocon(T_INT)); 947 Node* cset_bool = new BoolNode(cset_cmp, BoolTest::ne); 948 949 IfNode* cset_iff = new IfNode(old_ctrl, cset_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 950 ctrl = new IfTrueNode(cset_iff); 951 not_cset_ctrl = new IfFalseNode(cset_iff); 952 953 IdealLoopTree *loop = phase->get_loop(old_ctrl); 954 phase->register_control(cset_iff, loop, old_ctrl); 955 phase->register_control(ctrl, loop, cset_iff); 956 phase->register_control(not_cset_ctrl, loop, cset_iff); 957 958 phase->set_ctrl(cset_addr_ptr, phase->C->root()); 959 960 phase->register_new_node(raw_val, old_ctrl); 961 phase->register_new_node(cset_idx, old_ctrl); 962 phase->register_new_node(cset_addr, old_ctrl); 963 phase->register_new_node(cset_load_addr, old_ctrl); 964 phase->register_new_node(cset_load_ptr, old_ctrl); 965 phase->register_new_node(cset_load, old_ctrl); 966 phase->register_new_node(cset_cmp, old_ctrl); 967 phase->register_new_node(cset_bool, old_ctrl); 968 } 969 970 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, 971 DecoratorSet decorators, PhaseIdealLoop* phase) { 972 IdealLoopTree*loop = phase->get_loop(ctrl); 973 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); 974 975 address calladdr = nullptr; 976 const char* name = nullptr; 977 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); 978 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); 979 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); 980 bool is_native = ShenandoahBarrierSet::is_native_access(decorators); 981 bool is_narrow = UseCompressedOops && !is_native; 982 if (is_strong) { 983 if (is_narrow) { 984 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); 985 name = "load_reference_barrier_strong_narrow"; 986 } else { 987 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); 988 name = "load_reference_barrier_strong"; 989 } 990 } else if (is_weak) { 991 if (is_narrow) { 992 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); 993 name = "load_reference_barrier_weak_narrow"; 994 } else { 995 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); 996 name = "load_reference_barrier_weak"; 997 } 998 } else { 999 assert(is_phantom, "only remaining strength"); 1000 if (is_narrow) { 1001 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow); 1002 name = "load_reference_barrier_phantom_narrow"; 1003 } else { 1004 calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom); 1005 name = "load_reference_barrier_phantom"; 1006 } 1007 } 1008 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); 1009 1010 call->init_req(TypeFunc::Control, ctrl); 1011 call->init_req(TypeFunc::I_O, phase->C->top()); 1012 call->init_req(TypeFunc::Memory, phase->C->top()); 1013 call->init_req(TypeFunc::FramePtr, phase->C->top()); 1014 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 1015 call->init_req(TypeFunc::Parms, val); 1016 call->init_req(TypeFunc::Parms+1, load_addr); 1017 phase->register_control(call, loop, ctrl); 1018 ctrl = new ProjNode(call, TypeFunc::Control); 1019 phase->register_control(ctrl, loop, call); 1020 val = new ProjNode(call, TypeFunc::Parms); 1021 phase->register_new_node(val, call); 1022 val = new CheckCastPPNode(ctrl, val, obj_type); 1023 phase->register_new_node(val, ctrl); 1024 } 1025 1026 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { 1027 Node* ctrl = phase->get_ctrl(barrier); 1028 Node* init_raw_mem = fixer.find_mem(ctrl, barrier); 1029 1030 // Update the control of all nodes that should be after the 1031 // barrier control flow 1032 uses.clear(); 1033 // Every node that is control dependent on the barrier's input 1034 // control will be after the expanded barrier. The raw memory (if 1035 // its memory is control dependent on the barrier's input control) 1036 // must stay above the barrier. 1037 uses_to_ignore.clear(); 1038 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 1039 uses_to_ignore.push(init_raw_mem); 1040 } 1041 for (uint next = 0; next < uses_to_ignore.size(); next++) { 1042 Node *n = uses_to_ignore.at(next); 1043 for (uint i = 0; i < n->req(); i++) { 1044 Node* in = n->in(i); 1045 if (in != nullptr && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 1046 uses_to_ignore.push(in); 1047 } 1048 } 1049 } 1050 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 1051 Node* u = ctrl->fast_out(i); 1052 if (u->_idx < last && 1053 u != barrier && 1054 !uses_to_ignore.member(u) && 1055 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 1056 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 1057 Node* old_c = phase->ctrl_or_self(u); 1058 Node* c = old_c; 1059 if (c != ctrl || 1060 is_dominator_same_ctrl(old_c, barrier, u, phase) || 1061 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) { 1062 phase->igvn().rehash_node_delayed(u); 1063 int nb = u->replace_edge(ctrl, region, &phase->igvn()); 1064 if (u->is_CFG()) { 1065 if (phase->idom(u) == ctrl) { 1066 phase->set_idom(u, region, phase->dom_depth(region)); 1067 } 1068 } else if (phase->get_ctrl(u) == ctrl) { 1069 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 1070 uses.push(u); 1071 } 1072 assert(nb == 1, "more than 1 ctrl input?"); 1073 --i, imax -= nb; 1074 } 1075 } 1076 } 1077 } 1078 1079 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { 1080 Node* region = nullptr; 1081 while (c != ctrl) { 1082 if (c->is_Region()) { 1083 region = c; 1084 } 1085 c = phase->idom(c); 1086 } 1087 assert(region != nullptr, ""); 1088 Node* phi = new PhiNode(region, n->bottom_type()); 1089 for (uint j = 1; j < region->req(); j++) { 1090 Node* in = region->in(j); 1091 if (phase->is_dominator(projs.fallthrough_catchproj, in)) { 1092 phi->init_req(j, n); 1093 } else if (phase->is_dominator(projs.catchall_catchproj, in)) { 1094 phi->init_req(j, n_clone); 1095 } else { 1096 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); 1097 } 1098 } 1099 phase->register_new_node(phi, region); 1100 return phi; 1101 } 1102 1103 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { 1104 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 1105 1106 Unique_Node_List uses; 1107 for (int i = 0; i < state->iu_barriers_count(); i++) { 1108 Node* barrier = state->iu_barrier(i); 1109 Node* ctrl = phase->get_ctrl(barrier); 1110 IdealLoopTree* loop = phase->get_loop(ctrl); 1111 Node* head = loop->head(); 1112 if (head->is_OuterStripMinedLoop()) { 1113 // Expanding a barrier here will break loop strip mining 1114 // verification. Transform the loop so the loop nest doesn't 1115 // appear as strip mined. 1116 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop(); 1117 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1118 } 1119 } 1120 1121 Node_Stack stack(0); 1122 Node_List clones; 1123 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1124 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1125 1126 Node* ctrl = phase->get_ctrl(lrb); 1127 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1128 1129 CallStaticJavaNode* unc = nullptr; 1130 Node* unc_ctrl = nullptr; 1131 Node* uncasted_val = val; 1132 1133 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { 1134 Node* u = lrb->fast_out(i); 1135 if (u->Opcode() == Op_CastPP && 1136 u->in(0) != nullptr && 1137 phase->is_dominator(u->in(0), ctrl)) { 1138 const Type* u_t = phase->igvn().type(u); 1139 1140 if (u_t->meet(TypePtr::NULL_PTR) != u_t && 1141 u->in(0)->Opcode() == Op_IfTrue && 1142 u->in(0)->as_Proj()->is_uncommon_trap_if_pattern() && 1143 u->in(0)->in(0)->is_If() && 1144 u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 1145 u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 1146 u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 1147 u->in(0)->in(0)->in(1)->in(1)->in(1) == val && 1148 u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 1149 IdealLoopTree* loop = phase->get_loop(ctrl); 1150 IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); 1151 1152 if (!unc_loop->is_member(loop)) { 1153 continue; 1154 } 1155 1156 Node* branch = no_branches(ctrl, u->in(0), false, phase); 1157 assert(branch == nullptr || branch == NodeSentinel, "was not looking for a branch"); 1158 if (branch == NodeSentinel) { 1159 continue; 1160 } 1161 1162 Node* iff = u->in(0)->in(0); 1163 Node* bol = iff->in(1)->clone(); 1164 Node* cmp = bol->in(1)->clone(); 1165 cmp->set_req(1, lrb); 1166 bol->set_req(1, cmp); 1167 phase->igvn().replace_input_of(iff, 1, bol); 1168 phase->set_ctrl(lrb, iff->in(0)); 1169 phase->register_new_node(cmp, iff->in(0)); 1170 phase->register_new_node(bol, iff->in(0)); 1171 break; 1172 } 1173 } 1174 } 1175 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { 1176 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); 1177 if (call->entry_point() == OptoRuntime::rethrow_stub()) { 1178 // The rethrow call may have too many projections to be 1179 // properly handled here. Given there's no reason for a 1180 // barrier to depend on the call, move it above the call 1181 stack.push(lrb, 0); 1182 do { 1183 Node* n = stack.node(); 1184 uint idx = stack.index(); 1185 if (idx < n->req()) { 1186 Node* in = n->in(idx); 1187 stack.set_index(idx+1); 1188 if (in != nullptr) { 1189 if (phase->has_ctrl(in)) { 1190 if (phase->is_dominator(call, phase->get_ctrl(in))) { 1191 #ifdef ASSERT 1192 for (uint i = 0; i < stack.size(); i++) { 1193 assert(stack.node_at(i) != in, "node shouldn't have been seen yet"); 1194 } 1195 #endif 1196 stack.push(in, 0); 1197 } 1198 } else { 1199 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call"); 1200 } 1201 } 1202 } else { 1203 phase->set_ctrl(n, call->in(0)); 1204 stack.pop(); 1205 } 1206 } while(stack.size() > 0); 1207 continue; 1208 } 1209 CallProjections projs; 1210 call->extract_projections(&projs, false, false); 1211 1212 #ifdef ASSERT 1213 VectorSet cloned; 1214 #endif 1215 Node* lrb_clone = lrb->clone(); 1216 phase->register_new_node(lrb_clone, projs.catchall_catchproj); 1217 phase->set_ctrl(lrb, projs.fallthrough_catchproj); 1218 1219 stack.push(lrb, 0); 1220 clones.push(lrb_clone); 1221 1222 do { 1223 assert(stack.size() == clones.size(), ""); 1224 Node* n = stack.node(); 1225 #ifdef ASSERT 1226 if (n->is_Load()) { 1227 Node* mem = n->in(MemNode::Memory); 1228 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { 1229 Node* u = mem->fast_out(j); 1230 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); 1231 } 1232 } 1233 #endif 1234 uint idx = stack.index(); 1235 Node* n_clone = clones.at(clones.size()-1); 1236 if (idx < n->outcnt()) { 1237 Node* u = n->raw_out(idx); 1238 Node* c = phase->ctrl_or_self(u); 1239 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) { 1240 stack.set_index(idx+1); 1241 assert(!u->is_CFG(), ""); 1242 stack.push(u, 0); 1243 assert(!cloned.test_set(u->_idx), "only one clone"); 1244 Node* u_clone = u->clone(); 1245 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn()); 1246 assert(nb > 0, "should have replaced some uses"); 1247 phase->register_new_node(u_clone, projs.catchall_catchproj); 1248 clones.push(u_clone); 1249 phase->set_ctrl(u, projs.fallthrough_catchproj); 1250 } else { 1251 bool replaced = false; 1252 if (u->is_Phi()) { 1253 for (uint k = 1; k < u->req(); k++) { 1254 if (u->in(k) == n) { 1255 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { 1256 phase->igvn().replace_input_of(u, k, n_clone); 1257 replaced = true; 1258 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { 1259 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); 1260 replaced = true; 1261 } 1262 } 1263 } 1264 } else { 1265 if (phase->is_dominator(projs.catchall_catchproj, c)) { 1266 phase->igvn().rehash_node_delayed(u); 1267 int nb = u->replace_edge(n, n_clone, &phase->igvn()); 1268 assert(nb > 0, "should have replaced some uses"); 1269 replaced = true; 1270 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { 1271 if (u->is_If()) { 1272 // Can't break If/Bool/Cmp chain 1273 assert(n->is_Bool(), "unexpected If shape"); 1274 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape"); 1275 assert(n_clone->is_Bool(), "unexpected clone"); 1276 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone"); 1277 Node* bol_clone = n->clone(); 1278 Node* cmp_clone = stack.node_at(stack.size()-2)->clone(); 1279 bol_clone->set_req(1, cmp_clone); 1280 1281 Node* nn = stack.node_at(stack.size()-3); 1282 Node* nn_clone = clones.at(clones.size()-3); 1283 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch"); 1284 1285 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase), 1286 &phase->igvn()); 1287 assert(nb > 0, "should have replaced some uses"); 1288 1289 phase->register_new_node(bol_clone, u->in(0)); 1290 phase->register_new_node(cmp_clone, u->in(0)); 1291 1292 phase->igvn().replace_input_of(u, 1, bol_clone); 1293 1294 } else { 1295 phase->igvn().rehash_node_delayed(u); 1296 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase), &phase->igvn()); 1297 assert(nb > 0, "should have replaced some uses"); 1298 } 1299 replaced = true; 1300 } 1301 } 1302 if (!replaced) { 1303 stack.set_index(idx+1); 1304 } 1305 } 1306 } else { 1307 stack.pop(); 1308 clones.pop(); 1309 } 1310 } while (stack.size() > 0); 1311 assert(stack.size() == 0 && clones.size() == 0, ""); 1312 } 1313 } 1314 1315 for (int i = 0; i < state->load_reference_barriers_count(); i++) { 1316 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1317 Node* ctrl = phase->get_ctrl(lrb); 1318 IdealLoopTree* loop = phase->get_loop(ctrl); 1319 Node* head = loop->head(); 1320 if (head->is_OuterStripMinedLoop()) { 1321 // Expanding a barrier here will break loop strip mining 1322 // verification. Transform the loop so the loop nest doesn't 1323 // appear as strip mined. 1324 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop(); 1325 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1326 } 1327 } 1328 1329 // Expand load-reference-barriers 1330 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); 1331 Unique_Node_List uses_to_ignore; 1332 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1333 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1334 uint last = phase->C->unique(); 1335 Node* ctrl = phase->get_ctrl(lrb); 1336 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1337 1338 Node* orig_ctrl = ctrl; 1339 1340 Node* raw_mem = fixer.find_mem(ctrl, lrb); 1341 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr); 1342 1343 IdealLoopTree *loop = phase->get_loop(ctrl); 1344 1345 Node* heap_stable_ctrl = nullptr; 1346 Node* null_ctrl = nullptr; 1347 1348 assert(val->bottom_type()->make_oopptr(), "need oop"); 1349 assert(val->bottom_type()->make_oopptr()->const_oop() == nullptr, "expect non-constant"); 1350 1351 enum { _heap_stable = 1, _evac_path, _not_cset, PATH_LIMIT }; 1352 Node* region = new RegionNode(PATH_LIMIT); 1353 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()); 1354 1355 // Stable path. 1356 int flags = ShenandoahHeap::HAS_FORWARDED; 1357 if (!ShenandoahBarrierSet::is_strong_access(lrb->decorators())) { 1358 flags |= ShenandoahHeap::WEAK_ROOTS; 1359 } 1360 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, flags); 1361 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); 1362 1363 // Heap stable case 1364 region->init_req(_heap_stable, heap_stable_ctrl); 1365 val_phi->init_req(_heap_stable, val); 1366 1367 // Test for in-cset, unless it's a native-LRB. Native LRBs need to return null 1368 // even for non-cset objects to prevent resurrection of such objects. 1369 // Wires !in_cset(obj) to slot 2 of region and phis 1370 Node* not_cset_ctrl = nullptr; 1371 if (ShenandoahBarrierSet::is_strong_access(lrb->decorators())) { 1372 test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase); 1373 } 1374 if (not_cset_ctrl != nullptr) { 1375 region->init_req(_not_cset, not_cset_ctrl); 1376 val_phi->init_req(_not_cset, val); 1377 } else { 1378 region->del_req(_not_cset); 1379 val_phi->del_req(_not_cset); 1380 } 1381 1382 // Resolve object when orig-value is in cset. 1383 // Make the unconditional resolve for fwdptr. 1384 1385 // Call lrb-stub and wire up that path in slots 4 1386 Node* result_mem = nullptr; 1387 1388 Node* addr; 1389 if (ShenandoahSelfFixing) { 1390 VectorSet visited; 1391 addr = get_load_addr(phase, visited, lrb); 1392 } else { 1393 addr = phase->igvn().zerocon(T_OBJECT); 1394 } 1395 if (addr->Opcode() == Op_AddP) { 1396 Node* orig_base = addr->in(AddPNode::Base); 1397 Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), ConstraintCastNode::StrongDependency); 1398 phase->register_new_node(base, ctrl); 1399 if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { 1400 // Field access 1401 addr = addr->clone(); 1402 addr->set_req(AddPNode::Base, base); 1403 addr->set_req(AddPNode::Address, base); 1404 phase->register_new_node(addr, ctrl); 1405 } else { 1406 Node* addr2 = addr->in(AddPNode::Address); 1407 if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && 1408 addr2->in(AddPNode::Base) == orig_base) { 1409 addr2 = addr2->clone(); 1410 addr2->set_req(AddPNode::Base, base); 1411 addr2->set_req(AddPNode::Address, base); 1412 phase->register_new_node(addr2, ctrl); 1413 addr = addr->clone(); 1414 addr->set_req(AddPNode::Base, base); 1415 addr->set_req(AddPNode::Address, addr2); 1416 phase->register_new_node(addr, ctrl); 1417 } 1418 } 1419 } 1420 call_lrb_stub(ctrl, val, addr, lrb->decorators(), phase); 1421 region->init_req(_evac_path, ctrl); 1422 val_phi->init_req(_evac_path, val); 1423 1424 phase->register_control(region, loop, heap_stable_iff); 1425 Node* out_val = val_phi; 1426 phase->register_new_node(val_phi, region); 1427 1428 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); 1429 1430 ctrl = orig_ctrl; 1431 1432 phase->igvn().replace_node(lrb, out_val); 1433 1434 follow_barrier_uses(out_val, ctrl, uses, phase); 1435 1436 for(uint next = 0; next < uses.size(); next++ ) { 1437 Node *n = uses.at(next); 1438 assert(phase->get_ctrl(n) == ctrl, "bad control"); 1439 assert(n != raw_mem, "should leave input raw mem above the barrier"); 1440 phase->set_ctrl(n, region); 1441 follow_barrier_uses(n, ctrl, uses, phase); 1442 } 1443 fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl); 1444 } 1445 // Done expanding load-reference-barriers. 1446 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); 1447 1448 for (int i = state->iu_barriers_count() - 1; i >= 0; i--) { 1449 Node* barrier = state->iu_barrier(i); 1450 Node* pre_val = barrier->in(1); 1451 1452 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { 1453 ShouldNotReachHere(); 1454 continue; 1455 } 1456 1457 Node* ctrl = phase->get_ctrl(barrier); 1458 1459 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 1460 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move"); 1461 ctrl = ctrl->in(0)->in(0); 1462 phase->set_ctrl(barrier, ctrl); 1463 } else if (ctrl->is_CallRuntime()) { 1464 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move"); 1465 ctrl = ctrl->in(0); 1466 phase->set_ctrl(barrier, ctrl); 1467 } 1468 1469 Node* init_ctrl = ctrl; 1470 IdealLoopTree* loop = phase->get_loop(ctrl); 1471 Node* raw_mem = fixer.find_mem(ctrl, barrier); 1472 Node* init_raw_mem = raw_mem; 1473 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr); 1474 Node* heap_stable_ctrl = nullptr; 1475 Node* null_ctrl = nullptr; 1476 uint last = phase->C->unique(); 1477 1478 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; 1479 Node* region = new RegionNode(PATH_LIMIT); 1480 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1481 1482 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 }; 1483 Node* region2 = new RegionNode(PATH_LIMIT2); 1484 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1485 1486 // Stable path. 1487 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING); 1488 region->init_req(_heap_stable, heap_stable_ctrl); 1489 phi->init_req(_heap_stable, raw_mem); 1490 1491 // Null path 1492 Node* reg2_ctrl = nullptr; 1493 test_null(ctrl, pre_val, null_ctrl, phase); 1494 if (null_ctrl != nullptr) { 1495 reg2_ctrl = null_ctrl->in(0); 1496 region2->init_req(_null_path, null_ctrl); 1497 phi2->init_req(_null_path, raw_mem); 1498 } else { 1499 region2->del_req(_null_path); 1500 phi2->del_req(_null_path); 1501 } 1502 1503 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); 1504 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); 1505 Node* thread = new ThreadLocalNode(); 1506 phase->register_new_node(thread, ctrl); 1507 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset)); 1508 phase->register_new_node(buffer_adr, ctrl); 1509 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset)); 1510 phase->register_new_node(index_adr, ctrl); 1511 1512 BasicType index_bt = TypeX_X->basic_type(); 1513 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size."); 1514 const TypePtr* adr_type = TypeRawPtr::BOTTOM; 1515 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered); 1516 phase->register_new_node(index, ctrl); 1517 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0)); 1518 phase->register_new_node(index_cmp, ctrl); 1519 Node* index_test = new BoolNode(index_cmp, BoolTest::ne); 1520 phase->register_new_node(index_test, ctrl); 1521 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 1522 if (reg2_ctrl == nullptr) reg2_ctrl = queue_full_iff; 1523 phase->register_control(queue_full_iff, loop, ctrl); 1524 Node* not_full = new IfTrueNode(queue_full_iff); 1525 phase->register_control(not_full, loop, queue_full_iff); 1526 Node* full = new IfFalseNode(queue_full_iff); 1527 phase->register_control(full, loop, queue_full_iff); 1528 1529 ctrl = not_full; 1530 1531 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t))); 1532 phase->register_new_node(next_index, ctrl); 1533 1534 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered); 1535 phase->register_new_node(buffer, ctrl); 1536 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index); 1537 phase->register_new_node(log_addr, ctrl); 1538 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered); 1539 phase->register_new_node(log_store, ctrl); 1540 // update the index 1541 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered); 1542 phase->register_new_node(index_update, ctrl); 1543 1544 // Fast-path case 1545 region2->init_req(_fast_path, ctrl); 1546 phi2->init_req(_fast_path, index_update); 1547 1548 ctrl = full; 1549 1550 Node* base = find_bottom_mem(ctrl, phase); 1551 1552 MergeMemNode* mm = MergeMemNode::make(base); 1553 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 1554 phase->register_new_node(mm, ctrl); 1555 1556 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM); 1557 call->init_req(TypeFunc::Control, ctrl); 1558 call->init_req(TypeFunc::I_O, phase->C->top()); 1559 call->init_req(TypeFunc::Memory, mm); 1560 call->init_req(TypeFunc::FramePtr, phase->C->top()); 1561 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 1562 call->init_req(TypeFunc::Parms, pre_val); 1563 call->init_req(TypeFunc::Parms+1, thread); 1564 phase->register_control(call, loop, ctrl); 1565 1566 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 1567 phase->register_control(ctrl_proj, loop, call); 1568 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 1569 phase->register_new_node(mem_proj, call); 1570 1571 // Slow-path case 1572 region2->init_req(_slow_path, ctrl_proj); 1573 phi2->init_req(_slow_path, mem_proj); 1574 1575 phase->register_control(region2, loop, reg2_ctrl); 1576 phase->register_new_node(phi2, region2); 1577 1578 region->init_req(_heap_unstable, region2); 1579 phi->init_req(_heap_unstable, phi2); 1580 1581 phase->register_control(region, loop, heap_stable_ctrl->in(0)); 1582 phase->register_new_node(phi, region); 1583 1584 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase); 1585 for(uint next = 0; next < uses.size(); next++ ) { 1586 Node *n = uses.at(next); 1587 assert(phase->get_ctrl(n) == init_ctrl, "bad control"); 1588 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 1589 phase->set_ctrl(n, region); 1590 follow_barrier_uses(n, init_ctrl, uses, phase); 1591 } 1592 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); 1593 1594 phase->igvn().replace_node(barrier, pre_val); 1595 } 1596 assert(state->iu_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); 1597 1598 } 1599 1600 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { 1601 if (visited.test_set(in->_idx)) { 1602 return nullptr; 1603 } 1604 switch (in->Opcode()) { 1605 case Op_Proj: 1606 return get_load_addr(phase, visited, in->in(0)); 1607 case Op_CastPP: 1608 case Op_CheckCastPP: 1609 case Op_DecodeN: 1610 case Op_EncodeP: 1611 return get_load_addr(phase, visited, in->in(1)); 1612 case Op_LoadN: 1613 case Op_LoadP: 1614 return in->in(MemNode::Address); 1615 case Op_CompareAndExchangeN: 1616 case Op_CompareAndExchangeP: 1617 case Op_GetAndSetN: 1618 case Op_GetAndSetP: 1619 case Op_ShenandoahCompareAndExchangeP: 1620 case Op_ShenandoahCompareAndExchangeN: 1621 // Those instructions would just have stored a different 1622 // value into the field. No use to attempt to fix it at this point. 1623 return phase->igvn().zerocon(T_OBJECT); 1624 case Op_CMoveP: 1625 case Op_CMoveN: { 1626 Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); 1627 Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); 1628 // Handle unambiguous cases: single address reported on both branches. 1629 if (t != nullptr && f == nullptr) return t; 1630 if (t == nullptr && f != nullptr) return f; 1631 if (t != nullptr && t == f) return t; 1632 // Ambiguity. 1633 return phase->igvn().zerocon(T_OBJECT); 1634 } 1635 case Op_Phi: { 1636 Node* addr = nullptr; 1637 for (uint i = 1; i < in->req(); i++) { 1638 Node* addr1 = get_load_addr(phase, visited, in->in(i)); 1639 if (addr == nullptr) { 1640 addr = addr1; 1641 } 1642 if (addr != addr1) { 1643 return phase->igvn().zerocon(T_OBJECT); 1644 } 1645 } 1646 return addr; 1647 } 1648 case Op_ShenandoahLoadReferenceBarrier: 1649 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); 1650 case Op_ShenandoahIUBarrier: 1651 return get_load_addr(phase, visited, in->in(1)); 1652 case Op_CallDynamicJava: 1653 case Op_CallLeaf: 1654 case Op_CallStaticJava: 1655 case Op_ConN: 1656 case Op_ConP: 1657 case Op_Parm: 1658 case Op_CreateEx: 1659 return phase->igvn().zerocon(T_OBJECT); 1660 default: 1661 #ifdef ASSERT 1662 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); 1663 #endif 1664 return phase->igvn().zerocon(T_OBJECT); 1665 } 1666 1667 } 1668 1669 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 1670 IdealLoopTree *loop = phase->get_loop(iff); 1671 Node* loop_head = loop->_head; 1672 Node* entry_c = loop_head->in(LoopNode::EntryControl); 1673 1674 Node* bol = iff->in(1); 1675 Node* cmp = bol->in(1); 1676 Node* andi = cmp->in(1); 1677 Node* load = andi->in(1); 1678 1679 assert(is_gc_state_load(load), "broken"); 1680 if (!phase->is_dominator(load->in(0), entry_c)) { 1681 Node* mem_ctrl = nullptr; 1682 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 1683 load = load->clone(); 1684 load->set_req(MemNode::Memory, mem); 1685 load->set_req(0, entry_c); 1686 phase->register_new_node(load, entry_c); 1687 andi = andi->clone(); 1688 andi->set_req(1, load); 1689 phase->register_new_node(andi, entry_c); 1690 cmp = cmp->clone(); 1691 cmp->set_req(1, andi); 1692 phase->register_new_node(cmp, entry_c); 1693 bol = bol->clone(); 1694 bol->set_req(1, cmp); 1695 phase->register_new_node(bol, entry_c); 1696 1697 phase->igvn().replace_input_of(iff, 1, bol); 1698 } 1699 } 1700 1701 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { 1702 if (!n->is_If() || n->is_CountedLoopEnd()) { 1703 return false; 1704 } 1705 Node* region = n->in(0); 1706 1707 if (!region->is_Region()) { 1708 return false; 1709 } 1710 Node* dom = phase->idom(region); 1711 if (!dom->is_If()) { 1712 return false; 1713 } 1714 1715 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) { 1716 return false; 1717 } 1718 1719 IfNode* dom_if = dom->as_If(); 1720 Node* proj_true = dom_if->proj_out(1); 1721 Node* proj_false = dom_if->proj_out(0); 1722 1723 for (uint i = 1; i < region->req(); i++) { 1724 if (phase->is_dominator(proj_true, region->in(i))) { 1725 continue; 1726 } 1727 if (phase->is_dominator(proj_false, region->in(i))) { 1728 continue; 1729 } 1730 return false; 1731 } 1732 1733 return true; 1734 } 1735 1736 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { 1737 assert(is_heap_stable_test(n), "no other tests"); 1738 if (identical_backtoback_ifs(n, phase)) { 1739 Node* n_ctrl = n->in(0); 1740 if (phase->can_split_if(n_ctrl)) { 1741 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 1742 if (is_heap_stable_test(n)) { 1743 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1); 1744 assert(is_gc_state_load(gc_state_load), "broken"); 1745 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1); 1746 assert(is_gc_state_load(dom_gc_state_load), "broken"); 1747 if (gc_state_load != dom_gc_state_load) { 1748 phase->igvn().replace_node(gc_state_load, dom_gc_state_load); 1749 } 1750 } 1751 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 1752 Node* proj_true = dom_if->proj_out(1); 1753 Node* proj_false = dom_if->proj_out(0); 1754 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 1755 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 1756 1757 for (uint i = 1; i < n_ctrl->req(); i++) { 1758 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 1759 bolphi->init_req(i, con_true); 1760 } else { 1761 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 1762 bolphi->init_req(i, con_false); 1763 } 1764 } 1765 phase->register_new_node(bolphi, n_ctrl); 1766 phase->igvn().replace_input_of(n, 1, bolphi); 1767 phase->do_split_if(n); 1768 } 1769 } 1770 } 1771 1772 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { 1773 // Find first invariant test that doesn't exit the loop 1774 LoopNode *head = loop->_head->as_Loop(); 1775 IfNode* unswitch_iff = nullptr; 1776 Node* n = head->in(LoopNode::LoopBackControl); 1777 int loop_has_sfpts = -1; 1778 while (n != head) { 1779 Node* n_dom = phase->idom(n); 1780 if (n->is_Region()) { 1781 if (n_dom->is_If()) { 1782 IfNode* iff = n_dom->as_If(); 1783 if (iff->in(1)->is_Bool()) { 1784 BoolNode* bol = iff->in(1)->as_Bool(); 1785 if (bol->in(1)->is_Cmp()) { 1786 // If condition is invariant and not a loop exit, 1787 // then found reason to unswitch. 1788 if (is_heap_stable_test(iff) && 1789 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { 1790 assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); 1791 if (loop_has_sfpts == -1) { 1792 for(uint i = 0; i < loop->_body.size(); i++) { 1793 Node *m = loop->_body[i]; 1794 if (m->is_SafePoint() && !m->is_CallLeaf()) { 1795 loop_has_sfpts = 1; 1796 break; 1797 } 1798 } 1799 if (loop_has_sfpts == -1) { 1800 loop_has_sfpts = 0; 1801 } 1802 } 1803 if (!loop_has_sfpts) { 1804 unswitch_iff = iff; 1805 } 1806 } 1807 } 1808 } 1809 } 1810 } 1811 n = n_dom; 1812 } 1813 return unswitch_iff; 1814 } 1815 1816 1817 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { 1818 Node_List heap_stable_tests; 1819 stack.push(phase->C->start(), 0); 1820 do { 1821 Node* n = stack.node(); 1822 uint i = stack.index(); 1823 1824 if (i < n->outcnt()) { 1825 Node* u = n->raw_out(i); 1826 stack.set_index(i+1); 1827 if (!visited.test_set(u->_idx)) { 1828 stack.push(u, 0); 1829 } 1830 } else { 1831 stack.pop(); 1832 if (n->is_If() && is_heap_stable_test(n)) { 1833 heap_stable_tests.push(n); 1834 } 1835 } 1836 } while (stack.size() > 0); 1837 1838 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1839 Node* n = heap_stable_tests.at(i); 1840 assert(is_heap_stable_test(n), "only evacuation test"); 1841 merge_back_to_back_tests(n, phase); 1842 } 1843 1844 if (!phase->C->major_progress()) { 1845 VectorSet seen; 1846 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1847 Node* n = heap_stable_tests.at(i); 1848 IdealLoopTree* loop = phase->get_loop(n); 1849 if (loop != phase->ltree_root() && 1850 loop->_child == nullptr && 1851 !loop->_irreducible) { 1852 Node* head = loop->_head; 1853 if (head->is_Loop() && 1854 (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 1855 !seen.test_set(head->_idx)) { 1856 IfNode* iff = find_unswitching_candidate(loop, phase); 1857 if (iff != nullptr) { 1858 Node* bol = iff->in(1); 1859 if (head->as_Loop()->is_strip_mined()) { 1860 head->as_Loop()->verify_strip_mined(0); 1861 } 1862 move_gc_state_test_out_of_loop(iff, phase); 1863 1864 AutoNodeBudget node_budget(phase); 1865 1866 if (loop->policy_unswitching(phase)) { 1867 if (head->as_Loop()->is_strip_mined()) { 1868 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 1869 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); 1870 } 1871 phase->do_unswitching(loop, old_new); 1872 } else { 1873 // Not proceeding with unswitching. Move load back in 1874 // the loop. 1875 phase->igvn().replace_input_of(iff, 1, bol); 1876 } 1877 } 1878 } 1879 } 1880 } 1881 } 1882 } 1883 1884 ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(nullptr, val) { 1885 ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this); 1886 } 1887 1888 const Type* ShenandoahIUBarrierNode::bottom_type() const { 1889 if (in(1) == nullptr || in(1)->is_top()) { 1890 return Type::TOP; 1891 } 1892 const Type* t = in(1)->bottom_type(); 1893 if (t == TypePtr::NULL_PTR) { 1894 return t; 1895 } 1896 return t->is_oopptr(); 1897 } 1898 1899 const Type* ShenandoahIUBarrierNode::Value(PhaseGVN* phase) const { 1900 if (in(1) == nullptr) { 1901 return Type::TOP; 1902 } 1903 const Type* t = phase->type(in(1)); 1904 if (t == Type::TOP) { 1905 return Type::TOP; 1906 } 1907 if (t == TypePtr::NULL_PTR) { 1908 return t; 1909 } 1910 return t->is_oopptr(); 1911 } 1912 1913 int ShenandoahIUBarrierNode::needed(Node* n) { 1914 if (n == nullptr || 1915 n->is_Allocate() || 1916 n->Opcode() == Op_ShenandoahIUBarrier || 1917 n->bottom_type() == TypePtr::NULL_PTR || 1918 (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr)) { 1919 return NotNeeded; 1920 } 1921 if (n->is_Phi() || 1922 n->is_CMove()) { 1923 return MaybeNeeded; 1924 } 1925 return Needed; 1926 } 1927 1928 Node* ShenandoahIUBarrierNode::next(Node* n) { 1929 for (;;) { 1930 if (n == nullptr) { 1931 return n; 1932 } else if (n->bottom_type() == TypePtr::NULL_PTR) { 1933 return n; 1934 } else if (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr) { 1935 return n; 1936 } else if (n->is_ConstraintCast() || 1937 n->Opcode() == Op_DecodeN || 1938 n->Opcode() == Op_EncodeP) { 1939 n = n->in(1); 1940 } else if (n->is_Proj()) { 1941 n = n->in(0); 1942 } else { 1943 return n; 1944 } 1945 } 1946 ShouldNotReachHere(); 1947 return nullptr; 1948 } 1949 1950 Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) { 1951 PhaseIterGVN* igvn = phase->is_IterGVN(); 1952 1953 Node* n = next(in(1)); 1954 1955 int cont = needed(n); 1956 1957 if (cont == NotNeeded) { 1958 return in(1); 1959 } else if (cont == MaybeNeeded) { 1960 if (igvn == nullptr) { 1961 phase->record_for_igvn(this); 1962 return this; 1963 } else { 1964 ResourceMark rm; 1965 Unique_Node_List wq; 1966 uint wq_i = 0; 1967 1968 for (;;) { 1969 if (n->is_Phi()) { 1970 for (uint i = 1; i < n->req(); i++) { 1971 Node* m = n->in(i); 1972 if (m != nullptr) { 1973 wq.push(m); 1974 } 1975 } 1976 } else { 1977 assert(n->is_CMove(), "nothing else here"); 1978 Node* m = n->in(CMoveNode::IfFalse); 1979 wq.push(m); 1980 m = n->in(CMoveNode::IfTrue); 1981 wq.push(m); 1982 } 1983 Node* orig_n = nullptr; 1984 do { 1985 if (wq_i >= wq.size()) { 1986 return in(1); 1987 } 1988 n = wq.at(wq_i); 1989 wq_i++; 1990 orig_n = n; 1991 n = next(n); 1992 cont = needed(n); 1993 if (cont == Needed) { 1994 return this; 1995 } 1996 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n))); 1997 } 1998 } 1999 } 2000 2001 return this; 2002 } 2003 2004 #ifdef ASSERT 2005 static bool has_never_branch(Node* root) { 2006 for (uint i = 1; i < root->req(); i++) { 2007 Node* in = root->in(i); 2008 if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) { 2009 return true; 2010 } 2011 } 2012 return false; 2013 } 2014 #endif 2015 2016 void MemoryGraphFixer::collect_memory_nodes() { 2017 Node_Stack stack(0); 2018 VectorSet visited; 2019 Node_List regions; 2020 2021 // Walk the raw memory graph and create a mapping from CFG node to 2022 // memory node. Exclude phis for now. 2023 stack.push(_phase->C->root(), 1); 2024 do { 2025 Node* n = stack.node(); 2026 int opc = n->Opcode(); 2027 uint i = stack.index(); 2028 if (i < n->req()) { 2029 Node* mem = nullptr; 2030 if (opc == Op_Root) { 2031 Node* in = n->in(i); 2032 int in_opc = in->Opcode(); 2033 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 2034 mem = in->in(TypeFunc::Memory); 2035 } else if (in_opc == Op_Halt) { 2036 if (in->in(0)->is_Region()) { 2037 Node* r = in->in(0); 2038 for (uint j = 1; j < r->req(); j++) { 2039 assert(!r->in(j)->is_NeverBranch(), ""); 2040 } 2041 } else { 2042 Node* proj = in->in(0); 2043 assert(proj->is_Proj(), ""); 2044 Node* in = proj->in(0); 2045 assert(in->is_CallStaticJava() || in->is_NeverBranch() || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 2046 if (in->is_CallStaticJava()) { 2047 mem = in->in(TypeFunc::Memory); 2048 } else if (in->Opcode() == Op_Catch) { 2049 Node* call = in->in(0)->in(0); 2050 assert(call->is_Call(), ""); 2051 mem = call->in(TypeFunc::Memory); 2052 } else if (in->is_NeverBranch()) { 2053 mem = collect_memory_for_infinite_loop(in); 2054 } 2055 } 2056 } else { 2057 #ifdef ASSERT 2058 n->dump(); 2059 in->dump(); 2060 #endif 2061 ShouldNotReachHere(); 2062 } 2063 } else { 2064 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 2065 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, ""); 2066 mem = n->in(i); 2067 } 2068 i++; 2069 stack.set_index(i); 2070 if (mem == nullptr) { 2071 continue; 2072 } 2073 for (;;) { 2074 if (visited.test_set(mem->_idx) || mem->is_Start()) { 2075 break; 2076 } 2077 if (mem->is_Phi()) { 2078 stack.push(mem, 2); 2079 mem = mem->in(1); 2080 } else if (mem->is_Proj()) { 2081 stack.push(mem, mem->req()); 2082 mem = mem->in(0); 2083 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 2084 mem = mem->in(TypeFunc::Memory); 2085 } else if (mem->is_MergeMem()) { 2086 MergeMemNode* mm = mem->as_MergeMem(); 2087 mem = mm->memory_at(_alias); 2088 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 2089 assert(_alias == Compile::AliasIdxRaw, ""); 2090 stack.push(mem, mem->req()); 2091 mem = mem->in(MemNode::Memory); 2092 } else { 2093 #ifdef ASSERT 2094 mem->dump(); 2095 #endif 2096 ShouldNotReachHere(); 2097 } 2098 } 2099 } else { 2100 if (n->is_Phi()) { 2101 // Nothing 2102 } else if (!n->is_Root()) { 2103 Node* c = get_ctrl(n); 2104 _memory_nodes.map(c->_idx, n); 2105 } 2106 stack.pop(); 2107 } 2108 } while(stack.is_nonempty()); 2109 2110 // Iterate over CFG nodes in rpo and propagate memory state to 2111 // compute memory state at regions, creating new phis if needed. 2112 Node_List rpo_list; 2113 visited.clear(); 2114 _phase->rpo(_phase->C->root(), stack, visited, rpo_list); 2115 Node* root = rpo_list.pop(); 2116 assert(root == _phase->C->root(), ""); 2117 2118 const bool trace = false; 2119 #ifdef ASSERT 2120 if (trace) { 2121 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2122 Node* c = rpo_list.at(i); 2123 if (_memory_nodes[c->_idx] != nullptr) { 2124 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); 2125 } 2126 } 2127 } 2128 #endif 2129 uint last = _phase->C->unique(); 2130 2131 #ifdef ASSERT 2132 uint16_t max_depth = 0; 2133 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) { 2134 IdealLoopTree* lpt = iter.current(); 2135 max_depth = MAX2(max_depth, lpt->_nest); 2136 } 2137 #endif 2138 2139 bool progress = true; 2140 int iteration = 0; 2141 Node_List dead_phis; 2142 while (progress) { 2143 progress = false; 2144 iteration++; 2145 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2146 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 2147 2148 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2149 Node* c = rpo_list.at(i); 2150 2151 Node* prev_mem = _memory_nodes[c->_idx]; 2152 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2153 Node* prev_region = regions[c->_idx]; 2154 Node* unique = nullptr; 2155 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 2156 Node* m = _memory_nodes[c->in(j)->_idx]; 2157 assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state"); 2158 if (m != nullptr) { 2159 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 2160 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2161 // continue 2162 } else if (unique == nullptr) { 2163 unique = m; 2164 } else if (m == unique) { 2165 // continue 2166 } else { 2167 unique = NodeSentinel; 2168 } 2169 } 2170 } 2171 assert(unique != nullptr, "empty phi???"); 2172 if (unique != NodeSentinel) { 2173 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c) { 2174 dead_phis.push(prev_region); 2175 } 2176 regions.map(c->_idx, unique); 2177 } else { 2178 Node* phi = nullptr; 2179 if (prev_region != nullptr && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 2180 phi = prev_region; 2181 for (uint k = 1; k < c->req(); k++) { 2182 Node* m = _memory_nodes[c->in(k)->_idx]; 2183 assert(m != nullptr, "expect memory state"); 2184 phi->set_req(k, m); 2185 } 2186 } else { 2187 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == nullptr; j++) { 2188 Node* u = c->fast_out(j); 2189 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2190 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) { 2191 phi = u; 2192 for (uint k = 1; k < c->req() && phi != nullptr; k++) { 2193 Node* m = _memory_nodes[c->in(k)->_idx]; 2194 assert(m != nullptr, "expect memory state"); 2195 if (u->in(k) != m) { 2196 phi = NodeSentinel; 2197 } 2198 } 2199 } 2200 } 2201 if (phi == NodeSentinel) { 2202 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2203 for (uint k = 1; k < c->req(); k++) { 2204 Node* m = _memory_nodes[c->in(k)->_idx]; 2205 assert(m != nullptr, "expect memory state"); 2206 phi->init_req(k, m); 2207 } 2208 } 2209 } 2210 if (phi != nullptr) { 2211 regions.map(c->_idx, phi); 2212 } else { 2213 assert(c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 2214 } 2215 } 2216 Node* current_region = regions[c->_idx]; 2217 if (current_region != prev_region) { 2218 progress = true; 2219 if (prev_region == prev_mem) { 2220 _memory_nodes.map(c->_idx, current_region); 2221 } 2222 } 2223 } else if (prev_mem == nullptr || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { 2224 Node* m = _memory_nodes[_phase->idom(c)->_idx]; 2225 assert(m != nullptr || c->Opcode() == Op_Halt, "expect memory state"); 2226 if (m != prev_mem) { 2227 _memory_nodes.map(c->_idx, m); 2228 progress = true; 2229 } 2230 } 2231 #ifdef ASSERT 2232 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); } 2233 #endif 2234 } 2235 } 2236 2237 // Replace existing phi with computed memory state for that region 2238 // if different (could be a new phi or a dominating memory node if 2239 // that phi was found to be useless). 2240 while (dead_phis.size() > 0) { 2241 Node* n = dead_phis.pop(); 2242 n->replace_by(_phase->C->top()); 2243 n->destruct(&_phase->igvn()); 2244 } 2245 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2246 Node* c = rpo_list.at(i); 2247 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2248 Node* n = regions[c->_idx]; 2249 assert(n != nullptr || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 2250 if (n != nullptr && n->is_Phi() && n->_idx >= last && n->in(0) == c) { 2251 _phase->register_new_node(n, c); 2252 } 2253 } 2254 } 2255 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2256 Node* c = rpo_list.at(i); 2257 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2258 Node* n = regions[c->_idx]; 2259 assert(n != nullptr || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); 2260 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2261 Node* u = c->fast_out(i); 2262 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2263 u != n) { 2264 assert(c->unique_ctrl_out()->Opcode() != Op_Halt, "expected memory state"); 2265 if (u->adr_type() == TypePtr::BOTTOM) { 2266 fix_memory_uses(u, n, n, c); 2267 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2268 _phase->lazy_replace(u, n); 2269 --i; --imax; 2270 } 2271 } 2272 } 2273 } 2274 } 2275 } 2276 2277 Node* MemoryGraphFixer::collect_memory_for_infinite_loop(const Node* in) { 2278 Node* mem = nullptr; 2279 Node* head = in->in(0); 2280 assert(head->is_Region(), "unexpected infinite loop graph shape"); 2281 2282 Node* phi_mem = nullptr; 2283 for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) { 2284 Node* u = head->fast_out(j); 2285 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2286 if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2287 assert(phi_mem == nullptr || phi_mem->adr_type() == TypePtr::BOTTOM, ""); 2288 phi_mem = u; 2289 } else if (u->adr_type() == TypePtr::BOTTOM) { 2290 assert(phi_mem == nullptr || _phase->C->get_alias_index(phi_mem->adr_type()) == _alias, ""); 2291 if (phi_mem == nullptr) { 2292 phi_mem = u; 2293 } 2294 } 2295 } 2296 } 2297 if (phi_mem == nullptr) { 2298 ResourceMark rm; 2299 Node_Stack stack(0); 2300 stack.push(head, 1); 2301 do { 2302 Node* n = stack.node(); 2303 uint i = stack.index(); 2304 if (i >= n->req()) { 2305 stack.pop(); 2306 } else { 2307 stack.set_index(i + 1); 2308 Node* c = n->in(i); 2309 assert(c != head, "should have found a safepoint on the way"); 2310 if (stack.size() != 1 || _phase->is_dominator(head, c)) { 2311 for (;;) { 2312 if (c->is_Region()) { 2313 stack.push(c, 1); 2314 break; 2315 } else if (c->is_SafePoint() && !c->is_CallLeaf()) { 2316 Node* m = c->in(TypeFunc::Memory); 2317 if (m->is_MergeMem()) { 2318 m = m->as_MergeMem()->memory_at(_alias); 2319 } 2320 assert(mem == nullptr || mem == m, "several memory states"); 2321 mem = m; 2322 break; 2323 } else { 2324 assert(c != c->in(0), ""); 2325 c = c->in(0); 2326 } 2327 } 2328 } 2329 } 2330 } while (stack.size() > 0); 2331 assert(mem != nullptr, "should have found safepoint"); 2332 } else { 2333 mem = phi_mem; 2334 } 2335 return mem; 2336 } 2337 2338 Node* MemoryGraphFixer::get_ctrl(Node* n) const { 2339 Node* c = _phase->get_ctrl(n); 2340 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) { 2341 assert(c == n->in(0), ""); 2342 CallNode* call = c->as_Call(); 2343 CallProjections projs; 2344 call->extract_projections(&projs, true, false); 2345 if (projs.catchall_memproj != nullptr) { 2346 if (projs.fallthrough_memproj == n) { 2347 c = projs.fallthrough_catchproj; 2348 } else { 2349 assert(projs.catchall_memproj == n, ""); 2350 c = projs.catchall_catchproj; 2351 } 2352 } 2353 } 2354 return c; 2355 } 2356 2357 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const { 2358 if (_phase->has_ctrl(n)) 2359 return get_ctrl(n); 2360 else { 2361 assert (n->is_CFG(), "must be a CFG node"); 2362 return n; 2363 } 2364 } 2365 2366 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const { 2367 return m != nullptr && get_ctrl(m) == c; 2368 } 2369 2370 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const { 2371 assert(n == nullptr || _phase->ctrl_or_self(n) == ctrl, ""); 2372 assert(!ctrl->is_Call() || ctrl == n, "projection expected"); 2373 #ifdef ASSERT 2374 if ((ctrl->is_Proj() && ctrl->in(0)->is_Call()) || 2375 (ctrl->is_Catch() && ctrl->in(0)->in(0)->is_Call())) { 2376 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_Call() : ctrl->in(0)->in(0)->as_Call(); 2377 int mems = 0; 2378 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) { 2379 Node* u = call->fast_out(i); 2380 if (u->bottom_type() == Type::MEMORY) { 2381 mems++; 2382 } 2383 } 2384 assert(mems <= 1, "No node right after call if multiple mem projections"); 2385 } 2386 #endif 2387 Node* mem = _memory_nodes[ctrl->_idx]; 2388 Node* c = ctrl; 2389 while (!mem_is_valid(mem, c) && 2390 (!c->is_CatchProj() || mem == nullptr || c->in(0)->in(0)->in(0) != get_ctrl(mem))) { 2391 c = _phase->idom(c); 2392 mem = _memory_nodes[c->_idx]; 2393 } 2394 if (n != nullptr && mem_is_valid(mem, c)) { 2395 while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { 2396 mem = next_mem(mem, _alias); 2397 } 2398 if (mem->is_MergeMem()) { 2399 mem = mem->as_MergeMem()->memory_at(_alias); 2400 } 2401 if (!mem_is_valid(mem, c)) { 2402 do { 2403 c = _phase->idom(c); 2404 mem = _memory_nodes[c->_idx]; 2405 } while (!mem_is_valid(mem, c) && 2406 (!c->is_CatchProj() || mem == nullptr || c->in(0)->in(0)->in(0) != get_ctrl(mem))); 2407 } 2408 } 2409 assert(mem->bottom_type() == Type::MEMORY, ""); 2410 return mem; 2411 } 2412 2413 bool MemoryGraphFixer::has_mem_phi(Node* region) const { 2414 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 2415 Node* use = region->fast_out(i); 2416 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 2417 (_phase->C->get_alias_index(use->adr_type()) == _alias)) { 2418 return true; 2419 } 2420 } 2421 return false; 2422 } 2423 2424 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { 2425 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); 2426 const bool trace = false; 2427 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 2428 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); }); 2429 GrowableArray<Node*> phis; 2430 if (mem_for_ctrl != mem) { 2431 Node* old = mem_for_ctrl; 2432 Node* prev = nullptr; 2433 while (old != mem) { 2434 prev = old; 2435 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { 2436 assert(_alias == Compile::AliasIdxRaw, ""); 2437 old = old->in(MemNode::Memory); 2438 } else if (old->Opcode() == Op_SCMemProj) { 2439 assert(_alias == Compile::AliasIdxRaw, ""); 2440 old = old->in(0); 2441 } else { 2442 ShouldNotReachHere(); 2443 } 2444 } 2445 assert(prev != nullptr, ""); 2446 if (new_ctrl != ctrl) { 2447 _memory_nodes.map(ctrl->_idx, mem); 2448 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 2449 } 2450 uint input = (uint)MemNode::Memory; 2451 _phase->igvn().replace_input_of(prev, input, new_mem); 2452 } else { 2453 uses.clear(); 2454 _memory_nodes.map(new_ctrl->_idx, new_mem); 2455 uses.push(new_ctrl); 2456 for(uint next = 0; next < uses.size(); next++ ) { 2457 Node *n = uses.at(next); 2458 assert(n->is_CFG(), ""); 2459 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 2460 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2461 Node* u = n->fast_out(i); 2462 if (!u->is_Root() && u->is_CFG() && u != n) { 2463 Node* m = _memory_nodes[u->_idx]; 2464 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) && 2465 !has_mem_phi(u) && 2466 u->unique_ctrl_out()->Opcode() != Op_Halt) { 2467 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 2468 DEBUG_ONLY(if (trace && m != nullptr) { tty->print("ZZZ mem"); m->dump(); }); 2469 2470 if (!mem_is_valid(m, u) || !m->is_Phi()) { 2471 bool push = true; 2472 bool create_phi = true; 2473 if (_phase->is_dominator(new_ctrl, u)) { 2474 create_phi = false; 2475 } 2476 if (create_phi) { 2477 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2478 _phase->register_new_node(phi, u); 2479 phis.push(phi); 2480 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 2481 if (!mem_is_valid(m, u)) { 2482 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 2483 _memory_nodes.map(u->_idx, phi); 2484 } else { 2485 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 2486 for (;;) { 2487 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); 2488 Node* next = nullptr; 2489 if (m->is_Proj()) { 2490 next = m->in(0); 2491 } else { 2492 assert(m->is_Mem() || m->is_LoadStore(), ""); 2493 assert(_alias == Compile::AliasIdxRaw, ""); 2494 next = m->in(MemNode::Memory); 2495 } 2496 if (_phase->get_ctrl(next) != u) { 2497 break; 2498 } 2499 if (next->is_MergeMem()) { 2500 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, ""); 2501 break; 2502 } 2503 if (next->is_Phi()) { 2504 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 2505 break; 2506 } 2507 m = next; 2508 } 2509 2510 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 2511 assert(m->is_Mem() || m->is_LoadStore(), ""); 2512 uint input = (uint)MemNode::Memory; 2513 _phase->igvn().replace_input_of(m, input, phi); 2514 push = false; 2515 } 2516 } else { 2517 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 2518 } 2519 if (push) { 2520 uses.push(u); 2521 } 2522 } 2523 } else if (!mem_is_valid(m, u) && 2524 !(u->Opcode() == Op_CProj && u->in(0)->is_NeverBranch() && u->as_Proj()->_con == 1)) { 2525 uses.push(u); 2526 } 2527 } 2528 } 2529 } 2530 for (int i = 0; i < phis.length(); i++) { 2531 Node* n = phis.at(i); 2532 Node* r = n->in(0); 2533 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 2534 for (uint j = 1; j < n->req(); j++) { 2535 Node* m = find_mem(r->in(j), nullptr); 2536 _phase->igvn().replace_input_of(n, j, m); 2537 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 2538 } 2539 } 2540 } 2541 uint last = _phase->C->unique(); 2542 MergeMemNode* mm = nullptr; 2543 int alias = _alias; 2544 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); }); 2545 // Process loads first to not miss an anti-dependency: if the memory 2546 // edge of a store is updated before a load is processed then an 2547 // anti-dependency may be missed. 2548 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2549 Node* u = mem->out(i); 2550 if (u->_idx < last && u->is_Load() && _phase->C->get_alias_index(u->adr_type()) == alias) { 2551 Node* m = find_mem(_phase->get_ctrl(u), u); 2552 if (m != mem) { 2553 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2554 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2555 --i; 2556 } 2557 } 2558 } 2559 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2560 Node* u = mem->out(i); 2561 if (u->_idx < last) { 2562 if (u->is_Mem()) { 2563 if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2564 Node* m = find_mem(_phase->get_ctrl(u), u); 2565 if (m != mem) { 2566 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2567 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2568 --i; 2569 } 2570 } 2571 } else if (u->is_MergeMem()) { 2572 MergeMemNode* u_mm = u->as_MergeMem(); 2573 if (u_mm->memory_at(alias) == mem) { 2574 MergeMemNode* newmm = nullptr; 2575 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2576 Node* uu = u->fast_out(j); 2577 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2578 if (uu->is_Phi()) { 2579 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 2580 Node* region = uu->in(0); 2581 int nb = 0; 2582 for (uint k = 1; k < uu->req(); k++) { 2583 if (uu->in(k) == u) { 2584 Node* m = find_mem(region->in(k), nullptr); 2585 if (m != mem) { 2586 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 2587 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2588 if (newmm != u) { 2589 _phase->igvn().replace_input_of(uu, k, newmm); 2590 nb++; 2591 --jmax; 2592 } 2593 } 2594 } 2595 } 2596 if (nb > 0) { 2597 --j; 2598 } 2599 } else { 2600 Node* m = find_mem(_phase->ctrl_or_self(uu), uu); 2601 if (m != mem) { 2602 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 2603 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2604 if (newmm != u) { 2605 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2606 --j, --jmax; 2607 } 2608 } 2609 } 2610 } 2611 } 2612 } else if (u->is_Phi()) { 2613 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2614 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) { 2615 Node* region = u->in(0); 2616 bool replaced = false; 2617 for (uint j = 1; j < u->req(); j++) { 2618 if (u->in(j) == mem) { 2619 Node* m = find_mem(region->in(j), nullptr); 2620 Node* nnew = m; 2621 if (m != mem) { 2622 if (u->adr_type() == TypePtr::BOTTOM) { 2623 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m)); 2624 nnew = mm; 2625 } 2626 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 2627 _phase->igvn().replace_input_of(u, j, nnew); 2628 replaced = true; 2629 } 2630 } 2631 } 2632 if (replaced) { 2633 --i; 2634 } 2635 } 2636 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2637 u->adr_type() == nullptr) { 2638 assert(u->adr_type() != nullptr || 2639 u->Opcode() == Op_Rethrow || 2640 u->Opcode() == Op_Return || 2641 u->Opcode() == Op_SafePoint || 2642 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2643 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2644 u->Opcode() == Op_CallLeaf, ""); 2645 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2646 if (m != mem) { 2647 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m)); 2648 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2649 --i; 2650 } 2651 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2652 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2653 if (m != mem) { 2654 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2655 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2656 --i; 2657 } 2658 } else if (u->adr_type() != TypePtr::BOTTOM && 2659 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) { 2660 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2661 assert(m != mem, ""); 2662 // u is on the wrong slice... 2663 assert(u->is_ClearArray(), ""); 2664 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2665 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2666 --i; 2667 } 2668 } 2669 } 2670 #ifdef ASSERT 2671 assert(new_mem->outcnt() > 0, ""); 2672 for (int i = 0; i < phis.length(); i++) { 2673 Node* n = phis.at(i); 2674 assert(n->outcnt() > 0, "new phi must have uses now"); 2675 } 2676 #endif 2677 } 2678 2679 void MemoryGraphFixer::record_new_ctrl(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl) { 2680 if (mem_for_ctrl != mem && new_ctrl != ctrl) { 2681 _memory_nodes.map(ctrl->_idx, mem); 2682 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 2683 } 2684 } 2685 2686 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { 2687 MergeMemNode* mm = MergeMemNode::make(mem); 2688 mm->set_memory_at(_alias, rep_proj); 2689 _phase->register_new_node(mm, rep_ctrl); 2690 return mm; 2691 } 2692 2693 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const { 2694 MergeMemNode* newmm = nullptr; 2695 MergeMemNode* u_mm = u->as_MergeMem(); 2696 Node* c = _phase->get_ctrl(u); 2697 if (_phase->is_dominator(c, rep_ctrl)) { 2698 c = rep_ctrl; 2699 } else { 2700 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 2701 } 2702 if (u->outcnt() == 1) { 2703 if (u->req() > (uint)_alias && u->in(_alias) == mem) { 2704 _phase->igvn().replace_input_of(u, _alias, rep_proj); 2705 --i; 2706 } else { 2707 _phase->igvn().rehash_node_delayed(u); 2708 u_mm->set_memory_at(_alias, rep_proj); 2709 } 2710 newmm = u_mm; 2711 _phase->set_ctrl_and_loop(u, c); 2712 } else { 2713 // can't simply clone u and then change one of its input because 2714 // it adds and then removes an edge which messes with the 2715 // DUIterator 2716 newmm = MergeMemNode::make(u_mm->base_memory()); 2717 for (uint j = 0; j < u->req(); j++) { 2718 if (j < newmm->req()) { 2719 if (j == (uint)_alias) { 2720 newmm->set_req(j, rep_proj); 2721 } else if (newmm->in(j) != u->in(j)) { 2722 newmm->set_req(j, u->in(j)); 2723 } 2724 } else if (j == (uint)_alias) { 2725 newmm->add_req(rep_proj); 2726 } else { 2727 newmm->add_req(u->in(j)); 2728 } 2729 } 2730 if ((uint)_alias >= u->req()) { 2731 newmm->set_memory_at(_alias, rep_proj); 2732 } 2733 _phase->register_new_node(newmm, c); 2734 } 2735 return newmm; 2736 } 2737 2738 bool MemoryGraphFixer::should_process_phi(Node* phi) const { 2739 if (phi->adr_type() == TypePtr::BOTTOM) { 2740 Node* region = phi->in(0); 2741 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 2742 Node* uu = region->fast_out(j); 2743 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) { 2744 return false; 2745 } 2746 } 2747 return true; 2748 } 2749 return _phase->C->get_alias_index(phi->adr_type()) == _alias; 2750 } 2751 2752 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const { 2753 uint last = _phase-> C->unique(); 2754 MergeMemNode* mm = nullptr; 2755 assert(mem->bottom_type() == Type::MEMORY, ""); 2756 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2757 Node* u = mem->out(i); 2758 if (u != replacement && u->_idx < last) { 2759 if (u->is_MergeMem()) { 2760 MergeMemNode* u_mm = u->as_MergeMem(); 2761 if (u_mm->memory_at(_alias) == mem) { 2762 MergeMemNode* newmm = nullptr; 2763 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2764 Node* uu = u->fast_out(j); 2765 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2766 if (uu->is_Phi()) { 2767 if (should_process_phi(uu)) { 2768 Node* region = uu->in(0); 2769 int nb = 0; 2770 for (uint k = 1; k < uu->req(); k++) { 2771 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) { 2772 if (newmm == nullptr) { 2773 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2774 } 2775 if (newmm != u) { 2776 _phase->igvn().replace_input_of(uu, k, newmm); 2777 nb++; 2778 --jmax; 2779 } 2780 } 2781 } 2782 if (nb > 0) { 2783 --j; 2784 } 2785 } 2786 } else { 2787 if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { 2788 if (newmm == nullptr) { 2789 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2790 } 2791 if (newmm != u) { 2792 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2793 --j, --jmax; 2794 } 2795 } 2796 } 2797 } 2798 } 2799 } else if (u->is_Phi()) { 2800 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2801 Node* region = u->in(0); 2802 if (should_process_phi(u)) { 2803 bool replaced = false; 2804 for (uint j = 1; j < u->req(); j++) { 2805 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) { 2806 Node* nnew = rep_proj; 2807 if (u->adr_type() == TypePtr::BOTTOM) { 2808 if (mm == nullptr) { 2809 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2810 } 2811 nnew = mm; 2812 } 2813 _phase->igvn().replace_input_of(u, j, nnew); 2814 replaced = true; 2815 } 2816 } 2817 if (replaced) { 2818 --i; 2819 } 2820 2821 } 2822 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2823 u->adr_type() == nullptr) { 2824 assert(u->adr_type() != nullptr || 2825 u->Opcode() == Op_Rethrow || 2826 u->Opcode() == Op_Return || 2827 u->Opcode() == Op_SafePoint || 2828 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2829 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2830 u->Opcode() == Op_CallLeaf, "%s", u->Name()); 2831 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2832 if (mm == nullptr) { 2833 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2834 } 2835 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2836 --i; 2837 } 2838 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2839 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2840 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 2841 --i; 2842 } 2843 } 2844 } 2845 } 2846 } 2847 2848 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, DecoratorSet decorators) 2849 : Node(ctrl, obj), _decorators(decorators) { 2850 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); 2851 } 2852 2853 DecoratorSet ShenandoahLoadReferenceBarrierNode::decorators() const { 2854 return _decorators; 2855 } 2856 2857 uint ShenandoahLoadReferenceBarrierNode::size_of() const { 2858 return sizeof(*this); 2859 } 2860 2861 static DecoratorSet mask_decorators(DecoratorSet decorators) { 2862 return decorators & (ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF | IN_NATIVE); 2863 } 2864 2865 uint ShenandoahLoadReferenceBarrierNode::hash() const { 2866 uint hash = Node::hash(); 2867 hash += mask_decorators(_decorators); 2868 return hash; 2869 } 2870 2871 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { 2872 return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && 2873 mask_decorators(_decorators) == mask_decorators(((const ShenandoahLoadReferenceBarrierNode&)n)._decorators); 2874 } 2875 2876 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { 2877 if (in(ValueIn) == nullptr || in(ValueIn)->is_top()) { 2878 return Type::TOP; 2879 } 2880 const Type* t = in(ValueIn)->bottom_type(); 2881 if (t == TypePtr::NULL_PTR) { 2882 return t; 2883 } 2884 2885 if (ShenandoahBarrierSet::is_strong_access(decorators())) { 2886 return t; 2887 } 2888 2889 return t->meet(TypePtr::NULL_PTR); 2890 } 2891 2892 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { 2893 // Either input is TOP ==> the result is TOP 2894 const Type *t2 = phase->type(in(ValueIn)); 2895 if( t2 == Type::TOP ) return Type::TOP; 2896 2897 if (t2 == TypePtr::NULL_PTR) { 2898 return t2; 2899 } 2900 2901 if (ShenandoahBarrierSet::is_strong_access(decorators())) { 2902 return t2; 2903 } 2904 2905 return t2->meet(TypePtr::NULL_PTR); 2906 } 2907 2908 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { 2909 Node* value = in(ValueIn); 2910 if (!needs_barrier(phase, value)) { 2911 return value; 2912 } 2913 return this; 2914 } 2915 2916 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { 2917 Unique_Node_List visited; 2918 return needs_barrier_impl(phase, n, visited); 2919 } 2920 2921 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { 2922 if (n == nullptr) return false; 2923 if (visited.member(n)) { 2924 return false; // Been there. 2925 } 2926 visited.push(n); 2927 2928 if (n->is_Allocate()) { 2929 // tty->print_cr("optimize barrier on alloc"); 2930 return false; 2931 } 2932 if (n->is_Call()) { 2933 // tty->print_cr("optimize barrier on call"); 2934 return false; 2935 } 2936 2937 const Type* type = phase->type(n); 2938 if (type == Type::TOP) { 2939 return false; 2940 } 2941 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 2942 // tty->print_cr("optimize barrier on null"); 2943 return false; 2944 } 2945 if (type->make_oopptr() && type->make_oopptr()->const_oop() != nullptr) { 2946 // tty->print_cr("optimize barrier on constant"); 2947 return false; 2948 } 2949 2950 switch (n->Opcode()) { 2951 case Op_AddP: 2952 return true; // TODO: Can refine? 2953 case Op_LoadP: 2954 case Op_ShenandoahCompareAndExchangeN: 2955 case Op_ShenandoahCompareAndExchangeP: 2956 case Op_CompareAndExchangeN: 2957 case Op_CompareAndExchangeP: 2958 case Op_GetAndSetN: 2959 case Op_GetAndSetP: 2960 return true; 2961 case Op_Phi: { 2962 for (uint i = 1; i < n->req(); i++) { 2963 if (needs_barrier_impl(phase, n->in(i), visited)) return true; 2964 } 2965 return false; 2966 } 2967 case Op_CheckCastPP: 2968 case Op_CastPP: 2969 return needs_barrier_impl(phase, n->in(1), visited); 2970 case Op_Proj: 2971 return needs_barrier_impl(phase, n->in(0), visited); 2972 case Op_ShenandoahLoadReferenceBarrier: 2973 // tty->print_cr("optimize barrier on barrier"); 2974 return false; 2975 case Op_Parm: 2976 // tty->print_cr("optimize barrier on input arg"); 2977 return false; 2978 case Op_DecodeN: 2979 case Op_EncodeP: 2980 return needs_barrier_impl(phase, n->in(1), visited); 2981 case Op_LoadN: 2982 return true; 2983 case Op_CMoveN: 2984 case Op_CMoveP: 2985 return needs_barrier_impl(phase, n->in(2), visited) || 2986 needs_barrier_impl(phase, n->in(3), visited); 2987 case Op_ShenandoahIUBarrier: 2988 return needs_barrier_impl(phase, n->in(1), visited); 2989 case Op_CreateEx: 2990 return false; 2991 default: 2992 break; 2993 } 2994 #ifdef ASSERT 2995 tty->print("need barrier on?: "); 2996 tty->print_cr("ins:"); 2997 n->dump(2); 2998 tty->print_cr("outs:"); 2999 n->dump(-2); 3000 ShouldNotReachHere(); 3001 #endif 3002 return true; 3003 }