1 /*
   2  * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "code/vmreg.inline.hpp"
  28 #include "compiler/oopMap.hpp"
  29 #include "compiler/oopMap.inline.hpp"
  30 #include "compiler/oopMapStubGenerator.hpp"
  31 #include "gc/shared/collectedHeap.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #ifdef COMPILER1
  34 #include "c1/c1_Defs.hpp"
  35 #endif
  36 #ifdef COMPILER2
  37 #include "opto/optoreg.hpp"
  38 #endif
  39 
  40 // TODO PERF: Use non-temporal stores in freeze and non-temporal loads in thaw. See https://stackoverflow.com/a/45636785/750563
  41 
  42 static Register vsp = c_rarg0;
  43 static Register ref_stack = c_rarg1;
  44 static Register frame_ptr = c_rarg2;
  45 static Register frame_info = c_rarg2;
  46 static Register hsp = c_rarg3;
  47 
  48 #ifdef _WINDOWS
  49 static Register initial_index = rdi;
  50 static Register fp_oop_info = rsi;
  51 #else
  52 static Register initial_index = c_rarg4;
  53 static Register fp_oop_info = c_rarg5;
  54 #endif
  55 
  56 class OptOopMapStubGenerator : public StubCodeGenerator {
  57   // pmovzx to zero extend from compressed to uncompressed
  58   class MemSlice : public ResourceObj {
  59   private:
  60     Register _base;
  61     int _offset;
  62     int _used; // 1 bit for every dword
  63     int _uncompress; // 1 bit for every slot
  64     bool _is_read;
  65     MemSlice* _next;
  66 
  67     void add_read(int offset, int width) {
  68       int oops = width / heapOopSize;
  69       int n = offset - _offset;
  70       if (n > 0) {
  71         n /= heapOopSize;
  72       }
  73 
  74       while (oops-- > 0) {
  75         _used |= (1 << n);
  76         ++n;
  77       }
  78     }
  79 
  80     int max_read_offset() {
  81       return 16; // movdqu
  82     }
  83 
  84     bool need_movdqu() const {
  85       return (_used & 0xc);
  86     }
  87 
  88     bool need_movptr() const  {
  89       return (_used & 0x2);
  90     }
  91 
  92     void emit_read(MacroAssembler* masm) {
  93       if (need_movdqu()) {
  94         masm->movdqu(xmm1, Address(_base, _offset));
  95       } else if (need_movptr()) {
  96         masm->movptr(r13, Address(_base, _offset));
  97       } else {
  98         masm->xorptr(r13, r13);
  99         masm->movl(r13, Address(_base, _offset));
 100       }
 101     }
 102 
 103     void emit_extract(MacroAssembler* masm, int offset, int width) {
 104       if (need_movdqu()) {
 105         if (width == 8) {
 106           if (offset - _offset == 0) {
 107             masm->pextrq(rax, xmm1, 0);
 108           } else if (offset - _offset == 4) { /* |narrow|wide|narrow| */
 109             masm->pextrd(rax, xmm1, 1);
 110             masm->pextrd(r13, xmm1, 2);
 111             masm->shlptr(r13, 32);
 112             masm->orptr(rax, r13);
 113           } else if (offset - _offset == 8) {
 114             masm->pextrq(rax, xmm1, 1);
 115           } else {
 116             assert(false, "");
 117           }
 118         } else if (width == 4) {
 119           masm->xorptr(rax, rax);
 120           if (offset - _offset == 0) {
 121             masm->pextrd(rax, xmm1, 0);
 122           } else if (offset - _offset == 4) {
 123             masm->pextrd(rax, xmm1, 1);
 124           } else if (offset - _offset == 8) {
 125             masm->pextrd(rax, xmm1, 2);
 126           } else if (offset - _offset == 12) {
 127             masm->pextrd(rax, xmm1, 3);
 128           } else {
 129             assert(false, "");
 130           }
 131         } else {
 132           assert(false, "");
 133         }
 134       } else if (need_movptr()) {
 135         if (width == 8) {
 136           masm->movptr(rax, r13);
 137         } else {
 138           assert(width == 4, "");
 139           if (offset - _offset == 0) {
 140             masm->movl(rax, r13);
 141           } else if (offset - _offset == 4) {
 142             masm->movptr(rax, r13);
 143             masm->shrptr(rax, 32);
 144           } else {
 145             assert(false, "");
 146           }
 147         }
 148       } else {
 149         assert(width == 4, "");
 150         masm->xorptr(rax, rax);
 151         masm->movl(rax, r13);
 152       }
 153     }
 154 
 155     void read(MacroAssembler* masm, int offset, int width) {
 156       // offset is offset from base
 157       if (!_is_read) {
 158         _is_read = true;
 159         emit_read(masm);
 160       }
 161       emit_extract(masm, offset, width);
 162     }
 163 
 164   public:
 165     MemSlice(Register base, int offset) : _base(base), _offset(offset), _used(0), _is_read(false), _next(NULL) {}
 166 
 167     MemSlice* next() { return _next; }
 168 
 169     void read_oop(int offset) {
 170       add_read(offset, 8); // find the right constants
 171     }
 172 
 173     int offset() const {
 174       return _offset;
 175     }
 176 
 177     void set_next(MemSlice* slice) {
 178       _next = slice;
 179     }
 180 
 181     void read_narrow(MacroAssembler* masm, int offset) {
 182       read(masm, offset, 4);
 183     }
 184 
 185     void read_wide(MacroAssembler* masm, int offset) {
 186       read(masm, offset, 8);
 187     }
 188 
 189     void read_narrow(int offset) {
 190       add_read(offset, 4); // find the right constants
 191     }
 192 
 193     bool can_read(Register base, int offset, int width) {
 194       if (base != _base) {
 195         return false;
 196       }
 197 
 198       int diff = (offset + width) - _offset;
 199       if (offset - _offset < 0 || diff > max_read_offset()) {
 200         return false;
 201       }
 202       return true;
 203     }
 204   };
 205 
 206   class OMV : public ResourceObj {
 207   private:
 208     OopMapValue _omv;
 209     MemSlice* _memory;
 210 
 211     OMV* _next;
 212     OMV* _derived;
 213     OMV* _adjacent;
 214 
 215     int _id;
 216     bool _done;
 217 
 218     OMV* last_adjacent() {
 219       OMV* n = _adjacent;
 220       if (n == NULL) {
 221         return this;
 222       }
 223       while (n->_next != NULL) {
 224         n = n->_next;
 225       }
 226 
 227       return n;
 228     }
 229 
 230   public:
 231     OMV(OopMapValue omv, int id) : _omv(omv), _memory(NULL), _next(NULL), _derived(NULL), _adjacent(NULL), _id(id), _done(false) {}
 232 
 233     int id() { return _id; }
 234     OMV* next() { return _next; }
 235     OMV* adjacent() { return _adjacent; }
 236     OMV* derived() { return _derived; }
 237     OopMapValue omv() { return _omv; }
 238 
 239     bool need_derived() {
 240       OMV* d = _derived;
 241       while (d != NULL) {
 242         if (!d->is_done()) {
 243           return true;
 244         }
 245         d = d->next();
 246       }
 247       return false;
 248     }
 249 
 250     void set_memory(MemSlice* memory) {
 251       _memory = memory;
 252     }
 253 
 254     bool is_reg() const {
 255       return _omv.reg()->is_reg();
 256     }
 257 
 258     Register base() { // need something else than Register here
 259       return (is_reg() ? rdx : rdi);
 260     }
 261 
 262     int offset() {
 263       if (is_reg()) {
 264         return 0;
 265       }
 266       return _omv.reg()->reg2stack() * VMRegImpl::stack_slot_size;
 267     }
 268 
 269     MemSlice* memory() {
 270       assert(_memory != NULL, "");
 271       return _memory;
 272     }
 273 
 274     int number_adjacent() {
 275       int n = 0;
 276       OMV* a = _adjacent;
 277       while (a != NULL) {
 278         ++n;
 279         a = a->next();
 280       }
 281       return n;
 282     }
 283 
 284     void clear_done() {
 285       _done = false;
 286 
 287       if (_adjacent != NULL) {
 288         _adjacent->clear_done();
 289       }
 290 
 291       if (_derived != NULL) {
 292         _derived->clear_done();
 293       }
 294 
 295       if (_next != NULL) {
 296         _next->clear_done();
 297       }
 298     }
 299 
 300     void add_derived(OMV* o) {
 301       if (_derived == NULL) {
 302         _derived = o;
 303       } else {
 304         OMV* n = _derived;
 305         while (n->_next != NULL) {
 306           n = n->_next;
 307         }
 308         n->_next = o;
 309       }
 310     }
 311 
 312     void mark_done() {
 313       _done = true;
 314     }
 315 
 316     bool is_done() {
 317       return _done;
 318     }
 319 
 320     void add_adjacent(OMV* o) {
 321       if (_adjacent == NULL) {
 322         _adjacent = o;
 323       } else {
 324         OMV* n = _adjacent;
 325         while (n->_next != NULL) {
 326           n = n->_next;
 327         }
 328         n->_next = o;
 329       }
 330     }
 331 
 332     void set_next(OMV* o) {
 333       _next = o;
 334     }
 335 
 336     bool is_base(const OMV* o) const {
 337       return _omv.reg() == o->_omv.content_reg();
 338     }
 339 
 340     bool is_adjacent(const OMV* o) {
 341       if (_omv.type() != o->_omv.type()) {
 342         return false;
 343       }
 344 
 345       if (_omv.reg()->is_reg() || o->_omv.reg()->is_reg()) {
 346         return false;
 347       }
 348 
 349       OMV* adj = last_adjacent();
 350 
 351       int dist = (_omv.type() == OopMapValue::oop_value) ? 8 : 4;
 352       int v1_offset = adj->omv().reg()->reg2stack() * VMRegImpl::stack_slot_size;
 353       int o_offset = o->_omv.reg()->reg2stack() * VMRegImpl::stack_slot_size;
 354 
 355       if (o_offset > v1_offset) {
 356         return (o_offset - v1_offset) == dist;
 357       }
 358       return false;
 359     }
 360   };
 361 
 362   class OopWriter {
 363   public:
 364     ~OopWriter() {}
 365     virtual void write_narrow(Register reg) = 0;
 366     virtual void write(Register reg) = 0;
 367   };
 368 
 369 
 370 private:
 371   MacroAssembler* _masm;
 372   const ImmutableOopMap& _map;
 373   bool _link_offset_loaded;
 374   bool _written_rbp_index;
 375   address _freeze;
 376   address _thaw;
 377 
 378   int _num_oops;
 379   int _num_derived;
 380 
 381   OMV* _head;
 382   MemSlice* _mem_head;
 383   MemSlice* _mem_last;
 384 
 385 
 386   void append_mem_slice(MemSlice* slice) {
 387     if (_mem_head == NULL) {
 388       _mem_head = slice;
 389       _mem_last = slice;
 390     } else {
 391       _mem_last->set_next(slice);
 392       _mem_last = slice;
 393     }
 394   }
 395 
 396   MemSlice* find_slice(Register base, int offset, int width) {
 397     MemSlice* c = _mem_head;
 398     while (c != NULL) {
 399       if (c->can_read(base, offset, width)) {
 400         return c;
 401       }
 402       c = c->next();
 403     }
 404 
 405     c = new MemSlice(base, offset);
 406     append_mem_slice(c);
 407     return c;
 408   }
 409 
 410   MemSlice* read_wide(Register base, int offset) {
 411     MemSlice* slice = find_slice(base, offset, 8); // find constant
 412     assert(offset - slice->offset() >= 0, "");
 413     slice->read_oop(offset);
 414     return slice;
 415   }
 416 
 417   MemSlice* read_narrow(Register base, int offset) {
 418     MemSlice* slice = find_slice(base, offset, 4); // find constant
 419     assert(offset - slice->offset() >= 0, "");
 420     slice->read_narrow(offset);
 421     return slice;
 422   }
 423 
 424 public:
 425   OptOopMapStubGenerator(CodeBuffer* code, const ImmutableOopMap& map) : StubCodeGenerator(code), _map(map) {
 426     _masm = new MacroAssembler(code);
 427     _link_offset_loaded = false;
 428     _written_rbp_index = false;
 429     _freeze = NULL;
 430     _thaw = NULL;
 431     _head = NULL;
 432     _mem_head = NULL;
 433     _mem_last = NULL;
 434 
 435     _num_oops = 0;
 436     _num_derived = 0;
 437 
 438     OMV* last = NULL;
 439     OMV* last_insert = NULL;
 440     int count = 0;
 441 
 442     for (OopMapStream oms(&_map); !oms.is_done(); oms.next()) {
 443       OopMapValue omv = oms.current();
 444       OMV *o = new OMV(omv, count);
 445 
 446       if (omv.is_oop_or_narrow()) {
 447 
 448         if (omv.is_oop()) {
 449           o->set_memory(read_wide(o->base(), o->offset()));
 450         } else {
 451           // narrow
 452           o->set_memory(read_narrow(o->base(), o->offset()));
 453         }
 454 
 455         ++_num_oops;
 456         if (_head == NULL) {
 457           _head = o;
 458           last = o;
 459           last_insert = o;
 460         } else {
 461           if (last->is_adjacent(o)) {
 462             last->add_adjacent(o);
 463           } else {
 464             last->set_next(o);
 465             last = o;
 466           }
 467           last_insert = o;
 468         }
 469         ++count;
 470       } else if (omv.is_derived_oop()) {
 471         ++_num_derived;
 472         o->set_memory(read_wide(o->base(), o->offset()));
 473         assert(_head != NULL, "");
 474         assert(last != NULL, "");
 475         assert(last_insert != NULL, "");
 476         if (!last_insert->is_base(o)) {
 477           what(o);
 478         }
 479         assert(last_insert->is_base(o), "");
 480         last_insert->add_derived(o);
 481       }
 482       ++count;
 483     }
 484   }
 485 
 486   void what(OMV* last) {
 487     tty->print_cr("!omv %p", last);
 488   }
 489 
 490   address thaw_stub() {
 491     return _thaw;
 492   }
 493 
 494   bool has_rbp_index() {
 495     return _written_rbp_index;
 496   }
 497 
 498   void load_link_offset() {
 499     if (!_link_offset_loaded) {
 500       _link_offset_loaded = true;
 501       // _masm->movptr(frame_ptr, Address(frame_ptr, RegisterMap::link_offset()));
 502     }
 503   }
 504 
 505   void store_rbp_oop(Register idx) {
 506     assert(_written_rbp_index == false, "");
 507     _masm->movl(Address(fp_oop_info, 0), 1); // offset to bool has_fp_index
 508     _masm->movl(Address(fp_oop_info, 4), idx); // offset to int fp_index
 509     _written_rbp_index = true;
 510   }
 511 
 512   void thaw_single_oop(OMV* o, int& pos) {
 513     bool has_derived = o->derived() != NULL;
 514     OopMapValue omv = o->omv();
 515     VMReg reg = omv.reg();
 516     // read value from array
 517     if (UseCompressedOops) {
 518 
 519       _masm->movl(rax, Address(ref_stack, pos));
 520       if (omv.type() == OopMapValue::oop_value) {
 521         _masm->decode_heap_oop(rax);
 522         if (has_derived) {
 523           _masm->movptr(c_rarg3, rax);
 524         }
 525       } else if (omv.type() == OopMapValue::narrowoop_value && has_derived) {
 526         _masm->movptr(c_rarg3, rax);
 527         _masm->decode_heap_oop(c_rarg3);
 528       }
 529 
 530     } else {
 531       _masm->movptr(rax, Address(ref_stack, pos));
 532       if (has_derived) {
 533         _masm->movptr(c_rarg3, rax);
 534       }
 535     }
 536 
 537 
 538     if (reg->is_reg()) {
 539       assert(reg == rbp->as_VMReg(), "must");
 540       //load_link_offset();
 541       if (omv.type() == OopMapValue::oop_value) {
 542         _masm->movptr(Address(frame_info, 0), rax);
 543       } else {
 544         assert(UseCompressedOops, "");
 545         // narrow oop
 546         _masm->movl(Address(frame_info, 0), rax);
 547       }
 548 
 549       /* test
 550          store_rbp_oop(r8);
 551          _masm->subl(r8, 1);
 552          */
 553 
 554     } else {
 555       int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 556 
 557       if (omv.type() == OopMapValue::oop_value) {
 558         _masm->movptr(Address(vsp, sp_offset_in_bytes), rax);
 559       } else {
 560         // narrow oop
 561         assert(UseCompressedOops, "");
 562         _masm->movl(Address(vsp, sp_offset_in_bytes), rax);
 563       }
 564 
 565       // index is refStack_length - index
 566       /* test
 567          _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
 568          _masm->subl(r8, 1);
 569          */
 570     }
 571 
 572     if (UseCompressedOops) {
 573       pos += 4;
 574     } else {
 575       pos += 8;
 576     }
 577   }
 578 
 579   void thaw_single_derived(OopMapValue omv, Register base) {
 580     VMReg reg = omv.reg();
 581     bool derived_is_reg = false;
 582     // read the derived value into rax
 583     int derived_sp_offset_in_bytes = -1;
 584     if (reg->is_reg()) {
 585       _masm->movptr(rax, Address(frame_info, 0)); // offset -> rax
 586       derived_is_reg = true;
 587     } else {
 588       derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 589       _masm->movptr(rax, Address(vsp, derived_sp_offset_in_bytes)); // offset -> rax
 590     }
 591 
 592     address narrow_oop_base = CompressedOops::base();
 593     assert(narrow_oop_base == (address) NULL, "");
 594     // maybe we need to test for is_narrow_oop_base too...
 595 
 596     _masm->addptr(rax, base); // offset in (base + offset)
 597 
 598     if (reg->is_reg()) {
 599       _masm->movptr(Address(frame_info, 0), rax);
 600     } else {
 601       _masm->movptr(Address(vsp, derived_sp_offset_in_bytes), rax);
 602     }
 603   }
 604 
 605   void thaw_derived(OMV* o) {
 606     OMV* d = o->derived();
 607     if (d != NULL) {
 608       Label L_next;
 609       Register base = c_rarg3;
 610 
 611       _masm->testptr(base, base);
 612       _masm->jcc(Assembler::zero, L_next);
 613 
 614       while (d) {
 615         thaw_single_derived(d->omv(), base);
 616         d = d->next();
 617       }
 618 
 619       _masm->bind(L_next);
 620     }
 621   }
 622 
 623   bool need_heapbase() {
 624     return (UseCompressedOops && CompressedOops::base() != NULL) || CheckCompressedOops;
 625   }
 626 
 627   void generate_thaw(const ImmutableOopMap& map) {
 628     // need to reset these first
 629     _link_offset_loaded = false;
 630     _written_rbp_index = false;
 631     _head->clear_done();
 632 
 633     _masm->align(8);
 634     _thaw = _masm->pc();
 635 
 636     if (need_heapbase()) {
 637       _masm->push(r12);
 638       _masm->reinit_heapbase();
 639     }
 640 
 641     int pos = 0;
 642     {
 643 
 644       OMV* o = _head;
 645       while (o) {
 646         thaw_single_oop(o, pos);
 647         thaw_derived(o);
 648         OMV* a = o->adjacent();
 649         while (a) {
 650           thaw_single_oop(a, pos);
 651           thaw_derived(a);
 652           a = a->next();
 653         }
 654         o = o->next();
 655       }
 656     }
 657 
 658     if (need_heapbase()) {
 659       _masm->pop(r12);
 660     }
 661     _masm->movl(rax, map.num_oops());
 662     _masm->ret(0);
 663   }
 664 
 665   void freeze_single_oop(OMV* o, int& pos, OopWriter& writer) {
 666     if (o->is_done()) {
 667       return;
 668     }
 669     o->mark_done();
 670 
 671     OopMapValue omv = o->omv();
 672     VMReg reg = omv.reg();
 673 
 674     OMV* d = o->derived();
 675     if (reg->is_reg()) {
 676       assert(reg == rbp->as_VMReg(), "must");
 677       load_link_offset();
 678       if (omv.type() == OopMapValue::oop_value) {
 679         o->memory()->read_wide(_masm, 0);
 680 
 681         //_masm->movptr(rax, Address(rdx, 0));
 682 
 683         if (d != NULL) {
 684           assert(_map.has_derived(), "");
 685           _masm->movptr(r14, rax);
 686         }
 687 
 688         if (UseCompressedOops) {
 689           // compress
 690           _masm->encode_heap_oop(rax);
 691           writer.write_narrow(rax);
 692           //_masm->movl(Address(rsi, pos), rax);
 693         } else {
 694           writer.write(rax);
 695           //_masm->movptr(Address(rsi, pos), rax);
 696         }
 697       } else {
 698         assert(UseCompressedOops, "");
 699         // narrow oop
 700         o->memory()->read_narrow(_masm, 0);
 701         //_masm->movl(rax, Address(rdx, 0));
 702         if (d != NULL) {
 703           guarantee(false, "should not have narrow as base");
 704           assert(_map.has_derived(), "");
 705           _masm->movptr(r14, rax);
 706           _masm->decode_heap_oop(r14);
 707         }
 708         writer.write_narrow(rax);
 709         //_masm->movl(Address(rsi, pos), rax);
 710       }
 711 
 712       /* test
 713          store_rbp_oop(r8);
 714          _masm->subl(r8, 1);
 715          */
 716 
 717     } else {
 718       int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 719 
 720       if (omv.type() == OopMapValue::oop_value) {
 721         //_masm->movptr(rax, Address(rdi, sp_offset_in_bytes));
 722         o->memory()->read_wide(_masm, sp_offset_in_bytes);
 723 
 724         if (d != NULL) {
 725           assert(_map.has_derived(), "");
 726           _masm->movptr(r14, rax);
 727         }
 728 
 729         if (UseCompressedOops) {
 730           // compress
 731           _masm->encode_heap_oop(rax);
 732           writer.write_narrow(rax);
 733           //_masm->movl(Address(rsi, pos), rax);
 734         } else {
 735           writer.write(rax);
 736           //_masm->movptr(Address(rsi, pos), rax);
 737         }
 738       } else {
 739         // narrow oop
 740         assert(UseCompressedOops, "");
 741         o->memory()->read_narrow(_masm, sp_offset_in_bytes);
 742         //_masm->movl(rax, Address(rdi, sp_offset_in_bytes));
 743         writer.write_narrow(rax);
 744         //_masm->movl(Address(rsi, pos), rax);
 745 
 746         if (d != NULL) {
 747           guarantee(false, "should not have narrow as base");
 748           assert(_map.has_derived(), "");
 749           _masm->movptr(r14, rax);
 750           _masm->decode_heap_oop(r14);
 751         }
 752       }
 753 
 754       // index is refStack_length - index
 755       /* test
 756          _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
 757          _masm->subl(r8, 1);
 758          */
 759     }
 760     /*if (UseCompressedOops) {
 761       pos += 4;
 762     } else {
 763       pos += 8;
 764     }
 765     */
 766 
 767   }
 768 
 769   void freeze_single_derived(OMV* o) {
 770     if (o->is_done()) {
 771       return;
 772     }
 773     o->mark_done();
 774 
 775     OopMapValue omv = o->omv();
 776     VMReg reg = omv.reg();
 777     bool derived_is_reg = false;
 778     // read the derived value into rax
 779     int derived_sp_offset_in_bytes = -1;
 780     if (reg->is_reg()) {
 781       load_link_offset();
 782       o->memory()->read_wide(_masm, 0);
 783       //_masm->movptr(rax, Address(rdx, 0));
 784       derived_is_reg = true;
 785     } else {
 786       derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 787       o->memory()->read_wide(_masm, derived_sp_offset_in_bytes);
 788       //_masm->movptr(rax, Address(rdi, derived_sp_offset_in_bytes));
 789     }
 790 
 791     // read the base value into rax
 792     address narrow_oop_base = CompressedOops::base();
 793     assert(narrow_oop_base == (address) NULL, "");
 794     // maybe we need to test for is_narrow_oop_base too...
 795 
 796     _masm->subptr(rax, r14); // derived to rax (derived - base)
 797 
 798     if (reg->is_reg()) {
 799       store_rbp_oop(rax);
 800     } else {
 801       _masm->movptr(Address(rcx, derived_sp_offset_in_bytes), rax);
 802     }
 803   }
 804 
 805   void freeze(OMV* o, int& pos, OopWriter& writer) {
 806     freeze_single_oop(o, pos, writer);
 807     freeze_derived(o);
 808 
 809     OMV* a = o->adjacent();
 810     while (a != NULL) {
 811       freeze_single_oop(a, pos, writer);
 812       freeze_derived(a);
 813       a = a->next();
 814     }
 815   }
 816 
 817   void freeze_all_derived(OMV* d) {
 818     while (d != NULL) {
 819       freeze_single_derived(d);
 820       d = d->next();
 821     }
 822   }
 823 
 824   void freeze_derived(OMV* o) {
 825     if (o->need_derived()) {
 826       OMV* d = o->derived();
 827       Label L_next;
 828 
 829       _masm->testptr(r14, r14);
 830       _masm->jcc(Assembler::zero, L_next);
 831 
 832       freeze_all_derived(d);
 833 
 834       _masm->bind(L_next);
 835     }
 836   }
 837 
 838   class BatchWriter : public OopWriter {
 839   private:
 840     MacroAssembler* _masm;
 841     int _num_oops;
 842     int _pos;
 843     int _written;
 844 
 845     int _size;
 846     int _left;
 847     int _slot;
 848 
 849     int next_size() const {
 850       int bytes_remaining = 0;
 851       int left = _num_oops - _written;
 852       if (UseCompressedOops) {
 853         bytes_remaining = left * 4;
 854       } else {
 855         bytes_remaining = left * 8;
 856       }
 857 
 858       if (bytes_remaining >= 16) {
 859         return 16;
 860       } else if (bytes_remaining >= 8) {
 861         return 8;
 862       } else if (bytes_remaining >= 4) {
 863         return 4;
 864       } else {
 865         return 0;
 866       }
 867     }
 868 
 869     bool is_xmm() {
 870       return _size > 8;
 871     }
 872 
 873     bool is_quad() {
 874       return _size == 8;
 875     }
 876 
 877     bool is_word() {
 878       return _size == 4;
 879     }
 880 
 881     void write_narrow_xmm(Register reg) {
 882       _masm->pinsrd(xmm0, reg, _slot);
 883     }
 884 
 885     void write_narrow_quad(Register reg) {
 886       if (_slot == 0) {
 887         _masm->movl(rbx, reg);
 888       } else if (_slot == 1) {
 889         _masm->shlptr(reg, 32);
 890         _masm->orptr(reg, rbx);
 891       } else {
 892         assert(false, "");
 893       }
 894     }
 895 
 896     void write_narrow_word(Register reg) {
 897       // nop
 898     }
 899 
 900   public:
 901     BatchWriter(MacroAssembler* masm, int num_oops) : _masm(masm), _num_oops(num_oops), _pos(0), _written(0), _slot(0) {
 902       _left = _size = next_size();
 903     }
 904 
 905     void finish() {
 906       if (is_xmm()) {
 907         _masm->movdqu(Address(ref_stack, _pos), xmm0);
 908       } else if (is_quad()) {
 909         _masm->movptr(Address(ref_stack, _pos), rax);
 910       } else if (is_word()) {
 911         _masm->movl(Address(ref_stack, _pos), rax);
 912       }
 913       _pos += _size;
 914       _slot = 0;
 915       _left = _size = next_size();
 916     }
 917 
 918     void written(int bytes) {
 919       _written += 1;
 920       _left -= bytes;
 921       _slot += 1;
 922 
 923       if (_left == 0) {
 924         finish();
 925       }
 926     }
 927 
 928     virtual void write_narrow(Register reg) {
 929       if (is_xmm()) {
 930         write_narrow_xmm(reg);
 931       } else if (is_quad()) {
 932         write_narrow_quad(reg);
 933       } else if (is_word()) {
 934         write_narrow_word(reg);
 935       } else {
 936         assert(false, "");
 937       }
 938       written(4);
 939     }
 940 
 941     virtual void write(Register reg) {
 942       assert(false, "unimplemented");
 943       written(8);
 944     }
 945   };
 946 
 947   class SingleWriter : public OopWriter {
 948   private:
 949     MacroAssembler* _masm;
 950     int _pos;
 951 
 952   public:
 953     SingleWriter(MacroAssembler* masm) : _masm(masm), _pos(0) {}
 954     virtual void write_narrow(Register reg) {
 955       _masm->movl(Address(ref_stack, _pos), reg);
 956       _pos += 4;
 957     }
 958     virtual void write(Register reg) {
 959       _masm->movptr(Address(ref_stack, _pos), reg);
 960       _pos += 8;
 961     }
 962   };
 963 
 964   void generate_freeze(const ImmutableOopMap& map) {
 965     _masm->align(8);
 966     _freeze = _masm->pc();
 967 
 968     _masm->push(rbx);
 969 
 970     /* rdi is source (rsp), rsi is destination (first address), rdx (rbp address), rcx (hstack), r8 (initial index (refStack_length - index) ), r9 (fp_oop_info) */
 971     if (need_heapbase()) {
 972       _masm->push(r12);
 973       _masm->reinit_heapbase();
 974     }
 975 
 976     _masm->push(r13);
 977 
 978     if (map.has_derived()) {
 979       _masm->push(r14);
 980     }
 981 
 982     int pos = 0;
 983     {
 984       BatchWriter writer(_masm, _num_oops);
 985       OMV* o = _head;
 986       while (o != NULL) {
 987         freeze(o, pos, writer);
 988         o = o->next();
 989       }
 990     }
 991 
 992     if (map.has_derived()) {
 993       _masm->pop(r14);
 994     }
 995     _masm->pop(r13);
 996     if (need_heapbase()) {
 997       _masm->pop(r12);
 998     }
 999 
1000     _masm->pop(rbx);
1001 
1002     _masm->movl(rax, map.num_oops());
1003     _masm->ret(0);
1004   }
1005 };
1006 
1007 
1008 class OopMapStubGeneratorX86 : public StubCodeGenerator {
1009 private:
1010   MacroAssembler* _masm;
1011   bool _link_offset_loaded;
1012   bool _written_rbp_index;
1013   address _freeze;
1014   address _thaw;
1015   intptr_t _freeze_len, _thaw_len;
1016 
1017 public:
1018   OopMapStubGeneratorX86(CodeBuffer* code) : StubCodeGenerator(code) {
1019     _masm = new MacroAssembler(code);
1020     _link_offset_loaded = false;
1021     _written_rbp_index = false;
1022     _freeze = NULL;
1023     _thaw = NULL;
1024   }
1025 
1026   intptr_t thaw_length() {
1027     return _thaw_len;
1028   }
1029 
1030   address thaw_stub() {
1031     return _thaw;
1032   }
1033 
1034   address freeze_stub() {
1035     return _freeze;
1036   }
1037 
1038   bool has_rbp_index() {
1039     return _written_rbp_index;
1040   }
1041 
1042   void load_link_offset() {
1043     if (!_link_offset_loaded) {
1044       _link_offset_loaded = true;
1045       // _masm->movptr(c_rarg2, Address(frame_info, RegisterMap::link_offset()));
1046     }
1047   }
1048 
1049   void store_rbp_oop(Register idx) {
1050     assert(_written_rbp_index == false, "");
1051     _masm->movl(Address(fp_oop_info, 0), 1); // offset to bool has_fp_index
1052     _masm->movl(Address(fp_oop_info, 4), idx); // offset to int fp_index
1053     _written_rbp_index = true;
1054   }
1055 
1056   void generate_thaw(const ImmutableOopMap& map) {
1057     // need to reset these first
1058     _link_offset_loaded = false;
1059     _written_rbp_index = false;
1060 
1061     _masm->align(8);
1062     _masm->emit_int64(0); // make room for CodeBlob pointer
1063     _thaw = _masm->pc();
1064 
1065     if (UseCompressedOops) {
1066       _masm->push(r12);
1067       _masm->reinit_heapbase();
1068     }
1069 
1070     int pos = 0;
1071     {
1072       int mask = OopMapValue::oop_value | OopMapValue::narrowoop_value;
1073       for (OopMapStream oms(&map,mask); !oms.is_done(); oms.next()) {
1074         OopMapValue omv = oms.current();
1075         VMReg reg = omv.reg();
1076 
1077         // read value from array
1078         if (UseCompressedOops) {
1079           _masm->movl(rax, Address(ref_stack, pos));
1080 
1081           if (omv.type() == OopMapValue::oop_value) {
1082             _masm->decode_heap_oop(rax);
1083           }
1084         } else {
1085           _masm->movptr(rax, Address(ref_stack, pos));
1086         }
1087 
1088         if (reg->is_reg()) {
1089           assert(reg == rbp->as_VMReg(), "must"); // not true for safepoint stub, but we don't use oopmap stubs when preempting
1090           //load_link_offset();
1091           if (omv.type() == OopMapValue::oop_value) {
1092             _masm->movptr(Address(frame_info, 0), rax);
1093           } else {
1094             assert(UseCompressedOops, "");
1095             // narrow oop
1096             _masm->movl(Address(frame_info, 0), rax);
1097           }
1098 
1099           /* test
1100           store_rbp_oop(r8);
1101           _masm->subl(r8, 1);
1102           */
1103 
1104         } else {
1105           int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1106 
1107           if (omv.type() == OopMapValue::oop_value) {
1108             _masm->movptr(Address(vsp, sp_offset_in_bytes), rax);
1109           } else {
1110             // narrow oop
1111             assert(UseCompressedOops, "");
1112             _masm->movl(Address(vsp, sp_offset_in_bytes), rax);
1113           }
1114 
1115           // index is refStack_length - index
1116           /* test
1117           _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
1118           _masm->subl(r8, 1);
1119           */
1120         }
1121         if (UseCompressedOops) {
1122           pos += 4;
1123         } else {
1124           pos += 8;
1125         }
1126       }
1127     }
1128 
1129     {
1130       for (OopMapStream oms(&map,OopMapValue::derived_oop_value); !oms.is_done(); oms.next()) {
1131         OopMapValue omv = oms.current();
1132         VMReg reg = omv.reg();
1133         bool derived_is_reg = false;
1134         // read the derived value into rax
1135         int derived_sp_offset_in_bytes = -1;
1136         if (reg->is_reg()) {
1137           _masm->movptr(rax, Address(frame_info, 0)); // offset -> rax
1138           derived_is_reg = true;
1139         } else {
1140           derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1141           _masm->movptr(rax, Address(vsp, derived_sp_offset_in_bytes)); // offset -> rax
1142         }
1143 
1144         // read the base value into rcx
1145         VMReg content_reg = omv.content_reg();
1146         if (content_reg->is_reg()) {
1147           guarantee(content_reg == rbp->as_VMReg(), "must");
1148           guarantee(derived_is_reg == false, "must");
1149           _masm->movptr(c_rarg3, Address(frame_info, 0)); // base -> rcx
1150         } else {
1151           int sp_offset_in_bytes = content_reg->reg2stack() * VMRegImpl::stack_slot_size;
1152           _masm->movptr(c_rarg3, Address(vsp, sp_offset_in_bytes)); // base -> rdx
1153         }
1154 
1155         address narrow_oop_base = CompressedOops::base();
1156         assert(narrow_oop_base == (address) NULL, "");
1157         // maybe we need to test for is_narrow_oop_base too...
1158         Label L_next;
1159 
1160         _masm->testptr(c_rarg3, c_rarg3);
1161         _masm->jcc(Assembler::zero, L_next);
1162 
1163         _masm->addptr(c_rarg3, rax); // offset in (base + offset)
1164 
1165         if (reg->is_reg()) {
1166           _masm->movptr(Address(frame_info, 0), c_rarg3);
1167         } else {
1168           _masm->movptr(Address(vsp, derived_sp_offset_in_bytes), c_rarg3);
1169         }
1170 
1171         _masm->bind(L_next);
1172       }
1173     }
1174 
1175     if (UseCompressedOops) {
1176       _masm->pop(r12);
1177     }
1178     _masm->movl(rax, map.num_oops());
1179     _masm->ret(0);
1180 
1181     _thaw_len = (intptr_t) _masm->pc() - (intptr_t) _thaw;
1182 
1183   // #ifdef ASSERT
1184   //   tty->print_cr(">>> thaw stub:");
1185   //   _masm->code()->decode(_thaw, _masm->pc());
1186   // #endif
1187   }
1188 
1189   static void freeze_prologue(MacroAssembler* _masm) {
1190   #ifdef _WINDOWS
1191     // Callee save
1192     _masm->movptr(Address(rsp, wordSize * 3), initial_index);
1193     _masm->movptr(Address(rsp, wordSize * 4), fp_oop_info);
1194     _masm->load_sized_value(initial_index, Address(rsp, wordSize * 5), sizeof(int), true);
1195     _masm->movptr(fp_oop_info, Address(rsp, wordSize * 6));
1196   #endif
1197   }
1198 
1199   static void freeze_epilogue(MacroAssembler* _masm) {
1200   #ifdef _WINDOWS
1201     _masm->movptr(initial_index, Address(rsp, wordSize * 3));
1202     _masm->movptr(fp_oop_info, Address(rsp, wordSize * 4));
1203   #endif
1204   }
1205 
1206   void generate_freeze(const ImmutableOopMap& map) {
1207     _masm->align(8);
1208     _masm->emit_int64(0); // make room for thaw stub pointer
1209     _freeze = _masm->pc();
1210 
1211     freeze_prologue(_masm);
1212 
1213     /* rdi is source (rsp), rsi is destination (first address), rdx (rbp address), rcx (hstack), r8 (initial index (refStack_length - index) ), r9 (fp_oop_info) */
1214     if (UseCompressedOops) {
1215       _masm->push(r12);
1216       _masm->reinit_heapbase();
1217     }
1218 
1219     int pos = 0;
1220     {
1221       int mask = OopMapValue::oop_value | OopMapValue::narrowoop_value;
1222       for (OopMapStream oms(&map,mask); !oms.is_done(); oms.next()) {
1223         OopMapValue omv = oms.current();
1224         VMReg reg = omv.reg();
1225 
1226         if (reg->is_reg()) {
1227           assert(reg == rbp->as_VMReg(), "must"); // not true for safepoint stub, but we don't use oopmap stubs when preempting
1228           load_link_offset();
1229           if (omv.type() == OopMapValue::oop_value) {
1230             _masm->movptr(rax, Address(frame_ptr, 0));
1231             if (UseCompressedOops) {
1232               // compress
1233               _masm->encode_heap_oop(rax);
1234               _masm->movl(Address(ref_stack, pos), rax);
1235             } else {
1236               _masm->movptr(Address(ref_stack, pos), rax);
1237             }
1238           } else {
1239             assert(UseCompressedOops, "");
1240             // narrow oop
1241             _masm->movl(rax, Address(frame_ptr, 0));
1242             _masm->movl(Address(ref_stack, pos), rax);
1243           }
1244 
1245           /* test
1246           store_rbp_oop(r8);
1247           _masm->subl(r8, 1);
1248           */
1249 
1250         } else {
1251           int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1252 
1253           if (omv.type() == OopMapValue::oop_value) {
1254             _masm->movptr(rax, Address(vsp, sp_offset_in_bytes));
1255             if (UseCompressedOops) {
1256               // compress
1257               _masm->encode_heap_oop(rax);
1258               _masm->movl(Address(ref_stack, pos), rax);
1259             } else {
1260               _masm->movptr(Address(ref_stack, pos), rax);
1261             }
1262           } else {
1263             // narrow oop
1264             assert(UseCompressedOops, "");
1265             _masm->movl(rax, Address(vsp, sp_offset_in_bytes));
1266             _masm->movl(Address(ref_stack, pos), rax);
1267           }
1268 
1269           // index is refStack_length - index
1270           /* test
1271           _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
1272           _masm->subl(r8, 1);
1273           */
1274         }
1275         if (UseCompressedOops) {
1276           pos += 4;
1277         } else {
1278           pos += 8;
1279         }
1280       }
1281     }
1282 
1283     {
1284       bool derived_init = false;
1285       for (OopMapStream oms(&map,OopMapValue::derived_oop_value); !oms.is_done(); oms.next()) {
1286         if (!derived_init) {
1287           _masm->push(r11);
1288           derived_init = true;
1289         }
1290 
1291         OopMapValue omv = oms.current();
1292         VMReg reg = omv.reg();
1293         bool derived_is_reg = false;
1294         // read the derived value into r11
1295         int derived_sp_offset_in_bytes = -1;
1296         if (reg->is_reg()) {
1297           load_link_offset();
1298           _masm->movptr(r11, Address(frame_ptr, 0));
1299           derived_is_reg = true;
1300         } else {
1301           derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1302           _masm->movptr(r11, Address(vsp, derived_sp_offset_in_bytes));
1303         }
1304 
1305         // read the base value into rax
1306         VMReg content_reg = omv.content_reg();
1307         if (content_reg->is_reg()) {
1308           load_link_offset();
1309           guarantee(content_reg == rbp->as_VMReg(), "must");
1310           guarantee(derived_is_reg == false, "must");
1311           _masm->movptr(rax, Address(frame_ptr, 0));
1312         } else {
1313           int sp_offset_in_bytes = content_reg->reg2stack() * VMRegImpl::stack_slot_size;
1314           _masm->movptr(rax, Address(vsp, sp_offset_in_bytes));
1315         }
1316 
1317         address narrow_oop_base = CompressedOops::base();
1318         assert(narrow_oop_base == (address) NULL, "");
1319         // maybe we need to test for is_narrow_oop_base too...
1320         Label L_next;
1321 
1322         _masm->testptr(rax, rax);
1323         _masm->jcc(Assembler::zero, L_next);
1324 
1325         _masm->subptr(r11, rax); // derived to r11 (derived - base)
1326 
1327         if (reg->is_reg()) {
1328           store_rbp_oop(r11);
1329         } else {
1330           _masm->movptr(Address(hsp, derived_sp_offset_in_bytes), r11);
1331         }
1332 
1333         _masm->bind(L_next);
1334       }
1335       if (derived_init) {
1336         _masm->pop(r11);
1337       }
1338     }
1339 
1340     if (UseCompressedOops) {
1341       _masm->pop(r12);
1342     }
1343     _masm->movl(rax, map.num_oops());
1344 
1345     freeze_epilogue(_masm);
1346 
1347     _masm->ret(0);
1348     _freeze_len = (intptr_t) _masm->pc() - (intptr_t) _freeze;
1349 
1350   // #ifdef ASSERT
1351   //   tty->print_cr(">>> freeze stub:");
1352   //   _masm->code()->decode(_freeze, _masm->pc());
1353   // #endif
1354   }
1355 };
1356 
1357 bool OopMapStubGenerator::generate() {
1358   ResourceMark rm;
1359 
1360   int size = 64 + (_oopmap.count() * 6 * 15) + (CheckCompressedOops ? (256 * _oopmap.count()) : 0); // worst case, 6 instructions per oop, 15 bytes per instruction;
1361 
1362   _blob = BufferBlob::create("oopmap stub", size);
1363   if (_blob == NULL) {
1364     return false;
1365   }
1366 
1367   CodeBuffer buf(_blob);
1368   //OptOopMapStubGenerator cgen(&buf, this);
1369   OopMapStubGeneratorX86 cgen(&buf);
1370   cgen.generate_freeze(_oopmap);
1371   cgen.generate_thaw(_oopmap);
1372 
1373   _freeze_stub = cgen.freeze_stub(); // _blob->code_begin();
1374   _thaw_stub   = cgen.thaw_stub();
1375 
1376   *((address*)_freeze_stub - 1) = _thaw_stub;
1377   *((const CodeBlob**)_thaw_stub - 1) = _cb;
1378 
1379   assert (_cb != NULL, "");
1380 
1381   assert (thaw_stub(_freeze_stub) == _thaw_stub, "");
1382   assert ((intptr_t)_freeze_stub % 8 == 0, "");
1383   assert ((intptr_t)_thaw_stub   % 8 == 0, "");
1384 
1385   return true;
1386 }
1387 
1388 void OopMapStubGenerator::free() {
1389   if (_blob != NULL) {
1390     BufferBlob::free(_blob);
1391     _blob = NULL;
1392   }
1393   _freeze_stub = NULL;
1394   _thaw_stub = NULL;
1395 }
1396 
1397 address OopMapStubGenerator::thaw_stub(address freeze_stub_address) {
1398   return *((address*)freeze_stub_address - 1);
1399 }
1400 
1401 intptr_t OopMapStubGenerator::stub_to_offset(address stub) {
1402   assert (code_cache_base == (intptr_t)CodeCache::low_bound(CodeBlobType::NonNMethod), "");
1403   intptr_t offset = (intptr_t)stub - (intptr_t)code_cache_base;
1404   assert ((offset & 0xffffffff) == offset, "");
1405   return offset;
1406 }
1407 
1408 address OopMapStubGenerator::offset_to_stub(intptr_t offset) {
1409   assert (code_cache_base == (intptr_t)CodeCache::low_bound(CodeBlobType::NonNMethod), "");
1410   return (address)(code_cache_base + offset);
1411 }
1412 
1413 CodeBlob* OopMapStubGenerator::code_blob(address thaw_stub_address) {
1414   return (CodeBlob*)*((address*)thaw_stub_address - 1);
1415 }
1416 
1417 intptr_t OopMapStubGenerator::code_cache_base;
1418 
1419 void OopMapStubGenerator::init() {
1420   code_cache_base = (intptr_t)CodeCache::low_bound(CodeBlobType::NonNMethod);
1421 }