1 /*
   2  * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 // Old code. Might be deleted later if no uses appear.
  28 #if 0
  29 #include "asm/macroAssembler.hpp"
  30 #include "code/codeCache.hpp"
  31 #include "code/vmreg.inline.hpp"
  32 #include "compiler/oopMap.hpp"
  33 #include "compiler/oopMap.inline.hpp"
  34 #include "compiler/oopMapStubGenerator.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "memory/resourceArea.hpp"
  37 #include "runtime/stubRoutines.hpp"
  38 #ifdef COMPILER1
  39 #include "c1/c1_Defs.hpp"
  40 #endif
  41 #ifdef COMPILER2
  42 #include "opto/optoreg.hpp"
  43 #endif
  44 
  45 // TODO PERF: Use non-temporal stores in freeze and non-temporal loads in thaw. See https://stackoverflow.com/a/45636785/750563
  46 
  47 static Register vsp = c_rarg0;
  48 static Register ref_stack = c_rarg1;
  49 static Register frame_ptr = c_rarg2;
  50 static Register frame_info = c_rarg2;
  51 static Register hsp = c_rarg3;
  52 
  53 #ifdef _WINDOWS
  54 static Register initial_index = rdi;
  55 static Register fp_oop_info = rsi;
  56 #else
  57 static Register initial_index = c_rarg4;
  58 static Register fp_oop_info = c_rarg5;
  59 #endif
  60 
  61 class OptOopMapStubGenerator : public StubCodeGenerator {
  62   // pmovzx to zero extend from compressed to uncompressed
  63   class MemSlice : public ResourceObj {
  64   private:
  65     Register _base;
  66     int _offset;
  67     int _used; // 1 bit for every dword
  68     int _uncompress; // 1 bit for every slot
  69     bool _is_read;
  70     MemSlice* _next;
  71 
  72     void add_read(int offset, int width) {
  73       int oops = width / heapOopSize;
  74       int n = offset - _offset;
  75       if (n > 0) {
  76         n /= heapOopSize;
  77       }
  78 
  79       while (oops-- > 0) {
  80         _used |= (1 << n);
  81         ++n;
  82       }
  83     }
  84 
  85     int max_read_offset() {
  86       return 16; // movdqu
  87     }
  88 
  89     bool need_movdqu() const {
  90       return (_used & 0xc);
  91     }
  92 
  93     bool need_movptr() const  {
  94       return (_used & 0x2);
  95     }
  96 
  97     void emit_read(MacroAssembler* masm) {
  98       if (need_movdqu()) {
  99         masm->movdqu(xmm1, Address(_base, _offset));
 100       } else if (need_movptr()) {
 101         masm->movptr(r13, Address(_base, _offset));
 102       } else {
 103         masm->xorptr(r13, r13);
 104         masm->movl(r13, Address(_base, _offset));
 105       }
 106     }
 107 
 108     void emit_extract(MacroAssembler* masm, int offset, int width) {
 109       if (need_movdqu()) {
 110         if (width == 8) {
 111           if (offset - _offset == 0) {
 112             masm->pextrq(rax, xmm1, 0);
 113           } else if (offset - _offset == 4) { /* |narrow|wide|narrow| */
 114             masm->pextrd(rax, xmm1, 1);
 115             masm->pextrd(r13, xmm1, 2);
 116             masm->shlptr(r13, 32);
 117             masm->orptr(rax, r13);
 118           } else if (offset - _offset == 8) {
 119             masm->pextrq(rax, xmm1, 1);
 120           } else {
 121             assert(false, "");
 122           }
 123         } else if (width == 4) {
 124           masm->xorptr(rax, rax);
 125           if (offset - _offset == 0) {
 126             masm->pextrd(rax, xmm1, 0);
 127           } else if (offset - _offset == 4) {
 128             masm->pextrd(rax, xmm1, 1);
 129           } else if (offset - _offset == 8) {
 130             masm->pextrd(rax, xmm1, 2);
 131           } else if (offset - _offset == 12) {
 132             masm->pextrd(rax, xmm1, 3);
 133           } else {
 134             assert(false, "");
 135           }
 136         } else {
 137           assert(false, "");
 138         }
 139       } else if (need_movptr()) {
 140         if (width == 8) {
 141           masm->movptr(rax, r13);
 142         } else {
 143           assert(width == 4, "");
 144           if (offset - _offset == 0) {
 145             masm->movl(rax, r13);
 146           } else if (offset - _offset == 4) {
 147             masm->movptr(rax, r13);
 148             masm->shrptr(rax, 32);
 149           } else {
 150             assert(false, "");
 151           }
 152         }
 153       } else {
 154         assert(width == 4, "");
 155         masm->xorptr(rax, rax);
 156         masm->movl(rax, r13);
 157       }
 158     }
 159 
 160     void read(MacroAssembler* masm, int offset, int width) {
 161       // offset is offset from base
 162       if (!_is_read) {
 163         _is_read = true;
 164         emit_read(masm);
 165       }
 166       emit_extract(masm, offset, width);
 167     }
 168 
 169   public:
 170     MemSlice(Register base, int offset) : _base(base), _offset(offset), _used(0), _is_read(false), _next(NULL) {}
 171 
 172     MemSlice* next() { return _next; }
 173 
 174     void read_oop(int offset) {
 175       add_read(offset, 8); // find the right constants
 176     }
 177 
 178     int offset() const {
 179       return _offset;
 180     }
 181 
 182     void set_next(MemSlice* slice) {
 183       _next = slice;
 184     }
 185 
 186     void read_narrow(MacroAssembler* masm, int offset) {
 187       read(masm, offset, 4);
 188     }
 189 
 190     void read_wide(MacroAssembler* masm, int offset) {
 191       read(masm, offset, 8);
 192     }
 193 
 194     void read_narrow(int offset) {
 195       add_read(offset, 4); // find the right constants
 196     }
 197 
 198     bool can_read(Register base, int offset, int width) {
 199       if (base != _base) {
 200         return false;
 201       }
 202 
 203       int diff = (offset + width) - _offset;
 204       if (offset - _offset < 0 || diff > max_read_offset()) {
 205         return false;
 206       }
 207       return true;
 208     }
 209   };
 210 
 211   class OMV : public ResourceObj {
 212   private:
 213     OopMapValue _omv;
 214     MemSlice* _memory;
 215 
 216     OMV* _next;
 217     OMV* _derived;
 218     OMV* _adjacent;
 219 
 220     int _id;
 221     bool _done;
 222 
 223     OMV* last_adjacent() {
 224       OMV* n = _adjacent;
 225       if (n == NULL) {
 226         return this;
 227       }
 228       while (n->_next != NULL) {
 229         n = n->_next;
 230       }
 231 
 232       return n;
 233     }
 234 
 235   public:
 236     OMV(OopMapValue omv, int id) : _omv(omv), _memory(NULL), _next(NULL), _derived(NULL), _adjacent(NULL), _id(id), _done(false) {}
 237 
 238     int id() { return _id; }
 239     OMV* next() { return _next; }
 240     OMV* adjacent() { return _adjacent; }
 241     OMV* derived() { return _derived; }
 242     OopMapValue omv() { return _omv; }
 243 
 244     bool need_derived() {
 245       OMV* d = _derived;
 246       while (d != NULL) {
 247         if (!d->is_done()) {
 248           return true;
 249         }
 250         d = d->next();
 251       }
 252       return false;
 253     }
 254 
 255     void set_memory(MemSlice* memory) {
 256       _memory = memory;
 257     }
 258 
 259     bool is_reg() const {
 260       return _omv.reg()->is_reg();
 261     }
 262 
 263     Register base() { // need something else than Register here
 264       return (is_reg() ? rdx : rdi);
 265     }
 266 
 267     int offset() {
 268       if (is_reg()) {
 269         return 0;
 270       }
 271       return _omv.reg()->reg2stack() * VMRegImpl::stack_slot_size;
 272     }
 273 
 274     MemSlice* memory() {
 275       assert(_memory != NULL, "");
 276       return _memory;
 277     }
 278 
 279     int number_adjacent() {
 280       int n = 0;
 281       OMV* a = _adjacent;
 282       while (a != NULL) {
 283         ++n;
 284         a = a->next();
 285       }
 286       return n;
 287     }
 288 
 289     void clear_done() {
 290       _done = false;
 291 
 292       if (_adjacent != NULL) {
 293         _adjacent->clear_done();
 294       }
 295 
 296       if (_derived != NULL) {
 297         _derived->clear_done();
 298       }
 299 
 300       if (_next != NULL) {
 301         _next->clear_done();
 302       }
 303     }
 304 
 305     void add_derived(OMV* o) {
 306       if (_derived == NULL) {
 307         _derived = o;
 308       } else {
 309         OMV* n = _derived;
 310         while (n->_next != NULL) {
 311           n = n->_next;
 312         }
 313         n->_next = o;
 314       }
 315     }
 316 
 317     void mark_done() {
 318       _done = true;
 319     }
 320 
 321     bool is_done() {
 322       return _done;
 323     }
 324 
 325     void add_adjacent(OMV* o) {
 326       if (_adjacent == NULL) {
 327         _adjacent = o;
 328       } else {
 329         OMV* n = _adjacent;
 330         while (n->_next != NULL) {
 331           n = n->_next;
 332         }
 333         n->_next = o;
 334       }
 335     }
 336 
 337     void set_next(OMV* o) {
 338       _next = o;
 339     }
 340 
 341     bool is_base(const OMV* o) const {
 342       return _omv.reg() == o->_omv.content_reg();
 343     }
 344 
 345     bool is_adjacent(const OMV* o) {
 346       if (_omv.type() != o->_omv.type()) {
 347         return false;
 348       }
 349 
 350       if (_omv.reg()->is_reg() || o->_omv.reg()->is_reg()) {
 351         return false;
 352       }
 353 
 354       OMV* adj = last_adjacent();
 355 
 356       int dist = (_omv.type() == OopMapValue::oop_value) ? 8 : 4;
 357       int v1_offset = adj->omv().reg()->reg2stack() * VMRegImpl::stack_slot_size;
 358       int o_offset = o->_omv.reg()->reg2stack() * VMRegImpl::stack_slot_size;
 359 
 360       if (o_offset > v1_offset) {
 361         return (o_offset - v1_offset) == dist;
 362       }
 363       return false;
 364     }
 365   };
 366 
 367   class OopWriter {
 368   public:
 369     ~OopWriter() {}
 370     virtual void write_narrow(Register reg) = 0;
 371     virtual void write(Register reg) = 0;
 372   };
 373 
 374 
 375 private:
 376   MacroAssembler* _masm;
 377   const ImmutableOopMap& _map;
 378   bool _link_offset_loaded;
 379   bool _written_rbp_index;
 380   address _freeze;
 381   address _thaw;
 382 
 383   int _num_oops;
 384   int _num_derived;
 385 
 386   OMV* _head;
 387   MemSlice* _mem_head;
 388   MemSlice* _mem_last;
 389 
 390 
 391   void append_mem_slice(MemSlice* slice) {
 392     if (_mem_head == NULL) {
 393       _mem_head = slice;
 394       _mem_last = slice;
 395     } else {
 396       _mem_last->set_next(slice);
 397       _mem_last = slice;
 398     }
 399   }
 400 
 401   MemSlice* find_slice(Register base, int offset, int width) {
 402     MemSlice* c = _mem_head;
 403     while (c != NULL) {
 404       if (c->can_read(base, offset, width)) {
 405         return c;
 406       }
 407       c = c->next();
 408     }
 409 
 410     c = new MemSlice(base, offset);
 411     append_mem_slice(c);
 412     return c;
 413   }
 414 
 415   MemSlice* read_wide(Register base, int offset) {
 416     MemSlice* slice = find_slice(base, offset, 8); // find constant
 417     assert(offset - slice->offset() >= 0, "");
 418     slice->read_oop(offset);
 419     return slice;
 420   }
 421 
 422   MemSlice* read_narrow(Register base, int offset) {
 423     MemSlice* slice = find_slice(base, offset, 4); // find constant
 424     assert(offset - slice->offset() >= 0, "");
 425     slice->read_narrow(offset);
 426     return slice;
 427   }
 428 
 429 public:
 430   OptOopMapStubGenerator(CodeBuffer* code, const ImmutableOopMap& map) : StubCodeGenerator(code), _map(map) {
 431     _masm = new MacroAssembler(code);
 432     _link_offset_loaded = false;
 433     _written_rbp_index = false;
 434     _freeze = NULL;
 435     _thaw = NULL;
 436     _head = NULL;
 437     _mem_head = NULL;
 438     _mem_last = NULL;
 439 
 440     _num_oops = 0;
 441     _num_derived = 0;
 442 
 443     OMV* last = NULL;
 444     OMV* last_insert = NULL;
 445     int count = 0;
 446 
 447     for (OopMapStream oms(&_map); !oms.is_done(); oms.next()) {
 448       OopMapValue omv = oms.current();
 449       OMV *o = new OMV(omv, count);
 450 
 451       if (omv.is_oop_or_narrow()) {
 452 
 453         if (omv.is_oop()) {
 454           o->set_memory(read_wide(o->base(), o->offset()));
 455         } else {
 456           // narrow
 457           o->set_memory(read_narrow(o->base(), o->offset()));
 458         }
 459 
 460         ++_num_oops;
 461         if (_head == NULL) {
 462           _head = o;
 463           last = o;
 464           last_insert = o;
 465         } else {
 466           if (last->is_adjacent(o)) {
 467             last->add_adjacent(o);
 468           } else {
 469             last->set_next(o);
 470             last = o;
 471           }
 472           last_insert = o;
 473         }
 474         ++count;
 475       } else if (omv.is_derived_oop()) {
 476         ++_num_derived;
 477         o->set_memory(read_wide(o->base(), o->offset()));
 478         assert(_head != NULL, "");
 479         assert(last != NULL, "");
 480         assert(last_insert != NULL, "");
 481         if (!last_insert->is_base(o)) {
 482           what(o);
 483         }
 484         assert(last_insert->is_base(o), "");
 485         last_insert->add_derived(o);
 486       }
 487       ++count;
 488     }
 489   }
 490 
 491   void what(OMV* last) {
 492     tty->print_cr("!omv %p", last);
 493   }
 494 
 495   address thaw_stub() {
 496     return _thaw;
 497   }
 498 
 499   bool has_rbp_index() {
 500     return _written_rbp_index;
 501   }
 502 
 503   void load_link_offset() {
 504     if (!_link_offset_loaded) {
 505       _link_offset_loaded = true;
 506       // _masm->movptr(frame_ptr, Address(frame_ptr, RegisterMap::link_offset()));
 507     }
 508   }
 509 
 510   void store_rbp_oop(Register idx) {
 511     assert(_written_rbp_index == false, "");
 512     _masm->movl(Address(fp_oop_info, 0), 1); // offset to bool has_fp_index
 513     _masm->movl(Address(fp_oop_info, 4), idx); // offset to int fp_index
 514     _written_rbp_index = true;
 515   }
 516 
 517   void thaw_single_oop(OMV* o, int& pos) {
 518     bool has_derived = o->derived() != NULL;
 519     OopMapValue omv = o->omv();
 520     VMReg reg = omv.reg();
 521     // read value from array
 522     if (UseCompressedOops) {
 523 
 524       _masm->movl(rax, Address(ref_stack, pos));
 525       if (omv.type() == OopMapValue::oop_value) {
 526         _masm->decode_heap_oop(rax);
 527         if (has_derived) {
 528           _masm->movptr(c_rarg3, rax);
 529         }
 530       } else if (omv.type() == OopMapValue::narrowoop_value && has_derived) {
 531         _masm->movptr(c_rarg3, rax);
 532         _masm->decode_heap_oop(c_rarg3);
 533       }
 534 
 535     } else {
 536       _masm->movptr(rax, Address(ref_stack, pos));
 537       if (has_derived) {
 538         _masm->movptr(c_rarg3, rax);
 539       }
 540     }
 541 
 542 
 543     if (reg->is_reg()) {
 544       assert(reg == rbp->as_VMReg(), "must");
 545       //load_link_offset();
 546       if (omv.type() == OopMapValue::oop_value) {
 547         _masm->movptr(Address(frame_info, 0), rax);
 548       } else {
 549         assert(UseCompressedOops, "");
 550         // narrow oop
 551         _masm->movl(Address(frame_info, 0), rax);
 552       }
 553 
 554       /* test
 555          store_rbp_oop(r8);
 556          _masm->subl(r8, 1);
 557          */
 558 
 559     } else {
 560       int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 561 
 562       if (omv.type() == OopMapValue::oop_value) {
 563         _masm->movptr(Address(vsp, sp_offset_in_bytes), rax);
 564       } else {
 565         // narrow oop
 566         assert(UseCompressedOops, "");
 567         _masm->movl(Address(vsp, sp_offset_in_bytes), rax);
 568       }
 569 
 570       // index is refStack_length - index
 571       /* test
 572          _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
 573          _masm->subl(r8, 1);
 574          */
 575     }
 576 
 577     if (UseCompressedOops) {
 578       pos += 4;
 579     } else {
 580       pos += 8;
 581     }
 582   }
 583 
 584   void thaw_single_derived(OopMapValue omv, Register base) {
 585     VMReg reg = omv.reg();
 586     bool derived_is_reg = false;
 587     // read the derived value into rax
 588     int derived_sp_offset_in_bytes = -1;
 589     if (reg->is_reg()) {
 590       _masm->movptr(rax, Address(frame_info, 0)); // offset -> rax
 591       derived_is_reg = true;
 592     } else {
 593       derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 594       _masm->movptr(rax, Address(vsp, derived_sp_offset_in_bytes)); // offset -> rax
 595     }
 596 
 597     address narrow_oop_base = CompressedOops::base();
 598     assert(narrow_oop_base == (address) NULL, "");
 599     // maybe we need to test for is_narrow_oop_base too...
 600 
 601     _masm->addptr(rax, base); // offset in (base + offset)
 602 
 603     if (reg->is_reg()) {
 604       _masm->movptr(Address(frame_info, 0), rax);
 605     } else {
 606       _masm->movptr(Address(vsp, derived_sp_offset_in_bytes), rax);
 607     }
 608   }
 609 
 610   void thaw_derived(OMV* o) {
 611     OMV* d = o->derived();
 612     if (d != NULL) {
 613       Label L_next;
 614       Register base = c_rarg3;
 615 
 616       _masm->testptr(base, base);
 617       _masm->jcc(Assembler::zero, L_next);
 618 
 619       while (d) {
 620         thaw_single_derived(d->omv(), base);
 621         d = d->next();
 622       }
 623 
 624       _masm->bind(L_next);
 625     }
 626   }
 627 
 628   bool need_heapbase() {
 629     return (UseCompressedOops && CompressedOops::base() != NULL) || CheckCompressedOops;
 630   }
 631 
 632   void generate_thaw(const ImmutableOopMap& map) {
 633     // need to reset these first
 634     _link_offset_loaded = false;
 635     _written_rbp_index = false;
 636     _head->clear_done();
 637 
 638     _masm->align(8);
 639     _thaw = _masm->pc();
 640 
 641     if (need_heapbase()) {
 642       _masm->push(r12);
 643       _masm->reinit_heapbase();
 644     }
 645 
 646     int pos = 0;
 647     {
 648 
 649       OMV* o = _head;
 650       while (o) {
 651         thaw_single_oop(o, pos);
 652         thaw_derived(o);
 653         OMV* a = o->adjacent();
 654         while (a) {
 655           thaw_single_oop(a, pos);
 656           thaw_derived(a);
 657           a = a->next();
 658         }
 659         o = o->next();
 660       }
 661     }
 662 
 663     if (need_heapbase()) {
 664       _masm->pop(r12);
 665     }
 666     _masm->movl(rax, map.num_oops());
 667     _masm->ret(0);
 668   }
 669 
 670   void freeze_single_oop(OMV* o, int& pos, OopWriter& writer) {
 671     if (o->is_done()) {
 672       return;
 673     }
 674     o->mark_done();
 675 
 676     OopMapValue omv = o->omv();
 677     VMReg reg = omv.reg();
 678 
 679     OMV* d = o->derived();
 680     if (reg->is_reg()) {
 681       assert(reg == rbp->as_VMReg(), "must");
 682       load_link_offset();
 683       if (omv.type() == OopMapValue::oop_value) {
 684         o->memory()->read_wide(_masm, 0);
 685 
 686         //_masm->movptr(rax, Address(rdx, 0));
 687 
 688         if (d != NULL) {
 689           assert(_map.has_any(OopMapValue::derived_oop_value), "");
 690           _masm->movptr(r14, rax);
 691         }
 692 
 693         if (UseCompressedOops) {
 694           // compress
 695           _masm->encode_heap_oop(rax);
 696           writer.write_narrow(rax);
 697           //_masm->movl(Address(rsi, pos), rax);
 698         } else {
 699           writer.write(rax);
 700           //_masm->movptr(Address(rsi, pos), rax);
 701         }
 702       } else {
 703         assert(UseCompressedOops, "");
 704         // narrow oop
 705         o->memory()->read_narrow(_masm, 0);
 706         //_masm->movl(rax, Address(rdx, 0));
 707         if (d != NULL) {
 708           guarantee(false, "should not have narrow as base");
 709           assert(_map.has_any(OopMapValue::derived_oop_value), "");
 710           _masm->movptr(r14, rax);
 711           _masm->decode_heap_oop(r14);
 712         }
 713         writer.write_narrow(rax);
 714         //_masm->movl(Address(rsi, pos), rax);
 715       }
 716 
 717       /* test
 718          store_rbp_oop(r8);
 719          _masm->subl(r8, 1);
 720          */
 721 
 722     } else {
 723       int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 724 
 725       if (omv.type() == OopMapValue::oop_value) {
 726         //_masm->movptr(rax, Address(rdi, sp_offset_in_bytes));
 727         o->memory()->read_wide(_masm, sp_offset_in_bytes);
 728 
 729         if (d != NULL) {
 730           assert(_map.has_any(OopMapValue::derived_oop_value), "");
 731           _masm->movptr(r14, rax);
 732         }
 733 
 734         if (UseCompressedOops) {
 735           // compress
 736           _masm->encode_heap_oop(rax);
 737           writer.write_narrow(rax);
 738           //_masm->movl(Address(rsi, pos), rax);
 739         } else {
 740           writer.write(rax);
 741           //_masm->movptr(Address(rsi, pos), rax);
 742         }
 743       } else {
 744         // narrow oop
 745         assert(UseCompressedOops, "");
 746         o->memory()->read_narrow(_masm, sp_offset_in_bytes);
 747         //_masm->movl(rax, Address(rdi, sp_offset_in_bytes));
 748         writer.write_narrow(rax);
 749         //_masm->movl(Address(rsi, pos), rax);
 750 
 751         if (d != NULL) {
 752           guarantee(false, "should not have narrow as base");
 753           assert(_map.has_any(OopMapValue::derived_oop_value), "");
 754           _masm->movptr(r14, rax);
 755           _masm->decode_heap_oop(r14);
 756         }
 757       }
 758 
 759       // index is refStack_length - index
 760       /* test
 761          _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
 762          _masm->subl(r8, 1);
 763          */
 764     }
 765     /*if (UseCompressedOops) {
 766       pos += 4;
 767     } else {
 768       pos += 8;
 769     }
 770     */
 771 
 772   }
 773 
 774   void freeze_single_derived(OMV* o) {
 775     if (o->is_done()) {
 776       return;
 777     }
 778     o->mark_done();
 779 
 780     OopMapValue omv = o->omv();
 781     VMReg reg = omv.reg();
 782     bool derived_is_reg = false;
 783     // read the derived value into rax
 784     int derived_sp_offset_in_bytes = -1;
 785     if (reg->is_reg()) {
 786       load_link_offset();
 787       o->memory()->read_wide(_masm, 0);
 788       //_masm->movptr(rax, Address(rdx, 0));
 789       derived_is_reg = true;
 790     } else {
 791       derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
 792       o->memory()->read_wide(_masm, derived_sp_offset_in_bytes);
 793       //_masm->movptr(rax, Address(rdi, derived_sp_offset_in_bytes));
 794     }
 795 
 796     // read the base value into rax
 797     address narrow_oop_base = CompressedOops::base();
 798     assert(narrow_oop_base == (address) NULL, "");
 799     // maybe we need to test for is_narrow_oop_base too...
 800 
 801     _masm->subptr(rax, r14); // derived to rax (derived - base)
 802 
 803     if (reg->is_reg()) {
 804       store_rbp_oop(rax);
 805     } else {
 806       _masm->movptr(Address(rcx, derived_sp_offset_in_bytes), rax);
 807     }
 808   }
 809 
 810   void freeze(OMV* o, int& pos, OopWriter& writer) {
 811     freeze_single_oop(o, pos, writer);
 812     freeze_derived(o);
 813 
 814     OMV* a = o->adjacent();
 815     while (a != NULL) {
 816       freeze_single_oop(a, pos, writer);
 817       freeze_derived(a);
 818       a = a->next();
 819     }
 820   }
 821 
 822   void freeze_all_derived(OMV* d) {
 823     while (d != NULL) {
 824       freeze_single_derived(d);
 825       d = d->next();
 826     }
 827   }
 828 
 829   void freeze_derived(OMV* o) {
 830     if (o->need_derived()) {
 831       OMV* d = o->derived();
 832       Label L_next;
 833 
 834       _masm->testptr(r14, r14);
 835       _masm->jcc(Assembler::zero, L_next);
 836 
 837       freeze_all_derived(d);
 838 
 839       _masm->bind(L_next);
 840     }
 841   }
 842 
 843   class BatchWriter : public OopWriter {
 844   private:
 845     MacroAssembler* _masm;
 846     int _num_oops;
 847     int _pos;
 848     int _written;
 849 
 850     int _size;
 851     int _left;
 852     int _slot;
 853 
 854     int next_size() const {
 855       int bytes_remaining = 0;
 856       int left = _num_oops - _written;
 857       if (UseCompressedOops) {
 858         bytes_remaining = left * 4;
 859       } else {
 860         bytes_remaining = left * 8;
 861       }
 862 
 863       if (bytes_remaining >= 16) {
 864         return 16;
 865       } else if (bytes_remaining >= 8) {
 866         return 8;
 867       } else if (bytes_remaining >= 4) {
 868         return 4;
 869       } else {
 870         return 0;
 871       }
 872     }
 873 
 874     bool is_xmm() {
 875       return _size > 8;
 876     }
 877 
 878     bool is_quad() {
 879       return _size == 8;
 880     }
 881 
 882     bool is_word() {
 883       return _size == 4;
 884     }
 885 
 886     void write_narrow_xmm(Register reg) {
 887       _masm->pinsrd(xmm0, reg, _slot);
 888     }
 889 
 890     void write_narrow_quad(Register reg) {
 891       if (_slot == 0) {
 892         _masm->movl(rbx, reg);
 893       } else if (_slot == 1) {
 894         _masm->shlptr(reg, 32);
 895         _masm->orptr(reg, rbx);
 896       } else {
 897         assert(false, "");
 898       }
 899     }
 900 
 901     void write_narrow_word(Register reg) {
 902       // nop
 903     }
 904 
 905   public:
 906     BatchWriter(MacroAssembler* masm, int num_oops) : _masm(masm), _num_oops(num_oops), _pos(0), _written(0), _slot(0) {
 907       _left = _size = next_size();
 908     }
 909 
 910     void finish() {
 911       if (is_xmm()) {
 912         _masm->movdqu(Address(ref_stack, _pos), xmm0);
 913       } else if (is_quad()) {
 914         _masm->movptr(Address(ref_stack, _pos), rax);
 915       } else if (is_word()) {
 916         _masm->movl(Address(ref_stack, _pos), rax);
 917       }
 918       _pos += _size;
 919       _slot = 0;
 920       _left = _size = next_size();
 921     }
 922 
 923     void written(int bytes) {
 924       _written += 1;
 925       _left -= bytes;
 926       _slot += 1;
 927 
 928       if (_left == 0) {
 929         finish();
 930       }
 931     }
 932 
 933     virtual void write_narrow(Register reg) {
 934       if (is_xmm()) {
 935         write_narrow_xmm(reg);
 936       } else if (is_quad()) {
 937         write_narrow_quad(reg);
 938       } else if (is_word()) {
 939         write_narrow_word(reg);
 940       } else {
 941         assert(false, "");
 942       }
 943       written(4);
 944     }
 945 
 946     virtual void write(Register reg) {
 947       assert(false, "unimplemented");
 948       written(8);
 949     }
 950   };
 951 
 952   class SingleWriter : public OopWriter {
 953   private:
 954     MacroAssembler* _masm;
 955     int _pos;
 956 
 957   public:
 958     SingleWriter(MacroAssembler* masm) : _masm(masm), _pos(0) {}
 959     virtual void write_narrow(Register reg) {
 960       _masm->movl(Address(ref_stack, _pos), reg);
 961       _pos += 4;
 962     }
 963     virtual void write(Register reg) {
 964       _masm->movptr(Address(ref_stack, _pos), reg);
 965       _pos += 8;
 966     }
 967   };
 968 
 969   void generate_freeze(const ImmutableOopMap& map) {
 970     _masm->align(8);
 971     _freeze = _masm->pc();
 972 
 973     _masm->push(rbx);
 974 
 975     /* rdi is source (rsp), rsi is destination (first address), rdx (rbp address), rcx (hstack), r8 (initial index (refStack_length - index) ), r9 (fp_oop_info) */
 976     if (need_heapbase()) {
 977       _masm->push(r12);
 978       _masm->reinit_heapbase();
 979     }
 980 
 981     _masm->push(r13);
 982 
 983     const bool has_derived = map.has_any(OopMapValue::derived_oop_value);
 984     if (has_derived) {
 985       _masm->push(r14);
 986     }
 987 
 988     int pos = 0;
 989     {
 990       BatchWriter writer(_masm, _num_oops);
 991       OMV* o = _head;
 992       while (o != NULL) {
 993         freeze(o, pos, writer);
 994         o = o->next();
 995       }
 996     }
 997 
 998     if (has_derived) {
 999       _masm->pop(r14);
1000     }
1001     _masm->pop(r13);
1002     if (need_heapbase()) {
1003       _masm->pop(r12);
1004     }
1005 
1006     _masm->pop(rbx);
1007 
1008     _masm->movl(rax, map.num_oops());
1009     _masm->ret(0);
1010   }
1011 };
1012 
1013 
1014 class OopMapStubGeneratorX86 : public StubCodeGenerator {
1015 private:
1016   MacroAssembler* _masm;
1017   bool _link_offset_loaded;
1018   bool _written_rbp_index;
1019   address _freeze;
1020   address _thaw;
1021   intptr_t _freeze_len, _thaw_len;
1022 
1023 public:
1024   OopMapStubGeneratorX86(CodeBuffer* code) : StubCodeGenerator(code) {
1025     _masm = new MacroAssembler(code);
1026     _link_offset_loaded = false;
1027     _written_rbp_index = false;
1028     _freeze = NULL;
1029     _thaw = NULL;
1030   }
1031 
1032   intptr_t thaw_length() {
1033     return _thaw_len;
1034   }
1035 
1036   address thaw_stub() {
1037     return _thaw;
1038   }
1039 
1040   address freeze_stub() {
1041     return _freeze;
1042   }
1043 
1044   bool has_rbp_index() {
1045     return _written_rbp_index;
1046   }
1047 
1048   void load_link_offset() {
1049     if (!_link_offset_loaded) {
1050       _link_offset_loaded = true;
1051       // _masm->movptr(c_rarg2, Address(frame_info, RegisterMap::link_offset()));
1052     }
1053   }
1054 
1055   void store_rbp_oop(Register idx) {
1056     assert(_written_rbp_index == false, "");
1057     _masm->movl(Address(fp_oop_info, 0), 1); // offset to bool has_fp_index
1058     _masm->movl(Address(fp_oop_info, 4), idx); // offset to int fp_index
1059     _written_rbp_index = true;
1060   }
1061 
1062   void generate_thaw(const ImmutableOopMap& map) {
1063     // need to reset these first
1064     _link_offset_loaded = false;
1065     _written_rbp_index = false;
1066 
1067     _masm->align(8);
1068     _masm->emit_int64(0); // make room for CodeBlob pointer
1069     _thaw = _masm->pc();
1070 
1071     if (UseCompressedOops) {
1072       _masm->push(r12);
1073       _masm->reinit_heapbase();
1074     }
1075 
1076     int pos = 0;
1077     {
1078       for (OopMapStream oms(&map); !oms.is_done(); oms.next()) {
1079         OopMapValue omv = oms.current();
1080         if (omv.type() != OopMapValue::oop_value && omv.type() != OopMapValue::narrowoop_value)
1081           continue;
1082         
1083         VMReg reg = omv.reg();
1084 
1085         // read value from array
1086         if (UseCompressedOops) {
1087           _masm->movl(rax, Address(ref_stack, pos));
1088 
1089           if (omv.type() == OopMapValue::oop_value) {
1090             _masm->decode_heap_oop(rax);
1091           }
1092         } else {
1093           _masm->movptr(rax, Address(ref_stack, pos));
1094         }
1095 
1096         if (reg->is_reg()) {
1097           assert(reg == rbp->as_VMReg(), "must"); // not true for safepoint stub, but we don't use oopmap stubs when preempting
1098           //load_link_offset();
1099           if (omv.type() == OopMapValue::oop_value) {
1100             _masm->movptr(Address(frame_info, 0), rax);
1101           } else {
1102             assert(UseCompressedOops, "");
1103             // narrow oop
1104             _masm->movl(Address(frame_info, 0), rax);
1105           }
1106 
1107           /* test
1108           store_rbp_oop(r8);
1109           _masm->subl(r8, 1);
1110           */
1111 
1112         } else {
1113           int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1114 
1115           if (omv.type() == OopMapValue::oop_value) {
1116             _masm->movptr(Address(vsp, sp_offset_in_bytes), rax);
1117           } else {
1118             // narrow oop
1119             assert(UseCompressedOops, "");
1120             _masm->movl(Address(vsp, sp_offset_in_bytes), rax);
1121           }
1122 
1123           // index is refStack_length - index
1124           /* test
1125           _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
1126           _masm->subl(r8, 1);
1127           */
1128         }
1129         if (UseCompressedOops) {
1130           pos += 4;
1131         } else {
1132           pos += 8;
1133         }
1134       }
1135     }
1136 
1137     {
1138       for (OopMapStream oms(&map); !oms.is_done(); oms.next()) {
1139         OopMapValue omv = oms.current();
1140         if (omv.type() != OopMapValue::derived_oop_value)
1141           continue;
1142         VMReg reg = omv.reg();
1143         bool derived_is_reg = false;
1144         // read the derived value into rax
1145         int derived_sp_offset_in_bytes = -1;
1146         if (reg->is_reg()) {
1147           _masm->movptr(rax, Address(frame_info, 0)); // offset -> rax
1148           derived_is_reg = true;
1149         } else {
1150           derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1151           _masm->movptr(rax, Address(vsp, derived_sp_offset_in_bytes)); // offset -> rax
1152         }
1153 
1154         // read the base value into rcx
1155         VMReg content_reg = omv.content_reg();
1156         if (content_reg->is_reg()) {
1157           guarantee(content_reg == rbp->as_VMReg(), "must");
1158           guarantee(derived_is_reg == false, "must");
1159           _masm->movptr(c_rarg3, Address(frame_info, 0)); // base -> rcx
1160         } else {
1161           int sp_offset_in_bytes = content_reg->reg2stack() * VMRegImpl::stack_slot_size;
1162           _masm->movptr(c_rarg3, Address(vsp, sp_offset_in_bytes)); // base -> rdx
1163         }
1164 
1165         address narrow_oop_base = CompressedOops::base();
1166         assert(narrow_oop_base == (address) NULL, "");
1167         // maybe we need to test for is_narrow_oop_base too...
1168         Label L_next;
1169 
1170         _masm->testptr(c_rarg3, c_rarg3);
1171         _masm->jcc(Assembler::zero, L_next);
1172 
1173         _masm->addptr(c_rarg3, rax); // offset in (base + offset)
1174 
1175         if (reg->is_reg()) {
1176           _masm->movptr(Address(frame_info, 0), c_rarg3);
1177         } else {
1178           _masm->movptr(Address(vsp, derived_sp_offset_in_bytes), c_rarg3);
1179         }
1180 
1181         _masm->bind(L_next);
1182       }
1183     }
1184 
1185     if (UseCompressedOops) {
1186       _masm->pop(r12);
1187     }
1188     _masm->movl(rax, map.num_oops());
1189     _masm->ret(0);
1190 
1191     _thaw_len = (intptr_t) _masm->pc() - (intptr_t) _thaw;
1192 
1193   // #ifdef ASSERT
1194   //   tty->print_cr(">>> thaw stub:");
1195   //   _masm->code()->decode(_thaw, _masm->pc());
1196   // #endif
1197   }
1198 
1199   static void freeze_prologue(MacroAssembler* _masm) {
1200   #ifdef _WINDOWS
1201     // Callee save
1202     _masm->movptr(Address(rsp, wordSize * 3), initial_index);
1203     _masm->movptr(Address(rsp, wordSize * 4), fp_oop_info);
1204     _masm->load_sized_value(initial_index, Address(rsp, wordSize * 5), sizeof(int), true);
1205     _masm->movptr(fp_oop_info, Address(rsp, wordSize * 6));
1206   #endif
1207   }
1208 
1209   static void freeze_epilogue(MacroAssembler* _masm) {
1210   #ifdef _WINDOWS
1211     _masm->movptr(initial_index, Address(rsp, wordSize * 3));
1212     _masm->movptr(fp_oop_info, Address(rsp, wordSize * 4));
1213   #endif
1214   }
1215 
1216   void generate_freeze(const ImmutableOopMap& map) {
1217     _masm->align(8);
1218     _masm->emit_int64(0); // make room for thaw stub pointer
1219     _freeze = _masm->pc();
1220 
1221     freeze_prologue(_masm);
1222 
1223     /* rdi is source (rsp), rsi is destination (first address), rdx (rbp address), rcx (hstack), r8 (initial index (refStack_length - index) ), r9 (fp_oop_info) */
1224     if (UseCompressedOops) {
1225       _masm->push(r12);
1226       _masm->reinit_heapbase();
1227     }
1228 
1229     int pos = 0;
1230     {
1231       for (OopMapStream oms(&map); !oms.is_done(); oms.next()) {
1232         OopMapValue omv = oms.current();
1233         if (omv.type() != OopMapValue::oop_value && omv.type() != OopMapValue::narrowoop_value)
1234           continue;
1235         VMReg reg = omv.reg();
1236 
1237         if (reg->is_reg()) {
1238           assert(reg == rbp->as_VMReg(), "must"); // not true for safepoint stub, but we don't use oopmap stubs when preempting
1239           load_link_offset();
1240           if (omv.type() == OopMapValue::oop_value) {
1241             _masm->movptr(rax, Address(frame_ptr, 0));
1242             if (UseCompressedOops) {
1243               // compress
1244               _masm->encode_heap_oop(rax);
1245               _masm->movl(Address(ref_stack, pos), rax);
1246             } else {
1247               _masm->movptr(Address(ref_stack, pos), rax);
1248             }
1249           } else {
1250             assert(UseCompressedOops, "");
1251             // narrow oop
1252             _masm->movl(rax, Address(frame_ptr, 0));
1253             _masm->movl(Address(ref_stack, pos), rax);
1254           }
1255 
1256           /* test
1257           store_rbp_oop(r8);
1258           _masm->subl(r8, 1);
1259           */
1260 
1261         } else {
1262           int sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1263 
1264           if (omv.type() == OopMapValue::oop_value) {
1265             _masm->movptr(rax, Address(vsp, sp_offset_in_bytes));
1266             if (UseCompressedOops) {
1267               // compress
1268               _masm->encode_heap_oop(rax);
1269               _masm->movl(Address(ref_stack, pos), rax);
1270             } else {
1271               _masm->movptr(Address(ref_stack, pos), rax);
1272             }
1273           } else {
1274             // narrow oop
1275             assert(UseCompressedOops, "");
1276             _masm->movl(rax, Address(vsp, sp_offset_in_bytes));
1277             _masm->movl(Address(ref_stack, pos), rax);
1278           }
1279 
1280           // index is refStack_length - index
1281           /* test
1282           _masm->movl(Address(rcx, sp_offset_in_bytes), r8);
1283           _masm->subl(r8, 1);
1284           */
1285         }
1286         if (UseCompressedOops) {
1287           pos += 4;
1288         } else {
1289           pos += 8;
1290         }
1291       }
1292     }
1293 
1294     {
1295       bool derived_init = false;
1296       for (OopMapStream oms(&map); !oms.is_done(); oms.next()) {
1297         OopMapValue omv = oms.current();
1298         if (omv.type() != OopMapValue::derived_oop_value)
1299           continue;
1300           
1301         if (!derived_init) {
1302           _masm->push(r11);
1303           derived_init = true;
1304         }
1305 
1306         VMReg reg = omv.reg();
1307         bool derived_is_reg = false;
1308         // read the derived value into r11
1309         int derived_sp_offset_in_bytes = -1;
1310         if (reg->is_reg()) {
1311           load_link_offset();
1312           _masm->movptr(r11, Address(frame_ptr, 0));
1313           derived_is_reg = true;
1314         } else {
1315           derived_sp_offset_in_bytes = reg->reg2stack() * VMRegImpl::stack_slot_size;
1316           _masm->movptr(r11, Address(vsp, derived_sp_offset_in_bytes));
1317         }
1318 
1319         // read the base value into rax
1320         VMReg content_reg = omv.content_reg();
1321         if (content_reg->is_reg()) {
1322           load_link_offset();
1323           guarantee(content_reg == rbp->as_VMReg(), "must");
1324           guarantee(derived_is_reg == false, "must");
1325           _masm->movptr(rax, Address(frame_ptr, 0));
1326         } else {
1327           int sp_offset_in_bytes = content_reg->reg2stack() * VMRegImpl::stack_slot_size;
1328           _masm->movptr(rax, Address(vsp, sp_offset_in_bytes));
1329         }
1330 
1331         address narrow_oop_base = CompressedOops::base();
1332         assert(narrow_oop_base == (address) NULL, "");
1333         // maybe we need to test for is_narrow_oop_base too...
1334         Label L_next;
1335 
1336         _masm->testptr(rax, rax);
1337         _masm->jcc(Assembler::zero, L_next);
1338 
1339         _masm->subptr(r11, rax); // derived to r11 (derived - base)
1340 
1341         if (reg->is_reg()) {
1342           store_rbp_oop(r11);
1343         } else {
1344           _masm->movptr(Address(hsp, derived_sp_offset_in_bytes), r11);
1345         }
1346 
1347         _masm->bind(L_next);
1348       }
1349       if (derived_init) {
1350         _masm->pop(r11);
1351       }
1352     }
1353 
1354     if (UseCompressedOops) {
1355       _masm->pop(r12);
1356     }
1357     _masm->movl(rax, map.num_oops());
1358 
1359     freeze_epilogue(_masm);
1360 
1361     _masm->ret(0);
1362     _freeze_len = (intptr_t) _masm->pc() - (intptr_t) _freeze;
1363 
1364   // #ifdef ASSERT
1365   //   tty->print_cr(">>> freeze stub:");
1366   //   _masm->code()->decode(_freeze, _masm->pc());
1367   // #endif
1368   }
1369 };
1370 
1371 bool OopMapStubGenerator::generate() {
1372   ResourceMark rm;
1373 
1374   int size = 64 + (_oopmap.count() * 6 * 15) + (CheckCompressedOops ? (256 * _oopmap.count()) : 0); // worst case, 6 instructions per oop, 15 bytes per instruction;
1375 
1376 #ifdef _WINDOWS
1377   // spill instructions and alignment
1378   size += 16;
1379 #endif
1380 
1381   _blob = BufferBlob::create("oopmap stub", size);
1382   if (_blob == NULL) {
1383     return false;
1384   }
1385 
1386   CodeBuffer buf(_blob);
1387   //OptOopMapStubGenerator cgen(&buf, this);
1388   OopMapStubGeneratorX86 cgen(&buf);
1389   cgen.generate_freeze(_oopmap);
1390   cgen.generate_thaw(_oopmap);
1391 
1392   _freeze_stub = cgen.freeze_stub(); // _blob->code_begin();
1393   _thaw_stub   = cgen.thaw_stub();
1394 
1395   *((address*)_freeze_stub - 1) = _thaw_stub;
1396   *((const CodeBlob**)_thaw_stub - 1) = _cb;
1397 
1398   assert (_cb != NULL, "");
1399 
1400   assert (thaw_stub(_freeze_stub) == _thaw_stub, "");
1401   assert ((intptr_t)_freeze_stub % 8 == 0, "");
1402   assert ((intptr_t)_thaw_stub   % 8 == 0, "");
1403 
1404   return true;
1405 }
1406 
1407 void OopMapStubGenerator::free() {
1408   if (_blob != NULL) {
1409     BufferBlob::free(_blob);
1410     _blob = NULL;
1411   }
1412   _freeze_stub = NULL;
1413   _thaw_stub = NULL;
1414 }
1415 
1416 address OopMapStubGenerator::thaw_stub(address freeze_stub_address) {
1417   return *((address*)freeze_stub_address - 1);
1418 }
1419 
1420 #endif