1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/symbolTable.hpp"
  27 #include "classfile/systemDictionary.hpp"
  28 #include "classfile/vmSymbols.hpp"
  29 #include "jvmtifiles/jvmtiEnv.hpp"
  30 #include "oops/instanceMirrorKlass.hpp"
  31 #include "oops/objArrayKlass.hpp"
  32 #include "oops/oop.inline2.hpp"
  33 #include "prims/jvmtiEventController.hpp"
  34 #include "prims/jvmtiEventController.inline.hpp"
  35 #include "prims/jvmtiExport.hpp"
  36 #include "prims/jvmtiImpl.hpp"
  37 #include "prims/jvmtiTagMap.hpp"
  38 #include "runtime/biasedLocking.hpp"
  39 #include "runtime/javaCalls.hpp"
  40 #include "runtime/jniHandles.hpp"
  41 #include "runtime/mutex.hpp"
  42 #include "runtime/mutexLocker.hpp"
  43 #include "runtime/reflectionUtils.hpp"
  44 #include "runtime/vframe.hpp"
  45 #include "runtime/vmThread.hpp"
  46 #include "runtime/vm_operations.hpp"
  47 #include "services/serviceUtil.hpp"
  48 #include "utilities/macros.hpp"
  49 #if INCLUDE_ALL_GCS
  50 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  51 #include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp"
  52 #endif // INCLUDE_ALL_GCS
  53 
  54 // JvmtiTagHashmapEntry
  55 //
  56 // Each entry encapsulates a reference to the tagged object
  57 // and the tag value. In addition an entry includes a next pointer which
  58 // is used to chain entries together.
  59 
  60 class JvmtiTagHashmapEntry : public CHeapObj<mtInternal> {
  61  private:
  62   friend class JvmtiTagMap;
  63 
  64   oop _object;                          // tagged object
  65   jlong _tag;                           // the tag
  66   JvmtiTagHashmapEntry* _next;          // next on the list
  67 
  68   inline void init(oop object, jlong tag) {
  69     _object = object;
  70     _tag = tag;
  71     _next = NULL;
  72   }
  73 
  74   // constructor
  75   JvmtiTagHashmapEntry(oop object, jlong tag)         { init(object, tag); }
  76 
  77  public:
  78 
  79   // accessor methods
  80   inline oop object() const                           { return _object; }
  81   inline oop* object_addr()                           { return &_object; }
  82   inline jlong tag() const                            { return _tag; }
  83 
  84   inline void set_tag(jlong tag) {
  85     assert(tag != 0, "can't be zero");
  86     _tag = tag;
  87   }
  88 
  89   inline JvmtiTagHashmapEntry* next() const             { return _next; }
  90   inline void set_next(JvmtiTagHashmapEntry* next)      { _next = next; }
  91 };
  92 
  93 
  94 // JvmtiTagHashmap
  95 //
  96 // A hashmap is essentially a table of pointers to entries. Entries
  97 // are hashed to a location, or position in the table, and then
  98 // chained from that location. The "key" for hashing is address of
  99 // the object, or oop. The "value" is the tag value.
 100 //
 101 // A hashmap maintains a count of the number entries in the hashmap
 102 // and resizes if the number of entries exceeds a given threshold.
 103 // The threshold is specified as a percentage of the size - for
 104 // example a threshold of 0.75 will trigger the hashmap to resize
 105 // if the number of entries is >75% of table size.
 106 //
 107 // A hashmap provides functions for adding, removing, and finding
 108 // entries. It also provides a function to iterate over all entries
 109 // in the hashmap.
 110 
 111 class JvmtiTagHashmap : public CHeapObj<mtInternal> {
 112  private:
 113   friend class JvmtiTagMap;
 114 
 115   enum {
 116     small_trace_threshold  = 10000,                  // threshold for tracing
 117     medium_trace_threshold = 100000,
 118     large_trace_threshold  = 1000000,
 119     initial_trace_threshold = small_trace_threshold
 120   };
 121 
 122   static int _sizes[];                  // array of possible hashmap sizes
 123   int _size;                            // actual size of the table
 124   int _size_index;                      // index into size table
 125 
 126   int _entry_count;                     // number of entries in the hashmap
 127 
 128   float _load_factor;                   // load factor as a % of the size
 129   int _resize_threshold;                // computed threshold to trigger resizing.
 130   bool _resizing_enabled;               // indicates if hashmap can resize
 131 
 132   int _trace_threshold;                 // threshold for trace messages
 133 
 134   JvmtiTagHashmapEntry** _table;        // the table of entries.
 135 
 136   // private accessors
 137   int resize_threshold() const                  { return _resize_threshold; }
 138   int trace_threshold() const                   { return _trace_threshold; }
 139 
 140   // initialize the hashmap
 141   void init(int size_index=0, float load_factor=4.0f) {
 142     int initial_size =  _sizes[size_index];
 143     _size_index = size_index;
 144     _size = initial_size;
 145     _entry_count = 0;
 146     if (TraceJVMTIObjectTagging) {
 147       _trace_threshold = initial_trace_threshold;
 148     } else {
 149       _trace_threshold = -1;
 150     }
 151     _load_factor = load_factor;
 152     _resize_threshold = (int)(_load_factor * _size);
 153     _resizing_enabled = true;
 154     size_t s = initial_size * sizeof(JvmtiTagHashmapEntry*);
 155     _table = (JvmtiTagHashmapEntry**)os::malloc(s, mtInternal);
 156     if (_table == NULL) {
 157       vm_exit_out_of_memory(s, OOM_MALLOC_ERROR,
 158         "unable to allocate initial hashtable for jvmti object tags");
 159     }
 160     for (int i=0; i<initial_size; i++) {
 161       _table[i] = NULL;
 162     }
 163   }
 164 
 165   // hash a given key (oop) with the specified size
 166   static unsigned int hash(oop key, int size) {
 167     // shift right to get better distribution (as these bits will be zero
 168     // with aligned addresses)
 169     unsigned int addr = (unsigned int)(cast_from_oop<intptr_t>(key));
 170 #ifdef _LP64
 171     return (addr >> 3) % size;
 172 #else
 173     return (addr >> 2) % size;
 174 #endif
 175   }
 176 
 177   // hash a given key (oop)
 178   unsigned int hash(oop key) {
 179     return hash(key, _size);
 180   }
 181 
 182   // resize the hashmap - allocates a large table and re-hashes
 183   // all entries into the new table.
 184   void resize() {
 185     int new_size_index = _size_index+1;
 186     int new_size = _sizes[new_size_index];
 187     if (new_size < 0) {
 188       // hashmap already at maximum capacity
 189       return;
 190     }
 191 
 192     // allocate new table
 193     size_t s = new_size * sizeof(JvmtiTagHashmapEntry*);
 194     JvmtiTagHashmapEntry** new_table = (JvmtiTagHashmapEntry**)os::malloc(s, mtInternal);
 195     if (new_table == NULL) {
 196       warning("unable to allocate larger hashtable for jvmti object tags");
 197       set_resizing_enabled(false);
 198       return;
 199     }
 200 
 201     // initialize new table
 202     int i;
 203     for (i=0; i<new_size; i++) {
 204       new_table[i] = NULL;
 205     }
 206 
 207     // rehash all entries into the new table
 208     for (i=0; i<_size; i++) {
 209       JvmtiTagHashmapEntry* entry = _table[i];
 210       while (entry != NULL) {
 211         JvmtiTagHashmapEntry* next = entry->next();
 212         oop key = entry->object();
 213         assert(key != NULL, "jni weak reference cleared!!");
 214         unsigned int h = hash(key, new_size);
 215         JvmtiTagHashmapEntry* anchor = new_table[h];
 216         if (anchor == NULL) {
 217           new_table[h] = entry;
 218           entry->set_next(NULL);
 219         } else {
 220           entry->set_next(anchor);
 221           new_table[h] = entry;
 222         }
 223         entry = next;
 224       }
 225     }
 226 
 227     // free old table and update settings.
 228     os::free((void*)_table);
 229     _table = new_table;
 230     _size_index = new_size_index;
 231     _size = new_size;
 232 
 233     // compute new resize threshold
 234     _resize_threshold = (int)(_load_factor * _size);
 235   }
 236 
 237 
 238   // internal remove function - remove an entry at a given position in the
 239   // table.
 240   inline void remove(JvmtiTagHashmapEntry* prev, int pos, JvmtiTagHashmapEntry* entry) {
 241     assert(pos >= 0 && pos < _size, "out of range");
 242     if (prev == NULL) {
 243       _table[pos] = entry->next();
 244     } else {
 245       prev->set_next(entry->next());
 246     }
 247     assert(_entry_count > 0, "checking");
 248     _entry_count--;
 249   }
 250 
 251   // resizing switch
 252   bool is_resizing_enabled() const          { return _resizing_enabled; }
 253   void set_resizing_enabled(bool enable)    { _resizing_enabled = enable; }
 254 
 255   // debugging
 256   void print_memory_usage();
 257   void compute_next_trace_threshold();
 258 
 259  public:
 260 
 261   // create a JvmtiTagHashmap of a preferred size and optionally a load factor.
 262   // The preferred size is rounded down to an actual size.
 263   JvmtiTagHashmap(int size, float load_factor=0.0f) {
 264     int i=0;
 265     while (_sizes[i] < size) {
 266       if (_sizes[i] < 0) {
 267         assert(i > 0, "sanity check");
 268         i--;
 269         break;
 270       }
 271       i++;
 272     }
 273 
 274     // if a load factor is specified then use it, otherwise use default
 275     if (load_factor > 0.01f) {
 276       init(i, load_factor);
 277     } else {
 278       init(i);
 279     }
 280   }
 281 
 282   // create a JvmtiTagHashmap with default settings
 283   JvmtiTagHashmap() {
 284     init();
 285   }
 286 
 287   // release table when JvmtiTagHashmap destroyed
 288   ~JvmtiTagHashmap() {
 289     if (_table != NULL) {
 290       os::free((void*)_table);
 291       _table = NULL;
 292     }
 293   }
 294 
 295   // accessors
 296   int size() const                              { return _size; }
 297   JvmtiTagHashmapEntry** table() const          { return _table; }
 298   int entry_count() const                       { return _entry_count; }
 299 
 300   // find an entry in the hashmap, returns NULL if not found.
 301   inline JvmtiTagHashmapEntry* find(oop key) {
 302     unsigned int h = hash(key);
 303     JvmtiTagHashmapEntry* entry = _table[h];
 304     while (entry != NULL) {
 305       if (entry->object() == key) {
 306          return entry;
 307       }
 308       entry = entry->next();
 309     }
 310     return NULL;
 311   }
 312 
 313 
 314   // add a new entry to hashmap
 315   inline void add(oop key, JvmtiTagHashmapEntry* entry) {
 316     assert(key != NULL, "checking");
 317     assert(find(key) == NULL, "duplicate detected");
 318     unsigned int h = hash(key);
 319     JvmtiTagHashmapEntry* anchor = _table[h];
 320     if (anchor == NULL) {
 321       _table[h] = entry;
 322       entry->set_next(NULL);
 323     } else {
 324       entry->set_next(anchor);
 325       _table[h] = entry;
 326     }
 327 
 328     _entry_count++;
 329     if (trace_threshold() > 0 && entry_count() >= trace_threshold()) {
 330       assert(TraceJVMTIObjectTagging, "should only get here when tracing");
 331       print_memory_usage();
 332       compute_next_trace_threshold();
 333     }
 334 
 335     // if the number of entries exceed the threshold then resize
 336     if (entry_count() > resize_threshold() && is_resizing_enabled()) {
 337       resize();
 338     }
 339   }
 340 
 341   // remove an entry with the given key.
 342   inline JvmtiTagHashmapEntry* remove(oop key) {
 343     unsigned int h = hash(key);
 344     JvmtiTagHashmapEntry* entry = _table[h];
 345     JvmtiTagHashmapEntry* prev = NULL;
 346     while (entry != NULL) {
 347       if (key == entry->object()) {
 348         break;
 349       }
 350       prev = entry;
 351       entry = entry->next();
 352     }
 353     if (entry != NULL) {
 354       remove(prev, h, entry);
 355     }
 356     return entry;
 357   }
 358 
 359   // iterate over all entries in the hashmap
 360   void entry_iterate(JvmtiTagHashmapEntryClosure* closure);
 361 };
 362 
 363 // possible hashmap sizes - odd primes that roughly double in size.
 364 // To avoid excessive resizing the odd primes from 4801-76831 and
 365 // 76831-307261 have been removed. The list must be terminated by -1.
 366 int JvmtiTagHashmap::_sizes[] =  { 4801, 76831, 307261, 614563, 1228891,
 367     2457733, 4915219, 9830479, 19660831, 39321619, 78643219, -1 };
 368 
 369 
 370 // A supporting class for iterating over all entries in Hashmap
 371 class JvmtiTagHashmapEntryClosure {
 372  public:
 373   virtual void do_entry(JvmtiTagHashmapEntry* entry) = 0;
 374 };
 375 
 376 
 377 // iterate over all entries in the hashmap
 378 void JvmtiTagHashmap::entry_iterate(JvmtiTagHashmapEntryClosure* closure) {
 379   for (int i=0; i<_size; i++) {
 380     JvmtiTagHashmapEntry* entry = _table[i];
 381     JvmtiTagHashmapEntry* prev = NULL;
 382     while (entry != NULL) {
 383       // obtain the next entry before invoking do_entry - this is
 384       // necessary because do_entry may remove the entry from the
 385       // hashmap.
 386       JvmtiTagHashmapEntry* next = entry->next();
 387       closure->do_entry(entry);
 388       entry = next;
 389      }
 390   }
 391 }
 392 
 393 // debugging
 394 void JvmtiTagHashmap::print_memory_usage() {
 395   intptr_t p = (intptr_t)this;
 396   tty->print("[JvmtiTagHashmap @ " INTPTR_FORMAT, p);
 397 
 398   // table + entries in KB
 399   int hashmap_usage = (size()*sizeof(JvmtiTagHashmapEntry*) +
 400     entry_count()*sizeof(JvmtiTagHashmapEntry))/K;
 401 
 402   int weak_globals_usage = (int)(JNIHandles::weak_global_handle_memory_usage()/K);
 403   tty->print_cr(", %d entries (%d KB) <JNI weak globals: %d KB>]",
 404     entry_count(), hashmap_usage, weak_globals_usage);
 405 }
 406 
 407 // compute threshold for the next trace message
 408 void JvmtiTagHashmap::compute_next_trace_threshold() {
 409   if (trace_threshold() < medium_trace_threshold) {
 410     _trace_threshold += small_trace_threshold;
 411   } else {
 412     if (trace_threshold() < large_trace_threshold) {
 413       _trace_threshold += medium_trace_threshold;
 414     } else {
 415       _trace_threshold += large_trace_threshold;
 416     }
 417   }
 418 }
 419 
 420 // create a JvmtiTagMap
 421 JvmtiTagMap::JvmtiTagMap(JvmtiEnv* env) :
 422   _env(env),
 423   _lock(Mutex::nonleaf+2, "JvmtiTagMap._lock", false),
 424   _free_entries(NULL),
 425   _free_entries_count(0)
 426 {
 427   assert(JvmtiThreadState_lock->is_locked(), "sanity check");
 428   assert(((JvmtiEnvBase *)env)->tag_map() == NULL, "tag map already exists for environment");
 429 
 430   _hashmap = new JvmtiTagHashmap();
 431 
 432   // finally add us to the environment
 433   ((JvmtiEnvBase *)env)->release_set_tag_map(this);
 434 }
 435 
 436 
 437 // destroy a JvmtiTagMap
 438 JvmtiTagMap::~JvmtiTagMap() {
 439 
 440   // no lock acquired as we assume the enclosing environment is
 441   // also being destroryed.
 442   ((JvmtiEnvBase *)_env)->set_tag_map(NULL);
 443 
 444   JvmtiTagHashmapEntry** table = _hashmap->table();
 445   for (int j = 0; j < _hashmap->size(); j++) {
 446     JvmtiTagHashmapEntry* entry = table[j];
 447     while (entry != NULL) {
 448       JvmtiTagHashmapEntry* next = entry->next();
 449       delete entry;
 450       entry = next;
 451     }
 452   }
 453 
 454   // finally destroy the hashmap
 455   delete _hashmap;
 456   _hashmap = NULL;
 457 
 458   // remove any entries on the free list
 459   JvmtiTagHashmapEntry* entry = _free_entries;
 460   while (entry != NULL) {
 461     JvmtiTagHashmapEntry* next = entry->next();
 462     delete entry;
 463     entry = next;
 464   }
 465   _free_entries = NULL;
 466 }
 467 
 468 // create a hashmap entry
 469 // - if there's an entry on the (per-environment) free list then this
 470 // is returned. Otherwise an new entry is allocated.
 471 JvmtiTagHashmapEntry* JvmtiTagMap::create_entry(oop ref, jlong tag) {
 472   assert(Thread::current()->is_VM_thread() || is_locked(), "checking");
 473   JvmtiTagHashmapEntry* entry;
 474   if (_free_entries == NULL) {
 475     entry = new JvmtiTagHashmapEntry(ref, tag);
 476   } else {
 477     assert(_free_entries_count > 0, "mismatched _free_entries_count");
 478     _free_entries_count--;
 479     entry = _free_entries;
 480     _free_entries = entry->next();
 481     entry->init(ref, tag);
 482   }
 483   return entry;
 484 }
 485 
 486 // destroy an entry by returning it to the free list
 487 void JvmtiTagMap::destroy_entry(JvmtiTagHashmapEntry* entry) {
 488   assert(SafepointSynchronize::is_at_safepoint() || is_locked(), "checking");
 489   // limit the size of the free list
 490   if (_free_entries_count >= max_free_entries) {
 491     delete entry;
 492   } else {
 493     entry->set_next(_free_entries);
 494     _free_entries = entry;
 495     _free_entries_count++;
 496   }
 497 }
 498 
 499 // returns the tag map for the given environments. If the tag map
 500 // doesn't exist then it is created.
 501 JvmtiTagMap* JvmtiTagMap::tag_map_for(JvmtiEnv* env) {
 502   JvmtiTagMap* tag_map = ((JvmtiEnvBase*)env)->tag_map_acquire();
 503   if (tag_map == NULL) {
 504     MutexLocker mu(JvmtiThreadState_lock);
 505     tag_map = ((JvmtiEnvBase*)env)->tag_map();
 506     if (tag_map == NULL) {
 507       tag_map = new JvmtiTagMap(env);
 508     }
 509   } else {
 510     CHECK_UNHANDLED_OOPS_ONLY(Thread::current()->clear_unhandled_oops());
 511   }
 512   return tag_map;
 513 }
 514 
 515 // iterate over all entries in the tag map.
 516 void JvmtiTagMap::entry_iterate(JvmtiTagHashmapEntryClosure* closure) {
 517   hashmap()->entry_iterate(closure);
 518 }
 519 
 520 // returns true if the hashmaps are empty
 521 bool JvmtiTagMap::is_empty() {
 522   assert(SafepointSynchronize::is_at_safepoint() || is_locked(), "checking");
 523   return hashmap()->entry_count() == 0;
 524 }
 525 
 526 
 527 // Return the tag value for an object, or 0 if the object is
 528 // not tagged
 529 //
 530 static inline jlong tag_for(JvmtiTagMap* tag_map, oop o) {
 531   JvmtiTagHashmapEntry* entry = tag_map->hashmap()->find(o);
 532   if (entry == NULL) {
 533     return 0;
 534   } else {
 535     return entry->tag();
 536   }
 537 }
 538 
 539 
 540 // A CallbackWrapper is a support class for querying and tagging an object
 541 // around a callback to a profiler. The constructor does pre-callback
 542 // work to get the tag value, klass tag value, ... and the destructor
 543 // does the post-callback work of tagging or untagging the object.
 544 //
 545 // {
 546 //   CallbackWrapper wrapper(tag_map, o);
 547 //
 548 //   (*callback)(wrapper.klass_tag(), wrapper.obj_size(), wrapper.obj_tag_p(), ...)
 549 //
 550 // } // wrapper goes out of scope here which results in the destructor
 551 //      checking to see if the object has been tagged, untagged, or the
 552 //      tag value has changed.
 553 //
 554 class CallbackWrapper : public StackObj {
 555  private:
 556   JvmtiTagMap* _tag_map;
 557   JvmtiTagHashmap* _hashmap;
 558   JvmtiTagHashmapEntry* _entry;
 559   oop _o;
 560   jlong _obj_size;
 561   jlong _obj_tag;
 562   jlong _klass_tag;
 563 
 564  protected:
 565   JvmtiTagMap* tag_map() const      { return _tag_map; }
 566 
 567   // invoked post-callback to tag, untag, or update the tag of an object
 568   void inline post_callback_tag_update(oop o, JvmtiTagHashmap* hashmap,
 569                                        JvmtiTagHashmapEntry* entry, jlong obj_tag);
 570  public:
 571   CallbackWrapper(JvmtiTagMap* tag_map, oop o) {
 572     assert(Thread::current()->is_VM_thread() || tag_map->is_locked(),
 573            "MT unsafe or must be VM thread");
 574 
 575     // object to tag
 576     _o = o;
 577 
 578     // object size
 579     _obj_size = (jlong)_o->size() * wordSize;
 580 
 581     // record the context
 582     _tag_map = tag_map;
 583     _hashmap = tag_map->hashmap();
 584     _entry = _hashmap->find(_o);
 585 
 586     // get object tag
 587     _obj_tag = (_entry == NULL) ? 0 : _entry->tag();
 588 
 589     // get the class and the class's tag value
 590     assert(SystemDictionary::Class_klass()->oop_is_instanceMirror(), "Is not?");
 591 
 592     _klass_tag = tag_for(tag_map, _o->klass()->java_mirror());
 593   }
 594 
 595   ~CallbackWrapper() {
 596     post_callback_tag_update(_o, _hashmap, _entry, _obj_tag);
 597   }
 598 
 599   inline jlong* obj_tag_p()                     { return &_obj_tag; }
 600   inline jlong obj_size() const                 { return _obj_size; }
 601   inline jlong obj_tag() const                  { return _obj_tag; }
 602   inline jlong klass_tag() const                { return _klass_tag; }
 603 };
 604 
 605 
 606 
 607 // callback post-callback to tag, untag, or update the tag of an object
 608 void inline CallbackWrapper::post_callback_tag_update(oop o,
 609                                                       JvmtiTagHashmap* hashmap,
 610                                                       JvmtiTagHashmapEntry* entry,
 611                                                       jlong obj_tag) {
 612   if (entry == NULL) {
 613     if (obj_tag != 0) {
 614       // callback has tagged the object
 615       assert(Thread::current()->is_VM_thread(), "must be VMThread");
 616       entry = tag_map()->create_entry(o, obj_tag);
 617       hashmap->add(o, entry);
 618     }
 619   } else {
 620     // object was previously tagged - the callback may have untagged
 621     // the object or changed the tag value
 622     if (obj_tag == 0) {
 623 
 624       JvmtiTagHashmapEntry* entry_removed = hashmap->remove(o);
 625       assert(entry_removed == entry, "checking");
 626       tag_map()->destroy_entry(entry);
 627 
 628     } else {
 629       if (obj_tag != entry->tag()) {
 630          entry->set_tag(obj_tag);
 631       }
 632     }
 633   }
 634 }
 635 
 636 // An extended CallbackWrapper used when reporting an object reference
 637 // to the agent.
 638 //
 639 // {
 640 //   TwoOopCallbackWrapper wrapper(tag_map, referrer, o);
 641 //
 642 //   (*callback)(wrapper.klass_tag(),
 643 //               wrapper.obj_size(),
 644 //               wrapper.obj_tag_p()
 645 //               wrapper.referrer_tag_p(), ...)
 646 //
 647 // } // wrapper goes out of scope here which results in the destructor
 648 //      checking to see if the referrer object has been tagged, untagged,
 649 //      or the tag value has changed.
 650 //
 651 class TwoOopCallbackWrapper : public CallbackWrapper {
 652  private:
 653   bool _is_reference_to_self;
 654   JvmtiTagHashmap* _referrer_hashmap;
 655   JvmtiTagHashmapEntry* _referrer_entry;
 656   oop _referrer;
 657   jlong _referrer_obj_tag;
 658   jlong _referrer_klass_tag;
 659   jlong* _referrer_tag_p;
 660 
 661   bool is_reference_to_self() const             { return _is_reference_to_self; }
 662 
 663  public:
 664   TwoOopCallbackWrapper(JvmtiTagMap* tag_map, oop referrer, oop o) :
 665     CallbackWrapper(tag_map, o)
 666   {
 667     // self reference needs to be handled in a special way
 668     _is_reference_to_self = (referrer == o);
 669 
 670     if (_is_reference_to_self) {
 671       _referrer_klass_tag = klass_tag();
 672       _referrer_tag_p = obj_tag_p();
 673     } else {
 674       _referrer = referrer;
 675       // record the context
 676       _referrer_hashmap = tag_map->hashmap();
 677       _referrer_entry = _referrer_hashmap->find(_referrer);
 678 
 679       // get object tag
 680       _referrer_obj_tag = (_referrer_entry == NULL) ? 0 : _referrer_entry->tag();
 681       _referrer_tag_p = &_referrer_obj_tag;
 682 
 683       // get referrer class tag.
 684       _referrer_klass_tag = tag_for(tag_map, _referrer->klass()->java_mirror());
 685     }
 686   }
 687 
 688   ~TwoOopCallbackWrapper() {
 689     if (!is_reference_to_self()){
 690       post_callback_tag_update(_referrer,
 691                                _referrer_hashmap,
 692                                _referrer_entry,
 693                                _referrer_obj_tag);
 694     }
 695   }
 696 
 697   // address of referrer tag
 698   // (for a self reference this will return the same thing as obj_tag_p())
 699   inline jlong* referrer_tag_p()        { return _referrer_tag_p; }
 700 
 701   // referrer's class tag
 702   inline jlong referrer_klass_tag()     { return _referrer_klass_tag; }
 703 };
 704 
 705 // tag an object
 706 //
 707 // This function is performance critical. If many threads attempt to tag objects
 708 // around the same time then it's possible that the Mutex associated with the
 709 // tag map will be a hot lock.
 710 void JvmtiTagMap::set_tag(jobject object, jlong tag) {
 711   MutexLocker ml(lock());
 712 
 713   // resolve the object
 714   oop o = JNIHandles::resolve_non_null(object);
 715 
 716   // see if the object is already tagged
 717   JvmtiTagHashmap* hashmap = _hashmap;
 718   JvmtiTagHashmapEntry* entry = hashmap->find(o);
 719 
 720   // if the object is not already tagged then we tag it
 721   if (entry == NULL) {
 722     if (tag != 0) {
 723       entry = create_entry(o, tag);
 724       hashmap->add(o, entry);
 725     } else {
 726       // no-op
 727     }
 728   } else {
 729     // if the object is already tagged then we either update
 730     // the tag (if a new tag value has been provided)
 731     // or remove the object if the new tag value is 0.
 732     if (tag == 0) {
 733       hashmap->remove(o);
 734       destroy_entry(entry);
 735     } else {
 736       entry->set_tag(tag);
 737     }
 738   }
 739 }
 740 
 741 // get the tag for an object
 742 jlong JvmtiTagMap::get_tag(jobject object) {
 743   MutexLocker ml(lock());
 744 
 745   // resolve the object
 746   oop o = JNIHandles::resolve_non_null(object);
 747 
 748   return tag_for(this, o);
 749 }
 750 
 751 
 752 // Helper class used to describe the static or instance fields of a class.
 753 // For each field it holds the field index (as defined by the JVMTI specification),
 754 // the field type, and the offset.
 755 
 756 class ClassFieldDescriptor: public CHeapObj<mtInternal> {
 757  private:
 758   int _field_index;
 759   int _field_offset;
 760   char _field_type;
 761  public:
 762   ClassFieldDescriptor(int index, char type, int offset) :
 763     _field_index(index), _field_type(type), _field_offset(offset) {
 764   }
 765   int field_index()  const  { return _field_index; }
 766   char field_type()  const  { return _field_type; }
 767   int field_offset() const  { return _field_offset; }
 768 };
 769 
 770 class ClassFieldMap: public CHeapObj<mtInternal> {
 771  private:
 772   enum {
 773     initial_field_count = 5
 774   };
 775 
 776   // list of field descriptors
 777   GrowableArray<ClassFieldDescriptor*>* _fields;
 778 
 779   // constructor
 780   ClassFieldMap();
 781 
 782   // add a field
 783   void add(int index, char type, int offset);
 784 
 785   // returns the field count for the given class
 786   static int compute_field_count(instanceKlassHandle ikh);
 787 
 788  public:
 789   ~ClassFieldMap();
 790 
 791   // access
 792   int field_count()                     { return _fields->length(); }
 793   ClassFieldDescriptor* field_at(int i) { return _fields->at(i); }
 794 
 795   // functions to create maps of static or instance fields
 796   static ClassFieldMap* create_map_of_static_fields(Klass* k);
 797   static ClassFieldMap* create_map_of_instance_fields(oop obj);
 798 };
 799 
 800 ClassFieldMap::ClassFieldMap() {
 801   _fields = new (ResourceObj::C_HEAP, mtInternal)
 802     GrowableArray<ClassFieldDescriptor*>(initial_field_count, true);
 803 }
 804 
 805 ClassFieldMap::~ClassFieldMap() {
 806   for (int i=0; i<_fields->length(); i++) {
 807     delete _fields->at(i);
 808   }
 809   delete _fields;
 810 }
 811 
 812 void ClassFieldMap::add(int index, char type, int offset) {
 813   ClassFieldDescriptor* field = new ClassFieldDescriptor(index, type, offset);
 814   _fields->append(field);
 815 }
 816 
 817 // Returns a heap allocated ClassFieldMap to describe the static fields
 818 // of the given class.
 819 //
 820 ClassFieldMap* ClassFieldMap::create_map_of_static_fields(Klass* k) {
 821   HandleMark hm;
 822   instanceKlassHandle ikh = instanceKlassHandle(Thread::current(), k);
 823 
 824   // create the field map
 825   ClassFieldMap* field_map = new ClassFieldMap();
 826 
 827   FilteredFieldStream f(ikh, false, false);
 828   int max_field_index = f.field_count()-1;
 829 
 830   int index = 0;
 831   for (FilteredFieldStream fld(ikh, true, true); !fld.eos(); fld.next(), index++) {
 832     // ignore instance fields
 833     if (!fld.access_flags().is_static()) {
 834       continue;
 835     }
 836     field_map->add(max_field_index - index, fld.signature()->byte_at(0), fld.offset());
 837   }
 838   return field_map;
 839 }
 840 
 841 // Returns a heap allocated ClassFieldMap to describe the instance fields
 842 // of the given class. All instance fields are included (this means public
 843 // and private fields declared in superclasses and superinterfaces too).
 844 //
 845 ClassFieldMap* ClassFieldMap::create_map_of_instance_fields(oop obj) {
 846   HandleMark hm;
 847   instanceKlassHandle ikh = instanceKlassHandle(Thread::current(), obj->klass());
 848 
 849   // create the field map
 850   ClassFieldMap* field_map = new ClassFieldMap();
 851 
 852   FilteredFieldStream f(ikh, false, false);
 853 
 854   int max_field_index = f.field_count()-1;
 855 
 856   int index = 0;
 857   for (FilteredFieldStream fld(ikh, false, false); !fld.eos(); fld.next(), index++) {
 858     // ignore static fields
 859     if (fld.access_flags().is_static()) {
 860       continue;
 861     }
 862     field_map->add(max_field_index - index, fld.signature()->byte_at(0), fld.offset());
 863   }
 864 
 865   return field_map;
 866 }
 867 
 868 // Helper class used to cache a ClassFileMap for the instance fields of
 869 // a cache. A JvmtiCachedClassFieldMap can be cached by an InstanceKlass during
 870 // heap iteration and avoid creating a field map for each object in the heap
 871 // (only need to create the map when the first instance of a class is encountered).
 872 //
 873 class JvmtiCachedClassFieldMap : public CHeapObj<mtInternal> {
 874  private:
 875    enum {
 876      initial_class_count = 200
 877    };
 878   ClassFieldMap* _field_map;
 879 
 880   ClassFieldMap* field_map() const          { return _field_map; }
 881 
 882   JvmtiCachedClassFieldMap(ClassFieldMap* field_map);
 883   ~JvmtiCachedClassFieldMap();
 884 
 885   static GrowableArray<InstanceKlass*>* _class_list;
 886   static void add_to_class_list(InstanceKlass* ik);
 887 
 888  public:
 889   // returns the field map for a given object (returning map cached
 890   // by InstanceKlass if possible
 891   static ClassFieldMap* get_map_of_instance_fields(oop obj);
 892 
 893   // removes the field map from all instanceKlasses - should be
 894   // called before VM operation completes
 895   static void clear_cache();
 896 
 897   // returns the number of ClassFieldMap cached by instanceKlasses
 898   static int cached_field_map_count();
 899 };
 900 
 901 GrowableArray<InstanceKlass*>* JvmtiCachedClassFieldMap::_class_list;
 902 
 903 JvmtiCachedClassFieldMap::JvmtiCachedClassFieldMap(ClassFieldMap* field_map) {
 904   _field_map = field_map;
 905 }
 906 
 907 JvmtiCachedClassFieldMap::~JvmtiCachedClassFieldMap() {
 908   if (_field_map != NULL) {
 909     delete _field_map;
 910   }
 911 }
 912 
 913 // Marker class to ensure that the class file map cache is only used in a defined
 914 // scope.
 915 class ClassFieldMapCacheMark : public StackObj {
 916  private:
 917    static bool _is_active;
 918  public:
 919    ClassFieldMapCacheMark() {
 920      assert(Thread::current()->is_VM_thread(), "must be VMThread");
 921      assert(JvmtiCachedClassFieldMap::cached_field_map_count() == 0, "cache not empty");
 922      assert(!_is_active, "ClassFieldMapCacheMark cannot be nested");
 923      _is_active = true;
 924    }
 925    ~ClassFieldMapCacheMark() {
 926      JvmtiCachedClassFieldMap::clear_cache();
 927      _is_active = false;
 928    }
 929    static bool is_active() { return _is_active; }
 930 };
 931 
 932 bool ClassFieldMapCacheMark::_is_active;
 933 
 934 
 935 // record that the given InstanceKlass is caching a field map
 936 void JvmtiCachedClassFieldMap::add_to_class_list(InstanceKlass* ik) {
 937   if (_class_list == NULL) {
 938     _class_list = new (ResourceObj::C_HEAP, mtInternal)
 939       GrowableArray<InstanceKlass*>(initial_class_count, true);
 940   }
 941   _class_list->push(ik);
 942 }
 943 
 944 // returns the instance field map for the given object
 945 // (returns field map cached by the InstanceKlass if possible)
 946 ClassFieldMap* JvmtiCachedClassFieldMap::get_map_of_instance_fields(oop obj) {
 947   assert(Thread::current()->is_VM_thread(), "must be VMThread");
 948   assert(ClassFieldMapCacheMark::is_active(), "ClassFieldMapCacheMark not active");
 949 
 950   Klass* k = obj->klass();
 951   InstanceKlass* ik = InstanceKlass::cast(k);
 952 
 953   // return cached map if possible
 954   JvmtiCachedClassFieldMap* cached_map = ik->jvmti_cached_class_field_map();
 955   if (cached_map != NULL) {
 956     assert(cached_map->field_map() != NULL, "missing field list");
 957     return cached_map->field_map();
 958   } else {
 959     ClassFieldMap* field_map = ClassFieldMap::create_map_of_instance_fields(obj);
 960     cached_map = new JvmtiCachedClassFieldMap(field_map);
 961     ik->set_jvmti_cached_class_field_map(cached_map);
 962     add_to_class_list(ik);
 963     return field_map;
 964   }
 965 }
 966 
 967 // remove the fields maps cached from all instanceKlasses
 968 void JvmtiCachedClassFieldMap::clear_cache() {
 969   assert(Thread::current()->is_VM_thread(), "must be VMThread");
 970   if (_class_list != NULL) {
 971     for (int i = 0; i < _class_list->length(); i++) {
 972       InstanceKlass* ik = _class_list->at(i);
 973       JvmtiCachedClassFieldMap* cached_map = ik->jvmti_cached_class_field_map();
 974       assert(cached_map != NULL, "should not be NULL");
 975       ik->set_jvmti_cached_class_field_map(NULL);
 976       delete cached_map;  // deletes the encapsulated field map
 977     }
 978     delete _class_list;
 979     _class_list = NULL;
 980   }
 981 }
 982 
 983 // returns the number of ClassFieldMap cached by instanceKlasses
 984 int JvmtiCachedClassFieldMap::cached_field_map_count() {
 985   return (_class_list == NULL) ? 0 : _class_list->length();
 986 }
 987 
 988 // helper function to indicate if an object is filtered by its tag or class tag
 989 static inline bool is_filtered_by_heap_filter(jlong obj_tag,
 990                                               jlong klass_tag,
 991                                               int heap_filter) {
 992   // apply the heap filter
 993   if (obj_tag != 0) {
 994     // filter out tagged objects
 995     if (heap_filter & JVMTI_HEAP_FILTER_TAGGED) return true;
 996   } else {
 997     // filter out untagged objects
 998     if (heap_filter & JVMTI_HEAP_FILTER_UNTAGGED) return true;
 999   }
1000   if (klass_tag != 0) {
1001     // filter out objects with tagged classes
1002     if (heap_filter & JVMTI_HEAP_FILTER_CLASS_TAGGED) return true;
1003   } else {
1004     // filter out objects with untagged classes.
1005     if (heap_filter & JVMTI_HEAP_FILTER_CLASS_UNTAGGED) return true;
1006   }
1007   return false;
1008 }
1009 
1010 // helper function to indicate if an object is filtered by a klass filter
1011 static inline bool is_filtered_by_klass_filter(oop obj, KlassHandle klass_filter) {
1012   if (!klass_filter.is_null()) {
1013     if (obj->klass() != klass_filter()) {
1014       return true;
1015     }
1016   }
1017   return false;
1018 }
1019 
1020 // helper function to tell if a field is a primitive field or not
1021 static inline bool is_primitive_field_type(char type) {
1022   return (type != 'L' && type != '[');
1023 }
1024 
1025 // helper function to copy the value from location addr to jvalue.
1026 static inline void copy_to_jvalue(jvalue *v, address addr, jvmtiPrimitiveType value_type) {
1027   switch (value_type) {
1028     case JVMTI_PRIMITIVE_TYPE_BOOLEAN : { v->z = *(jboolean*)addr; break; }
1029     case JVMTI_PRIMITIVE_TYPE_BYTE    : { v->b = *(jbyte*)addr;    break; }
1030     case JVMTI_PRIMITIVE_TYPE_CHAR    : { v->c = *(jchar*)addr;    break; }
1031     case JVMTI_PRIMITIVE_TYPE_SHORT   : { v->s = *(jshort*)addr;   break; }
1032     case JVMTI_PRIMITIVE_TYPE_INT     : { v->i = *(jint*)addr;     break; }
1033     case JVMTI_PRIMITIVE_TYPE_LONG    : { v->j = *(jlong*)addr;    break; }
1034     case JVMTI_PRIMITIVE_TYPE_FLOAT   : { v->f = *(jfloat*)addr;   break; }
1035     case JVMTI_PRIMITIVE_TYPE_DOUBLE  : { v->d = *(jdouble*)addr;  break; }
1036     default: ShouldNotReachHere();
1037   }
1038 }
1039 
1040 // helper function to invoke string primitive value callback
1041 // returns visit control flags
1042 static jint invoke_string_value_callback(jvmtiStringPrimitiveValueCallback cb,
1043                                          CallbackWrapper* wrapper,
1044                                          oop str,
1045                                          void* user_data)
1046 {
1047   assert(str->klass() == SystemDictionary::String_klass(), "not a string");
1048 
1049   typeArrayOop s_value = java_lang_String::value(str);
1050 
1051   // JDK-6584008: the value field may be null if a String instance is
1052   // partially constructed.
1053   if (s_value == NULL) {
1054     return 0;
1055   }
1056   // get the string value and length
1057   // (string value may be offset from the base)
1058   int s_len = java_lang_String::length(str);
1059   int s_offset = java_lang_String::offset(str);
1060   jchar* value;
1061   if (s_len > 0) {
1062     value = s_value->char_at_addr(s_offset);
1063   } else {
1064     value = (jchar*) s_value->base(T_CHAR);
1065   }
1066 
1067   // invoke the callback
1068   return (*cb)(wrapper->klass_tag(),
1069                wrapper->obj_size(),
1070                wrapper->obj_tag_p(),
1071                value,
1072                (jint)s_len,
1073                user_data);
1074 }
1075 
1076 // helper function to invoke string primitive value callback
1077 // returns visit control flags
1078 static jint invoke_array_primitive_value_callback(jvmtiArrayPrimitiveValueCallback cb,
1079                                                   CallbackWrapper* wrapper,
1080                                                   oop obj,
1081                                                   void* user_data)
1082 {
1083   assert(obj->is_typeArray(), "not a primitive array");
1084 
1085   // get base address of first element
1086   typeArrayOop array = typeArrayOop(obj);
1087   BasicType type = TypeArrayKlass::cast(array->klass())->element_type();
1088   void* elements = array->base(type);
1089 
1090   // jvmtiPrimitiveType is defined so this mapping is always correct
1091   jvmtiPrimitiveType elem_type = (jvmtiPrimitiveType)type2char(type);
1092 
1093   return (*cb)(wrapper->klass_tag(),
1094                wrapper->obj_size(),
1095                wrapper->obj_tag_p(),
1096                (jint)array->length(),
1097                elem_type,
1098                elements,
1099                user_data);
1100 }
1101 
1102 // helper function to invoke the primitive field callback for all static fields
1103 // of a given class
1104 static jint invoke_primitive_field_callback_for_static_fields
1105   (CallbackWrapper* wrapper,
1106    oop obj,
1107    jvmtiPrimitiveFieldCallback cb,
1108    void* user_data)
1109 {
1110   // for static fields only the index will be set
1111   static jvmtiHeapReferenceInfo reference_info = { 0 };
1112 
1113   assert(obj->klass() == SystemDictionary::Class_klass(), "not a class");
1114   if (java_lang_Class::is_primitive(obj)) {
1115     return 0;
1116   }
1117   Klass* klass = java_lang_Class::as_Klass(obj);
1118 
1119   // ignore classes for object and type arrays
1120   if (!klass->oop_is_instance()) {
1121     return 0;
1122   }
1123 
1124   // ignore classes which aren't linked yet
1125   InstanceKlass* ik = InstanceKlass::cast(klass);
1126   if (!ik->is_linked()) {
1127     return 0;
1128   }
1129 
1130   // get the field map
1131   ClassFieldMap* field_map = ClassFieldMap::create_map_of_static_fields(klass);
1132 
1133   // invoke the callback for each static primitive field
1134   for (int i=0; i<field_map->field_count(); i++) {
1135     ClassFieldDescriptor* field = field_map->field_at(i);
1136 
1137     // ignore non-primitive fields
1138     char type = field->field_type();
1139     if (!is_primitive_field_type(type)) {
1140       continue;
1141     }
1142     // one-to-one mapping
1143     jvmtiPrimitiveType value_type = (jvmtiPrimitiveType)type;
1144 
1145     // get offset and field value
1146     int offset = field->field_offset();
1147     address addr = (address)klass->java_mirror() + offset;
1148     jvalue value;
1149     copy_to_jvalue(&value, addr, value_type);
1150 
1151     // field index
1152     reference_info.field.index = field->field_index();
1153 
1154     // invoke the callback
1155     jint res = (*cb)(JVMTI_HEAP_REFERENCE_STATIC_FIELD,
1156                      &reference_info,
1157                      wrapper->klass_tag(),
1158                      wrapper->obj_tag_p(),
1159                      value,
1160                      value_type,
1161                      user_data);
1162     if (res & JVMTI_VISIT_ABORT) {
1163       delete field_map;
1164       return res;
1165     }
1166   }
1167 
1168   delete field_map;
1169   return 0;
1170 }
1171 
1172 // helper function to invoke the primitive field callback for all instance fields
1173 // of a given object
1174 static jint invoke_primitive_field_callback_for_instance_fields(
1175   CallbackWrapper* wrapper,
1176   oop obj,
1177   jvmtiPrimitiveFieldCallback cb,
1178   void* user_data)
1179 {
1180   // for instance fields only the index will be set
1181   static jvmtiHeapReferenceInfo reference_info = { 0 };
1182 
1183   // get the map of the instance fields
1184   ClassFieldMap* fields = JvmtiCachedClassFieldMap::get_map_of_instance_fields(obj);
1185 
1186   // invoke the callback for each instance primitive field
1187   for (int i=0; i<fields->field_count(); i++) {
1188     ClassFieldDescriptor* field = fields->field_at(i);
1189 
1190     // ignore non-primitive fields
1191     char type = field->field_type();
1192     if (!is_primitive_field_type(type)) {
1193       continue;
1194     }
1195     // one-to-one mapping
1196     jvmtiPrimitiveType value_type = (jvmtiPrimitiveType)type;
1197 
1198     // get offset and field value
1199     int offset = field->field_offset();
1200     address addr = (address)obj + offset;
1201     jvalue value;
1202     copy_to_jvalue(&value, addr, value_type);
1203 
1204     // field index
1205     reference_info.field.index = field->field_index();
1206 
1207     // invoke the callback
1208     jint res = (*cb)(JVMTI_HEAP_REFERENCE_FIELD,
1209                      &reference_info,
1210                      wrapper->klass_tag(),
1211                      wrapper->obj_tag_p(),
1212                      value,
1213                      value_type,
1214                      user_data);
1215     if (res & JVMTI_VISIT_ABORT) {
1216       return res;
1217     }
1218   }
1219   return 0;
1220 }
1221 
1222 
1223 // VM operation to iterate over all objects in the heap (both reachable
1224 // and unreachable)
1225 class VM_HeapIterateOperation: public VM_Operation {
1226  private:
1227   ObjectClosure* _blk;
1228  public:
1229   VM_HeapIterateOperation(ObjectClosure* blk) { _blk = blk; }
1230 
1231   VMOp_Type type() const { return VMOp_HeapIterateOperation; }
1232   void doit() {
1233     // allows class files maps to be cached during iteration
1234     ClassFieldMapCacheMark cm;
1235 
1236     // make sure that heap is parsable (fills TLABs with filler objects)
1237     Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1238 
1239     // Verify heap before iteration - if the heap gets corrupted then
1240     // JVMTI's IterateOverHeap will crash.
1241     if (VerifyBeforeIteration) {
1242       Universe::verify();
1243     }
1244 
1245     // do the iteration
1246     // If this operation encounters a bad object when using CMS,
1247     // consider using safe_object_iterate() which avoids perm gen
1248     // objects that may contain bad references.
1249     Universe::heap()->object_iterate(_blk);
1250   }
1251 
1252 };
1253 
1254 
1255 // An ObjectClosure used to support the deprecated IterateOverHeap and
1256 // IterateOverInstancesOfClass functions
1257 class IterateOverHeapObjectClosure: public ObjectClosure {
1258  private:
1259   JvmtiTagMap* _tag_map;
1260   KlassHandle _klass;
1261   jvmtiHeapObjectFilter _object_filter;
1262   jvmtiHeapObjectCallback _heap_object_callback;
1263   const void* _user_data;
1264 
1265   // accessors
1266   JvmtiTagMap* tag_map() const                    { return _tag_map; }
1267   jvmtiHeapObjectFilter object_filter() const     { return _object_filter; }
1268   jvmtiHeapObjectCallback object_callback() const { return _heap_object_callback; }
1269   KlassHandle klass() const                       { return _klass; }
1270   const void* user_data() const                   { return _user_data; }
1271 
1272   // indicates if iteration has been aborted
1273   bool _iteration_aborted;
1274   bool is_iteration_aborted() const               { return _iteration_aborted; }
1275   void set_iteration_aborted(bool aborted)        { _iteration_aborted = aborted; }
1276 
1277  public:
1278   IterateOverHeapObjectClosure(JvmtiTagMap* tag_map,
1279                                KlassHandle klass,
1280                                jvmtiHeapObjectFilter object_filter,
1281                                jvmtiHeapObjectCallback heap_object_callback,
1282                                const void* user_data) :
1283     _tag_map(tag_map),
1284     _klass(klass),
1285     _object_filter(object_filter),
1286     _heap_object_callback(heap_object_callback),
1287     _user_data(user_data),
1288     _iteration_aborted(false)
1289   {
1290   }
1291 
1292   void do_object(oop o);
1293 };
1294 
1295 // invoked for each object in the heap
1296 void IterateOverHeapObjectClosure::do_object(oop o) {
1297   // check if iteration has been halted
1298   if (is_iteration_aborted()) return;
1299 
1300   // ignore any objects that aren't visible to profiler
1301   if (!ServiceUtil::visible_oop(o)) return;
1302 
1303   // instanceof check when filtering by klass
1304   if (!klass().is_null() && !o->is_a(klass()())) {
1305     return;
1306   }
1307   // prepare for the calllback
1308   CallbackWrapper wrapper(tag_map(), o);
1309 
1310   // if the object is tagged and we're only interested in untagged objects
1311   // then don't invoke the callback. Similiarly, if the object is untagged
1312   // and we're only interested in tagged objects we skip the callback.
1313   if (wrapper.obj_tag() != 0) {
1314     if (object_filter() == JVMTI_HEAP_OBJECT_UNTAGGED) return;
1315   } else {
1316     if (object_filter() == JVMTI_HEAP_OBJECT_TAGGED) return;
1317   }
1318 
1319   // invoke the agent's callback
1320   jvmtiIterationControl control = (*object_callback())(wrapper.klass_tag(),
1321                                                        wrapper.obj_size(),
1322                                                        wrapper.obj_tag_p(),
1323                                                        (void*)user_data());
1324   if (control == JVMTI_ITERATION_ABORT) {
1325     set_iteration_aborted(true);
1326   }
1327 }
1328 
1329 // An ObjectClosure used to support the IterateThroughHeap function
1330 class IterateThroughHeapObjectClosure: public ObjectClosure {
1331  private:
1332   JvmtiTagMap* _tag_map;
1333   KlassHandle _klass;
1334   int _heap_filter;
1335   const jvmtiHeapCallbacks* _callbacks;
1336   const void* _user_data;
1337 
1338   // accessor functions
1339   JvmtiTagMap* tag_map() const                     { return _tag_map; }
1340   int heap_filter() const                          { return _heap_filter; }
1341   const jvmtiHeapCallbacks* callbacks() const      { return _callbacks; }
1342   KlassHandle klass() const                        { return _klass; }
1343   const void* user_data() const                    { return _user_data; }
1344 
1345   // indicates if the iteration has been aborted
1346   bool _iteration_aborted;
1347   bool is_iteration_aborted() const                { return _iteration_aborted; }
1348 
1349   // used to check the visit control flags. If the abort flag is set
1350   // then we set the iteration aborted flag so that the iteration completes
1351   // without processing any further objects
1352   bool check_flags_for_abort(jint flags) {
1353     bool is_abort = (flags & JVMTI_VISIT_ABORT) != 0;
1354     if (is_abort) {
1355       _iteration_aborted = true;
1356     }
1357     return is_abort;
1358   }
1359 
1360  public:
1361   IterateThroughHeapObjectClosure(JvmtiTagMap* tag_map,
1362                                   KlassHandle klass,
1363                                   int heap_filter,
1364                                   const jvmtiHeapCallbacks* heap_callbacks,
1365                                   const void* user_data) :
1366     _tag_map(tag_map),
1367     _klass(klass),
1368     _heap_filter(heap_filter),
1369     _callbacks(heap_callbacks),
1370     _user_data(user_data),
1371     _iteration_aborted(false)
1372   {
1373   }
1374 
1375   void do_object(oop o);
1376 };
1377 
1378 // invoked for each object in the heap
1379 void IterateThroughHeapObjectClosure::do_object(oop obj) {
1380   // check if iteration has been halted
1381   if (is_iteration_aborted()) return;
1382 
1383   // ignore any objects that aren't visible to profiler
1384   if (!ServiceUtil::visible_oop(obj)) return;
1385 
1386   // apply class filter
1387   if (is_filtered_by_klass_filter(obj, klass())) return;
1388 
1389   // prepare for callback
1390   CallbackWrapper wrapper(tag_map(), obj);
1391 
1392   // check if filtered by the heap filter
1393   if (is_filtered_by_heap_filter(wrapper.obj_tag(), wrapper.klass_tag(), heap_filter())) {
1394     return;
1395   }
1396 
1397   // for arrays we need the length, otherwise -1
1398   bool is_array = obj->is_array();
1399   int len = is_array ? arrayOop(obj)->length() : -1;
1400 
1401   // invoke the object callback (if callback is provided)
1402   if (callbacks()->heap_iteration_callback != NULL) {
1403     jvmtiHeapIterationCallback cb = callbacks()->heap_iteration_callback;
1404     jint res = (*cb)(wrapper.klass_tag(),
1405                      wrapper.obj_size(),
1406                      wrapper.obj_tag_p(),
1407                      (jint)len,
1408                      (void*)user_data());
1409     if (check_flags_for_abort(res)) return;
1410   }
1411 
1412   // for objects and classes we report primitive fields if callback provided
1413   if (callbacks()->primitive_field_callback != NULL && obj->is_instance()) {
1414     jint res;
1415     jvmtiPrimitiveFieldCallback cb = callbacks()->primitive_field_callback;
1416     if (obj->klass() == SystemDictionary::Class_klass()) {
1417       res = invoke_primitive_field_callback_for_static_fields(&wrapper,
1418                                                                     obj,
1419                                                                     cb,
1420                                                                     (void*)user_data());
1421     } else {
1422       res = invoke_primitive_field_callback_for_instance_fields(&wrapper,
1423                                                                       obj,
1424                                                                       cb,
1425                                                                       (void*)user_data());
1426     }
1427     if (check_flags_for_abort(res)) return;
1428   }
1429 
1430   // string callback
1431   if (!is_array &&
1432       callbacks()->string_primitive_value_callback != NULL &&
1433       obj->klass() == SystemDictionary::String_klass()) {
1434     jint res = invoke_string_value_callback(
1435                 callbacks()->string_primitive_value_callback,
1436                 &wrapper,
1437                 obj,
1438                 (void*)user_data() );
1439     if (check_flags_for_abort(res)) return;
1440   }
1441 
1442   // array callback
1443   if (is_array &&
1444       callbacks()->array_primitive_value_callback != NULL &&
1445       obj->is_typeArray()) {
1446     jint res = invoke_array_primitive_value_callback(
1447                callbacks()->array_primitive_value_callback,
1448                &wrapper,
1449                obj,
1450                (void*)user_data() );
1451     if (check_flags_for_abort(res)) return;
1452   }
1453 };
1454 
1455 
1456 // Deprecated function to iterate over all objects in the heap
1457 void JvmtiTagMap::iterate_over_heap(jvmtiHeapObjectFilter object_filter,
1458                                     KlassHandle klass,
1459                                     jvmtiHeapObjectCallback heap_object_callback,
1460                                     const void* user_data)
1461 {
1462   MutexLocker ml(Heap_lock);
1463   IterateOverHeapObjectClosure blk(this,
1464                                    klass,
1465                                    object_filter,
1466                                    heap_object_callback,
1467                                    user_data);
1468   VM_HeapIterateOperation op(&blk);
1469   VMThread::execute(&op);
1470 }
1471 
1472 
1473 // Iterates over all objects in the heap
1474 void JvmtiTagMap::iterate_through_heap(jint heap_filter,
1475                                        KlassHandle klass,
1476                                        const jvmtiHeapCallbacks* callbacks,
1477                                        const void* user_data)
1478 {
1479   MutexLocker ml(Heap_lock);
1480   IterateThroughHeapObjectClosure blk(this,
1481                                       klass,
1482                                       heap_filter,
1483                                       callbacks,
1484                                       user_data);
1485   VM_HeapIterateOperation op(&blk);
1486   VMThread::execute(&op);
1487 }
1488 
1489 // support class for get_objects_with_tags
1490 
1491 class TagObjectCollector : public JvmtiTagHashmapEntryClosure {
1492  private:
1493   JvmtiEnv* _env;
1494   jlong* _tags;
1495   jint _tag_count;
1496 
1497   GrowableArray<jobject>* _object_results;  // collected objects (JNI weak refs)
1498   GrowableArray<uint64_t>* _tag_results;    // collected tags
1499 
1500  public:
1501   TagObjectCollector(JvmtiEnv* env, const jlong* tags, jint tag_count) {
1502     _env = env;
1503     _tags = (jlong*)tags;
1504     _tag_count = tag_count;
1505     _object_results = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<jobject>(1,true);
1506     _tag_results = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<uint64_t>(1,true);
1507   }
1508 
1509   ~TagObjectCollector() {
1510     delete _object_results;
1511     delete _tag_results;
1512   }
1513 
1514   // for each tagged object check if the tag value matches
1515   // - if it matches then we create a JNI local reference to the object
1516   // and record the reference and tag value.
1517   //
1518   void do_entry(JvmtiTagHashmapEntry* entry) {
1519     for (int i=0; i<_tag_count; i++) {
1520       if (_tags[i] == entry->tag()) {
1521         oop o = entry->object();
1522         assert(o != NULL && Universe::heap()->is_in_reserved(o), "sanity check");
1523 #if INCLUDE_ALL_GCS
1524         if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) {
1525           // The reference in this tag map could be the only (implicitly weak)
1526           // reference to that object. If we hand it out, we need to keep it live wrt
1527           // SATB marking similar to other j.l.ref.Reference referents.
1528           G1SATBCardTableModRefBS::enqueue(o);
1529         }
1530 #endif
1531         jobject ref = JNIHandles::make_local(JavaThread::current(), o);
1532         _object_results->append(ref);
1533         _tag_results->append((uint64_t)entry->tag());
1534       }
1535     }
1536   }
1537 
1538   // return the results from the collection
1539   //
1540   jvmtiError result(jint* count_ptr, jobject** object_result_ptr, jlong** tag_result_ptr) {
1541     jvmtiError error;
1542     int count = _object_results->length();
1543     assert(count >= 0, "sanity check");
1544 
1545     // if object_result_ptr is not NULL then allocate the result and copy
1546     // in the object references.
1547     if (object_result_ptr != NULL) {
1548       error = _env->Allocate(count * sizeof(jobject), (unsigned char**)object_result_ptr);
1549       if (error != JVMTI_ERROR_NONE) {
1550         return error;
1551       }
1552       for (int i=0; i<count; i++) {
1553         (*object_result_ptr)[i] = _object_results->at(i);
1554       }
1555     }
1556 
1557     // if tag_result_ptr is not NULL then allocate the result and copy
1558     // in the tag values.
1559     if (tag_result_ptr != NULL) {
1560       error = _env->Allocate(count * sizeof(jlong), (unsigned char**)tag_result_ptr);
1561       if (error != JVMTI_ERROR_NONE) {
1562         if (object_result_ptr != NULL) {
1563           _env->Deallocate((unsigned char*)object_result_ptr);
1564         }
1565         return error;
1566       }
1567       for (int i=0; i<count; i++) {
1568         (*tag_result_ptr)[i] = (jlong)_tag_results->at(i);
1569       }
1570     }
1571 
1572     *count_ptr = count;
1573     return JVMTI_ERROR_NONE;
1574   }
1575 };
1576 
1577 // return the list of objects with the specified tags
1578 jvmtiError JvmtiTagMap::get_objects_with_tags(const jlong* tags,
1579   jint count, jint* count_ptr, jobject** object_result_ptr, jlong** tag_result_ptr) {
1580 
1581   TagObjectCollector collector(env(), tags, count);
1582   {
1583     // iterate over all tagged objects
1584     MutexLocker ml(lock());
1585     entry_iterate(&collector);
1586   }
1587   return collector.result(count_ptr, object_result_ptr, tag_result_ptr);
1588 }
1589 
1590 
1591 // ObjectMarker is used to support the marking objects when walking the
1592 // heap.
1593 //
1594 // This implementation uses the existing mark bits in an object for
1595 // marking. Objects that are marked must later have their headers restored.
1596 // As most objects are unlocked and don't have their identity hash computed
1597 // we don't have to save their headers. Instead we save the headers that
1598 // are "interesting". Later when the headers are restored this implementation
1599 // restores all headers to their initial value and then restores the few
1600 // objects that had interesting headers.
1601 //
1602 // Future work: This implementation currently uses growable arrays to save
1603 // the oop and header of interesting objects. As an optimization we could
1604 // use the same technique as the GC and make use of the unused area
1605 // between top() and end().
1606 //
1607 
1608 // An ObjectClosure used to restore the mark bits of an object
1609 class RestoreMarksClosure : public ObjectClosure {
1610  public:
1611   void do_object(oop o) {
1612     if (o != NULL) {
1613       markOop mark = o->mark();
1614       if (mark->is_marked()) {
1615         o->init_mark();
1616       }
1617     }
1618   }
1619 };
1620 
1621 // ObjectMarker provides the mark and visited functions
1622 class ObjectMarker : AllStatic {
1623  private:
1624   // saved headers
1625   static GrowableArray<oop>* _saved_oop_stack;
1626   static GrowableArray<markOop>* _saved_mark_stack;
1627   static bool _needs_reset;                  // do we need to reset mark bits?
1628 
1629  public:
1630   static void init();                       // initialize
1631   static void done();                       // clean-up
1632 
1633   static inline void mark(oop o);           // mark an object
1634   static inline bool visited(oop o);        // check if object has been visited
1635 
1636   static inline bool needs_reset()            { return _needs_reset; }
1637   static inline void set_needs_reset(bool v)  { _needs_reset = v; }
1638 };
1639 
1640 GrowableArray<oop>* ObjectMarker::_saved_oop_stack = NULL;
1641 GrowableArray<markOop>* ObjectMarker::_saved_mark_stack = NULL;
1642 bool ObjectMarker::_needs_reset = true;  // need to reset mark bits by default
1643 
1644 // initialize ObjectMarker - prepares for object marking
1645 void ObjectMarker::init() {
1646   assert(Thread::current()->is_VM_thread(), "must be VMThread");
1647 
1648   // prepare heap for iteration
1649   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1650 
1651   // create stacks for interesting headers
1652   _saved_mark_stack = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<markOop>(4000, true);
1653   _saved_oop_stack = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<oop>(4000, true);
1654 
1655   if (UseBiasedLocking) {
1656     BiasedLocking::preserve_marks();
1657   }
1658 }
1659 
1660 // Object marking is done so restore object headers
1661 void ObjectMarker::done() {
1662   // iterate over all objects and restore the mark bits to
1663   // their initial value
1664   RestoreMarksClosure blk;
1665   if (needs_reset()) {
1666     Universe::heap()->object_iterate(&blk);
1667   } else {
1668     // We don't need to reset mark bits on this call, but reset the
1669     // flag to the default for the next call.
1670     set_needs_reset(true);
1671   }
1672 
1673   // now restore the interesting headers
1674   for (int i = 0; i < _saved_oop_stack->length(); i++) {
1675     oop o = _saved_oop_stack->at(i);
1676     markOop mark = _saved_mark_stack->at(i);
1677     o->set_mark(mark);
1678   }
1679 
1680   if (UseBiasedLocking) {
1681     BiasedLocking::restore_marks();
1682   }
1683 
1684   // free the stacks
1685   delete _saved_oop_stack;
1686   delete _saved_mark_stack;
1687 }
1688 
1689 // mark an object
1690 inline void ObjectMarker::mark(oop o) {
1691   assert(Universe::heap()->is_in(o), "sanity check");
1692   assert(!o->mark()->is_marked(), "should only mark an object once");
1693 
1694   // object's mark word
1695   markOop mark = o->mark();
1696 
1697   if (mark->must_be_preserved(o)) {
1698     _saved_mark_stack->push(mark);
1699     _saved_oop_stack->push(o);
1700   }
1701 
1702   // mark the object
1703   o->set_mark(markOopDesc::prototype()->set_marked());
1704 }
1705 
1706 // return true if object is marked
1707 inline bool ObjectMarker::visited(oop o) {
1708   return o->mark()->is_marked();
1709 }
1710 
1711 // Stack allocated class to help ensure that ObjectMarker is used
1712 // correctly. Constructor initializes ObjectMarker, destructor calls
1713 // ObjectMarker's done() function to restore object headers.
1714 class ObjectMarkerController : public StackObj {
1715  public:
1716   ObjectMarkerController() {
1717     ObjectMarker::init();
1718   }
1719   ~ObjectMarkerController() {
1720     ObjectMarker::done();
1721   }
1722 };
1723 
1724 
1725 // helper to map a jvmtiHeapReferenceKind to an old style jvmtiHeapRootKind
1726 // (not performance critical as only used for roots)
1727 static jvmtiHeapRootKind toJvmtiHeapRootKind(jvmtiHeapReferenceKind kind) {
1728   switch (kind) {
1729     case JVMTI_HEAP_REFERENCE_JNI_GLOBAL:   return JVMTI_HEAP_ROOT_JNI_GLOBAL;
1730     case JVMTI_HEAP_REFERENCE_SYSTEM_CLASS: return JVMTI_HEAP_ROOT_SYSTEM_CLASS;
1731     case JVMTI_HEAP_REFERENCE_MONITOR:      return JVMTI_HEAP_ROOT_MONITOR;
1732     case JVMTI_HEAP_REFERENCE_STACK_LOCAL:  return JVMTI_HEAP_ROOT_STACK_LOCAL;
1733     case JVMTI_HEAP_REFERENCE_JNI_LOCAL:    return JVMTI_HEAP_ROOT_JNI_LOCAL;
1734     case JVMTI_HEAP_REFERENCE_THREAD:       return JVMTI_HEAP_ROOT_THREAD;
1735     case JVMTI_HEAP_REFERENCE_OTHER:        return JVMTI_HEAP_ROOT_OTHER;
1736     default: ShouldNotReachHere();          return JVMTI_HEAP_ROOT_OTHER;
1737   }
1738 }
1739 
1740 // Base class for all heap walk contexts. The base class maintains a flag
1741 // to indicate if the context is valid or not.
1742 class HeapWalkContext VALUE_OBJ_CLASS_SPEC {
1743  private:
1744   bool _valid;
1745  public:
1746   HeapWalkContext(bool valid)                   { _valid = valid; }
1747   void invalidate()                             { _valid = false; }
1748   bool is_valid() const                         { return _valid; }
1749 };
1750 
1751 // A basic heap walk context for the deprecated heap walking functions.
1752 // The context for a basic heap walk are the callbacks and fields used by
1753 // the referrer caching scheme.
1754 class BasicHeapWalkContext: public HeapWalkContext {
1755  private:
1756   jvmtiHeapRootCallback _heap_root_callback;
1757   jvmtiStackReferenceCallback _stack_ref_callback;
1758   jvmtiObjectReferenceCallback _object_ref_callback;
1759 
1760   // used for caching
1761   oop _last_referrer;
1762   jlong _last_referrer_tag;
1763 
1764  public:
1765   BasicHeapWalkContext() : HeapWalkContext(false) { }
1766 
1767   BasicHeapWalkContext(jvmtiHeapRootCallback heap_root_callback,
1768                        jvmtiStackReferenceCallback stack_ref_callback,
1769                        jvmtiObjectReferenceCallback object_ref_callback) :
1770     HeapWalkContext(true),
1771     _heap_root_callback(heap_root_callback),
1772     _stack_ref_callback(stack_ref_callback),
1773     _object_ref_callback(object_ref_callback),
1774     _last_referrer(NULL),
1775     _last_referrer_tag(0) {
1776   }
1777 
1778   // accessors
1779   jvmtiHeapRootCallback heap_root_callback() const         { return _heap_root_callback; }
1780   jvmtiStackReferenceCallback stack_ref_callback() const   { return _stack_ref_callback; }
1781   jvmtiObjectReferenceCallback object_ref_callback() const { return _object_ref_callback;  }
1782 
1783   oop last_referrer() const               { return _last_referrer; }
1784   void set_last_referrer(oop referrer)    { _last_referrer = referrer; }
1785   jlong last_referrer_tag() const         { return _last_referrer_tag; }
1786   void set_last_referrer_tag(jlong value) { _last_referrer_tag = value; }
1787 };
1788 
1789 // The advanced heap walk context for the FollowReferences functions.
1790 // The context is the callbacks, and the fields used for filtering.
1791 class AdvancedHeapWalkContext: public HeapWalkContext {
1792  private:
1793   jint _heap_filter;
1794   KlassHandle _klass_filter;
1795   const jvmtiHeapCallbacks* _heap_callbacks;
1796 
1797  public:
1798   AdvancedHeapWalkContext() : HeapWalkContext(false) { }
1799 
1800   AdvancedHeapWalkContext(jint heap_filter,
1801                            KlassHandle klass_filter,
1802                            const jvmtiHeapCallbacks* heap_callbacks) :
1803     HeapWalkContext(true),
1804     _heap_filter(heap_filter),
1805     _klass_filter(klass_filter),
1806     _heap_callbacks(heap_callbacks) {
1807   }
1808 
1809   // accessors
1810   jint heap_filter() const         { return _heap_filter; }
1811   KlassHandle klass_filter() const { return _klass_filter; }
1812 
1813   const jvmtiHeapReferenceCallback heap_reference_callback() const {
1814     return _heap_callbacks->heap_reference_callback;
1815   };
1816   const jvmtiPrimitiveFieldCallback primitive_field_callback() const {
1817     return _heap_callbacks->primitive_field_callback;
1818   }
1819   const jvmtiArrayPrimitiveValueCallback array_primitive_value_callback() const {
1820     return _heap_callbacks->array_primitive_value_callback;
1821   }
1822   const jvmtiStringPrimitiveValueCallback string_primitive_value_callback() const {
1823     return _heap_callbacks->string_primitive_value_callback;
1824   }
1825 };
1826 
1827 // The CallbackInvoker is a class with static functions that the heap walk can call
1828 // into to invoke callbacks. It works in one of two modes. The "basic" mode is
1829 // used for the deprecated IterateOverReachableObjects functions. The "advanced"
1830 // mode is for the newer FollowReferences function which supports a lot of
1831 // additional callbacks.
1832 class CallbackInvoker : AllStatic {
1833  private:
1834   // heap walk styles
1835   enum { basic, advanced };
1836   static int _heap_walk_type;
1837   static bool is_basic_heap_walk()           { return _heap_walk_type == basic; }
1838   static bool is_advanced_heap_walk()        { return _heap_walk_type == advanced; }
1839 
1840   // context for basic style heap walk
1841   static BasicHeapWalkContext _basic_context;
1842   static BasicHeapWalkContext* basic_context() {
1843     assert(_basic_context.is_valid(), "invalid");
1844     return &_basic_context;
1845   }
1846 
1847   // context for advanced style heap walk
1848   static AdvancedHeapWalkContext _advanced_context;
1849   static AdvancedHeapWalkContext* advanced_context() {
1850     assert(_advanced_context.is_valid(), "invalid");
1851     return &_advanced_context;
1852   }
1853 
1854   // context needed for all heap walks
1855   static JvmtiTagMap* _tag_map;
1856   static const void* _user_data;
1857   static GrowableArray<oop>* _visit_stack;
1858 
1859   // accessors
1860   static JvmtiTagMap* tag_map()                        { return _tag_map; }
1861   static const void* user_data()                       { return _user_data; }
1862   static GrowableArray<oop>* visit_stack()             { return _visit_stack; }
1863 
1864   // if the object hasn't been visited then push it onto the visit stack
1865   // so that it will be visited later
1866   static inline bool check_for_visit(oop obj) {
1867     if (!ObjectMarker::visited(obj)) visit_stack()->push(obj);
1868     return true;
1869   }
1870 
1871   // invoke basic style callbacks
1872   static inline bool invoke_basic_heap_root_callback
1873     (jvmtiHeapRootKind root_kind, oop obj);
1874   static inline bool invoke_basic_stack_ref_callback
1875     (jvmtiHeapRootKind root_kind, jlong thread_tag, jint depth, jmethodID method,
1876      int slot, oop obj);
1877   static inline bool invoke_basic_object_reference_callback
1878     (jvmtiObjectReferenceKind ref_kind, oop referrer, oop referree, jint index);
1879 
1880   // invoke advanced style callbacks
1881   static inline bool invoke_advanced_heap_root_callback
1882     (jvmtiHeapReferenceKind ref_kind, oop obj);
1883   static inline bool invoke_advanced_stack_ref_callback
1884     (jvmtiHeapReferenceKind ref_kind, jlong thread_tag, jlong tid, int depth,
1885      jmethodID method, jlocation bci, jint slot, oop obj);
1886   static inline bool invoke_advanced_object_reference_callback
1887     (jvmtiHeapReferenceKind ref_kind, oop referrer, oop referree, jint index);
1888 
1889   // used to report the value of primitive fields
1890   static inline bool report_primitive_field
1891     (jvmtiHeapReferenceKind ref_kind, oop obj, jint index, address addr, char type);
1892 
1893  public:
1894   // initialize for basic mode
1895   static void initialize_for_basic_heap_walk(JvmtiTagMap* tag_map,
1896                                              GrowableArray<oop>* visit_stack,
1897                                              const void* user_data,
1898                                              BasicHeapWalkContext context);
1899 
1900   // initialize for advanced mode
1901   static void initialize_for_advanced_heap_walk(JvmtiTagMap* tag_map,
1902                                                 GrowableArray<oop>* visit_stack,
1903                                                 const void* user_data,
1904                                                 AdvancedHeapWalkContext context);
1905 
1906    // functions to report roots
1907   static inline bool report_simple_root(jvmtiHeapReferenceKind kind, oop o);
1908   static inline bool report_jni_local_root(jlong thread_tag, jlong tid, jint depth,
1909     jmethodID m, oop o);
1910   static inline bool report_stack_ref_root(jlong thread_tag, jlong tid, jint depth,
1911     jmethodID method, jlocation bci, jint slot, oop o);
1912 
1913   // functions to report references
1914   static inline bool report_array_element_reference(oop referrer, oop referree, jint index);
1915   static inline bool report_class_reference(oop referrer, oop referree);
1916   static inline bool report_class_loader_reference(oop referrer, oop referree);
1917   static inline bool report_signers_reference(oop referrer, oop referree);
1918   static inline bool report_protection_domain_reference(oop referrer, oop referree);
1919   static inline bool report_superclass_reference(oop referrer, oop referree);
1920   static inline bool report_interface_reference(oop referrer, oop referree);
1921   static inline bool report_static_field_reference(oop referrer, oop referree, jint slot);
1922   static inline bool report_field_reference(oop referrer, oop referree, jint slot);
1923   static inline bool report_constant_pool_reference(oop referrer, oop referree, jint index);
1924   static inline bool report_primitive_array_values(oop array);
1925   static inline bool report_string_value(oop str);
1926   static inline bool report_primitive_instance_field(oop o, jint index, address value, char type);
1927   static inline bool report_primitive_static_field(oop o, jint index, address value, char type);
1928 };
1929 
1930 // statics
1931 int CallbackInvoker::_heap_walk_type;
1932 BasicHeapWalkContext CallbackInvoker::_basic_context;
1933 AdvancedHeapWalkContext CallbackInvoker::_advanced_context;
1934 JvmtiTagMap* CallbackInvoker::_tag_map;
1935 const void* CallbackInvoker::_user_data;
1936 GrowableArray<oop>* CallbackInvoker::_visit_stack;
1937 
1938 // initialize for basic heap walk (IterateOverReachableObjects et al)
1939 void CallbackInvoker::initialize_for_basic_heap_walk(JvmtiTagMap* tag_map,
1940                                                      GrowableArray<oop>* visit_stack,
1941                                                      const void* user_data,
1942                                                      BasicHeapWalkContext context) {
1943   _tag_map = tag_map;
1944   _visit_stack = visit_stack;
1945   _user_data = user_data;
1946   _basic_context = context;
1947   _advanced_context.invalidate();       // will trigger assertion if used
1948   _heap_walk_type = basic;
1949 }
1950 
1951 // initialize for advanced heap walk (FollowReferences)
1952 void CallbackInvoker::initialize_for_advanced_heap_walk(JvmtiTagMap* tag_map,
1953                                                         GrowableArray<oop>* visit_stack,
1954                                                         const void* user_data,
1955                                                         AdvancedHeapWalkContext context) {
1956   _tag_map = tag_map;
1957   _visit_stack = visit_stack;
1958   _user_data = user_data;
1959   _advanced_context = context;
1960   _basic_context.invalidate();      // will trigger assertion if used
1961   _heap_walk_type = advanced;
1962 }
1963 
1964 
1965 // invoke basic style heap root callback
1966 inline bool CallbackInvoker::invoke_basic_heap_root_callback(jvmtiHeapRootKind root_kind, oop obj) {
1967   assert(ServiceUtil::visible_oop(obj), "checking");
1968 
1969   // if we heap roots should be reported
1970   jvmtiHeapRootCallback cb = basic_context()->heap_root_callback();
1971   if (cb == NULL) {
1972     return check_for_visit(obj);
1973   }
1974 
1975   CallbackWrapper wrapper(tag_map(), obj);
1976   jvmtiIterationControl control = (*cb)(root_kind,
1977                                         wrapper.klass_tag(),
1978                                         wrapper.obj_size(),
1979                                         wrapper.obj_tag_p(),
1980                                         (void*)user_data());
1981   // push root to visit stack when following references
1982   if (control == JVMTI_ITERATION_CONTINUE &&
1983       basic_context()->object_ref_callback() != NULL) {
1984     visit_stack()->push(obj);
1985   }
1986   return control != JVMTI_ITERATION_ABORT;
1987 }
1988 
1989 // invoke basic style stack ref callback
1990 inline bool CallbackInvoker::invoke_basic_stack_ref_callback(jvmtiHeapRootKind root_kind,
1991                                                              jlong thread_tag,
1992                                                              jint depth,
1993                                                              jmethodID method,
1994                                                              jint slot,
1995                                                              oop obj) {
1996   assert(ServiceUtil::visible_oop(obj), "checking");
1997 
1998   // if we stack refs should be reported
1999   jvmtiStackReferenceCallback cb = basic_context()->stack_ref_callback();
2000   if (cb == NULL) {
2001     return check_for_visit(obj);
2002   }
2003 
2004   CallbackWrapper wrapper(tag_map(), obj);
2005   jvmtiIterationControl control = (*cb)(root_kind,
2006                                         wrapper.klass_tag(),
2007                                         wrapper.obj_size(),
2008                                         wrapper.obj_tag_p(),
2009                                         thread_tag,
2010                                         depth,
2011                                         method,
2012                                         slot,
2013                                         (void*)user_data());
2014   // push root to visit stack when following references
2015   if (control == JVMTI_ITERATION_CONTINUE &&
2016       basic_context()->object_ref_callback() != NULL) {
2017     visit_stack()->push(obj);
2018   }
2019   return control != JVMTI_ITERATION_ABORT;
2020 }
2021 
2022 // invoke basic style object reference callback
2023 inline bool CallbackInvoker::invoke_basic_object_reference_callback(jvmtiObjectReferenceKind ref_kind,
2024                                                                     oop referrer,
2025                                                                     oop referree,
2026                                                                     jint index) {
2027 
2028   assert(ServiceUtil::visible_oop(referrer), "checking");
2029   assert(ServiceUtil::visible_oop(referree), "checking");
2030 
2031   BasicHeapWalkContext* context = basic_context();
2032 
2033   // callback requires the referrer's tag. If it's the same referrer
2034   // as the last call then we use the cached value.
2035   jlong referrer_tag;
2036   if (referrer == context->last_referrer()) {
2037     referrer_tag = context->last_referrer_tag();
2038   } else {
2039     referrer_tag = tag_for(tag_map(), referrer);
2040   }
2041 
2042   // do the callback
2043   CallbackWrapper wrapper(tag_map(), referree);
2044   jvmtiObjectReferenceCallback cb = context->object_ref_callback();
2045   jvmtiIterationControl control = (*cb)(ref_kind,
2046                                         wrapper.klass_tag(),
2047                                         wrapper.obj_size(),
2048                                         wrapper.obj_tag_p(),
2049                                         referrer_tag,
2050                                         index,
2051                                         (void*)user_data());
2052 
2053   // record referrer and referrer tag. For self-references record the
2054   // tag value from the callback as this might differ from referrer_tag.
2055   context->set_last_referrer(referrer);
2056   if (referrer == referree) {
2057     context->set_last_referrer_tag(*wrapper.obj_tag_p());
2058   } else {
2059     context->set_last_referrer_tag(referrer_tag);
2060   }
2061 
2062   if (control == JVMTI_ITERATION_CONTINUE) {
2063     return check_for_visit(referree);
2064   } else {
2065     return control != JVMTI_ITERATION_ABORT;
2066   }
2067 }
2068 
2069 // invoke advanced style heap root callback
2070 inline bool CallbackInvoker::invoke_advanced_heap_root_callback(jvmtiHeapReferenceKind ref_kind,
2071                                                                 oop obj) {
2072   assert(ServiceUtil::visible_oop(obj), "checking");
2073 
2074   AdvancedHeapWalkContext* context = advanced_context();
2075 
2076   // check that callback is provided
2077   jvmtiHeapReferenceCallback cb = context->heap_reference_callback();
2078   if (cb == NULL) {
2079     return check_for_visit(obj);
2080   }
2081 
2082   // apply class filter
2083   if (is_filtered_by_klass_filter(obj, context->klass_filter())) {
2084     return check_for_visit(obj);
2085   }
2086 
2087   // setup the callback wrapper
2088   CallbackWrapper wrapper(tag_map(), obj);
2089 
2090   // apply tag filter
2091   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2092                                  wrapper.klass_tag(),
2093                                  context->heap_filter())) {
2094     return check_for_visit(obj);
2095   }
2096 
2097   // for arrays we need the length, otherwise -1
2098   jint len = (jint)(obj->is_array() ? arrayOop(obj)->length() : -1);
2099 
2100   // invoke the callback
2101   jint res  = (*cb)(ref_kind,
2102                     NULL, // referrer info
2103                     wrapper.klass_tag(),
2104                     0,    // referrer_class_tag is 0 for heap root
2105                     wrapper.obj_size(),
2106                     wrapper.obj_tag_p(),
2107                     NULL, // referrer_tag_p
2108                     len,
2109                     (void*)user_data());
2110   if (res & JVMTI_VISIT_ABORT) {
2111     return false;// referrer class tag
2112   }
2113   if (res & JVMTI_VISIT_OBJECTS) {
2114     check_for_visit(obj);
2115   }
2116   return true;
2117 }
2118 
2119 // report a reference from a thread stack to an object
2120 inline bool CallbackInvoker::invoke_advanced_stack_ref_callback(jvmtiHeapReferenceKind ref_kind,
2121                                                                 jlong thread_tag,
2122                                                                 jlong tid,
2123                                                                 int depth,
2124                                                                 jmethodID method,
2125                                                                 jlocation bci,
2126                                                                 jint slot,
2127                                                                 oop obj) {
2128   assert(ServiceUtil::visible_oop(obj), "checking");
2129 
2130   AdvancedHeapWalkContext* context = advanced_context();
2131 
2132   // check that callback is provider
2133   jvmtiHeapReferenceCallback cb = context->heap_reference_callback();
2134   if (cb == NULL) {
2135     return check_for_visit(obj);
2136   }
2137 
2138   // apply class filter
2139   if (is_filtered_by_klass_filter(obj, context->klass_filter())) {
2140     return check_for_visit(obj);
2141   }
2142 
2143   // setup the callback wrapper
2144   CallbackWrapper wrapper(tag_map(), obj);
2145 
2146   // apply tag filter
2147   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2148                                  wrapper.klass_tag(),
2149                                  context->heap_filter())) {
2150     return check_for_visit(obj);
2151   }
2152 
2153   // setup the referrer info
2154   jvmtiHeapReferenceInfo reference_info;
2155   reference_info.stack_local.thread_tag = thread_tag;
2156   reference_info.stack_local.thread_id = tid;
2157   reference_info.stack_local.depth = depth;
2158   reference_info.stack_local.method = method;
2159   reference_info.stack_local.location = bci;
2160   reference_info.stack_local.slot = slot;
2161 
2162   // for arrays we need the length, otherwise -1
2163   jint len = (jint)(obj->is_array() ? arrayOop(obj)->length() : -1);
2164 
2165   // call into the agent
2166   int res = (*cb)(ref_kind,
2167                   &reference_info,
2168                   wrapper.klass_tag(),
2169                   0,    // referrer_class_tag is 0 for heap root (stack)
2170                   wrapper.obj_size(),
2171                   wrapper.obj_tag_p(),
2172                   NULL, // referrer_tag is 0 for root
2173                   len,
2174                   (void*)user_data());
2175 
2176   if (res & JVMTI_VISIT_ABORT) {
2177     return false;
2178   }
2179   if (res & JVMTI_VISIT_OBJECTS) {
2180     check_for_visit(obj);
2181   }
2182   return true;
2183 }
2184 
2185 // This mask is used to pass reference_info to a jvmtiHeapReferenceCallback
2186 // only for ref_kinds defined by the JVM TI spec. Otherwise, NULL is passed.
2187 #define REF_INFO_MASK  ((1 << JVMTI_HEAP_REFERENCE_FIELD)         \
2188                       | (1 << JVMTI_HEAP_REFERENCE_STATIC_FIELD)  \
2189                       | (1 << JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT) \
2190                       | (1 << JVMTI_HEAP_REFERENCE_CONSTANT_POOL) \
2191                       | (1 << JVMTI_HEAP_REFERENCE_STACK_LOCAL)   \
2192                       | (1 << JVMTI_HEAP_REFERENCE_JNI_LOCAL))
2193 
2194 // invoke the object reference callback to report a reference
2195 inline bool CallbackInvoker::invoke_advanced_object_reference_callback(jvmtiHeapReferenceKind ref_kind,
2196                                                                        oop referrer,
2197                                                                        oop obj,
2198                                                                        jint index)
2199 {
2200   // field index is only valid field in reference_info
2201   static jvmtiHeapReferenceInfo reference_info = { 0 };
2202 
2203   assert(ServiceUtil::visible_oop(referrer), "checking");
2204   assert(ServiceUtil::visible_oop(obj), "checking");
2205 
2206   AdvancedHeapWalkContext* context = advanced_context();
2207 
2208   // check that callback is provider
2209   jvmtiHeapReferenceCallback cb = context->heap_reference_callback();
2210   if (cb == NULL) {
2211     return check_for_visit(obj);
2212   }
2213 
2214   // apply class filter
2215   if (is_filtered_by_klass_filter(obj, context->klass_filter())) {
2216     return check_for_visit(obj);
2217   }
2218 
2219   // setup the callback wrapper
2220   TwoOopCallbackWrapper wrapper(tag_map(), referrer, obj);
2221 
2222   // apply tag filter
2223   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2224                                  wrapper.klass_tag(),
2225                                  context->heap_filter())) {
2226     return check_for_visit(obj);
2227   }
2228 
2229   // field index is only valid field in reference_info
2230   reference_info.field.index = index;
2231 
2232   // for arrays we need the length, otherwise -1
2233   jint len = (jint)(obj->is_array() ? arrayOop(obj)->length() : -1);
2234 
2235   // invoke the callback
2236   int res = (*cb)(ref_kind,
2237                   (REF_INFO_MASK & (1 << ref_kind)) ? &reference_info : NULL,
2238                   wrapper.klass_tag(),
2239                   wrapper.referrer_klass_tag(),
2240                   wrapper.obj_size(),
2241                   wrapper.obj_tag_p(),
2242                   wrapper.referrer_tag_p(),
2243                   len,
2244                   (void*)user_data());
2245 
2246   if (res & JVMTI_VISIT_ABORT) {
2247     return false;
2248   }
2249   if (res & JVMTI_VISIT_OBJECTS) {
2250     check_for_visit(obj);
2251   }
2252   return true;
2253 }
2254 
2255 // report a "simple root"
2256 inline bool CallbackInvoker::report_simple_root(jvmtiHeapReferenceKind kind, oop obj) {
2257   assert(kind != JVMTI_HEAP_REFERENCE_STACK_LOCAL &&
2258          kind != JVMTI_HEAP_REFERENCE_JNI_LOCAL, "not a simple root");
2259   assert(ServiceUtil::visible_oop(obj), "checking");
2260 
2261   if (is_basic_heap_walk()) {
2262     // map to old style root kind
2263     jvmtiHeapRootKind root_kind = toJvmtiHeapRootKind(kind);
2264     return invoke_basic_heap_root_callback(root_kind, obj);
2265   } else {
2266     assert(is_advanced_heap_walk(), "wrong heap walk type");
2267     return invoke_advanced_heap_root_callback(kind, obj);
2268   }
2269 }
2270 
2271 
2272 // invoke the primitive array values
2273 inline bool CallbackInvoker::report_primitive_array_values(oop obj) {
2274   assert(obj->is_typeArray(), "not a primitive array");
2275 
2276   AdvancedHeapWalkContext* context = advanced_context();
2277   assert(context->array_primitive_value_callback() != NULL, "no callback");
2278 
2279   // apply class filter
2280   if (is_filtered_by_klass_filter(obj, context->klass_filter())) {
2281     return true;
2282   }
2283 
2284   CallbackWrapper wrapper(tag_map(), obj);
2285 
2286   // apply tag filter
2287   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2288                                  wrapper.klass_tag(),
2289                                  context->heap_filter())) {
2290     return true;
2291   }
2292 
2293   // invoke the callback
2294   int res = invoke_array_primitive_value_callback(context->array_primitive_value_callback(),
2295                                                   &wrapper,
2296                                                   obj,
2297                                                   (void*)user_data());
2298   return (!(res & JVMTI_VISIT_ABORT));
2299 }
2300 
2301 // invoke the string value callback
2302 inline bool CallbackInvoker::report_string_value(oop str) {
2303   assert(str->klass() == SystemDictionary::String_klass(), "not a string");
2304 
2305   AdvancedHeapWalkContext* context = advanced_context();
2306   assert(context->string_primitive_value_callback() != NULL, "no callback");
2307 
2308   // apply class filter
2309   if (is_filtered_by_klass_filter(str, context->klass_filter())) {
2310     return true;
2311   }
2312 
2313   CallbackWrapper wrapper(tag_map(), str);
2314 
2315   // apply tag filter
2316   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2317                                  wrapper.klass_tag(),
2318                                  context->heap_filter())) {
2319     return true;
2320   }
2321 
2322   // invoke the callback
2323   int res = invoke_string_value_callback(context->string_primitive_value_callback(),
2324                                          &wrapper,
2325                                          str,
2326                                          (void*)user_data());
2327   return (!(res & JVMTI_VISIT_ABORT));
2328 }
2329 
2330 // invoke the primitive field callback
2331 inline bool CallbackInvoker::report_primitive_field(jvmtiHeapReferenceKind ref_kind,
2332                                                     oop obj,
2333                                                     jint index,
2334                                                     address addr,
2335                                                     char type)
2336 {
2337   // for primitive fields only the index will be set
2338   static jvmtiHeapReferenceInfo reference_info = { 0 };
2339 
2340   AdvancedHeapWalkContext* context = advanced_context();
2341   assert(context->primitive_field_callback() != NULL, "no callback");
2342 
2343   // apply class filter
2344   if (is_filtered_by_klass_filter(obj, context->klass_filter())) {
2345     return true;
2346   }
2347 
2348   CallbackWrapper wrapper(tag_map(), obj);
2349 
2350   // apply tag filter
2351   if (is_filtered_by_heap_filter(wrapper.obj_tag(),
2352                                  wrapper.klass_tag(),
2353                                  context->heap_filter())) {
2354     return true;
2355   }
2356 
2357   // the field index in the referrer
2358   reference_info.field.index = index;
2359 
2360   // map the type
2361   jvmtiPrimitiveType value_type = (jvmtiPrimitiveType)type;
2362 
2363   // setup the jvalue
2364   jvalue value;
2365   copy_to_jvalue(&value, addr, value_type);
2366 
2367   jvmtiPrimitiveFieldCallback cb = context->primitive_field_callback();
2368   int res = (*cb)(ref_kind,
2369                   &reference_info,
2370                   wrapper.klass_tag(),
2371                   wrapper.obj_tag_p(),
2372                   value,
2373                   value_type,
2374                   (void*)user_data());
2375   return (!(res & JVMTI_VISIT_ABORT));
2376 }
2377 
2378 
2379 // instance field
2380 inline bool CallbackInvoker::report_primitive_instance_field(oop obj,
2381                                                              jint index,
2382                                                              address value,
2383                                                              char type) {
2384   return report_primitive_field(JVMTI_HEAP_REFERENCE_FIELD,
2385                                 obj,
2386                                 index,
2387                                 value,
2388                                 type);
2389 }
2390 
2391 // static field
2392 inline bool CallbackInvoker::report_primitive_static_field(oop obj,
2393                                                            jint index,
2394                                                            address value,
2395                                                            char type) {
2396   return report_primitive_field(JVMTI_HEAP_REFERENCE_STATIC_FIELD,
2397                                 obj,
2398                                 index,
2399                                 value,
2400                                 type);
2401 }
2402 
2403 // report a JNI local (root object) to the profiler
2404 inline bool CallbackInvoker::report_jni_local_root(jlong thread_tag, jlong tid, jint depth, jmethodID m, oop obj) {
2405   if (is_basic_heap_walk()) {
2406     return invoke_basic_stack_ref_callback(JVMTI_HEAP_ROOT_JNI_LOCAL,
2407                                            thread_tag,
2408                                            depth,
2409                                            m,
2410                                            -1,
2411                                            obj);
2412   } else {
2413     return invoke_advanced_stack_ref_callback(JVMTI_HEAP_REFERENCE_JNI_LOCAL,
2414                                               thread_tag, tid,
2415                                               depth,
2416                                               m,
2417                                               (jlocation)-1,
2418                                               -1,
2419                                               obj);
2420   }
2421 }
2422 
2423 
2424 // report a local (stack reference, root object)
2425 inline bool CallbackInvoker::report_stack_ref_root(jlong thread_tag,
2426                                                    jlong tid,
2427                                                    jint depth,
2428                                                    jmethodID method,
2429                                                    jlocation bci,
2430                                                    jint slot,
2431                                                    oop obj) {
2432   if (is_basic_heap_walk()) {
2433     return invoke_basic_stack_ref_callback(JVMTI_HEAP_ROOT_STACK_LOCAL,
2434                                            thread_tag,
2435                                            depth,
2436                                            method,
2437                                            slot,
2438                                            obj);
2439   } else {
2440     return invoke_advanced_stack_ref_callback(JVMTI_HEAP_REFERENCE_STACK_LOCAL,
2441                                               thread_tag,
2442                                               tid,
2443                                               depth,
2444                                               method,
2445                                               bci,
2446                                               slot,
2447                                               obj);
2448   }
2449 }
2450 
2451 // report an object referencing a class.
2452 inline bool CallbackInvoker::report_class_reference(oop referrer, oop referree) {
2453   if (is_basic_heap_walk()) {
2454     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_CLASS, referrer, referree, -1);
2455   } else {
2456     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_CLASS, referrer, referree, -1);
2457   }
2458 }
2459 
2460 // report a class referencing its class loader.
2461 inline bool CallbackInvoker::report_class_loader_reference(oop referrer, oop referree) {
2462   if (is_basic_heap_walk()) {
2463     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_CLASS_LOADER, referrer, referree, -1);
2464   } else {
2465     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_CLASS_LOADER, referrer, referree, -1);
2466   }
2467 }
2468 
2469 // report a class referencing its signers.
2470 inline bool CallbackInvoker::report_signers_reference(oop referrer, oop referree) {
2471   if (is_basic_heap_walk()) {
2472     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_SIGNERS, referrer, referree, -1);
2473   } else {
2474     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_SIGNERS, referrer, referree, -1);
2475   }
2476 }
2477 
2478 // report a class referencing its protection domain..
2479 inline bool CallbackInvoker::report_protection_domain_reference(oop referrer, oop referree) {
2480   if (is_basic_heap_walk()) {
2481     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_PROTECTION_DOMAIN, referrer, referree, -1);
2482   } else {
2483     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_PROTECTION_DOMAIN, referrer, referree, -1);
2484   }
2485 }
2486 
2487 // report a class referencing its superclass.
2488 inline bool CallbackInvoker::report_superclass_reference(oop referrer, oop referree) {
2489   if (is_basic_heap_walk()) {
2490     // Send this to be consistent with past implementation
2491     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_CLASS, referrer, referree, -1);
2492   } else {
2493     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_SUPERCLASS, referrer, referree, -1);
2494   }
2495 }
2496 
2497 // report a class referencing one of its interfaces.
2498 inline bool CallbackInvoker::report_interface_reference(oop referrer, oop referree) {
2499   if (is_basic_heap_walk()) {
2500     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_INTERFACE, referrer, referree, -1);
2501   } else {
2502     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_INTERFACE, referrer, referree, -1);
2503   }
2504 }
2505 
2506 // report a class referencing one of its static fields.
2507 inline bool CallbackInvoker::report_static_field_reference(oop referrer, oop referree, jint slot) {
2508   if (is_basic_heap_walk()) {
2509     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_STATIC_FIELD, referrer, referree, slot);
2510   } else {
2511     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_STATIC_FIELD, referrer, referree, slot);
2512   }
2513 }
2514 
2515 // report an array referencing an element object
2516 inline bool CallbackInvoker::report_array_element_reference(oop referrer, oop referree, jint index) {
2517   if (is_basic_heap_walk()) {
2518     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_ARRAY_ELEMENT, referrer, referree, index);
2519   } else {
2520     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT, referrer, referree, index);
2521   }
2522 }
2523 
2524 // report an object referencing an instance field object
2525 inline bool CallbackInvoker::report_field_reference(oop referrer, oop referree, jint slot) {
2526   if (is_basic_heap_walk()) {
2527     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_FIELD, referrer, referree, slot);
2528   } else {
2529     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_FIELD, referrer, referree, slot);
2530   }
2531 }
2532 
2533 // report an array referencing an element object
2534 inline bool CallbackInvoker::report_constant_pool_reference(oop referrer, oop referree, jint index) {
2535   if (is_basic_heap_walk()) {
2536     return invoke_basic_object_reference_callback(JVMTI_REFERENCE_CONSTANT_POOL, referrer, referree, index);
2537   } else {
2538     return invoke_advanced_object_reference_callback(JVMTI_HEAP_REFERENCE_CONSTANT_POOL, referrer, referree, index);
2539   }
2540 }
2541 
2542 // A supporting closure used to process simple roots
2543 class SimpleRootsClosure : public OopClosure {
2544  private:
2545   jvmtiHeapReferenceKind _kind;
2546   bool _continue;
2547 
2548   jvmtiHeapReferenceKind root_kind()    { return _kind; }
2549 
2550  public:
2551   void set_kind(jvmtiHeapReferenceKind kind) {
2552     _kind = kind;
2553     _continue = true;
2554   }
2555 
2556   inline bool stopped() {
2557     return !_continue;
2558   }
2559 
2560   void do_oop(oop* obj_p) {
2561     // iteration has terminated
2562     if (stopped()) {
2563       return;
2564     }
2565 
2566     // ignore null or deleted handles
2567     oop o = *obj_p;
2568     if (o == NULL || o == JNIHandles::deleted_handle()) {
2569       return;
2570     }
2571 
2572     assert(Universe::heap()->is_in_reserved(o), "should be impossible");
2573 
2574     jvmtiHeapReferenceKind kind = root_kind();
2575     if (kind == JVMTI_HEAP_REFERENCE_SYSTEM_CLASS) {
2576       // SystemDictionary::always_strong_oops_do reports the application
2577       // class loader as a root. We want this root to be reported as
2578       // a root kind of "OTHER" rather than "SYSTEM_CLASS".
2579       if (!o->is_instanceMirror()) {
2580         kind = JVMTI_HEAP_REFERENCE_OTHER;
2581       }
2582     }
2583 
2584     // some objects are ignored - in the case of simple
2585     // roots it's mostly Symbol*s that we are skipping
2586     // here.
2587     if (!ServiceUtil::visible_oop(o)) {
2588       return;
2589     }
2590 
2591     // invoke the callback
2592     _continue = CallbackInvoker::report_simple_root(kind, o);
2593 
2594   }
2595   virtual void do_oop(narrowOop* obj_p) { ShouldNotReachHere(); }
2596 };
2597 
2598 // A supporting closure used to process JNI locals
2599 class JNILocalRootsClosure : public OopClosure {
2600  private:
2601   jlong _thread_tag;
2602   jlong _tid;
2603   jint _depth;
2604   jmethodID _method;
2605   bool _continue;
2606  public:
2607   void set_context(jlong thread_tag, jlong tid, jint depth, jmethodID method) {
2608     _thread_tag = thread_tag;
2609     _tid = tid;
2610     _depth = depth;
2611     _method = method;
2612     _continue = true;
2613   }
2614 
2615   inline bool stopped() {
2616     return !_continue;
2617   }
2618 
2619   void do_oop(oop* obj_p) {
2620     // iteration has terminated
2621     if (stopped()) {
2622       return;
2623     }
2624 
2625     // ignore null or deleted handles
2626     oop o = *obj_p;
2627     if (o == NULL || o == JNIHandles::deleted_handle()) {
2628       return;
2629     }
2630 
2631     if (!ServiceUtil::visible_oop(o)) {
2632       return;
2633     }
2634 
2635     // invoke the callback
2636     _continue = CallbackInvoker::report_jni_local_root(_thread_tag, _tid, _depth, _method, o);
2637   }
2638   virtual void do_oop(narrowOop* obj_p) { ShouldNotReachHere(); }
2639 };
2640 
2641 
2642 // A VM operation to iterate over objects that are reachable from
2643 // a set of roots or an initial object.
2644 //
2645 // For VM_HeapWalkOperation the set of roots used is :-
2646 //
2647 // - All JNI global references
2648 // - All inflated monitors
2649 // - All classes loaded by the boot class loader (or all classes
2650 //     in the event that class unloading is disabled)
2651 // - All java threads
2652 // - For each java thread then all locals and JNI local references
2653 //      on the thread's execution stack
2654 // - All visible/explainable objects from Universes::oops_do
2655 //
2656 class VM_HeapWalkOperation: public VM_Operation {
2657  private:
2658   enum {
2659     initial_visit_stack_size = 4000
2660   };
2661 
2662   bool _is_advanced_heap_walk;                      // indicates FollowReferences
2663   JvmtiTagMap* _tag_map;
2664   Handle _initial_object;
2665   GrowableArray<oop>* _visit_stack;                 // the visit stack
2666 
2667   bool _collecting_heap_roots;                      // are we collecting roots
2668   bool _following_object_refs;                      // are we following object references
2669 
2670   bool _reporting_primitive_fields;                 // optional reporting
2671   bool _reporting_primitive_array_values;
2672   bool _reporting_string_values;
2673 
2674   GrowableArray<oop>* create_visit_stack() {
2675     return new (ResourceObj::C_HEAP, mtInternal) GrowableArray<oop>(initial_visit_stack_size, true);
2676   }
2677 
2678   // accessors
2679   bool is_advanced_heap_walk() const               { return _is_advanced_heap_walk; }
2680   JvmtiTagMap* tag_map() const                     { return _tag_map; }
2681   Handle initial_object() const                    { return _initial_object; }
2682 
2683   bool is_following_references() const             { return _following_object_refs; }
2684 
2685   bool is_reporting_primitive_fields()  const      { return _reporting_primitive_fields; }
2686   bool is_reporting_primitive_array_values() const { return _reporting_primitive_array_values; }
2687   bool is_reporting_string_values() const          { return _reporting_string_values; }
2688 
2689   GrowableArray<oop>* visit_stack() const          { return _visit_stack; }
2690 
2691   // iterate over the various object types
2692   inline bool iterate_over_array(oop o);
2693   inline bool iterate_over_type_array(oop o);
2694   inline bool iterate_over_class(oop o);
2695   inline bool iterate_over_object(oop o);
2696 
2697   // root collection
2698   inline bool collect_simple_roots();
2699   inline bool collect_stack_roots();
2700   inline bool collect_stack_roots(JavaThread* java_thread, JNILocalRootsClosure* blk);
2701 
2702   // visit an object
2703   inline bool visit(oop o);
2704 
2705  public:
2706   VM_HeapWalkOperation(JvmtiTagMap* tag_map,
2707                        Handle initial_object,
2708                        BasicHeapWalkContext callbacks,
2709                        const void* user_data);
2710 
2711   VM_HeapWalkOperation(JvmtiTagMap* tag_map,
2712                        Handle initial_object,
2713                        AdvancedHeapWalkContext callbacks,
2714                        const void* user_data);
2715 
2716   ~VM_HeapWalkOperation();
2717 
2718   VMOp_Type type() const { return VMOp_HeapWalkOperation; }
2719   void doit();
2720 };
2721 
2722 
2723 VM_HeapWalkOperation::VM_HeapWalkOperation(JvmtiTagMap* tag_map,
2724                                            Handle initial_object,
2725                                            BasicHeapWalkContext callbacks,
2726                                            const void* user_data) {
2727   _is_advanced_heap_walk = false;
2728   _tag_map = tag_map;
2729   _initial_object = initial_object;
2730   _following_object_refs = (callbacks.object_ref_callback() != NULL);
2731   _reporting_primitive_fields = false;
2732   _reporting_primitive_array_values = false;
2733   _reporting_string_values = false;
2734   _visit_stack = create_visit_stack();
2735 
2736 
2737   CallbackInvoker::initialize_for_basic_heap_walk(tag_map, _visit_stack, user_data, callbacks);
2738 }
2739 
2740 VM_HeapWalkOperation::VM_HeapWalkOperation(JvmtiTagMap* tag_map,
2741                                            Handle initial_object,
2742                                            AdvancedHeapWalkContext callbacks,
2743                                            const void* user_data) {
2744   _is_advanced_heap_walk = true;
2745   _tag_map = tag_map;
2746   _initial_object = initial_object;
2747   _following_object_refs = true;
2748   _reporting_primitive_fields = (callbacks.primitive_field_callback() != NULL);;
2749   _reporting_primitive_array_values = (callbacks.array_primitive_value_callback() != NULL);;
2750   _reporting_string_values = (callbacks.string_primitive_value_callback() != NULL);;
2751   _visit_stack = create_visit_stack();
2752 
2753   CallbackInvoker::initialize_for_advanced_heap_walk(tag_map, _visit_stack, user_data, callbacks);
2754 }
2755 
2756 VM_HeapWalkOperation::~VM_HeapWalkOperation() {
2757   if (_following_object_refs) {
2758     assert(_visit_stack != NULL, "checking");
2759     delete _visit_stack;
2760     _visit_stack = NULL;
2761   }
2762 }
2763 
2764 // an array references its class and has a reference to
2765 // each element in the array
2766 inline bool VM_HeapWalkOperation::iterate_over_array(oop o) {
2767   objArrayOop array = objArrayOop(o);
2768 
2769   // array reference to its class
2770   oop mirror = ObjArrayKlass::cast(array->klass())->java_mirror();
2771   if (!CallbackInvoker::report_class_reference(o, mirror)) {
2772     return false;
2773   }
2774 
2775   // iterate over the array and report each reference to a
2776   // non-null element
2777   for (int index=0; index<array->length(); index++) {
2778     oop elem = array->obj_at(index);
2779     if (elem == NULL) {
2780       continue;
2781     }
2782 
2783     // report the array reference o[index] = elem
2784     if (!CallbackInvoker::report_array_element_reference(o, elem, index)) {
2785       return false;
2786     }
2787   }
2788   return true;
2789 }
2790 
2791 // a type array references its class
2792 inline bool VM_HeapWalkOperation::iterate_over_type_array(oop o) {
2793   Klass* k = o->klass();
2794   oop mirror = k->java_mirror();
2795   if (!CallbackInvoker::report_class_reference(o, mirror)) {
2796     return false;
2797   }
2798 
2799   // report the array contents if required
2800   if (is_reporting_primitive_array_values()) {
2801     if (!CallbackInvoker::report_primitive_array_values(o)) {
2802       return false;
2803     }
2804   }
2805   return true;
2806 }
2807 
2808 // verify that a static oop field is in range
2809 static inline bool verify_static_oop(InstanceKlass* ik,
2810                                      oop mirror, int offset) {
2811   address obj_p = (address)mirror + offset;
2812   address start = (address)InstanceMirrorKlass::start_of_static_fields(mirror);
2813   address end = start + (java_lang_Class::static_oop_field_count(mirror) * heapOopSize);
2814   assert(end >= start, "sanity check");
2815 
2816   if (obj_p >= start && obj_p < end) {
2817     return true;
2818   } else {
2819     return false;
2820   }
2821 }
2822 
2823 // a class references its super class, interfaces, class loader, ...
2824 // and finally its static fields
2825 inline bool VM_HeapWalkOperation::iterate_over_class(oop java_class) {
2826   int i;
2827   Klass* klass = java_lang_Class::as_Klass(java_class);
2828 
2829   if (klass->oop_is_instance()) {
2830     InstanceKlass* ik = InstanceKlass::cast(klass);
2831 
2832     // ignore the class if it's has been initialized yet
2833     if (!ik->is_linked()) {
2834       return true;
2835     }
2836 
2837     // get the java mirror
2838     oop mirror = klass->java_mirror();
2839 
2840     // super (only if something more interesting than java.lang.Object)
2841     Klass* java_super = ik->java_super();
2842     if (java_super != NULL && java_super != SystemDictionary::Object_klass()) {
2843       oop super = java_super->java_mirror();
2844       if (!CallbackInvoker::report_superclass_reference(mirror, super)) {
2845         return false;
2846       }
2847     }
2848 
2849     // class loader
2850     oop cl = ik->class_loader();
2851     if (cl != NULL) {
2852       if (!CallbackInvoker::report_class_loader_reference(mirror, cl)) {
2853         return false;
2854       }
2855     }
2856 
2857     // protection domain
2858     oop pd = ik->protection_domain();
2859     if (pd != NULL) {
2860       if (!CallbackInvoker::report_protection_domain_reference(mirror, pd)) {
2861         return false;
2862       }
2863     }
2864 
2865     // signers
2866     oop signers = ik->signers();
2867     if (signers != NULL) {
2868       if (!CallbackInvoker::report_signers_reference(mirror, signers)) {
2869         return false;
2870       }
2871     }
2872 
2873     // references from the constant pool
2874     {
2875       ConstantPool* pool = ik->constants();
2876       for (int i = 1; i < pool->length(); i++) {
2877         constantTag tag = pool->tag_at(i).value();
2878         if (tag.is_string() || tag.is_klass()) {
2879           oop entry;
2880           if (tag.is_string()) {
2881             entry = pool->resolved_string_at(i);
2882             // If the entry is non-null it is resolved.
2883             if (entry == NULL) continue;
2884           } else {
2885             entry = pool->resolved_klass_at(i)->java_mirror();
2886           }
2887           if (!CallbackInvoker::report_constant_pool_reference(mirror, entry, (jint)i)) {
2888             return false;
2889           }
2890         }
2891       }
2892     }
2893 
2894     // interfaces
2895     // (These will already have been reported as references from the constant pool
2896     //  but are specified by IterateOverReachableObjects and must be reported).
2897     Array<Klass*>* interfaces = ik->local_interfaces();
2898     for (i = 0; i < interfaces->length(); i++) {
2899       oop interf = ((Klass*)interfaces->at(i))->java_mirror();
2900       if (interf == NULL) {
2901         continue;
2902       }
2903       if (!CallbackInvoker::report_interface_reference(mirror, interf)) {
2904         return false;
2905       }
2906     }
2907 
2908     // iterate over the static fields
2909 
2910     ClassFieldMap* field_map = ClassFieldMap::create_map_of_static_fields(klass);
2911     for (i=0; i<field_map->field_count(); i++) {
2912       ClassFieldDescriptor* field = field_map->field_at(i);
2913       char type = field->field_type();
2914       if (!is_primitive_field_type(type)) {
2915         oop fld_o = mirror->obj_field(field->field_offset());
2916         assert(verify_static_oop(ik, mirror, field->field_offset()), "sanity check");
2917         if (fld_o != NULL) {
2918           int slot = field->field_index();
2919           if (!CallbackInvoker::report_static_field_reference(mirror, fld_o, slot)) {
2920             delete field_map;
2921             return false;
2922           }
2923         }
2924       } else {
2925          if (is_reporting_primitive_fields()) {
2926            address addr = (address)mirror + field->field_offset();
2927            int slot = field->field_index();
2928            if (!CallbackInvoker::report_primitive_static_field(mirror, slot, addr, type)) {
2929              delete field_map;
2930              return false;
2931           }
2932         }
2933       }
2934     }
2935     delete field_map;
2936 
2937     return true;
2938   }
2939 
2940   return true;
2941 }
2942 
2943 // an object references a class and its instance fields
2944 // (static fields are ignored here as we report these as
2945 // references from the class).
2946 inline bool VM_HeapWalkOperation::iterate_over_object(oop o) {
2947   // reference to the class
2948   if (!CallbackInvoker::report_class_reference(o, o->klass()->java_mirror())) {
2949     return false;
2950   }
2951 
2952   // iterate over instance fields
2953   ClassFieldMap* field_map = JvmtiCachedClassFieldMap::get_map_of_instance_fields(o);
2954   for (int i=0; i<field_map->field_count(); i++) {
2955     ClassFieldDescriptor* field = field_map->field_at(i);
2956     char type = field->field_type();
2957     if (!is_primitive_field_type(type)) {
2958       oop fld_o = o->obj_field(field->field_offset());
2959       // ignore any objects that aren't visible to profiler
2960       if (fld_o != NULL && ServiceUtil::visible_oop(fld_o)) {
2961         assert(Universe::heap()->is_in_reserved(fld_o), "unsafe code should not "
2962                "have references to Klass* anymore");
2963         int slot = field->field_index();
2964         if (!CallbackInvoker::report_field_reference(o, fld_o, slot)) {
2965           return false;
2966         }
2967       }
2968     } else {
2969       if (is_reporting_primitive_fields()) {
2970         // primitive instance field
2971         address addr = (address)o + field->field_offset();
2972         int slot = field->field_index();
2973         if (!CallbackInvoker::report_primitive_instance_field(o, slot, addr, type)) {
2974           return false;
2975         }
2976       }
2977     }
2978   }
2979 
2980   // if the object is a java.lang.String
2981   if (is_reporting_string_values() &&
2982       o->klass() == SystemDictionary::String_klass()) {
2983     if (!CallbackInvoker::report_string_value(o)) {
2984       return false;
2985     }
2986   }
2987   return true;
2988 }
2989 
2990 
2991 // Collects all simple (non-stack) roots except for threads;
2992 // threads are handled in collect_stack_roots() as an optimization.
2993 // if there's a heap root callback provided then the callback is
2994 // invoked for each simple root.
2995 // if an object reference callback is provided then all simple
2996 // roots are pushed onto the marking stack so that they can be
2997 // processed later
2998 //
2999 inline bool VM_HeapWalkOperation::collect_simple_roots() {
3000   SimpleRootsClosure blk;
3001 
3002   // JNI globals
3003   blk.set_kind(JVMTI_HEAP_REFERENCE_JNI_GLOBAL);
3004   JNIHandles::oops_do(&blk);
3005   if (blk.stopped()) {
3006     return false;
3007   }
3008 
3009   // Preloaded classes and loader from the system dictionary
3010   blk.set_kind(JVMTI_HEAP_REFERENCE_SYSTEM_CLASS);
3011   SystemDictionary::always_strong_oops_do(&blk);
3012   KlassToOopClosure klass_blk(&blk);
3013   ClassLoaderDataGraph::always_strong_oops_do(&blk, &klass_blk, false);
3014   if (blk.stopped()) {
3015     return false;
3016   }
3017 
3018   // Inflated monitors
3019   blk.set_kind(JVMTI_HEAP_REFERENCE_MONITOR);
3020   ObjectSynchronizer::oops_do(&blk);
3021   if (blk.stopped()) {
3022     return false;
3023   }
3024 
3025   // threads are now handled in collect_stack_roots()
3026 
3027   // Other kinds of roots maintained by HotSpot
3028   // Many of these won't be visible but others (such as instances of important
3029   // exceptions) will be visible.
3030   blk.set_kind(JVMTI_HEAP_REFERENCE_OTHER);
3031   Universe::oops_do(&blk);
3032 
3033   // If there are any non-perm roots in the code cache, visit them.
3034   blk.set_kind(JVMTI_HEAP_REFERENCE_OTHER);
3035   CodeBlobToOopClosure look_in_blobs(&blk, !CodeBlobToOopClosure::FixRelocations);
3036   CodeCache::scavenge_root_nmethods_do(&look_in_blobs);
3037 
3038   return true;
3039 }
3040 
3041 // Walk the stack of a given thread and find all references (locals
3042 // and JNI calls) and report these as stack references
3043 inline bool VM_HeapWalkOperation::collect_stack_roots(JavaThread* java_thread,
3044                                                       JNILocalRootsClosure* blk)
3045 {
3046   oop threadObj = java_thread->threadObj();
3047   assert(threadObj != NULL, "sanity check");
3048 
3049   // only need to get the thread's tag once per thread
3050   jlong thread_tag = tag_for(_tag_map, threadObj);
3051 
3052   // also need the thread id
3053   jlong tid = java_lang_Thread::thread_id(threadObj);
3054 
3055 
3056   if (java_thread->has_last_Java_frame()) {
3057 
3058     // vframes are resource allocated
3059     Thread* current_thread = Thread::current();
3060     ResourceMark rm(current_thread);
3061     HandleMark hm(current_thread);
3062 
3063     RegisterMap reg_map(java_thread);
3064     frame f = java_thread->last_frame();
3065     vframe* vf = vframe::new_vframe(&f, &reg_map, java_thread);
3066 
3067     bool is_top_frame = true;
3068     int depth = 0;
3069     frame* last_entry_frame = NULL;
3070 
3071     while (vf != NULL) {
3072       if (vf->is_java_frame()) {
3073 
3074         // java frame (interpreted, compiled, ...)
3075         javaVFrame *jvf = javaVFrame::cast(vf);
3076 
3077         // the jmethodID
3078         jmethodID method = jvf->method()->jmethod_id();
3079 
3080         if (!(jvf->method()->is_native())) {
3081           jlocation bci = (jlocation)jvf->bci();
3082           StackValueCollection* locals = jvf->locals();
3083           for (int slot=0; slot<locals->size(); slot++) {
3084             if (locals->at(slot)->type() == T_OBJECT) {
3085               oop o = locals->obj_at(slot)();
3086               if (o == NULL) {
3087                 continue;
3088               }
3089 
3090               // stack reference
3091               if (!CallbackInvoker::report_stack_ref_root(thread_tag, tid, depth, method,
3092                                                    bci, slot, o)) {
3093                 return false;
3094               }
3095             }
3096           }
3097         } else {
3098           blk->set_context(thread_tag, tid, depth, method);
3099           if (is_top_frame) {
3100             // JNI locals for the top frame.
3101             java_thread->active_handles()->oops_do(blk);
3102           } else {
3103             if (last_entry_frame != NULL) {
3104               // JNI locals for the entry frame
3105               assert(last_entry_frame->is_entry_frame(), "checking");
3106               last_entry_frame->entry_frame_call_wrapper()->handles()->oops_do(blk);
3107             }
3108           }
3109         }
3110         last_entry_frame = NULL;
3111         depth++;
3112       } else {
3113         // externalVFrame - for an entry frame then we report the JNI locals
3114         // when we find the corresponding javaVFrame
3115         frame* fr = vf->frame_pointer();
3116         assert(fr != NULL, "sanity check");
3117         if (fr->is_entry_frame()) {
3118           last_entry_frame = fr;
3119         }
3120       }
3121 
3122       vf = vf->sender();
3123       is_top_frame = false;
3124     }
3125   } else {
3126     // no last java frame but there may be JNI locals
3127     blk->set_context(thread_tag, tid, 0, (jmethodID)NULL);
3128     java_thread->active_handles()->oops_do(blk);
3129   }
3130   return true;
3131 }
3132 
3133 
3134 // Collects the simple roots for all threads and collects all
3135 // stack roots - for each thread it walks the execution
3136 // stack to find all references and local JNI refs.
3137 inline bool VM_HeapWalkOperation::collect_stack_roots() {
3138   JNILocalRootsClosure blk;
3139   for (JavaThread* thread = Threads::first(); thread != NULL ; thread = thread->next()) {
3140     oop threadObj = thread->threadObj();
3141     if (threadObj != NULL && !thread->is_exiting() && !thread->is_hidden_from_external_view()) {
3142       // Collect the simple root for this thread before we
3143       // collect its stack roots
3144       if (!CallbackInvoker::report_simple_root(JVMTI_HEAP_REFERENCE_THREAD,
3145                                                threadObj)) {
3146         return false;
3147       }
3148       if (!collect_stack_roots(thread, &blk)) {
3149         return false;
3150       }
3151     }
3152   }
3153   return true;
3154 }
3155 
3156 // visit an object
3157 // first mark the object as visited
3158 // second get all the outbound references from this object (in other words, all
3159 // the objects referenced by this object).
3160 //
3161 bool VM_HeapWalkOperation::visit(oop o) {
3162   // mark object as visited
3163   assert(!ObjectMarker::visited(o), "can't visit same object more than once");
3164   ObjectMarker::mark(o);
3165 
3166   // instance
3167   if (o->is_instance()) {
3168     if (o->klass() == SystemDictionary::Class_klass()) {
3169       if (!java_lang_Class::is_primitive(o)) {
3170         // a java.lang.Class
3171         return iterate_over_class(o);
3172       }
3173     } else {
3174       return iterate_over_object(o);
3175     }
3176   }
3177 
3178   // object array
3179   if (o->is_objArray()) {
3180     return iterate_over_array(o);
3181   }
3182 
3183   // type array
3184   if (o->is_typeArray()) {
3185     return iterate_over_type_array(o);
3186   }
3187 
3188   return true;
3189 }
3190 
3191 void VM_HeapWalkOperation::doit() {
3192   ResourceMark rm;
3193   ObjectMarkerController marker;
3194   ClassFieldMapCacheMark cm;
3195 
3196   assert(visit_stack()->is_empty(), "visit stack must be empty");
3197 
3198   // the heap walk starts with an initial object or the heap roots
3199   if (initial_object().is_null()) {
3200     // If either collect_stack_roots() or collect_simple_roots()
3201     // returns false at this point, then there are no mark bits
3202     // to reset.
3203     ObjectMarker::set_needs_reset(false);
3204 
3205     // Calling collect_stack_roots() before collect_simple_roots()
3206     // can result in a big performance boost for an agent that is
3207     // focused on analyzing references in the thread stacks.
3208     if (!collect_stack_roots()) return;
3209 
3210     if (!collect_simple_roots()) return;
3211 
3212     // no early return so enable heap traversal to reset the mark bits
3213     ObjectMarker::set_needs_reset(true);
3214   } else {
3215     visit_stack()->push(initial_object()());
3216   }
3217 
3218   // object references required
3219   if (is_following_references()) {
3220 
3221     // visit each object until all reachable objects have been
3222     // visited or the callback asked to terminate the iteration.
3223     while (!visit_stack()->is_empty()) {
3224       oop o = visit_stack()->pop();
3225       if (!ObjectMarker::visited(o)) {
3226         if (!visit(o)) {
3227           break;
3228         }
3229       }
3230     }
3231   }
3232 }
3233 
3234 // iterate over all objects that are reachable from a set of roots
3235 void JvmtiTagMap::iterate_over_reachable_objects(jvmtiHeapRootCallback heap_root_callback,
3236                                                  jvmtiStackReferenceCallback stack_ref_callback,
3237                                                  jvmtiObjectReferenceCallback object_ref_callback,
3238                                                  const void* user_data) {
3239   MutexLocker ml(Heap_lock);
3240   BasicHeapWalkContext context(heap_root_callback, stack_ref_callback, object_ref_callback);
3241   VM_HeapWalkOperation op(this, Handle(), context, user_data);
3242   VMThread::execute(&op);
3243 }
3244 
3245 // iterate over all objects that are reachable from a given object
3246 void JvmtiTagMap::iterate_over_objects_reachable_from_object(jobject object,
3247                                                              jvmtiObjectReferenceCallback object_ref_callback,
3248                                                              const void* user_data) {
3249   oop obj = JNIHandles::resolve(object);
3250   Handle initial_object(Thread::current(), obj);
3251 
3252   MutexLocker ml(Heap_lock);
3253   BasicHeapWalkContext context(NULL, NULL, object_ref_callback);
3254   VM_HeapWalkOperation op(this, initial_object, context, user_data);
3255   VMThread::execute(&op);
3256 }
3257 
3258 // follow references from an initial object or the GC roots
3259 void JvmtiTagMap::follow_references(jint heap_filter,
3260                                     KlassHandle klass,
3261                                     jobject object,
3262                                     const jvmtiHeapCallbacks* callbacks,
3263                                     const void* user_data)
3264 {
3265   oop obj = JNIHandles::resolve(object);
3266   Handle initial_object(Thread::current(), obj);
3267 
3268   MutexLocker ml(Heap_lock);
3269   AdvancedHeapWalkContext context(heap_filter, klass, callbacks);
3270   VM_HeapWalkOperation op(this, initial_object, context, user_data);
3271   VMThread::execute(&op);
3272 }
3273 
3274 
3275 void JvmtiTagMap::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* f) {
3276   // No locks during VM bring-up (0 threads) and no safepoints after main
3277   // thread creation and before VMThread creation (1 thread); initial GC
3278   // verification can happen in that window which gets to here.
3279   assert(Threads::number_of_threads() <= 1 ||
3280          SafepointSynchronize::is_at_safepoint(),
3281          "must be executed at a safepoint");
3282   if (JvmtiEnv::environments_might_exist()) {
3283     JvmtiEnvIterator it;
3284     for (JvmtiEnvBase* env = it.first(); env != NULL; env = it.next(env)) {
3285       JvmtiTagMap* tag_map = env->tag_map_acquire();
3286       if (tag_map != NULL && !tag_map->is_empty()) {
3287         tag_map->do_weak_oops(is_alive, f);
3288       }
3289     }
3290   }
3291 }
3292 
3293 void JvmtiTagMap::do_weak_oops(BoolObjectClosure* is_alive, OopClosure* f) {
3294 
3295   // does this environment have the OBJECT_FREE event enabled
3296   bool post_object_free = env()->is_enabled(JVMTI_EVENT_OBJECT_FREE);
3297 
3298   // counters used for trace message
3299   int freed = 0;
3300   int moved = 0;
3301 
3302   JvmtiTagHashmap* hashmap = this->hashmap();
3303 
3304   // reenable sizing (if disabled)
3305   hashmap->set_resizing_enabled(true);
3306 
3307   // if the hashmap is empty then we can skip it
3308   if (hashmap->_entry_count == 0) {
3309     return;
3310   }
3311 
3312   // now iterate through each entry in the table
3313 
3314   JvmtiTagHashmapEntry** table = hashmap->table();
3315   int size = hashmap->size();
3316 
3317   JvmtiTagHashmapEntry* delayed_add = NULL;
3318 
3319   for (int pos = 0; pos < size; ++pos) {
3320     JvmtiTagHashmapEntry* entry = table[pos];
3321     JvmtiTagHashmapEntry* prev = NULL;
3322 
3323     while (entry != NULL) {
3324       JvmtiTagHashmapEntry* next = entry->next();
3325 
3326       oop* obj = entry->object_addr();
3327 
3328       // has object been GC'ed
3329       if (!is_alive->do_object_b(entry->object())) {
3330         // grab the tag
3331         jlong tag = entry->tag();
3332         guarantee(tag != 0, "checking");
3333 
3334         // remove GC'ed entry from hashmap and return the
3335         // entry to the free list
3336         hashmap->remove(prev, pos, entry);
3337         destroy_entry(entry);
3338 
3339         // post the event to the profiler
3340         if (post_object_free) {
3341           JvmtiExport::post_object_free(env(), tag);
3342         }
3343 
3344         ++freed;
3345       } else {
3346         f->do_oop(entry->object_addr());
3347         oop new_oop = entry->object();
3348 
3349         // if the object has moved then re-hash it and move its
3350         // entry to its new location.
3351         unsigned int new_pos = JvmtiTagHashmap::hash(new_oop, size);
3352         if (new_pos != (unsigned int)pos) {
3353           if (prev == NULL) {
3354             table[pos] = next;
3355           } else {
3356             prev->set_next(next);
3357           }
3358           if (new_pos < (unsigned int)pos) {
3359             entry->set_next(table[new_pos]);
3360             table[new_pos] = entry;
3361           } else {
3362             // Delay adding this entry to it's new position as we'd end up
3363             // hitting it again during this iteration.
3364             entry->set_next(delayed_add);
3365             delayed_add = entry;
3366           }
3367           moved++;
3368         } else {
3369           // object didn't move
3370           prev = entry;
3371         }
3372       }
3373 
3374       entry = next;
3375     }
3376   }
3377 
3378   // Re-add all the entries which were kept aside
3379   while (delayed_add != NULL) {
3380     JvmtiTagHashmapEntry* next = delayed_add->next();
3381     unsigned int pos = JvmtiTagHashmap::hash(delayed_add->object(), size);
3382     delayed_add->set_next(table[pos]);
3383     table[pos] = delayed_add;
3384     delayed_add = next;
3385   }
3386 
3387   // stats
3388   if (TraceJVMTIObjectTagging) {
3389     int post_total = hashmap->_entry_count;
3390     int pre_total = post_total + freed;
3391 
3392     tty->print_cr("(%d->%d, %d freed, %d total moves)",
3393         pre_total, post_total, freed, moved);
3394   }
3395 }