changeset 56676:765ecbffe88a

8232575: Shenandoah: asynchronous object/region pinning Reviewed-by: rkennke
author shade
date Mon, 21 Oct 2019 15:11:43 +0200
parents ff3eab78554f
children 14e098407bb0
files src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
diffstat 10 files changed, 124 insertions(+), 39 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -75,6 +75,9 @@
   RegionData *data = get_region_data_cache(heap->num_regions());
   size_t cnt = 0;
 
+  // About to choose the collection set, make sure we have pinned regions in correct state
+  heap->assert_pinned_region_status();
+
   // Step 0. Prepare all regions
   for (size_t i = 0; i < heap->num_regions(); i++) {
     ShenandoahHeapRegion* r = heap->get_region(i);
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -59,6 +59,9 @@
   RegionData *data = get_region_data_cache(heap->num_regions());
   size_t cnt = 0;
 
+  // About to choose the collection set, make sure we have pinned regions in correct state
+  heap->assert_pinned_region_status();
+
   // Step 0. Prepare all regions
 
   for (size_t i = 0; i < heap->num_regions(); i++) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -1507,6 +1507,14 @@
       make_parsable(true);
     }
 
+    // We are about to select the collection set, make sure it knows about
+    // current pinning status. Also, this allows trashing more regions that
+    // now have their pinning status dropped.
+    {
+      ShenandoahGCPhase phase(ShenandoahPhaseTimings::sync_pinned);
+      sync_pinned_region_status();
+    }
+
     // Trash the collection set left over from previous cycle, if any.
     {
       ShenandoahGCPhase phase(ShenandoahPhaseTimings::trash_cset);
@@ -1784,6 +1792,7 @@
         // it, we fail degeneration right away and slide into Full GC to recover.
 
         {
+          sync_pinned_region_status();
           collection_set()->clear_current_index();
 
           ShenandoahHeapRegion* r;
@@ -2148,16 +2157,45 @@
 }
 
 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
-  ShenandoahHeapLocker locker(lock());
-  heap_region_containing(o)->make_pinned();
+  heap_region_containing(o)->record_pin();
   return o;
 }
 
 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
+  heap_region_containing(o)->record_unpin();
+}
+
+void ShenandoahHeap::sync_pinned_region_status() {
   ShenandoahHeapLocker locker(lock());
-  heap_region_containing(o)->make_unpinned();
+
+  for (size_t i = 0; i < num_regions(); i++) {
+    ShenandoahHeapRegion *r = get_region(i);
+    if (r->is_active()) {
+      if (r->is_pinned()) {
+        if (r->pin_count() == 0) {
+          r->make_unpinned();
+        }
+      } else {
+        if (r->pin_count() > 0) {
+          r->make_pinned();
+        }
+      }
+    }
+  }
+
+  assert_pinned_region_status();
 }
 
+#ifdef ASSERT
+void ShenandoahHeap::assert_pinned_region_status() {
+  for (size_t i = 0; i < num_regions(); i++) {
+    ShenandoahHeapRegion* r = get_region(i);
+    assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
+           "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+  }
+}
+#endif
+
 GCTimer* ShenandoahHeap::gc_timer() const {
   return _gc_timer;
 }
@@ -2316,6 +2354,13 @@
     verifier()->verify_roots_in_to_space();
   }
 
+  // Drop unnecessary "pinned" state from regions that does not have CP marks
+  // anymore, as this would allow trashing them below.
+  {
+    ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_sync_pinned);
+    sync_pinned_region_status();
+  }
+
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
     trash_cset_regions();
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Mon Oct 21 15:11:43 2019 +0200
@@ -43,6 +43,7 @@
 class ShenandoahGCStateResetter;
 class ShenandoahHeuristics;
 class ShenandoahMarkingContext;
+class ShenandoahMarkCompact;
 class ShenandoahMode;
 class ShenandoahPhaseTimings;
 class ShenandoahHeap;
@@ -574,6 +575,9 @@
   oop pin_object(JavaThread* thread, oop obj);
   void unpin_object(JavaThread* thread, oop obj);
 
+  void sync_pinned_region_status();
+  void assert_pinned_region_status() NOT_DEBUG_RETURN;
+
 // ---------- Allocation support
 //
 private:
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -59,7 +59,6 @@
   _reserved(MemRegion(start, size_words)),
   _region_number(index),
   _new_top(NULL),
-  _critical_pins(0),
   _empty_time(os::elapsedTime()),
   _state(committed ? _empty_committed : _empty_uncommitted),
   _tlab_allocs(0),
@@ -69,7 +68,8 @@
   _seqnum_first_alloc_gc(0),
   _seqnum_last_alloc_mutator(0),
   _seqnum_last_alloc_gc(0),
-  _live_data(0) {
+  _live_data(0),
+  _critical_pins(0) {
 
   ContiguousSpace::initialize(_reserved, true, committed);
 }
@@ -187,25 +187,21 @@
 
 void ShenandoahHeapRegion::make_pinned() {
   _heap->assert_heaplock_owned_by_current_thread();
+  assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
+
   switch (_state) {
     case _regular:
-      assert (_critical_pins == 0, "sanity");
       set_state(_pinned);
     case _pinned_cset:
     case _pinned:
-      _critical_pins++;
       return;
     case _humongous_start:
-      assert (_critical_pins == 0, "sanity");
       set_state(_pinned_humongous_start);
     case _pinned_humongous_start:
-      _critical_pins++;
       return;
     case _cset:
       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
-      assert (_critical_pins == 0, "sanity");
       _state = _pinned_cset;
-      _critical_pins++;
       return;
     default:
       report_illegal_transition("pinning");
@@ -214,32 +210,21 @@
 
 void ShenandoahHeapRegion::make_unpinned() {
   _heap->assert_heaplock_owned_by_current_thread();
+  assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
+
   switch (_state) {
     case _pinned:
-      assert (_critical_pins > 0, "sanity");
-      _critical_pins--;
-      if (_critical_pins == 0) {
-        set_state(_regular);
-      }
+      set_state(_regular);
       return;
     case _regular:
     case _humongous_start:
-      assert (_critical_pins == 0, "sanity");
       return;
     case _pinned_cset:
       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
-      assert (_critical_pins > 0, "sanity");
-      _critical_pins--;
-      if (_critical_pins == 0) {
-        set_state(_cset);
-      }
+      set_state(_cset);
       return;
     case _pinned_humongous_start:
-      assert (_critical_pins > 0, "sanity");
-      _critical_pins--;
-      if (_critical_pins == 0) {
-        set_state(_humongous_start);
-      }
+      set_state(_humongous_start);
       return;
     default:
       report_illegal_transition("unpinning");
@@ -434,7 +419,7 @@
   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
-  st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
+  st->print("|CP " SIZE_FORMAT_W(3), pin_count());
   st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
             seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
             seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
@@ -702,3 +687,16 @@
   }
   _state = to;
 }
+
+void ShenandoahHeapRegion::record_pin() {
+  Atomic::add((size_t)1, &_critical_pins);
+}
+
+void ShenandoahHeapRegion::record_unpin() {
+  assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", region_number());
+  Atomic::sub((size_t)1, &_critical_pins);
+}
+
+size_t ShenandoahHeapRegion::pin_count() const {
+  return Atomic::load(&_critical_pins);
+}
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp	Mon Oct 21 15:11:43 2019 +0200
@@ -203,6 +203,10 @@
   RegionState state()              const { return _state; }
   int  state_ordinal()             const { return region_state_to_ordinal(_state); }
 
+  void record_pin();
+  void record_unpin();
+  size_t pin_count() const;
+
 private:
   static size_t RegionCount;
   static size_t RegionSizeBytes;
@@ -238,7 +242,6 @@
 
   // Rarely updated fields
   HeapWord* _new_top;
-  size_t _critical_pins;
   double _empty_time;
 
   // Seldom updated fields
@@ -255,6 +258,7 @@
   uint64_t _seqnum_last_alloc_gc;
 
   volatile size_t _live_data;
+  volatile size_t _critical_pins;
 
   // Claim some space at the end to protect next region
   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -120,6 +120,9 @@
 
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
+  // Check all pinned regions have updated status before choosing the collection set.
+  heap->assert_pinned_region_status();
+
   // Step 1. Build up the region candidates we care about, rejecting losers and accepting winners right away.
 
   size_t num_regions = heap->num_regions();
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -128,6 +128,9 @@
     // e. Set back forwarded objects bit back, in case some steps above dropped it.
     heap->set_has_forwarded_objects(has_forwarded_objects);
 
+    // f. Sync pinned region status from the CP marks
+    heap->sync_pinned_region_status();
+
     // The rest of prologue:
     BiasedLocking::preserve_marks();
     _preserved_marks->init(heap->workers()->active_workers());
@@ -505,6 +508,10 @@
 
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
+  // About to figure out which regions can be compacted, make sure pinning status
+  // had been updated in GC prologue.
+  heap->assert_pinned_region_status();
+
   {
     // Trash the immediately collectible regions before computing addresses
     ShenandoahTrashImmediateGarbageClosure tigcl;
--- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp	Mon Oct 21 15:11:43 2019 +0200
@@ -101,6 +101,7 @@
   f(purge_cldg,                                     "    CLDG")                         \
   f(complete_liveness,                              "  Complete Liveness")              \
   f(retire_tlabs,                                   "  Retire TLABs")                   \
+  f(sync_pinned,                                    "  Sync Pinned")                    \
   f(trash_cset,                                     "  Trash CSet")                     \
   f(prepare_evac,                                   "  Prepare Evacuation")             \
                                                                                         \
@@ -161,6 +162,7 @@
   f(final_update_refs_string_dedup_queue_roots,      "    UR: Dedup Queue Roots")       \
   f(final_update_refs_finish_queues,                 "    UR: Finish Queues")           \
                                                                                         \
+  f(final_update_refs_sync_pinned,                   "  Sync Pinned")                   \
   f(final_update_refs_trash_cset,                    "  Trash CSet")                    \
                                                                                         \
   f(degen_gc_gross,                                  "Pause Degenerated GC (G)")        \
@@ -193,6 +195,7 @@
   f(traversal_gc_prepare,                            "  Prepare")                       \
   f(traversal_gc_make_parsable,                      "    Make Parsable")               \
   f(traversal_gc_resize_tlabs,                       "    Resize TLABs")                \
+  f(traversal_gc_prepare_sync_pinned,                "    Sync Pinned")                 \
                                                                                         \
   /* Per-thread timer block, should have "roots" counters in consistent order */        \
   f(init_traversal_gc_work,                          "  Work")                          \
@@ -264,6 +267,7 @@
   f(final_traversal_update_string_dedup_queue_roots,    "    TU: Dedup Queue Roots")    \
   f(final_traversal_update_finish_queues,               "    TU: Finish Queues")        \
                                                                                         \
+  f(traversal_gc_sync_pinned,                        "  Sync Pinned")                   \
   f(traversal_gc_cleanup,                            "  Cleanup")                       \
                                                                                         \
   f(full_gc_gross,                                   "Pause Full GC (G)")               \
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Mon Oct 21 15:11:43 2019 +0200
@@ -340,9 +340,6 @@
 }
 
 void ShenandoahTraversalGC::prepare() {
-  _heap->collection_set()->clear();
-  assert(_heap->collection_set()->count() == 0, "collection set not clear");
-
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
     _heap->make_parsable(true);
@@ -356,15 +353,26 @@
   assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
   assert(!_heap->marking_context()->is_complete(), "should not be complete");
 
-  ShenandoahFreeSet* free_set = _heap->free_set();
+  // About to choose the collection set, make sure we know which regions are pinned.
+  {
+    ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned);
+    _heap->sync_pinned_region_status();
+  }
+
   ShenandoahCollectionSet* collection_set = _heap->collection_set();
+  {
+    ShenandoahHeapLocker lock(_heap->lock());
 
-  // Find collection set
-  _heap->heuristics()->choose_collection_set(collection_set);
-  prepare_regions();
+    collection_set->clear();
+    assert(collection_set->count() == 0, "collection set not clear");
 
-  // Rebuild free set
-  free_set->rebuild();
+    // Find collection set
+    _heap->heuristics()->choose_collection_set(collection_set);
+    prepare_regions();
+
+    // Rebuild free set
+    _heap->free_set()->rebuild();
+  }
 
   log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions",
                      byte_size_in_proper_unit(collection_set->garbage()),   proper_unit_for_byte_size(collection_set->garbage()),
@@ -385,7 +393,6 @@
 
   {
     ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
-    ShenandoahHeapLocker lock(_heap->lock());
     prepare();
   }
 
@@ -613,6 +620,13 @@
     // Resize metaspace
     MetaspaceGC::compute_new_size();
 
+    // Need to see that pinned region status is updated: newly pinned regions must not
+    // be trashed. New unpinned regions should be trashed.
+    {
+      ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned);
+      _heap->sync_pinned_region_status();
+    }
+
     // Still good? We can now trash the cset, and make final verification
     {
       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);