changeset 58372:cc0ffb1d0458

8240873: Shenandoah: Short-cut arraycopy barriers Reviewed-by: shade
author rkennke
date Thu, 12 Mar 2020 17:52:47 +0100
parents c0f672668596
children 67a2ce1f3a0a be3aeb6e766d
files src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
diffstat 8 files changed, 22 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -124,6 +124,7 @@
   template <class T>
   oop load_reference_barrier_native_impl(oop obj, T* load_addr);
 
+  inline bool skip_bulk_update(HeapWord* dst);
 public:
   // Callbacks for runtime accesses.
   template <DecoratorSet decorators, typename BarrierSetT = ShenandoahBarrierSet>
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -293,7 +293,8 @@
 
 template <class T>
 void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) {
-  if (_heap->is_concurrent_mark_in_progress()) {
+  if (_heap->is_concurrent_mark_in_progress() &&
+      !_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(dst))) {
     if (_heap->has_forwarded_objects()) {
       arraycopy_work<T, true, false, true>(dst, count);
     } else {
@@ -301,7 +302,9 @@
     }
   }
 
-  arraycopy_update_impl(src, count);
+  if (_heap->has_forwarded_objects()) {
+    arraycopy_update_impl(src, count);
+  }
 }
 
 void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) {
@@ -312,8 +315,13 @@
   arraycopy_pre_work(src, dst, count);
 }
 
+inline bool ShenandoahBarrierSet::skip_bulk_update(HeapWord* dst) {
+  return dst >= _heap->heap_region_containing(dst)->get_update_watermark();
+}
+
 template <class T>
 void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) {
+  if (skip_bulk_update(reinterpret_cast<HeapWord*>(src))) return;
   if (_heap->is_evacuation_in_progress()) {
     ShenandoahEvacOOMScope oom_evac;
     arraycopy_work<T, true, true, false>(src, count);
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -82,6 +82,7 @@
   // that potentially need to be updated.
 
   shenandoah_assert_correct(NULL, obj);
+  if (skip_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
   if (_heap->is_evacuation_in_progress()) {
     ShenandoahEvacOOMScope evac_scope;
     ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Thu Mar 12 17:52:47 2020 +0100
@@ -2403,6 +2403,7 @@
       if (r->is_active() && !r->is_cset()) {
         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
       }
+      r->set_update_watermark(r->bottom());
       if (ShenandoahPacing) {
         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
       }
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -261,7 +261,7 @@
   volatile size_t _live_data;
   volatile size_t _critical_pins;
 
-  HeapWord* _update_watermark;
+  HeapWord* volatile _update_watermark;
 
   // Claim some space at the end to protect next region
   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
@@ -432,12 +432,12 @@
 
   HeapWord* get_update_watermark() const {
     assert(bottom() <= _update_watermark && _update_watermark <= top(), "within bounds");
-    return _update_watermark;
+    return Atomic::load_acquire(&_update_watermark);
   }
 
   void set_update_watermark(HeapWord* w) {
     assert(bottom() <= w && w <= top(), "within bounds");
-    _update_watermark = w;
+    Atomic::release_store(&_update_watermark, w);
   }
 
 private:
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -56,6 +56,7 @@
   inline bool is_marked(oop obj) const;
 
   inline bool allocated_after_mark_start(oop obj) const;
+  inline bool allocated_after_mark_start(HeapWord* addr) const;
 
   inline MarkBitMap* mark_bit_map();
 
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp	Thu Mar 12 17:52:47 2020 +0100
@@ -42,6 +42,10 @@
 
 inline bool ShenandoahMarkingContext::allocated_after_mark_start(oop obj) const {
   HeapWord* addr = cast_from_oop<HeapWord*>(obj);
+  return allocated_after_mark_start(addr);
+}
+
+inline bool ShenandoahMarkingContext::allocated_after_mark_start(HeapWord* addr) const {
   uintx index = ((uintx) addr) >> ShenandoahHeapRegion::region_size_bytes_shift();
   HeapWord* top_at_mark_start = _top_at_mark_starts[index];
   bool alloc_after_mark_start = addr >= top_at_mark_start;
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Thu Mar 12 17:52:10 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Thu Mar 12 17:52:47 2020 +0100
@@ -314,6 +314,7 @@
   ShenandoahMarkingContext* const ctx = _heap->marking_context();
   for (size_t i = 0; i < num_regions; i++) {
     ShenandoahHeapRegion* region = _heap->get_region(i);
+    region->set_update_watermark(region->top());
     if (_heap->is_bitmap_slice_committed(region)) {
       if (_traversal_set.is_in(i)) {
         ctx->capture_top_at_mark_start(region);