OpenJDK / jdk / jdk
changeset 58372:cc0ffb1d0458
8240873: Shenandoah: Short-cut arraycopy barriers
Reviewed-by: shade
author | rkennke |
---|---|
date | Thu, 12 Mar 2020 17:52:47 +0100 |
parents | c0f672668596 |
children | 67a2ce1f3a0a be3aeb6e766d |
files | src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp |
diffstat | 8 files changed, 22 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -124,6 +124,7 @@ template <class T> oop load_reference_barrier_native_impl(oop obj, T* load_addr); + inline bool skip_bulk_update(HeapWord* dst); public: // Callbacks for runtime accesses. template <DecoratorSet decorators, typename BarrierSetT = ShenandoahBarrierSet>
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -293,7 +293,8 @@ template <class T> void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) { - if (_heap->is_concurrent_mark_in_progress()) { + if (_heap->is_concurrent_mark_in_progress() && + !_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(dst))) { if (_heap->has_forwarded_objects()) { arraycopy_work<T, true, false, true>(dst, count); } else { @@ -301,7 +302,9 @@ } } - arraycopy_update_impl(src, count); + if (_heap->has_forwarded_objects()) { + arraycopy_update_impl(src, count); + } } void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) { @@ -312,8 +315,13 @@ arraycopy_pre_work(src, dst, count); } +inline bool ShenandoahBarrierSet::skip_bulk_update(HeapWord* dst) { + return dst >= _heap->heap_region_containing(dst)->get_update_watermark(); +} + template <class T> void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) { + if (skip_bulk_update(reinterpret_cast<HeapWord*>(src))) return; if (_heap->is_evacuation_in_progress()) { ShenandoahEvacOOMScope oom_evac; arraycopy_work<T, true, true, false>(src, count);
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -82,6 +82,7 @@ // that potentially need to be updated. shenandoah_assert_correct(NULL, obj); + if (skip_bulk_update(cast_from_oop<HeapWord*>(obj))) return; if (_heap->is_evacuation_in_progress()) { ShenandoahEvacOOMScope evac_scope; ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Mar 12 17:52:47 2020 +0100 @@ -2403,6 +2403,7 @@ if (r->is_active() && !r->is_cset()) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } + r->set_update_watermark(r->bottom()); if (ShenandoahPacing) { _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom())); }
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -261,7 +261,7 @@ volatile size_t _live_data; volatile size_t _critical_pins; - HeapWord* _update_watermark; + HeapWord* volatile _update_watermark; // Claim some space at the end to protect next region DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0); @@ -432,12 +432,12 @@ HeapWord* get_update_watermark() const { assert(bottom() <= _update_watermark && _update_watermark <= top(), "within bounds"); - return _update_watermark; + return Atomic::load_acquire(&_update_watermark); } void set_update_watermark(HeapWord* w) { assert(bottom() <= w && w <= top(), "within bounds"); - _update_watermark = w; + Atomic::release_store(&_update_watermark, w); } private:
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -56,6 +56,7 @@ inline bool is_marked(oop obj) const; inline bool allocated_after_mark_start(oop obj) const; + inline bool allocated_after_mark_start(HeapWord* addr) const; inline MarkBitMap* mark_bit_map();
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp Thu Mar 12 17:52:47 2020 +0100 @@ -42,6 +42,10 @@ inline bool ShenandoahMarkingContext::allocated_after_mark_start(oop obj) const { HeapWord* addr = cast_from_oop<HeapWord*>(obj); + return allocated_after_mark_start(addr); +} + +inline bool ShenandoahMarkingContext::allocated_after_mark_start(HeapWord* addr) const { uintx index = ((uintx) addr) >> ShenandoahHeapRegion::region_size_bytes_shift(); HeapWord* top_at_mark_start = _top_at_mark_starts[index]; bool alloc_after_mark_start = addr >= top_at_mark_start;
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Thu Mar 12 17:52:10 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Thu Mar 12 17:52:47 2020 +0100 @@ -314,6 +314,7 @@ ShenandoahMarkingContext* const ctx = _heap->marking_context(); for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = _heap->get_region(i); + region->set_update_watermark(region->top()); if (_heap->is_bitmap_slice_committed(region)) { if (_traversal_set.is_in(i)) { ctx->capture_top_at_mark_start(region);