changeset 53397:959d0caa9b7d

8250784: Shenandoah: A Low-Pause-Time Garbage Collector Reviewed-by: aph, kdnilsen, adityam
author rkennke
date Wed, 29 Jul 2020 17:15:58 +0200
parents 958423127de9
children c93a0088d193
files make/autoconf/hotspot.m4 make/hotspot/gensrc/GensrcAdlc.gmk make/hotspot/gensrc/GensrcJfr.gmk make/hotspot/lib/JvmFeatures.gmk make/hotspot/lib/JvmOverrideFiles.gmk make/src/classes/build/tools/jfr/GenerateJfrFiles.java src/hotspot/cpu/aarch64/aarch64.ad src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_32.ad src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp src/hotspot/share/adlc/formssel.cpp src/hotspot/share/ci/ciInstanceKlass.cpp src/hotspot/share/ci/ciInstanceKlass.hpp src/hotspot/share/code/codeCache.hpp src/hotspot/share/gc/shared/barrierSetConfig.hpp src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp src/hotspot/share/gc/shared/collectedHeap.hpp src/hotspot/share/gc/shared/gcCause.cpp src/hotspot/share/gc/shared/gcCause.hpp src/hotspot/share/gc/shared/gcConfig.cpp src/hotspot/share/gc/shared/gcConfiguration.cpp src/hotspot/share/gc/shared/gcName.hpp src/hotspot/share/gc/shared/gc_globals.hpp src/hotspot/share/gc/shared/referenceProcessor.cpp src/hotspot/share/gc/shared/taskqueue.hpp src/hotspot/share/gc/shared/vmStructs_gc.hpp src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp src/hotspot/share/gc/shenandoah/markBitMap.cpp src/hotspot/share/gc/shenandoah/markBitMap.hpp src/hotspot/share/gc/shenandoah/markBitMap.inline.hpp src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.hpp src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp src/hotspot/share/gc/shenandoah/parallelCleaning.cpp src/hotspot/share/gc/shenandoah/parallelCleaning.hpp src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahJfrSupport.cpp src/hotspot/share/gc/shenandoah/shenandoahJfrSupport.hpp src/hotspot/share/gc/shenandoah/shenandoahLock.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp src/hotspot/share/gc/shenandoah/shenandoahMetrics.cpp src/hotspot/share/gc/shenandoah/shenandoahMetrics.hpp src/hotspot/share/gc/shenandoah/shenandoahMonitoringSupport.cpp src/hotspot/share/gc/shenandoah/shenandoahMonitoringSupport.hpp src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.cpp src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.hpp src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp src/hotspot/share/gc/shenandoah/shenandoahSATBMarkQueue.cpp src/hotspot/share/gc/shenandoah/shenandoahSATBMarkQueue.hpp src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.cpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.hpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp src/hotspot/share/gc/shenandoah/shenandoahStringDedup.hpp src/hotspot/share/gc/shenandoah/shenandoahStringDedup.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.cpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp src/hotspot/share/gc/shenandoah/shenandoahTracer.hpp src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp src/hotspot/share/gc/shenandoah/shenandoahWorkGroup.cpp src/hotspot/share/gc/shenandoah/shenandoahWorkGroup.hpp src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp src/hotspot/share/jfr/metadata/metadata-shenandoah.xml src/hotspot/share/jfr/periodic/jfrPeriodic.cpp src/hotspot/share/memory/metaspace.hpp src/hotspot/share/opto/arraycopynode.cpp src/hotspot/share/opto/cfgnode.hpp src/hotspot/share/opto/classes.cpp src/hotspot/share/opto/classes.hpp src/hotspot/share/opto/compile.cpp src/hotspot/share/opto/compile.hpp src/hotspot/share/opto/escape.cpp src/hotspot/share/opto/graphKit.cpp src/hotspot/share/opto/loopTransform.cpp src/hotspot/share/opto/loopnode.cpp src/hotspot/share/opto/loopnode.hpp src/hotspot/share/opto/macro.cpp src/hotspot/share/opto/macroArrayCopy.cpp src/hotspot/share/opto/matcher.cpp src/hotspot/share/opto/memnode.cpp src/hotspot/share/opto/mulnode.cpp src/hotspot/share/opto/node.hpp src/hotspot/share/opto/phaseX.cpp src/hotspot/share/opto/phasetype.hpp src/hotspot/share/opto/subnode.cpp src/hotspot/share/opto/type.hpp src/hotspot/share/runtime/mutexLocker.cpp src/hotspot/share/runtime/sharedRuntime.cpp src/hotspot/share/runtime/sharedRuntime.hpp src/hotspot/share/runtime/stackValue.cpp src/hotspot/share/runtime/vmOperations.hpp src/hotspot/share/utilities/globalDefinitions.hpp src/hotspot/share/utilities/macros.hpp src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeapName.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/GCCause.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/GCName.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeapRegion.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/VMOps.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java src/jdk.jfr/share/conf/jfr/default.jfc src/jdk.jfr/share/conf/jfr/profile.jfc test/hotspot/jtreg/TEST.ROOT test/hotspot/jtreg/TEST.groups test/hotspot/jtreg/compiler/c2/aarch64/TestVolatiles.java test/hotspot/jtreg/compiler/c2/aarch64/TestVolatilesShenandoah.java test/hotspot/jtreg/gc/TestFullGCCount.java test/hotspot/jtreg/gc/TestHumongousReferenceObject.java test/hotspot/jtreg/gc/TestSystemGC.java test/hotspot/jtreg/gc/arguments/TestAlignmentToUseLargePages.java test/hotspot/jtreg/gc/arguments/TestMaxMinHeapFreeRatioFlags.java test/hotspot/jtreg/gc/arguments/TestNewRatioFlag.java test/hotspot/jtreg/gc/arguments/TestNewSizeFlags.java test/hotspot/jtreg/gc/arguments/TestShrinkHeapInSteps.java test/hotspot/jtreg/gc/arguments/TestSurvivorRatioFlag.java test/hotspot/jtreg/gc/arguments/TestTargetSurvivorRatioFlag.java test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsErgo.java test/hotspot/jtreg/gc/arguments/TestVerifyBeforeAndAfterGCFlags.java test/hotspot/jtreg/gc/class_unloading/TestClassUnloadingDisabled.java test/hotspot/jtreg/gc/ergonomics/TestDynamicNumberOfGCThreads.java test/hotspot/jtreg/gc/ergonomics/TestInitialGCThreadLogging.java test/hotspot/jtreg/gc/logging/TestGCId.java test/hotspot/jtreg/gc/metaspace/TestMetaspacePerfCounters.java test/hotspot/jtreg/gc/shenandoah/TestAllocHumongousFragment.java test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java test/hotspot/jtreg/gc/shenandoah/compiler/BarrierInInfiniteLoop.java test/hotspot/jtreg/gc/shenandoah/compiler/CallMultipleCatchProjs.java test/hotspot/jtreg/gc/shenandoah/compiler/FoldIfAfterExpansion.java test/hotspot/jtreg/gc/shenandoah/compiler/LRBRightAfterMemBar.java test/hotspot/jtreg/gc/shenandoah/compiler/TestC1ArrayCopyNPE.java test/hotspot/jtreg/gc/shenandoah/compiler/TestC1VectorizedMismatch.java test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java test/hotspot/jtreg/gc/shenandoah/compiler/TestExpandedWBLostNullCheckDep.java test/hotspot/jtreg/gc/shenandoah/compiler/TestMaybeNullUnsafeAccess.java test/hotspot/jtreg/gc/shenandoah/compiler/TestNullCheck.java test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java test/hotspot/jtreg/gc/shenandoah/compiler/TestShenandoahCmpPAfterCall.java test/hotspot/jtreg/gc/shenandoah/compiler/TestUnsafeOffheapSwap.java test/hotspot/jtreg/gc/shenandoah/compiler/TestWriteBarrierClearControl.java test/hotspot/jtreg/gc/shenandoah/jni/CriticalNativeArgs.java test/hotspot/jtreg/gc/shenandoah/jni/CriticalNativeStress.java test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java test/hotspot/jtreg/gc/shenandoah/jni/libCriticalNative.c test/hotspot/jtreg/gc/shenandoah/jni/libTestJNICritical.c test/hotspot/jtreg/gc/shenandoah/jni/libTestJNIGlobalRefs.c test/hotspot/jtreg/gc/shenandoah/jni/libTestPinnedGarbage.c test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java test/hotspot/jtreg/gc/shenandoah/jvmti/libTestHeapDump.c test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java test/hotspot/jtreg/gc/shenandoah/options/TestAlwaysPreTouch.java test/hotspot/jtreg/gc/shenandoah/options/TestArgumentRanges.java test/hotspot/jtreg/gc/shenandoah/options/TestClassUnloadingArguments.java test/hotspot/jtreg/gc/shenandoah/options/TestCodeCacheRootStyles.java test/hotspot/jtreg/gc/shenandoah/options/TestCriticalControlThreadPriority.java test/hotspot/jtreg/gc/shenandoah/options/TestEnabled.java test/hotspot/jtreg/gc/shenandoah/options/TestExplicitGC.java test/hotspot/jtreg/gc/shenandoah/options/TestExplicitGCNoConcurrent.java test/hotspot/jtreg/gc/shenandoah/options/TestHeuristicsUnlock.java test/hotspot/jtreg/gc/shenandoah/options/TestHumongousMoves.java test/hotspot/jtreg/gc/shenandoah/options/TestHumongousThresholdArgs.java test/hotspot/jtreg/gc/shenandoah/options/TestLoopMiningArguments.java test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java test/hotspot/jtreg/gc/shenandoah/options/TestObjectAlignment.java test/hotspot/jtreg/gc/shenandoah/options/TestPacing.java test/hotspot/jtreg/gc/shenandoah/options/TestParallelRegionStride.java test/hotspot/jtreg/gc/shenandoah/options/TestRegionSizeArgs.java test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java test/hotspot/jtreg/gc/shenandoah/options/TestSingleThreaded.java test/hotspot/jtreg/gc/shenandoah/options/TestThreadCounts.java test/hotspot/jtreg/gc/shenandoah/options/TestThreadCountsOverride.java test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierEnable.java test/hotspot/jtreg/gc/startup_warnings/TestShenandoah.java test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java test/hotspot/jtreg/gc/survivorAlignment/TestAllocationInEden.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromEdenToTenured.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromSurvivorToTenuredAfterFullGC.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromSurvivorToTenuredAfterMinorGC.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionToSurvivor.java test/hotspot/jtreg/gc/whitebox/TestWBGC.java test/hotspot/jtreg/resourcehogs/serviceability/sa/TestHeapDumpForLargeArray.java test/hotspot/jtreg/runtime/CompressedOops/UseCompressedOops.java test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java test/hotspot/jtreg/serviceability/sa/ClhsdbJhisto.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCapacityTest.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest01.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest02.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest03.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcNewTest.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcTest01.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcTest02.java test/hotspot/jtreg/vmTestbase/nsk/jvmti/scenarios/general_functions/GF08/gf08t001/TestDriver.java test/jdk/TEST.ROOT test/jdk/jdk/jfr/event/gc/detailed/TestShenandoahHeapRegionInformationEvent.java test/jdk/jdk/jfr/event/gc/detailed/TestShenandoahHeapRegionStateChangeEvent.java test/lib/jdk/test/lib/jfr/EventNames.java test/lib/jdk/test/lib/jfr/GCHelper.java test/lib/sun/hotspot/gc/GC.java
diffstat 347 files changed, 43006 insertions(+), 105 deletions(-) [+]
line wrap: on
line diff
--- a/make/autoconf/hotspot.m4	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/autoconf/hotspot.m4	Wed Jul 29 17:15:58 2020 +0200
@@ -25,7 +25,7 @@
 
 # All valid JVM features, regardless of platform
 VALID_JVM_FEATURES="compiler1 compiler2 zero minimal dtrace jvmti jvmci \
-    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc zgc nmt cds \
+    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc shenandoahgc zgc nmt cds \
     static-build link-time-opt aot jfr"
 
 # Deprecated JVM features (these are ignored, but with a warning)
@@ -352,6 +352,19 @@
     fi
   fi
 
+  # Only enable Shenandoah on supported arches, and only if requested
+  AC_MSG_CHECKING([if shenandoah can be built])
+  if HOTSPOT_CHECK_JVM_FEATURE(shenandoahgc); then
+    if test "x$OPENJDK_TARGET_CPU_ARCH" = "xx86" || test "x$OPENJDK_TARGET_CPU" = "xaarch64" ; then
+      AC_MSG_RESULT([yes])
+    else
+      DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES shenandoahgc"
+      AC_MSG_RESULT([no, platform not supported])
+    fi
+  else
+      DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES shenandoahgc"
+  fi
+
   # Only enable ZGC on supported platforms
   AC_MSG_CHECKING([if zgc can be built])
   if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
@@ -363,7 +376,7 @@
 
   # Disable unsupported GCs for Zero
   if HOTSPOT_CHECK_JVM_VARIANT(zero); then
-    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc zgc"
+    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc shenandoahgc zgc"
   fi
 
   # Turn on additional features based on other parts of configure
@@ -494,7 +507,7 @@
   fi
 
   # All variants but minimal (and custom) get these features
-  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc jni-check jvmti management nmt services vm-structs zgc"
+  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc shenandoahgc jni-check jvmti management nmt services vm-structs zgc"
 
   AC_MSG_CHECKING([if cds should be enabled])
   if test "x$ENABLE_CDS" = "xtrue"; then
--- a/make/hotspot/gensrc/GensrcAdlc.gmk	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk	Wed Jul 29 17:15:58 2020 +0200
@@ -62,6 +62,10 @@
 
   ADLC_CFLAGS += -I$(TOPDIR)/src/hotspot/share
 
+  ifneq ($(call check-jvm-feature, shenandoahgc), true)
+    ADLC_CFLAGS += -DINCLUDE_SHENANDOAHGC=0
+  endif
+
   $(eval $(call SetupNativeCompilation, BUILD_ADLC, \
       NAME := adlc, \
       TYPE := EXECUTABLE, \
@@ -136,6 +140,12 @@
       $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \
     )))
 
+  ifeq ($(call check-jvm-feature, shenandoahgc), true)
+    AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \
+      )))
+  endif
+
   SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad
 
   INSERT_FILENAME_AWK_SCRIPT := \
--- a/make/hotspot/gensrc/GensrcJfr.gmk	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/hotspot/gensrc/GensrcJfr.gmk	Wed Jul 29 17:15:58 2020 +0200
@@ -57,11 +57,15 @@
 METADATA_XML := $(JFR_SRCDIR)/metadata.xml
 METADATA_XSD := $(JFR_SRCDIR)/metadata.xsd
 
-$(JFR_OUTPUTDIR)/jfrEventClasses.hpp: $(METADATA_XML) $(METADATA_XSD) \
-    $(BUILD_JFR_TOOLS)
+ifeq ($(call check-jvm-feature, shenandoahgc), true)
+  METADATA_SHENANDOAH_XML := $(JFR_SRCDIR)/metadata-shenandoah.xml
+endif
+
+$(JFR_OUTPUTDIR)/jfrEventClasses.hpp: $(METADATA_XSD) \
+    $(BUILD_JFR_TOOLS) $(METADATA_XML) $(METADATA_SHENANDOAH_XML)
 	$(call LogInfo, Generating $(@F))
 	$(call MakeDir, $(@D))
-	$(call ExecuteWithLog, $@, $(TOOL_JFR_GEN) $(METADATA_XML) $(METADATA_XSD) $(JFR_OUTPUTDIR))
+	$(call ExecuteWithLog, $@, $(TOOL_JFR_GEN) $(METADATA_XSD) $(JFR_OUTPUTDIR) $(METADATA_XML) $(METADATA_SHENANDOAH_XML) )
 	test -f $@
 
 TARGETS += $(JFR_OUTPUTDIR)/jfrEventClasses.hpp
--- a/make/hotspot/lib/JvmFeatures.gmk	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/hotspot/lib/JvmFeatures.gmk	Wed Jul 29 17:15:58 2020 +0200
@@ -166,6 +166,11 @@
   JVM_EXCLUDE_PATTERNS += gc/z
 endif
 
+ifneq ($(call check-jvm-feature, shenandoahgc), true)
+  JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0
+  JVM_EXCLUDE_PATTERNS += gc/shenandoah
+endif
+
 ifneq ($(call check-jvm-feature, jfr), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_JFR=0
   JVM_EXCLUDE_PATTERNS += jfr
--- a/make/hotspot/lib/JvmOverrideFiles.gmk	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/hotspot/lib/JvmOverrideFiles.gmk	Wed Jul 29 17:15:58 2020 +0200
@@ -36,6 +36,10 @@
   BUILD_LIBJVM_assembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_cardTableBarrierSetAssembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_interp_masm_x86.cpp_CXXFLAGS := -Wno-uninitialized
+  ifeq ($(DEBUG_LEVEL), release)
+    # Need extra inlining to collapse all marking code into the hot marking loop
+    BUILD_LIBJVM_shenandoahConcurrentMark.cpp_CXXFLAGS := --param inline-unit-growth=1000
+  endif
 endif
 
 LIBJVM_FDLIBM_COPY_OPT_FLAG := $(CXX_O_FLAG_NONE)
--- a/make/src/classes/build/tools/jfr/GenerateJfrFiles.java	Tue Jan 14 19:27:03 2020 -0800
+++ b/make/src/classes/build/tools/jfr/GenerateJfrFiles.java	Wed Jul 29 17:15:58 2020 +0200
@@ -28,18 +28,20 @@
 public class GenerateJfrFiles {
 
     public static void main(String... args) throws Exception {
-        if (args.length != 3) {
+        if (args.length < 3) {
             System.err.println("Incorrect number of command line arguments.");
             System.err.println("Usage:");
-            System.err.println("java GenerateJfrFiles[.java] <path-to-metadata.xml> <path-to-metadata.xsd> <output-directory>");
+            System.err.println("java GenerateJfrFiles[.java] <path-to-metadata.xsd> <output-directory> <path-to-metadata.xml> [<path-to-metadata.xml> ...]");
             System.exit(1);
         }
         try {
-            File metadataXml = new File(args[0]);
-            File metadataSchema = new File(args[1]);
-            File outputDirectory = new File(args[2]);
-
-            Metadata metadata = new Metadata(metadataXml, metadataSchema);
+            File metadataSchema = new File(args[0]);
+            File outputDirectory = new File(args[1]);
+            File[] metadataXml = new File[args.length - 2];
+            for (int i = 2; i < args.length; i++) {
+              metadataXml[i - 2] = new File(args[i]);
+            }
+            Metadata metadata = new Metadata(metadataSchema, metadataXml);
             metadata.verify();
             metadata.wireUpTypes();
 
@@ -75,12 +77,14 @@
     static class Metadata {
         final Map<String, TypeElement> types = new LinkedHashMap<>();
         final Map<String, XmlType> xmlTypes = new HashMap<>();
-        Metadata(File metadataXml, File metadataSchema) throws ParserConfigurationException, SAXException, FileNotFoundException, IOException {
+        Metadata(File metadataSchema, File[] metadataXml) throws ParserConfigurationException, SAXException, FileNotFoundException, IOException {
             SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
             SAXParserFactory factory = SAXParserFactory.newInstance();
             factory.setSchema(schemaFactory.newSchema(metadataSchema));
             SAXParser sp = factory.newSAXParser();
-            sp.parse(metadataXml, new MetadataHandler(this));
+            for (File file : metadataXml) {
+              sp.parse(file, new MetadataHandler(this));
+            }
         }
 
         List<EventElement> getEvents() {
--- a/src/hotspot/cpu/aarch64/aarch64.ad	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/aarch64/aarch64.ad	Wed Jul 29 17:15:58 2020 +0200
@@ -1280,6 +1280,10 @@
     case Op_GetAndSetN:
     case Op_GetAndAddI:
     case Op_GetAndAddL:
+#if INCLUDE_SHENANDOAHGC
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
+#endif
       return true;
     case Op_CompareAndExchangeI:
     case Op_CompareAndExchangeN:
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -42,6 +42,7 @@
 #include "oops/objArrayKlass.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
 #include "vmreg_aarch64.inline.hpp"
 
 
@@ -2818,6 +2819,12 @@
 
 
 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC && patch_code != lir_patch_none) {
+    deoptimize_trap(info);
+    return;
+  }
+#endif
   assert(patch_code == lir_patch_none, "Patch code not supported");
   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+#define __ masm->masm()->
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  Register addr = _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  Register result = result_opr()->as_register();
+
+  ShenandoahBarrierSet::assembler()->storeval_barrier(masm->masm(), newval, rscratch2);
+
+  if (UseCompressedOops) {
+    __ encode_heap_oop(tmp1, cmpval);
+    cmpval = tmp1;
+    __ encode_heap_oop(tmp2, newval);
+    newval = tmp2;
+  }
+
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, result);
+}
+
+#undef __
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+  BasicType bt = access.type();
+  if (access.is_oop()) {
+    LIRGenerator *gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item();
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+      LIR_Opr result = gen->new_register(T_INT);
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  LIR_Opr tmp = gen->new_register(T_INT);
+  __ xchg(access.resolved_addr(), value_opr, result, tmp);
+
+  if (access.is_oop()) {
+    result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0));
+    LIR_Opr tmp = gen->new_register(type);
+    __ move(result, tmp);
+    result = tmp;
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,689 @@
+/*
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahForwarding.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register src, Register dst, Register count, RegSet saved_regs) {
+  if (is_oop) {
+    bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
+
+      Label done;
+
+      // Avoid calling runtime if count == 0
+      __ cbz(count, done);
+
+      // Is GC active?
+      Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+      __ ldrb(rscratch1, gc_state);
+      if (ShenandoahSATBBarrier && dest_uninitialized) {
+        __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
+      } else {
+        __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
+        __ tst(rscratch1, rscratch2);
+        __ br(Assembler::EQ, done);
+      }
+
+      __ push(saved_regs, sp);
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
+      } else {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
+      }
+      __ pop(saved_regs, sp);
+      __ bind(done);
+    }
+  }
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+  assert(thread == rthread, "must be");
+
+  Label done;
+  Label runtime;
+
+  assert_different_registers(obj, pre_val, tmp, rscratch1);
+  assert(pre_val != noreg &&  tmp != noreg, "expecting a register");
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  // Is marking active?
+  if (in_bytes(ShenandoahSATBMarkQueue::byte_width_of_active()) == 4) {
+    __ ldrw(tmp, in_progress);
+  } else {
+    assert(in_bytes(ShenandoahSATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+    __ ldrb(tmp, in_progress);
+  }
+  __ cbzw(tmp, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cbz(pre_val, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ ldr(tmp, index);                      // tmp := *index_adr
+  __ cbz(tmp, runtime);                    // tmp == 0?
+                                        // If yes, goto runtime
+
+  __ sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
+  __ str(tmp, index);                      // *index_adr := tmp
+  __ ldr(rscratch1, buffer);
+  __ add(tmp, tmp, rscratch1);             // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ str(pre_val, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(pre_val);
+  if (tosca_live) saved += RegSet::of(r0);
+  if (obj != noreg) saved += RegSet::of(obj);
+
+  __ push(saved, sp);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  if (expand_call) {
+    assert(pre_val != c_rarg1, "smashed arg");
+    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  }
+
+  __ pop(saved, sp);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
+  assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
+  Label is_null;
+  __ cbz(dst, is_null);
+  resolve_forward_pointer_not_null(masm, dst, tmp);
+  __ bind(is_null);
+}
+
+// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitely
+// passed in.
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
+  assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
+  // The below loads the mark word, checks if the lowest two bits are
+  // set, and if so, clear the lowest two bits and copy the result
+  // to dst. Otherwise it leaves dst alone.
+  // Implementing this is surprisingly awkward. I do it here by:
+  // - Inverting the mark word
+  // - Test lowest two bits == 0
+  // - If so, set the lowest two bits
+  // - Invert the result back, and copy to dst
+
+  bool borrow_reg = (tmp == noreg);
+  if (borrow_reg) {
+    // No free registers available. Make one useful.
+    tmp = rscratch1;
+    if (tmp == dst) {
+      tmp = rscratch2;
+    }
+    __ push(RegSet::of(tmp), sp);
+  }
+
+  assert_different_registers(tmp, dst);
+
+  Label done;
+  __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp, tmp, zr);
+  __ ands(zr, tmp, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, done);
+  __ orr(tmp, tmp, markOopDesc::marked_value);
+  __ eon(dst, tmp, zr);
+  __ bind(done);
+
+  if (borrow_reg) {
+    __ pop(RegSet::of(tmp), sp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address load_addr) {
+  assert(ShenandoahLoadRefBarrier, "Should be enabled");
+  assert(dst != rscratch2, "need rscratch2");
+  assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
+
+  Label done;
+  __ enter();
+  Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(rscratch2, gc_state);
+
+  // Check for heap stability
+  __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
+
+  // use r1 for load address
+  Register result_dst = dst;
+  if (dst == r1) {
+    __ mov(rscratch1, dst);
+    dst = rscratch1;
+  }
+
+  // Save r0 and r1, unless it is an output register
+  RegSet to_save = RegSet::of(r0, r1) - result_dst;
+  __ push(to_save, sp);
+  __ lea(r1, load_addr);
+  __ mov(r0, dst);
+
+  __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
+
+  __ mov(result_dst, r0);
+  __ pop(to_save, sp);
+
+  __ bind(done);
+  __ leave();
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    // Save possibly live regs.
+    RegSet live_regs = RegSet::range(r0, r4) - dst;
+    __ push(live_regs, sp);
+    __ strd(v0, __ pre(sp, 2 * -wordSize));
+
+    satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
+
+    // Restore possibly live regs.
+    __ ldrd(v0, __ post(sp, 2 * wordSize));
+    __ pop(live_regs, sp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) {
+  if (ShenandoahLoadRefBarrier) {
+    Label is_null;
+    __ cbz(dst, is_null);
+    load_reference_barrier_not_null(masm, dst, load_addr);
+    __ bind(is_null);
+  }
+}
+
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location to load from, might be clobbered
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   rscratch1 (scratch reg)
+//
+// Alias:
+//   dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
+//
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                            Register dst, Address src, Register tmp1, Register tmp_thread) {
+  // 1: non-reference load, no additional barrier is needed
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    return;
+  }
+
+  // 2: load a reference from src location and apply LRB if needed
+  if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
+    Register result_dst = dst;
+
+    // Preserve src location for LRB
+    if (dst == src.base() || dst == src.index()) {
+      dst = rscratch1;
+    }
+    assert_different_registers(dst, src.base(), src.index());
+
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+
+    load_reference_barrier(masm, dst, src);
+
+    if (dst != result_dst) {
+      __ mov(result_dst, dst);
+      dst = result_dst;
+    }
+  } else {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  }
+
+  // 3: apply keep-alive barrier if needed
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    __ enter();
+    __ push_call_clobbered_registers();
+    satb_write_barrier_pre(masm /* masm */,
+                           noreg /* obj */,
+                           dst /* pre_val */,
+                           rthread /* thread */,
+                           tmp1 /* tmp */,
+                           true /* tosca_live */,
+                           true /* expand_call */);
+    __ pop_call_clobbered_registers();
+    __ leave();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                             Address dst, Register val, Register tmp1, Register tmp2) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  if (!on_oop) {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+    return;
+  }
+
+  // flatten object address if needed
+  if (dst.index() == noreg && dst.offset() == 0) {
+    if (dst.base() != r3) {
+      __ mov(r3, dst.base());
+    }
+  } else {
+    __ lea(r3, dst);
+  }
+
+  shenandoah_write_barrier_pre(masm,
+                               r3 /* obj */,
+                               tmp2 /* pre_val */,
+                               rthread /* thread */,
+                               tmp1  /* tmp */,
+                               val != noreg /* tosca_live */,
+                               false /* expand_call */);
+
+  if (val == noreg) {
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
+  } else {
+    storeval_barrier(masm, val, tmp1);
+    // G1 barrier needs uncompressed oop for region cross check.
+    Register new_val = val;
+    if (UseCompressedOops) {
+      new_val = rscratch2;
+      __ mov(new_val, val);
+    }
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
+                                                                  Register obj, Register tmp, Label& slowpath) {
+  Label done;
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
+
+  // Check for null.
+  __ cbz(obj, done);
+
+  assert(obj != rscratch2, "need rscratch2");
+  Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
+  __ lea(rscratch2, gc_state);
+  __ ldrb(rscratch2, Address(rscratch2));
+
+  // Check for heap in evacuation phase
+  __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
+
+  __ bind(done);
+}
+
+
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                                                bool acquire, bool release, bool weak, bool is_cae,
+                                                Register result) {
+  Register tmp1 = rscratch1;
+  Register tmp2 = rscratch2;
+  bool is_narrow = UseCompressedOops;
+  Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
+
+  assert_different_registers(addr, expected, new_val, tmp1, tmp2);
+
+  Label retry, done, fail;
+
+  // CAS, using LL/SC pair.
+  __ bind(retry);
+  __ load_exclusive(tmp1, addr, size, acquire);
+  if (is_narrow) {
+    __ cmpw(tmp1, expected);
+  } else {
+    __ cmp(tmp1, expected);
+  }
+  __ br(Assembler::NE, fail);
+  __ store_exclusive(tmp2, new_val, addr, size, release);
+  if (weak) {
+    __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
+  } else {
+    __ cbnzw(tmp2, retry);
+  }
+  __ b(done);
+
+ __  bind(fail);
+  // Check if rb(expected)==rb(tmp1)
+  // Shuffle registers so that we have memory value ready for next expected.
+  __ mov(tmp2, expected);
+  __ mov(expected, tmp1);
+  if (is_narrow) {
+    __ decode_heap_oop(tmp1, tmp1);
+    __ decode_heap_oop(tmp2, tmp2);
+  }
+  resolve_forward_pointer(masm, tmp1);
+  resolve_forward_pointer(masm, tmp2);
+  __ cmp(tmp1, tmp2);
+  // Retry with expected now being the value we just loaded from addr.
+  __ br(Assembler::EQ, retry);
+  if (is_cae && is_narrow) {
+    // For cmp-and-exchange and narrow oops, we need to restore
+    // the compressed old-value. We moved it to 'expected' a few lines up.
+    __ mov(tmp1, expected);
+  }
+  __ bind(done);
+
+  if (is_cae) {
+    __ mov(result, tmp1);
+  } else {
+    __ cset(result, Assembler::EQ);
+  }
+}
+
+#undef __
+
+#ifdef COMPILER1
+
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+  __ cbz(pre_val_reg, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ b(*stub->continuation());
+}
+
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+  Register addr = stub->addr()->as_pointer_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
+
+  assert(res == r0, "result must arrive in r0");
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  __ cbz(res, *stub->continuation());
+
+  // Check for object in cset.
+  __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(tmp2, Address(tmp2, tmp1));
+  __ cbz(tmp2, *stub->continuation());
+
+  // Check if object is already forwarded.
+  Label slow_path;
+  __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp1, tmp1, zr);
+  __ ands(zr, tmp1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
+
+  // Decode forwarded object.
+  __ orr(tmp1, tmp1, markOopDesc::marked_value);
+  __ eon(res, tmp1, zr);
+  __ b(*stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+  ce->store_parameter(addr, 1);
+  __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
+  __ b(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+
+  // arg0 : previous value of memory
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+
+  const Register pre_val = r0;
+  const Register thread = rthread;
+  const Register tmp = rscratch1;
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is marking still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(tmp, gc_state);
+  __ mov(rscratch2, ShenandoahHeap::MARKING);
+  __ tst(tmp, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  // Can we store original value in the thread's buffer?
+  __ ldr(tmp, queue_index);
+  __ cbz(tmp, runtime);
+
+  __ sub(tmp, tmp, wordSize);
+  __ str(tmp, queue_index);
+  __ ldr(rscratch2, buffer);
+  __ add(tmp, tmp, rscratch2);
+  __ load_parameter(0, rscratch2);
+  __ str(rscratch2, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+  __ load_parameter(0, pre_val);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  __ pop_call_clobbered_registers();
+  __ bind(done);
+
+  __ epilogue();
+}
+
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_load_reference_barrier", false);
+  // arg0 : object to be resolved
+
+  __ push_call_clobbered_registers();
+  __ load_parameter(0, r0);
+  __ load_parameter(1, r1);
+  if (UseCompressedOops) {
+    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
+  } else {
+    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+  }
+  __ blr(lr);
+  __ mov(rscratch1, r0);
+  __ pop_call_clobbered_registers();
+  __ mov(r0, rscratch1);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+  assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+  return _shenandoah_lrb;
+}
+
+#define __ cgen->assembler()->
+
+// Shenandoah load reference barrier.
+//
+// Input:
+//   r0: OOP to evacuate.  Not null.
+//   r1: load address
+//
+// Output:
+//   r0: Pointer to evacuated OOP.
+//
+// Trash rscratch1, rscratch2.  Preserve everything else.
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
+
+  __ align(6);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
+  address start = __ pc();
+
+  Label work, done;
+  __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(rscratch2, Address(rscratch2, rscratch1));
+  __ tbnz(rscratch2, 0, work);
+  __ ret(lr);
+  __ bind(work);
+
+  Label slow_path;
+  __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+  __ eon(rscratch1, rscratch1, zr);
+  __ ands(zr, rscratch1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
+
+  // Decode forwarded object.
+  __ orr(rscratch1, rscratch1, markOopDesc::marked_value);
+  __ eon(r0, rscratch1, zr);
+  __ ret(lr);
+
+  __ bind(slow_path);
+  __ enter(); // required for proper stackwalking of RuntimeStub frame
+
+  __ push_call_clobbered_registers();
+
+  if (UseCompressedOops) {
+    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
+  } else {
+    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+  }
+  __ blr(lr);
+  __ mov(rscratch1, r0);
+  __ pop_call_clobbered_registers();
+  __ mov(r0, rscratch1);
+
+  __ leave(); // required for proper stackwalking of RuntimeStub frame
+  __ bind(done);
+  __ ret(lr);
+
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahLoadRefBarrier) {
+    int stub_code_size = 2048;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
+  }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahLoadReferenceBarrierStub;
+class StubAssembler;
+#endif
+class StubCodeGenerator;
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_lrb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
+  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
+  void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr);
+  void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address load_addr);
+
+  address generate_shenandoah_lrb(StubCodeGenerator* cgen);
+
+public:
+  static address shenandoah_lrb();
+
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+  void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register src, Register dst, Register count, RegSet saved_regs);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
+                                             Register obj, Register tmp, Label& slowpath);
+  virtual void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                           bool acquire, bool release, bool weak, bool is_cae, Register result);
+
+  virtual void barrier_stubs_init();
+};
+
+#endif // CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,187 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+%}
+
+encode %{
+  enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
+  enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+%}
+
+instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+ format %{
+    "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+ %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+
--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -49,6 +49,7 @@
 #include "runtime/timer.hpp"
 #include "runtime/vframeArray.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/macros.hpp"
 #include <sys/types.h>
 
 #ifndef PRODUCT
@@ -872,8 +873,16 @@
   }
 
   // Get mirror and store it in the frame as GC root for this Method*
-  __ load_mirror(rscratch1, rmethod);
-  __ stp(rscratch1, zr, Address(sp, 4 * wordSize));
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    __ load_mirror(r10, rmethod);
+    __ stp(r10, zr, Address(sp, 4 * wordSize));
+  } else
+#endif
+  {
+    __ load_mirror(rscratch1, rmethod);
+    __ stp(rscratch1, zr, Address(sp, 4 * wordSize));
+  }
 
   __ ldr(rcpool, Address(rmethod, Method::const_offset()));
   __ ldr(rcpool, Address(rcpool, ConstMethod::constants_offset()));
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -36,6 +36,7 @@
 #include "gc/shared/c1/barrierSetC1.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
+#include "utilities/macros.hpp"
 #include "vmreg_x86.inline.hpp"
 
 #ifdef ASSERT
@@ -674,6 +675,11 @@
   if (type == T_OBJECT || type == T_ARRAY) {
     cmp_value.load_item_force(FrameMap::rax_oop_opr);
     new_value.load_item();
+#if INCLUDE_SHENANDOAHGC
+    if (UseShenandoahGC) {
+      __ cas_obj(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), new_register(T_OBJECT), new_register(T_OBJECT));
+    } else
+#endif
     __ cas_obj(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill);
   } else if (type == T_INT) {
     cmp_value.load_item_force(FrameMap::rax_opr);
@@ -699,6 +705,12 @@
   // Because we want a 2-arg form of xchg and xadd
   __ move(value.result(), result);
   assert(type == T_INT || is_oop LP64_ONLY( || type == T_LONG ), "unexpected type");
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    LIR_Opr tmp = is_oop ? new_register(type) : LIR_OprFact::illegalOpr;
+    __ xchg(addr, result, result, tmp);
+  } else
+#endif
   __ xchg(addr, result, result, LIR_OprFact::illegalOpr);
   return result;
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+#define __ masm->masm()->
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  NOT_LP64(assert(_addr->is_single_cpu(), "must be single");)
+  Register addr = _addr->is_single_cpu() ? _addr->as_register() : _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  Register result = result_opr()->as_register();
+  assert(cmpval == rax, "wrong register");
+  assert(newval != NULL, "new val must be register");
+  assert(cmpval != newval, "cmp and new values must be in different registers");
+  assert(cmpval != addr, "cmp and addr must be in different registers");
+  assert(newval != addr, "new value and addr must be in different registers");
+
+  // Apply storeval barrier to newval.
+  ShenandoahBarrierSet::assembler()->storeval_barrier(masm->masm(), newval, tmp1);
+
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ encode_heap_oop(cmpval);
+    __ mov(rscratch1, newval);
+    __ encode_heap_oop(rscratch1);
+    newval = rscratch1;
+  }
+#endif
+
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), result, Address(addr, 0), cmpval, newval, false, tmp1, tmp2);
+}
+
+#undef __
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+
+  if (access.is_oop()) {
+    LIRGenerator* gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item_force(FrameMap::rax_oop_opr);
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+      LIR_Opr result = gen->new_register(T_INT);
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  // Because we want a 2-arg form of xchg and xadd
+  __ move(value_opr, result);
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
+
+  if (access.is_oop()) {
+    result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0));
+    LIR_Opr tmp = gen->new_register(type);
+    __ move(result, tmp);
+    result = tmp;
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,1240 @@
+/*
+ * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahForwarding.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/macros.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register src, Register dst, Register count) {
+
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  if (type == T_OBJECT || type == T_ARRAY) {
+
+    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
+#ifdef _LP64
+      Register thread = r15_thread;
+#else
+      Register thread = rax;
+      if (thread == src || thread == dst || thread == count) {
+        thread = rbx;
+      }
+      if (thread == src || thread == dst || thread == count) {
+        thread = rcx;
+      }
+      if (thread == src || thread == dst || thread == count) {
+        thread = rdx;
+      }
+      __ push(thread);
+      __ get_thread(thread);
+#endif
+      assert_different_registers(src, dst, count, thread);
+
+      Label done;
+      // Short-circuit if count == 0.
+      __ testptr(count, count);
+      __ jcc(Assembler::zero, done);
+
+      // Avoid runtime call when not active.
+      Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+      int flags;
+      if (ShenandoahSATBBarrier && dest_uninitialized) {
+        flags = ShenandoahHeap::HAS_FORWARDED;
+      } else {
+        flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
+      }
+      __ testb(gc_state, flags);
+      __ jcc(Assembler::zero, done);
+
+      __ pusha();                      // push registers
+
+#ifdef _LP64
+      assert(src == rdi, "expected");
+      assert(dst == rsi, "expected");
+      assert(count == rdx, "expected");
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
+                        src, dst, count);
+      } else
+#endif
+      {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
+                        src, dst, count);
+      }
+
+      __ popa();
+      __ bind(done);
+      NOT_LP64(__ pop(thread);)
+    }
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+#ifdef _LP64
+  assert(thread == r15_thread, "must be");
+#endif // _LP64
+
+  Label done;
+  Label runtime;
+
+  assert(pre_val != noreg, "check this code");
+
+  if (obj != noreg) {
+    assert_different_registers(obj, pre_val, tmp);
+    assert(pre_val != rax, "check this code");
+  }
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING);
+  __ jcc(Assembler::zero, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cmpptr(pre_val, (int32_t) NULL_WORD);
+  __ jcc(Assembler::equal, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ movptr(tmp, index);                   // tmp := *index_adr
+  __ cmpptr(tmp, 0);                       // tmp == 0?
+  __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
+
+  __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
+  __ movptr(index, tmp);                   // *index_adr := tmp
+  __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+  // save the live input values
+  if(tosca_live) __ push(rax);
+
+  if (obj != noreg && obj != rax)
+    __ push(obj);
+
+  if (pre_val != rax)
+    __ push(pre_val);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  NOT_LP64( __ push(thread); )
+
+#ifdef _LP64
+  // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
+  // pre_val be c_rarg1 (where the call prologue would copy thread argument).
+  // Note: this should not accidentally smash thread, because thread is always r15.
+  assert(thread != c_rarg0, "smashed arg");
+  if (c_rarg0 != pre_val) {
+    __ mov(c_rarg0, pre_val);
+  }
+#endif
+
+  if (expand_call) {
+    LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
+#ifdef _LP64
+    if (c_rarg1 != thread) {
+      __ mov(c_rarg1, thread);
+    }
+    // Already moved pre_val into c_rarg0 above
+#else
+    __ push(thread);
+    __ push(pre_val);
+#endif
+    __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
+  }
+
+  NOT_LP64( __ pop(thread); )
+
+  // save the live input values
+  if (pre_val != rax)
+    __ pop(pre_val);
+
+  if (obj != noreg && obj != rax)
+    __ pop(obj);
+
+  if(tosca_live) __ pop(rax);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src) {
+  assert(ShenandoahLoadRefBarrier, "Should be enabled");
+
+  Label done;
+
+#ifdef _LP64
+  Register thread = r15_thread;
+#else
+  Register thread = rcx;
+  if (thread == dst) {
+    thread = rbx;
+  }
+  __ push(thread);
+  __ get_thread(thread);
+#endif
+
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
+  __ jccb(Assembler::zero, done);
+
+  // Use rsi for src address
+  const Register src_addr = rsi;
+  // Setup address parameter first, if it does not clobber oop in dst
+  bool need_addr_setup = (src_addr != dst);
+
+  if (need_addr_setup) {
+    __ push(src_addr);
+    __ lea(src_addr, src);
+
+    if (dst != rax) {
+      // Move obj into rax and save rax
+      __ push(rax);
+      __ movptr(rax, dst);
+    }
+  } else {
+    // dst == rsi
+    __ push(rax);
+    __ movptr(rax, dst);
+
+    // we can clobber it, since it is outgoing register
+    __ lea(src_addr, src);
+  }
+
+  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
+
+  if (need_addr_setup) {
+    if (dst != rax) {
+      __ movptr(dst, rax);
+      __ pop(rax);
+    }
+    __ pop(src_addr);
+  } else {
+    __ movptr(dst, rax);
+    __ pop(rax);
+  }
+
+  __ bind(done);
+
+#ifndef _LP64
+    __ pop(thread);
+#endif
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    storeval_barrier_impl(masm, dst, tmp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
+  assert(ShenandoahStoreValEnqueueBarrier, "should be enabled");
+
+  if (dst == noreg) return;
+
+  if (ShenandoahStoreValEnqueueBarrier) {
+    // The set of registers to be saved+restored is the same as in the write-barrier above.
+    // Those are the commonly used registers in the interpreter.
+    __ pusha();
+    // __ push_callee_saved_registers();
+    __ subptr(rsp, 2 * Interpreter::stackElementSize);
+    __ movdbl(Address(rsp, 0), xmm0);
+
+#ifdef _LP64
+    Register thread = r15_thread;
+#else
+    Register thread = rcx;
+    if (thread == dst || thread == tmp) {
+      thread = rdi;
+    }
+    if (thread == dst || thread == tmp) {
+      thread = rbx;
+    }
+    __ get_thread(thread);
+#endif
+    assert_different_registers(dst, tmp, thread);
+
+    satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false);
+    __ movdbl(xmm0, Address(rsp, 0));
+    __ addptr(rsp, 2 * Interpreter::stackElementSize);
+    //__ pop_callee_saved_registers();
+    __ popa();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src) {
+  if (ShenandoahLoadRefBarrier) {
+    Label done;
+    __ testptr(dst, dst);
+    __ jcc(Assembler::zero, done);
+    load_reference_barrier_not_null(masm, dst, src);
+    __ bind(done);
+  }
+}
+
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location, might be clobbered
+//   tmp1:       scratch register, might not be valid.
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   tmp1 (if it is valid)
+//
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+             Register dst, Address src, Register tmp1, Register tmp_thread) {
+  // 1: non-reference load, no additional barrier is needed
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    return;
+  }
+
+  assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected");
+
+  // 2: load a reference from src location and apply LRB if needed
+  if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
+    Register result_dst = dst;
+    bool use_tmp1_for_dst = false;
+
+    // Preserve src location for LRB
+    if (dst == src.base() || dst == src.index()) {
+      // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
+      if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
+        dst = tmp1;
+        use_tmp1_for_dst = true;
+      } else {
+        dst = rdi;
+        __ push(dst);
+      }
+      assert_different_registers(dst, src.base(), src.index());
+    }
+
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+
+    load_reference_barrier(masm, dst, src);
+
+    // Move loaded oop to final destination
+    if (dst != result_dst) {
+      __ movptr(result_dst, dst);
+
+      if (!use_tmp1_for_dst) {
+        __ pop(dst);
+      }
+
+      dst = result_dst;
+    }
+  } else {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  }
+
+  // 3: apply keep-alive barrier if needed
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    __ push_IU_state();
+    // That path can be reached from the c2i adapter with live fp
+    // arguments in registers.
+    LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
+    __ subptr(rsp, 64);
+    __ movdbl(Address(rsp, 0), xmm0);
+    __ movdbl(Address(rsp, 8), xmm1);
+    __ movdbl(Address(rsp, 16), xmm2);
+    __ movdbl(Address(rsp, 24), xmm3);
+    __ movdbl(Address(rsp, 32), xmm4);
+    __ movdbl(Address(rsp, 40), xmm5);
+    __ movdbl(Address(rsp, 48), xmm6);
+    __ movdbl(Address(rsp, 56), xmm7);
+
+    Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
+    assert_different_registers(dst, tmp1, tmp_thread);
+    if (!thread->is_valid()) {
+      thread = rdx;
+    }
+    NOT_LP64(__ get_thread(thread));
+    // Generate the SATB pre-barrier code to log the value of
+    // the referent field in an SATB buffer.
+    shenandoah_write_barrier_pre(masm /* masm */,
+                                 noreg /* obj */,
+                                 dst /* pre_val */,
+                                 thread /* thread */,
+                                 tmp1 /* tmp */,
+                                 true /* tosca_live */,
+                                 true /* expand_call */);
+    __ movdbl(xmm0, Address(rsp, 0));
+    __ movdbl(xmm1, Address(rsp, 8));
+    __ movdbl(xmm2, Address(rsp, 16));
+    __ movdbl(xmm3, Address(rsp, 24));
+    __ movdbl(xmm4, Address(rsp, 32));
+    __ movdbl(xmm5, Address(rsp, 40));
+    __ movdbl(xmm6, Address(rsp, 48));
+    __ movdbl(xmm7, Address(rsp, 56));
+    __ addptr(rsp, 64);
+    __ pop_IU_state();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+              Address dst, Register val, Register tmp1, Register tmp2) {
+
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool as_normal = (decorators & AS_NORMAL) != 0;
+  if (on_oop && in_heap) {
+    bool needs_pre_barrier = as_normal;
+
+    Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
+    Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
+    // flatten object address if needed
+    // We do it regardless of precise because we need the registers
+    if (dst.index() == noreg && dst.disp() == 0) {
+      if (dst.base() != tmp1) {
+        __ movptr(tmp1, dst.base());
+      }
+    } else {
+      __ lea(tmp1, dst);
+    }
+
+    assert_different_registers(val, tmp1, tmp2, tmp3, rthread);
+
+#ifndef _LP64
+    __ get_thread(rthread);
+    InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
+    imasm->save_bcp();
+#endif
+
+    if (needs_pre_barrier) {
+      shenandoah_write_barrier_pre(masm /*masm*/,
+                                   tmp1 /* obj */,
+                                   tmp2 /* pre_val */,
+                                   rthread /* thread */,
+                                   tmp3  /* tmp */,
+                                   val != noreg /* tosca_live */,
+                                   false /* expand_call */);
+    }
+    if (val == noreg) {
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    } else {
+      storeval_barrier(masm, val, tmp3);
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    }
+    NOT_LP64(imasm->restore_bcp());
+  } else {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
+                                                                  Register obj, Register tmp, Label& slowpath) {
+  Label done;
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
+
+  // Check for null.
+  __ testptr(obj, obj);
+  __ jcc(Assembler::zero, done);
+
+  Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
+  __ testb(gc_state, ShenandoahHeap::EVACUATION);
+  __ jccb(Assembler::notZero, slowpath);
+  __ bind(done);
+}
+
+// Special Shenandoah CAS implementation that handles false negatives
+// due to concurrent evacuation.
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
+                                                Register res, Address addr, Register oldval, Register newval,
+                                                bool exchange, Register tmp1, Register tmp2) {
+  assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
+  assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
+  assert_different_registers(oldval, newval, tmp1, tmp2);
+
+  Label L_success, L_failure;
+
+  // Remember oldval for retry logic below
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ movl(tmp1, oldval);
+  } else
+#endif
+  {
+    __ movptr(tmp1, oldval);
+  }
+
+  // Step 1. Fast-path.
+  //
+  // Try to CAS with given arguments. If successful, then we are done.
+
+  if (os::is_MP()) __ lock();
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else
+#endif
+  {
+    __ cmpxchgptr(newval, addr);
+  }
+  __ jcc(Assembler::equal, L_success);
+
+  // Step 2. CAS had failed. This may be a false negative.
+  //
+  // The trouble comes when we compare the to-space pointer with the from-space
+  // pointer to the same object. To resolve this, it will suffice to resolve
+  // the value from memory -- this will give both to-space pointers.
+  // If they mismatch, then it was a legitimate failure.
+  //
+  // Before reaching to resolve sequence, see if we can avoid the whole shebang
+  // with filters.
+
+  // Filter: when offending in-memory value is NULL, the failure is definitely legitimate
+  __ testptr(oldval, oldval);
+  __ jcc(Assembler::zero, L_failure);
+
+  // Filter: when heap is stable, the failure is definitely legitimate
+#ifdef _LP64
+  const Register thread = r15_thread;
+#else
+  const Register thread = tmp2;
+  __ get_thread(thread);
+#endif
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
+  __ jcc(Assembler::zero, L_failure);
+
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ movl(tmp2, oldval);
+    __ decode_heap_oop(tmp2);
+  } else
+#endif
+  {
+    __ movptr(tmp2, oldval);
+  }
+
+  // Decode offending in-memory value.
+  // Test if-forwarded
+  __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markOopDesc::marked_value);
+  __ jcc(Assembler::noParity, L_failure);  // When odd number of bits, then not forwarded
+  __ jcc(Assembler::zero, L_failure);      // When it is 00, then also not forwarded
+
+  // Load and mask forwarding pointer
+  __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
+  __ shrptr(tmp2, 2);
+  __ shlptr(tmp2, 2);
+
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ decode_heap_oop(tmp1); // decode for comparison
+  }
+#endif
+
+  // Now we have the forwarded offender in tmp2.
+  // Compare and if they don't match, we have legitimate failure
+  __ cmpptr(tmp1, tmp2);
+  __ jcc(Assembler::notEqual, L_failure);
+
+  // Step 3. Need to fix the memory ptr before continuing.
+  //
+  // At this point, we have from-space oldval in the register, and its to-space
+  // address is in tmp2. Let's try to update it into memory. We don't care if it
+  // succeeds or not. If it does, then the retrying CAS would see it and succeed.
+  // If this fixup fails, this means somebody else beat us to it, and necessarily
+  // with to-space ptr store. We still have to do the retry, because the GC might
+  // have updated the reference for us.
+
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ encode_heap_oop(tmp2); // previously decoded at step 2.
+  }
+#endif
+
+  if (os::is_MP()) __ lock();
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ cmpxchgl(tmp2, addr);
+  } else
+#endif
+  {
+    __ cmpxchgptr(tmp2, addr);
+  }
+
+  // Step 4. Try to CAS again.
+  //
+  // This is guaranteed not to have false negatives, because oldval is definitely
+  // to-space, and memory pointer is to-space as well. Nothing is able to store
+  // from-space ptr into memory anymore. Make sure oldval is restored, after being
+  // garbled during retries.
+  //
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ movl(oldval, tmp2);
+  } else
+#endif
+  {
+    __ movptr(oldval, tmp2);
+  }
+
+  if (os::is_MP()) __ lock();
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else
+#endif
+  {
+    __ cmpxchgptr(newval, addr);
+  }
+  if (!exchange) {
+    __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
+  }
+
+  // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
+  // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
+  // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
+
+  if (exchange) {
+    __ bind(L_failure);
+    __ bind(L_success);
+  } else {
+    assert(res != NULL, "need result register");
+
+    Label exit;
+    __ bind(L_failure);
+    __ xorptr(res, res);
+    __ jmpb(exit);
+
+    __ bind(L_success);
+    __ movptr(res, 1);
+    __ bind(exit);
+  }
+}
+
+#ifdef _LP64
+// Copied from sharedRuntime_x86_64.cpp
+static int reg2offset_in(VMReg r) {
+  // Account for saved rbp and return address
+  // This should really be in_preserve_stack_slots
+  return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
+}
+
+// Copied from sharedRuntime_x86_64.cpp
+static int reg2offset_out(VMReg r) {
+  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
+}
+
+// Copied from sharedRuntime_x86_64.cpp
+static void move_ptr(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
+  if (src.first()->is_stack()) {
+    if (dst.first()->is_stack()) {
+      // stack to stack
+      __ movq(rax, Address(rbp, reg2offset_in(src.first())));
+      __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
+    } else {
+      // stack to reg
+      __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
+    }
+  } else if (dst.first()->is_stack()) {
+    // reg to stack
+    __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
+  } else {
+    if (dst.first() != src.first()) {
+      __ movq(dst.first()->as_Register(), src.first()->as_Register());
+    }
+  }
+}
+
+// Pin incoming array argument of java critical method
+void ShenandoahBarrierSetAssembler::pin_critical_native_array(MacroAssembler* masm,
+                                                              VMRegPair reg,
+                                                              int& pinned_slot) {
+  __ block_comment("pin_critical_native_array {");
+  Register tmp_reg = rax;
+
+  Label is_null;
+  VMRegPair tmp;
+  VMRegPair in_reg = reg;
+  bool on_stack = false;
+
+  tmp.set_ptr(tmp_reg->as_VMReg());
+  if (reg.first()->is_stack()) {
+    // Load the arg up from the stack
+    move_ptr(masm, reg, tmp);
+    reg = tmp;
+    on_stack = true;
+  } else {
+    __ movptr(rax, reg.first()->as_Register());
+  }
+  __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
+  __ jccb(Assembler::equal, is_null);
+
+  __ push(c_rarg0);
+  __ push(c_rarg1);
+  __ push(c_rarg2);
+  __ push(c_rarg3);
+#ifdef _WIN64
+  // caller-saved registers on Windows
+  __ push(r10);
+  __ push(r11);
+#else
+  __ push(c_rarg4);
+  __ push(c_rarg5);
+#endif
+
+  if (reg.first()->as_Register() != c_rarg1) {
+    __ movptr(c_rarg1, reg.first()->as_Register());
+  }
+  __ movptr(c_rarg0, r15_thread);
+  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::pin_object)));
+
+#ifdef _WIN64
+  __ pop(r11);
+  __ pop(r10);
+#else
+  __ pop(c_rarg5);
+  __ pop(c_rarg4);
+#endif
+  __ pop(c_rarg3);
+  __ pop(c_rarg2);
+  __ pop(c_rarg1);
+  __ pop(c_rarg0);
+
+  if (on_stack) {
+    __ movptr(Address(rbp, reg2offset_in(in_reg.first())), rax);
+    __ bind(is_null);
+  } else {
+    __ movptr(reg.first()->as_Register(), rax);
+
+    // save on stack for unpinning later
+    __ bind(is_null);
+    assert(reg.first()->is_Register(), "Must be a register");
+    int offset = pinned_slot * VMRegImpl::stack_slot_size;
+    pinned_slot += VMRegImpl::slots_per_word;
+    __ movq(Address(rsp, offset), rax);
+  }
+  __ block_comment("} pin_critical_native_array");
+}
+
+// Unpin array argument of java critical method
+void ShenandoahBarrierSetAssembler::unpin_critical_native_array(MacroAssembler* masm,
+                                                                VMRegPair reg,
+                                                                int& pinned_slot) {
+  __ block_comment("unpin_critical_native_array {");
+  Label is_null;
+
+  if (reg.first()->is_stack()) {
+    __ movptr(c_rarg1, Address(rbp, reg2offset_in(reg.first())));
+  } else {
+    int offset = pinned_slot * VMRegImpl::stack_slot_size;
+    pinned_slot += VMRegImpl::slots_per_word;
+    __ movq(c_rarg1, Address(rsp, offset));
+  }
+  __ testptr(c_rarg1, c_rarg1);
+  __ jccb(Assembler::equal, is_null);
+
+  __ movptr(c_rarg0, r15_thread);
+  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object)));
+
+  __ bind(is_null);
+  __ block_comment("} unpin_critical_native_array");
+}
+#else
+// Copied from sharedRuntime_x86_32.cpp
+static int reg2offset_in(VMReg r) {
+  // Account for saved rbp, and return address
+  // This should really be in_preserve_stack_slots
+  return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;
+}
+
+// Copied from sharedRuntime_x86_32.cpp
+static int reg2offset_out(VMReg r) {
+  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
+}
+
+// Copied from sharedRuntime_x86_32.cpp
+static void simple_move32(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
+  if (src.first()->is_stack()) {
+    if (dst.first()->is_stack()) {
+      // stack to stack
+      // __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
+      // __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
+      __ movl2ptr(rax, Address(rbp, reg2offset_in(src.first())));
+      __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
+    } else {
+      // stack to reg
+      __ movl2ptr(dst.first()->as_Register(),  Address(rbp, reg2offset_in(src.first())));
+    }
+  } else if (dst.first()->is_stack()) {
+    // reg to stack
+    // no need to sign extend on 64bit
+    __ movptr(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
+  } else {
+    if (dst.first() != src.first()) {
+      __ mov(dst.first()->as_Register(), src.first()->as_Register());
+    }
+  }
+}
+
+// Registers need to be saved for runtime call
+static Register caller_saved_registers[] = {
+  rcx, rdx, rsi, rdi
+};
+
+// Save caller saved registers except r1 and r2
+static void save_registers_except(MacroAssembler* masm, Register r1, Register r2) {
+  int reg_len = (int)(sizeof(caller_saved_registers) / sizeof(Register));
+  for (int index = 0; index < reg_len; index ++) {
+    Register this_reg = caller_saved_registers[index];
+    if (this_reg != r1 && this_reg != r2) {
+      __ push(this_reg);
+    }
+  }
+}
+
+// Restore caller saved registers except r1 and r2
+static void restore_registers_except(MacroAssembler* masm, Register r1, Register r2) {
+  int reg_len = (int)(sizeof(caller_saved_registers) / sizeof(Register));
+  for (int index = reg_len - 1; index >= 0; index --) {
+    Register this_reg = caller_saved_registers[index];
+    if (this_reg != r1 && this_reg != r2) {
+      __ pop(this_reg);
+    }
+  }
+}
+
+// Pin object, return pinned object or null in rax
+void ShenandoahBarrierSetAssembler::gen_pin_object(MacroAssembler* masm,
+                                                   Register thread, VMRegPair reg) {
+  __ block_comment("gen_pin_object {");
+
+  Label is_null;
+  Register tmp_reg = rax;
+  VMRegPair tmp(tmp_reg->as_VMReg());
+  if (reg.first()->is_stack()) {
+    // Load the arg up from the stack
+    simple_move32(masm, reg, tmp);
+    reg = tmp;
+  } else {
+    __ movl(tmp_reg, reg.first()->as_Register());
+  }
+  __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
+  __ jccb(Assembler::equal, is_null);
+
+  // Save registers that may be used by runtime call
+  Register arg = reg.first()->is_Register() ? reg.first()->as_Register() : noreg;
+  save_registers_except(masm, arg, thread);
+
+  __ call_VM_leaf(
+    CAST_FROM_FN_PTR(address, SharedRuntime::pin_object),
+    thread, reg.first()->as_Register());
+
+  // Restore saved registers
+  restore_registers_except(masm, arg, thread);
+
+  __ bind(is_null);
+  __ block_comment("} gen_pin_object");
+}
+
+// Unpin object
+void ShenandoahBarrierSetAssembler::gen_unpin_object(MacroAssembler* masm,
+                                                     Register thread, VMRegPair reg) {
+  __ block_comment("gen_unpin_object {");
+  Label is_null;
+
+  // temp register
+  __ push(rax);
+  Register tmp_reg = rax;
+  VMRegPair tmp(tmp_reg->as_VMReg());
+
+  simple_move32(masm, reg, tmp);
+
+  __ testptr(rax, rax);
+  __ jccb(Assembler::equal, is_null);
+
+  // Save registers that may be used by runtime call
+  Register arg = reg.first()->is_Register() ? reg.first()->as_Register() : noreg;
+  save_registers_except(masm, arg, thread);
+
+  __ call_VM_leaf(
+    CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object),
+    thread, rax);
+
+  // Restore saved registers
+  restore_registers_except(masm, arg, thread);
+  __ bind(is_null);
+  __ pop(rax);
+  __ block_comment("} gen_unpin_object");
+}
+#endif
+
+#undef __
+
+#ifdef COMPILER1
+
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+
+  __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
+  __ jcc(Assembler::equal, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ jmp(*stub->continuation());
+
+}
+
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+  Register addr = stub->addr()->as_pointer_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
+  assert_different_registers(obj, res, addr, tmp1, tmp2);
+
+  Label slow_path;
+
+  assert(res == rax, "result must arrive in rax");
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  __ testptr(res, res);
+  __ jcc(Assembler::zero, *stub->continuation());
+
+  // Check for object being in the collection set.
+  __ mov(tmp1, res);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+#ifdef _LP64
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+#else
+  // On x86_32, C1 register allocator can give us the register without 8-bit support.
+  // Do the full-register access and test to avoid compilation failures.
+  __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testptr(tmp2, 0xFF);
+#endif
+  __ jcc(Assembler::zero, *stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+  ce->store_parameter(addr, 1);
+  __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
+  __ jmp(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+  // arg0 : previous value of memory
+
+  __ push(rax);
+  __ push(rdx);
+
+  const Register pre_val = rax;
+  const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+  const Register tmp = rdx;
+
+  NOT_LP64(__ get_thread(thread);)
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is SATB still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING);
+  __ jcc(Assembler::zero, done);
+
+  // Can we store original value in the thread's buffer?
+
+  __ movptr(tmp, queue_index);
+  __ testptr(tmp, tmp);
+  __ jcc(Assembler::zero, runtime);
+  __ subptr(tmp, wordSize);
+  __ movptr(queue_index, tmp);
+  __ addptr(tmp, buffer);
+
+  // prev_val (rax)
+  __ load_parameter(0, pre_val);
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+
+  __ save_live_registers_no_oop_map(true);
+
+  // load the pre-value
+  __ load_parameter(0, rcx);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
+
+  __ restore_live_registers(true);
+
+  __ bind(done);
+
+  __ pop(rdx);
+  __ pop(rax);
+
+  __ epilogue();
+}
+
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_load_reference_barrier", false);
+  // arg0 : object to be resolved
+
+  __ save_live_registers_no_oop_map(true);
+
+#ifdef _LP64
+  __ load_parameter(0, c_rarg0);
+  __ load_parameter(1, c_rarg1);
+  if (UseCompressedOops) {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
+  }
+#else
+  __ load_parameter(0, rax);
+  __ load_parameter(1, rbx);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
+#endif
+
+  __ restore_live_registers_except_rax(true);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+  assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+  return _shenandoah_lrb;
+}
+
+#define __ cgen->assembler()->
+
+/*
+ *  Incoming parameters:
+ *  rax: oop
+ *  rsi: load address
+ */
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
+  __ align(CodeEntryAlignment);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
+  address start = __ pc();
+
+  Label resolve_oop, slow_path;
+
+  // We use RDI, which also serves as argument register for slow call.
+  // RAX always holds the src object ptr, except after the slow call,
+  // then it holds the result. R8/RBX is used as temporary register.
+
+  Register tmp1 = rdi;
+  Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
+
+  __ push(tmp1);
+  __ push(tmp2);
+
+  // Check for object being in the collection set.
+  __ mov(tmp1, rax);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+  __ jccb(Assembler::notZero, resolve_oop);
+  __ pop(tmp2);
+  __ pop(tmp1);
+  __ ret(0);
+
+  // Test if object is already resolved.
+  __ bind(resolve_oop);
+  __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
+  // Test if both lowest bits are set. We trick it by negating the bits
+  // then test for both bits clear.
+  __ notptr(tmp2);
+  __ testb(tmp2, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, slow_path);
+  // Clear both lower bits. It's still inverted, so set them, and then invert back.
+  __ orptr(tmp2, markOopDesc::marked_value);
+  __ notptr(tmp2);
+  // At this point, tmp2 contains the decoded forwarding pointer.
+  __ mov(rax, tmp2);
+
+  __ pop(tmp2);
+  __ pop(tmp1);
+  __ ret(0);
+
+  __ bind(slow_path);
+
+  __ push(rcx);
+  __ push(rdx);
+  __ push(rdi);
+#ifdef _LP64
+  __ push(r8);
+  __ push(r9);
+  __ push(r10);
+  __ push(r11);
+  __ push(r12);
+  __ push(r13);
+  __ push(r14);
+  __ push(r15);
+#endif
+  __ push(rbp);
+  __ movptr(rbp, rsp);
+  __ andptr(rsp, -StackAlignmentInBytes);
+  __ push_FPU_state();
+  if (UseCompressedOops) {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), rax, rsi);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rsi);
+  }
+  __ pop_FPU_state();
+  __ movptr(rsp, rbp);
+  __ pop(rbp);
+#ifdef _LP64
+  __ pop(r15);
+  __ pop(r14);
+  __ pop(r13);
+  __ pop(r12);
+  __ pop(r11);
+  __ pop(r10);
+  __ pop(r9);
+  __ pop(r8);
+#endif
+  __ pop(rdi);
+  __ pop(rdx);
+  __ pop(rcx);
+
+  __ pop(tmp2);
+  __ pop(tmp1);
+  __ ret(0);
+
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahLoadRefBarrier) {
+    int stub_code_size = 4096;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
+  }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+#define CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahLoadReferenceBarrierStub;
+class StubAssembler;
+#endif
+class StubCodeGenerator;
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_lrb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src);
+
+  void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp);
+
+  address generate_shenandoah_lrb(StubCodeGenerator* cgen);
+
+public:
+  static address shenandoah_lrb();
+
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+  void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  void load_reference_barrier(MacroAssembler* masm, Register dst, Address src);
+
+  virtual void cmpxchg_oop(MacroAssembler* masm,
+                           Register res, Address addr, Register oldval, Register newval,
+                           bool exchange, Register tmp1, Register tmp2);
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
+                                             Register obj, Register tmp, Label& slowpath);
+
+  virtual void barrier_stubs_init();
+
+#ifdef _LP64
+  void pin_critical_native_array(MacroAssembler* masm,
+                                 VMRegPair reg,
+                                 int& pinned_slot);
+  void unpin_critical_native_array(MacroAssembler* masm,
+                                   VMRegPair reg,
+                                   int& pinned_slot);
+#else
+  void gen_pin_object(MacroAssembler* masm,
+                      Register thread, VMRegPair reg);
+  void gen_unpin_object(MacroAssembler* masm,
+                        Register thread, VMRegPair reg);
+#endif
+};
+
+#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_32.ad	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,70 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+%}
+
+instruct compareAndSwapP_shenandoah(rRegI res,
+                                    memory mem_ptr,
+                                    eRegP tmp1, eRegP tmp2,
+                                    eAXRegP oldval, eRegP newval,
+                                    eFlagsReg cr)
+%{
+  match(Set res (ShenandoahCompareAndSwapP mem_ptr (Binary oldval newval)));
+  match(Set res (ShenandoahWeakCompareAndSwapP mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   false, // swap
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndExchangeP_shenandoah(memory mem_ptr,
+                                        eAXRegP oldval, eRegP newval,
+                                        eRegP tmp1, eRegP tmp2,
+                                        eFlagsReg cr)
+%{
+  match(Set oldval (ShenandoahCompareAndExchangeP mem_ptr (Binary oldval newval)));
+  effect(KILL cr, TEMP tmp1, TEMP tmp2);
+  ins_cost(1000);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   true,  // exchange
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,112 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+%}
+
+instruct compareAndSwapP_shenandoah(rRegI res,
+                                    memory mem_ptr,
+                                    rRegP tmp1, rRegP tmp2,
+                                    rax_RegP oldval, rRegP newval,
+                                    rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_cx8());
+  match(Set res (ShenandoahCompareAndSwapP mem_ptr (Binary oldval newval)));
+  match(Set res (ShenandoahWeakCompareAndSwapP mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   false, // swap
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndSwapN_shenandoah(rRegI res,
+                                    memory mem_ptr,
+                                    rRegP tmp1, rRegP tmp2,
+                                    rax_RegN oldval, rRegN newval,
+                                    rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndSwapN mem_ptr (Binary oldval newval)));
+  match(Set res (ShenandoahWeakCompareAndSwapN mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   false, // swap
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndExchangeN_shenandoah(memory mem_ptr,
+                                        rax_RegN oldval, rRegN newval,
+                                        rRegP tmp1, rRegP tmp2,
+                                        rFlagsReg cr) %{
+  match(Set oldval (ShenandoahCompareAndExchangeN mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   true, // exchange
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndExchangeP_shenandoah(memory mem_ptr,
+                                        rax_RegP oldval, rRegP newval,
+                                        rRegP tmp1, rRegP tmp2,
+                                        rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_cx8());
+  match(Set oldval (ShenandoahCompareAndExchangeP mem_ptr (Binary oldval newval)));
+  effect(KILL cr, TEMP tmp1, TEMP tmp2);
+  ins_cost(1000);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   true,  // exchange
+                                                   $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,7 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/vframeArray.hpp"
 #include "utilities/align.hpp"
+#include "utilities/macros.hpp"
 #include "vmreg_x86.inline.hpp"
 #ifdef COMPILER1
 #include "c1/c1_Runtime1.hpp"
@@ -46,6 +47,10 @@
 #include "opto/runtime.hpp"
 #endif
 #include "vm_version_x86.hpp"
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#endif
 
 #define __ masm->
 
@@ -1838,7 +1843,7 @@
 
   __ get_thread(thread);
 
-  if (is_critical_native) {
+  if (is_critical_native SHENANDOAHGC_ONLY(&& !UseShenandoahGC)) {
     check_needs_gc_for_critical_native(masm, thread, stack_slots, total_c_args, total_in_args,
                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
   }
@@ -1876,6 +1881,12 @@
   //
   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
 
+#if INCLUDE_SHENANDOAHGC
+  // Inbound arguments that need to be pinned for critical natives
+  GrowableArray<int> pinned_args(total_in_args);
+  // Current stack slot for storing register based array argument
+  int pinned_slot = oop_handle_offset;
+#endif
   // Mark location of rbp,
   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, rbp->as_VMReg());
 
@@ -1887,7 +1898,31 @@
     switch (in_sig_bt[i]) {
       case T_ARRAY:
         if (is_critical_native) {
+#if INCLUDE_SHENANDOAHGC
+          VMRegPair in_arg = in_regs[i];
+          if (UseShenandoahGC) {
+            // gen_pin_object handles save and restore
+            // of any clobbered registers
+            ShenandoahBarrierSet::assembler()->gen_pin_object(masm, thread, in_arg);
+            pinned_args.append(i);
+
+            // rax has pinned array
+            VMRegPair result_reg(rax->as_VMReg());
+            if (!in_arg.first()->is_stack()) {
+              assert(pinned_slot <= stack_slots, "overflow");
+              simple_move32(masm, result_reg, VMRegImpl::stack2reg(pinned_slot));
+              pinned_slot += VMRegImpl::slots_per_word;
+            } else {
+              // Write back pinned value, it will be used to unpin this argument
+              __ movptr(Address(rbp, reg2offset_in(in_arg.first())), result_reg.first()->as_Register());
+            }
+            // We have the array in register, use it
+            in_arg = result_reg;
+          }
+          unpack_array_argument(masm, in_arg, in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
+#else
           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
+#endif
           c_arg++;
           break;
         }
@@ -2083,6 +2118,29 @@
   default       : ShouldNotReachHere();
   }
 
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    // unpin pinned arguments
+    pinned_slot = oop_handle_offset;
+    if (pinned_args.length() > 0) {
+      // save return value that may be overwritten otherwise.
+      save_native_result(masm, ret_type, stack_slots);
+      for (int index = 0; index < pinned_args.length(); index ++) {
+        int i = pinned_args.at(index);
+        assert(pinned_slot <= stack_slots, "overflow");
+        if (!in_regs[i].first()->is_stack()) {
+          int offset = pinned_slot * VMRegImpl::stack_slot_size;
+          __ movl(in_regs[i].first()->as_Register(), Address(rsp, offset));
+          pinned_slot += VMRegImpl::slots_per_word;
+        }
+        // gen_pin_object handles save and restore
+        // of any other clobbered registers
+        ShenandoahBarrierSet::assembler()->gen_unpin_object(masm, thread, in_regs[i]);
+      }
+      restore_native_result(masm, ret_type, stack_slots);
+    }
+  }
+#endif
   // Switch thread to "native transition" state before reading the synchronization state.
   // This additional state is necessary because reading and testing the synchronization
   // state is not atomic w.r.t. GC, as this scenario demonstrates:
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
 #include "runtime/vframeArray.hpp"
 #include "utilities/align.hpp"
 #include "utilities/formatBuffer.hpp"
+#include "utilities/macros.hpp"
 #include "vm_version_x86.hpp"
 #include "vmreg_x86.inline.hpp"
 #ifdef COMPILER1
@@ -53,6 +54,10 @@
 #if INCLUDE_JVMCI
 #include "jvmci/jvmciJavaClasses.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#endif
 
 #define __ masm->
 
@@ -2130,7 +2135,7 @@
 
   const Register oop_handle_reg = r14;
 
-  if (is_critical_native) {
+  if (is_critical_native SHENANDOAHGC_ONLY(&& !UseShenandoahGC)) {
     check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
   }
@@ -2187,6 +2192,12 @@
   // the incoming and outgoing registers are offset upwards and for
   // critical natives they are offset down.
   GrowableArray<int> arg_order(2 * total_in_args);
+#if INCLUDE_SHENANDOAHGC
+  // Inbound arguments that need to be pinned for critical natives
+  GrowableArray<int> pinned_args(total_in_args);
+  // Current stack slot for storing register based array argument
+  int pinned_slot = oop_handle_offset;
+#endif
   VMRegPair tmp_vmreg;
   tmp_vmreg.set2(rbx->as_VMReg());
 
@@ -2234,6 +2245,14 @@
     switch (in_sig_bt[i]) {
       case T_ARRAY:
         if (is_critical_native) {
+#if INCLUDE_SHENANDOAHGC
+          // pin before unpack
+          if (UseShenandoahGC) {
+            assert(pinned_slot <= stack_slots, "overflow");
+            ShenandoahBarrierSet::assembler()->pin_critical_native_array(masm, in_regs[i], pinned_slot);
+            pinned_args.append(i);
+          }
+#endif
           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
           c_arg++;
 #ifdef ASSERT
@@ -2450,6 +2469,22 @@
   default       : ShouldNotReachHere();
   }
 
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    // unpin pinned arguments
+    pinned_slot = oop_handle_offset;
+    if (pinned_args.length() > 0) {
+      // save return value that may be overwritten otherwise.
+      save_native_result(masm, ret_type, stack_slots);
+      for (int index = 0; index < pinned_args.length(); index ++) {
+        int i = pinned_args.at(index);
+        assert(pinned_slot <= stack_slots, "overflow");
+        ShenandoahBarrierSet::assembler()->unpin_critical_native_array(masm, in_regs[i], pinned_slot);
+      }
+      restore_native_result(masm, ret_type, stack_slots);
+    }
+  }
+#endif
   // Switch thread to "native transition" state before reading the synchronization state.
   // This additional state is necessary because reading and testing the synchronization
   // state is not atomic w.r.t. GC, as this scenario demonstrates:
--- a/src/hotspot/share/adlc/formssel.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/adlc/formssel.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -778,6 +778,10 @@
        !strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") ||
        !strcmp(_matrule->_rChild->_opType,"LoadBarrierWeakSlowReg") ||
 #endif
+#if INCLUDE_SHENANDOAHGC
+       !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") ||
+       !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") ||
+#endif
        !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") ||
        !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN")))  return true;
   else if ( is_ideal_load() == Form::idealP )                return true;
@@ -3502,6 +3506,9 @@
     "CompareAndSwapB", "CompareAndSwapS", "CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN",
     "WeakCompareAndSwapB", "WeakCompareAndSwapS", "WeakCompareAndSwapI", "WeakCompareAndSwapL", "WeakCompareAndSwapP", "WeakCompareAndSwapN",
     "CompareAndExchangeB", "CompareAndExchangeS", "CompareAndExchangeI", "CompareAndExchangeL", "CompareAndExchangeP", "CompareAndExchangeN",
+#if INCLUDE_SHENANDOAHGC
+    "ShenandoahCompareAndSwapN", "ShenandoahCompareAndSwapP", "ShenandoahWeakCompareAndSwapP", "ShenandoahWeakCompareAndSwapN", "ShenandoahCompareAndExchangeP", "ShenandoahCompareAndExchangeN",
+#endif
     "StoreCM",
     "ClearArray",
     "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
--- a/src/hotspot/share/ci/ciInstanceKlass.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -554,6 +554,14 @@
   _has_injected_fields = has_injected_fields;
 }
 
+#if INCLUDE_SHENANDOAHGC
+bool ciInstanceKlass::has_object_fields() const {
+  GUARDED_VM_ENTRY(
+      return get_instanceKlass()->nonstatic_oop_map_size() > 0;
+    );
+}
+#endif
+
 // ------------------------------------------------------------------
 // ciInstanceKlass::find_method
 //
--- a/src/hotspot/share/ci/ciInstanceKlass.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/ci/ciInstanceKlass.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -202,6 +202,10 @@
     return _has_injected_fields > 0 ? true : false;
   }
 
+#if INCLUDE_SHENANDOAHGC
+  bool has_object_fields() const;
+#endif
+
   // nth nonstatic field (presented by ascending address)
   ciField* nonstatic_field_at(int i) {
     assert(_nonstatic_fields != NULL, "");
--- a/src/hotspot/share/code/codeCache.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/code/codeCache.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -32,6 +32,7 @@
 #include "oops/instanceKlass.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "runtime/mutexLocker.hpp"
+#include "utilities/macros.hpp"
 
 // The CodeCache implements the code cache for various pieces of generated
 // code, e.g., compiled java methods, runtime stubs, transition frames, etc.
@@ -80,6 +81,9 @@
   template <class T, class Filter> friend class CodeBlobIterator;
   friend class WhiteBox;
   friend class CodeCacheLoader;
+#if INCLUDE_SHENANDOAHGC
+  friend class ShenandoahParallelCodeHeapIterator;
+#endif
  private:
   // CodeHeaps of the cache
   static GrowableArray<CodeHeap*>* _heaps;
--- a/src/hotspot/share/gc/shared/barrierSetConfig.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -32,6 +32,7 @@
   f(CardTableBarrierSet)                             \
   EPSILONGC_ONLY(f(EpsilonBarrierSet))               \
   G1GC_ONLY(f(G1BarrierSet))                         \
+  SHENANDOAHGC_ONLY(f(ShenandoahBarrierSet))         \
   ZGC_ONLY(f(ZBarrierSet))
 
 #define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f)          \
--- a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -36,6 +36,9 @@
 #if INCLUDE_G1GC
 #include "gc/g1/g1BarrierSet.inline.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/zBarrierSet.inline.hpp"
 #endif
--- a/src/hotspot/share/gc/shared/collectedHeap.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/collectedHeap.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -35,6 +35,7 @@
 #include "utilities/events.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/growableArray.hpp"
+#include "utilities/macros.hpp"
 
 // A "CollectedHeap" is an implementation of a java heap for HotSpot.  This
 // is an abstract class: there may be many different kinds of heaps.  This
@@ -89,6 +90,7 @@
 //     CMSHeap
 //   G1CollectedHeap
 //   ParallelScavengeHeap
+//   ShenandoahHeap
 //   ZCollectedHeap
 //
 class CollectedHeap : public CHeapObj<mtInternal> {
@@ -179,6 +181,9 @@
     G1,
     Epsilon,
     Z
+#if INCLUDE_SHENANDOAHGC
+    ,Shenandoah
+#endif
   };
 
   static inline size_t filler_array_max_size() {
--- a/src/hotspot/share/gc/shared/gcCause.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gcCause.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gcCause.hpp"
+#include "utilities/macros.hpp"
 
 const char* GCCause::to_string(GCCause::Cause cause) {
   switch (cause) {
@@ -105,6 +106,20 @@
     case _dcmd_gc_run:
       return "Diagnostic Command";
 
+#if INCLUDE_SHENANDOAHGC
+    case _shenandoah_allocation_failure_evac:
+      return "Allocation Failure During Evacuation";
+
+    case _shenandoah_stop_vm:
+      return "Stopping VM";
+
+    case _shenandoah_concurrent_gc:
+      return "Concurrent GC";
+
+    case _shenandoah_upgrade_to_full_gc:
+      return "Upgrade To Full GC";
+#endif
+
     case _z_timer:
       return "Timer";
 
--- a/src/hotspot/share/gc/shared/gcCause.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gcCause.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 #define SHARE_VM_GC_SHARED_GCCAUSE_HPP
 
 #include "memory/allocation.hpp"
+#include "utilities/macros.hpp"
 
 //
 // This class exposes implementation details of the various
@@ -78,6 +79,13 @@
 
     _dcmd_gc_run,
 
+#if INCLUDE_SHENANDOAHGC
+    _shenandoah_stop_vm,
+    _shenandoah_allocation_failure_evac,
+    _shenandoah_concurrent_gc,
+    _shenandoah_upgrade_to_full_gc,
+#endif
+
     _z_timer,
     _z_warmup,
     _z_allocation_rate,
@@ -121,7 +129,9 @@
     // _allocation_failure is the generic cause a collection for allocation failure
     // _adaptive_size_policy is for a collecton done before a full GC
     return (cause == GCCause::_allocation_failure ||
-            cause == GCCause::_adaptive_size_policy);
+            cause == GCCause::_adaptive_size_policy
+            SHENANDOAHGC_ONLY(|| cause == GCCause::_shenandoah_allocation_failure_evac)
+            );
   }
 
   // Return a string describing the GCCause.
--- a/src/hotspot/share/gc/shared/gcConfig.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gcConfig.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -43,6 +43,9 @@
 #if INCLUDE_SERIALGC
 #include "gc/serial/serialArguments.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahArguments.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/zArguments.hpp"
 #endif
@@ -57,23 +60,25 @@
       _flag(flag), _name(name), _arguments(arguments), _hs_err_name(hs_err_name) {}
 };
 
-     CMSGC_ONLY(static CMSArguments      cmsArguments;)
- EPSILONGC_ONLY(static EpsilonArguments  epsilonArguments;)
-      G1GC_ONLY(static G1Arguments       g1Arguments;)
-PARALLELGC_ONLY(static ParallelArguments parallelArguments;)
-  SERIALGC_ONLY(static SerialArguments   serialArguments;)
-       ZGC_ONLY(static ZArguments        zArguments;)
+       CMSGC_ONLY(static CMSArguments        cmsArguments;)
+   EPSILONGC_ONLY(static EpsilonArguments    epsilonArguments;)
+        G1GC_ONLY(static G1Arguments         g1Arguments;)
+  PARALLELGC_ONLY(static ParallelArguments   parallelArguments;)
+    SERIALGC_ONLY(static SerialArguments     serialArguments;)
+SHENANDOAHGC_ONLY(static ShenandoahArguments shenandoahArguments;)
+         ZGC_ONLY(static ZArguments          zArguments;)
 
 // Table of supported GCs, for translating between command
 // line flag, CollectedHeap::Name and GCArguments instance.
 static const SupportedGC SupportedGCs[] = {
-       CMSGC_ONLY_ARG(SupportedGC(UseConcMarkSweepGC, CollectedHeap::CMS,      cmsArguments,      "concurrent mark sweep gc"))
-   EPSILONGC_ONLY_ARG(SupportedGC(UseEpsilonGC,       CollectedHeap::Epsilon,  epsilonArguments,  "epsilon gc"))
-        G1GC_ONLY_ARG(SupportedGC(UseG1GC,            CollectedHeap::G1,       g1Arguments,       "g1 gc"))
-  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelGC,      CollectedHeap::Parallel, parallelArguments, "parallel gc"))
-  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelOldGC,   CollectedHeap::Parallel, parallelArguments, "parallel gc"))
-    SERIALGC_ONLY_ARG(SupportedGC(UseSerialGC,        CollectedHeap::Serial,   serialArguments,   "serial gc"))
-         ZGC_ONLY_ARG(SupportedGC(UseZGC,             CollectedHeap::Z,        zArguments,        "z gc"))
+       CMSGC_ONLY_ARG(SupportedGC(UseConcMarkSweepGC, CollectedHeap::CMS,        cmsArguments,        "concurrent mark sweep gc"))
+   EPSILONGC_ONLY_ARG(SupportedGC(UseEpsilonGC,       CollectedHeap::Epsilon,    epsilonArguments,    "epsilon gc"))
+        G1GC_ONLY_ARG(SupportedGC(UseG1GC,            CollectedHeap::G1,         g1Arguments,         "g1 gc"))
+  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelGC,      CollectedHeap::Parallel,   parallelArguments,   "parallel gc"))
+  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelOldGC,   CollectedHeap::Parallel,   parallelArguments,   "parallel gc"))
+    SERIALGC_ONLY_ARG(SupportedGC(UseSerialGC,        CollectedHeap::Serial,     serialArguments,     "serial gc"))
+SHENANDOAHGC_ONLY_ARG(SupportedGC(UseShenandoahGC,    CollectedHeap::Shenandoah, shenandoahArguments, "shenandoah gc"))
+         ZGC_ONLY_ARG(SupportedGC(UseZGC,             CollectedHeap::Z,          zArguments,          "z gc"))
 };
 
 #define FOR_EACH_SUPPORTED_GC(var)                                          \
--- a/src/hotspot/share/gc/shared/gcConfiguration.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -43,7 +43,7 @@
     return ParNew;
   }
 
-  if (UseZGC) {
+  if (UseZGC SHENANDOAHGC_ONLY(|| UseShenandoahGC)) {
     return NA;
   }
 
@@ -67,6 +67,12 @@
     return Z;
   }
 
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    return Shenandoah;
+  }
+#endif
+
   return SerialOld;
 }
 
--- a/src/hotspot/share/gc/shared/gcName.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gcName.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,9 @@
   G1Old,
   G1Full,
   Z,
+#if INCLUDE_SHENANDOAHGC
+  Shenandoah,
+#endif
   NA,
   GCNameEndSentinel
 };
@@ -58,6 +61,9 @@
       case G1Old: return "G1Old";
       case G1Full: return "G1Full";
       case Z: return "Z";
+#if INCLUDE_SHENANDOAHGC
+      case Shenandoah: return "Shenandoah";
+#endif
       case NA: return "N/A";
       default: ShouldNotReachHere(); return NULL;
     }
--- a/src/hotspot/share/gc/shared/gc_globals.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -41,10 +41,19 @@
 #if INCLUDE_SERIALGC
 #include "gc/serial/serial_globals.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoah_globals.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/z_globals.hpp"
 #endif
 
+#if INCLUDE_SHENANDOAHGC
+#define shproduct(product, type, name, value, doc) product(type, name, value, doc)
+#else
+#define shproduct(product, type, name, value, doc)
+#endif
+
 #define GC_FLAGS(develop,                                                   \
                  develop_pd,                                                \
                  product,                                                   \
@@ -140,6 +149,22 @@
     constraint,                                                             \
     writeable))                                                             \
                                                                             \
+  SHENANDOAHGC_ONLY(GC_SHENANDOAH_FLAGS(                                    \
+    develop,                                                                \
+    develop_pd,                                                             \
+    product,                                                                \
+    product_pd,                                                             \
+    diagnostic,                                                             \
+    diagnostic_pd,                                                          \
+    experimental,                                                           \
+    notproduct,                                                             \
+    manageable,                                                             \
+    product_rw,                                                             \
+    lp64_product,                                                           \
+    range,                                                                  \
+    constraint,                                                             \
+    writeable))                                                             \
+                                                                            \
   ZGC_ONLY(GC_Z_FLAGS(                                                      \
     develop,                                                                \
     develop_pd,                                                             \
@@ -179,6 +204,9 @@
   experimental(bool, UseZGC, false,                                         \
           "Use the Z garbage collector")                                    \
                                                                             \
+  shproduct(product, bool, UseShenandoahGC, false,                          \
+          "Use the Shenandoah garbage collector")                           \
+                                                                            \
   product(uint, ParallelGCThreads, 0,                                       \
           "Number of parallel threads parallel gc will use")                \
           constraint(ParallelGCThreadsConstraintFunc,AfterErgo)             \
--- a/src/hotspot/share/gc/shared/referenceProcessor.cpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/referenceProcessor.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -1156,7 +1156,7 @@
       // Check assumption that an object is not potentially
       // discovered twice except by concurrent collectors that potentially
       // trace the same Reference object twice.
-      assert(UseConcMarkSweepGC || UseG1GC,
+      assert(UseConcMarkSweepGC || UseG1GC SHENANDOAHGC_ONLY(|| UseShenandoahGC),
              "Only possible with a concurrent marking collector");
       return true;
     }
--- a/src/hotspot/share/gc/shared/taskqueue.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/taskqueue.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -359,6 +359,9 @@
 public:
   // Returns "true" if some TaskQueue in the set contains a task.
   virtual bool peek() = 0;
+#if INCLUDE_SHENANDOAHGC
+  virtual size_t tasks() = 0;
+#endif
 };
 
 template <MEMFLAGS F> class TaskQueueSetSuperImpl: public CHeapObj<F>, public TaskQueueSetSuper {
@@ -390,6 +393,9 @@
   bool steal(uint queue_num, int* seed, E& t);
 
   bool peek();
+#if INCLUDE_SHENANDOAHGC
+  size_t tasks();
+#endif
 
   uint size() const { return _n; }
 };
@@ -415,6 +421,17 @@
   return false;
 }
 
+#if INCLUDE_SHENANDOAHGC
+template<class T, MEMFLAGS F>
+size_t GenericTaskQueueSet<T, F>::tasks() {
+  size_t n = 0;
+  for (uint j = 0; j < _n; j++) {
+    n += _queues[j]->size();
+  }
+  return n;
+}
+#endif
+
 // When to terminate from the termination protocol.
 class TerminatorTerminator: public CHeapObj<mtInternal> {
 public:
@@ -428,6 +445,9 @@
 
 class ParallelTaskTerminator: public StackObj {
 private:
+#if INCLUDE_SHENANDOAHGC
+protected:
+#endif
   uint _n_threads;
   TaskQueueSetSuper* _queue_set;
 
@@ -463,7 +483,7 @@
   // As above, but it also terminates if the should_exit_termination()
   // method of the terminator parameter returns true. If terminator is
   // NULL, then it is ignored.
-  bool offer_termination(TerminatorTerminator* terminator);
+  SHENANDOAHGC_ONLY(virtual) bool offer_termination(TerminatorTerminator* terminator);
 
   // Reset the terminator, so that it may be reused again.
   // The caller is responsible for ensuring that this is done
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp	Tue Jan 14 19:27:03 2020 -0800
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -50,6 +50,9 @@
 #include "gc/serial/defNewGeneration.hpp"
 #include "gc/serial/vmStructs_serial.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/vmStructs_shenandoah.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/vmStructs_z.hpp"
 #endif
@@ -73,6 +76,9 @@
   SERIALGC_ONLY(VM_STRUCTS_SERIALGC(nonstatic_field,                                                                                 \
                                     volatile_nonstatic_field,                                                                        \
                                     static_field))                                                                                   \
+  SHENANDOAHGC_ONLY(VM_STRUCTS_SHENANDOAH(nonstatic_field,                                                                           \
+                                          volatile_nonstatic_field,                                                                  \
+                                          static_field))                                                                             \
   ZGC_ONLY(VM_STRUCTS_ZGC(nonstatic_field,                                                                                           \
                           volatile_nonstatic_field,                                                                                  \
                           static_field))                                                                                             \
@@ -178,6 +184,9 @@
   SERIALGC_ONLY(VM_TYPES_SERIALGC(declare_type,                           \
                                   declare_toplevel_type,                  \
                                   declare_integer_type))                  \
+  SHENANDOAHGC_ONLY(VM_TYPES_SHENANDOAH(declare_type,                     \
+                                        declare_toplevel_type,            \
+                                        declare_integer_type))            \
   ZGC_ONLY(VM_TYPES_ZGC(declare_type,                                     \
                         declare_toplevel_type,                            \
                         declare_integer_type))                            \
@@ -253,6 +262,8 @@
                                               declare_constant_with_value)) \
   SERIALGC_ONLY(VM_INT_CONSTANTS_SERIALGC(declare_constant,                 \
                                           declare_constant_with_value))     \
+  SHENANDOAHGC_ONLY(VM_INT_CONSTANTS_SHENANDOAH(declare_constant,           \
+                                              declare_constant_with_value)) \
   ZGC_ONLY(VM_INT_CONSTANTS_ZGC(declare_constant,                           \
                                 declare_constant_with_value))               \
                                                                             \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_IR.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahSATBMarkQueue.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+void ShenandoahPreBarrierStub::emit_code(LIR_Assembler* ce) {
+  ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->gen_pre_barrier_stub(ce, this);
+}
+
+void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) {
+  ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->gen_load_reference_barrier_stub(ce, this);
+}
+
+ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() :
+  _pre_barrier_c1_runtime_code_blob(NULL),
+  _load_reference_barrier_rt_code_blob(NULL) {}
+
+void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
+  // First we test whether marking is in progress.
+  BasicType flag_type;
+  bool patch = (decorators & C1_NEEDS_PATCHING) != 0;
+  bool do_load = pre_val == LIR_OprFact::illegalOpr;
+  if (in_bytes(ShenandoahSATBMarkQueue::byte_width_of_active()) == 4) {
+    flag_type = T_INT;
+  } else {
+    guarantee(in_bytes(ShenandoahSATBMarkQueue::byte_width_of_active()) == 1,
+              "Assumption");
+    // Use unsigned type T_BOOLEAN here rather than signed T_BYTE since some platforms, eg. ARM,
+    // need to use unsigned instructions to use the large offset to load the satb_mark_queue.
+    flag_type = T_BOOLEAN;
+  }
+  LIR_Opr thrd = gen->getThreadPointer();
+  LIR_Address* mark_active_flag_addr =
+    new LIR_Address(thrd,
+                    in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()),
+                    flag_type);
+  // Read the marking-in-progress flag.
+  LIR_Opr flag_val = gen->new_register(T_INT);
+  __ load(mark_active_flag_addr, flag_val);
+  __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+
+  LIR_PatchCode pre_val_patch_code = lir_patch_none;
+
+  CodeStub* slow;
+
+  if (do_load) {
+    assert(pre_val == LIR_OprFact::illegalOpr, "sanity");
+    assert(addr_opr != LIR_OprFact::illegalOpr, "sanity");
+
+    if (patch)
+      pre_val_patch_code = lir_patch_normal;
+
+    pre_val = gen->new_register(T_OBJECT);
+
+    if (!addr_opr->is_address()) {
+      assert(addr_opr->is_register(), "must be");
+      addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT));
+    }
+    slow = new ShenandoahPreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info ? new CodeEmitInfo(info) : NULL);
+  } else {
+    assert(addr_opr == LIR_OprFact::illegalOpr, "sanity");
+    assert(pre_val->is_register(), "must be");
+    assert(pre_val->type() == T_OBJECT, "must be an object");
+
+    slow = new ShenandoahPreBarrierStub(pre_val);
+  }
+
+  __ branch(lir_cond_notEqual, T_INT, slow);
+  __ branch_destination(slow->continuation());
+}
+
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr) {
+  if (ShenandoahLoadRefBarrier) {
+    return load_reference_barrier_impl(gen, obj, addr);
+  } else {
+    return obj;
+  }
+}
+
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr) {
+  assert(ShenandoahLoadRefBarrier, "Should be enabled");
+
+  obj = ensure_in_register(gen, obj, T_OBJECT);
+  assert(obj->is_register(), "must be a register at this point");
+  addr = ensure_in_register(gen, addr, T_ADDRESS);
+  assert(addr->is_register(), "must be a register at this point");
+  LIR_Opr result = gen->result_register_for(obj->value_type());
+  __ move(obj, result);
+  LIR_Opr tmp1 = gen->new_register(T_ADDRESS);
+  LIR_Opr tmp2 = gen->new_register(T_ADDRESS);
+
+  LIR_Opr thrd = gen->getThreadPointer();
+  LIR_Address* active_flag_addr =
+    new LIR_Address(thrd,
+                    in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
+                    T_BYTE);
+  // Read and check the gc-state-flag.
+  LIR_Opr flag_val = gen->new_register(T_INT);
+  __ load(active_flag_addr, flag_val);
+  LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED);
+  LIR_Opr mask_reg = gen->new_register(T_INT);
+  __ move(mask, mask_reg);
+
+  if (TwoOperandLIRForm) {
+    __ logical_and(flag_val, mask_reg, flag_val);
+  } else {
+    LIR_Opr masked_flag = gen->new_register(T_INT);
+    __ logical_and(flag_val, mask_reg, masked_flag);
+    flag_val = masked_flag;
+  }
+  __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+
+  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2);
+  __ branch(lir_cond_notEqual, T_INT, slow);
+  __ branch_destination(slow->continuation());
+
+  return result;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type) {
+  if (!obj->is_register()) {
+    LIR_Opr obj_reg;
+    if (obj->is_constant()) {
+      obj_reg = gen->new_register(type);
+      __ move(obj, obj_reg);
+    } else {
+      obj_reg = gen->new_pointer_register();
+      __ leal(obj, obj_reg);
+    }
+    obj = obj_reg;
+  }
+  return obj;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    obj = ensure_in_register(gen, obj, T_OBJECT);
+    pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj);
+  }
+  return obj;
+}
+
+void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    value = storeval_barrier(access.gen(), value, access.access_emit_info(), access.decorators());
+  }
+  BarrierSetC1::store_at_resolved(access, value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
+  // We must resolve in register when patching. This is to avoid
+  // having a patch area in the load barrier stub, since the call
+  // into the runtime to patch will not have the proper oop map.
+  const bool patch_before_barrier = access.is_oop() && (access.decorators() & C1_NEEDS_PATCHING) != 0;
+  return BarrierSetC1::resolve_address(access, resolve_in_register || patch_before_barrier);
+}
+
+void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
+  // 1: non-reference load, no additional barrier is needed
+  if (!access.is_oop()) {
+    BarrierSetC1::load_at_resolved(access, result);
+    return;
+  }
+
+  LIRGenerator* gen = access.gen();
+  DecoratorSet decorators = access.decorators();
+  BasicType type = access.type();
+
+  // 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
+  if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
+    LIR_Opr tmp = gen->new_register(T_OBJECT);
+    BarrierSetC1::load_at_resolved(access, tmp);
+    tmp = load_reference_barrier(gen, tmp, access.resolved_addr());
+    __ move(tmp, result);
+  } else {
+    BarrierSetC1::load_at_resolved(access, result);
+  }
+
+  // 3: apply keep-alive barrier if ShenandoahSATBBarrier is set
+  if (ShenandoahSATBBarrier) {
+    DecoratorSet decorators = access.decorators();
+    bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+    bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+    bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+    if (is_weak || is_phantom || is_anonymous) {
+      // Register the value in the referent field with the pre-barrier
+      LabelObj *Lcont_anonymous;
+      if (is_anonymous) {
+        Lcont_anonymous = new LabelObj();
+        generate_referent_check(access, Lcont_anonymous);
+      }
+      pre_barrier(gen, access.access_emit_info(), decorators, LIR_OprFact::illegalOpr /* addr_opr */,
+                  result /* pre_val */);
+      if (is_anonymous) {
+        __ branch_destination(Lcont_anonymous->label());
+      }
+    }
+ }
+}
+
+class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+    bs->generate_c1_pre_barrier_runtime_stub(sasm);
+    return NULL;
+  }
+};
+
+class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+    bs->generate_c1_load_reference_barrier_runtime_stub(sasm);
+    return NULL;
+  }
+};
+
+void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) {
+  C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl;
+  _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1,
+                                                              "shenandoah_pre_barrier_slow",
+                                                              false, &pre_code_gen_cl);
+  if (ShenandoahLoadRefBarrier) {
+    C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl;
+    _load_reference_barrier_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
+                                                                  "shenandoah_load_reference_barrier_slow",
+                                                                  false, &lrb_code_gen_cl);
+  }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,230 @@
+/*
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
+#define SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
+
+#include "c1/c1_CodeStubs.hpp"
+#include "gc/shared/c1/barrierSetC1.hpp"
+
+class ShenandoahPreBarrierStub: public CodeStub {
+  friend class ShenandoahBarrierSetC1;
+private:
+  bool _do_load;
+  LIR_Opr _addr;
+  LIR_Opr _pre_val;
+  LIR_PatchCode _patch_code;
+  CodeEmitInfo* _info;
+
+public:
+  // Version that _does_ generate a load of the previous value from addr.
+  // addr (the address of the field to be read) must be a LIR_Address
+  // pre_val (a temporary register) must be a register;
+  ShenandoahPreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) :
+    _do_load(true), _addr(addr), _pre_val(pre_val),
+    _patch_code(patch_code), _info(info)
+  {
+    assert(_pre_val->is_register(), "should be temporary register");
+    assert(_addr->is_address(), "should be the address of the field");
+  }
+
+  // Version that _does not_ generate load of the previous value; the
+  // previous value is assumed to have already been loaded into pre_val.
+  ShenandoahPreBarrierStub(LIR_Opr pre_val) :
+    _do_load(false), _addr(LIR_OprFact::illegalOpr), _pre_val(pre_val),
+    _patch_code(lir_patch_none), _info(NULL)
+  {
+    assert(_pre_val->is_register(), "should be a register");
+  }
+
+  LIR_Opr addr() const { return _addr; }
+  LIR_Opr pre_val() const { return _pre_val; }
+  LIR_PatchCode patch_code() const { return _patch_code; }
+  CodeEmitInfo* info() const { return _info; }
+  bool do_load() const { return _do_load; }
+
+  virtual void emit_code(LIR_Assembler* e);
+  virtual void visit(LIR_OpVisitState* visitor) {
+    if (_do_load) {
+      // don't pass in the code emit info since it's processed in the fast
+      // path
+      if (_info != NULL)
+        visitor->do_slow_case(_info);
+      else
+        visitor->do_slow_case();
+
+      visitor->do_input(_addr);
+      visitor->do_temp(_pre_val);
+    } else {
+      visitor->do_slow_case();
+      visitor->do_input(_pre_val);
+    }
+  }
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const { out->print("ShenandoahPreBarrierStub"); }
+#endif // PRODUCT
+};
+
+class ShenandoahLoadReferenceBarrierStub: public CodeStub {
+  friend class ShenandoahBarrierSetC1;
+private:
+  LIR_Opr _obj;
+  LIR_Opr _addr;
+  LIR_Opr _result;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
+
+public:
+  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) :
+    _obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2)
+  {
+    assert(_obj->is_register(), "should be register");
+    assert(_addr->is_register(), "should be register");
+    assert(_result->is_register(), "should be register");
+    assert(_tmp1->is_register(), "should be register");
+    assert(_tmp2->is_register(), "should be register");
+  }
+
+  LIR_Opr obj() const { return _obj; }
+  LIR_Opr addr() const { return _addr; }
+  LIR_Opr result() const { return _result; }
+  LIR_Opr tmp1() const { return _tmp1; }
+  LIR_Opr tmp2() const { return _tmp2; }
+
+  virtual void emit_code(LIR_Assembler* e);
+  virtual void visit(LIR_OpVisitState* visitor) {
+    visitor->do_slow_case();
+    visitor->do_input(_obj);
+    visitor->do_temp(_obj);
+    visitor->do_input(_addr);
+    visitor->do_temp(_addr);
+    visitor->do_temp(_result);
+    visitor->do_temp(_tmp1);
+    visitor->do_temp(_tmp2);
+  }
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); }
+#endif // PRODUCT
+};
+
+class LIR_OpShenandoahCompareAndSwap : public LIR_Op {
+ friend class LIR_OpVisitState;
+
+private:
+  LIR_Opr _addr;
+  LIR_Opr _cmp_value;
+  LIR_Opr _new_value;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
+
+public:
+  LIR_OpShenandoahCompareAndSwap(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value,
+                                 LIR_Opr t1, LIR_Opr t2, LIR_Opr result)
+    : LIR_Op(lir_none, result, NULL)  // no info
+    , _addr(addr)
+    , _cmp_value(cmp_value)
+    , _new_value(new_value)
+    , _tmp1(t1)
+    , _tmp2(t2)                                  { }
+
+  LIR_Opr addr()        const                    { return _addr;  }
+  LIR_Opr cmp_value()   const                    { return _cmp_value; }
+  LIR_Opr new_value()   const                    { return _new_value; }
+  LIR_Opr tmp1()        const                    { return _tmp1;      }
+  LIR_Opr tmp2()        const                    { return _tmp2;      }
+
+  virtual void visit(LIR_OpVisitState* state) {
+      assert(_addr->is_valid(),      "used");
+      assert(_cmp_value->is_valid(), "used");
+      assert(_new_value->is_valid(), "used");
+      if (_info)                    state->do_info(_info);
+                                    state->do_input(_addr);
+                                    state->do_temp(_addr);
+                                    state->do_input(_cmp_value);
+                                    state->do_temp(_cmp_value);
+                                    state->do_input(_new_value);
+                                    state->do_temp(_new_value);
+      if (_tmp1->is_valid())        state->do_temp(_tmp1);
+      if (_tmp2->is_valid())        state->do_temp(_tmp2);
+      if (_result->is_valid())      state->do_output(_result);
+  }
+
+  virtual void emit_code(LIR_Assembler* masm);
+
+  virtual void print_instr(outputStream* out) const {
+    addr()->print(out);      out->print(" ");
+    cmp_value()->print(out); out->print(" ");
+    new_value()->print(out); out->print(" ");
+    tmp1()->print(out);      out->print(" ");
+    tmp2()->print(out);      out->print(" ");
+  }
+#ifndef PRODUCT
+  virtual const char* name() const {
+    return "shenandoah_cas_obj";
+  }
+#endif // PRODUCT
+};
+
+class ShenandoahBarrierSetC1 : public BarrierSetC1 {
+private:
+  CodeBlob* _pre_barrier_c1_runtime_code_blob;
+  CodeBlob* _load_reference_barrier_rt_code_blob;
+
+  void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
+
+  LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr);
+  LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);
+
+  LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr);
+
+  LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type);
+
+public:
+  ShenandoahBarrierSetC1();
+
+  CodeBlob* pre_barrier_c1_runtime_code_blob() {
+    assert(_pre_barrier_c1_runtime_code_blob != NULL, "");
+    return _pre_barrier_c1_runtime_code_blob;
+  }
+
+  CodeBlob* load_reference_barrier_rt_code_blob() {
+    assert(_load_reference_barrier_rt_code_blob != NULL, "");
+    return _load_reference_barrier_rt_code_blob;
+  }
+
+protected:
+
+  virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);
+  virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register);
+  virtual void load_at_resolved(LIRAccess& access, LIR_Opr result);
+
+  virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value);
+
+  virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
+
+public:
+
+  virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob);
+};
+
+#endif // SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,1059 @@
+/*
+ * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahForwarding.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "opto/arraycopynode.hpp"
+#include "opto/escape.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/idealKit.hpp"
+#include "opto/macro.hpp"
+#include "opto/movenode.hpp"
+#include "opto/narrowptrnode.hpp"
+#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
+
+ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
+  return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
+}
+
+ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
+  : _enqueue_barriers(new (comp_arena) GrowableArray<ShenandoahEnqueueBarrierNode*>(comp_arena, 8,  0, NULL)),
+    _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
+}
+
+int ShenandoahBarrierSetC2State::enqueue_barriers_count() const {
+  return _enqueue_barriers->length();
+}
+
+ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const {
+  return _enqueue_barriers->at(idx);
+}
+
+void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+  assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
+  _enqueue_barriers->append(n);
+}
+
+void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+  if (_enqueue_barriers->contains(n)) {
+    _enqueue_barriers->remove(n);
+  }
+}
+
+int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
+  return _load_reference_barriers->length();
+}
+
+ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const {
+  return _load_reference_barriers->at(idx);
+}
+
+void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+  assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list");
+  _load_reference_barriers->append(n);
+}
+
+void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+  if (_load_reference_barriers->contains(n)) {
+    _load_reference_barriers->remove(n);
+  }
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    obj = shenandoah_enqueue_barrier(kit, obj);
+  }
+  return obj;
+}
+
+#define __ kit->
+
+bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
+                                                         BasicType bt, uint adr_idx) const {
+  intptr_t offset = 0;
+  Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
+  AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
+
+  if (offset == Type::OffsetBot) {
+    return false; // cannot unalias unless there are precise offsets
+  }
+
+  if (alloc == NULL) {
+    return false; // No allocation found
+  }
+
+  intptr_t size_in_bytes = type2aelembytes(bt);
+
+  Node* mem = __ memory(adr_idx); // start searching here...
+
+  for (int cnt = 0; cnt < 50; cnt++) {
+
+    if (mem->is_Store()) {
+
+      Node* st_adr = mem->in(MemNode::Address);
+      intptr_t st_offset = 0;
+      Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
+
+      if (st_base == NULL) {
+        break; // inscrutable pointer
+      }
+
+      // Break we have found a store with same base and offset as ours so break
+      if (st_base == base && st_offset == offset) {
+        break;
+      }
+
+      if (st_offset != offset && st_offset != Type::OffsetBot) {
+        const int MAX_STORE = BytesPerLong;
+        if (st_offset >= offset + size_in_bytes ||
+            st_offset <= offset - MAX_STORE ||
+            st_offset <= offset - mem->as_Store()->memory_size()) {
+          // Success:  The offsets are provably independent.
+          // (You may ask, why not just test st_offset != offset and be done?
+          // The answer is that stores of different sizes can co-exist
+          // in the same sequence of RawMem effects.  We sometimes initialize
+          // a whole 'tile' of array elements with a single jint or jlong.)
+          mem = mem->in(MemNode::Memory);
+          continue; // advance through independent store memory
+        }
+      }
+
+      if (st_base != base
+          && MemNode::detect_ptr_independence(base, alloc, st_base,
+                                              AllocateNode::Ideal_allocation(st_base, phase),
+                                              phase)) {
+        // Success:  The bases are provably independent.
+        mem = mem->in(MemNode::Memory);
+        continue; // advance through independent store memory
+      }
+    } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
+
+      InitializeNode* st_init = mem->in(0)->as_Initialize();
+      AllocateNode* st_alloc = st_init->allocation();
+
+      // Make sure that we are looking at the same allocation site.
+      // The alloc variable is guaranteed to not be null here from earlier check.
+      if (alloc == st_alloc) {
+        // Check that the initialization is storing NULL so that no previous store
+        // has been moved up and directly write a reference
+        Node* captured_store = st_init->find_captured_store(offset,
+                                                            type2aelembytes(T_OBJECT),
+                                                            phase);
+        if (captured_store == NULL || captured_store == st_init->zero_memory()) {
+          return true;
+        }
+      }
+    }
+
+    // Unless there is an explicit 'continue', we must bail out here,
+    // because 'mem' is an inscrutable memory state (e.g., a call).
+    break;
+  }
+
+  return false;
+}
+
+#undef __
+#define __ ideal.
+
+void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,
+                                                    bool do_load,
+                                                    Node* obj,
+                                                    Node* adr,
+                                                    uint alias_idx,
+                                                    Node* val,
+                                                    const TypeOopPtr* val_type,
+                                                    Node* pre_val,
+                                                    BasicType bt) const {
+  // Some sanity checks
+  // Note: val is unused in this routine.
+
+  if (do_load) {
+    // We need to generate the load of the previous value
+    assert(obj != NULL, "must have a base");
+    assert(adr != NULL, "where are loading from?");
+    assert(pre_val == NULL, "loaded already?");
+    assert(val_type != NULL, "need a type");
+
+    if (ReduceInitialCardMarks
+        && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
+      return;
+    }
+
+  } else {
+    // In this case both val_type and alias_idx are unused.
+    assert(pre_val != NULL, "must be loaded already");
+    // Nothing to be done if pre_val is null.
+    if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
+    assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
+  }
+  assert(bt == T_OBJECT, "or we shouldn't be here");
+
+  IdealKit ideal(kit, true);
+
+  Node* tls = __ thread(); // ThreadLocalStorage
+
+  Node* no_base = __ top();
+  Node* zero  = __ ConI(0);
+  Node* zeroX = __ ConX(0);
+
+  float likely  = PROB_LIKELY(0.999);
+  float unlikely  = PROB_UNLIKELY(0.999);
+
+  // Offsets into the thread
+  const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
+  const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
+
+  // Now the actual pointers into the thread
+  Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
+  Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
+
+  // Now some of the values
+  Node* marking;
+  Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
+  Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
+  marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
+  assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
+
+  // if (!marking)
+  __ if_then(marking, BoolTest::ne, zero, unlikely); {
+    BasicType index_bt = TypeX_X->basic_type();
+    assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
+    Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
+
+    if (do_load) {
+      // load original value
+      // alias_idx correct??
+      pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
+    }
+
+    // if (pre_val != NULL)
+    __ if_then(pre_val, BoolTest::ne, kit->null()); {
+      Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
+
+      // is the queue for this thread full?
+      __ if_then(index, BoolTest::ne, zeroX, likely); {
+
+        // decrement the index
+        Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
+
+        // Now get the buffer location we will log the previous value into and store it
+        Node *log_addr = __ AddP(no_base, buffer, next_index);
+        __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
+        // update the index
+        __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
+
+      } __ else_(); {
+
+        // logging buffer is full, call the runtime
+        const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();
+        __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);
+      } __ end_if();  // (!index)
+    } __ end_if();  // (pre_val != NULL)
+  } __ end_if();  // (!marking)
+
+  // Final sync IdealKit and GraphKit.
+  kit->final_sync(ideal);
+
+  if (ShenandoahSATBBarrier && adr != NULL) {
+    Node* c = kit->control();
+    Node* call = c->in(1)->in(1)->in(1)->in(0);
+    assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
+    call->add_req(adr);
+  }
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
+  return call->is_CallLeaf() &&
+         call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
+  if (!call->is_CallLeaf()) {
+    return false;
+  }
+
+  address entry_point = call->as_CallLeaf()->entry_point();
+  return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)) ||
+         (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
+  if (n->Opcode() != Op_If) {
+    return false;
+  }
+
+  Node* bol = n->in(1);
+  assert(bol->is_Bool(), "");
+  Node* cmpx = bol->in(1);
+  if (bol->as_Bool()->_test._test == BoolTest::ne &&
+      cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
+      is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
+      cmpx->in(1)->in(2)->is_Con() &&
+      cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
+    return true;
+  }
+
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
+  if (!n->is_Load()) return false;
+  const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
+  return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
+         && n->in(2)->in(3)->is_Con()
+         && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
+}
+
+void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
+                                                          bool do_load,
+                                                          Node* obj,
+                                                          Node* adr,
+                                                          uint alias_idx,
+                                                          Node* val,
+                                                          const TypeOopPtr* val_type,
+                                                          Node* pre_val,
+                                                          BasicType bt) const {
+  if (ShenandoahSATBBarrier) {
+    IdealKit ideal(kit);
+    kit->sync_kit(ideal);
+
+    satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
+
+    ideal.sync_kit(kit);
+    kit->final_sync(ideal);
+  }
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {
+  return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));
+}
+
+// Helper that guards and inserts a pre-barrier.
+void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
+                                                Node* pre_val, bool need_mem_bar) const {
+  // We could be accessing the referent field of a reference object. If so, when G1
+  // is enabled, we need to log the value in the referent field in an SATB buffer.
+  // This routine performs some compile time filters and generates suitable
+  // runtime filters that guard the pre-barrier code.
+  // Also add memory barrier for non volatile load from the referent field
+  // to prevent commoning of loads across safepoint.
+
+  // Some compile time checks.
+
+  // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
+  const TypeX* otype = offset->find_intptr_t_type();
+  if (otype != NULL && otype->is_con() &&
+      otype->get_con() != java_lang_ref_Reference::referent_offset) {
+    // Constant offset but not the reference_offset so just return
+    return;
+  }
+
+  // We only need to generate the runtime guards for instances.
+  const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
+  if (btype != NULL) {
+    if (btype->isa_aryptr()) {
+      // Array type so nothing to do
+      return;
+    }
+
+    const TypeInstPtr* itype = btype->isa_instptr();
+    if (itype != NULL) {
+      // Can the klass of base_oop be statically determined to be
+      // _not_ a sub-class of Reference and _not_ Object?
+      ciKlass* klass = itype->klass();
+      if ( klass->is_loaded() &&
+          !klass->is_subtype_of(kit->env()->Reference_klass()) &&
+          !kit->env()->Object_klass()->is_subtype_of(klass)) {
+        return;
+      }
+    }
+  }
+
+  // The compile time filters did not reject base_oop/offset so
+  // we need to generate the following runtime filters
+  //
+  // if (offset == java_lang_ref_Reference::_reference_offset) {
+  //   if (instance_of(base, java.lang.ref.Reference)) {
+  //     pre_barrier(_, pre_val, ...);
+  //   }
+  // }
+
+  float likely   = PROB_LIKELY(  0.999);
+  float unlikely = PROB_UNLIKELY(0.999);
+
+  IdealKit ideal(kit);
+
+  Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
+
+  __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
+      // Update graphKit memory and control from IdealKit.
+      kit->sync_kit(ideal);
+
+      Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
+      Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
+
+      // Update IdealKit memory and control from graphKit.
+      __ sync_kit(kit);
+
+      Node* one = __ ConI(1);
+      // is_instof == 0 if base_oop == NULL
+      __ if_then(is_instof, BoolTest::eq, one, unlikely); {
+
+        // Update graphKit from IdeakKit.
+        kit->sync_kit(ideal);
+
+        // Use the pre-barrier to record the value in the referent field
+        satb_write_barrier_pre(kit, false /* do_load */,
+                               NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
+                               pre_val /* pre_val */,
+                               T_OBJECT);
+        if (need_mem_bar) {
+          // Add memory barrier to prevent commoning reads from this field
+          // across safepoint since GC can change its value.
+          kit->insert_mem_bar(Op_MemBarCPUOrder);
+        }
+        // Update IdealKit from graphKit.
+        __ sync_kit(kit);
+
+      } __ end_if(); // _ref_type != ref_none
+  } __ end_if(); // offset == referent_offset
+
+  // Final sync IdealKit and GraphKit.
+  kit->final_sync(ideal);
+}
+
+#undef __
+
+const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
+  const Type **fields = TypeTuple::fields(2);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() {
+  const Type **fields = TypeTuple::fields(2);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM;   // original load address
+
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
+  DecoratorSet decorators = access.decorators();
+
+  const TypePtr* adr_type = access.addr().type();
+  Node* adr = access.addr().node();
+
+  bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+  bool on_heap = (decorators & IN_HEAP) != 0;
+
+  if (!access.is_oop() || (!on_heap && !anonymous)) {
+    return BarrierSetC2::store_at_resolved(access, val);
+  }
+
+  GraphKit* kit = access.kit();
+
+  uint adr_idx = kit->C->get_alias_index(adr_type);
+  assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
+  Node* value = val.node();
+  value = shenandoah_storeval_barrier(kit, value);
+  val.set_node(value);
+  shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
+                               static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
+  return BarrierSetC2::store_at_resolved(access, val);
+}
+
+Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
+  // 1: non-reference load, no additional barrier is needed
+  if (!access.is_oop()) {
+    return BarrierSetC2::load_at_resolved(access, val_type);;
+  }
+
+  Node* load = BarrierSetC2::load_at_resolved(access, val_type);
+  DecoratorSet decorators = access.decorators();
+  BasicType type = access.type();
+
+  // 2: apply LRB if needed
+  if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
+    load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
+    load = access.kit()->gvn().transform(load);
+  }
+
+  // 3: apply keep-alive barrier if needed
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    Node* top = Compile::current()->top();
+    Node* adr = access.addr().node();
+    Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
+    Node* obj = access.base();
+
+    bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+    bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
+    bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
+
+    // If we are reading the value of the referent field of a Reference
+    // object (either by using Unsafe directly or through reflection)
+    // then, if SATB is enabled, we need to record the referent in an
+    // SATB log buffer using the pre-barrier mechanism.
+    // Also we need to add memory barrier to prevent commoning reads
+    // from this field across safepoint since GC can change its value.
+    if (!on_weak_ref || (unknown && (offset == top || obj == top)) || !keep_alive) {
+      return load;
+    }
+    GraphKit* kit = access.kit();
+    bool mismatched = (decorators & C2_MISMATCHED) != 0;
+    bool is_unordered = (decorators & MO_UNORDERED) != 0;
+    bool need_cpu_mem_bar = !is_unordered || mismatched;
+
+    if (on_weak_ref) {
+      // Use the pre-barrier to record the value in the referent field
+      satb_write_barrier_pre(kit, false /* do_load */,
+                             NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
+                             load /* pre_val */, T_OBJECT);
+      // Add memory barrier to prevent commoning reads from this field
+      // across safepoint since GC can change its value.
+      kit->insert_mem_bar(Op_MemBarCPUOrder);
+    } else if (unknown) {
+      // We do not require a mem bar inside pre_barrier if need_mem_bar
+      // is set: the barriers would be emitted by us.
+      insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
+    }
+  }
+
+  return load;
+}
+
+static void pin_atomic_op(C2AtomicAccess& access) {
+  if (!access.needs_pinning()) {
+    return;
+  }
+  // SCMemProjNodes represent the memory state of a LoadStore. Their
+  // main role is to prevent LoadStore nodes from being optimized away
+  // when their results aren't used.
+  GraphKit* kit = access.kit();
+  Node* load_store = access.raw_access();
+  assert(load_store != NULL, "must pin atomic op");
+  Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
+  kit->set_memory(proj, access.alias_idx());
+}
+
+Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+                                                   Node* new_val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    new_val = shenandoah_storeval_barrier(kit, new_val);
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 expected_val /* pre_val */, T_OBJECT);
+
+    MemNode::MemOrd mo = access.mem_node_mo();
+    Node* mem = access.memory();
+    Node* adr = access.addr().node();
+    const TypePtr* adr_type = access.addr().type();
+    Node* load_store = NULL;
+
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
+      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
+      if (ShenandoahCASBarrier) {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+      } else {
+        load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+      }
+    } else
+#endif
+    {
+      if (ShenandoahCASBarrier) {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+      } else {
+        load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+      }
+    }
+
+    access.set_raw_access(load_store);
+    pin_atomic_op(access);
+
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
+    }
+#endif
+    load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
+    return load_store;
+  }
+  return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
+}
+
+Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+                                                              Node* new_val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    new_val = shenandoah_storeval_barrier(kit, new_val);
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 expected_val /* pre_val */, T_OBJECT);
+    DecoratorSet decorators = access.decorators();
+    MemNode::MemOrd mo = access.mem_node_mo();
+    Node* mem = access.memory();
+    bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
+    Node* load_store = NULL;
+    Node* adr = access.addr().node();
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
+      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
+      if (ShenandoahCASBarrier) {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        } else {
+          load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        }
+      } else {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        } else {
+          load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        }
+      }
+    } else
+#endif
+    {
+      if (ShenandoahCASBarrier) {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        } else {
+          load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        }
+      } else {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        } else {
+          load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        }
+      }
+    }
+    access.set_raw_access(load_store);
+    pin_atomic_op(access);
+    return load_store;
+  }
+  return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
+}
+
+Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    val = shenandoah_storeval_barrier(kit, val);
+  }
+  Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
+  if (access.is_oop()) {
+    result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 result /* pre_val */, T_OBJECT);
+  }
+  return result;
+}
+
+// Support for GC barriers emitted during parsing
+bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
+  if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
+    return false;
+  }
+  CallLeafNode *call = node->as_CallLeaf();
+  if (call->_name == NULL) {
+    return false;
+  }
+
+  return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
+         strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
+         strcmp(call->_name, "shenandoah_wb_pre") == 0;
+}
+
+Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
+  if (c == NULL) {
+    return c;
+  }
+  if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+  }
+  if (c->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    c = c->in(1);
+  }
+  return c;
+}
+
+bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
+  return !ShenandoahBarrierC2Support::expand(C, igvn);
+}
+
+bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
+  if (mode == LoopOptsShenandoahExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    ShenandoahBarrierC2Support::pin_and_expand(phase);
+    return true;
+  } else if (mode == LoopOptsShenandoahPostExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    visited.Clear();
+    ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase);
+    return true;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const {
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
+  const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
+  if (src_type->isa_instptr() != NULL) {
+    ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
+    if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
+      if (ik->has_object_fields()) {
+        return true;
+      } else {
+        if (!src_type->klass_is_exact()) {
+          Compile::current()->dependencies()->assert_leaf_type(ik);
+        }
+      }
+    } else {
+      return true;
+        }
+  } else if (src_type->isa_aryptr()) {
+    BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
+    if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
+      return true;
+    }
+  } else {
+    return true;
+  }
+  return false;
+}
+
+void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
+  Node* ctrl = ac->in(TypeFunc::Control);
+  Node* mem = ac->in(TypeFunc::Memory);
+  Node* src = ac->in(ArrayCopyNode::Src);
+  Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+  Node* dest = ac->in(ArrayCopyNode::Dest);
+  Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+  Node* length = ac->in(ArrayCopyNode::Length);
+  assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+  assert (src->is_AddP(), "for clone the src should be the interior ptr");
+  assert (dest->is_AddP(), "for clone the dst should be the interior ptr");
+
+  if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
+    // Check if heap is has forwarded objects. If it does, we need to call into the special
+    // routine that would fix up source references before we can continue.
+
+    enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    Node* thread = phase->transform_later(new ThreadLocalNode());
+    Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+    Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
+
+    uint gc_state_idx = Compile::AliasIdxRaw;
+    const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
+    debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
+
+    Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
+    int flags = ShenandoahHeap::HAS_FORWARDED;
+    if (ShenandoahStoreValEnqueueBarrier) {
+      flags |= ShenandoahHeap::MARKING;
+    }
+    Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
+    Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
+    Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
+
+    IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
+    Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
+    Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
+
+    // Heap is stable, no need to do anything additional
+    region->init_req(_heap_stable, stable_ctrl);
+    mem_phi->init_req(_heap_stable, mem);
+
+    // Heap is unstable, call into clone barrier stub
+    Node* call = phase->make_leaf_call(unstable_ctrl, mem,
+                    ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
+                    CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
+                    "shenandoah_clone",
+                    TypeRawPtr::BOTTOM,
+                    src->in(AddPNode::Base));
+    call = phase->transform_later(call);
+
+    ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
+    mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
+    region->init_req(_heap_unstable, ctrl);
+    mem_phi->init_req(_heap_unstable, mem);
+
+    // Wire up the actual arraycopy stub now
+    ctrl = phase->transform_later(region);
+    mem = phase->transform_later(mem_phi);
+
+    const char* name = "arraycopy";
+    call = phase->make_leaf_call(ctrl, mem,
+                                 OptoRuntime::fast_arraycopy_Type(),
+                                 phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
+                                 name, TypeRawPtr::BOTTOM,
+                                 src, dest, length
+                                 LP64_ONLY(COMMA phase->top()));
+    call = phase->transform_later(call);
+
+    // Hook up the whole thing into the graph
+    phase->igvn().replace_node(ac, call);
+  } else {
+    BarrierSetC2::clone_at_expansion(phase, ac);
+  }
+}
+
+// Support for macro expanded GC barriers
+void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+  }
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
+  }
+}
+
+void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+  }
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
+  }
+}
+
+void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
+  if (is_shenandoah_wb_pre_call(n)) {
+    shenandoah_eliminate_wb_pre(n, &macro->igvn());
+  }
+}
+
+void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
+  assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
+  Node* c = call->as_Call()->proj_out(TypeFunc::Control);
+  c = c->unique_ctrl_out();
+  assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+  c = c->unique_ctrl_out();
+  assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+  Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
+  assert(iff->is_If(), "expect test");
+  if (!is_shenandoah_marking_if(igvn, iff)) {
+    c = c->unique_ctrl_out();
+    assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+    iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
+    assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
+  }
+  Node* cmpx = iff->in(1)->in(1);
+  igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
+  igvn->rehash_node_delayed(call);
+  call->del_req(call->req()-1);
+}
+
+void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {
+  if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {
+    for (DUIterator_Fast imax, i = node->fast_outs(imax); i < imax; i++) {
+      Node* use = node->fast_out(i);
+      worklist.push(use);
+    }
+  }
+}
+
+void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
+  for (uint i = 0; i < useful.size(); i++) {
+    Node* n = useful.at(i);
+    if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {
+      for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+        Compile::current()->record_for_igvn(n->fast_out(i));
+      }
+    }
+  }
+  for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i);
+    if (!useful.member(n)) {
+      state()->remove_enqueue_barrier(n);
+    }
+  }
+  for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
+    if (!useful.member(n)) {
+      state()->remove_load_reference_barrier(n);
+    }
+  }
+}
+
+void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
+
+void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+  return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
+}
+
+ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
+  return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
+}
+
+// If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
+// expanded later, then now is the time to do so.
+bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
+
+#ifdef ASSERT
+void ShenandoahBarrierSetC2::verify_gc_barriers(bool post_parse) const {
+  if (ShenandoahVerifyOptoBarriers && !post_parse) {
+    ShenandoahBarrierC2Support::verify(Compile::current()->root());
+  }
+}
+#endif
+
+Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
+  if (is_shenandoah_wb_pre_call(n)) {
+    uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
+    if (n->req() > cnt) {
+      Node* addp = n->in(cnt);
+      if (has_only_shenandoah_wb_pre_uses(addp)) {
+        n->del_req(cnt);
+        if (can_reshape) {
+          phase->is_IterGVN()->_worklist.push(addp);
+        }
+        return n;
+      }
+    }
+  }
+  if (n->Opcode() == Op_CmpP) {
+    Node* in1 = n->in(1);
+    Node* in2 = n->in(2);
+    if (in1->bottom_type() == TypePtr::NULL_PTR) {
+      in2 = step_over_gc_barrier(in2);
+    }
+    if (in2->bottom_type() == TypePtr::NULL_PTR) {
+      in1 = step_over_gc_barrier(in1);
+    }
+    PhaseIterGVN* igvn = phase->is_IterGVN();
+    if (in1 != n->in(1)) {
+      if (igvn != NULL) {
+        n->set_req_X(1, in1, igvn);
+      } else {
+        n->set_req(1, in1);
+      }
+      assert(in2 == n->in(2), "only one change");
+      return n;
+    }
+    if (in2 != n->in(2)) {
+      if (igvn != NULL) {
+        n->set_req_X(2, in2, igvn);
+      } else {
+        n->set_req(2, in2);
+      }
+      return n;
+    }
+  } else if (can_reshape &&
+             n->Opcode() == Op_If &&
+             ShenandoahBarrierC2Support::is_heap_stable_test(n) &&
+             n->in(0) != NULL) {
+    Node* dom = n->in(0);
+    Node* prev_dom = n;
+    int op = n->Opcode();
+    int dist = 16;
+    // Search up the dominator tree for another heap stable test
+    while (dom->Opcode() != op    ||  // Not same opcode?
+           !ShenandoahBarrierC2Support::is_heap_stable_test(dom) ||  // Not same input 1?
+           prev_dom->in(0) != dom) {  // One path of test does not dominate?
+      if (dist < 0) return NULL;
+
+      dist--;
+      prev_dom = dom;
+      dom = IfNode::up_one_dom(dom);
+      if (!dom) return NULL;
+    }
+
+    // Check that we did not follow a loop back to ourselves
+    if (n == dom) {
+      return NULL;
+    }
+
+    return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN());
+  }
+  return NULL;
+}
+
+bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
+  for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+    Node* u = n->fast_out(i);
+    if (!is_shenandoah_wb_pre_call(u)) {
+      return false;
+    }
+  }
+  return n->outcnt() > 0;
+}
+
+Node* ShenandoahBarrierSetC2::arraycopy_load_reference_barrier(PhaseGVN *phase, Node* v) {
+  if (ShenandoahLoadRefBarrier) {
+    return phase->transform(new ShenandoahLoadReferenceBarrierNode(NULL, v));
+  }
+  if (ShenandoahStoreValEnqueueBarrier) {
+    return phase->transform(new ShenandoahEnqueueBarrierNode(v));
+  }
+  return v;
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
+#define SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
+
+#include "gc/shared/c2/barrierSetC2.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "utilities/growableArray.hpp"
+
+class ShenandoahBarrierSetC2State : public ResourceObj {
+private:
+  GrowableArray<ShenandoahEnqueueBarrierNode*>* _enqueue_barriers;
+  GrowableArray<ShenandoahLoadReferenceBarrierNode*>* _load_reference_barriers;
+
+public:
+  ShenandoahBarrierSetC2State(Arena* comp_arena);
+
+  int enqueue_barriers_count() const;
+  ShenandoahEnqueueBarrierNode* enqueue_barrier(int idx) const;
+  void add_enqueue_barrier(ShenandoahEnqueueBarrierNode* n);
+  void remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n);
+
+  int load_reference_barriers_count() const;
+  ShenandoahLoadReferenceBarrierNode* load_reference_barrier(int idx) const;
+  void add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode* n);
+  void remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n);
+};
+
+class ShenandoahBarrierSetC2 : public BarrierSetC2 {
+private:
+  void shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const;
+
+  bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
+                                   BasicType bt, uint adr_idx) const;
+  void satb_write_barrier_pre(GraphKit* kit, bool do_load,
+                              Node* obj,
+                              Node* adr,
+                              uint alias_idx,
+                              Node* val,
+                              const TypeOopPtr* val_type,
+                              Node* pre_val,
+                              BasicType bt) const;
+
+  void shenandoah_write_barrier_pre(GraphKit* kit,
+                                    bool do_load,
+                                    Node* obj,
+                                    Node* adr,
+                                    uint alias_idx,
+                                    Node* val,
+                                    const TypeOopPtr* val_type,
+                                    Node* pre_val,
+                                    BasicType bt) const;
+
+  Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const;
+  Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const;
+
+  void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
+                          Node* pre_val, bool need_mem_bar) const;
+
+  static bool clone_needs_barrier(Node* src, PhaseGVN& gvn);
+
+protected:
+  virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
+  virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
+  virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+                                               Node* new_val, const Type* val_type) const;
+  virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+                                                Node* new_val, const Type* value_type) const;
+  virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const;
+
+public:
+  static ShenandoahBarrierSetC2* bsc2();
+
+  static bool is_shenandoah_wb_pre_call(Node* call);
+  static bool is_shenandoah_lrb_call(Node* call);
+  static bool is_shenandoah_marking_if(PhaseTransform *phase, Node* n);
+  static bool is_shenandoah_state_load(Node* n);
+  static bool has_only_shenandoah_wb_pre_uses(Node* n);
+
+  ShenandoahBarrierSetC2State* state() const;
+
+  static const TypeFunc* write_ref_field_pre_entry_Type();
+  static const TypeFunc* shenandoah_clone_barrier_Type();
+  static const TypeFunc* shenandoah_load_reference_barrier_Type();
+  virtual bool has_load_barriers() const { return true; }
+
+  // This is the entry-point for the backend to perform accesses through the Access API.
+  virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
+
+  // These are general helper methods used by C2
+  virtual bool array_copy_requires_gc_barriers(BasicType type) const;
+
+  // Support for GC barriers emitted during parsing
+  virtual bool is_gc_barrier_node(Node* node) const;
+  virtual Node* step_over_gc_barrier(Node* c) const;
+  virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const;
+  virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const;
+  virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; }
+  virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; }
+
+  // Support for macro expanded GC barriers
+  virtual void register_potential_barrier_node(Node* node) const;
+  virtual void unregister_potential_barrier_node(Node* node) const;
+  virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
+  virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const;
+  virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const;
+  virtual void add_users_to_worklist(Unique_Node_List* worklist) const;
+
+  // Allow barrier sets to have shared state that is preserved across a compilation unit.
+  // This could for example comprise macro nodes to be expanded during macro expansion.
+  virtual void* create_barrier_state(Arena* comp_arena) const;
+  // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
+  // expanded later, then now is the time to do so.
+  virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const;
+
+#ifdef ASSERT
+  virtual void verify_gc_barriers(bool post_parse) const;
+#endif
+
+  virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const;
+
+  Node* arraycopy_load_reference_barrier(PhaseGVN *phase, Node* v);
+
+};
+
+#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Wed Jul 29 17:15:58 2020 +0200
@@ -0,0 +1,3381 @@
+/*
+ * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahForwarding.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "opto/arraycopynode.hpp"
+#include "opto/block.hpp"
+#include "opto/callnode.hpp"
+#include "opto/castnode.hpp"
+#include "opto/movenode.hpp"
+#include "opto/phaseX.hpp"
+#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
+#include "opto/subnode.hpp"
+
+bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
+  ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
+  if ((state->enqueue_barriers_count() +
+       state->load_reference_barriers_count()) > 0) {
+    bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
+    C->clear_major_progress();
+    PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
+    if (C->failing()) return false;
+    PhaseIdealLoop::verify(igvn);
+    DEBUG_ONLY(verify_raw_mem(C->root());)
+    if (attempt_more_loopopts) {
+      C->set_major_progress();
+      int cnt = 0;
+      if (!C->optimize_loops(cnt, igvn, LoopOptsShenandoahPostExpand)) {
+        return false;
+      }
+      C->clear_major_progress();
+      if (C->range_check_cast_count() > 0) {
+        // No more loop optimizations. Remove all range check dependent CastIINodes.
+        C->remove_range_check_casts(igvn);
+        igvn.optimize();
+      }
+    }
+  }
+  return true;
+}
+
+bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) {
+  if (!UseShenandoahGC) {
+    return false;
+  }
+  assert(iff->is_If(), "bad input");
+  if (iff->Opcode() != Op_If) {
+    return false;
+  }
+  Node* bol = iff->in(1);
+  if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
+    return false;
+  }
+  Node* cmp = bol->in(1);
+  if (cmp->Opcode() != Op_CmpI) {
+    return false;
+  }
+  Node* in1 = cmp->in(1);
+  Node* in2 = cmp->in(2);
+  if (in2->find_int_con(-1) != 0) {
+    return false;
+  }
+  if (in1->Opcode() != Op_AndI) {
+    return false;
+  }
+  in2 = in1->in(2);
+  if (in2->find_int_con(-1) != mask) {
+    return false;
+  }
+  in1 = in1->in(1);
+
+  return is_gc_state_load(in1);
+}
+
+bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) {
+  return is_gc_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
+}
+
+bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) {
+  if (!UseShenandoahGC) {
+    return false;
+  }
+  if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {
+    return false;
+  }
+  Node* addp = n->in(MemNode::Address);
+  if (!addp->is_AddP()) {
+    return false;
+  }
+  Node* base = addp->in(AddPNode::Address);
+  Node* off = addp->in(AddPNode::Offset);
+  if (base->Opcode() != Op_ThreadLocal) {
+    return false;
+  }
+  if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {
+    return false;
+  }
+  return true;
+}
+
+bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
+  assert(phase->is_dominator(stop, start), "bad inputs");
+  ResourceMark rm;
+  Unique_Node_List wq;
+  wq.push(start);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *m = wq.at(next);
+    if (m == stop) {
+      continue;
+    }
+    if (m->is_SafePoint() && !m->is_CallLeaf()) {
+      return true;
+    }
+    if (m->is_Region()) {
+      for (uint i = 1; i < m->req(); i++) {
+        wq.push(m->in(i));
+      }
+    } else {
+      wq.push(m->in(0));
+    }
+  }
+  return false;
+}
+
+#ifdef ASSERT
+bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
+  assert(phis.size() == 0, "");
+
+  while (true) {
+    if (in->bottom_type() == TypePtr::NULL_PTR) {
+      if (trace) {tty->print_cr("NULL");}
+    } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
+      if (trace) {tty->print_cr("Non oop");}
+    } else if (in->bottom_type()->make_ptr()->make_oopptr() == TypeInstPtr::MIRROR) {
+      if (trace) {tty->print_cr("Java mirror");}
+    } else {
+      if (in->is_ConstraintCast()) {
+        in = in->in(1);
+        continue;
+      } else if (in->is_AddP()) {
+        assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");
+        in = in->in(AddPNode::Address);
+        continue;
+      } else if (in->is_Con()) {
+        if (trace) {
+          tty->print("Found constant");
+          in->dump();
+        }
+      } else if (in->Opcode() == Op_Parm) {
+        if (trace) {
+          tty->print("Found argument");
+        }
+      } else if (in->Opcode() == Op_CreateEx) {
+        if (trace) {
+          tty->print("Found create-exception");
+        }
+      } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
+        if (trace) {
+          tty->print("Found raw LoadP (OSR argument?)");
+        }
+      } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+        if (t == ShenandoahOopStore) {
+          uint i = 0;
+          for (; i < phis.size(); i++) {
+            Node* n = phis.node_at(i);
+            if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+              break;
+            }
+          }
+          if (i == phis.size()) {
+            return false;
+          }
+        }
+        barriers_used.push(in);
+        if (trace) {tty->print("Found barrier"); in->dump();}
+      } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {
+        if (t != ShenandoahOopStore) {
+          in = in->in(1);
+          continue;
+        }
+        if (trace) {tty->print("Found enqueue barrier"); in->dump();}
+        phis.push(in, in->req());
+        in = in->in(1);
+        continue;
+      } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
+        if (trace) {
+          tty->print("Found alloc");
+          in->in(0)->dump();
+        }
+      } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
+        if (trace) {
+          tty->print("Found Java call");
+        }
+      } else if (in->is_Phi()) {
+        if (!visited.test_set(in->_idx)) {
+          if (trace) {tty->print("Pushed phi:"); in->dump();}
+          phis.push(in, 2);
+          in = in->in(1);
+          continue;
+        }
+        if (trace) {tty->print("Already seen phi:"); in->dump();}
+      } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
+        if (!visited.test_set(in->_idx)) {
+          if (trace) {tty->print("Pushed cmovep:"); in->dump();}
+          phis.push(in, CMoveNode::IfTrue);
+          in = in->in(CMoveNode::IfFalse);
+          continue;
+        }
+        if (trace) {tty->print("Already seen cmovep:"); in->dump();}
+      } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {
+        in = in->in(1);
+        continue;
+      } else {
+        return false;
+      }
+    }
+    bool cont = false;
+    while (phis.is_nonempty()) {
+      uint idx = phis.index();
+      Node* phi = phis.node();
+      if (idx >= phi->req()) {
+        if (trace) {tty->print("Popped phi:"); phi->dump();}
+        phis.pop();
+        continue;
+      }
+      if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}
+      in = phi->in(idx);
+      phis.set_index(idx+1);
+      cont = true;
+      break;
+    }
+    if (!cont) {
+      break;
+    }
+  }
+  return true;
+}
+
+void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) {
+  if (n1 != NULL) {
+    n1->dump(+10);
+  }
+  if (n2 != NULL) {
+    n2->dump(+10);
+  }
+  fatal("%s", msg);
+}
+
+void ShenandoahBarrierC2Support::verify(RootNode* root) {
+  ResourceMark rm;
+  Unique_Node_List wq;
+  GrowableArray<Node*> barriers;
+  Unique_Node_List barriers_used;
+  Node_Stack phis(0);
+  VectorSet visited(Thread::current()->resource_area());
+  const bool trace = false;
+  const bool verify_no_useless_barrier = false;
+
+  wq.push(root);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *n = wq.at(next);
+    if (n->is_Load()) {
+      const bool trace = false;
+      if (trace) {tty->print("Verifying"); n->dump();}
+      if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {
+        if (trace) {tty->print_cr("Load range/klass");}
+      } else {
+        const TypePtr* adr_type = n->as_Load()->adr_type();
+
+        if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) {
+          if (trace) {tty->print_cr("Mark load");}
+        } else if (adr_type->isa_instptr() &&
+                   adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
+                   adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {
+          if (trace) {tty->print_cr("Reference.get()");}
+        } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: Load should have barriers", n);
+        }
+      }
+    } else if (n->is_Store()) {
+      const bool trace = false;
+
+      if (trace) {tty->print("Verifying"); n->dump();}
+      if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
+        Node* adr = n->in(MemNode::Address);
+        bool verify = true;
+
+        if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
+          adr = adr->in(AddPNode::Address);
+          if (adr->is_AddP()) {
+            assert(adr->in(AddPNode::Base)->is_top(), "");
+            adr = adr->in(AddPNode::Address);
+            if (adr->Opcode() == Op_LoadP &&
+                adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
+                adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
+                adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
+              if (trace) {tty->print_cr("SATB prebarrier");}
+              verify = false;
+            }
+          }
+        }
+
+        if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: Store should have barriers", n);
+        }
+      }
+      if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
+      }
+    } else if (n->Opcode() == Op_CmpP) {
+      const bool trace = false;
+
+      Node* in1 = n->in(1);
+      Node* in2 = n->in(2);
+      if (in1->bottom_type()->isa_oopptr()) {
+        if (trace) {tty->print("Verifying"); n->dump();}
+
+        bool mark_inputs = false;
+        if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
+            (in1->is_Con() || in2->is_Con())) {
+          if (trace) {tty->print_cr("Comparison against a constant");}
+          mark_inputs = true;
+        } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
+                   (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {
+          if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
+          mark_inputs = true;
+        } else {
+          assert(in2->bottom_type()->isa_oopptr(), "");
+
+          if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
+              !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
+            report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
+          }
+        }
+        if (verify_no_useless_barrier &&
+            mark_inputs &&
+            (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
+             !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
+          phis.clear();
+          visited.Reset();
+        }
+      }
+    } else if (n->is_LoadStore()) {
+      if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
+          !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
+      }
+
+      if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
+      }
+    } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
+      CallNode* call = n->as_Call();
+
+      static struct {
+        const char* name;
+        struct {
+          int pos;
+          verify_type t;
+        } args[6];
+      } calls[] = {
+        "aescrypt_encryptBlock",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "aescrypt_decryptBlock",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "multiplyToLen",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { TypeFunc::Parms+4, ShenandoahStore },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "squareToLen",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "montgomery_multiply",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+6, ShenandoahStore }, { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "montgomery_square",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+5, ShenandoahStore },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "mulAdd",
+        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "vectorizedMismatch",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesCRC32",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesAdler32",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesCRC32C",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad},    { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "counterMode_AESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
+        "cipherBlockChaining_encryptAESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "cipherBlockChaining_decryptAESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "shenandoah_clone_barrier",
+        { { TypeFunc::Parms, ShenandoahLoad },   { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "ghash_processBlocks",
+        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha1_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha256_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha512_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha1_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha256_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha512_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "encodeBlock",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+      };
+
+      if (call->is_call_to_arraycopystub()) {
+        Node* dest = NULL;
+        const TypeTuple* args = n->as_Call()->_tf->domain();
+        for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
+          if (args->field_at(i)->isa_ptr()) {
+            j++;
+            if (j == 2) {
+              dest = n->in(i);
+              break;
+            }
+          }
+        }
+        if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
+            !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
+        }
+      } else if (strlen(call->_name) > 5 &&
+                 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
+        if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: _fill should have barriers", n);
+        }
+      } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
+        // skip
+      } else {
+        const int calls_len = sizeof(calls) / sizeof(calls[0]);
+        int i = 0;
+        for (; i < calls_len; i++) {
+          if (!strcmp(calls[i].name, call->_name)) {
+            break;
+          }
+        }
+        if (i != calls_len) {
+          const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);
+          for (uint j = 0; j < args_len; j++) {
+            int pos = calls[i].args[j].pos;
+            if (pos == -1) {
+              break;
+            }
+            if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
+              report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
+            }
+          }
+          for (uint j = TypeFunc::Parms; j < call->req(); j++) {
+            if (call->in(j)->bottom_type()->make_ptr() &&
+                call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
+              uint k = 0;
+              for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
+              if (k == args_len) {
+                fatal("arg %d for call %s not covered", j, call->_name);
+              }
+            }
+          }
+        } else {
+          for (uint j = TypeFunc::Parms; j < call->req(); j++) {
+            if (call->in(j)->bottom_type()->make_ptr() &&
+                call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
+              fatal("%s not covered", call->_name);
+            }
+          }
+        }
+      }
+    } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+      // skip
+    } else if (n->is_AddP()
+               || n->is_Phi()
+               || n->is_ConstraintCast()
+               || n->Opcode() == Op_Return
+               || n->Opcode() == Op_CMoveP
+               || n->Opcode() == Op_CMoveN
+               || n->Opcode() == Op_Rethrow
+               || n->is_MemBar()
+               || n->Opcode() == Op_Conv2B
+               || n->Opcode() == Op_SafePoint
+               || n->is_CallJava()
+               || n->Opcode() == Op_Unlock
+               || n->Opcode() == Op_EncodeP
+               || n->Opcode() == Op_DecodeN) {
+      // nothing to do
+    } else {
+      static struct {
+        int opcode;
+        struct {
+          int pos;
+          verify_type t;
+        } inputs[2];
+      } others[] = {
+        Op_FastLock,
+        { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_Lock,
+        { { TypeFunc::Parms, ShenandoahLoad },    { -1, ShenandoahNone} },
+        Op_ArrayCopy,
+        { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } },
+        Op_StrCompressedCopy,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_StrInflatedCopy,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_AryEq,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
+        Op_StrIndexOf,
+        { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
+        Op_StrComp,
+        { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
+        Op_StrEquals,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
+        Op_EncodeISOArray,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_HasNegatives,
+        { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_CastP2X,
+        { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_StrIndexOfChar,
+        { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
+      };
+
+      const int others_len = sizeof(others) / sizeof(others[0]);
+      int i = 0;
+      for (; i < others_len; i++) {
+        if (others[i].opcode == n->Opcode()) {
+          break;
+        }
+      }
+      uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
+      if (i != others_len) {
+        const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
+        for (uint j = 0; j < inputs_len; j++) {
+          int pos = others[i].inputs[j].pos;
+          if (pos == -1) {
+            break;
+          }
+          if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
+            report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
+          }
+        }
+        for (uint j = 1; j < stop; j++) {
+          if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
+              n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
+            uint k = 0;
+            for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
+            if (k == inputs_len) {
+              fatal("arg %d for node %s not covered", j, n->Name());
+            }
+          }
+        }
+      } else {
+        for (uint j = 1; j < stop; j++) {
+          if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
+              n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
+            fatal("%s not covered", n->Name());
+          }
+        }
+      }
+    }
+
+    if (n->is_SafePoint()) {
+      SafePointNode* sfpt = n->as_SafePoint();
+      if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
+        for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
+          if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+            phis.clear();
+            visited.Reset();
+          }
+        }
+      }
+    }
+  }
+
+  if (verify_no_useless_barrier) {
+    for (int i = 0; i < barriers.length(); i++) {
+      Node* n = barriers.at(i);
+      if (!barriers_used.member(n)) {
+        tty->print("XXX useless barrier"); n->dump(-2);
+        ShouldNotReachHere();
+      }
+    }
+  }
+}
+#endif
+
+bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) {
+  // That both nodes have the same control is not sufficient to prove
+  // domination, verify that there's no path from d to n
+  ResourceMark rm;
+  Unique_Node_List wq;
+  wq.push(d);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *m = wq.at(next);
+    if (m == n) {
+      return false;
+    }
+    if (m->is_Phi() && m->in(0)->is_Loop()) {
+      assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
+    } else {
+      if (m->is_Store() || m->is_LoadStore()) {
+        // Take anti-dependencies into account
+        Node* mem = m->in(MemNode::Memory);
+        for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
+          Node* u = mem->fast_out(i);
+          if (u->is_Load() && phase->C->can_alias(m->adr_type(), phase->C->get_alias_index(u->adr_type())) &&
+              phase->ctrl_or_self(u) == c) {
+            wq.push(u);
+          }
+        }
+      }
+      for (uint i = 0; i < m->req(); i++) {
+        if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) {
+          wq.push(m->in(i));
+        }
+      }
+    }
+  }
+  return true;
+}
+
+bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
+  if (d_c != n_c) {
+    return phase->is_dominator(d_c, n_c);
+  }
+  return is_dominator_same_ctrl(d_c, d, n, phase);
+}
+
+Node* next_mem(Node* mem, int alias) {
+  Node* res = NULL;
+  if (mem->is_Proj()) {
+    res = mem->in(0);
+  } else if (mem->is_SafePoint() || mem->is_MemBar()) {
+    res = mem->in(TypeFunc::Memory);
+  } else if (mem->is_Phi()) {
+    res = mem->in(1);
+  } else if (mem->is_MergeMem()) {
+    res = mem->as_MergeMem()->memory_at(alias);
+  } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
+    assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
+    res = mem->in(MemNode::Memory);
+  } else {
+#ifdef ASSERT
+    mem->dump();
+#endif
+    ShouldNotReachHere();
+  }
+  return res;
+}
+
+Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
+  Node* iffproj = NULL;
+  while (c != dom) {
+    Node* next = phase->idom(c);
+    assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?");
+    if (c->is_Region()) {
+      ResourceMark rm;
+      Unique_Node_List wq;
+      wq.push(c);
+      for (uint i = 0; i < wq.size(); i++) {
+        Node *n = wq.at(i);
+        if (n == next) {
+          continue;
+        }
+        if (n->is_Region()) {
+          for (uint j = 1; j < n->req(); j++) {
+            wq.push(n->in(j));
+          }
+        } else {
+          wq.push(n->in(0));
+        }
+      }
+      for (uint i = 0; i < wq.size(); i++) {
+        Node *n = wq.at(i);
+        assert(n->is_CFG(), "");
+        if (n->is_Multi()) {
+          for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
+            Node* u = n->fast_out(j);
+            if (u->is_CFG()) {
+              if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
+                return NodeSentinel;
+              }
+            }
+          }
+        }
+      }
+    } else  if (c->is_Proj()) {
+      if (c->is_IfProj()) {
+        if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) {
+          // continue;
+        } else {
+          if (!allow_one_proj) {
+            return NodeSentinel;
+          }
+          if (iffproj == NULL) {
+            iffproj = c;
+          } else {
+            return NodeSentinel;
+          }
+        }
+      } else if (c->Opcode() == Op_JumpProj) {
+        return NodeSentinel; // unsupported
+      } else if (c->Opcode() == Op_CatchProj) {
+        return NodeSentinel; // unsupported
+      } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
+        return NodeSentinel; // unsupported
+      } else {
+        assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
+      }
+    }
+    c = next;
+  }
+  return iffproj;
+}
+
+Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
+  ResourceMark rm;
+  VectorSet wq(Thread::current()->resource_area());
+  wq.set(mem->_idx);
+  mem_ctrl = phase->ctrl_or_self(mem);
+  while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) {
+    mem = next_mem(mem, alias);
+    if (wq.test_set(mem->_idx)) {
+      return NULL;
+    }
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  if (mem->is_MergeMem()) {
+    mem = mem->as_MergeMem()->memory_at(alias);
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  return mem;
+}
+
+Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
+  Node* mem = NULL;
+  Node* c = ctrl;
+  do {
+    if (c->is_Region()) {
+      Node* phi_bottom = NULL;
+      for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
+        Node* u = c->fast_out(i);
+        if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
+          if (u->adr_type() == TypePtr::BOTTOM) {
+            mem = u;
+          }
+        }
+      }
+    } else {
+      if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
+        CallProjections projs;
+        c->as_Call()->extract_projections(&projs, true, false);
+        if (projs.fallthrough_memproj != NULL) {
+          if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
+            if (projs.catchall_memproj == NULL) {
+              mem = projs.fallthrough_memproj;
+            } else {
+              if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
+                mem = projs.fallthrough_memproj;
+              } else {
+                assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
+                mem = projs.catchall_memproj;
+              }
+            }
+          }
+        } else {
+          Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
+          if (proj != NULL &&
+              proj->adr_type() == TypePtr::BOTTOM) {
+            mem = proj;
+          }
+        }
+      } else {
+        for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
+          Node* u = c->fast_out(i);
+          if (u->is_Proj() &&
+              u->bottom_type() == Type::MEMORY &&
+              u->adr_type() == TypePtr::BOTTOM) {
+              assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
+              assert(mem == NULL, "only one proj");
+              mem = u;
+          }
+        }
+        assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected");
+      }
+    }
+    c = phase->idom(c);
+  } while (mem == NULL);
+  return mem;
+}
+
+void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
+  for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+    Node* u = n->fast_out(i);
+    if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
+      uses.push(u);
+    }
+  }
+}
+
+static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
+  OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
+  Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
+  phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
+  Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
+  phase->register_control(new_le, phase->get_loop(le), le->in(0));
+  phase->lazy_replace(outer, new_outer);
+  phase->lazy_replace(le, new_le);
+  inner->clear_strip_mined();
+}
+
+void ShenandoahBarrierC2Support::test_gc_state(Node*& ctrl, Node* raw_mem, Node*& test_fail_ctrl,
+                                               PhaseIdealLoop* phase, int flags) {
+  PhaseIterGVN& igvn = phase->igvn();
+  Node* old_ctrl = ctrl;
+
+  Node* thread          = new ThreadLocalNode();
+  Node* gc_state_offset = igvn.MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  Node* gc_state_addr   = new AddPNode(phase->C->top(), thread, gc_state_offset);
+  Node* gc_state        = new LoadBNode(old_ctrl, raw_mem, gc_state_addr,
+                                        DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL),
+                                        TypeInt::BYTE, MemNode::unordered);
+  Node* gc_state_and    = new AndINode(gc_state, igvn.intcon(flags));
+  Node* gc_state_cmp    = new CmpINode(gc_state_and, igvn.zerocon(T_INT));
+  Node* gc_state_bool   = new BoolNode(gc_state_cmp, BoolTest::ne);
+
+  IfNode* gc_state_iff  = new IfNode(old_ctrl, gc_state_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+  ctrl                  = new IfTrueNode(gc_state_iff);
+  test_fail_ctrl        = new IfFalseNode(gc_state_iff);
+
+  IdealLoopTree* loop = phase->get_loop(old_ctrl);
+  phase->register_control(gc_state_iff,   loop, old_ctrl);
+  phase->register_control(ctrl,           loop, gc_state_iff);
+  phase->register_control(test_fail_ctrl, loop, gc_state_iff);
+
+  phase->register_new_node(thread,        old_ctrl);
+  phase->register_new_node(gc_state_addr, old_ctrl);
+  phase->register_new_node(gc_state,      old_ctrl);
+  phase->register_new_node(gc_state_and,  old_ctrl);
+  phase->register_new_node(gc_state_cmp,  old_ctrl);
+  phase->register_new_node(gc_state_bool, old_ctrl);
+
+  phase->set_ctrl(gc_state_offset, phase->C->root());
+
+  assert(is_gc_state_test(gc_state_iff, flags), "Should match the shape");
+}
+
+void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
+  Node* old_ctrl = ctrl;
+  PhaseIterGVN& igvn = phase->igvn();
+
+  const Type* val_t = igvn.type(val);
+  if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
+    Node* null_cmp   = new CmpPNode(val, igvn.zerocon(T_OBJECT));
+    Node* null_test  = new BoolNode(null_cmp, BoolTest::ne);
+
+    IfNode* null_iff = new IfNode(old_ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
+    ctrl             = new IfTrueNode(null_iff);
+    null_ctrl        = new IfFalseNode(null_iff);
+
+    IdealLoopTree* loop = phase->get_loop(old_ctrl);
+    phase->register_control(null_iff,  loop, old_ctrl);
+    phase->register_control(ctrl,      loop, null_iff);
+    phase->register_control(null_ctrl, loop, null_iff);
+
+    phase->register_new_node(null_cmp,  old_ctrl);
+    phase->register_new_node(null_test, old_ctrl);
+  }
+}
+
+Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
+  IdealLoopTree *loop = phase->get_loop(c);
+  Node* iff = unc_ctrl->in(0);
+  assert(iff->is_If(), "broken");
+  Node* new_iff = iff->clone();
+  new_iff->set_req(0, c);
+  phase->register_control(new_iff, loop, c);
+  Node* iffalse = new IfFalseNode(new_iff->as_If());
+  phase->register_control(iffalse, loop, new_iff);
+  Node* iftrue = new IfTrueNode(new_iff->as_If());
+  phase->register_control(iftrue, loop, new_iff);
+  c = iftrue;
+  const Type *t = phase->igvn().type(val);
+  assert(val->Opcode() == Op_CastPP, "expect cast to non null here");
+  Node* uncasted_val = val->in(1);
+  val = new CastPPNode(uncasted_val, t);
+  val->init_req(0, c);
+  phase->register_new_node(val, c);
+  return val;
+}
+
+void ShenandoahBarrierC2Support::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
+                                                Unique_Node_List& uses, PhaseIdealLoop* phase) {
+  IfNode* iff = unc_ctrl->in(0)->as_If();
+  Node* proj = iff->proj_out(0);
+  assert(proj != unc_ctrl, "bad projection");
+  Node* use = proj->unique_ctrl_out();
+
+  assert(use == unc || use->is_Region(), "what else?");
+
+  uses.clear();
+  if (use == unc) {
+    phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use));
+    for (uint i = 1; i < unc->req(); i++) {
+      Node* n = unc->in(i);
+      if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) {
+        uses.push(n);
+      }
+    }
+  } else {
+    assert(use->is_Region(), "what else?");
+    uint idx = 1;
+    for (; use->in(idx) != proj; idx++);
+    for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) {
+      Node* u = use->fast_out(i);
+      if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) {
+        uses.push(u->in(idx));
+      }
+    }
+  }
+  for(uint next = 0; next < uses.size(); next++ ) {
+    Node *n = uses.at(next);
+    assert(phase->get_ctrl(n) == proj, "bad control");
+    phase->set_ctrl_and_loop(n, new_unc_ctrl);
+    if (n->in(0) == proj) {
+      phase->igvn().replace_input_of(n, 0, new_unc_ctrl);
+    }
+    for (uint i = 0; i < n->req(); i++) {
+      Node* m = n->in(i);
+      if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) {
+        uses.push(m);
+      }
+    }
+  }
+
+  phase->igvn().rehash_node_delayed(use);
+  int nb = use->replace_edge(proj, new_unc_ctrl);
+  assert(nb == 1, "only use expected");
+}
+
+void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
+  Node* old_ctrl = ctrl;
+  PhaseIterGVN& igvn = phase->igvn();
+
+  Node* raw_val        = new CastP2XNode(old_ctrl, val);
+  Node* cset_idx       = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
+
+  // Figure out the target cset address with raw pointer math.
+  // This avoids matching AddP+LoadB that would emit inefficient code.
+  // See JDK-8245465.
+  Node* cset_addr_ptr  = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+  Node* cset_addr      = new CastP2XNode(old_ctrl, cset_addr_ptr);
+  Node* cset_load_addr = new AddXNode(cset_addr, cset_idx);
+  Node* cset_load_ptr  = new CastX2PNode(cset_load_addr);
+
+  Node* cset_load      = new LoadBNode(old_ctrl, raw_mem, cset_load_ptr,
+                                       DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL),
+                                       TypeInt::BYTE, MemNode::unordered);
+  Node* cset_cmp       = new CmpINode(cset_load, igvn.zerocon(T_INT));
+  Node* cset_bool      = new BoolNode(cset_cmp, BoolTest::ne);
+
+  IfNode* cset_iff     = new IfNode(old_ctrl, cset_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+  ctrl                 = new IfTrueNode(cset_iff);
+  not_cset_ctrl        = new IfFalseNode(cset_iff);
+
+  IdealLoopTree *loop = phase->get_loop(old_ctrl);
+  phase->register_control(cset_iff,      loop, old_ctrl);
+  phase->register_control(ctrl,          loop, cset_iff);
+  phase->register_control(not_cset_ctrl, loop, cset_iff);
+
+  phase->set_ctrl(cset_addr_ptr, phase->C->root());
+
+  phase->register_new_node(raw_val,        old_ctrl);
+  phase->register_new_node(cset_idx,       old_ctrl);
+  phase->register_new_node(cset_addr,      old_ctrl);
+  phase->register_new_node(cset_load_addr, old_ctrl);
+  phase->register_new_node(cset_load_ptr,  old_ctrl);
+  phase->register_new_node(cset_load,      old_ctrl);
+  phase->register_new_node(cset_cmp,       old_ctrl);
+  phase->register_new_node(cset_bool,      old_ctrl);
+}
+
+void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
+  IdealLoopTree*loop = phase->get_loop(ctrl);
+  const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
+
+  // The slow path stub consumes and produces raw memory in addition
+  // to the existing memory edges
+  Node* base = find_bottom_mem(ctrl, phase);
+  MergeMemNode* mm = MergeMemNode::make(base);
+  mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
+  phase->register_new_node(mm, ctrl);
+
+  address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
+          CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
+          CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
+
+  Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(),
+                                target,
+                                "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);
+  call->init_req(TypeFunc::Control, ctrl);
+  call->init_req(TypeFunc::I_O, phase->C->top());
+  call->init_req(TypeFunc::Memory, mm);
+  call->init_req(TypeFunc::FramePtr, phase->C->top());
+  call->init_req(TypeFunc::ReturnAdr, phase->C->top());
+  call->init_req(TypeFunc::Parms, val);
+  call->init_req(TypeFunc::Parms+1, load_addr);
+  phase->register_control(call, loop, ctrl);
+  ctrl = new ProjNode(call, TypeFunc::Control);
+  phase->register_control(ctrl, loop, call);
+  result_mem = new ProjNode(call, TypeFunc::Memory);
+  phase->register_new_node(result_mem, call);
+  val = new ProjNode(call, TypeFunc::Parms);
+  phase->register_new_node(val, call);
+  val = new CheckCastPPNode(ctrl, val, obj_type);
+  phase->register_new_node(val, ctrl);
+}
+
+void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
+  Node* ctrl = phase->get_ctrl(barrier);
+  Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
+
+  // Update the control of all nodes that should be after the
+  // barrier control flow
+  uses.clear();
+  // Every node that is control dependent on the barrier's input
+  // control will be after the expanded barrier. The raw memory (if
+  // its memory is control dependent on the barrier's input control)
+  // must stay above the barrier.
+  uses_to_ignore.clear();
+  if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) {
+    uses_to_ignore.push(init_raw_mem);
+  }
+  for (uint next = 0; next < uses_to_ignore.size(); next++) {
+    Node *n = uses_to_ignore.at(next);
+    for (uint i = 0; i < n->req(); i++) {
+      Node* in = n->in(i);
+      if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
+        uses_to_ignore.push(in);
+      }
+    }
+  }
+  for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) {
+    Node* u = ctrl->fast_out(i);
+    if (u->_idx < last &&
+        u != barrier &&
+        !uses_to_ignore.member(u) &&
+        (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) &&
+        (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) {
+      Node* old_c = phase->ctrl_or_self(u);
+      Node* c = old_c;
+      if (c != ctrl ||
+          is_dominator_same_ctrl(old_c, barrier, u, phase) ||
+          ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
+        phase->igvn().rehash_node_delayed(u);
+        int nb = u->replace_edge(ctrl, region);
+        if (u->is_CFG()) {
+          if (phase->idom(u) == ctrl) {
+            phase->set_idom(u, region, phase->dom_depth(region));
+          }
+        } else if (phase->get_ctrl(u) == ctrl) {
+          assert(u != init_raw_mem, "should leave input raw mem above the barrier");
+          uses.push(u);
+        }
+        assert(nb == 1, "more than 1 ctrl input?");
+        --i, imax -= nb;
+      }
+    }
+  }
+}
+
+static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
+  Node* region = NULL;
+  while (c != ctrl) {
+    if (c->is_Region()) {
+      region = c;
+    }
+    c = phase->idom(c);
+  }
+  assert(region != NULL, "");
+  Node* phi = new PhiNode(region, n->bottom_type());
+  for (uint j = 1; j < region->req(); j++) {
+    Node* in = region->in(j);
+    if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
+      phi->init_req(j, n);
+    } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
+      phi->init_req(j, n_clone);
+    } else {
+      phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
+    }
+  }
+  phase->register_new_node(phi, region);
+  return phi;
+}
+
+void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
+  ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
+
+  Unique_Node_List uses;
+  for (int i = 0; i < state->enqueue_barriers_count(); i++) {
+    Node* barrier = state->enqueue_barrier(i);
+    Node* ctrl = phase->get_ctrl(barrier);
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    if (loop->_head->is_OuterStripMinedLoop()) {
+      // Expanding a barrier here will break loop strip mining
+      // verification. Transform the loop so the loop nest doesn't
+      // appear as strip mined.
+      OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
+      hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
+    }
+  }
+
+  Node_Stack stack(0);
+  Node_List clones;
+  for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
+    if (lrb->is_redundant()) {
+      continue;
+    }
+
+    Node* ctrl = phase->get_ctrl(lrb);
+    Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+
+    CallStaticJavaNode* unc = NULL;
+    Node* unc_ctrl = NULL;
+    Node* uncasted_val = val;
+
+    for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
+      Node* u = lrb->fast_out(i);
+      if (u->Opcode() == Op_CastPP &&
+          u->in(0) != NULL &&
+          phase->is_dominator(u->in(0), ctrl)) {
+        const Type* u_t = phase->igvn().type(u);
+
+        if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
+            u->in(0)->Opcode() == Op_IfTrue &&
+            u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
+            u->in(0)->in(0)->is_If() &&
+            u->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
+            u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
+            u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
+            u->in(0)->in(0)->in(1)->in(1)->in(1) == val &&
+            u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
+          IdealLoopTree* loop = phase->get_loop(ctrl);
+          IdealLoopTree* unc_loop = phase->get_loop(u->in(0));
+
+          if (!unc_loop->is_member(loop)) {
+            continue;
+          }
+
+          Node* branch = no_branches(ctrl, u->in(0), false, phase);
+          assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
+          if (branch == NodeSentinel) {
+            continue;
+          }
+
+          phase->igvn().replace_input_of(u, 1, val);
+          phase->igvn().replace_input_of(lrb, ShenandoahLoadReferenceBarrierNode::ValueIn, u);
+          phase->set_ctrl(u, u->in(0));
+          phase->set_ctrl(lrb, u->in(0));
+          unc = u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
+          unc_ctrl = u->in(0);
+          val = u;
+
+          for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
+            Node* u = val->fast_out(j);
+            if (u == lrb) continue;
+            phase->igvn().rehash_node_delayed(u);
+            int nb = u->replace_edge(val, lrb);
+            --j; jmax -= nb;
+          }
+
+          RegionNode* r = new RegionNode(3);
+          IfNode* iff = unc_ctrl->in(0)->as_If();
+
+          Node* ctrl_use = unc_ctrl->unique_ctrl_out();
+          Node* unc_ctrl_clone = unc_ctrl->clone();
+          phase->register_control(unc_ctrl_clone, loop, iff);
+          Node* c = unc_ctrl_clone;
+          Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
+          r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
+
+          phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
+          phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
+          phase->lazy_replace(c, unc_ctrl);
+          c = NULL;;
+          phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
+          phase->set_ctrl(val, unc_ctrl_clone);
+
+          IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
+          fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
+          Node* iff_proj = iff->proj_out(0);
+          r->init_req(2, iff_proj);
+          phase->register_control(r, phase->ltree_root(), iff);
+
+          Node* new_bol = new_iff->in(1)->clone();
+          Node* new_cmp = new_bol->in(1)->clone();
+          assert(new_cmp->Opcode() == Op_CmpP, "broken");
+          assert(new_cmp->in(1) == val->in(1), "broken");
+          new_bol->set_req(1, new_cmp);
+          new_cmp->set_req(1, lrb);
+          phase->register_new_node(new_bol, new_iff->in(0));
+          phase->register_new_node(new_cmp, new_iff->in(0));
+          phase->igvn().replace_input_of(new_iff, 1, new_bol);
+          phase->igvn().replace_input_of(new_cast, 1, lrb);
+
+          for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
+            Node* u = lrb->fast_out(i);
+            if (u == new_cast || u == new_cmp) {
+              continue;
+            }
+            phase->igvn().rehash_node_delayed(u);
+            int nb = u->replace_edge(lrb, new_cast);
+            assert(nb > 0, "no update?");
+            --i; imax -= nb;
+          }
+
+          for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+            Node* u = val->fast_out(i);
+            if (u == lrb) {
+              continue;
+            }
+            phase->igvn().rehash_node_delayed(u);
+            int nb = u->replace_edge(val, new_cast);
+            assert(nb > 0, "no update?");
+            --i; imax -= nb;
+          }
+
+          ctrl = unc_ctrl_clone;
+          phase->set_ctrl_and_loop(lrb, ctrl);
+          break;
+        }
+      }
+    }
+    if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) {
+      CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava();
+      if (call->entry_point() == OptoRuntime::rethrow_stub()) {
+        // The rethrow call may have too many projections to be
+        // properly handled here. Given there's no reason for a
+        // barrier to depend on the call, move it above the call
+        stack.push(lrb, 0);
+        do {
+          Node* n = stack.node();
+          uint idx = stack.index();
+          if (idx < n->req()) {
+            Node* in = n->in(idx);
+            stack.set_index(idx+1);
+            if (in != NULL) {
+              if (phase->has_ctrl(in)) {
+                if (phase->is_dominator(call, phase->get_ctrl(in))) {
+#ifdef ASSERT
+                  for (uint i = 0; i < stack.size(); i++) {
+                    assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
+                  }
+#endif
+                  stack.push(in, 0);
+                }
+              } else {
+                assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
+              }
+            }
+          } else {
+            phase->set_ctrl(n, call->in(0));
+            stack.pop();
+          }
+        } while(stack.size() > 0);
+        continue;
+      }
+      CallProjections projs;
+      call->extract_projections(&projs, false, false);
+
+#ifdef ASSERT
+      VectorSet cloned(Thread::current()->resource_area());
+#endif
+      Node* lrb_clone = lrb->clone();
+      phase->register_new_node(lrb_clone, projs.catchall_catchproj);
+      phase->set_ctrl(lrb, projs.fallthrough_catchproj);
+
+      stack.push(lrb, 0);
+      clones.push(lrb_clone);
+
+      do {
+        assert(stack.size() == clones.size(), "");
+        Node* n = stack.node();
+#ifdef ASSERT
+        if (n->is_Load()) {
+          Node* mem = n->in(MemNode::Memory);
+          for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
+            Node* u = mem->fast_out(j);
+            assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
+          }
+        }
+#endif
+        uint idx = stack.index();
+        Node* n_clone = clones.at(clones.size()-1);
+        if (idx < n->outcnt()) {
+          Node* u = n->raw_out(idx);
+          Node* c = phase->ctrl_or_self(u);
+          if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
+            stack.set_index(idx+1);
+            assert(!u->is_CFG(), "");
+            stack.push(u, 0);
+            assert(!cloned.test_set(u->_idx), "only one clone");
+            Node* u_clone = u->clone();
+            int nb = u_clone->replace_edge(n, n_clone);
+            assert(nb > 0, "should have replaced some uses");
+            phase->register_new_node(u_clone, projs.catchall_catchproj);
+            clones.push(u_clone);
+            phase->set_ctrl(u, projs.fallthrough_catchproj);
+          } else {
+            bool replaced = false;
+            if (u->is_Phi()) {
+              for (uint k = 1; k < u->req(); k++) {
+                if (u->in(k) == n) {
+                  if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
+                    phase->igvn().replace_input_of(u, k, n_clone);
+                    replaced = true;
+                  } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
+                    phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
+                    replaced = true;
+                  }
+                }
+              }
+            } else {
+              if (phase->is_dominator(projs.catchall_catchproj, c)) {
+                phase->igvn().rehash_node_delayed(u);
+                int nb = u->replace_edge(n, n_clone);
+                assert(nb > 0, "should have replaced some uses");
+                replaced = true;
+              } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
+                if (u->is_If()) {
+                  // Can't break If/Bool/Cmp chain
+                  assert(n->is_Bool(), "unexpected If shape");
+                  assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
+                  assert(n_clone->is_Bool(), "unexpected clone");
+                  assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
+                  Node* bol_clone = n->clone();
+                  Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
+                  bol_clone->set_req(1, cmp_clone);
+
+                  Node* nn = stack.node_at(stack.size()-3);
+                  Node* nn_clone = clones.at(clones.size()-3);
+                  assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
+
+                  int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase));
+                  assert(nb > 0, "should have replaced some uses");
+
+                  phase->register_new_node(bol_clone, u->in(0));
+                  phase->register_new_node(cmp_clone, u->in(0));
+
+                  phase->igvn().replace_input_of(u, 1, bol_clone);
+
+                } else {
+                  phase->igvn().rehash_node_delayed(u);
+                  int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
+                  assert(nb > 0, "should have replaced some uses");
+                }
+                replaced = true;
+              }
+            }
+            if (!replaced) {
+              stack.set_index(idx+1);
+            }
+          }
+        } else {
+          stack.pop();
+          clones.pop();
+        }
+      } while (stack.size() > 0);
+      assert(stack.size() == 0 && clones.size() == 0, "");
+    }
+  }
+
+  for (int i = 0; i < state->load_reference_barriers_count(); i++) {
+    ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
+    if (lrb->is_redundant()) {
+      continue;
+    }
+    Node* ctrl = phase->get_ctrl(lrb);
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    if (loop->_head->is_OuterStripMinedLoop()) {
+      // Expanding a barrier here will break loop strip mining
+      // verification. Transform the loop so the loop nest doesn't
+      // appear as strip mined.
+      OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
+      hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
+    }
+  }
+
+  // Expand load-reference-barriers
+  MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
+  Unique_Node_List uses_to_ignore;
+  for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
+    if (lrb->is_redundant()) {
+      phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
+      continue;
+    }
+    uint last = phase->C->unique();
+    Node* ctrl = phase->get_ctrl(lrb);
+    Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+
+
+    Node* orig_ctrl = ctrl;
+
+    Node* raw_mem = fixer.find_mem(ctrl, lrb);
+    Node* init_raw_mem = raw_mem;
+    Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
+
+    IdealLoopTree *loop = phase->get_loop(ctrl);
+    CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn());
+    Node* unc_ctrl = NULL;
+    if (unc != NULL) {
+      if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
+        unc = NULL;
+      } else {
+        unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
+      }
+    }
+
+    Node* uncasted_val = val;
+    if (unc != NULL) {
+      uncasted_val = val->in(1);
+    }
+
+    Node* heap_stable_ctrl = NULL;
+    Node* null_ctrl = NULL;
+
+    assert(val->bottom_type()->make_oopptr(), "need oop");
+    assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
+
+    enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
+    Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    // Stable path.
+    test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
+    IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
+
+    // Heap stable case
+    region->init_req(_heap_stable, heap_stable_ctrl);
+    val_phi->init_req(_heap_stable, uncasted_val);
+    raw_mem_phi->init_req(_heap_stable, raw_mem);
+
+    Node* reg2_ctrl = NULL;
+    // Null case
+    test_null(ctrl, val, null_ctrl, phase);
+    if (null_ctrl != NULL) {
+      reg2_ctrl = null_ctrl->in(0);
+      region->init_req(_null_path, null_ctrl);
+      val_phi->init_req(_null_path, uncasted_val);
+      raw_mem_phi->init_req(_null_path, raw_mem);
+    } else {
+      region->del_req(_null_path);
+      val_phi->del_req(_null_path);
+      raw_mem_phi->del_req(_null_path);
+    }
+
+    // Test for in-cset.
+    // Wires !in_cset(obj) to slot 2 of region and phis
+    Node* not_cset_ctrl = NULL;
+    test_in_cset(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
+    if (not_cset_ctrl != NULL) {
+      if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
+      region->init_req(_not_cset, not_cset_ctrl);
+      val_phi->init_req(_not_cset, uncasted_val);
+      raw_mem_phi->init_req(_not_cset, raw_mem);
+    }
+
+    // Resolve object when orig-value is in cset.
+    // Make the unconditional resolve for fwdptr.
+    Node* new_val = uncasted_val;
+    if (unc_ctrl != NULL) {
+      // Clone the null check in this branch to allow implicit null check
+      new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
+      fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
+
+      IfNode* iff = unc_ctrl->in(0)->as_If();
+      phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
+    }
+
+    // Call lrb-stub and wire up that path in slots 4
+    Node* result_mem = NULL;
+
+    Node* fwd = new_val;
+    Node* addr;
+    if (ShenandoahSelfFixing) {
+      VectorSet visited(Thread::current()->resource_area());
+      addr = get_load_addr(phase, visited, lrb);
+    } else {
+      addr = phase->igvn().zerocon(T_OBJECT);
+    }
+    if (addr->Opcode() == Op_AddP) {
+      Node* orig_base = addr->in(AddPNode::Base);
+      Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true);
+      phase->register_new_node(base, ctrl);
+      if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) {
+        // Field access
+        addr = addr->clone();
+        addr->set_req(AddPNode::Base, base);
+        addr->set_req(AddPNode::Address, base);
+        phase->register_new_node(addr, ctrl);
+      } else {
+        Node* addr2 = addr->in(AddPNode::Address);
+        if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) &&
+              addr2->in(AddPNode::Base) == orig_base) {
+          addr2 = addr2->clone();
+          addr2->set_req(AddPNode::Base, base);
+          addr2->set_req(AddPNode::Address, base);
+          phase->register_new_node(addr2, ctrl);
+          addr = addr->clone();
+          addr->set_req(AddPNode::Base, base);
+          addr->set_req(AddPNode::Address, addr2);
+          phase->register_new_node(addr, ctrl);
+        }
+      }
+    }
+    call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, false, phase);
+    region->init_req(_evac_path, ctrl);
+    val_phi->init_req(_evac_path, fwd);
+    raw_mem_phi->init_req(_evac_path, result_mem);
+
+    phase->register_control(region, loop, heap_stable_iff);
+    Node* out_val = val_phi;
+    phase->register_new_node(val_phi, region);
+    phase->register_new_node(raw_mem_phi, region);
+
+    fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
+
+    ctrl = orig_ctrl;
+
+    if (unc != NULL) {
+      for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+        Node* u = val->fast_out(i);
+        Node* c = phase->ctrl_or_self(u);
+        if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
+          phase->igvn().rehash_node_delayed(u);
+          int nb = u->replace_edge(val, out_val);
+          --i, imax -= nb;
+        }
+      }
+      if (val->outcnt() == 0) {
+        phase->igvn()._worklist.push(val);
+      }
+    }
+    phase->igvn().replace_node(lrb, out_val);
+
+    follow_barrier_uses(out_val, ctrl, uses, phase);
+
+    for(uint next = 0; next < uses.size(); next++ ) {
+      Node *n = uses.at(next);
+      assert(phase->get_ctrl(n) == ctrl, "bad control");
+      assert(n != init_raw_mem, "should leave input raw mem above the barrier");
+      phase->set_ctrl(n, region);
+      follow_barrier_uses(n, ctrl, uses, phase);
+    }
+
+    // The slow path call produces memory: hook the raw memory phi
+    // from the expanded load reference barrier with the rest of the graph
+    // which may require adding memory phis at every post dominated
+    // region and at enclosing loop heads. Use the memory state
+    // collected in memory_nodes to fix the memory graph. Update that
+    // memory state as we go.
+    fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
+  }
+  // Done expanding load-reference-barriers.
+  assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
+
+  for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) {
+    Node* barrier = state->enqueue_barrier(i);
+    Node* pre_val = barrier->in(1);
+
+    if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
+      ShouldNotReachHere();
+      continue;
+    }
+
+    Node* ctrl = phase->get_ctrl(barrier);
+
+    if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
+      assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
+      ctrl = ctrl->in(0)->in(0);
+      phase->set_ctrl(barrier, ctrl);
+    } else if (ctrl->is_CallRuntime()) {
+      assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
+      ctrl = ctrl->in(0);
+      phase->set_ctrl(barrier, ctrl);
+    }
+
+    Node* init_ctrl = ctrl;
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    Node* raw_mem = fixer.find_mem(ctrl, barrier);
+    Node* init_raw_mem = raw_mem;
+    Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
+    Node* heap_stable_ctrl = NULL;
+    Node* null_ctrl = NULL;
+    uint last = phase->C->unique();
+
+    enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
+    Node* region2 = new RegionNode(PATH_LIMIT2);
+    Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    // Stable path.
+    test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
+    region->init_req(_heap_stable, heap_stable_ctrl);
+    phi->init_req(_heap_stable, raw_mem);
+
+    // Null path
+    Node* reg2_ctrl = NULL;
+    test_null(ctrl, pre_val, null_ctrl, phase);
+    if (null_ctrl != NULL) {
+      reg2_ctrl = null_ctrl->in(0);
+      region2->init_req(_null_path, null_ctrl);
+      phi2->init_req(_null_path, raw_mem);
+    } else {
+      region2->del_req(_null_path);
+      phi2->del_req(_null_path);
+    }
+
+    const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
+    const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
+    Node* thread = new ThreadLocalNode();
+    phase->register_new_node(thread, ctrl);
+    Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
+    phase->register_new_node(buffer_adr, ctrl);
+    Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
+    phase->register_new_node(index_adr, ctrl);
+
+    BasicType index_bt = TypeX_X->basic_type();
+    assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
+    const TypePtr* adr_type = TypeRawPtr::BOTTOM;
+    Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
+    phase->register_new_node(index, ctrl);
+    Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
+    phase->register_new_node(index_cmp, ctrl);
+    Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
+    phase->register_new_node(index_test, ctrl);
+    IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
+    if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff;
+    phase->register_control(queue_full_iff, loop, ctrl);
+    Node* not_full = new IfTrueNode(queue_full_iff);
+    phase->register_control(not_full, loop, queue_full_iff);
+    Node* full = new IfFalseNode(queue_full_iff);
+    phase->register_control(full, loop, queue_full_iff);
+
+    ctrl = not_full;
+
+    Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
+    phase->register_new_node(next_index, ctrl);
+
+    Node* buffer  = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
+    phase->register_new_node(buffer, ctrl);
+    Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
+    phase->register_new_node(log_addr, ctrl);
+    Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
+    phase->register_new_node(log_store, ctrl);
+    // update the index
+    Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
+    phase->register_new_node(index_update, ctrl);
+
+    // Fast-path case
+    region2->init_req(_fast_path, ctrl);
+    phi2->init_req(_fast_path, index_update);
+
+    ctrl = full;
+
+    Node* base = find_bottom_mem(ctrl, phase);
+
+    MergeMemNode* mm = MergeMemNode::make(base);
+    mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
+    phase->register_new_node(mm, ctrl);
+
+    Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
+    call->init_req(TypeFunc::Control, ctrl);
+    call->init_req(TypeFunc::I_O, phase->C->top());
+    call->init_req(TypeFunc::Memory, mm);
+    call->init_req(TypeFunc::FramePtr, phase->C->top());
+    call->init_req(TypeFunc::ReturnAdr, phase->C->top());
+    call->init_req(TypeFunc::Parms, pre_val);
+    call->init_req(TypeFunc::Parms+1, thread);
+    phase->register_control(call, loop, ctrl);
+
+    Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
+    phase->register_control(ctrl_proj, loop, call);
+    Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
+    phase->register_new_node(mem_proj, call);
+
+    // Slow-path case
+    region2->init_req(_slow_path, ctrl_proj);
+    phi2->init_req(_slow_path, mem_proj);
+
+    phase->register_control(region2, loop, reg2_ctrl);
+    phase->register_new_node(phi2, region2);
+
+    region->init_req(_heap_unstable, region2);
+    phi->init_req(_heap_unstable, phi2);
+
+    phase->register_control(region, loop, heap_stable_ctrl->in(0));
+    phase->register_new_node(phi, region);
+
+    fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
+    for(uint next = 0; next < uses.size(); next++ ) {
+      Node *n = uses.at(next);
+      assert(phase->get_ctrl(n) == init_ctrl, "bad control");
+      assert(n != init_raw_mem, "should leave input raw mem above the barrier");
+      phase->set_ctrl(n, region);
+      follow_barrier_uses(n, init_ctrl, uses, phase);
+    }
+    fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
+
+    phase->igvn().replace_node(barrier, pre_val);
+  }
+  assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
+
+}
+
+Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
+  if (visited.test_set(in->_idx)) {
+    return NULL;
+  }
+  switch (in->Opcode()) {
+    case Op_Proj:
+      return get_load_addr(phase, visited, in->in(0));
+    case Op_CastPP:
+    case Op_CheckCastPP:
+    case Op_DecodeN:
+    case Op_EncodeP:
+      return get_load_addr(phase, visited, in->in(1));
+    case Op_LoadN:
+    case Op_LoadP:
+      return in->in(MemNode::Address);
+    case Op_CompareAndExchangeN:
+    case Op_CompareAndExchangeP:
+    case Op_GetAndSetN:
+    case Op_GetAndSetP:
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
+      // Those instructions would just have stored a different
+      // value into the field. No use to attempt to fix it at this point.
+      return phase->igvn().zerocon(T_OBJECT);
+    case Op_CMoveP:
+    case Op_CMoveN: {
+      Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue));
+      Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse));
+      // Handle unambiguous cases: single address reported on both branches.
+      if (t != NULL && f == NULL) return t;
+      if (t == NULL && f != NULL) return f;
+      if (t != NULL && t == f)    return t;
+      // Ambiguity.
+      return phase->igvn().zerocon(T_OBJECT);
+    }
+    case Op_Phi: {
+      Node* addr = NULL;
+      for (uint i = 1; i < in->req(); i++) {
+        Node* addr1 = get_load_addr(phase, visited, in->in(i));
+        if (addr == NULL) {
+          addr = addr1;
+        }
+        if (addr != addr1) {
+          return phase->igvn().zerocon(T_OBJECT);
+        }
+      }
+      return addr;
+    }
+    case Op_ShenandoahLoadReferenceBarrier:
+      return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
+    case Op_ShenandoahEnqueueBarrier:
+      return get_load_addr(phase, visited, in->in(1));
+    case Op_CallDynamicJava:
+    case Op_CallLeaf:
+    case Op_CallStaticJava:
+    case Op_ConN:
+    case Op_ConP:
+    case Op_Parm:
+    case Op_CreateEx:
+      return phase->igvn().zerocon(T_OBJECT);
+    default:
+#ifdef ASSERT
+      fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
+#endif
+      return phase->igvn().zerocon(T_OBJECT);
+  }
+
+}
+
+void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
+  IdealLoopTree *loop = phase->get_loop(iff);
+  Node* loop_head = loop->_head;
+  Node* entry_c = loop_head->in(LoopNode::EntryControl);
+
+  Node* bol = iff->in(1);
+  Node* cmp = bol->in(1);
+  Node* andi = cmp->in(1);
+  Node* load = andi->in(1);
+
+  assert(is_gc_state_load(load), "broken");
+  if (!phase->is_dominator(load->in(0), entry_c)) {
+    Node* mem_ctrl = NULL;
+    Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
+    load = load->clone();
+    load->set_req(MemNode::Memory, mem);
+    load->set_req(0, entry_c);
+    phase->register_new_node(load, entry_c);
+    andi = andi->clone();
+    andi->set_req(1, load);
+    phase->register_new_node(andi, entry_c);
+    cmp = cmp->clone();
+    cmp->set_req(1, andi);
+    phase->register_new_node(cmp, entry_c);
+    bol = bol->clone();
+    bol->set_req(1, cmp);
+    phase->register_new_node(bol, entry_c);
+
+    Node* old_bol =iff->in(1);
+    phase->igvn().replace_input_of(iff, 1, bol);
+  }
+}
+
+bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
+  if (!n->is_If() || n->is_CountedLoopEnd()) {
+    return false;
+  }
+  Node* region = n->in(0);
+
+  if (!region->is_Region()) {
+    return false;
+  }
+  Node* dom = phase->idom(region);
+  if (!dom->is_If()) {
+    return false;
+  }
+
+  if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
+    return false;
+  }
+
+  IfNode* dom_if = dom->as_If();
+  Node* proj_true = dom_if->proj_out(1);
+  Node* proj_false = dom_if->proj_out(0);
+
+  for (uint i = 1; i < region->req(); i++) {
+    if (phase->is_dominator(proj_true, region->in(i))) {
+      continue;
+    }
+    if (phase->is_dominator(proj_false, region->in(i))) {
+      continue;
+    }
+    return false;
+  }
+
+  return true;
+}
+
+void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
+  assert(is_heap_stable_test(n), "no other tests");
+  if (identical_backtoback_ifs(n, phase)) {
+    Node* n_ctrl = n->in(0);
+    if (phase->can_split_if(n_ctrl)) {
+      IfNode* dom_if = phase->idom(n_ctrl)->as_If();
+      if (is_heap_stable_test(n)) {
+        Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
+        assert(is_gc_state_load(gc_state_load), "broken");
+        Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
+        assert(is_gc_state_load(dom_gc_state_load), "broken");
+        if (gc_state_load != dom_gc_state_load) {
+          phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
+        }
+      }
+      PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
+      Node* proj_true = dom_if->proj_out(1);
+      Node* proj_false = dom_if->proj_out(0);
+      Node* con_true = phase->igvn().makecon(TypeInt::ONE);
+      Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
+
+      for (uint i = 1; i < n_ctrl->req(); i++) {
+        if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
+          bolphi->init_req(i, con_true);
+        } else {
+          assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
+          bolphi->init_req(i, con_false);
+        }
+      }
+      phase->register_new_node(bolphi, n_ctrl);
+      phase->igvn().replace_input_of(n, 1, bolphi);
+      phase->do_split_if(n);
+    }
+  }
+}
+
+IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
+  // Find first invariant test that doesn't exit the loop
+  LoopNode *head = loop->_head->as_Loop();
+  IfNode* unswitch_iff = NULL;
+  Node* n = head->in(LoopNode::LoopBackControl);
+  int loop_has_sfpts = -1;
+  while (n != head) {
+    Node* n_dom = phase->idom(n);
+    if (n->is_Region()) {
+      if (n_dom->is_If()) {
+        IfNode* iff = n_dom->as_If();
+        if (iff->in(1)->is_Bool()) {
+          BoolNode* bol = iff->in(1)->as_Bool();
+          if (bol->in(1)->is_Cmp()) {
+            // If condition is invariant and not a loop exit,
+            // then found reason to unswitch.
+            if (is_heap_stable_test(iff) &&
+                (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
+              assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
+              if (loop_has_sfpts == -1) {
+                for(uint i = 0; i < loop->_body.size(); i++) {
+                  Node *m = loop->_body[i];
+                  if (m->is_SafePoint() && !m->is_CallLeaf()) {
+                    loop_has_sfpts = 1;
+                    break;
+                  }
+                }
+                if (loop_has_sfpts == -1) {
+                  loop_has_sfpts = 0;
+                }
+              }
+              if (!loop_has_sfpts) {
+                unswitch_iff = iff;
+              }
+            }
+          }
+        }
+      }
+    }
+    n = n_dom;
+  }
+  return unswitch_iff;
+}
+
+
+void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
+  Node_List heap_stable_tests;
+  stack.push(phase->C->start(), 0);
+  do {
+    Node* n = stack.node();
+    uint i = stack.index();
+
+    if (i < n->outcnt()) {
+      Node* u = n->raw_out(i);
+      stack.set_index(i+1);
+      if (!visited.test_set(u->_idx)) {
+        stack.push(u, 0);
+      }
+    } else {
+      stack.pop();
+      if (n->is_If() && is_heap_stable_test(n)) {
+        heap_stable_tests.push(n);
+      }
+    }
+  } while (stack.size() > 0);
+
+  for (uint i = 0; i < heap_stable_tests.size(); i++) {
+    Node* n = heap_stable_tests.at(i);
+    assert(is_heap_stable_test(n), "only evacuation test");
+    merge_back_to_back_tests(n, phase);
+  }
+
+  if (!phase->C->major_progress()) {
+    VectorSet seen(Thread::current()->resource_area());
+    for (uint i = 0; i < heap_stable_tests.size(); i++) {
+      Node* n = heap_stable_tests.at(i);
+      IdealLoopTree* loop = phase->get_loop(n);
+      if (loop != phase->ltree_root() &&
+          loop->_child == NULL &&
+          !loop->_irreducible) {
+        Node* head = loop->_head;
+        if (head->is_Loop() &&
+            (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
+            !seen.test_set(head->_idx)) {
+          IfNode* iff = find_unswitching_candidate(loop, phase);
+          if (iff != NULL) {
+            Node* bol = iff->in(1);
+            if (head->as_Loop()->is_strip_mined()) {
+              head->as_Loop()->verify_strip_mined(0);
+            }
+            move_gc_state_test_out_of_loop(iff, phase);
+            if (loop->policy_unswitching(phase)) {
+              if (head->as_Loop()->is_strip_mined()) {
+                OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
+                hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
+              }
+              phase->do_unswitching(loop, old_new);
+            } else {
+              // Not proceeding with unswitching. Move load back in
+              // the loop.
+              phase->igvn().replace_input_of(iff, 1, bol);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+#ifdef ASSERT
+void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
+  const bool trace = false;
+  ResourceMark rm;
+  Unique_Node_List nodes;
+  Unique_Node_List controls;
+  Unique_Node_List memories;
+
+  nodes.push(root);
+  for (uint next = 0; next < nodes.size(); next++) {
+    Node *n  = nodes.at(next);
+    if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) {
+      controls.push(n);
+      if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
+      for (uint next2 = 0; next2 < controls.size(); next2++) {
+        Node *m = controls.at(next2);
+        for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
+          Node* u = m->fast_out(i);
+          if (u->is_CFG() && !u->is_Root() &&
+              !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
+              !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
+            if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
+            controls.push(u);
+          }
+        }
+      }
+      memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
+      for (uint next2 = 0; next2 < memories.size(); next2++) {
+        Node *m = memories.at(next2);
+        assert(m->bottom_type() == Type::MEMORY, "");
+        for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
+          Node* u = m->fast_out(i);
+          if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) {
+            if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
+            memories.push(u);
+          } else if (u->is_LoadStore()) {
+            if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); }
+            memories.push(u->find_out_with(Op_SCMemProj));
+          } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) {
+            if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
+            memories.push(u);
+          } else if (u->is_Phi()) {
+            assert(u->bottom_type() == Type::MEMORY, "");
+            if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) {
+              assert(controls.member(u->in(0)), "");
+              if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
+              memories.push(u);
+            }
+          } else if (u->is_SafePoint() || u->is_MemBar()) {
+            for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
+              Node* uu = u->fast_out(j);
+              if (uu->bottom_type() == Type::MEMORY) {
+                if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); }
+                memories.push(uu);
+              }
+            }
+          }
+        }
+      }
+      for (uint next2 = 0; next2 < controls.size(); next2++) {
+        Node *m = controls.at(next2);
+        if (m->is_Region()) {
+          bool all_in = true;
+          for (uint i = 1; i < m->req(); i++) {
+            if (!controls.member(m->in(i))) {
+              all_in = false;
+              break;
+            }
+          }
+          if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); }
+          bool found_phi = false;
+          for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) {
+            Node* u = m->fast_out(j);
+            if (u->is_Phi() && memories.member(u)) {
+              found_phi = true;
+              for (uint i = 1; i < u->req() && found_phi; i++) {
+                Node* k = u->in(i);
+                if (memories.member(k) != controls.member(m->in(i))) {
+                  found_phi = false;
+                }
+              }
+            }
+          }
+          assert(found_phi || all_in, "");
+        }
+      }
+      controls.clear();
+      memories.clear();
+    }
+    for( uint i = 0; i < n->len(); ++i ) {
+      Node *m = n->in(i);
+      if (m != NULL) {
+        nodes.push(m);
+      }
+    }
+  }
+}
+#endif
+
+ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
+  ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this);
+}
+
+const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
+  if (in(1) == NULL || in(1)->is_top()) {
+    return Type::TOP;
+  }
+  const Type* t = in(1)->bottom_type();
+  if (t == TypePtr::NULL_PTR) {
+    return t;
+  }
+  return t->is_oopptr();
+}
+
+const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
+  if (in(1) == NULL) {
+    return Type::TOP;
+  }
+  const Type* t = phase->type(in(1));
+  if (t == Type::TOP) {
+    return Type::TOP;
+  }
+  if (t == TypePtr::NULL_PTR) {
+    return t;
+  }
+  return t->is_oopptr();
+}
+
+int ShenandoahEnqueueBarrierNode::needed(Node* n) {
+  if (n == NULL ||
+      n->is_Allocate() ||
+      n->Opcode() == Op_ShenandoahEnqueueBarrier ||
+      n->bottom_type() == TypePtr::NULL_PTR ||
+      (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
+    return NotNeeded;
+  }
+  if (n->is_Phi() ||
+      n->is_CMove()) {
+    return MaybeNeeded;
+  }
+  return Needed;
+}
+
+Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
+  for (;;) {
+    if (n == NULL) {
+      return n;
+    } else if (n->bottom_type() == TypePtr::NULL_PTR) {
+      return n;
+    } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) {
+      return n;
+    } else if (n->is_ConstraintCast() ||
+               n->Opcode() == Op_DecodeN ||
+               n->Opcode() == Op_EncodeP) {
+      n = n->in(1);
+    } else if (n->is_Proj()) {
+      n = n->in(0);
+    } else {
+      return n;
+    }
+  }
+  ShouldNotReachHere();
+  return NULL;
+}
+
+Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) {
+  PhaseIterGVN* igvn = phase->is_IterGVN();
+
+  Node* n = next(in(1));
+
+  int cont = needed(n);
+
+  if (cont == NotNeeded) {
+    return in(1);
+  } else if (cont == MaybeNeeded) {
+    if (igvn == NULL) {
+      phase->record_for_igvn(this);
+      return this;
+    } else {
+      ResourceMark rm;
+      Unique_Node_List wq;
+      uint wq_i = 0;
+
+      for (;;) {
+        if (n->is_Phi()) {
+          for (uint i = 1; i < n->req(); i++) {
+            Node* m = n->in(i);
+            if (m != NULL) {
+              wq.push(m);
+            }
+          }
+        } else {
+          assert(n->is_CMove(), "nothing else here");
+          Node* m = n->in(CMoveNode::IfFalse);
+          wq.push(m);
+          m = n->in(CMoveNode::IfTrue);
+          wq.push(m);
+        }
+        Node* orig_n = NULL;
+        do {
+          if (wq_i >= wq.size()) {
+            return in(1);
+          }
+          n = wq.at(wq_i);
+          wq_i++;
+          orig_n = n;
+          n = next(n);
+          cont = needed(n);
+          if (cont == Needed) {
+            return this;
+          }
+        } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
+      }
+    }
+  }
+
+  return this;
+}
+
+#ifdef ASSERT
+static bool has_never_branch(Node* root) {
+  for (uint i = 1; i < root->req(); i++) {
+    Node* in = root->in(i);
+    if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) {
+      return true;
+    }
+  }
+  return false;
+}
+#endif
+
+void MemoryGraphFixer::collect_memory_nodes() {
+  Node_Stack stack(0);
+  VectorSet visited(Thread::current()->resource_area());
+  Node_List regions;
+
+  // Walk the raw memory graph and create a mapping from CFG node to
+  // memory node. Exclude phis for now.
+  stack.push(_phase->C->root(), 1);
+  do {
+    Node* n = stack.node();
+    int opc = n->Opcode();
+    uint i = stack.index();
+    if (i < n->req()) {
+      Node* mem = NULL;
+      if (opc == Op_Root) {
+        Node* in = n->in(i);
+        int in_opc = in->Opcode();
+        if (in_opc == Op_Return || in_opc == Op_Rethrow) {
+          mem = in->in(TypeFunc::Memory);
+        } else if (in_opc == Op_Halt) {
+          if (in->in(0)->is_Region()) {
+            Node* r = in->in(0);
+            for (uint j = 1; j < r->req(); j++) {
+              assert(r->in(j)->Opcode() != Op_NeverBranch, "");
+            }
+          } else {
+            Node* proj = in->in(0);
+            assert(proj->is_Proj(), "");
+            Node* in = proj->in(0);
+            assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
+            if (in->is_CallStaticJava()) {
+              mem = in->in(TypeFunc::Memory);
+            } else if (in->Opcode() == Op_Catch) {
+              Node* call = in->in(0)->in(0);
+              assert(call->is_Call(), "");
+              mem = call->in(TypeFunc::Memory);
+            } else if (in->Opcode() == Op_NeverBranch) {
+              Node* head = in->in(0);
+              assert(head->is_Region(), "unexpected infinite loop graph shape");
+
+              Node* phi_mem = NULL;
+              for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
+                Node* u = head->fast_out(j);
+                if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
+                  if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
+                    assert(phi_mem == NULL || phi_mem->adr_type() == TypePtr::BOTTOM, "");
+                    phi_mem = u;
+                  } else if (u->adr_type() == TypePtr::BOTTOM) {
+                    assert(phi_mem == NULL || _phase->C->get_alias_index(phi_mem->adr_type()) == _alias, "");
+                    if (phi_mem == NULL) {
+                      phi_mem = u;
+                    }
+                  }
+                }
+              }
+              if (phi_mem == NULL) {
+                for (uint j = 1; j < head->req(); j++) {
+                  Node* tail = head->in(j);
+                  if (!_phase->is_dominator(head, tail)) {
+                    continue;
+                  }
+                  Node* c = tail;
+                  while (c != head) {
+                    if (c->is_SafePoint() && !c->is_CallLeaf()) {
+                      Node* m =c->in(TypeFunc::Memory);
+                      if (m->is_MergeMem()) {
+                        m = m->as_MergeMem()->memory_at(_alias);
+                      }
+                      assert(mem == NULL || mem == m, "several memory states");
+                      mem = m;
+                    }
+                    c = _phase->idom(c);
+                  }
+                  assert(mem != NULL, "should have found safepoint");
+                }
+                assert(mem != NULL, "should have found safepoint");
+              } else {
+                mem = phi_mem;
+              }
+            }
+          }
+        } else {
+#ifdef ASSERT
+          n->dump();
+          in->dump();
+#endif
+          ShouldNotReachHere();
+        }
+      } else {
+        assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, "");
+        assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, "");
+        mem = n->in(i);
+      }
+      i++;
+      stack.set_index(i);
+      if (mem == NULL) {
+        continue;
+      }
+      for (;;) {
+        if (visited.test_set(mem->_idx) || mem->is_Start()) {
+          break;
+        }
+        if (mem->is_Phi()) {
+          stack.push(mem, 2);
+          mem = mem->in(1);
+        } else if (mem->is_Proj()) {
+          stack.push(mem, mem->req());
+          mem = mem->in(0);
+        } else if (mem->is_SafePoint() || mem->is_MemBar()) {
+          mem = mem->in(TypeFunc::Memory);
+        } else if (mem->is_MergeMem()) {
+          MergeMemNode* mm = mem->as_MergeMem();
+          mem = mm->memory_at(_alias);
+        } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
+          assert(_alias == Compile::AliasIdxRaw, "");
+          stack.push(mem, mem->req());
+          mem = mem->in(MemNode::Memory);
+        } else {
+#ifdef ASSERT
+          mem->dump();
+#endif
+          ShouldNotReachHere();
+        }
+      }
+    } else {
+      if (n->is_Phi()) {
+        // Nothing
+      } else if (!n->is_Root()) {
+        Node* c = get_ctrl(n);
+        _memory_nodes.map(c->_idx, n);
+      }
+      stack.pop();
+    }
+  } while(stack.is_nonempty());
+
+  // Iterate over CFG nodes in rpo and propagate memory state to
+  // compute memory state at regions, creating new phis if needed.
+  Node_List rpo_list;
+  visited.Clear();
+  _phase->rpo(_phase->C->root(), stack, visited, rpo_list);
+  Node* root = rpo_list.pop();
+  assert(root == _phase->C->root(), "");
+
+  const bool trace = false;
+#ifdef ASSERT
+  if (trace) {
+    for (int i = rpo_list.size() - 1; i >= 0; i--) {
+      Node* c = rpo_list.at(i);
+      if (_memory_nodes[c->_idx] != NULL) {
+        tty->print("X %d", c->_idx);  _memory_nodes[c->_idx]->dump();
+      }
+    }
+  }
+#endif
+  uint last = _phase->C->unique();
+
+#ifdef ASSERT
+  uint8_t max_depth = 0;
+  for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) {
+    IdealLoopTree* lpt = iter.current();
+    max_depth = MAX2(max_depth, lpt->_nest);
+  }
+#endif
+
+  bool progress = true;
+  int iteration = 0;
+  Node_List dead_phis;
+  while (progress) {
+    progress = false;
+    iteration++;
+    assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
+    if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
+    IdealLoopTree* last_updated_ilt = NULL;
+    for (int i = rpo_list.size() - 1; i >= 0; i--) {
+      Node* c = rpo_list.at(i);
+
+      Node* prev_mem = _memory_nodes[c->_idx];
+      if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
+        Node* prev_region = regions[c->_idx];
+        Node* unique = NULL;
+        for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
+          Node* m = _memory_nodes[c->in(j)->_idx];
+          assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
+          if (m != NULL) {
+            if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
+              assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
+              // continue
+            } else if (unique == NULL) {
+              unique = m;
+            } else if (m == unique) {
+              // continue
+            } else {
+              unique = NodeSentinel;
+            }
+          }
+        }
+        assert(unique != NULL, "empty phi???");
+        if (unique != NodeSentinel) {
+          if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) {
+            dead_phis.push(prev_region);
+          }
+          regions.map(c->_idx, unique);
+        } else {
+          Node* phi = NULL;
+          if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
+            phi = prev_region;
+            for (uint k = 1; k < c->req(); k++) {
+              Node* m = _memory_nodes[c->in(k)->_idx];
+              assert(m != NULL, "expect memory state");
+              phi->set_req(k, m);
+            }
+          } else {
+            for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) {
+              Node* u = c->fast_out(j);
+              if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
+                  (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) {
+                phi = u;
+                for (uint k = 1; k < c->req() && phi != NULL; k++) {
+                  Node* m = _memory_nodes[c->in(k)->_idx];
+                  assert(m != NULL, "expect memory state");
+                  if (u->in(k) != m) {
+                    phi = NULL;
+                  }
+                }
+              }
+            }
+            if (phi == NULL) {
+              phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias));
+              for (uint k = 1; k < c->req(); k++) {
+                Node* m = _memory_nodes[c->in(k)->_idx];
+                assert(m != NULL, "expect memory state");
+                phi->init_req(k, m);
+              }
+            }
+          }
+          assert(phi != NULL, "");
+          regions.map(c->_idx, phi);
+        }
+        Node* current_region = regions[c->_idx];
+        if (current_region != prev_region) {
+          progress = true;
+          if (prev_region == prev_mem) {
+            _memory_nodes.map(c->_idx, current_region);
+          }
+        }
+      } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) {
+        Node* m = _memory_nodes[_phase->idom(c)->_idx];
+        assert(m != NULL, "expect memory state");
+        if (m != prev_mem) {
+          _memory_nodes.map(c->_idx, m);
+          progress = true;
+        }
+      }
+#ifdef ASSERT
+      if (trace) { tty->print("X %d", c->_idx);  _memory_nodes[c->_idx]->dump(); }
+#endif
+    }
+  }
+
+  // Replace existing phi with computed memory state for that region
+  // if different (could be a new phi or a dominating memory node if
+  // that phi was found to be useless).
+  while (dead_phis.size() > 0) {
+    Node* n = dead_phis.pop();
+    n->replace_by(_phase->C->top());
+    n->destruct();
+  }
+  for (int i = rpo_list.size() - 1; i >= 0; i--) {
+    Node* c = rpo_list.at(i);
+    if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
+      Node* n = regions[c->_idx];
+      if (n->is_Phi() && n->_idx >= last && n->in(0) == c) {
+        _phase->register_new_node(n, c);
+      }
+    }
+  }
+  for (int i = rpo_list.size() - 1; i >= 0; i--) {
+    Node* c = rpo_list.at(i);
+    if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
+      Node* n = regions[c->_idx];
+      for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
+        Node* u = c->fast_out(i);
+        if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
+            u != n) {
+          if (u->adr_type() == TypePtr::BOTTOM) {
+            fix_memory_uses(u, n, n, c);
+          } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
+            _phase->lazy_replace(u, n);
+            --i; --imax;
+          }
+        }
+      }
+    }
+  }
+}
+
+Node* MemoryGraphFixer::get_ctrl(Node* n) const {
+  Node* c = _phase->get_ctrl(n);
+  if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
+    assert(c == n->in(0), "");
+    CallNode* call = c->as_Call();
+    CallProjections projs;
+    call->extract_projections(&projs, true, false);
+    if (projs.catchall_memproj != NULL) {
+      if (projs.fallthrough_memproj == n) {
+        c = projs.fallthrough_catchproj;
+      } else {
+        assert(projs.catchall_memproj == n, "");
+        c = projs.catchall_catchproj;
+      }
+    }
+  }
+  return c;
+}
+
+Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
+  if (_phase->has_ctrl(n))
+    return get_ctrl(n);
+  else {
+    assert (n->is_CFG(), "must be a CFG node");
+    return n;
+  }
+}
+
+bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
+  return m != NULL && get_ctrl(m) == c;
+}
+
+Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
+  assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, "");
+  Node* mem = _memory_nodes[ctrl->_idx];
+  Node* c = ctrl;
+  while (!mem_is_valid(mem, c) &&
+         (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) {
+    c = _phase->idom(c);
+    mem = _memory_nodes[c->_idx];
+  }
+  if (n != NULL && mem_is_valid(mem, c)) {
+    while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
+      mem = next_mem(mem, _alias);
+    }
+    if (mem->is_MergeMem()) {
+      mem = mem->as_MergeMem()->memory_at(_alias);
+    }
+    if (!mem_is_valid(mem, c)) {
+      do {
+        c = _phase->idom(c);
+        mem = _memory_nodes[c->_idx];
+      } while (!mem_is_va