changeset 53962:9c18c9d839d3

8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental) Reviewed-by: kvn, roland, shade, coleenp, lmesnik, pliden, jgeorge, ihse, erikj Contributed-by: Christine Flood <chf@redhat.com>, Aleksey Shipilev <shade@redhat.com>, Roland Westrelin <rwestrel@redhat.com>, Zhenygu Gu <zgu@redhat.com>, Andrew Haley <aph@redhat.com>, Andrew Dinn <adinn@redhat.com>, Mario Torre <mtorre@redhat.com>, Roman Kennke <rkennke@redhat.com>
author rkennke
date Mon, 10 Dec 2018 15:47:44 +0100
parents 420ff459906f
children 38bee05fb0e4
files make/autoconf/hotspot.m4 make/hotspot/gensrc/GensrcAdlc.gmk make/hotspot/lib/JvmFeatures.gmk make/hotspot/lib/JvmOverrideFiles.gmk src/hotspot/cpu/aarch64/aarch64.ad src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad src/hotspot/share/adlc/formssel.cpp src/hotspot/share/ci/ciInstanceKlass.cpp src/hotspot/share/ci/ciInstanceKlass.hpp src/hotspot/share/code/codeCache.hpp src/hotspot/share/gc/shared/barrierSetConfig.hpp src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp src/hotspot/share/gc/shared/c2/barrierSetC2.hpp src/hotspot/share/gc/shared/collectedHeap.hpp src/hotspot/share/gc/shared/gcCause.cpp src/hotspot/share/gc/shared/gcCause.hpp src/hotspot/share/gc/shared/gcConfig.cpp src/hotspot/share/gc/shared/gcConfiguration.cpp src/hotspot/share/gc/shared/gcName.hpp src/hotspot/share/gc/shared/gc_globals.hpp src/hotspot/share/gc/shared/referenceProcessor.cpp src/hotspot/share/gc/shared/vmStructs_gc.hpp src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp src/hotspot/share/gc/shenandoah/shenandoahAllocTracker.cpp src/hotspot/share/gc/shenandoah/shenandoahAllocTracker.hpp src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.hpp src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.hpp src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapLock.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.hpp src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp src/hotspot/share/gc/shenandoah/shenandoahHeuristics.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp src/hotspot/share/gc/shenandoah/shenandoahMetrics.cpp src/hotspot/share/gc/shenandoah/shenandoahMetrics.hpp src/hotspot/share/gc/shenandoah/shenandoahMonitoringSupport.cpp src/hotspot/share/gc/shenandoah/shenandoahMonitoringSupport.hpp src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp src/hotspot/share/gc/shenandoah/shenandoahSATBMarkQueueSet.cpp src/hotspot/share/gc/shenandoah/shenandoahSATBMarkQueueSet.hpp src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.cpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.hpp src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp src/hotspot/share/gc/shenandoah/shenandoahStringDedup.hpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.cpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp src/hotspot/share/gc/shenandoah/shenandoahTimingTracker.cpp src/hotspot/share/gc/shenandoah/shenandoahTimingTracker.hpp src/hotspot/share/gc/shenandoah/shenandoahTracer.hpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp src/hotspot/share/gc/shenandoah/shenandoahWorkGroup.cpp src/hotspot/share/gc/shenandoah/shenandoahWorkGroup.hpp src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp src/hotspot/share/memory/metaspace.hpp src/hotspot/share/opto/arraycopynode.cpp src/hotspot/share/opto/cfgnode.hpp src/hotspot/share/opto/classes.cpp src/hotspot/share/opto/classes.hpp src/hotspot/share/opto/compile.cpp src/hotspot/share/opto/compile.hpp src/hotspot/share/opto/lcm.cpp src/hotspot/share/opto/library_call.cpp src/hotspot/share/opto/loopPredicate.cpp src/hotspot/share/opto/loopTransform.cpp src/hotspot/share/opto/loopnode.cpp src/hotspot/share/opto/loopnode.hpp src/hotspot/share/opto/loopopts.cpp src/hotspot/share/opto/macro.cpp src/hotspot/share/opto/node.hpp src/hotspot/share/opto/type.cpp src/hotspot/share/opto/type.hpp src/hotspot/share/runtime/fieldDescriptor.hpp src/hotspot/share/runtime/mutexLocker.cpp src/hotspot/share/runtime/vmOperations.hpp src/hotspot/share/utilities/globalDefinitions.hpp src/hotspot/share/utilities/macros.hpp src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeapName.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/GCCause.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/GCName.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeapRegion.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/VMOps.java src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java test/hotspot/jtreg/TEST.ROOT test/hotspot/jtreg/TEST.groups test/hotspot/jtreg/compiler/c2/aarch64/TestVolatiles.java test/hotspot/jtreg/compiler/c2/aarch64/TestVolatilesShenandoah.java test/hotspot/jtreg/gc/CriticalNativeArgs.java test/hotspot/jtreg/gc/TestFullGCCount.java test/hotspot/jtreg/gc/TestHumongousReferenceObject.java test/hotspot/jtreg/gc/TestSystemGC.java test/hotspot/jtreg/gc/arguments/TestAlignmentToUseLargePages.java test/hotspot/jtreg/gc/arguments/TestDisableDefaultGC.java test/hotspot/jtreg/gc/arguments/TestMaxMinHeapFreeRatioFlags.java test/hotspot/jtreg/gc/arguments/TestNewRatioFlag.java test/hotspot/jtreg/gc/arguments/TestNewSizeFlags.java test/hotspot/jtreg/gc/arguments/TestShrinkHeapInSteps.java test/hotspot/jtreg/gc/arguments/TestSurvivorRatioFlag.java test/hotspot/jtreg/gc/arguments/TestTargetSurvivorRatioFlag.java test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsErgo.java test/hotspot/jtreg/gc/arguments/TestVerifyBeforeAndAfterGCFlags.java test/hotspot/jtreg/gc/class_unloading/TestClassUnloadingDisabled.java test/hotspot/jtreg/gc/epsilon/CriticalNativeArgs.java test/hotspot/jtreg/gc/epsilon/CriticalNativeStress.java test/hotspot/jtreg/gc/epsilon/libCriticalNative.c test/hotspot/jtreg/gc/ergonomics/TestDynamicNumberOfGCThreads.java test/hotspot/jtreg/gc/ergonomics/TestInitialGCThreadLogging.java test/hotspot/jtreg/gc/libCriticalNative.c test/hotspot/jtreg/gc/logging/TestGCId.java test/hotspot/jtreg/gc/metaspace/TestMetaspacePerfCounters.java test/hotspot/jtreg/gc/shenandoah/TestAllocHumongousFragment.java test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java test/hotspot/jtreg/gc/shenandoah/compiler/TestC1ArrayCopyNPE.java test/hotspot/jtreg/gc/shenandoah/compiler/TestC1VectorizedMismatch.java test/hotspot/jtreg/gc/shenandoah/compiler/TestCommonGCLoads.java test/hotspot/jtreg/gc/shenandoah/compiler/TestExpandedWBLostNullCheckDep.java test/hotspot/jtreg/gc/shenandoah/compiler/TestMaybeNullUnsafeAccess.java test/hotspot/jtreg/gc/shenandoah/compiler/TestNullCheck.java test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java test/hotspot/jtreg/gc/shenandoah/compiler/TestWriteBarrierClearControl.java test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java test/hotspot/jtreg/gc/shenandoah/jni/libTestJNICritical.c test/hotspot/jtreg/gc/shenandoah/jni/libTestJNIGlobalRefs.c test/hotspot/jtreg/gc/shenandoah/jni/libTestPinnedGarbage.c test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java test/hotspot/jtreg/gc/shenandoah/jvmti/libTestHeapDump.c test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java test/hotspot/jtreg/gc/shenandoah/options/TestAlwaysPreTouch.java test/hotspot/jtreg/gc/shenandoah/options/TestArgumentRanges.java test/hotspot/jtreg/gc/shenandoah/options/TestClassUnloadingArguments.java test/hotspot/jtreg/gc/shenandoah/options/TestCodeCacheRootStyles.java test/hotspot/jtreg/gc/shenandoah/options/TestEnabled.java test/hotspot/jtreg/gc/shenandoah/options/TestExplicitGC.java test/hotspot/jtreg/gc/shenandoah/options/TestExplicitGCNoConcurrent.java test/hotspot/jtreg/gc/shenandoah/options/TestHeuristicsUnlock.java test/hotspot/jtreg/gc/shenandoah/options/TestHumongousThresholdArgs.java test/hotspot/jtreg/gc/shenandoah/options/TestLoopMiningArguments.java test/hotspot/jtreg/gc/shenandoah/options/TestObjectAlignment.java test/hotspot/jtreg/gc/shenandoah/options/TestPacing.java test/hotspot/jtreg/gc/shenandoah/options/TestParallelRegionStride.java test/hotspot/jtreg/gc/shenandoah/options/TestRegionSizeArgs.java test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java test/hotspot/jtreg/gc/shenandoah/options/TestSingleThreaded.java test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java test/hotspot/jtreg/gc/startup_warnings/TestShenandoah.java test/hotspot/jtreg/gc/stress/CriticalNativeStress.java test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java test/hotspot/jtreg/gc/survivorAlignment/TestAllocationInEden.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromEdenToTenured.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromSurvivorToTenuredAfterFullGC.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionFromSurvivorToTenuredAfterMinorGC.java test/hotspot/jtreg/gc/survivorAlignment/TestPromotionToSurvivor.java test/hotspot/jtreg/gc/whitebox/TestWBGC.java test/hotspot/jtreg/runtime/CompressedOops/UseCompressedOops.java test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java test/hotspot/jtreg/serviceability/sa/ClhsdbJhisto.java test/hotspot/jtreg/serviceability/sa/TestHeapDumpForLargeArray.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCapacityTest.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest01.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest02.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcCauseTest03.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcNewTest.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcTest01.java test/hotspot/jtreg/serviceability/tmtools/jstat/GcTest02.java test/hotspot/jtreg/vmTestbase/nsk/jvmti/scenarios/general_functions/GF08/gf08t001/TestDriver.java test/lib/sun/hotspot/gc/GC.java
diffstat 299 files changed, 40728 insertions(+), 484 deletions(-) [+]
line wrap: on
line diff
--- a/make/autoconf/hotspot.m4	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/autoconf/hotspot.m4	Mon Dec 10 15:47:44 2018 +0100
@@ -25,7 +25,7 @@
 
 # All valid JVM features, regardless of platform
 VALID_JVM_FEATURES="compiler1 compiler2 zero minimal dtrace jvmti jvmci \
-    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc zgc nmt cds \
+    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc shenandoahgc zgc nmt cds \
     static-build link-time-opt aot jfr"
 
 # Deprecated JVM features (these are ignored, but with a warning)
@@ -325,6 +325,15 @@
     fi
   fi
 
+  # Only enable Shenandoah on supported arches
+  AC_MSG_CHECKING([if shenandoah can be built])
+  if test "x$OPENJDK_TARGET_CPU_ARCH" = "xx86" || test "x$OPENJDK_TARGET_CPU" = "xaarch64" ; then
+    AC_MSG_RESULT([yes])
+  else
+    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES shenandoahgc"
+    AC_MSG_RESULT([no, platform not supported])
+  fi
+
   # Only enable ZGC on supported platforms
   AC_MSG_CHECKING([if zgc can be built])
   if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
@@ -336,7 +345,7 @@
 
   # Disable unsupported GCs for Zero
   if HOTSPOT_CHECK_JVM_VARIANT(zero); then
-    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc zgc"
+    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc zgc shenandoahgc"
   fi
 
   # Turn on additional features based on other parts of configure
@@ -470,7 +479,7 @@
   fi
 
   # All variants but minimal (and custom) get these features
-  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc jni-check jvmti management nmt services vm-structs zgc"
+  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc shenandoahgc jni-check jvmti management nmt services vm-structs zgc"
 
   # Disable CDS on AIX.
   if test "x$OPENJDK_TARGET_OS" = "xaix"; then
--- a/make/hotspot/gensrc/GensrcAdlc.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -136,6 +136,12 @@
       $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \
     )))
 
+  ifeq ($(call check-jvm-feature, shenandoahgc), true)
+    AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \
+      )))
+  endif
+
   SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad
 
   INSERT_FILENAME_AWK_SCRIPT := \
--- a/make/hotspot/lib/JvmFeatures.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/lib/JvmFeatures.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -166,6 +166,13 @@
   JVM_EXCLUDE_PATTERNS += gc/z
 endif
 
+ifneq ($(call check-jvm-feature, shenandoahgc), true)
+  JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0
+  JVM_EXCLUDE_PATTERNS += gc/shenandoah
+else
+  JVM_CFLAGS_FEATURES += -DSUPPORT_BARRIER_ON_PRIMITIVES -DSUPPORT_NOT_TO_SPACE_INVARIANT
+endif
+
 ifneq ($(call check-jvm-feature, jfr), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_JFR=0
   JVM_EXCLUDE_PATTERNS += jfr
--- a/make/hotspot/lib/JvmOverrideFiles.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/lib/JvmOverrideFiles.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -36,6 +36,11 @@
   BUILD_LIBJVM_assembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_cardTableBarrierSetAssembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_interp_masm_x86.cpp_CXXFLAGS := -Wno-uninitialized
+  ifeq ($(DEBUG_LEVEL), release)
+    # Need extra inlining to collapse all marking code into the hot marking loop
+    BUILD_LIBJVM_shenandoahConcurrentMark.cpp_CXXFLAGS := --param inline-unit-growth=1000
+    BUILD_LIBJVM_shenandoahTraversalGC.cpp_CXXFLAGS := --param inline-unit-growth=1000
+  endif
 endif
 
 LIBJVM_FDLIBM_COPY_OPT_FLAG := $(CXX_O_FLAG_NONE)
--- a/src/hotspot/cpu/aarch64/aarch64.ad	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/cpu/aarch64/aarch64.ad	Mon Dec 10 15:47:44 2018 +0100
@@ -1,6 +1,6 @@
 //
 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2014, Red Hat Inc. All rights reserved.
+// Copyright (c) 2014, 2018, Red Hat, Inc. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -1272,6 +1272,8 @@
     case Op_CompareAndSwapL:
     case Op_CompareAndSwapP:
     case Op_CompareAndSwapN:
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
     case Op_CompareAndSwapB:
     case Op_CompareAndSwapS:
     case Op_GetAndSetI:
@@ -1293,6 +1295,10 @@
     case Op_WeakCompareAndSwapL:
     case Op_WeakCompareAndSwapP:
     case Op_WeakCompareAndSwapN:
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahWeakCompareAndSwapN:
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
       return maybe_volatile;
     default:
       return false;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,671 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register addr, Register count, RegSet saved_regs) {
+  if (is_oop) {
+    bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+    if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
+      __ push(saved_regs, sp);
+      if (count == c_rarg0) {
+        if (addr == c_rarg1) {
+          // exactly backwards!!
+          __ mov(rscratch1, c_rarg0);
+          __ mov(c_rarg0, c_rarg1);
+          __ mov(c_rarg1, rscratch1);
+        } else {
+          __ mov(c_rarg1, count);
+          __ mov(c_rarg0, addr);
+        }
+      } else {
+        __ mov(c_rarg0, addr);
+        __ mov(c_rarg1, count);
+      }
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
+      } else {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
+      }
+      __ pop(saved_regs, sp);
+    }
+  }
+}
+
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register start, Register end, Register scratch, RegSet saved_regs) {
+  if (is_oop) {
+    __ push(saved_regs, sp);
+    // must compute element count unless barrier set interface is changed (other platforms supply count)
+    assert_different_registers(start, end, scratch);
+    __ lea(scratch, Address(end, BytesPerHeapOop));
+    __ sub(scratch, scratch, start);               // subtract start to get #bytes
+    __ lsr(scratch, scratch, LogBytesPerHeapOop);  // convert to element count
+    __ mov(c_rarg0, start);
+    __ mov(c_rarg1, scratch);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
+    __ pop(saved_regs, sp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+  assert(thread == rthread, "must be");
+
+  Label done;
+  Label runtime;
+
+  assert_different_registers(obj, pre_val, tmp, rscratch1);
+  assert(pre_val != noreg &&  tmp != noreg, "expecting a register");
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  // Is marking active?
+  if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+    __ ldrw(tmp, in_progress);
+  } else {
+    assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+    __ ldrb(tmp, in_progress);
+  }
+  __ cbzw(tmp, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cbz(pre_val, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ ldr(tmp, index);                      // tmp := *index_adr
+  __ cbz(tmp, runtime);                    // tmp == 0?
+                                        // If yes, goto runtime
+
+  __ sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
+  __ str(tmp, index);                      // *index_adr := tmp
+  __ ldr(rscratch1, buffer);
+  __ add(tmp, tmp, rscratch1);             // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ str(pre_val, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(pre_val);
+  if (tosca_live) saved += RegSet::of(r0);
+  if (obj != noreg) saved += RegSet::of(obj);
+
+  __ push(saved, sp);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  if (expand_call) {
+    assert(pre_val != c_rarg1, "smashed arg");
+    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  }
+
+  __ pop(saved, sp);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  Label is_null;
+  __ cbz(dst, is_null);
+  read_barrier_not_null_impl(masm, dst);
+  __ bind(is_null);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_not_null_impl(masm, dst);
+  }
+}
+
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahWriteBarrier) {
+    write_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+  assert(dst != rscratch1, "need rscratch1");
+  assert(dst != rscratch2, "need rscratch2");
+
+  Label done;
+
+  Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(rscratch1, gc_state);
+
+  // Check for heap stability
+  __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ tst(rscratch1, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  // Heap is unstable, need to perform the read-barrier even if WB is inactive
+  __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+
+  // Check for evacuation-in-progress and jump to WB slow-path if needed
+  __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ tst(rscratch1, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  RegSet to_save = RegSet::of(r0);
+  if (dst != r0) {
+    __ push(to_save, sp);
+    __ mov(r0, dst);
+  }
+
+  __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+
+  if (dst != r0) {
+    __ mov(dst, r0);
+    __ pop(to_save, sp);
+  }
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    Label is_null;
+    __ cbz(dst, is_null);
+    write_barrier_impl(masm, dst);
+    __ bind(is_null);
+    // Save possibly live regs.
+    RegSet live_regs = RegSet::range(r0, r4) - dst;
+    __ push(live_regs, sp);
+    __ strd(v0, __ pre(sp, 2 * -wordSize));
+
+    satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
+
+    // Restore possibly live regs.
+    __ ldrd(v0, __ post(sp, 2 * wordSize));
+    __ pop(live_regs, sp);
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                            Register dst, Address src, Register tmp1, Register tmp_thread) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  bool on_reference = on_weak || on_phantom;
+
+  if (in_heap) {
+    read_barrier_not_null(masm, src.base());
+  }
+
+  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
+    __ enter();
+    satb_write_barrier_pre(masm /* masm */,
+                           noreg /* obj */,
+                           dst /* pre_val */,
+                           rthread /* thread */,
+                           tmp1 /* tmp */,
+                           true /* tosca_live */,
+                           true /* expand_call */);
+    __ leave();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                             Address dst, Register val, Register tmp1, Register tmp2) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  if (in_heap) {
+    write_barrier(masm, dst.base());
+  }
+  if (!on_oop) {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+    return;
+  }
+
+  // flatten object address if needed
+  if (dst.index() == noreg && dst.offset() == 0) {
+    if (dst.base() != r3) {
+      __ mov(r3, dst.base());
+    }
+  } else {
+    __ lea(r3, dst);
+  }
+
+  shenandoah_write_barrier_pre(masm,
+                               r3 /* obj */,
+                               tmp2 /* pre_val */,
+                               rthread /* thread */,
+                               tmp1  /* tmp */,
+                               val != noreg /* tosca_live */,
+                               false /* expand_call */);
+
+  if (val == noreg) {
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
+  } else {
+    storeval_barrier(masm, val, tmp1);
+    // G1 barrier needs uncompressed oop for region cross check.
+    Register new_val = val;
+    if (UseCompressedOops) {
+      new_val = rscratch2;
+      __ mov(new_val, val);
+    }
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
+  __ cmp(op1, op2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ br(Assembler::EQ, done);
+    // The object may have been evacuated, but we won't see it without a
+    // membar here.
+    __ membar(Assembler::LoadStore| Assembler::LoadLoad);
+    read_barrier(masm, op1);
+    read_barrier(masm, op2);
+    __ cmp(op1, op2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
+                                                  Register var_size_in_bytes,
+                                                  int con_size_in_bytes,
+                                                  Register t1,
+                                                  Register t2,
+                                                  Label& slow_case) {
+
+  assert_different_registers(obj, t2);
+  assert_different_registers(obj, var_size_in_bytes);
+  Register end = t2;
+
+  __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
+  if (var_size_in_bytes == noreg) {
+    __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
+  } else {
+    __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
+    __ lea(end, Address(obj, var_size_in_bytes));
+  }
+  __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
+  __ cmp(end, rscratch1);
+  __ br(Assembler::HI, slow_case);
+
+  // update the tlab top pointer
+  __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
+
+  __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
+  __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
+
+  // recover var_size_in_bytes if necessary
+  if (var_size_in_bytes == end) {
+    __ sub(var_size_in_bytes, var_size_in_bytes, obj);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
+  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+  bool is_write = (decorators & ACCESS_WRITE) != 0;
+  if (is_write) {
+    if (oop_not_null) {
+      write_barrier(masm, obj);
+    } else {
+      Label done;
+      __ cbz(obj, done);
+      write_barrier(masm, obj);
+      __ bind(done);
+    }
+  } else {
+    if (oop_not_null) {
+      read_barrier_not_null(masm, obj);
+    } else {
+      read_barrier(masm, obj);
+    }
+  }
+}
+
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                                                bool acquire, bool release, bool weak, bool encode,
+                                                Register tmp1, Register tmp2, Register tmp3,
+                                                Register result) {
+
+  if (!ShenandoahCASBarrier) {
+    if (UseCompressedOops) {
+      if (encode) {
+        __ encode_heap_oop(tmp1, expected);
+        expected = tmp1;
+        __ encode_heap_oop(tmp3, new_val);
+        new_val = tmp3;
+      }
+      __ cmpxchg(addr, expected, new_val, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
+      __ membar(__ AnyAny);
+    } else {
+      __ cmpxchg(addr, expected, new_val, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
+      __ membar(__ AnyAny);
+    }
+    return;
+  }
+
+  if (encode) {
+    storeval_barrier(masm, new_val, tmp3);
+  }
+
+  if (UseCompressedOops) {
+    if (encode) {
+      __ encode_heap_oop(tmp1, expected);
+      expected = tmp1;
+      __ encode_heap_oop(tmp2, new_val);
+      new_val = tmp2;
+    }
+  }
+  bool is_cae = (result != noreg);
+  bool is_narrow = UseCompressedOops;
+  Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
+  if (! is_cae) result = rscratch1;
+
+  assert_different_registers(addr, expected, new_val, result, tmp3);
+
+  Label retry, done, fail;
+
+  // CAS, using LL/SC pair.
+  __ bind(retry);
+  __ load_exclusive(result, addr, size, acquire);
+  if (is_narrow) {
+    __ cmpw(result, expected);
+  } else {
+    __ cmp(result, expected);
+  }
+  __ br(Assembler::NE, fail);
+  __ store_exclusive(tmp3, new_val, addr, size, release);
+  if (weak) {
+    __ cmpw(tmp3, 0u); // If the store fails, return NE to our caller
+  } else {
+    __ cbnzw(tmp3, retry);
+  }
+  __ b(done);
+
+ __  bind(fail);
+  // Check if rb(expected)==rb(result)
+  // Shuffle registers so that we have memory value ready for next expected.
+  __ mov(tmp3, expected);
+  __ mov(expected, result);
+  if (is_narrow) {
+    __ decode_heap_oop(result, result);
+    __ decode_heap_oop(tmp3, tmp3);
+  }
+  read_barrier_impl(masm, result);
+  read_barrier_impl(masm, tmp3);
+  __ cmp(result, tmp3);
+  // Retry with expected now being the value we just loaded from addr.
+  __ br(Assembler::EQ, retry);
+  if (is_narrow && is_cae) {
+    // For cmp-and-exchange and narrow oops, we need to restore
+    // the compressed old-value. We moved it to 'expected' a few lines up.
+    __ mov(result, expected);
+  }
+  __ bind(done);
+
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+  __ cbz(pre_val_reg, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ b(*stub->continuation());
+}
+
+void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+
+  Label done;
+
+  __ bind(*stub->entry());
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+  // Check for null.
+  if (stub->needs_null_check()) {
+    __ cbz(res, done);
+  }
+
+  write_barrier(ce->masm(), res);
+
+  __ bind(done);
+  __ b(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+
+  // arg0 : previous value of memory
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+
+  const Register pre_val = r0;
+  const Register thread = rthread;
+  const Register tmp = rscratch1;
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is marking still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(tmp, gc_state);
+  __ mov(rscratch2, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ tst(tmp, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  // Can we store original value in the thread's buffer?
+  __ ldr(tmp, queue_index);
+  __ cbz(tmp, runtime);
+
+  __ sub(tmp, tmp, wordSize);
+  __ str(tmp, queue_index);
+  __ ldr(rscratch2, buffer);
+  __ add(tmp, tmp, rscratch2);
+  __ load_parameter(0, rscratch2);
+  __ str(rscratch2, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+  __ load_parameter(0, pre_val);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  __ pop_call_clobbered_registers();
+  __ bind(done);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_wb() {
+  assert(_shenandoah_wb != NULL, "need write barrier stub");
+  return _shenandoah_wb;
+}
+
+#define __ cgen->assembler()->
+
+// Shenandoah write barrier.
+//
+// Input:
+//   r0: OOP to evacuate.  Not null.
+//
+// Output:
+//   r0: Pointer to evacuated OOP.
+//
+// Trash rscratch1, rscratch2.  Preserve everything else.
+address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+
+  __ align(6);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+  address start = __ pc();
+
+  Label work;
+  __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(rscratch2, Address(rscratch2, rscratch1));
+  __ tbnz(rscratch2, 0, work);
+  __ ret(lr);
+  __ bind(work);
+
+  Register obj = r0;
+
+  __ enter(); // required for proper stackwalking of RuntimeStub frame
+
+  __ push_call_clobbered_registers();
+
+  __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
+  __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
+  __ mov(rscratch1, obj);
+  __ pop_call_clobbered_registers();
+  __ mov(obj, rscratch1);
+
+  __ leave(); // required for proper stackwalking of RuntimeStub frame
+  __ ret(lr);
+
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+    int stub_code_size = 2048;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_wb = generate_shenandoah_wb(&cgen);
+  }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahWriteBarrierStub;
+class StubAssembler;
+class StubCodeGenerator;
+#endif
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_wb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void read_barrier(MacroAssembler* masm, Register dst);
+  void read_barrier_impl(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
+  void write_barrier(MacroAssembler* masm, Register dst);
+  void write_barrier_impl(MacroAssembler* masm, Register dst);
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+  void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2);
+
+  address generate_shenandoah_wb(StubCodeGenerator* cgen);
+
+public:
+  static address shenandoah_wb();
+
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register addr, Register count, RegSet saved_regs);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register start, Register end, Register tmp, RegSet saved_regs);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
+  virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
+  virtual void tlab_allocate(MacroAssembler* masm, Register obj,
+                             Register var_size_in_bytes,
+                             int con_size_in_bytes,
+                             Register t1,
+                             Register t2,
+                             Label& slow_case);
+
+  void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                   bool acquire, bool release, bool weak, bool encode,
+                   Register tmp1, Register tmp2, Register tmp3 = rscratch2,
+                   Register result = noreg);
+
+  virtual void barrier_stubs_init();
+};
+
+#endif // CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  Register addr = _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, true, tmp1, tmp2);
+}
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+  BasicType bt = access.type();
+  if (access.is_oop()) {
+    LIRGenerator *gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item();
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2,
+                                                   LIR_OprFact::illegalOpr));
+
+      LIR_Opr result = gen->new_register(T_INT);
+      __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+               result, T_INT);
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  LIR_Opr tmp = gen->new_register(T_INT);
+  __ xchg(access.resolved_addr(), value_opr, result, tmp);
+
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,281 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+%}
+
+encode %{
+  enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+  %}
+
+  enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+  %}
+%}
+
+instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahReadBarrier src));
+  format %{ "shenandoah_rb $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
+  %}
+  ins_pipe(pipe_class_memory);
+%}
+
+
+instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
+             aarch64_enc_cset_eq(res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+    __ cset($res$$Register, Assembler::EQ);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
+             aarch64_enc_cset_eq(res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+ format %{
+    "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+ %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+    __ cset($res$$Register, Assembler::EQ);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,1041 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register src, Register dst, Register count) {
+
+  bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+  bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+  bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  if (type == T_OBJECT || type == T_ARRAY) {
+#ifdef _LP64
+    if (!checkcast && !obj_int) {
+      // Save count for barrier
+      __ movptr(r11, count);
+    } else if (disjoint && obj_int) {
+      // Save dst in r11 in the disjoint case
+      __ movq(r11, dst);
+    }
+#else
+    if (disjoint) {
+      __ mov(rdx, dst);          // save 'to'
+    }
+#endif
+
+    if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
+      Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+#ifndef _LP64
+      __ push(thread);
+      __ get_thread(thread);
+#endif
+
+      Label filtered;
+      Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+      // Is marking active?
+      if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+        __ cmpl(in_progress, 0);
+      } else {
+        assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+        __ cmpb(in_progress, 0);
+      }
+
+      NOT_LP64(__ pop(thread);)
+
+        __ jcc(Assembler::equal, filtered);
+
+      __ pusha();                      // push registers
+#ifdef _LP64
+      if (count == c_rarg0) {
+        if (dst == c_rarg1) {
+          // exactly backwards!!
+          __ xchgptr(c_rarg1, c_rarg0);
+        } else {
+          __ movptr(c_rarg1, count);
+          __ movptr(c_rarg0, dst);
+        }
+      } else {
+        __ movptr(c_rarg0, dst);
+        __ movptr(c_rarg1, count);
+      }
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
+      } else {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
+      }
+#else
+      __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
+                      dst, count);
+#endif
+      __ popa();
+      __ bind(filtered);
+    }
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register src, Register dst, Register count) {
+  bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+  bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+  bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+  Register tmp = rax;
+
+  if (type == T_OBJECT || type == T_ARRAY) {
+#ifdef _LP64
+    if (!checkcast && !obj_int) {
+      // Save count for barrier
+      count = r11;
+    } else if (disjoint && obj_int) {
+      // Use the saved dst in the disjoint case
+      dst = r11;
+    } else if (checkcast) {
+      tmp = rscratch1;
+    }
+#else
+    if (disjoint) {
+      __ mov(dst, rdx); // restore 'to'
+    }
+#endif
+
+    __ pusha();             // push registers (overkill)
+#ifdef _LP64
+    if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
+      assert_different_registers(c_rarg1, dst);
+      __ mov(c_rarg1, count);
+      __ mov(c_rarg0, dst);
+    } else {
+      assert_different_registers(c_rarg0, count);
+      __ mov(c_rarg0, dst);
+      __ mov(c_rarg1, count);
+    }
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
+#else
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
+                    dst, count);
+#endif
+    __ popa();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+#ifdef _LP64
+  assert(thread == r15_thread, "must be");
+#endif // _LP64
+
+  Label done;
+  Label runtime;
+
+  assert(pre_val != noreg, "check this code");
+
+  if (obj != noreg) {
+    assert_different_registers(obj, pre_val, tmp);
+    assert(pre_val != rax, "check this code");
+  }
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ jcc(Assembler::zero, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cmpptr(pre_val, (int32_t) NULL_WORD);
+  __ jcc(Assembler::equal, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ movptr(tmp, index);                   // tmp := *index_adr
+  __ cmpptr(tmp, 0);                       // tmp == 0?
+  __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
+
+  __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
+  __ movptr(index, tmp);                   // *index_adr := tmp
+  __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+  // save the live input values
+  if(tosca_live) __ push(rax);
+
+  if (obj != noreg && obj != rax)
+    __ push(obj);
+
+  if (pre_val != rax)
+    __ push(pre_val);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  NOT_LP64( __ push(thread); )
+
+#ifdef _LP64
+  // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
+  // pre_val be c_rarg1 (where the call prologue would copy thread argument).
+  // Note: this should not accidentally smash thread, because thread is always r15.
+  assert(thread != c_rarg0, "smashed arg");
+  if (c_rarg0 != pre_val) {
+    __ mov(c_rarg0, pre_val);
+  }
+#endif
+
+  if (expand_call) {
+    LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
+#ifdef _LP64
+    if (c_rarg1 != thread) {
+      __ mov(c_rarg1, thread);
+    }
+    // Already moved pre_val into c_rarg0 above
+#else
+    __ push(thread);
+    __ push(pre_val);
+#endif
+    __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
+  }
+
+  NOT_LP64( __ pop(thread); )
+
+  // save the live input values
+  if (pre_val != rax)
+    __ pop(pre_val);
+
+  if (obj != noreg && obj != rax)
+    __ pop(obj);
+
+  if(tosca_live) __ pop(rax);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  Label is_null;
+  __ testptr(dst, dst);
+  __ jcc(Assembler::zero, is_null);
+  read_barrier_not_null_impl(masm, dst);
+  __ bind(is_null);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_not_null_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+}
+
+
+void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahWriteBarrier) {
+    write_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+#ifdef _LP64
+  Label done;
+
+  Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ jccb(Assembler::zero, done);
+
+  // Heap is unstable, need to perform the read-barrier even if WB is inactive
+  read_barrier_not_null(masm, dst);
+
+  __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ jccb(Assembler::zero, done);
+
+   if (dst != rax) {
+     __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
+   }
+
+   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+
+   if (dst != rax) {
+     __ xchgptr(rax, dst); // Swap back obj with rax.
+   }
+
+  __ bind(done);
+#else
+  Unimplemented();
+#endif
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
+    storeval_barrier_impl(masm, dst, tmp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
+  assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
+
+  if (dst == noreg) return;
+
+#ifdef _LP64
+  if (ShenandoahStoreValEnqueueBarrier) {
+    Label is_null;
+    __ testptr(dst, dst);
+    __ jcc(Assembler::zero, is_null);
+    write_barrier_impl(masm, dst);
+    __ bind(is_null);
+
+    // The set of registers to be saved+restored is the same as in the write-barrier above.
+    // Those are the commonly used registers in the interpreter.
+    __ pusha();
+    // __ push_callee_saved_registers();
+    __ subptr(rsp, 2 * Interpreter::stackElementSize);
+    __ movdbl(Address(rsp, 0), xmm0);
+
+    satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
+    __ movdbl(xmm0, Address(rsp, 0));
+    __ addptr(rsp, 2 * Interpreter::stackElementSize);
+    //__ pop_callee_saved_registers();
+    __ popa();
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+#else
+  Unimplemented();
+#endif
+}
+
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+             Register dst, Address src, Register tmp1, Register tmp_thread) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  bool on_reference = on_weak || on_phantom;
+  if (in_heap) {
+    read_barrier_not_null(masm, src.base());
+  }
+  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
+    const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
+    NOT_LP64(__ get_thread(thread));
+
+    // Generate the SATB pre-barrier code to log the value of
+    // the referent field in an SATB buffer.
+    shenandoah_write_barrier_pre(masm /* masm */,
+                                 noreg /* obj */,
+                                 dst /* pre_val */,
+                                 thread /* thread */,
+                                 tmp1 /* tmp */,
+                                 true /* tosca_live */,
+                                 true /* expand_call */);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+              Address dst, Register val, Register tmp1, Register tmp2) {
+
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool as_normal = (decorators & AS_NORMAL) != 0;
+  if (in_heap) {
+    write_barrier(masm, dst.base());
+  }
+  if (type == T_OBJECT || type == T_ARRAY) {
+    bool needs_pre_barrier = as_normal;
+
+    Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
+    Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
+    // flatten object address if needed
+    // We do it regardless of precise because we need the registers
+    if (dst.index() == noreg && dst.disp() == 0) {
+      if (dst.base() != tmp1) {
+        __ movptr(tmp1, dst.base());
+      }
+    } else {
+      __ lea(tmp1, dst);
+    }
+
+#ifndef _LP64
+    InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
+#endif
+
+    NOT_LP64(__ get_thread(rcx));
+    NOT_LP64(imasm->save_bcp());
+
+    if (needs_pre_barrier) {
+      shenandoah_write_barrier_pre(masm /*masm*/,
+                                   tmp1 /* obj */,
+                                   tmp2 /* pre_val */,
+                                   rthread /* thread */,
+                                   tmp3  /* tmp */,
+                                   val != noreg /* tosca_live */,
+                                   false /* expand_call */);
+    }
+    if (val == noreg) {
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    } else {
+      storeval_barrier(masm, val, tmp3);
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    }
+    NOT_LP64(imasm->restore_bcp());
+  } else {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+  }
+}
+
+#ifndef _LP64
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
+                                               Address obj1, jobject obj2) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
+                                               Register obj1, jobject obj2) {
+  Unimplemented();
+}
+#endif
+
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
+  __ cmpptr(op1, op2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ jccb(Assembler::equal, done);
+    read_barrier(masm, op1);
+    read_barrier(masm, op2);
+    __ cmpptr(op1, op2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
+  __ cmpptr(src1, src2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ jccb(Assembler::equal, done);
+    __ movptr(rscratch2, src2);
+    read_barrier(masm, src1);
+    read_barrier(masm, rscratch2);
+    __ cmpptr(src1, rscratch2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
+                                                  Register thread, Register obj,
+                                                  Register var_size_in_bytes,
+                                                  int con_size_in_bytes,
+                                                  Register t1, Register t2,
+                                                  Label& slow_case) {
+  assert_different_registers(obj, t1, t2);
+  assert_different_registers(obj, var_size_in_bytes, t1);
+  Register end = t2;
+  if (!thread->is_valid()) {
+#ifdef _LP64
+    thread = r15_thread;
+#else
+    assert(t1->is_valid(), "need temp reg");
+    thread = t1;
+    __ get_thread(thread);
+#endif
+  }
+
+  __ verify_tlab();
+
+  __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
+  if (var_size_in_bytes == noreg) {
+    __ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
+  } else {
+    __ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
+    __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
+  }
+  __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
+  __ jcc(Assembler::above, slow_case);
+
+  // update the tlab top pointer
+  __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
+
+  // Initialize brooks pointer
+#ifdef _LP64
+  __ incrementq(obj, ShenandoahBrooksPointer::byte_size());
+#else
+  __ incrementl(obj, ShenandoahBrooksPointer::byte_size());
+#endif
+  __ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
+
+  // recover var_size_in_bytes if necessary
+  if (var_size_in_bytes == end) {
+    __ subptr(var_size_in_bytes, obj);
+  }
+  __ verify_tlab();
+}
+
+void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
+  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+  bool is_write = (decorators & ACCESS_WRITE) != 0;
+  if (is_write) {
+    if (oop_not_null) {
+      write_barrier(masm, obj);
+    } else {
+      Label done;
+      __ testptr(obj, obj);
+      __ jcc(Assembler::zero, done);
+      write_barrier(masm, obj);
+      __ bind(done);
+    }
+  } else {
+    if (oop_not_null) {
+      read_barrier_not_null(masm, obj);
+    } else {
+      read_barrier(masm, obj);
+    }
+  }
+}
+
+// Special Shenandoah CAS implementation that handles false negatives
+// due to concurrent evacuation.
+#ifndef _LP64
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
+                                                Register res, Address addr, Register oldval, Register newval,
+                                                bool exchange, bool encode, Register tmp1, Register tmp2) {
+  // Shenandoah has no 32-bit version for this.
+  Unimplemented();
+}
+#else
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
+                                                Register res, Address addr, Register oldval, Register newval,
+                                                bool exchange, bool encode, Register tmp1, Register tmp2) {
+  if (!ShenandoahCASBarrier) {
+#ifdef _LP64
+    if (UseCompressedOops) {
+      if (encode) {
+        __ encode_heap_oop(oldval);
+        __ mov(rscratch1, newval);
+        __ encode_heap_oop(rscratch1);
+        newval = rscratch1;
+      }
+      if (os::is_MP()) {
+        __ lock();
+      }
+      // oldval (rax) is implicitly used by this instruction
+      __ cmpxchgl(newval, addr);
+    } else
+#endif
+      {
+        if (os::is_MP()) {
+          __ lock();
+        }
+        __ cmpxchgptr(newval, addr);
+      }
+
+    if (!exchange) {
+      assert(res != NULL, "need result register");
+      __ setb(Assembler::equal, res);
+      __ movzbl(res, res);
+    }
+    return;
+  }
+
+  assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
+  assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
+
+  Label retry, done;
+
+  // Apply storeval barrier to newval.
+  if (encode) {
+    storeval_barrier(masm, newval, tmp1);
+  }
+
+  if (UseCompressedOops) {
+    if (encode) {
+      __ encode_heap_oop(oldval);
+      __ mov(rscratch1, newval);
+      __ encode_heap_oop(rscratch1);
+      newval = rscratch1;
+    }
+  }
+
+  // Remember oldval for retry logic below
+  if (UseCompressedOops) {
+    __ movl(tmp1, oldval);
+  } else {
+    __ movptr(tmp1, oldval);
+  }
+
+  // Step 1. Try to CAS with given arguments. If successful, then we are done,
+  // and can safely return.
+  if (os::is_MP()) __ lock();
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else {
+    __ cmpxchgptr(newval, addr);
+  }
+  __ jcc(Assembler::equal, done, true);
+
+  // Step 2. CAS had failed. This may be a false negative.
+  //
+  // The trouble comes when we compare the to-space pointer with the from-space
+  // pointer to the same object. To resolve this, it will suffice to read both
+  // oldval and the value from memory through the read barriers -- this will give
+  // both to-space pointers. If they mismatch, then it was a legitimate failure.
+  //
+  if (UseCompressedOops) {
+    __ decode_heap_oop(tmp1);
+  }
+  read_barrier_impl(masm, tmp1);
+
+  if (UseCompressedOops) {
+    __ movl(tmp2, oldval);
+    __ decode_heap_oop(tmp2);
+  } else {
+    __ movptr(tmp2, oldval);
+  }
+  read_barrier_impl(masm, tmp2);
+
+  __ cmpptr(tmp1, tmp2);
+  __ jcc(Assembler::notEqual, done, true);
+
+  // Step 3. Try to CAS again with resolved to-space pointers.
+  //
+  // Corner case: it may happen that somebody stored the from-space pointer
+  // to memory while we were preparing for retry. Therefore, we can fail again
+  // on retry, and so need to do this in loop, always re-reading the failure
+  // witness through the read barrier.
+  __ bind(retry);
+  if (os::is_MP()) __ lock();
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else {
+    __ cmpxchgptr(newval, addr);
+  }
+  __ jcc(Assembler::equal, done, true);
+
+  if (UseCompressedOops) {
+    __ movl(tmp2, oldval);
+    __ decode_heap_oop(tmp2);
+  } else {
+    __ movptr(tmp2, oldval);
+  }
+  read_barrier_impl(masm, tmp2);
+
+  __ cmpptr(tmp1, tmp2);
+  __ jcc(Assembler::equal, retry, true);
+
+  // Step 4. If we need a boolean result out of CAS, check the flag again,
+  // and promote the result. Note that we handle the flag from both the CAS
+  // itself and from the retry loop.
+  __ bind(done);
+  if (!exchange) {
+    assert(res != NULL, "need result register");
+    __ setb(Assembler::equal, res);
+    __ movzbl(res, res);
+  }
+}
+#endif // LP64
+
+void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
+  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
+  if (UseAVX > 2) {
+    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
+  }
+
+  if (UseSSE == 1)  {
+    __ subptr(rsp, sizeof(jdouble)*8);
+    for (int n = 0; n < 8; n++) {
+      __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
+    }
+  } else if (UseSSE >= 2)  {
+    if (UseAVX > 2) {
+      __ push(rbx);
+      __ movl(rbx, 0xffff);
+      __ kmovwl(k1, rbx);
+      __ pop(rbx);
+    }
+#ifdef COMPILER2
+    if (MaxVectorSize > 16) {
+      if(UseAVX > 2) {
+        // Save upper half of ZMM registers
+        __ subptr(rsp, 32*num_xmm_regs);
+        for (int n = 0; n < num_xmm_regs; n++) {
+          __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
+        }
+      }
+      assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
+      // Save upper half of YMM registers
+      __ subptr(rsp, 16*num_xmm_regs);
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
+      }
+    }
+#endif
+    // Save whole 128bit (16 bytes) XMM registers
+    __ subptr(rsp, 16*num_xmm_regs);
+#ifdef _LP64
+    if (VM_Version::supports_evex()) {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
+      }
+    } else {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
+      }
+    }
+#else
+    for (int n = 0; n < num_xmm_regs; n++) {
+      __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
+    }
+#endif
+  }
+}
+
+void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
+  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
+  if (UseAVX > 2) {
+    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
+  }
+  if (UseSSE == 1)  {
+    for (int n = 0; n < 8; n++) {
+      __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
+    }
+    __ addptr(rsp, sizeof(jdouble)*8);
+  } else if (UseSSE >= 2)  {
+    // Restore whole 128bit (16 bytes) XMM registers
+#ifdef _LP64
+    if (VM_Version::supports_evex()) {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
+      }
+    } else {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
+      }
+    }
+#else
+    for (int n = 0; n < num_xmm_regs; n++) {
+      __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
+    }
+#endif
+    __ addptr(rsp, 16*num_xmm_regs);
+
+#ifdef COMPILER2
+    if (MaxVectorSize > 16) {
+      // Restore upper half of YMM registers.
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
+      }
+      __ addptr(rsp, 16*num_xmm_regs);
+      if (UseAVX > 2) {
+        for (int n = 0; n < num_xmm_regs; n++) {
+          __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
+        }
+        __ addptr(rsp, 32*num_xmm_regs);
+      }
+    }
+#endif
+  }
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+
+  __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
+  __ jcc(Assembler::equal, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ jmp(*stub->continuation());
+
+}
+
+void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+  __ bind(*stub->entry());
+
+  Label done;
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  if (stub->needs_null_check()) {
+    __ testptr(res, res);
+    __ jcc(Assembler::zero, done);
+  }
+
+  write_barrier(ce->masm(), res);
+
+  __ bind(done);
+  __ jmp(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+  // arg0 : previous value of memory
+
+  __ push(rax);
+  __ push(rdx);
+
+  const Register pre_val = rax;
+  const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+  const Register tmp = rdx;
+
+  NOT_LP64(__ get_thread(thread);)
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is SATB still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ jcc(Assembler::zero, done);
+
+  // Can we store original value in the thread's buffer?
+
+  __ movptr(tmp, queue_index);
+  __ testptr(tmp, tmp);
+  __ jcc(Assembler::zero, runtime);
+  __ subptr(tmp, wordSize);
+  __ movptr(queue_index, tmp);
+  __ addptr(tmp, buffer);
+
+  // prev_val (rax)
+  __ load_parameter(0, pre_val);
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+
+  __ save_live_registers_no_oop_map(true);
+
+  // load the pre-value
+  __ load_parameter(0, rcx);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
+
+  __ restore_live_registers(true);
+
+  __ bind(done);
+
+  __ pop(rdx);
+  __ pop(rax);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_wb() {
+  assert(_shenandoah_wb != NULL, "need write barrier stub");
+  return _shenandoah_wb;
+}
+
+#define __ cgen->assembler()->
+
+address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+  __ align(CodeEntryAlignment);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+  address start = __ pc();
+
+#ifdef _LP64
+  Label not_done;
+
+  // We use RDI, which also serves as argument register for slow call.
+  // RAX always holds the src object ptr, except after the slow call and
+  // the cmpxchg, then it holds the result.
+  // R8 and RCX are used as temporary registers.
+  __ push(rdi);
+  __ push(r8);
+
+  // Check for object beeing in the collection set.
+  // TODO: Can we use only 1 register here?
+  // The source object arrives here in rax.
+  // live: rax
+  // live: rdi
+  __ mov(rdi, rax);
+  __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  // live: r8
+  __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(r8, Address(r8, rdi, Address::times_1));
+  // unlive: rdi
+  __ testbool(r8);
+  // unlive: r8
+  __ jccb(Assembler::notZero, not_done);
+
+  __ pop(r8);
+  __ pop(rdi);
+  __ ret(0);
+
+  __ bind(not_done);
+
+  __ push(rcx);
+  __ push(rdx);
+  __ push(rdi);
+  __ push(rsi);
+  __ push(r8);
+  __ push(r9);
+  __ push(r10);
+  __ push(r11);
+  __ push(r12);
+  __ push(r13);
+  __ push(r14);
+  __ push(r15);
+  save_vector_registers(cgen->assembler());
+  __ movptr(rdi, rax);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
+  restore_vector_registers(cgen->assembler());
+  __ pop(r15);
+  __ pop(r14);
+  __ pop(r13);
+  __ pop(r12);
+  __ pop(r11);
+  __ pop(r10);
+  __ pop(r9);
+  __ pop(r8);
+  __ pop(rsi);
+  __ pop(rdi);
+  __ pop(rdx);
+  __ pop(rcx);
+
+  __ pop(r8);
+  __ pop(rdi);
+  __ ret(0);
+#else
+  ShouldNotReachHere();
+#endif
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+    int stub_code_size = 4096;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_wb = generate_shenandoah_wb(&cgen);
+  }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+#define CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahWriteBarrierStub;
+class StubAssembler;
+class StubCodeGenerator;
+#endif
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_wb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void read_barrier(MacroAssembler* masm, Register dst);
+  void read_barrier_impl(MacroAssembler* masm, Register dst);
+
+  void read_barrier_not_null(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
+
+  void write_barrier(MacroAssembler* masm, Register dst);
+  void write_barrier_impl(MacroAssembler* masm, Register dst);
+
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+  void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp);
+
+  address generate_shenandoah_wb(StubCodeGenerator* cgen);
+
+  void save_vector_registers(MacroAssembler* masm);
+  void restore_vector_registers(MacroAssembler* masm);
+
+public:
+  static address shenandoah_wb();
+
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  void cmpxchg_oop(MacroAssembler* masm,
+                   Register res, Address addr, Register oldval, Register newval,
+                   bool exchange, bool encode, Register tmp1, Register tmp2);
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+
+#ifndef _LP64
+  virtual void obj_equals(MacroAssembler* masm,
+                          Address obj1, jobject obj2);
+  virtual void obj_equals(MacroAssembler* masm,
+                          Register obj1, jobject obj2);
+#endif
+
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2);
+
+  virtual void tlab_allocate(MacroAssembler* masm,
+                             Register thread, Register obj,
+                             Register var_size_in_bytes,
+                             int con_size_in_bytes,
+                             Register t1, Register t2,
+                             Label& slow_case);
+
+  virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
+
+  virtual void barrier_stubs_init();
+
+};
+
+#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  Register addr = _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  assert(cmpval == rax, "wrong register");
+  assert(newval != NULL, "new val must be register");
+  assert(cmpval != newval, "cmp and new values must be in different registers");
+  assert(cmpval != addr, "cmp and addr must be in different registers");
+  assert(newval != addr, "new value and addr must be in different registers");
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), NULL, Address(addr, 0), cmpval, newval, true, true, tmp1, tmp2);
+}
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+
+  if (access.is_oop()) {
+    LIRGenerator* gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item_force(FrameMap::rax_oop_opr);
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, LIR_OprFact::illegalOpr));
+
+      LIR_Opr result = gen->new_register(T_INT);
+      __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+               result, T_INT);
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  // Because we want a 2-arg form of xchg and xadd
+  __ move(value_opr, result);
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
+
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,152 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+%}
+
+instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahReadBarrier src));
+  effect(DEF dst, USE src);
+  ins_cost(125); // XXX
+  format %{ "shenandoah_rb $dst, $src" %}
+  ins_encode %{
+    Register d = $dst$$Register;
+    Register s = $src$$Register;
+    __ movptr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+instruct shenandoahRBNarrow(rRegP dst, rRegN src) %{
+  predicate(UseCompressedOops && (Universe::narrow_oop_shift() == 0));
+  match(Set dst (ShenandoahReadBarrier (DecodeN src)));
+  effect(DEF dst, USE src);
+  ins_cost(125); // XXX
+  format %{ "shenandoah_rb $dst, $src" %}
+  ins_encode %{
+    Register d = $dst$$Register;
+    Register s = $src$$Register;
+    __ movptr(d, Address(r12, s, Address::times_1, ShenandoahBrooksPointer::byte_offset()));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+instruct shenandoahRBNarrowShift(rRegP dst, rRegN src) %{
+  predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
+  match(Set dst (ShenandoahReadBarrier (DecodeN src)));
+  effect(DEF dst, USE src);
+  ins_cost(125); // XXX
+  format %{ "shenandoah_rb $dst, $src" %}
+  ins_encode %{
+    Register d = $dst$$Register;
+    Register s = $src$$Register;
+    __ movptr(d, Address(r12, s, Address::times_8, ShenandoahBrooksPointer::byte_offset()));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+instruct compareAndSwapP_shenandoah(rRegI res,
+                                    memory mem_ptr,
+                                    rRegP tmp1, rRegP tmp2,
+                                    rax_RegP oldval, rRegP newval,
+                                    rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_cx8());
+  match(Set res (ShenandoahCompareAndSwapP mem_ptr (Binary oldval newval)));
+  match(Set res (ShenandoahWeakCompareAndSwapP mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   false, // swap
+                                                   false, $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndSwapN_shenandoah(rRegI res,
+                                    memory mem_ptr,
+                                    rRegP tmp1, rRegP tmp2,
+                                    rax_RegN oldval, rRegN newval,
+                                    rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndSwapN mem_ptr (Binary oldval newval)));
+  match(Set res (ShenandoahWeakCompareAndSwapN mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   false, // swap
+                                                   false, $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndExchangeN_shenandoah(memory mem_ptr,
+                                        rax_RegN oldval, rRegN newval,
+                                        rRegP tmp1, rRegP tmp2,
+                                        rFlagsReg cr) %{
+  match(Set oldval (ShenandoahCompareAndExchangeN mem_ptr (Binary oldval newval)));
+  effect(TEMP tmp1, TEMP tmp2, KILL cr);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   true, // exchange
+                                                   false, $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
+
+instruct compareAndExchangeP_shenandoah(memory mem_ptr,
+                                        rax_RegP oldval, rRegP newval,
+                                        rRegP tmp1, rRegP tmp2,
+                                        rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_cx8());
+  match(Set oldval (ShenandoahCompareAndExchangeP mem_ptr (Binary oldval newval)));
+  effect(KILL cr, TEMP tmp1, TEMP tmp2);
+  ins_cost(1000);
+
+  format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
+
+  ins_encode %{
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
+                                                   NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
+                                                   true,  // exchange
+                                                   false, $tmp1$$Register, $tmp2$$Register
+                                                   );
+  %}
+  ins_pipe( pipe_cmpxchg );
+%}
--- a/src/hotspot/share/adlc/formssel.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/adlc/formssel.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -775,7 +775,10 @@
        !strcmp(_matrule->_rChild->_opType,"GetAndSetP")   ||
        !strcmp(_matrule->_rChild->_opType,"GetAndSetN")   ||
        !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") ||
-       !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN")))  return true;
+       !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN") ||
+       !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") ||
+       !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") ||
+       !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier")))  return true;
   else if ( is_ideal_load() == Form::idealP )                return true;
   else if ( is_ideal_store() != Form::none  )                return true;
 
@@ -3498,10 +3501,12 @@
     "CompareAndSwapB", "CompareAndSwapS", "CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN",
     "WeakCompareAndSwapB", "WeakCompareAndSwapS", "WeakCompareAndSwapI", "WeakCompareAndSwapL", "WeakCompareAndSwapP", "WeakCompareAndSwapN",
     "CompareAndExchangeB", "CompareAndExchangeS", "CompareAndExchangeI", "CompareAndExchangeL", "CompareAndExchangeP", "CompareAndExchangeN",
+    "ShenandoahCompareAndSwapN", "ShenandoahCompareAndSwapP", "ShenandoahWeakCompareAndSwapP", "ShenandoahWeakCompareAndSwapN", "ShenandoahCompareAndExchangeP", "ShenandoahCompareAndExchangeN",
     "StoreCM",
     "ClearArray",
     "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
     "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN",
+    "ShenandoahReadBarrier",
     "LoadBarrierSlowReg", "LoadBarrierWeakSlowReg"
   };
   int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*);
--- a/src/hotspot/share/ci/ciInstanceKlass.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -742,3 +742,27 @@
     ik->do_local_static_fields(&sffp);
   }
 }
+
+#ifdef ASSERT
+bool ciInstanceKlass::debug_final_field_at(int offset) {
+  GUARDED_VM_ENTRY(
+    InstanceKlass* ik = get_instanceKlass();
+    fieldDescriptor fd;
+    if (ik->find_field_from_offset(offset, false, &fd)) {
+      return fd.is_final();
+    }
+  );
+  return false;
+}
+
+bool ciInstanceKlass::debug_stable_field_at(int offset) {
+  GUARDED_VM_ENTRY(
+    InstanceKlass* ik = get_instanceKlass();
+    fieldDescriptor fd;
+    if (ik->find_field_from_offset(offset, false, &fd)) {
+      return fd.is_stable();
+    }
+  );
+  return false;
+}
+#endif
--- a/src/hotspot/share/ci/ciInstanceKlass.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/ci/ciInstanceKlass.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -274,6 +274,11 @@
 
   // Dump the current state of this klass for compilation replay.
   virtual void dump_replay_data(outputStream* out);
+
+#ifdef ASSERT
+  bool debug_final_field_at(int offset);
+  bool debug_stable_field_at(int offset);
+#endif
 };
 
 #endif // SHARE_VM_CI_CIINSTANCEKLASS_HPP
--- a/src/hotspot/share/code/codeCache.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/code/codeCache.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -75,6 +75,7 @@
 class ExceptionCache;
 class KlassDepChange;
 class OopClosure;
+class ShenandoahParallelCodeHeapIterator;
 
 class CodeCache : AllStatic {
   friend class VMStructs;
@@ -82,6 +83,7 @@
   template <class T, class Filter> friend class CodeBlobIterator;
   friend class WhiteBox;
   friend class CodeCacheLoader;
+  friend class ShenandoahParallelCodeHeapIterator;
  private:
   // CodeHeaps of the cache
   static GrowableArray<CodeHeap*>* _heaps;
--- a/src/hotspot/share/gc/shared/barrierSetConfig.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -32,6 +32,7 @@
   f(CardTableBarrierSet)                             \
   EPSILONGC_ONLY(f(EpsilonBarrierSet))               \
   G1GC_ONLY(f(G1BarrierSet))                         \
+  SHENANDOAHGC_ONLY(f(ShenandoahBarrierSet))         \
   ZGC_ONLY(f(ZBarrierSet))
 
 #define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f)          \
--- a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -36,6 +36,9 @@
 #if INCLUDE_G1GC
 #include "gc/g1/g1BarrierSet.inline.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/zBarrierSet.inline.hpp"
 #endif
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -314,6 +314,8 @@
   virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {}
 
   virtual Node* split_if_pre(PhaseIdealLoop* phase, Node* n) const { return NULL; }
+  virtual bool build_loop_late_post(PhaseIdealLoop* phase, Node* n) const { return false; }
+  virtual bool sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const { return false; }
 };
 
 #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP
--- a/src/hotspot/share/gc/shared/collectedHeap.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/collectedHeap.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -90,6 +90,7 @@
 //     CMSHeap
 //   G1CollectedHeap
 //   ParallelScavengeHeap
+//   ShenandoahHeap
 //   ZCollectedHeap
 //
 class CollectedHeap : public CHeapObj<mtInternal> {
@@ -176,7 +177,8 @@
     CMS,
     G1,
     Epsilon,
-    Z
+    Z,
+    Shenandoah
   };
 
   static inline size_t filler_array_max_size() {
--- a/src/hotspot/share/gc/shared/gcCause.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gcCause.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -111,6 +111,21 @@
     case _dcmd_gc_run:
       return "Diagnostic Command";
 
+    case _shenandoah_allocation_failure_evac:
+      return "Allocation Failure During Evacuation";
+
+    case _shenandoah_stop_vm:
+      return "Stopping VM";
+
+    case _shenandoah_concurrent_gc:
+      return "Concurrent GC";
+
+    case _shenandoah_traversal_gc:
+      return "Traversal GC";
+
+    case _shenandoah_upgrade_to_full_gc:
+      return "Upgrade To Full GC";
+
     case _z_timer:
       return "Timer";
 
--- a/src/hotspot/share/gc/shared/gcCause.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gcCause.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -80,6 +80,12 @@
 
     _dcmd_gc_run,
 
+    _shenandoah_stop_vm,
+    _shenandoah_allocation_failure_evac,
+    _shenandoah_concurrent_gc,
+    _shenandoah_traversal_gc,
+    _shenandoah_upgrade_to_full_gc,
+
     _z_timer,
     _z_warmup,
     _z_allocation_rate,
@@ -123,7 +129,8 @@
     // _allocation_failure is the generic cause a collection for allocation failure
     // _adaptive_size_policy is for a collecton done before a full GC
     return (cause == GCCause::_allocation_failure ||
-            cause == GCCause::_adaptive_size_policy);
+            cause == GCCause::_adaptive_size_policy ||
+            cause == GCCause::_shenandoah_allocation_failure_evac);
   }
 
   // Return a string describing the GCCause.
--- a/src/hotspot/share/gc/shared/gcConfig.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gcConfig.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -43,6 +43,9 @@
 #if INCLUDE_SERIALGC
 #include "gc/serial/serialArguments.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahArguments.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/zArguments.hpp"
 #endif
@@ -57,23 +60,25 @@
       _flag(flag), _name(name), _arguments(arguments), _hs_err_name(hs_err_name) {}
 };
 
-     CMSGC_ONLY(static CMSArguments      cmsArguments;)
- EPSILONGC_ONLY(static EpsilonArguments  epsilonArguments;)
-      G1GC_ONLY(static G1Arguments       g1Arguments;)
-PARALLELGC_ONLY(static ParallelArguments parallelArguments;)
-  SERIALGC_ONLY(static SerialArguments   serialArguments;)
-       ZGC_ONLY(static ZArguments        zArguments;)
+       CMSGC_ONLY(static CMSArguments      cmsArguments;)
+   EPSILONGC_ONLY(static EpsilonArguments  epsilonArguments;)
+        G1GC_ONLY(static G1Arguments       g1Arguments;)
+  PARALLELGC_ONLY(static ParallelArguments parallelArguments;)
+    SERIALGC_ONLY(static SerialArguments   serialArguments;)
+SHENANDOAHGC_ONLY(static ShenandoahArguments shenandoahArguments;)
+         ZGC_ONLY(static ZArguments        zArguments;)
 
 // Table of supported GCs, for translating between command
 // line flag, CollectedHeap::Name and GCArguments instance.
 static const SupportedGC SupportedGCs[] = {
-       CMSGC_ONLY_ARG(SupportedGC(UseConcMarkSweepGC, CollectedHeap::CMS,      cmsArguments,      "concurrent mark sweep gc"))
-   EPSILONGC_ONLY_ARG(SupportedGC(UseEpsilonGC,       CollectedHeap::Epsilon,  epsilonArguments,  "epsilon gc"))
-        G1GC_ONLY_ARG(SupportedGC(UseG1GC,            CollectedHeap::G1,       g1Arguments,       "g1 gc"))
-  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelGC,      CollectedHeap::Parallel, parallelArguments, "parallel gc"))
-  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelOldGC,   CollectedHeap::Parallel, parallelArguments, "parallel gc"))
-    SERIALGC_ONLY_ARG(SupportedGC(UseSerialGC,        CollectedHeap::Serial,   serialArguments,   "serial gc"))
-         ZGC_ONLY_ARG(SupportedGC(UseZGC,             CollectedHeap::Z,        zArguments,        "z gc"))
+       CMSGC_ONLY_ARG(SupportedGC(UseConcMarkSweepGC, CollectedHeap::CMS,        cmsArguments,        "concurrent mark sweep gc"))
+   EPSILONGC_ONLY_ARG(SupportedGC(UseEpsilonGC,       CollectedHeap::Epsilon,    epsilonArguments,    "epsilon gc"))
+        G1GC_ONLY_ARG(SupportedGC(UseG1GC,            CollectedHeap::G1,         g1Arguments,         "g1 gc"))
+  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelGC,      CollectedHeap::Parallel,   parallelArguments,   "parallel gc"))
+  PARALLELGC_ONLY_ARG(SupportedGC(UseParallelOldGC,   CollectedHeap::Parallel,   parallelArguments,   "parallel gc"))
+    SERIALGC_ONLY_ARG(SupportedGC(UseSerialGC,        CollectedHeap::Serial,     serialArguments,     "serial gc"))
+SHENANDOAHGC_ONLY_ARG(SupportedGC(UseShenandoahGC,    CollectedHeap::Shenandoah, shenandoahArguments, "shenandoah gc"))
+         ZGC_ONLY_ARG(SupportedGC(UseZGC,             CollectedHeap::Z,          zArguments,          "z gc"))
 };
 
 #define FOR_EACH_SUPPORTED_GC(var)                                          \
@@ -90,14 +95,15 @@
 bool GCConfig::_gc_selected_ergonomically = false;
 
 void GCConfig::fail_if_unsupported_gc_is_selected() {
-  NOT_CMSGC(     FAIL_IF_SELECTED(UseConcMarkSweepGC, true));
-  NOT_EPSILONGC( FAIL_IF_SELECTED(UseEpsilonGC,       true));
-  NOT_G1GC(      FAIL_IF_SELECTED(UseG1GC,            true));
-  NOT_PARALLELGC(FAIL_IF_SELECTED(UseParallelGC,      true));
-  NOT_PARALLELGC(FAIL_IF_SELECTED(UseParallelOldGC,   true));
-  NOT_SERIALGC(  FAIL_IF_SELECTED(UseSerialGC,        true));
-  NOT_SERIALGC(  FAIL_IF_SELECTED(UseParallelOldGC,   false));
-  NOT_ZGC(       FAIL_IF_SELECTED(UseZGC,             true));
+  NOT_CMSGC(       FAIL_IF_SELECTED(UseConcMarkSweepGC, true));
+  NOT_EPSILONGC(   FAIL_IF_SELECTED(UseEpsilonGC,       true));
+  NOT_G1GC(        FAIL_IF_SELECTED(UseG1GC,            true));
+  NOT_PARALLELGC(  FAIL_IF_SELECTED(UseParallelGC,      true));
+  NOT_PARALLELGC(  FAIL_IF_SELECTED(UseParallelOldGC,   true));
+  NOT_SERIALGC(    FAIL_IF_SELECTED(UseSerialGC,        true));
+  NOT_SERIALGC(    FAIL_IF_SELECTED(UseParallelOldGC,   false));
+  NOT_SHENANDOAHGC(FAIL_IF_SELECTED(UseShenandoahGC,    true));
+  NOT_ZGC(         FAIL_IF_SELECTED(UseZGC,             true));
 }
 
 void GCConfig::select_gc_ergonomically() {
--- a/src/hotspot/share/gc/shared/gcConfiguration.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -43,7 +43,7 @@
     return ParNew;
   }
 
-  if (UseZGC) {
+  if (UseZGC || UseShenandoahGC) {
     return NA;
   }
 
@@ -67,6 +67,10 @@
     return Z;
   }
 
+  if (UseShenandoahGC) {
+    return Shenandoah;
+  }
+
   return SerialOld;
 }
 
--- a/src/hotspot/share/gc/shared/gcName.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gcName.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,7 @@
   G1Old,
   G1Full,
   Z,
+  Shenandoah,
   NA,
   GCNameEndSentinel
 };
@@ -58,6 +59,7 @@
       case G1Old: return "G1Old";
       case G1Full: return "G1Full";
       case Z: return "Z";
+      case Shenandoah: return "Shenandoah";
       case NA: return "N/A";
       default: ShouldNotReachHere(); return NULL;
     }
--- a/src/hotspot/share/gc/shared/gc_globals.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -41,6 +41,9 @@
 #if INCLUDE_SERIALGC
 #include "gc/serial/serial_globals.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoah_globals.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/z_globals.hpp"
 #endif
@@ -140,6 +143,22 @@
     constraint,                                                             \
     writeable))                                                             \
                                                                             \
+  SHENANDOAHGC_ONLY(GC_SHENANDOAH_FLAGS(                                    \
+    develop,                                                                \
+    develop_pd,                                                             \
+    product,                                                                \
+    product_pd,                                                             \
+    diagnostic,                                                             \
+    diagnostic_pd,                                                          \
+    experimental,                                                           \
+    notproduct,                                                             \
+    manageable,                                                             \
+    product_rw,                                                             \
+    lp64_product,                                                           \
+    range,                                                                  \
+    constraint,                                                             \
+    writeable))                                                             \
+                                                                            \
   ZGC_ONLY(GC_Z_FLAGS(                                                      \
     develop,                                                                \
     develop_pd,                                                             \
@@ -179,6 +198,9 @@
   experimental(bool, UseZGC, false,                                         \
           "Use the Z garbage collector")                                    \
                                                                             \
+  experimental(bool, UseShenandoahGC, false,                                \
+          "Use the Shenandoah garbage collector")                           \
+                                                                            \
   product(uint, ParallelGCThreads, 0,                                       \
           "Number of parallel threads parallel gc will use")                \
           constraint(ParallelGCThreadsConstraintFunc,AfterErgo)             \
--- a/src/hotspot/share/gc/shared/referenceProcessor.cpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/referenceProcessor.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -1156,7 +1156,7 @@
       // Check assumption that an object is not potentially
       // discovered twice except by concurrent collectors that potentially
       // trace the same Reference object twice.
-      assert(UseConcMarkSweepGC || UseG1GC,
+      assert(UseConcMarkSweepGC || UseG1GC || UseShenandoahGC,
              "Only possible with a concurrent marking collector");
       return true;
     }
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -50,6 +50,9 @@
 #include "gc/serial/defNewGeneration.hpp"
 #include "gc/serial/vmStructs_serial.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/vmStructs_shenandoah.hpp"
+#endif
 #if INCLUDE_ZGC
 #include "gc/z/vmStructs_z.hpp"
 #endif
@@ -73,6 +76,9 @@
   SERIALGC_ONLY(VM_STRUCTS_SERIALGC(nonstatic_field,                                                                                 \
                                     volatile_nonstatic_field,                                                                        \
                                     static_field))                                                                                   \
+  SHENANDOAHGC_ONLY(VM_STRUCTS_SHENANDOAH(nonstatic_field,                                                                           \
+                               volatile_nonstatic_field,                                                                             \
+                               static_field))                                                                                        \
   ZGC_ONLY(VM_STRUCTS_ZGC(nonstatic_field,                                                                                           \
                           volatile_nonstatic_field,                                                                                  \
                           static_field))                                                                                             \
@@ -178,6 +184,9 @@
   SERIALGC_ONLY(VM_TYPES_SERIALGC(declare_type,                           \
                                   declare_toplevel_type,                  \
                                   declare_integer_type))                  \
+  SHENANDOAHGC_ONLY(VM_TYPES_SHENANDOAH(declare_type,                     \
+                             declare_toplevel_type,                       \
+                             declare_integer_type))                       \
   ZGC_ONLY(VM_TYPES_ZGC(declare_type,                                     \
                         declare_toplevel_type,                            \
                         declare_integer_type))                            \
@@ -253,6 +262,8 @@
                                               declare_constant_with_value)) \
   SERIALGC_ONLY(VM_INT_CONSTANTS_SERIALGC(declare_constant,                 \
                                           declare_constant_with_value))     \
+  SHENANDOAHGC_ONLY(VM_INT_CONSTANTS_SHENANDOAH(declare_constant,           \
+                                     declare_constant_with_value))          \
   ZGC_ONLY(VM_INT_CONSTANTS_ZGC(declare_constant,                           \
                                 declare_constant_with_value))               \
                                                                             \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_IR.hpp"
+#include "gc/shared/satbMarkQueue.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+#ifndef PATCHED_ADDR
+#define PATCHED_ADDR  (max_jint)
+#endif
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+void ShenandoahPreBarrierStub::emit_code(LIR_Assembler* ce) {
+  ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->gen_pre_barrier_stub(ce, this);
+}
+
+void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) {
+  ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->gen_write_barrier_stub(ce, this);
+}
+
+void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
+  // First we test whether marking is in progress.
+  BasicType flag_type;
+  bool patch = (decorators & C1_NEEDS_PATCHING) != 0;
+  bool do_load = pre_val == LIR_OprFact::illegalOpr;
+  if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+    flag_type = T_INT;
+  } else {
+    guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1,
+              "Assumption");
+    // Use unsigned type T_BOOLEAN here rather than signed T_BYTE since some platforms, eg. ARM,
+    // need to use unsigned instructions to use the large offset to load the satb_mark_queue.
+    flag_type = T_BOOLEAN;
+  }
+  LIR_Opr thrd = gen->getThreadPointer();
+  LIR_Address* mark_active_flag_addr =
+    new LIR_Address(thrd,
+                    in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()),
+                    flag_type);
+  // Read the marking-in-progress flag.
+  LIR_Opr flag_val = gen->new_register(T_INT);
+  __ load(mark_active_flag_addr, flag_val);
+  __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+
+  LIR_PatchCode pre_val_patch_code = lir_patch_none;
+
+  CodeStub* slow;
+
+  if (do_load) {
+    assert(pre_val == LIR_OprFact::illegalOpr, "sanity");
+    assert(addr_opr != LIR_OprFact::illegalOpr, "sanity");
+
+    if (patch)
+      pre_val_patch_code = lir_patch_normal;
+
+    pre_val = gen->new_register(T_OBJECT);
+
+    if (!addr_opr->is_address()) {
+      assert(addr_opr->is_register(), "must be");
+      addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT));
+    }
+    slow = new ShenandoahPreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info ? new CodeEmitInfo(info) : NULL);
+  } else {
+    assert(addr_opr == LIR_OprFact::illegalOpr, "sanity");
+    assert(pre_val->is_register(), "must be");
+    assert(pre_val->type() == T_OBJECT, "must be an object");
+
+    slow = new ShenandoahPreBarrierStub(pre_val);
+  }
+
+  __ branch(lir_cond_notEqual, T_INT, slow);
+  __ branch_destination(slow->continuation());
+}
+
+LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  if (UseShenandoahGC && ShenandoahReadBarrier) {
+    return read_barrier_impl(gen, obj, info, need_null_check);
+  } else {
+    return obj;
+  }
+}
+
+LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled");
+  LabelObj* done = new LabelObj();
+  LIR_Opr result = gen->new_register(T_OBJECT);
+  __ move(obj, result);
+  if (need_null_check) {
+    __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
+    __ branch(lir_cond_equal, T_LONG, done->label());
+  }
+  LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS);
+  __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none);
+
+  __ branch_destination(done->label());
+  return result;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  if (UseShenandoahGC && ShenandoahWriteBarrier) {
+    return write_barrier_impl(gen, obj, info, need_null_check);
+  } else {
+    return obj;
+  }
+}
+
+LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+
+  obj = ensure_in_register(gen, obj);
+  assert(obj->is_register(), "must be a register at this point");
+  LIR_Opr result = gen->new_register(T_OBJECT);
+  __ move(obj, result);
+
+  LIR_Opr thrd = gen->getThreadPointer();
+  LIR_Address* active_flag_addr =
+    new LIR_Address(thrd,
+                    in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
+                    T_BYTE);
+  // Read and check the gc-state-flag.
+  LIR_Opr flag_val = gen->new_register(T_INT);
+  __ load(active_flag_addr, flag_val);
+  LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED |
+                                       ShenandoahHeap::EVACUATION |
+                                       ShenandoahHeap::TRAVERSAL);
+  LIR_Opr mask_reg = gen->new_register(T_INT);
+  __ move(mask, mask_reg);
+
+  if (TwoOperandLIRForm) {
+    __ logical_and(flag_val, mask_reg, flag_val);
+  } else {
+    LIR_Opr masked_flag = gen->new_register(T_INT);
+    __ logical_and(flag_val, mask_reg, masked_flag);
+    flag_val = masked_flag;
+  }
+  __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+
+  CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check);
+  __ branch(lir_cond_notEqual, T_INT, slow);
+  __ branch_destination(slow->continuation());
+
+  return result;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) {
+  if (!obj->is_register()) {
+    LIR_Opr obj_reg = gen->new_register(T_OBJECT);
+    if (obj->is_constant()) {
+      __ move(obj, obj_reg);
+    } else {
+      __ leal(obj, obj_reg);
+    }
+    obj = obj_reg;
+  }
+  return obj;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) {
+  bool need_null_check = (decorators & IS_NOT_NULL) == 0;
+  if (ShenandoahStoreValEnqueueBarrier) {
+    obj = write_barrier_impl(gen, obj, info, need_null_check);
+    pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj);
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/);
+  }
+  return obj;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
+  DecoratorSet decorators = access.decorators();
+  bool is_array = (decorators & IS_ARRAY) != 0;
+  bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0;
+
+  bool is_write = (decorators & ACCESS_WRITE) != 0;
+  bool needs_null_check = (decorators & IS_NOT_NULL) == 0;
+
+  LIR_Opr base = access.base().item().result();
+  LIR_Opr offset = access.offset().opr();
+  LIRGenerator* gen = access.gen();
+
+  if (is_write) {
+    base = write_barrier(gen, base, access.access_emit_info(), needs_null_check);
+  } else {
+    base = read_barrier(gen, base, access.access_emit_info(), needs_null_check);
+  }
+
+  LIR_Opr addr_opr;
+  if (is_array) {
+    addr_opr = LIR_OprFact::address(gen->emit_array_address(base, offset, access.type()));
+  } else if (needs_patching) {
+    // we need to patch the offset in the instruction so don't allow
+    // generate_address to try to be smart about emitting the -1.
+    // Otherwise the patching code won't know how to find the
+    // instruction to patch.
+    addr_opr = LIR_OprFact::address(new LIR_Address(base, PATCHED_ADDR, access.type()));
+  } else {
+    addr_opr = LIR_OprFact::address(gen->generate_address(base, offset, 0, 0, access.type()));
+  }
+
+  if (resolve_in_register) {
+    LIR_Opr resolved_addr = gen->new_pointer_register();
+    __ leal(addr_opr, resolved_addr);
+    resolved_addr = LIR_OprFact::address(new LIR_Address(resolved_addr, access.type()));
+    return resolved_addr;
+  } else {
+    return addr_opr;
+  }
+}
+
+void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    value = storeval_barrier(access.gen(), value, access.access_emit_info(), access.decorators());
+  }
+  BarrierSetC1::store_at_resolved(access, value);
+}
+
+void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
+  BarrierSetC1::load_at_resolved(access, result);
+
+  if (ShenandoahKeepAliveBarrier) {
+    DecoratorSet decorators = access.decorators();
+    bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+    bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+    bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+    LIRGenerator *gen = access.gen();
+    if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) {
+      // Register the value in the referent field with the pre-barrier
+      LabelObj *Lcont_anonymous;
+      if (is_anonymous) {
+        Lcont_anonymous = new LabelObj();
+        generate_referent_check(access, Lcont_anonymous);
+      }
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
+                  result /* pre_val */);
+      if (is_anonymous) {
+        __ branch_destination(Lcont_anonymous->label());
+      }
+    }
+  }
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_add_at_resolved(LIRAccess& access, LIRItem& value) {
+  return BarrierSetC1::atomic_add_at_resolved(access, value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj) {
+  bool is_write = decorators & ACCESS_WRITE;
+  if (is_write) {
+    return write_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
+  } else {
+    return read_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
+  }
+}
+
+class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+    bs->generate_c1_pre_barrier_runtime_stub(sasm);
+    return NULL;
+  }
+};
+
+void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) {
+  C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl;
+  _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1,
+                                                              "shenandoah_pre_barrier_slow",
+                                                              false, &pre_code_gen_cl);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
+#define SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
+
+#include "c1/c1_CodeStubs.hpp"
+#include "gc/shared/c1/barrierSetC1.hpp"
+
+class ShenandoahPreBarrierStub: public CodeStub {
+  friend class ShenandoahBarrierSetC1;
+private:
+  bool _do_load;
+  LIR_Opr _addr;
+  LIR_Opr _pre_val;
+  LIR_PatchCode _patch_code;
+  CodeEmitInfo* _info;
+
+public:
+  // Version that _does_ generate a load of the previous value from addr.
+  // addr (the address of the field to be read) must be a LIR_Address
+  // pre_val (a temporary register) must be a register;
+  ShenandoahPreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) :
+    _do_load(true), _addr(addr), _pre_val(pre_val),
+    _patch_code(patch_code), _info(info)
+  {
+    assert(_pre_val->is_register(), "should be temporary register");
+    assert(_addr->is_address(), "should be the address of the field");
+  }
+
+  // Version that _does not_ generate load of the previous value; the
+  // previous value is assumed to have already been loaded into pre_val.
+  ShenandoahPreBarrierStub(LIR_Opr pre_val) :
+    _do_load(false), _addr(LIR_OprFact::illegalOpr), _pre_val(pre_val),
+    _patch_code(lir_patch_none), _info(NULL)
+  {
+    assert(_pre_val->is_register(), "should be a register");
+  }
+
+  LIR_Opr addr() const { return _addr; }
+  LIR_Opr pre_val() const { return _pre_val; }
+  LIR_PatchCode patch_code() const { return _patch_code; }
+  CodeEmitInfo* info() const { return _info; }
+  bool do_load() const { return _do_load; }
+
+  virtual void emit_code(LIR_Assembler* e);
+  virtual void visit(LIR_OpVisitState* visitor) {
+    if (_do_load) {
+      // don't pass in the code emit info since it's processed in the fast
+      // path
+      if (_info != NULL)
+        visitor->do_slow_case(_info);
+      else
+        visitor->do_slow_case();
+
+      visitor->do_input(_addr);
+      visitor->do_temp(_pre_val);
+    } else {
+      visitor->do_slow_case();
+      visitor->do_input(_pre_val);
+    }
+  }
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const { out->print("ShenandoahPreBarrierStub"); }
+#endif // PRODUCT
+};
+
+class ShenandoahWriteBarrierStub: public CodeStub {
+  friend class ShenandoahBarrierSetC1;
+private:
+  LIR_Opr _obj;
+  LIR_Opr _result;
+  CodeEmitInfo* _info;
+  bool _needs_null_check;
+
+public:
+  ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) :
+    _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check)
+  {
+    assert(_obj->is_register(), "should be register");
+    assert(_result->is_register(), "should be register");
+  }
+
+  LIR_Opr obj() const { return _obj; }
+  LIR_Opr result() const { return _result; }
+  CodeEmitInfo* info() const { return _info; }
+  bool needs_null_check() const { return _needs_null_check; }
+
+  virtual void emit_code(LIR_Assembler* e);
+  virtual void visit(LIR_OpVisitState* visitor) {
+    visitor->do_slow_case();
+    visitor->do_input(_obj);
+    visitor->do_temp(_result);
+  }
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); }
+#endif // PRODUCT
+};
+
+class LIR_OpShenandoahCompareAndSwap : public LIR_Op {
+ friend class LIR_OpVisitState;
+
+private:
+  LIR_Opr _addr;
+  LIR_Opr _cmp_value;
+  LIR_Opr _new_value;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
+
+public:
+  LIR_OpShenandoahCompareAndSwap(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value,
+                                 LIR_Opr t1, LIR_Opr t2, LIR_Opr result)
+    : LIR_Op(lir_none, result, NULL)  // no info
+    , _addr(addr)
+    , _cmp_value(cmp_value)
+    , _new_value(new_value)
+    , _tmp1(t1)
+    , _tmp2(t2)                                  { }
+
+  LIR_Opr addr()        const                    { return _addr;  }
+  LIR_Opr cmp_value()   const                    { return _cmp_value; }
+  LIR_Opr new_value()   const                    { return _new_value; }
+  LIR_Opr tmp1()        const                    { return _tmp1;      }
+  LIR_Opr tmp2()        const                    { return _tmp2;      }
+
+  virtual void visit(LIR_OpVisitState* state) {
+      assert(_addr->is_valid(),      "used");
+      assert(_cmp_value->is_valid(), "used");
+      assert(_new_value->is_valid(), "used");
+      if (_info)                    state->do_info(_info);
+                                    state->do_input(_addr);
+                                    state->do_temp(_addr);
+                                    state->do_input(_cmp_value);
+                                    state->do_temp(_cmp_value);
+                                    state->do_input(_new_value);
+                                    state->do_temp(_new_value);
+      if (_tmp1->is_valid())        state->do_temp(_tmp1);
+      if (_tmp2->is_valid())        state->do_temp(_tmp2);
+      if (_result->is_valid())      state->do_output(_result);
+  }
+
+  virtual void emit_code(LIR_Assembler* masm);
+
+  virtual void print_instr(outputStream* out) const {
+    addr()->print(out);      out->print(" ");
+    cmp_value()->print(out); out->print(" ");
+    new_value()->print(out); out->print(" ");
+    tmp1()->print(out);      out->print(" ");
+    tmp2()->print(out);      out->print(" ");
+  }
+#ifndef PRODUCT
+  virtual const char* name() const {
+    return "shenandoah_cas_obj";
+  }
+#endif // PRODUCT
+};
+
+class ShenandoahBarrierSetC1 : public BarrierSetC1 {
+private:
+  CodeBlob* _pre_barrier_c1_runtime_code_blob;
+
+  void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
+
+  LIR_Opr read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+  LIR_Opr write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+  LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);
+
+  LIR_Opr read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+  LIR_Opr write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+
+  LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj);
+
+public:
+  CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; }
+
+protected:
+  virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register);
+
+  virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);
+  virtual void load_at_resolved(LIRAccess& access, LIR_Opr result);
+
+  virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value);
+
+  virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
+  virtual LIR_Opr atomic_add_at_resolved(LIRAccess& access, LIRItem& value);
+
+public:
+  virtual LIR_Opr resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj);
+
+  virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob);
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,1545 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "opto/arraycopynode.hpp"
+#include "opto/escape.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/idealKit.hpp"
+#include "opto/macro.hpp"
+#include "opto/movenode.hpp"
+#include "opto/narrowptrnode.hpp"
+#include "opto/rootnode.hpp"
+
+ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
+  return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
+}
+
+ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
+  : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8,  0, NULL)) {
+}
+
+int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {
+  return _shenandoah_barriers->length();
+}
+
+ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {
+  return _shenandoah_barriers->at(idx);
+}
+
+void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
+  assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");
+  _shenandoah_barriers->append(n);
+}
+
+void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
+  if (_shenandoah_barriers->contains(n)) {
+    _shenandoah_barriers->remove(n);
+  }
+}
+
+#define __ kit->
+
+Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {
+  if (ShenandoahReadBarrier) {
+    obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);
+  }
+  return obj;
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    obj = shenandoah_write_barrier(kit, obj);
+    obj = shenandoah_enqueue_barrier(kit, obj);
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
+  }
+  return obj;
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {
+  const Type* obj_type = obj->bottom_type();
+  if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
+    return obj;
+  }
+  const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
+  Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
+
+  if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
+    // We know it is null, no barrier needed.
+    return obj;
+  }
+
+  if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
+
+    // We don't know if it's null or not. Need null-check.
+    enum { _not_null_path = 1, _null_path, PATH_LIMIT };
+    RegionNode* region = new RegionNode(PATH_LIMIT);
+    Node*       phi    = new PhiNode(region, obj_type);
+    Node* null_ctrl = __ top();
+    Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
+
+    region->init_req(_null_path, null_ctrl);
+    phi   ->init_req(_null_path, __ zerocon(T_OBJECT));
+
+    Node* ctrl = use_ctrl ? __ control() : NULL;
+    ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
+    Node* n = __ gvn().transform(rb);
+
+    region->init_req(_not_null_path, __ control());
+    phi   ->init_req(_not_null_path, n);
+
+    __ set_control(__ gvn().transform(region));
+    __ record_for_igvn(region);
+    return __ gvn().transform(phi);
+
+  } else {
+    // We know it is not null. Simple barrier is sufficient.
+    Node* ctrl = use_ctrl ? __ control() : NULL;
+    ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
+    Node* n = __ gvn().transform(rb);
+    __ record_for_igvn(n);
+    return n;
+  }
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
+  ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
+  Node* n = __ gvn().transform(wb);
+  if (n == wb) { // New barrier needs memory projection.
+    Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
+    __ set_memory(proj, adr_type);
+  }
+  return n;
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
+  if (ShenandoahWriteBarrier) {
+    obj = shenandoah_write_barrier_impl(kit, obj);
+  }
+  return obj;
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
+  if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
+    return obj;
+  }
+  const Type* obj_type = obj->bottom_type();
+  const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
+  Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
+  __ record_for_igvn(n);
+  return n;
+}
+
+bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
+                                                         BasicType bt, uint adr_idx) const {
+  intptr_t offset = 0;
+  Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
+  AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
+
+  if (offset == Type::OffsetBot) {
+    return false; // cannot unalias unless there are precise offsets
+  }
+
+  if (alloc == NULL) {
+    return false; // No allocation found
+  }
+
+  intptr_t size_in_bytes = type2aelembytes(bt);
+
+  Node* mem = __ memory(adr_idx); // start searching here...
+
+  for (int cnt = 0; cnt < 50; cnt++) {
+
+    if (mem->is_Store()) {
+
+      Node* st_adr = mem->in(MemNode::Address);
+      intptr_t st_offset = 0;
+      Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
+
+      if (st_base == NULL) {
+        break; // inscrutable pointer
+      }
+
+      // Break we have found a store with same base and offset as ours so break
+      if (st_base == base && st_offset == offset) {
+        break;
+      }
+
+      if (st_offset != offset && st_offset != Type::OffsetBot) {
+        const int MAX_STORE = BytesPerLong;
+        if (st_offset >= offset + size_in_bytes ||
+            st_offset <= offset - MAX_STORE ||
+            st_offset <= offset - mem->as_Store()->memory_size()) {
+          // Success:  The offsets are provably independent.
+          // (You may ask, why not just test st_offset != offset and be done?
+          // The answer is that stores of different sizes can co-exist
+          // in the same sequence of RawMem effects.  We sometimes initialize
+          // a whole 'tile' of array elements with a single jint or jlong.)
+          mem = mem->in(MemNode::Memory);
+          continue; // advance through independent store memory
+        }
+      }
+
+      if (st_base != base
+          && MemNode::detect_ptr_independence(base, alloc, st_base,
+                                              AllocateNode::Ideal_allocation(st_base, phase),
+                                              phase)) {
+        // Success:  The bases are provably independent.
+        mem = mem->in(MemNode::Memory);
+        continue; // advance through independent store memory
+      }
+    } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
+
+      InitializeNode* st_init = mem->in(0)->as_Initialize();
+      AllocateNode* st_alloc = st_init->allocation();
+
+      // Make sure that we are looking at the same allocation site.
+      // The alloc variable is guaranteed to not be null here from earlier check.
+      if (alloc == st_alloc) {
+        // Check that the initialization is storing NULL so that no previous store
+        // has been moved up and directly write a reference
+        Node* captured_store = st_init->find_captured_store(offset,
+                                                            type2aelembytes(T_OBJECT),
+                                                            phase);
+        if (captured_store == NULL || captured_store == st_init->zero_memory()) {
+          return true;
+        }
+      }
+    }
+
+    // Unless there is an explicit 'continue', we must bail out here,
+    // because 'mem' is an inscrutable memory state (e.g., a call).
+    break;
+  }
+
+  return false;
+}
+
+#undef __
+#define __ ideal.
+
+void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,
+                                                    bool do_load,
+                                                    Node* obj,
+                                                    Node* adr,
+                                                    uint alias_idx,
+                                                    Node* val,
+                                                    const TypeOopPtr* val_type,
+                                                    Node* pre_val,
+                                                    BasicType bt) const {
+  // Some sanity checks
+  // Note: val is unused in this routine.
+
+  if (do_load) {
+    // We need to generate the load of the previous value
+    assert(obj != NULL, "must have a base");
+    assert(adr != NULL, "where are loading from?");
+    assert(pre_val == NULL, "loaded already?");
+    assert(val_type != NULL, "need a type");
+
+    if (ReduceInitialCardMarks
+        && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
+      return;
+    }
+
+  } else {
+    // In this case both val_type and alias_idx are unused.
+    assert(pre_val != NULL, "must be loaded already");
+    // Nothing to be done if pre_val is null.
+    if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
+    assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
+  }
+  assert(bt == T_OBJECT, "or we shouldn't be here");
+
+  IdealKit ideal(kit, true);
+
+  Node* tls = __ thread(); // ThreadLocalStorage
+
+  Node* no_base = __ top();
+  Node* zero  = __ ConI(0);
+  Node* zeroX = __ ConX(0);
+
+  float likely  = PROB_LIKELY(0.999);
+  float unlikely  = PROB_UNLIKELY(0.999);
+
+  // Offsets into the thread
+  const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
+  const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
+
+  // Now the actual pointers into the thread
+  Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
+  Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
+
+  // Now some of the values
+  Node* marking;
+  Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
+  Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
+  marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
+  assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");
+
+  // if (!marking)
+  __ if_then(marking, BoolTest::ne, zero, unlikely); {
+    BasicType index_bt = TypeX_X->basic_type();
+    assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
+    Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
+
+    if (do_load) {
+      // load original value
+      // alias_idx correct??
+      pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
+    }
+
+    // if (pre_val != NULL)
+    __ if_then(pre_val, BoolTest::ne, kit->null()); {
+      Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
+
+      // is the queue for this thread full?
+      __ if_then(index, BoolTest::ne, zeroX, likely); {
+
+        // decrement the index
+        Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
+
+        // Now get the buffer location we will log the previous value into and store it
+        Node *log_addr = __ AddP(no_base, buffer, next_index);
+        __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
+        // update the index
+        __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
+
+      } __ else_(); {
+
+        // logging buffer is full, call the runtime
+        const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();
+        __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);
+      } __ end_if();  // (!index)
+    } __ end_if();  // (pre_val != NULL)
+  } __ end_if();  // (!marking)
+
+  // Final sync IdealKit and GraphKit.
+  kit->final_sync(ideal);
+
+  if (ShenandoahSATBBarrier && adr != NULL) {
+    Node* c = kit->control();
+    Node* call = c->in(1)->in(1)->in(1)->in(0);
+    assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
+    call->add_req(adr);
+  }
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
+  return call->is_CallLeaf() &&
+         call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
+  return call->is_CallLeaf() &&
+         call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
+  if (n->Opcode() != Op_If) {
+    return false;
+  }
+
+  Node* bol = n->in(1);
+  assert(bol->is_Bool(), "");
+  Node* cmpx = bol->in(1);
+  if (bol->as_Bool()->_test._test == BoolTest::ne &&
+      cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
+      is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
+      cmpx->in(1)->in(2)->is_Con() &&
+      cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
+    return true;
+  }
+
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
+  if (!n->is_Load()) return false;
+  const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
+  return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
+         && n->in(2)->in(3)->is_Con()
+         && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
+}
+
+void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
+                                                          bool do_load,
+                                                          Node* obj,
+                                                          Node* adr,
+                                                          uint alias_idx,
+                                                          Node* val,
+                                                          const TypeOopPtr* val_type,
+                                                          Node* pre_val,
+                                                          BasicType bt) const {
+  if (ShenandoahSATBBarrier) {
+    IdealKit ideal(kit);
+    kit->sync_kit(ideal);
+
+    satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
+
+    ideal.sync_kit(kit);
+    kit->final_sync(ideal);
+  }
+}
+
+Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {
+  return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));
+}
+
+// Helper that guards and inserts a pre-barrier.
+void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
+                                                Node* pre_val, bool need_mem_bar) const {
+  // We could be accessing the referent field of a reference object. If so, when G1
+  // is enabled, we need to log the value in the referent field in an SATB buffer.
+  // This routine performs some compile time filters and generates suitable
+  // runtime filters that guard the pre-barrier code.
+  // Also add memory barrier for non volatile load from the referent field
+  // to prevent commoning of loads across safepoint.
+
+  // Some compile time checks.
+
+  // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
+  const TypeX* otype = offset->find_intptr_t_type();
+  if (otype != NULL && otype->is_con() &&
+      otype->get_con() != java_lang_ref_Reference::referent_offset) {
+    // Constant offset but not the reference_offset so just return
+    return;
+  }
+
+  // We only need to generate the runtime guards for instances.
+  const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
+  if (btype != NULL) {
+    if (btype->isa_aryptr()) {
+      // Array type so nothing to do
+      return;
+    }
+
+    const TypeInstPtr* itype = btype->isa_instptr();
+    if (itype != NULL) {
+      // Can the klass of base_oop be statically determined to be
+      // _not_ a sub-class of Reference and _not_ Object?
+      ciKlass* klass = itype->klass();
+      if ( klass->is_loaded() &&
+          !klass->is_subtype_of(kit->env()->Reference_klass()) &&
+          !kit->env()->Object_klass()->is_subtype_of(klass)) {
+        return;
+      }
+    }
+  }
+
+  // The compile time filters did not reject base_oop/offset so
+  // we need to generate the following runtime filters
+  //
+  // if (offset == java_lang_ref_Reference::_reference_offset) {
+  //   if (instance_of(base, java.lang.ref.Reference)) {
+  //     pre_barrier(_, pre_val, ...);
+  //   }
+  // }
+
+  float likely   = PROB_LIKELY(  0.999);
+  float unlikely = PROB_UNLIKELY(0.999);
+
+  IdealKit ideal(kit);
+
+  Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
+
+  __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
+      // Update graphKit memory and control from IdealKit.
+      kit->sync_kit(ideal);
+
+      Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
+      Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
+
+      // Update IdealKit memory and control from graphKit.
+      __ sync_kit(kit);
+
+      Node* one = __ ConI(1);
+      // is_instof == 0 if base_oop == NULL
+      __ if_then(is_instof, BoolTest::eq, one, unlikely); {
+
+        // Update graphKit from IdeakKit.
+        kit->sync_kit(ideal);
+
+        // Use the pre-barrier to record the value in the referent field
+        satb_write_barrier_pre(kit, false /* do_load */,
+                               NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
+                               pre_val /* pre_val */,
+                               T_OBJECT);
+        if (need_mem_bar) {
+          // Add memory barrier to prevent commoning reads from this field
+          // across safepoint since GC can change its value.
+          kit->insert_mem_bar(Op_MemBarCPUOrder);
+        }
+        // Update IdealKit from graphKit.
+        __ sync_kit(kit);
+
+      } __ end_if(); // _ref_type != ref_none
+  } __ end_if(); // offset == referent_offset
+
+  // Final sync IdealKit and GraphKit.
+  kit->final_sync(ideal);
+}
+
+#undef __
+
+const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
+  const Type **fields = TypeTuple::fields(2);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* ShenandoahBarrierSetC2::shenandoah_write_barrier_Type() {
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
+  const TypePtr* adr_type = access.addr().type();
+
+  if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
+    int off = adr_type->is_ptr()->offset();
+    int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
+      arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
+    assert(off != Type::OffsetTop, "unexpected offset");
+    if (off == Type::OffsetBot || off >= base_off) {
+      DecoratorSet decorators = access.decorators();
+      bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
+      GraphKit* kit = NULL;
+      if (access.is_parse_access()) {
+        C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+        kit = parse_access.kit();
+      }
+      Node* adr = access.addr().node();
+      assert(adr->is_AddP(), "unexpected address shape");
+      Node* base = adr->in(AddPNode::Base);
+
+      if (is_write) {
+        if (kit != NULL) {
+          base = shenandoah_write_barrier(kit, base);
+        } else {
+          assert(access.is_opt_access(), "either parse or opt access");
+          assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
+        }
+      } else {
+        if (adr_type->isa_instptr()) {
+          Compile* C = access.gvn().C;
+          ciField* field = C->alias_type(adr_type)->field();
+
+          // Insert read barrier for Shenandoah.
+          if (field != NULL &&
+              ((ShenandoahOptimizeStaticFinals   && field->is_static()  && field->is_final()) ||
+               (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
+               (ShenandoahOptimizeStableFinals   && field->is_stable()))) {
+            // Skip the barrier for special fields
+          } else {
+            if (kit != NULL) {
+              base = shenandoah_read_barrier(kit, base);
+            } else {
+              assert(access.is_opt_access(), "either parse or opt access");
+              assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
+            }
+          }
+        } else {
+          if (kit != NULL) {
+            base = shenandoah_read_barrier(kit, base);
+          } else {
+            assert(access.is_opt_access(), "either parse or opt access");
+            assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
+          }
+        }
+      }
+      if (base != adr->in(AddPNode::Base)) {
+        assert(kit != NULL, "no barrier should have been added");
+
+        Node* address = adr->in(AddPNode::Address);
+
+        if (address->is_AddP()) {
+          assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
+          assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
+          assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
+          address = address->clone();
+          address->set_req(AddPNode::Base, base);
+          address->set_req(AddPNode::Address, base);
+          address = kit->gvn().transform(address);
+        } else {
+          assert(address == adr->in(AddPNode::Base), "unexpected address shape");
+          address = base;
+        }
+        adr = adr->clone();
+        adr->set_req(AddPNode::Base, base);
+        adr->set_req(AddPNode::Address, address);
+        adr = kit->gvn().transform(adr);
+        access.addr().set_node(adr);
+      }
+    }
+  }
+}
+
+Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
+  DecoratorSet decorators = access.decorators();
+
+  const TypePtr* adr_type = access.addr().type();
+  Node* adr = access.addr().node();
+
+  bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+  bool on_heap = (decorators & IN_HEAP) != 0;
+
+  if (!access.is_oop() || (!on_heap && !anonymous)) {
+    return BarrierSetC2::store_at_resolved(access, val);
+  }
+
+  if (access.is_parse_access()) {
+    C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+    GraphKit* kit = parse_access.kit();
+
+    uint adr_idx = kit->C->get_alias_index(adr_type);
+    assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
+    Node* value = val.node();
+    value = shenandoah_storeval_barrier(kit, value);
+    val.set_node(value);
+    shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
+                                 static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
+  } else {
+    assert(access.is_opt_access(), "only for optimization passes");
+    assert(((decorators & C2_TIGHLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
+    C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
+    PhaseGVN& gvn =  opt_access.gvn();
+    MergeMemNode* mm = opt_access.mem();
+
+    if (ShenandoahStoreValReadBarrier) {
+      RegionNode* region = new RegionNode(3);
+      const Type* v_t = gvn.type(val.node());
+      Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
+      Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
+      Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
+      IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
+
+      gvn.transform(iff);
+      if (gvn.is_IterGVN()) {
+        gvn.is_IterGVN()->_worklist.push(iff);
+      } else {
+        gvn.record_for_igvn(iff);
+      }
+
+      Node* null_true = gvn.transform(new IfFalseNode(iff));
+      Node* null_false = gvn.transform(new IfTrueNode(iff));
+      region->init_req(1, null_true);
+      region->init_req(2, null_false);
+      phi->init_req(1, gvn.zerocon(T_OBJECT));
+      Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
+      cast->set_req(0, null_false);
+      cast = gvn.transform(cast);
+      Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
+      phi->init_req(2, rb);
+      opt_access.set_ctl(gvn.transform(region));
+      val.set_node(gvn.transform(phi));
+    }
+    if (ShenandoahStoreValEnqueueBarrier) {
+      const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));
+      int alias = gvn.C->get_alias_index(adr_type);
+      Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
+      Node* wb_transformed = gvn.transform(wb);
+      Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
+      if (wb_transformed == wb) {
+        Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
+        mm->set_memory_at(alias, proj);
+      }
+      val.set_node(enqueue);
+    }
+  }
+  return BarrierSetC2::store_at_resolved(access, val);
+}
+
+Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
+  DecoratorSet decorators = access.decorators();
+
+  Node* adr = access.addr().node();
+  Node* obj = access.base();
+
+  bool mismatched = (decorators & C2_MISMATCHED) != 0;
+  bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+  bool on_heap = (decorators & IN_HEAP) != 0;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool is_unordered = (decorators & MO_UNORDERED) != 0;
+  bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
+
+  Node* top = Compile::current()->top();
+
+  Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
+  Node* load = BarrierSetC2::load_at_resolved(access, val_type);
+
+  // If we are reading the value of the referent field of a Reference
+  // object (either by using Unsafe directly or through reflection)
+  // then, if SATB is enabled, we need to record the referent in an
+  // SATB log buffer using the pre-barrier mechanism.
+  // Also we need to add memory barrier to prevent commoning reads
+  // from this field across safepoint since GC can change its value.
+  bool need_read_barrier = ShenandoahKeepAliveBarrier &&
+    (on_heap && (on_weak || (unknown && offset != top && obj != top)));
+
+  if (!access.is_oop() || !need_read_barrier) {
+    return load;
+  }
+
+  assert(access.is_parse_access(), "entry not supported at optimization time");
+  C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+  GraphKit* kit = parse_access.kit();
+
+  if (on_weak) {
+    // Use the pre-barrier to record the value in the referent field
+    satb_write_barrier_pre(kit, false /* do_load */,
+                           NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
+                           load /* pre_val */, T_OBJECT);
+    // Add memory barrier to prevent commoning reads from this field
+    // across safepoint since GC can change its value.
+    kit->insert_mem_bar(Op_MemBarCPUOrder);
+  } else if (unknown) {
+    // We do not require a mem bar inside pre_barrier if need_mem_bar
+    // is set: the barriers would be emitted by us.
+    insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
+  }
+
+  return load;
+}
+
+Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                                   Node* new_val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    new_val = shenandoah_storeval_barrier(kit, new_val);
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 expected_val /* pre_val */, T_OBJECT);
+
+    MemNode::MemOrd mo = access.mem_node_mo();
+    Node* mem = access.memory();
+    Node* adr = access.addr().node();
+    const TypePtr* adr_type = access.addr().type();
+    Node* load_store = NULL;
+
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
+      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
+      load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+    } else
+#endif
+    {
+      load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+    }
+
+    access.set_raw_access(load_store);
+    pin_atomic_op(access);
+
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
+    }
+#endif
+    return load_store;
+  }
+  return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
+}
+
+Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                                              Node* new_val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    new_val = shenandoah_storeval_barrier(kit, new_val);
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 expected_val /* pre_val */, T_OBJECT);
+    DecoratorSet decorators = access.decorators();
+    MemNode::MemOrd mo = access.mem_node_mo();
+    Node* mem = access.memory();
+    bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
+    Node* load_store = NULL;
+    Node* adr = access.addr().node();
+#ifdef _LP64
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
+      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
+      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
+      if (is_weak_cas) {
+        load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+      } else {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+      }
+    } else
+#endif
+    {
+      if (is_weak_cas) {
+        load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+      } else {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+      }
+    }
+    access.set_raw_access(load_store);
+    pin_atomic_op(access);
+    return load_store;
+  }
+  return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
+}
+
+Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
+  GraphKit* kit = access.kit();
+  if (access.is_oop()) {
+    val = shenandoah_storeval_barrier(kit, val);
+  }
+  Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
+  if (access.is_oop()) {
+    shenandoah_write_barrier_pre(kit, false /* do_load */,
+                                 NULL, NULL, max_juint, NULL, NULL,
+                                 result /* pre_val */, T_OBJECT);
+  }
+  return result;
+}
+
+void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
+  assert(!src->is_AddP(), "unexpected input");
+  src = shenandoah_read_barrier(kit, src);
+  BarrierSetC2::clone(kit, src, dst, size, is_array);
+}
+
+Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
+  bool is_write = decorators & ACCESS_WRITE;
+  if (is_write) {
+    return shenandoah_write_barrier(kit, n);
+  } else {
+  return shenandoah_read_barrier(kit, n);
+  }
+}
+
+Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
+                                           Node*& i_o, Node*& needgc_ctrl,
+                                           Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
+                                           intx prefetch_lines) const {
+  PhaseIterGVN& igvn = macro->igvn();
+
+  // Allocate several words more for the Shenandoah brooks pointer.
+  size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
+  macro->transform_later(size_in_bytes);
+
+  Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,
+                                              i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem,
+                                              prefetch_lines);
+
+  // Bump up object for Shenandoah brooks pointer.
+  fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
+  macro->transform_later(fast_oop);
+
+  // Initialize Shenandoah brooks pointer to point to the object itself.
+  fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);
+
+  return fast_oop;
+}
+
+// Support for GC barriers emitted during parsing
+bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
+  if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
+    return false;
+  }
+  CallLeafNode *call = node->as_CallLeaf();
+  if (call->_name == NULL) {
+    return false;
+  }
+
+  return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
+         strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
+         strcmp(call->_name, "shenandoah_wb_pre") == 0;
+}
+
+Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
+  return ShenandoahBarrierNode::skip_through_barrier(c);
+}
+
+bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
+  return !ShenandoahWriteBarrierNode::expand(C, igvn);
+}
+
+bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
+  if (mode == LoopOptsShenandoahExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    ShenandoahWriteBarrierNode::pin_and_expand(phase);
+    return true;
+  } else if (mode == LoopOptsShenandoahPostExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    visited.Clear();
+    ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase);
+    return true;
+  }
+  GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
+  ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
+  bool is_oop = type == T_OBJECT || type == T_ARRAY;
+  if (!is_oop) {
+    return false;
+  }
+
+  if (tightly_coupled_alloc) {
+    if (phase == Optimization) {
+      return false;
+    }
+    return !is_clone;
+  }
+  if (phase == Optimization) {
+    return !ShenandoahStoreValEnqueueBarrier;
+  }
+  return true;
+}
+
+bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
+  Node* src = ac->in(ArrayCopyNode::Src);
+  const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
+  if (src_type->isa_instptr() != NULL) {
+    ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
+    if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
+      if (ik->has_object_fields()) {
+        return true;
+      } else {
+        if (!src_type->klass_is_exact()) {
+          igvn.C->dependencies()->assert_leaf_type(ik);
+        }
+      }
+    } else {
+      return true;
+    }
+  } else if (src_type->isa_aryptr()) {
+    BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
+    if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
+      return true;
+    }
+  } else {
+    return true;
+  }
+  return false;
+}
+
+void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
+  assert(ac->is_clonebasic(), "no other kind of arraycopy here");
+
+  if (!clone_needs_postbarrier(ac, igvn)) {
+    BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);
+    return;
+  }
+
+  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+  Node* c = new ProjNode(call,TypeFunc::Control);
+  c = igvn.transform(c);
+  Node* m = new ProjNode(call, TypeFunc::Memory);
+  c = igvn.transform(m);
+
+  Node* dest = ac->in(ArrayCopyNode::Dest);
+  assert(dest->is_AddP(), "bad input");
+  Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
+                                        CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
+                                        "shenandoah_clone_barrier", raw_adr_type);
+  barrier_call->init_req(TypeFunc::Control, c);
+  barrier_call->init_req(TypeFunc::I_O    , igvn.C->top());
+  barrier_call->init_req(TypeFunc::Memory , m);
+  barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());
+  barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());
+  barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));
+
+  barrier_call = igvn.transform(barrier_call);
+  c = new ProjNode(barrier_call,TypeFunc::Control);
+  c = igvn.transform(c);
+  m = new ProjNode(barrier_call, TypeFunc::Memory);
+  m = igvn.transform(m);
+
+  Node* out_c = ac->proj_out(TypeFunc::Control);
+  Node* out_m = ac->proj_out(TypeFunc::Memory);
+  igvn.replace_node(out_c, c);
+  igvn.replace_node(out_m, m);
+}
+
+
+// Support for macro expanded GC barriers
+void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahWriteBarrier) {
+    state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+  }
+}
+
+void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahWriteBarrier) {
+    state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+  }
+}
+
+void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
+  if (is_shenandoah_wb_pre_call(n)) {
+    shenandoah_eliminate_wb_pre(n, &macro->igvn());
+  }
+}
+
+void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
+  assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
+  Node* c = call->as_Call()->proj_out(TypeFunc::Control);
+  c = c->unique_ctrl_out();
+  assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+  c = c->unique_ctrl_out();
+  assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+  Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
+  assert(iff->is_If(), "expect test");
+  if (!is_shenandoah_marking_if(igvn, iff)) {
+    c = c->unique_ctrl_out();
+    assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
+    iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
+    assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
+  }
+  Node* cmpx = iff->in(1)->in(1);
+  igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
+  igvn->rehash_node_delayed(call);
+  call->del_req(call->req()-1);
+}
+
+void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
+  if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {
+    igvn->add_users_to_worklist(node);
+  }
+}
+
+void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
+  for (uint i = 0; i < useful.size(); i++) {
+    Node* n = useful.at(i);
+    if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {
+      for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+        C->record_for_igvn(n->fast_out(i));
+      }
+    }
+  }
+  for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {
+    ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);
+    if (!useful.member(n)) {
+      state()->remove_shenandoah_barrier(n);
+    }
+  }
+
+}
+
+bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {
+  assert(node->outcnt() == 1, "match only for unique out");
+  Node* n = node->unique_out();
+  return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
+}
+
+void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
+
+void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+  return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
+}
+
+ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
+  return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
+}
+
+// If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
+// expanded later, then now is the time to do so.
+bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
+
+#ifdef ASSERT
+void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
+  if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
+    ShenandoahBarrierNode::verify(Compile::current()->root());
+  } else if (phase == BarrierSetC2::BeforeCodeGen) {
+    // Verify G1 pre-barriers
+    const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
+
+    ResourceArea *area = Thread::current()->resource_area();
+    Unique_Node_List visited(area);
+    Node_List worklist(area);
+    // We're going to walk control flow backwards starting from the Root
+    worklist.push(compile->root());
+    while (worklist.size() > 0) {
+      Node *x = worklist.pop();
+      if (x == NULL || x == compile->top()) continue;
+      if (visited.member(x)) {
+        continue;
+      } else {
+        visited.push(x);
+      }
+
+      if (x->is_Region()) {
+        for (uint i = 1; i < x->req(); i++) {
+          worklist.push(x->in(i));
+        }
+      } else {
+        worklist.push(x->in(0));
+        // We are looking for the pattern:
+        //                            /->ThreadLocal
+        // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
+        //              \->ConI(0)
+        // We want to verify that the If and the LoadB have the same control
+        // See GraphKit::g1_write_barrier_pre()
+        if (x->is_If()) {
+          IfNode *iff = x->as_If();
+          if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
+            CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
+            if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
+                && cmp->in(1)->is_Load()) {
+              LoadNode *load = cmp->in(1)->as_Load();
+              if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
+                  && load->in(2)->in(3)->is_Con()
+                  && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
+
+                Node *if_ctrl = iff->in(0);
+                Node *load_ctrl = load->in(0);
+
+                if (if_ctrl != load_ctrl) {
+                  // Skip possible CProj->NeverBranch in infinite loops
+                  if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
+                      && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
+                    if_ctrl = if_ctrl->in(0)->in(0);
+                  }
+                }
+                assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+#endif
+
+Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
+  if (is_shenandoah_wb_pre_call(n)) {
+    uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
+    if (n->req() > cnt) {
+      Node* addp = n->in(cnt);
+      if (has_only_shenandoah_wb_pre_uses(addp)) {
+        n->del_req(cnt);
+        if (can_reshape) {
+          phase->is_IterGVN()->_worklist.push(addp);
+        }
+        return n;
+      }
+    }
+  }
+  if (n->Opcode() == Op_CmpP) {
+    Node* in1 = n->in(1);
+    Node* in2 = n->in(2);
+    if (in1->bottom_type() == TypePtr::NULL_PTR) {
+      in2 = step_over_gc_barrier(in2);
+    }
+    if (in2->bottom_type() == TypePtr::NULL_PTR) {
+      in1 = step_over_gc_barrier(in1);
+    }
+    PhaseIterGVN* igvn = phase->is_IterGVN();
+    if (in1 != n->in(1)) {
+      if (igvn != NULL) {
+        n->set_req_X(1, in1, igvn);
+      } else {
+        n->set_req(1, in1);
+      }
+      assert(in2 == n->in(2), "only one change");
+      return n;
+    }
+    if (in2 != n->in(2)) {
+      if (igvn != NULL) {
+        n->set_req_X(2, in2, igvn);
+      } else {
+        n->set_req(2, in2);
+      }
+      return n;
+    }
+  } else if (can_reshape &&
+             n->Opcode() == Op_If &&
+             ShenandoahWriteBarrierNode::is_heap_stable_test(n) &&
+             n->in(0) != NULL) {
+    Node* dom = n->in(0);
+    Node* prev_dom = n;
+    int op = n->Opcode();
+    int dist = 16;
+    // Search up the dominator tree for another heap stable test
+    while (dom->Opcode() != op    ||  // Not same opcode?
+           !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) ||  // Not same input 1?
+           prev_dom->in(0) != dom) {  // One path of test does not dominate?
+      if (dist < 0) return NULL;
+
+      dist--;
+      prev_dom = dom;
+      dom = IfNode::up_one_dom(dom);
+      if (!dom) return NULL;
+    }
+
+    // Check that we did not follow a loop back to ourselves
+    if (n == dom) {
+      return NULL;
+    }
+
+    return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN());
+  }
+
+  return NULL;
+}
+
+Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
+  if (n->is_Load()) {
+    Node *mem = n->in(MemNode::Memory);
+    Node *value = n->as_Load()->can_see_stored_value(mem, phase);
+    if (value) {
+      PhaseIterGVN *igvn = phase->is_IterGVN();
+      if (igvn != NULL &&
+          value->is_Phi() &&
+          value->req() > 2 &&
+          value->in(1) != NULL &&
+          value->in(1)->is_ShenandoahBarrier()) {
+        if (igvn->_worklist.member(value) ||
+            igvn->_worklist.member(value->in(0)) ||
+            (value->in(0)->in(1) != NULL &&
+             value->in(0)->in(1)->is_IfProj() &&
+             (igvn->_worklist.member(value->in(0)->in(1)) ||
+              (value->in(0)->in(1)->in(0) != NULL &&
+               igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
+          igvn->_worklist.push(n);
+          return n;
+        }
+      }
+      // (This works even when value is a Con, but LoadNode::Value
+      // usually runs first, producing the singleton type of the Con.)
+      Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
+      if (value->Opcode() == Op_EncodeP) {
+        if (value_no_barrier != value->in(1)) {
+          Node *encode = value->clone();
+          encode->set_req(1, value_no_barrier);
+          encode = phase->transform(encode);
+          return encode;
+        }
+      } else {
+        return value_no_barrier;
+      }
+    }
+  }
+  return n;
+}
+
+bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
+  for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+    Node* u = n->fast_out(i);
+    if (!is_shenandoah_wb_pre_call(u)) {
+      return false;
+    }
+  }
+  return n->outcnt() > 0;
+}
+
+bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
+  int offset = adr_type->offset();
+  if (offset == ShenandoahBrooksPointer::byte_offset()) {
+    if (adr_type->isa_aryptr()) {
+      adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
+    } else if (adr_type->isa_instptr()) {
+      adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
+  switch (opcode) {
+    case Op_CallLeaf:
+    case Op_CallLeafNoFP: {
+      assert (n->is_Call(), "");
+      CallNode *call = n->as_Call();
+      if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
+        uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
+        if (call->req() > cnt) {
+          assert(call->req() == cnt + 1, "only one extra input");
+          Node *addp = call->in(cnt);
+          assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
+          call->del_req(cnt);
+        }
+      }
+      return false;
+    }
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
+    case Op_ShenandoahWeakCompareAndSwapN:
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
+#ifdef ASSERT
+      if( VerifyOptoOopOffsets ) {
+        MemNode* mem  = n->as_Mem();
+        // Check to see if address types have grounded out somehow.
+        const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
+        ciInstanceKlass *k = tp->klass()->as_instance_klass();
+        bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
+        assert( !tp || oop_offset_is_sane, "" );
+      }
+#endif
+      return true;
+    case Op_ShenandoahReadBarrier:
+      return true;
+    case Op_ShenandoahWriteBarrier:
+      assert(false, "should have been expanded already");
+      return true;
+    default:
+      return false;
+  }
+}
+
+#ifdef ASSERT
+bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
+  if (offset == ShenandoahBrooksPointer::byte_offset() &&
+      (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
+    return true;
+  } else {
+    return false;
+  }
+}
+#endif
+
+bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
+  switch (opcode) {
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
+      conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
+      // fallthrough
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahWeakCompareAndSwapN:
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
+      conn_graph->add_to_congraph_unsafe_access(n, opcode, delayed_worklist);
+      return true;
+    case Op_StoreP: {
+      Node* adr = n->in(MemNode::Address);
+      const Type* adr_type = gvn->type(adr);
+      // Pointer stores in G1 barriers looks like unsafe access.
+      // Ignore such stores to be able scalar replace non-escaping
+      // allocations.
+      if (adr_type->isa_rawptr() && adr->is_AddP()) {
+        Node* base = conn_graph->get_addp_base(adr);
+        if (base->Opcode() == Op_LoadP &&
+          base->in(MemNode::Address)->is_AddP()) {
+          adr = base->in(MemNode::Address);
+          Node* tls = conn_graph->get_addp_base(adr);
+          if (tls->Opcode() == Op_ThreadLocal) {
+             int offs = (int) gvn->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
+             const int buf_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
+             if (offs == buf_offset) {
+               return true; // Pre barrier previous oop value store.
+             }
+          }
+        }
+      }
+      return false;
+    }
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
+      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
+      // It doesn't escape.
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
+      break;
+    case Op_ShenandoahEnqueueBarrier:
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
+      break;
+    default:
+      // Nothing
+      break;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
+  switch (opcode) {
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN: {
+      Node *adr = n->in(MemNode::Address);
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL);
+      // fallthrough
+    }
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahWeakCompareAndSwapN:
+      return conn_graph->add_final_edges_unsafe_access(n, opcode);
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
+      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
+      // It doesn't escape.
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
+      return true;
+    case Op_ShenandoahEnqueueBarrier:
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
+      return true;
+    default:
+      // Nothing
+      break;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::escape_has_out_with_unsafe_object(Node* n) const {
+  return n->has_out_with(Op_ShenandoahCompareAndExchangeP) || n->has_out_with(Op_ShenandoahCompareAndExchangeN) ||
+         n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN);
+
+}
+
+bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
+  return n->is_ShenandoahBarrier();
+}
+
+bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
+  switch (opcode) {
+    case Op_ShenandoahReadBarrier:
+      if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
+        matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
+      }
+      matcher->set_shared(n);
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
+  switch (opcode) {
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahWeakCompareAndSwapN:
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN: {   // Convert trinary to binary-tree
+      Node* newval = n->in(MemNode::ValueIn);
+      Node* oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
+      Node* pair = new BinaryNode(oldval, newval);
+      n->set_req(MemNode::ValueIn,pair);
+      n->del_req(LoadStoreConditionalNode::ExpectedIn);
+      return true;
+    }
+    default:
+      break;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierSetC2::matcher_is_store_load_barrier(Node* x, uint xop) const {
+  return xop == Op_ShenandoahCompareAndExchangeP ||
+         xop == Op_ShenandoahCompareAndExchangeN ||
+         xop == Op_ShenandoahWeakCompareAndSwapP ||
+         xop == Op_ShenandoahWeakCompareAndSwapN ||
+         xop == Op_ShenandoahCompareAndSwapN ||
+         xop == Op_ShenandoahCompareAndSwapP;
+}
+
+void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
+  if (use->is_ShenandoahBarrier()) {
+    for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
+      Node* u = use->fast_out(i2);
+      Node* cmp = use->find_out_with(Op_CmpP);
+      if (u->Opcode() == Op_CmpP) {
+        igvn->_worklist.push(cmp);
+      }
+    }
+  }
+}
+
+void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
+  if (use->is_ShenandoahBarrier()) {
+    for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
+      Node* p = use->fast_out(i2);
+      if (p->Opcode() == Op_AddP) {
+        for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
+          Node* q = p->fast_out(i3);
+          if (q->is_Load()) {
+            if(q->bottom_type() != ccp->type(q)) {
+              worklist.push(q);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const {
+  if (n->Opcode() == Op_ShenandoahReadBarrier) {
+    ((ShenandoahReadBarrierNode*)n)->try_move(phase);
+  } else if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase);
+  }
+
+  return NULL;
+}
+
+bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const {
+  return ShenandoahBarrierNode::build_loop_late_post(phase, n);
+}
+
+bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const {
+  if (n->is_ShenandoahBarrier()) {
+    return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl);
+  }
+  if (n->is_MergeMem()) {
+    // PhaseIdealLoop::split_if_with_blocks_post() would:
+    // _igvn._worklist.yank(x);
+    // which sometimes causes chains of MergeMem which some of
+    // shenandoah specific code doesn't support
+    phase->register_new_node(x, x_ctrl);
+    return true;
+  }
+  return false;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
+#define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
+
+#include "gc/shared/c2/barrierSetC2.hpp"
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "utilities/growableArray.hpp"
+
+class ShenandoahBarrierSetC2State : public ResourceObj {
+private:
+  GrowableArray<ShenandoahWriteBarrierNode*>* _shenandoah_barriers;
+
+public:
+  ShenandoahBarrierSetC2State(Arena* comp_arena);
+  int shenandoah_barriers_count() const;
+  ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const;
+  void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n);
+  void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n);
+};
+
+class ShenandoahBarrierSetC2 : public BarrierSetC2 {
+private:
+  void shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const;
+
+  bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
+                                   BasicType bt, uint adr_idx) const;
+  void satb_write_barrier_pre(GraphKit* kit, bool do_load,
+                              Node* obj,
+                              Node* adr,
+                              uint alias_idx,
+                              Node* val,
+                              const TypeOopPtr* val_type,
+                              Node* pre_val,
+                              BasicType bt) const;
+
+  void shenandoah_write_barrier_pre(GraphKit* kit,
+                                    bool do_load,
+                                    Node* obj,
+                                    Node* adr,
+                                    uint alias_idx,
+                                    Node* val,
+                                    const TypeOopPtr* val_type,
+                                    Node* pre_val,
+                                    BasicType bt) const;
+
+  Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const;
+  Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const;
+  Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const;
+  Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const;
+  Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const;
+  Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const;
+  Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const;
+
+  void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
+                          Node* pre_val, bool need_mem_bar) const;
+
+  static bool clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn);
+
+protected:
+  virtual void resolve_address(C2Access& access) const;
+  virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
+  virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
+  virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                               Node* new_val, const Type* val_type) const;
+  virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                                Node* new_val, const Type* value_type) const;
+  virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
+
+public:
+  static ShenandoahBarrierSetC2* bsc2();
+
+  static bool is_shenandoah_wb_pre_call(Node* call);
+  static bool is_shenandoah_wb_call(Node* call);
+  static bool is_shenandoah_marking_if(PhaseTransform *phase, Node* n);
+  static bool is_shenandoah_state_load(Node* n);
+  static bool has_only_shenandoah_wb_pre_uses(Node* n);
+
+  ShenandoahBarrierSetC2State* state() const;
+
+  static const TypeFunc* write_ref_field_pre_entry_Type();
+  static const TypeFunc* shenandoah_clone_barrier_Type();
+  static const TypeFunc* shenandoah_write_barrier_Type();
+
+  // This is the entry-point for the backend to perform accesses through the Access API.
+  virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const;
+
+  virtual Node* resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const;
+
+  virtual Node* obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
+                             Node*& i_o, Node*& needgc_ctrl,
+                             Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
+                             intx prefetch_lines) const;
+
+  // These are general helper methods used by C2
+  virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const;
+  virtual void clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const;
+
+  // Support for GC barriers emitted during parsing
+  virtual bool is_gc_barrier_node(Node* node) const;
+  virtual Node* step_over_gc_barrier(Node* c) const;
+  virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const;
+  virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const;
+  virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; }
+  virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; }
+
+  // Support for macro expanded GC barriers
+  virtual void register_potential_barrier_node(Node* node) const;
+  virtual void unregister_potential_barrier_node(Node* node) const;
+  virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
+  virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
+  virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const;
+  virtual void add_users_to_worklist(Unique_Node_List* worklist) const;
+
+  // Allow barrier sets to have shared state that is preserved across a compilation unit.
+  // This could for example comprise macro nodes to be expanded during macro expansion.
+  virtual void* create_barrier_state(Arena* comp_arena) const;
+  // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
+  // expanded later, then now is the time to do so.
+  virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const;
+
+#ifdef ASSERT
+  virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
+#endif
+
+  virtual bool flatten_gc_alias_type(const TypePtr*& adr_type) const;
+#ifdef ASSERT
+  virtual bool verify_gc_alias_type(const TypePtr* adr_type, int offset) const;
+#endif
+
+  virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const;
+  virtual Node* identity_node(PhaseGVN* phase, Node* n) const;
+  virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const;
+
+  virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const;
+  virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const;
+  virtual bool escape_has_out_with_unsafe_object(Node* n) const;
+  virtual bool escape_is_barrier_node(Node* n) const;
+
+  virtual bool matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const;
+  virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const;
+  virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const;
+
+  virtual void igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const;
+  virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const;
+
+  virtual bool has_special_unique_user(const Node* node) const;
+  virtual Node* split_if_pre(PhaseIdealLoop* phase, Node* n) const;
+  virtual bool build_loop_late_post(PhaseIdealLoop* phase, Node* n) const;
+  virtual bool sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const;
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,4278 @@
+/*
+ * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "opto/arraycopynode.hpp"
+#include "opto/block.hpp"
+#include "opto/callnode.hpp"
+#include "opto/castnode.hpp"
+#include "opto/movenode.hpp"
+#include "opto/phaseX.hpp"
+#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
+#include "opto/subnode.hpp"
+
+Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
+  if (n == NULL) {
+    return NULL;
+  }
+  if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    n = n->in(1);
+  }
+
+  if (n->is_ShenandoahBarrier()) {
+    return n->in(ValueIn);
+  } else if (n->is_Phi() &&
+             n->req() == 3 &&
+             n->in(1) != NULL &&
+             n->in(1)->is_ShenandoahBarrier() &&
+             n->in(2) != NULL &&
+             n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
+             n->in(0) != NULL &&
+             n->in(0)->in(1) != NULL &&
+             n->in(0)->in(1)->is_IfProj() &&
+             n->in(0)->in(2) != NULL &&
+             n->in(0)->in(2)->is_IfProj() &&
+             n->in(0)->in(1)->in(0) != NULL &&
+             n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) &&
+             n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) {
+    Node* iff = n->in(0)->in(1)->in(0);
+    Node* res = n->in(1)->in(ValueIn)->in(1);
+    if (iff->is_If() &&
+        iff->in(1) != NULL &&
+        iff->in(1)->is_Bool() &&
+        iff->in(1)->as_Bool()->_test._test == BoolTest::ne &&
+        iff->in(1)->in(1) != NULL &&
+        iff->in(1)->in(1)->Opcode() == Op_CmpP &&
+        iff->in(1)->in(1)->in(1) != NULL &&
+        iff->in(1)->in(1)->in(1) == res &&
+        iff->in(1)->in(1)->in(2) != NULL &&
+        iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
+      return res;
+    }
+  }
+  return n;
+}
+
+bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) {
+  Unique_Node_List visited;
+  return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited);
+}
+
+bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) {
+  if (visited.member(n)) {
+    return false; // Been there.
+  }
+  visited.push(n);
+
+  if (n->is_Allocate()) {
+    return false;
+  }
+
+  if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) {
+    return true;
+  }
+
+  const Type* type = phase->type(n);
+  if (type == Type::TOP) {
+    return false;
+  }
+  if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
+    return false;
+  }
+  if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
+    return false;
+  }
+
+  if (ShenandoahOptimizeStableFinals) {
+    const TypeAryPtr* ary = type->isa_aryptr();
+    if (ary && ary->is_stable() && allow_fromspace) {
+      return false;
+    }
+  }
+
+  if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
+  }
+  if (n->is_Parm()) {
+    return true;
+  }
+  if (n->is_Proj()) {
+    return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited);
+  }
+
+  if (n->Opcode() == Op_ShenandoahWBMemProj) {
+    return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited);
+  }
+  if (n->is_Phi()) {
+    bool need_barrier = false;
+    for (uint i = 1; i < n->req() && ! need_barrier; i++) {
+      Node* input = n->in(i);
+      if (input == NULL) {
+        need_barrier = true; // Phi not complete yet?
+      } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) {
+        need_barrier = true;
+      }
+    }
+    return need_barrier;
+  }
+  if (n->is_CMove()) {
+    return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) ||
+           needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited);
+  }
+  if (n->Opcode() == Op_CreateEx) {
+    return true;
+  }
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    return false;
+  }
+  if (n->Opcode() == Op_ShenandoahReadBarrier) {
+    if (rb_mem == n->in(Memory)) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  if (n->Opcode() == Op_LoadP ||
+      n->Opcode() == Op_LoadN ||
+      n->Opcode() == Op_GetAndSetP ||
+      n->Opcode() == Op_CompareAndExchangeP ||
+      n->Opcode() == Op_ShenandoahCompareAndExchangeP ||
+      n->Opcode() == Op_GetAndSetN ||
+      n->Opcode() == Op_CompareAndExchangeN ||
+      n->Opcode() == Op_ShenandoahCompareAndExchangeN) {
+    return true;
+  }
+  if (n->Opcode() == Op_DecodeN ||
+      n->Opcode() == Op_EncodeP) {
+    return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
+  }
+
+#ifdef ASSERT
+  tty->print("need barrier on?: "); n->dump();
+  ShouldNotReachHere();
+#endif
+  return true;
+}
+
+bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
+                                                         Node* b1,
+                                                         Node* b2,
+                                                         Node* current,
+                                                         bool linear) {
+  ResourceMark rm;
+  VectorSet visited(Thread::current()->resource_area());
+  Node_Stack phis(0);
+
+  for(int i = 0; i < 10; i++) {
+    if (current == NULL) {
+      return false;
+    } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
+      current = NULL;
+      while (phis.is_nonempty() && current == NULL) {
+        uint idx = phis.index();
+        Node* phi = phis.node();
+        if (idx >= phi->req()) {
+          phis.pop();
+        } else {
+          current = phi->in(idx);
+          phis.set_index(idx+1);
+        }
+      }
+      if (current == NULL) {
+        return true;
+      }
+    } else if (current == phase->C->immutable_memory()) {
+      return false;
+    } else if (current->isa_Phi()) {
+      if (!linear) {
+        return false;
+      }
+      phis.push(current, 2);
+      current = current->in(1);
+    } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
+      const Type* in_type = current->bottom_type();
+      const Type* this_type = b2->bottom_type();
+      if (is_independent(in_type, this_type)) {
+        current = current->in(Memory);
+      } else {
+        return false;
+      }
+    } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
+      current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
+    } else if (current->is_Proj()) {
+      current = current->in(0);
+    } else if (current->is_Call()) {
+      return false; // TODO: Maybe improve by looking at the call's memory effects?
+    } else if (current->is_MemBar()) {
+      return false; // TODO: Do we need to stop at *any* membar?
+    } else if (current->is_MergeMem()) {
+      const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
+      uint alias_idx = phase->C->get_alias_index(adr_type);
+      current = current->as_MergeMem()->memory_at(alias_idx);
+    } else {
+#ifdef ASSERT
+      current->dump();
+#endif
+      ShouldNotReachHere();
+      return false;
+    }
+  }
+  return false;
+}
+
+bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
+  if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
+    return true;
+  } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
+    return true;
+  } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
+    const Type* mem_type = mem->bottom_type();
+    const Type* this_type = bottom_type();
+    if (is_independent(mem_type, this_type)) {
+      return true;
+    } else {
+      return false;
+    }
+  } else if (mem->is_Call() || mem->is_MemBar()) {
+    return false;
+  }
+#ifdef ASSERT
+  mem->dump();
+#endif
+  ShouldNotReachHere();
+  return true;
+}
+
+bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
+  return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
+}
+
+bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
+  assert(in_type->isa_oopptr(), "expect oop ptr");
+  assert(this_type->isa_oopptr(), "expect oop ptr");
+
+  ciKlass* in_kls = in_type->is_oopptr()->klass();
+  ciKlass* this_kls = this_type->is_oopptr()->klass();
+  if (in_kls != NULL && this_kls != NULL &&
+      in_kls->is_loaded() && this_kls->is_loaded() &&
+      (!in_kls->is_subclass_of(this_kls)) &&
+      (!this_kls->is_subclass_of(in_kls))) {
+    return true;
+  }
+  return false;
+}
+
+Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  if (! can_reshape) {
+    return NULL;
+  }
+
+  if (in(Memory) == phase->C->immutable_memory()) return NULL;
+
+  // If memory input is a MergeMem, take the appropriate slice out of it.
+  Node* mem_in = in(Memory);
+  if (mem_in->isa_MergeMem()) {
+    const TypePtr* adr_type = brooks_pointer_type(bottom_type());
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
+    set_req(Memory, mem_in);
+    return this;
+  }
+
+  Node* input = in(Memory);
+  if (input->Opcode() == Op_ShenandoahWBMemProj) {
+    ResourceMark rm;
+    VectorSet seen(Thread::current()->resource_area());
+    Node* n = in(Memory);
+    while (n->Opcode() == Op_ShenandoahWBMemProj &&
+           n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL &&
+           n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier &&
+           n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) {
+      if (seen.test_set(n->_idx)) {
+        return NULL; // loop
+      }
+      n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory);
+    }
+
+    Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier);
+    const Type* in_type = phase->type(wb);
+    // is_top() test not sufficient here: we can come here after CCP
+    // in a dead branch of the graph that has not yet been removed.
+    if (in_type == Type::TOP) return NULL; // Dead path.
+    assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
+    if (is_independent(in_type, _type)) {
+      phase->igvn_rehash_node_delayed(wb);
+      set_req(Memory, wb->in(Memory));
+      if (can_reshape && input->outcnt() == 0) {
+        phase->is_IterGVN()->_worklist.push(input);
+      }
+      return this;
+    }
+  }
+  return NULL;
+}
+
+ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj)
+  : ShenandoahBarrierNode(ctrl, mem, obj, false) {
+  assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
+  ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this);
+}
+
+Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) {
+  assert(in(0) != NULL, "should have control");
+  PhaseIterGVN* igvn = phase->is_IterGVN();
+  Node* mem_in = in(Memory);
+  Node* mem_proj = NULL;
+
+  if (igvn != NULL) {
+    mem_proj = find_out_with(Op_ShenandoahWBMemProj);
+    if (mem_in == mem_proj) {
+      return this;
+    }
+  }
+
+  Node* replacement = Identity_impl(phase);
+  if (igvn != NULL) {
+    if (replacement != NULL && replacement != this && mem_proj != NULL) {
+      igvn->replace_node(mem_proj, mem_in);
+    }
+  }
+  return replacement;
+}
+
+Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  assert(in(0) != NULL, "should have control");
+  if (!can_reshape) {
+    return NULL;
+  }
+
+  Node* mem_in = in(Memory);
+
+  if (mem_in->isa_MergeMem()) {
+    const TypePtr* adr_type = brooks_pointer_type(bottom_type());
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
+    set_req(Memory, mem_in);
+    return this;
+  }
+
+  Node* val = in(ValueIn);
+  if (val->is_ShenandoahBarrier()) {
+    set_req(ValueIn, val->in(ValueIn));
+    return this;
+  }
+
+  return NULL;
+}
+
+bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn) {
+  if (UseShenandoahGC) {
+    if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) {
+      bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
+      C->clear_major_progress();
+      PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
+      if (C->failing()) return false;
+      PhaseIdealLoop::verify(igvn);
+      DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());)
+      if (attempt_more_loopopts) {
+        C->set_major_progress();
+        if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
+          return false;
+        }
+        C->clear_major_progress();
+      }
+    }
+  }
+  return true;
+}
+
+bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) {
+  if (!UseShenandoahGC) {
+    return false;
+  }
+  assert(iff->is_If(), "bad input");
+  if (iff->Opcode() != Op_If) {
+    return false;
+  }
+  Node* bol = iff->in(1);
+  if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
+    return false;
+  }
+  Node* cmp = bol->in(1);
+  if (cmp->Opcode() != Op_CmpI) {
+    return false;
+  }
+  Node* in1 = cmp->in(1);
+  Node* in2 = cmp->in(2);
+  if (in2->find_int_con(-1) != 0) {
+    return false;
+  }
+  if (in1->Opcode() != Op_AndI) {
+    return false;
+  }
+  in2 = in1->in(2);
+  if (in2->find_int_con(-1) != mask) {
+    return false;
+  }
+  in1 = in1->in(1);
+
+  return is_gc_state_load(in1);
+}
+
+bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) {
+  return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
+}
+
+bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
+  if (!UseShenandoahGC) {
+    return false;
+  }
+  if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {
+    return false;
+  }
+  Node* addp = n->in(MemNode::Address);
+  if (!addp->is_AddP()) {
+    return false;
+  }
+  Node* base = addp->in(AddPNode::Address);
+  Node* off = addp->in(AddPNode::Offset);
+  if (base->Opcode() != Op_ThreadLocal) {
+    return false;
+  }
+  if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {
+    return false;
+  }
+  return true;
+}
+
+bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
+  assert(phase->is_dominator(stop, start), "bad inputs");
+  ResourceMark rm;
+  Unique_Node_List wq;
+  wq.push(start);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *m = wq.at(next);
+    if (m == stop) {
+      continue;
+    }
+    if (m->is_SafePoint() && !m->is_CallLeaf()) {
+      return true;
+    }
+    if (m->is_Region()) {
+      for (uint i = 1; i < m->req(); i++) {
+        wq.push(m->in(i));
+      }
+    } else {
+      wq.push(m->in(0));
+    }
+  }
+  return false;
+}
+
+bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
+  assert(is_gc_state_load(n), "inconsistent");
+  Node* addp = n->in(MemNode::Address);
+  Node* dominator = NULL;
+  for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
+    Node* u = addp->fast_out(i);
+    assert(is_gc_state_load(u), "inconsistent");
+    if (u != n && phase->is_dominator(u->in(0), n->in(0))) {
+      if (dominator == NULL) {
+        dominator = u;
+      } else {
+        if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {
+          dominator = u;
+        }
+      }
+    }
+  }
+  if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {
+    return false;
+  }
+  phase->igvn().replace_node(n, dominator);
+
+  return true;
+}
+
+bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
+                                                  Node* b1,
+                                                  Node* b2,
+                                                  Node* current,
+                                                  bool linear) {
+  ResourceMark rm;
+  VectorSet visited(Thread::current()->resource_area());
+  Node_Stack phis(0);
+
+  for(int i = 0; i < 10; i++) {
+    if (current == NULL) {
+      return false;
+    } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
+      current = NULL;
+      while (phis.is_nonempty() && current == NULL) {
+        uint idx = phis.index();
+        Node* phi = phis.node();
+        if (idx >= phi->req()) {
+          phis.pop();
+        } else {
+          current = phi->in(idx);
+          phis.set_index(idx+1);
+        }
+      }
+      if (current == NULL) {
+        return true;
+      }
+    } else if (current == b2) {
+      return false;
+    } else if (current == phase->C->immutable_memory()) {
+      return false;
+    } else if (current->isa_Phi()) {
+      if (!linear) {
+        return false;
+      }
+      phis.push(current, 2);
+      current = current->in(1);
+    } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
+      current = current->in(Memory);
+    } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
+      current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
+    } else if (current->is_Proj()) {
+      current = current->in(0);
+    } else if (current->is_Call()) {
+      current = current->in(TypeFunc::Memory);
+    } else if (current->is_MemBar()) {
+      current = current->in(TypeFunc::Memory);
+    } else if (current->is_MergeMem()) {
+      const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
+      uint alias_idx = phase->C->get_alias_index(adr_type);
+      current = current->as_MergeMem()->memory_at(alias_idx);
+    } else {
+#ifdef ASSERT
+      current->dump();
+#endif
+      ShouldNotReachHere();
+      return false;
+    }
+  }
+  return false;
+}
+
+/**
+ * Determines if b1 dominates b2 through memory inputs. It returns true if:
+ * - b1 can be reached by following each branch in b2's memory input (through phis, etc)
+ * - or we get back to b2 (i.e. through a loop) without seeing b1
+ * In all other cases, (in particular, if we reach immutable_memory without having seen b1)
+ * we return false.
+ */
+bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
+  return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear);
+}
+
+Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) {
+  Node* n = in(ValueIn);
+
+  Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL;
+  if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) {
+    return n;
+  }
+
+  // Try to find a write barrier sibling with identical inputs that we can fold into.
+  for (DUIterator i = n->outs(); n->has_out(i); i++) {
+    Node* sibling = n->out(i);
+    if (sibling == this) {
+      continue;
+    }
+    if (sibling->Opcode() != Op_ShenandoahWriteBarrier) {
+      continue;
+    }
+
+    assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
+    assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity");
+
+    if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) {
+      return sibling;
+    }
+  }
+  return this;
+}
+
+#ifndef PRODUCT
+void ShenandoahBarrierNode::dump_spec(outputStream *st) const {
+  const TypePtr* adr = adr_type();
+  if (adr == NULL) {
+    return;
+  }
+  st->print(" @");
+  adr->dump_on(st);
+  st->print(" (");
+  Compile::current()->alias_type(adr)->adr_type()->dump_on(st);
+  st->print(") ");
+}
+#endif
+
+Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) {
+  Node* id = Identity_impl(phase);
+
+  if (id == this && phase->is_IterGVN()) {
+    Node* n = in(ValueIn);
+    // No success in super call. Try to combine identical read barriers.
+    for (DUIterator i = n->outs(); n->has_out(i); i++) {
+      Node* sibling = n->out(i);
+      if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) {
+        continue;
+      }
+      assert(sibling->in(ValueIn)  == in(ValueIn), "sanity");
+      if (phase->is_IterGVN()->hash_find(sibling) &&
+          sibling->bottom_type() == bottom_type() &&
+          sibling->in(Control) == in(Control) &&
+          dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
+        return sibling;
+      }
+    }
+  }
+  return id;
+}
+
+const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
+  // Either input is TOP ==> the result is TOP
+  const Type *t1 = phase->type(in(Memory));
+  if (t1 == Type::TOP) return Type::TOP;
+  const Type *t2 = phase->type(in(ValueIn));
+  if( t2 == Type::TOP ) return Type::TOP;
+
+  if (t2 == TypePtr::NULL_PTR) {
+    return _type;
+  }
+
+  const Type* type = t2->is_oopptr()->cast_to_nonconst();
+  return type;
+}
+
+uint ShenandoahBarrierNode::hash() const {
+  return TypeNode::hash() + _allow_fromspace;
+}
+
+uint ShenandoahBarrierNode::cmp(const Node& n) const {
+  return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
+    && TypeNode::cmp(n);
+}
+
+uint ShenandoahBarrierNode::size_of() const {
+  return sizeof(*this);
+}
+
+Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
+  Node* wb = in(WriteBarrier);
+  if (wb->is_top()) return phase->C->top(); // Dead path.
+
+  assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
+  PhaseIterGVN* igvn = phase->is_IterGVN();
+  // We can't do the below unless the graph is fully constructed.
+  if (igvn == NULL) {
+    return this;
+  }
+
+  // If the mem projection has no barrier users, it's not needed anymore.
+  if (wb->outcnt() == 1) {
+    return wb->in(ShenandoahBarrierNode::Memory);
+  }
+
+  return this;
+}
+
+#ifdef ASSERT
+bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
+  assert(phis.size() == 0, "");
+
+  while (true) {
+    if (in->bottom_type() == TypePtr::NULL_PTR) {
+      if (trace) {tty->print_cr("NULL");}
+    } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
+      if (trace) {tty->print_cr("Non oop");}
+    } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&
+               in->bottom_type()->make_ptr()->isa_aryptr() &&
+               in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {
+      if (trace) {tty->print_cr("Stable array load");}
+    } else {
+      if (in->is_ConstraintCast()) {
+        in = in->in(1);
+        continue;
+      } else if (in->is_AddP()) {
+        assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");
+        in = in->in(AddPNode::Address);
+        continue;
+      } else if (in->is_Con()) {
+        if (trace) {tty->print("Found constant"); in->dump();}
+      } else if (in->is_ShenandoahBarrier()) {
+        if (t == ShenandoahOopStore) {
+          if (in->Opcode() != Op_ShenandoahWriteBarrier) {
+            return false;
+          }
+          uint i = 0;
+          for (; i < phis.size(); i++) {
+            Node* n = phis.node_at(i);
+            if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+              break;
+            }
+          }
+          if (i == phis.size()) {
+            return false;
+          }
+        } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) {
+          return false;
+        }
+        barriers_used.push(in);
+        if (trace) {tty->print("Found barrier"); in->dump();}
+      } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {
+        if (t != ShenandoahOopStore) {
+          in = in->in(1);
+          continue;
+        }
+        if (trace) {tty->print("Found enqueue barrier"); in->dump();}
+        phis.push(in, in->req());
+        in = in->in(1);
+        continue;
+      } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
+        if (trace) {tty->print("Found alloc"); in->in(0)->dump();}
+      } else if (in->is_Phi()) {
+        if (!visited.test_set(in->_idx)) {
+          if (trace) {tty->print("Pushed phi:"); in->dump();}
+          phis.push(in, 2);
+          in = in->in(1);
+          continue;
+        }
+        if (trace) {tty->print("Already seen phi:"); in->dump();}
+      } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
+        if (!visited.test_set(in->_idx)) {
+          if (trace) {tty->print("Pushed cmovep:"); in->dump();}
+          phis.push(in, CMoveNode::IfTrue);
+          in = in->in(CMoveNode::IfFalse);
+          continue;
+        }
+        if (trace) {tty->print("Already seen cmovep:"); in->dump();}
+      } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {
+        in = in->in(1);
+        continue;
+      } else {
+        return false;
+      }
+    }
+    bool cont = false;
+    while (phis.is_nonempty()) {
+      uint idx = phis.index();
+      Node* phi = phis.node();
+      if (idx >= phi->req()) {
+        if (trace) {tty->print("Popped phi:"); phi->dump();}
+        phis.pop();
+        continue;
+      }
+      if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}
+      in = phi->in(idx);
+      phis.set_index(idx+1);
+      cont = true;
+      break;
+    }
+    if (!cont) {
+      break;
+    }
+  }
+  return true;
+}
+
+void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) {
+  if (n1 != NULL) {
+    n1->dump(+10);
+  }
+  if (n2 != NULL) {
+    n2->dump(+10);
+  }
+  fatal("%s", msg);
+}
+
+void ShenandoahBarrierNode::verify(RootNode* root) {
+  ResourceMark rm;
+  Unique_Node_List wq;
+  GrowableArray<Node*> barriers;
+  Unique_Node_List barriers_used;
+  Node_Stack phis(0);
+  VectorSet visited(Thread::current()->resource_area());
+  const bool trace = false;
+  const bool verify_no_useless_barrier = false;
+
+  wq.push(root);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *n = wq.at(next);
+    if (n->is_Load()) {
+      const bool trace = false;
+      if (trace) {tty->print("Verifying"); n->dump();}
+      if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {
+        if (trace) {tty->print_cr("Load range/klass");}
+      } else {
+        const TypePtr* adr_type = n->as_Load()->adr_type();
+
+        if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) {
+          if (trace) {tty->print_cr("Mark load");}
+        } else if (adr_type->isa_instptr() &&
+                   adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
+                   adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {
+          if (trace) {tty->print_cr("Reference.get()");}
+        } else {
+          bool verify = true;
+          if (adr_type->isa_instptr()) {
+            const TypeInstPtr* tinst = adr_type->is_instptr();
+            ciKlass* k = tinst->klass();
+            assert(k->is_instance_klass(), "");
+            ciInstanceKlass* ik = (ciInstanceKlass*)k;
+            int offset = adr_type->offset();
+
+            if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||
+                (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {
+              if (trace) {tty->print_cr("Final/stable");}
+              verify = false;
+            } else if (k == ciEnv::current()->Class_klass() &&
+                       tinst->const_oop() != NULL &&
+                       tinst->offset() >= (ik->size_helper() * wordSize)) {
+              ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
+              ciField* field = k->get_field_by_offset(tinst->offset(), true);
+              if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||
+                  (ShenandoahOptimizeStableFinals && field->is_stable())) {
+                verify = false;
+              }
+            }
+          }
+
+          if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+            report_verify_failure("Shenandoah verification: Load should have barriers", n);
+          }
+        }
+      }
+    } else if (n->is_Store()) {
+      const bool trace = false;
+
+      if (trace) {tty->print("Verifying"); n->dump();}
+      if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
+        Node* adr = n->in(MemNode::Address);
+        bool verify = true;
+
+        if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
+          adr = adr->in(AddPNode::Address);
+          if (adr->is_AddP()) {
+            assert(adr->in(AddPNode::Base)->is_top(), "");
+            adr = adr->in(AddPNode::Address);
+            if (adr->Opcode() == Op_LoadP &&
+                adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
+                adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
+                adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
+              if (trace) {tty->print_cr("SATB prebarrier");}
+              verify = false;
+            }
+          }
+        }
+
+        if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: Store should have barriers", n);
+        }
+      }
+      if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
+      }
+    } else if (n->Opcode() == Op_CmpP) {
+      const bool trace = false;
+
+      Node* in1 = n->in(1);
+      Node* in2 = n->in(2);
+      if (in1->bottom_type()->isa_oopptr()) {
+        if (trace) {tty->print("Verifying"); n->dump();}
+
+        bool mark_inputs = false;
+        if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
+            (in1->is_Con() || in2->is_Con())) {
+          if (trace) {tty->print_cr("Comparison against a constant");}
+          mark_inputs = true;
+        } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
+                   (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {
+          if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
+          mark_inputs = true;
+        } else {
+          assert(in2->bottom_type()->isa_oopptr(), "");
+
+          if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
+              !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
+            report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
+          }
+        }
+        if (verify_no_useless_barrier &&
+            mark_inputs &&
+            (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
+             !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
+          phis.clear();
+          visited.Reset();
+        }
+      }
+    } else if (n->is_LoadStore()) {
+      if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
+          !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
+      }
+
+      if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+        report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
+      }
+    } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
+      CallNode* call = n->as_Call();
+
+      static struct {
+        const char* name;
+        struct {
+          int pos;
+          verify_type t;
+        } args[6];
+      } calls[] = {
+        "aescrypt_encryptBlock",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "aescrypt_decryptBlock",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "multiplyToLen",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { TypeFunc::Parms+4, ShenandoahStore },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "squareToLen",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "montgomery_multiply",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+6, ShenandoahStore }, { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "montgomery_square",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+5, ShenandoahStore },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "mulAdd",
+        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "vectorizedMismatch",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesCRC32",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesAdler32",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "updateBytesCRC32C",
+        { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad},    { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "counterMode_AESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
+        "cipherBlockChaining_encryptAESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "cipherBlockChaining_decryptAESCrypt",
+        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
+          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "shenandoah_clone_barrier",
+        { { TypeFunc::Parms, ShenandoahLoad },   { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "ghash_processBlocks",
+        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha1_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha256_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha512_implCompress",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha1_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha256_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+        "sha512_implCompressMB",
+        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
+          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
+      };
+
+      if (call->is_call_to_arraycopystub()) {
+        Node* dest = NULL;
+        const TypeTuple* args = n->as_Call()->_tf->domain();
+        for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
+          if (args->field_at(i)->isa_ptr()) {
+            j++;
+            if (j == 2) {
+              dest = n->in(i);
+              break;
+            }
+          }
+        }
+        if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
+            !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
+        }
+      } else if (strlen(call->_name) > 5 &&
+                 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
+        if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
+          report_verify_failure("Shenandoah verification: _fill should have barriers", n);
+        }
+      } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
+        // skip
+      } else {
+        const int calls_len = sizeof(calls) / sizeof(calls[0]);
+        int i = 0;
+        for (; i < calls_len; i++) {
+          if (!strcmp(calls[i].name, call->_name)) {
+            break;
+          }
+        }
+        if (i != calls_len) {
+          const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);
+          for (uint j = 0; j < args_len; j++) {
+            int pos = calls[i].args[j].pos;
+            if (pos == -1) {
+              break;
+            }
+            if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
+              report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
+            }
+          }
+          for (uint j = TypeFunc::Parms; j < call->req(); j++) {
+            if (call->in(j)->bottom_type()->make_ptr() &&
+                call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
+              uint k = 0;
+              for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
+              if (k == args_len) {
+                fatal("arg %d for call %s not covered", j, call->_name);
+              }
+            }
+          }
+        } else {
+          for (uint j = TypeFunc::Parms; j < call->req(); j++) {
+            if (call->in(j)->bottom_type()->make_ptr() &&
+                call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
+              fatal("%s not covered", call->_name);
+            }
+          }
+        }
+      }
+    } else if (n->is_ShenandoahBarrier()) {
+      assert(!barriers.contains(n), "");
+      assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier");
+      assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier");
+      barriers.push(n);
+    } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+      // skip
+    } else if (n->Opcode() == Op_ShenandoahWBMemProj) {
+      assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj");
+    } else if (n->is_AddP()
+               || n->is_Phi()
+               || n->is_ConstraintCast()
+               || n->Opcode() == Op_Return
+               || n->Opcode() == Op_CMoveP
+               || n->Opcode() == Op_CMoveN
+               || n->Opcode() == Op_Rethrow
+               || n->is_MemBar()
+               || n->Opcode() == Op_Conv2B
+               || n->Opcode() == Op_SafePoint
+               || n->is_CallJava()
+               || n->Opcode() == Op_Unlock
+               || n->Opcode() == Op_EncodeP
+               || n->Opcode() == Op_DecodeN) {
+      // nothing to do
+    } else {
+      static struct {
+        int opcode;
+        struct {
+          int pos;
+          verify_type t;
+        } inputs[2];
+      } others[] = {
+        Op_FastLock,
+        { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_Lock,
+        { { TypeFunc::Parms, ShenandoahLoad },    { -1, ShenandoahNone} },
+        Op_ArrayCopy,
+        { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } },
+        Op_StrCompressedCopy,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_StrInflatedCopy,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_AryEq,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
+        Op_StrIndexOf,
+        { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
+        Op_StrComp,
+        { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
+        Op_StrEquals,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
+        Op_EncodeISOArray,
+        { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
+        Op_HasNegatives,
+        { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_CastP2X,
+        { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
+        Op_StrIndexOfChar,
+        { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
+      };
+
+      const int others_len = sizeof(others) / sizeof(others[0]);
+      int i = 0;
+      for (; i < others_len; i++) {
+        if (others[i].opcode == n->Opcode()) {
+          break;
+        }
+      }
+      uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
+      if (i != others_len) {
+        const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
+        for (uint j = 0; j < inputs_len; j++) {
+          int pos = others[i].inputs[j].pos;
+          if (pos == -1) {
+            break;
+          }
+          if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
+            report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
+          }
+        }
+        for (uint j = 1; j < stop; j++) {
+          if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
+              n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
+            uint k = 0;
+            for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
+            if (k == inputs_len) {
+              fatal("arg %d for node %s not covered", j, n->Name());
+            }
+          }
+        }
+      } else {
+        for (uint j = 1; j < stop; j++) {
+          if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
+              n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
+            fatal("%s not covered", n->Name());
+          }
+        }
+      }
+    }
+
+    if (n->is_SafePoint()) {
+      SafePointNode* sfpt = n->as_SafePoint();
+      if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
+        for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
+          if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+            phis.clear();
+            visited.Reset();
+          }
+        }
+      }
+    }
+    for( uint i = 0; i < n->len(); ++i ) {
+      Node *m = n->in(i);
+      if (m == NULL) continue;
+
+      // In most cases, inputs should be known to be non null. If it's
+      // not the case, it could be a missing cast_not_null() in an
+      // intrinsic or support might be needed in AddPNode::Ideal() to
+      // avoid a NULL+offset input.
+      if (!(n->is_Phi() ||
+            (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) ||
+            n->Opcode() == Op_CmpP ||
+            n->Opcode() == Op_CmpN ||
+            (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) ||
+            (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) ||
+            n->is_ConstraintCast() ||
+            n->Opcode() == Op_Return ||
+            n->Opcode() == Op_Conv2B ||
+            n->is_AddP() ||
+            n->Opcode() == Op_CMoveP ||
+            n->Opcode() == Op_CMoveN ||
+            n->Opcode() == Op_Rethrow ||
+            n->is_MemBar() ||
+            n->is_Mem() ||
+            n->Opcode() == Op_AryEq ||
+            n->Opcode() == Op_SCMemProj ||
+            n->Opcode() == Op_EncodeP ||
+            n->Opcode() == Op_DecodeN ||
+            n->Opcode() == Op_ShenandoahWriteBarrier ||
+            n->Opcode() == Op_ShenandoahWBMemProj ||
+            n->Opcode() == Op_ShenandoahEnqueueBarrier)) {
+        if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
+          report_verify_failure("Shenandoah verification: null input", n, m);
+        }
+      }
+
+      wq.push(m);
+    }
+  }
+
+  if (verify_no_useless_barrier) {
+    for (int i = 0; i < barriers.length(); i++) {
+      Node* n = barriers.at(i);
+      if (!barriers_used.member(n)) {
+        tty->print("XXX useless barrier"); n->dump(-2);
+        ShouldNotReachHere();
+      }
+    }
+  }
+}
+#endif
+
+bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) {
+  // That both nodes have the same control is not sufficient to prove
+  // domination, verify that there's no path from d to n
+  ResourceMark rm;
+  Unique_Node_List wq;
+  wq.push(d);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *m = wq.at(next);
+    if (m == n) {
+      return false;
+    }
+    if (m->is_Phi() && m->in(0)->is_Loop()) {
+      assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
+    } else {
+      for (uint i = 0; i < m->req(); i++) {
+        if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) {
+          wq.push(m->in(i));
+        }
+      }
+    }
+  }
+  return true;
+}
+
+bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
+  if (d_c != n_c) {
+    return phase->is_dominator(d_c, n_c);
+  }
+  return is_dominator_same_ctrl(d_c, d, n, phase);
+}
+
+Node* next_mem(Node* mem, int alias) {
+  Node* res = NULL;
+  if (mem->is_Proj()) {
+    res = mem->in(0);
+  } else if (mem->is_SafePoint() || mem->is_MemBar()) {
+    res = mem->in(TypeFunc::Memory);
+  } else if (mem->is_Phi()) {
+    res = mem->in(1);
+  } else if (mem->is_ShenandoahBarrier()) {
+    res = mem->in(ShenandoahBarrierNode::Memory);
+  } else if (mem->is_MergeMem()) {
+    res = mem->as_MergeMem()->memory_at(alias);
+  } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
+    assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
+    res = mem->in(MemNode::Memory);
+  } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
+    res = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
+  } else {
+#ifdef ASSERT
+    mem->dump();
+#endif
+    ShouldNotReachHere();
+  }
+  return res;
+}
+
+Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
+  Node* iffproj = NULL;
+  while (c != dom) {
+    Node* next = phase->idom(c);
+    assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?");
+    if (c->is_Region()) {
+      ResourceMark rm;
+      Unique_Node_List wq;
+      wq.push(c);
+      for (uint i = 0; i < wq.size(); i++) {
+        Node *n = wq.at(i);
+        if (n == next) {
+          continue;
+        }
+        if (n->is_Region()) {
+          for (uint j = 1; j < n->req(); j++) {
+            wq.push(n->in(j));
+          }
+        } else {
+          wq.push(n->in(0));
+        }
+      }
+      for (uint i = 0; i < wq.size(); i++) {
+        Node *n = wq.at(i);
+        assert(n->is_CFG(), "");
+        if (n->is_Multi()) {
+          for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
+            Node* u = n->fast_out(j);
+            if (u->is_CFG()) {
+              if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
+                return NodeSentinel;
+              }
+            }
+          }
+        }
+      }
+    } else  if (c->is_Proj()) {
+      if (c->is_IfProj()) {
+        if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) {
+          // continue;
+        } else {
+          if (!allow_one_proj) {
+            return NodeSentinel;
+          }
+          if (iffproj == NULL) {
+            iffproj = c;
+          } else {
+            return NodeSentinel;
+          }
+        }
+      } else if (c->Opcode() == Op_JumpProj) {
+        return NodeSentinel; // unsupported
+      } else if (c->Opcode() == Op_CatchProj) {
+        return NodeSentinel; // unsupported
+      } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
+        return NodeSentinel; // unsupported
+      } else {
+        assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
+      }
+    }
+    c = next;
+  }
+  return iffproj;
+}
+
+bool ShenandoahBarrierNode::build_loop_late_post(PhaseIdealLoop* phase, Node* n) {
+  if (n->Opcode() == Op_ShenandoahReadBarrier ||
+      n->Opcode() == Op_ShenandoahWriteBarrier ||
+      n->Opcode() == Op_ShenandoahWBMemProj) {
+
+    phase->build_loop_late_post_work(n, false);
+
+    if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+      // The write barrier and its memory proj must have the same
+      // control otherwise some loop opts could put nodes (Phis) between
+      // them
+      Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
+      if (proj != NULL) {
+        phase->set_ctrl_and_loop(proj, phase->get_ctrl(n));
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool ShenandoahBarrierNode::sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl) {
+  ctrl = phase->find_non_split_ctrl(ctrl);
+  assert(phase->dom_depth(n_ctrl) <= phase->dom_depth(ctrl), "n is later than its clone");
+  set_req(0, ctrl);
+  phase->register_new_node(this, ctrl);
+  return true;
+}
+
+#ifdef ASSERT
+void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) {
+  const bool trace = false;
+  if (trace) { tty->print("X control is"); c->dump(); }
+
+  uint start = controls.size();
+  controls.push(c);
+  for (uint i = start; i < controls.size(); i++) {
+    Node *n = controls.at(i);
+
+    if (trace) { tty->print("X from"); n->dump(); }
+
+    if (n == rep_ctrl) {
+      continue;
+    }
+
+    if (n->is_Proj()) {
+      Node* n_dom = n->in(0);
+      IdealLoopTree* n_dom_loop = phase->get_loop(n_dom);
+      if (n->is_IfProj() && n_dom->outcnt() == 2) {
+        n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0));
+      }
+      if (n_dom_loop != phase->ltree_root()) {
+        Node* tail = n_dom_loop->tail();
+        if (tail->is_Region()) {
+          for (uint j = 1; j < tail->req(); j++) {
+            if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) {
+              assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?");
+              // entering loop from below, mark backedge
+              if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); }
+              controls.push(tail->in(j));
+              //assert(n->in(0) == n_dom, "strange flow control");
+            }
+          }
+        } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) {
+          // entering loop from below, mark backedge
+          if (trace) { tty->print("X pushing backedge"); tail->dump(); }
+          controls.push(tail);
+          //assert(n->in(0) == n_dom, "strange flow control");
+        }
+      }
+    }
+
+    if (n->is_Loop()) {
+      Node* c = n->in(LoopNode::EntryControl);
+      if (trace) { tty->print("X pushing"); c->dump(); }
+      controls.push(c);
+    } else if (n->is_Region()) {
+      for (uint i = 1; i < n->req(); i++) {
+        Node* c = n->in(i);
+        if (trace) { tty->print("X pushing"); c->dump(); }
+        controls.push(c);
+      }
+    } else {
+      Node* c = n->in(0);
+      if (trace) { tty->print("X pushing"); c->dump(); }
+      controls.push(c);
+    }
+  }
+}
+
+bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
+  const bool trace = false;
+  if (trace) {
+    tty->print("XXX mem is"); mem->dump();
+    tty->print("XXX rep ctrl is"); rep_ctrl->dump();
+    tty->print_cr("XXX alias is %d", alias);
+  }
+  ResourceMark rm;
+  Unique_Node_List wq;
+  Unique_Node_List controls;
+  wq.push(mem);
+  for (uint next = 0; next < wq.size(); next++) {
+    Node *nn = wq.at(next);
+    if (trace) { tty->print("XX from mem"); nn->dump(); }
+    assert(nn->bottom_type() == Type::MEMORY, "memory only");
+
+    if (nn->is_Phi()) {
+      Node* r = nn->in(0);
+      for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) {
+        Node* u = r->fast_out(j);
+        if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn &&
+            (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) {
+          if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); }
+          wq.push(u);
+        }
+      }
+    }
+
+    for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) {
+      Node* use = nn->fast_out(i);
+
+      if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); }
+      if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) {
+        Node* c = use->in(0);
+        if (phase->is_dominator(rep_ctrl, c)) {
+          memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
+        } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) {
+          Node* region = c;
+          if (trace) { tty->print("XX unc region"); region->dump(); }
+          for (uint j = 1; j < region->req(); j++) {
+            if (phase->is_dominator(rep_ctrl, region->in(j))) {
+              if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); }
+              memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase);
+            }
+          }
+        }
+        //continue;
+      } else if (use->is_Phi()) {
+        assert(use->bottom_type() == Type::MEMORY, "bad phi");
+        if ((use->adr_type() == TypePtr::BOTTOM) ||
+            phase->C->get_alias_index(use->adr_type()) == alias) {
+          for (uint j = 1; j < use->req(); j++) {
+            if (use->in(j) == nn) {
+              Node* c = use->in(0)->in(j);
+              if (phase->is_dominator(rep_ctrl, c)) {
+                memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
+              }
+            }
+          }
+        }
+        //        continue;
+      }
+
+      if (use->is_MergeMem()) {
+        if (use->as_MergeMem()->memory_at(alias) == nn) {
+          if (trace) { tty->print("XX Next mem"); use->dump(); }
+          // follow the memory edges
+          wq.push(use);
+        }
+      } else if (use->is_Phi()) {
+        assert(use->bottom_type() == Type::MEMORY, "bad phi");
+        if ((use->adr_type() == TypePtr::BOTTOM) ||
+            phase->C->get_alias_index(use->adr_type()) == alias) {
+          if (trace) { tty->print("XX Next mem"); use->dump(); }
+          // follow the memory edges
+          wq.push(use);
+        }
+      } else if (use->bottom_type() == Type::MEMORY &&
+                 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
+        if (trace) { tty->print("XX Next mem"); use->dump(); }
+        // follow the memory edges
+        wq.push(use);
+      } else if ((use->is_SafePoint() || use->is_MemBar()) &&
+                 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
+        for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
+          Node* u = use->fast_out(j);
+          if (u->bottom_type() == Type::MEMORY) {
+            if (trace) { tty->print("XX Next mem"); u->dump(); }
+            // follow the memory edges
+            wq.push(u);
+          }
+        }
+      } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) {
+        Node* m = use->find_out_with(Op_ShenandoahWBMemProj);
+        if (m != NULL) {
+          if (trace) { tty->print("XX Next mem"); m->dump(); }
+          // follow the memory edges
+          wq.push(m);
+        }
+      }
+    }
+  }
+
+  if (controls.size() == 0) {
+    return false;
+  }
+
+  for (uint i = 0; i < controls.size(); i++) {
+    Node *n = controls.at(i);
+
+    if (trace) { tty->print("X checking"); n->dump(); }
+
+    if (n->unique_ctrl_out() != NULL) {
+      continue;
+    }
+
+    if (n->Opcode() == Op_NeverBranch) {
+      Node* taken = n->as_Multi()->proj_out(0);
+      if (!controls.member(taken)) {
+        if (trace) { tty->print("X not seen"); taken->dump(); }
+        return false;
+      }
+      continue;
+    }
+
+    for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
+      Node* u = n->fast_out(j);
+
+      if (u->is_CFG()) {
+        if (!controls.member(u)) {
+          if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
+            if (trace) { tty->print("X not seen but unc"); u->dump(); }
+          } else {
+            Node* c = u;
+            do {
+              c = c->unique_ctrl_out();
+            } while (c != NULL && c->is_Region());
+            if (c != NULL && c->Opcode() == Op_Halt) {
+              if (trace) { tty->print("X not seen but halt"); c->dump(); }
+            } else {
+              if (trace) { tty->print("X not seen"); u->dump(); }
+              return false;
+            }
+          }
+        } else {
+          if (trace) { tty->print("X seen"); u->dump(); }
+        }
+      }
+    }
+  }
+  return true;
+}
+#endif
+
+Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
+  ResourceMark rm;
+  VectorSet wq(Thread::current()->resource_area());
+  wq.set(mem->_idx);
+  mem_ctrl = phase->get_ctrl(mem);
+  while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) {
+    mem = next_mem(mem, alias);
+    if (wq.test_set(mem->_idx)) {
+      return NULL; // hit an unexpected loop
+    }
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  if (mem->is_MergeMem()) {
+    mem = mem->as_MergeMem()->memory_at(alias);
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  return mem;
+}
+
+Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
+  ResourceMark rm;
+  VectorSet wq(Thread::current()->resource_area());
+  wq.set(mem->_idx);
+  mem_ctrl = phase->ctrl_or_self(mem);
+  while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) {
+    mem = next_mem(mem, alias);
+    if (wq.test_set(mem->_idx)) {
+      return NULL;
+    }
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  if (mem->is_MergeMem()) {
+    mem = mem->as_MergeMem()->memory_at(alias);
+    mem_ctrl = phase->ctrl_or_self(mem);
+  }
+  return mem;
+}
+
+static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) {
+  Node* mem_in = wb->in(ShenandoahBarrierNode::Memory);
+  Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj);
+
+  for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) {
+    Node* u = proj->last_out(i);
+    igvn.rehash_node_delayed(u);
+    int nb = u->replace_edge(proj, mem_in);
+    assert(nb > 0, "no replacement?");
+    i -= nb;
+  }
+}
+
+Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) {
+  Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl);
+  Node* above_pred = phase->skip_all_loop_predicates(entry);
+  Node* ctrl = entry;
+  while (ctrl != above_pred) {
+    Node* next = ctrl->in(0);
+    if (!phase->is_dominator(val_ctrl, next)) {
+      break;
+    }
+    ctrl = next;
+  }
+  return ctrl;
+}
+
+static MemoryGraphFixer* find_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias) {
+  for (int i = 0; i < memory_graph_fixers.length(); i++) {
+    if (memory_graph_fixers.at(i)->alias() == alias) {
+      return memory_graph_fixers.at(i);
+    }
+  }
+  return NULL;
+}
+
+static MemoryGraphFixer* create_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) {
+  assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet");
+  MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase);
+  memory_graph_fixers.push(fixer);
+  return fixer;
+}
+
+void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
+  assert(cl->is_Loop(), "bad control");
+  Node* ctrl = move_above_predicates(cl, val_ctrl, phase);
+  Node* mem_ctrl = NULL;
+  int alias = phase->C->get_alias_index(adr_type());
+
+  MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
+  if (fixer == NULL) {
+    fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
+  }
+
+  Node* proj = find_out_with(Op_ShenandoahWBMemProj);
+
+  fixer->remove(proj);
+  Node* mem = fixer->find_mem(ctrl, NULL);
+
+  assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph");
+
+  phase->set_ctrl_and_loop(this, ctrl);
+  phase->igvn().replace_input_of(this, Control, ctrl);
+
+  disconnect_barrier_mem(this, phase->igvn());
+
+  phase->igvn().replace_input_of(this, Memory, mem);
+  phase->set_ctrl_and_loop(proj, ctrl);
+
+  fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses);
+  assert(proj->outcnt() > 0, "disconnected write barrier");
+}
+
+LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) {
+  // A write barrier between a pre and main loop can get in the way of
+  // vectorization. Move it above the pre loop if possible
+  CountedLoopNode* cl = NULL;
+  if (c->is_IfFalse() &&
+      c->in(0)->is_CountedLoopEnd()) {
+    cl = c->in(0)->as_CountedLoopEnd()->loopnode();
+  } else if (c->is_IfProj() &&
+             c->in(0)->is_If() &&
+             c->in(0)->in(0)->is_IfFalse() &&
+             c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) {
+    cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode();
+  }
+  if (cl != NULL &&
+      cl->is_pre_loop() &&
+      val_ctrl != cl &&
+      phase->is_dominator(val_ctrl, cl)) {
+    return cl;
+  }
+  return NULL;
+}
+
+void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
+  Node *n_ctrl = phase->get_ctrl(this);
+  IdealLoopTree *n_loop = phase->get_loop(n_ctrl);
+  Node* val = in(ValueIn);
+  Node* val_ctrl = phase->get_ctrl(val);
+  if (n_loop != phase->ltree_root() && !n_loop->_irreducible) {
+    IdealLoopTree *val_loop = phase->get_loop(val_ctrl);
+    Node* mem = in(Memory);
+    IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem));
+    if (!n_loop->is_member(val_loop) &&
+        n_loop->is_member(mem_loop)) {
+      Node* n_loop_head = n_loop->_head;
+
+      if (n_loop_head->is_Loop()) {
+        LoopNode* loop = n_loop_head->as_Loop();
+        if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
+          LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase);
+          if (res != NULL) {
+            loop = res;
+          }
+        }
+
+        try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
+      }
+    }
+  }
+  LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase);
+  if (ctrl != NULL) {
+    try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
+  }
+}
+
+Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) {
+  Node* val = in(ValueIn);
+  Node* val_ctrl = phase->get_ctrl(val);
+  Node* other_mem = other->in(Memory);
+  Node* other_ctrl = phase->get_ctrl(other);
+  Node* this_ctrl = phase->get_ctrl(this);
+  IdealLoopTree* this_loop = phase->get_loop(this_ctrl);
+  IdealLoopTree* other_loop = phase->get_loop(other_ctrl);
+
+  Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl);
+
+  if (ctrl->is_Proj() &&
+      ctrl->in(0)->is_Call() &&
+      ctrl->unique_ctrl_out() != NULL &&
+      ctrl->unique_ctrl_out()->Opcode() == Op_Catch &&
+      !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) {
+    return NULL;
+  }
+
+  IdealLoopTree* loop = phase->get_loop(ctrl);
+
+  // We don't want to move a write barrier in a loop
+  // If the LCA is in a inner loop, try a control out of loop if possible
+  while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) {
+    ctrl = phase->idom(ctrl);
+    if (ctrl->is_MultiBranch()) {
+      ctrl = ctrl->in(0);
+    }
+    if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) {
+      return NULL;
+    }
+    loop = phase->get_loop(ctrl);
+  }
+
+  if (ShenandoahDontIncreaseWBFreq) {
+    Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase);
+    if (other->Opcode() == Op_ShenandoahWriteBarrier) {
+      Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase);
+      if (other_iffproj == NULL || this_iffproj == NULL) {
+        return ctrl;
+      } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel &&
+                 other_iffproj->in(0) == this_iffproj->in(0)) {
+        return ctrl;
+      }
+    } else if (this_iffproj == NULL) {
+      return ctrl;
+    }
+    return NULL;
+  }
+
+  return ctrl;
+}
+
+void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm) {
+  bool progress = false;
+  Unique_Node_List uses;
+  do {
+    progress = false;
+    for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
+      ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
+
+      wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses);
+
+      Node* val = wb->in(ValueIn);
+
+      for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
+        Node* u = val->fast_out(j);
+        if (u != wb && u->is_ShenandoahBarrier()) {
+          Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase);
+
+          if (rep_ctrl != NULL) {
+            Node* other = u;
+            Node* val_ctrl = phase->get_ctrl(val);
+            if (rep_ctrl->is_Proj() &&
+                rep_ctrl->in(0)->is_Call() &&
+                rep_ctrl->unique_ctrl_out() != NULL &&
+                rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) {
+              rep_ctrl = rep_ctrl->in(0)->in(0);
+
+              assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control");
+            } else {
+              LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase);
+              if (c != NULL) {
+                rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase);
+              } else {
+                while (rep_ctrl->is_IfProj()) {
+                  CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
+                  if (unc != NULL) {
+                    int req = unc->uncommon_trap_request();
+                    Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
+                    if ((trap_reason == Deoptimization::Reason_loop_limit_check ||
+                         trap_reason == Deoptimization::Reason_predicate ||
+                         trap_reason == Deoptimization::Reason_profile_predicate) &&
+                        phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) {
+                      rep_ctrl = rep_ctrl->in(0)->in(0);
+                      continue;
+                    }
+                  }
+                  break;
+                }
+              }
+            }
+
+            Node* wb_ctrl = phase->get_ctrl(wb);
+            Node* other_ctrl = phase->get_ctrl(other);
+            int alias = phase->C->get_alias_index(wb->adr_type());
+            MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);;
+            if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) {
+              if (fixer == NULL) {
+                fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
+              }
+              Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL);
+
+              if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) {
+                continue;
+              }
+
+              Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj);
+              fixer->remove(wb_proj);
+              Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL);
+
+              if (wb->in(Memory) != mem) {
+                disconnect_barrier_mem(wb, phase->igvn());
+                phase->igvn().replace_input_of(wb, Memory, mem);
+              }
+              if (rep_ctrl != wb_ctrl) {
+                phase->set_ctrl_and_loop(wb, rep_ctrl);
+                phase->igvn().replace_input_of(wb, Control, rep_ctrl);
+                phase->set_ctrl_and_loop(wb_proj, rep_ctrl);
+                progress = true;
+              }
+
+              fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses);
+
+              assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph");
+            }
+
+            if (other->Opcode() == Op_ShenandoahWriteBarrier) {
+              Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj);
+              if (fixer != NULL) {
+                fixer->remove(other_proj);
+              }
+              phase->igvn().replace_node(other_proj, other->in(Memory));
+            }
+            phase->igvn().replace_node(other, wb);
+            --j; --jmax;
+          }
+        }
+      }
+    }
+  } while(progress);
+}
+
+// Some code duplication with PhaseIdealLoop::split_if_with_blocks_pre()
+Node* ShenandoahWriteBarrierNode::try_split_thru_phi(PhaseIdealLoop* phase) {
+  Node *ctrl = phase->get_ctrl(this);
+  if (ctrl == NULL) {
+    return this;
+  }
+  Node *blk = phase->has_local_phi_input(this);
+  if (blk == NULL) {
+    return this;
+  }
+
+  if (in(0) != blk) {
+    return this;
+  }
+
+  int policy = blk->req() >> 2;
+
+  if (blk->is_CountedLoop()) {
+    IdealLoopTree *lp = phase->get_loop(blk);
+    if (lp && lp->_rce_candidate) {
+      return this;
+    }
+  }
+
+  if (phase->C->live_nodes() > 35000) {
+    return this;
+  }
+
+  uint unique = phase->C->unique();
+  Node *phi = phase->split_thru_phi(this, blk, policy);
+  if (phi == NULL) {
+    return this;
+  }
+
+  Node* mem_phi = new PhiNode(blk, Type::MEMORY, phase->C->alias_type(adr_type())->adr_type());
+  for (uint i = 1; i < blk->req(); i++) {
+    Node* n = phi->in(i);
+    if (n->Opcode() == Op_ShenandoahWriteBarrier &&
+        n->_idx >= unique) {
+      Node* proj = new ShenandoahWBMemProjNode(n);
+      phase->register_new_node(proj, phase->get_ctrl(n));
+      mem_phi->init_req(i, proj);
+    } else {
+      Node* mem = in(ShenandoahBarrierNode::Memory);
+      if (mem->is_Phi() && mem->in(0) == blk) {
+        mem = mem->in(i);
+      }
+      mem_phi->init_req(i, mem);
+    }
+  }
+  phase->register_new_node(mem_phi, blk);
+
+
+  Node* proj = find_out_with(Op_ShenandoahWBMemProj);
+  phase->igvn().replace_node(proj, mem_phi);
+  phase->igvn().replace_node(this, phi);
+
+  return phi;
+}
+
+void ShenandoahReadBarrierNode::try_move(PhaseIdealLoop* phase) {
+  Node *n_ctrl = phase->get_ctrl(this);
+  if (n_ctrl == NULL) {
+    return;
+  }
+  Node* mem = in(MemNode::Memory);
+  int alias = phase->C->get_alias_index(adr_type());
+  const bool trace = false;
+
+#ifdef ASSERT
+  if (trace) { tty->print("Trying to move mem of"); dump(); }
+#endif
+
+  Node* new_mem = mem;
+
+  ResourceMark rm;
+  VectorSet seen(Thread::current()->resource_area());
+  Node_List phis;
+
+  for (;;) {
+#ifdef ASSERT
+    if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
+#endif
+    if (mem->is_Proj() && mem->in(0)->is_Start()) {
+      if (new_mem != in(MemNode::Memory)) {
+#ifdef ASSERT
+        if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
+#endif
+        phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
+      }
+      return;
+    }
+
+    Node* candidate = mem;
+    do {
+      if (!is_independent(mem)) {
+        if (trace) { tty->print_cr("Not independent"); }
+        if (new_mem != in(MemNode::Memory)) {
+#ifdef ASSERT
+          if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
+#endif
+          phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
+        }
+        return;
+      }
+      if (seen.test_set(mem->_idx)) {
+        if (trace) { tty->print_cr("Already seen"); }
+        ShouldNotReachHere();
+        // Strange graph
+        if (new_mem != in(MemNode::Memory)) {
+#ifdef ASSERT
+          if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
+#endif
+          phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
+        }
+        return;
+      }
+      if (mem->is_Phi()) {
+        phis.push(mem);
+      }
+      mem = next_mem(mem, alias);
+      if (mem->bottom_type() == Type::MEMORY) {
+        candidate = mem;
+      }
+      assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator");
+#ifdef ASSERT
+      if (trace) { tty->print("Next mem is"); mem->dump(); }
+#endif
+    } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl));
+
+    assert(mem->bottom_type() == Type::MEMORY, "bad mem");
+
+    bool not_dom = false;
+    for (uint i = 0; i < phis.size() && !not_dom; i++) {
+      Node* nn = phis.at(i);
+
+#ifdef ASSERT
+      if (trace) { tty->print("Looking from phi"); nn->dump(); }
+#endif
+      assert(nn->is_Phi(), "phis only");
+      for (uint j = 2; j < nn->req() && !not_dom; j++) {
+        Node* m = nn->in(j);
+#ifdef ASSERT
+        if (trace) { tty->print("Input %d is", j); m->dump(); }
+#endif
+        while (m != mem && !seen.test_set(m->_idx)) {
+          if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) {
+            not_dom = true;
+            // Scheduling anomaly
+#ifdef ASSERT
+            if (trace) { tty->print("Giving up"); m->dump(); }
+#endif
+            break;
+          }
+          if (!is_independent(m)) {
+            if (trace) { tty->print_cr("Not independent"); }
+            if (new_mem != in(MemNode::Memory)) {
+#ifdef ASSERT
+              if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
+#endif
+              phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
+            }
+            return;
+          }
+          if (m->is_Phi()) {
+            phis.push(m);
+          }
+          m = next_mem(m, alias);
+#ifdef ASSERT
+          if (trace) { tty->print("Next mem is"); m->dump(); }
+#endif
+        }
+      }
+    }
+    if (!not_dom) {
+      new_mem = mem;
+      phis.clear();
+    } else {
+      seen.Clear();
+    }
+  }
+}
+
+CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
+  Node* val = in(ValueIn);
+
+  const Type* val_t = igvn.type(val);
+
+  if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
+      val->Opcode() == Op_CastPP &&
+      val->in(0) != NULL &&
+      val->in(0)->Opcode() == Op_IfTrue &&
+      val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
+      val->in(0)->in(0)->is_If() &&
+      val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
+      val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
+      val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
+      val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
+      val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
+    assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
+    CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
+    return unc;
+  }
+  return NULL;
+}
+
+void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses) {
+  Node* unc = pin_and_expand_null_check(phase->igvn());
+  Node* val = in(ValueIn);
+
+  if (unc != NULL) {
+    Node* ctrl = phase->get_ctrl(this);
+    Node* unc_ctrl = val->in(0);
+
+    // Don't move write barrier in a loop
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl);
+
+    if (!unc_loop->is_member(loop)) {
+      return;
+    }
+
+    Node* branch = no_branches(ctrl, unc_ctrl, false, phase);
+    assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
+    if (branch == NodeSentinel) {
+      return;
+    }
+
+    RegionNode* r = new RegionNode(3);
+    IfNode* iff = unc_ctrl->in(0)->as_If();
+
+    Node* ctrl_use = unc_ctrl->unique_ctrl_out();
+    Node* unc_ctrl_clone = unc_ctrl->clone();
+    phase->register_control(unc_ctrl_clone, loop, iff);
+    Node* c = unc_ctrl_clone;
+    Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
+    r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
+
+    phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
+    phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
+    phase->lazy_replace(c, unc_ctrl);
+    c = NULL;;
+    phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
+    phase->set_ctrl(val, unc_ctrl_clone);
+
+    IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
+    fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
+    Node* iff_proj = iff->proj_out(0);
+    r->init_req(2, iff_proj);
+    phase->register_control(r, phase->ltree_root(), iff);
+
+    Node* new_bol = new_iff->in(1)->clone();
+    Node* new_cmp = new_bol->in(1)->clone();
+    assert(new_cmp->Opcode() == Op_CmpP, "broken");
+    assert(new_cmp->in(1) == val->in(1), "broken");
+    new_bol->set_req(1, new_cmp);
+    new_cmp->set_req(1, this);
+    phase->register_new_node(new_bol, new_iff->in(0));
+    phase->register_new_node(new_cmp, new_iff->in(0));
+    phase->igvn().replace_input_of(new_iff, 1, new_bol);
+    phase->igvn().replace_input_of(new_cast, 1, this);
+
+    for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) {
+      Node* u = this->fast_out(i);
+      if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) {
+        continue;
+      }
+      phase->igvn().rehash_node_delayed(u);
+      int nb = u->replace_edge(this, new_cast);
+      assert(nb > 0, "no update?");
+      --i; imax -= nb;
+    }
+
+    for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+      Node* u = val->fast_out(i);
+      if (u == this) {
+        continue;
+      }
+      phase->igvn().rehash_node_delayed(u);
+      int nb = u->replace_edge(val, new_cast);
+      assert(nb > 0, "no update?");
+      --i; imax -= nb;
+    }
+
+    Node* new_ctrl = unc_ctrl_clone;
+
+    int alias = phase->C->get_alias_index(adr_type());
+    MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
+    if (fixer == NULL) {
+      fixer = create_fixer(memory_graph_fixers, alias, phase, true);
+    }
+
+    Node* proj = find_out_with(Op_ShenandoahWBMemProj);
+    fixer->remove(proj);
+    Node* mem = fixer->find_mem(new_ctrl, NULL);
+
+    if (in(Memory) != mem) {
+      disconnect_barrier_mem(this, phase->igvn());
+      phase->igvn().replace_input_of(this, Memory, mem);
+    }
+
+    phase->set_ctrl_and_loop(this, new_ctrl);
+    phase->igvn().replace_input_of(this, Control, new_ctrl);
+    phase->set_ctrl_and_loop(proj, new_ctrl);
+
+    fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses);
+  }
+}
+
+void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) {
+  Node* val = in(ValueIn);
+  CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn());
+  Node* rep = this;
+  Node* ctrl = phase->get_ctrl(this);
+  if (unc != NULL && val->in(0) == ctrl) {
+    Node* unc_ctrl = val->in(0);
+    IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If();
+    ProjNode* other_unc_ctrl = other_iff->proj_out(1);
+    Node* cast = NULL;
+    for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) {
+      Node* u = other_unc_ctrl->fast_out(i);
+      if (u->Opcode() == Op_CastPP && u->in(1) == this) {
+        cast = u;
+      }
+    }
+    assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken");
+    rep = cast;
+  }
+
+  // Replace all uses of barrier's input that are dominated by ctrl
+  // with the value returned by the barrier: no need to keep both
+  // live.
+  for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+    Node* u = val->fast_out(i);
+    if (u != this) {
+      if (u->is_Phi()) {
+        int nb = 0;
+        for (uint j = 1; j < u->req(); j++) {
+          if (u->in(j) == val) {
+            Node* c = u->in(0)->in(j);
+            if (phase->is_dominator(ctrl, c)) {
+              phase->igvn().replace_input_of(u, j, rep);
+              nb++;
+            }
+          }
+        }
+        if (nb > 0) {
+          imax -= nb;
+          --i;
+        }
+      } else {
+        Node* c = phase->ctrl_or_self(u);
+        if (is_dominator(ctrl, c, this, u, phase)) {
+          phase->igvn().rehash_node_delayed(u);
+          int nb = u->replace_edge(val, rep);
+          assert(nb > 0, "no update?");
+          --i, imax -= nb;
+        }
+      }
+    }
+  }
+}
+
+Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
+  Node* mem = NULL;
+  Node* c = ctrl;
+  do {
+    if (c->is_Region()) {
+      Node* phi_bottom = NULL;
+      for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
+        Node* u = c->fast_out(i);
+        if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
+          if (u->adr_type() == TypePtr::BOTTOM) {
+            mem = u;
+          }
+        }
+      }
+    } else {
+      if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
+        CallProjections projs;
+        c->as_Call()->extract_projections(&projs, true, false);
+        if (projs.fallthrough_memproj != NULL) {
+          if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
+            if (projs.catchall_memproj == NULL) {
+              mem = projs.fallthrough_memproj;
+            } else {
+              if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
+                mem = projs.fallthrough_memproj;
+              } else {
+                assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
+                mem = projs.catchall_memproj;
+              }
+            }
+          }
+        } else {
+          Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
+          if (proj != NULL &&
+              proj->adr_type() == TypePtr::BOTTOM) {
+            mem = proj;
+          }
+        }
+      } else {
+        for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
+          Node* u = c->fast_out(i);
+          if (u->is_Proj() &&
+              u->bottom_type() == Type::MEMORY &&
+              u->adr_type() == TypePtr::BOTTOM) {
+              assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
+              assert(mem == NULL, "only one proj");
+              mem = u;
+          }
+        }
+        assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected");
+      }
+    }
+    c = phase->idom(c);
+  } while (mem == NULL);
+  return mem;
+}
+
+void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
+  for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+    Node* u = n->fast_out(i);
+    if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
+      uses.push(u);
+    }
+  }
+}
+
+static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
+  OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
+  Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
+  phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
+  Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
+  phase->register_control(new_le, phase->get_loop(le), le->in(0));
+  phase->lazy_replace(outer, new_outer);
+  phase->lazy_replace(le, new_le);
+  inner->clear_strip_mined();
+}
+
+void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
+                                                  PhaseIdealLoop* phase) {
+  IdealLoopTree* loop = phase->get_loop(ctrl);
+  Node* thread = new ThreadLocalNode();
+  phase->register_new_node(thread, ctrl);
+  Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  phase->set_ctrl(offset, phase->C->root());
+  Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
+  phase->register_new_node(gc_state_addr, ctrl);
+  uint gc_state_idx = Compile::AliasIdxRaw;
+  const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
+  debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
+
+  Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
+  phase->register_new_node(gc_state, ctrl);
+  Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
+  phase->register_new_node(heap_stable_and, ctrl);
+  Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
+  phase->register_new_node(heap_stable_cmp, ctrl);
+  Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
+  phase->register_new_node(heap_stable_test, ctrl);
+  IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+  phase->register_control(heap_stable_iff, loop, ctrl);
+
+  heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
+  phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
+  ctrl = new IfTrueNode(heap_stable_iff);
+  phase->register_control(ctrl, loop, heap_stable_iff);
+
+  assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
+}
+
+void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
+  const Type* val_t = phase->igvn().type(val);
+  if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
+    phase->register_new_node(null_cmp, ctrl);
+    Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
+    phase->register_new_node(null_test, ctrl);
+    IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
+    phase->register_control(null_iff, loop, ctrl);
+    ctrl = new IfTrueNode(null_iff);
+    phase->register_control(ctrl, loop, null_iff);
+    null_ctrl = new IfFalseNode(null_iff);
+    phase->register_control(null_ctrl, loop, null_iff);
+  }
+}
+
+Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
+  IdealLoopTree *loop = phase->get_loop(c);
+  Node* iff = unc_ctrl->in(0);
+  assert(iff->is_If(), "broken");
+  Node* new_iff = iff->clone();
+  new_iff->set_req(0, c);
+  phase->register_control(new_iff, loop, c);
+  Node* iffalse = new IfFalseNode(new_iff->as_If());
+  phase->register_control(iffalse, loop, new_iff);
+  Node* iftrue = new IfTrueNode(new_iff->as_If());
+  phase->register_control(iftrue, loop, new_iff);
+  c = iftrue;
+  const Type *t = phase->igvn().type(val);
+  assert(val->Opcode() == Op_CastPP, "expect cast to non null here");
+  Node* uncasted_val = val->in(1);
+  val = new CastPPNode(uncasted_val, t);
+  val->init_req(0, c);
+  phase->register_new_node(val, c);
+  return val;
+}
+
+void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
+                                                Unique_Node_List& uses, PhaseIdealLoop* phase) {
+  IfNode* iff = unc_ctrl->in(0)->as_If();
+  Node* proj = iff->proj_out(0);
+  assert(proj != unc_ctrl, "bad projection");
+  Node* use = proj->unique_ctrl_out();
+
+  assert(use == unc || use->is_Region(), "what else?");
+
+  uses.clear();
+  if (use == unc) {
+    phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use));
+    for (uint i = 1; i < unc->req(); i++) {
+      Node* n = unc->in(i);
+      if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) {
+        uses.push(n);
+      }
+    }
+  } else {
+    assert(use->is_Region(), "what else?");
+    uint idx = 1;
+    for (; use->in(idx) != proj; idx++);
+    for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) {
+      Node* u = use->fast_out(i);
+      if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) {
+        uses.push(u->in(idx));
+      }
+    }
+  }
+  for(uint next = 0; next < uses.size(); next++ ) {
+    Node *n = uses.at(next);
+    assert(phase->get_ctrl(n) == proj, "bad control");
+    phase->set_ctrl_and_loop(n, new_unc_ctrl);
+    if (n->in(0) == proj) {
+      phase->igvn().replace_input_of(n, 0, new_unc_ctrl);
+    }
+    for (uint i = 0; i < n->req(); i++) {
+      Node* m = n->in(i);
+      if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) {
+        uses.push(m);
+      }
+    }
+  }
+
+  phase->igvn().rehash_node_delayed(use);
+  int nb = use->replace_edge(proj, new_unc_ctrl);
+  assert(nb == 1, "only use expected");
+}
+
+void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
+  IdealLoopTree *loop = phase->get_loop(ctrl);
+  Node* raw_rbtrue = new CastP2XNode(ctrl, val);
+  phase->register_new_node(raw_rbtrue, ctrl);
+  Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
+  phase->register_new_node(cset_offset, ctrl);
+  Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+  phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root());
+  Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset);
+  phase->register_new_node(in_cset_fast_test_adr, ctrl);
+  uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
+  const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
+  debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
+  Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
+  phase->register_new_node(in_cset_fast_test_load, ctrl);
+  Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
+  phase->register_new_node(in_cset_fast_test_cmp, ctrl);
+  Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
+  phase->register_new_node(in_cset_fast_test_test, ctrl);
+  IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+  phase->register_control(in_cset_fast_test_iff, loop, ctrl);
+
+  not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
+  phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
+
+  ctrl = new IfFalseNode(in_cset_fast_test_iff);
+  phase->register_control(ctrl, loop, in_cset_fast_test_iff);
+}
+
+void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
+                                              Node* raw_mem, Node* wb_mem,
+                                              int alias,
+                                              PhaseIdealLoop* phase) {
+  IdealLoopTree*loop = phase->get_loop(ctrl);
+  const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
+
+  // The slow path stub consumes and produces raw memory in addition
+  // to the existing memory edges
+  Node* base = find_bottom_mem(ctrl, phase);
+
+  MergeMemNode* mm = MergeMemNode::make(base);
+  mm->set_memory_at(alias, wb_mem);
+  mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
+  phase->register_new_node(mm, ctrl);
+
+  Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
+  call->init_req(TypeFunc::Control, ctrl);
+  call->init_req(TypeFunc::I_O, phase->C->top());
+  call->init_req(TypeFunc::Memory, mm);
+  call->init_req(TypeFunc::FramePtr, phase->C->top());
+  call->init_req(TypeFunc::ReturnAdr, phase->C->top());
+  call->init_req(TypeFunc::Parms, val);
+  phase->register_control(call, loop, ctrl);
+  ctrl = new ProjNode(call, TypeFunc::Control);
+  phase->register_control(ctrl, loop, call);
+  result_mem = new ProjNode(call, TypeFunc::Memory);
+  phase->register_new_node(result_mem, call);
+  val = new ProjNode(call, TypeFunc::Parms);
+  phase->register_new_node(val, call);
+  val = new CheckCastPPNode(ctrl, val, obj_type);
+  phase->register_new_node(val, ctrl);
+}
+
+void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
+  Node* ctrl = phase->get_ctrl(barrier);
+  Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
+
+  // Update the control of all nodes that should be after the
+  // barrier control flow
+  uses.clear();
+  // Every node that is control dependent on the barrier's input
+  // control will be after the expanded barrier. The raw memory (if
+  // its memory is control dependent on the barrier's input control)
+  // must stay above the barrier.
+  uses_to_ignore.clear();
+  if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) {
+    uses_to_ignore.push(init_raw_mem);
+  }
+  for (uint next = 0; next < uses_to_ignore.size(); next++) {
+    Node *n = uses_to_ignore.at(next);
+    for (uint i = 0; i < n->req(); i++) {
+      Node* in = n->in(i);
+      if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
+        uses_to_ignore.push(in);
+      }
+    }
+  }
+  for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) {
+    Node* u = ctrl->fast_out(i);
+    if (u->_idx < last &&
+        u != barrier &&
+        !uses_to_ignore.member(u) &&
+        (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) &&
+        (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) {
+      Node* old_c = phase->ctrl_or_self(u);
+      Node* c = old_c;
+      if (c != ctrl ||
+          is_dominator_same_ctrl(old_c, barrier, u, phase) ||
+          ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
+        phase->igvn().rehash_node_delayed(u);
+        int nb = u->replace_edge(ctrl, region);
+        if (u->is_CFG()) {
+          if (phase->idom(u) == ctrl) {
+            phase->set_idom(u, region, phase->dom_depth(region));
+          }
+        } else if (phase->get_ctrl(u) == ctrl) {
+          assert(u != init_raw_mem, "should leave input raw mem above the barrier");
+          uses.push(u);
+        }
+        assert(nb == 1, "more than 1 ctrl input?");
+        --i, imax -= nb;
+      }
+    }
+  }
+}
+
+void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) {
+  Node_List enqueue_barriers;
+  if (ShenandoahStoreValEnqueueBarrier) {
+    Unique_Node_List wq;
+    wq.push(phase->C->root());
+    for (uint i = 0; i < wq.size(); i++) {
+      Node* n = wq.at(i);
+      if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+        enqueue_barriers.push(n);
+      }
+      for (uint i = 0; i < n->req(); i++) {
+        Node* in = n->in(i);
+        if (in != NULL) {
+          wq.push(in);
+        }
+      }
+    }
+  }
+
+  const bool trace = false;
+
+  // Collect raw memory state at CFG points in the entire graph and
+  // record it in memory_nodes. Optimize the raw memory graph in the
+  // process. Optimizing the memory graph also makes the memory graph
+  // simpler.
+  GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
+
+  // Let's try to common write barriers again
+  optimize_before_expansion(phase, memory_graph_fixers, true);
+
+  Unique_Node_List uses;
+  for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
+    ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
+    Node* ctrl = phase->get_ctrl(wb);
+
+    Node* val = wb->in(ValueIn);
+    if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
+      assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move");
+      phase->set_ctrl(wb, ctrl->in(0)->in(0));
+    } else if (ctrl->is_CallRuntime()) {
+      assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move");
+      phase->set_ctrl(wb, ctrl->in(0));
+    }
+
+    assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers");
+    // Look for a null check that dominates this barrier and move the
+    // barrier right after the null check to enable implicit null
+    // checks
+    wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses);
+
+    wb->pin_and_expand_helper(phase);
+  }
+
+  MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
+  Unique_Node_List uses_to_ignore;
+  Unique_Node_List outer_lsms;
+  for (uint i = 0; i < enqueue_barriers.size(); i++) {
+    Node* barrier = enqueue_barriers.at(i);
+    Node* pre_val = barrier->in(1);
+
+    if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
+      ShouldNotReachHere();
+      continue;
+    }
+
+    Node* ctrl = phase->get_ctrl(barrier);
+
+    if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
+      assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
+      ctrl = ctrl->in(0)->in(0);
+      phase->set_ctrl(barrier, ctrl);
+    } else if (ctrl->is_CallRuntime()) {
+      assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
+      ctrl = ctrl->in(0);
+      phase->set_ctrl(barrier, ctrl);
+    }
+
+    Node* init_ctrl = ctrl;
+    IdealLoopTree* loop = phase->get_loop(ctrl);
+    if (loop->_head->is_OuterStripMinedLoop()) {
+      outer_lsms.push(loop->_head);
+    }
+    Node* raw_mem = fixer.find_mem(ctrl, barrier);
+    Node* init_raw_mem = raw_mem;
+    Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
+    Node* heap_stable_ctrl = NULL;
+    Node* null_ctrl = NULL;
+    uint last = phase->C->unique();
+
+    enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
+    Node* region2 = new RegionNode(PATH_LIMIT2);
+    Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    // Stable path.
+    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    region->init_req(_heap_stable, heap_stable_ctrl);
+    phi->init_req(_heap_stable, raw_mem);
+
+    // Null path
+    Node* reg2_ctrl = NULL;
+    test_null(ctrl, pre_val, null_ctrl, phase);
+    if (null_ctrl != NULL) {
+      reg2_ctrl = null_ctrl->in(0);
+      region2->init_req(_null_path, null_ctrl);
+      phi2->init_req(_null_path, raw_mem);
+    } else {
+      region2->del_req(_null_path);
+      phi2->del_req(_null_path);
+    }
+
+    const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
+    const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
+    Node* thread = new ThreadLocalNode();
+    phase->register_new_node(thread, ctrl);
+    Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
+    phase->register_new_node(buffer_adr, ctrl);
+    Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
+    phase->register_new_node(index_adr, ctrl);
+
+    BasicType index_bt = TypeX_X->basic_type();
+    assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
+    const TypePtr* adr_type = TypeRawPtr::BOTTOM;
+    Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
+    phase->register_new_node(index, ctrl);
+    Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
+    phase->register_new_node(index_cmp, ctrl);
+    Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
+    phase->register_new_node(index_test, ctrl);
+    IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
+    if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff;
+    phase->register_control(queue_full_iff, loop, ctrl);
+    Node* not_full = new IfTrueNode(queue_full_iff);
+    phase->register_control(not_full, loop, queue_full_iff);
+    Node* full = new IfFalseNode(queue_full_iff);
+    phase->register_control(full, loop, queue_full_iff);
+
+    ctrl = not_full;
+
+    Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
+    phase->register_new_node(next_index, ctrl);
+
+    Node* buffer  = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
+    phase->register_new_node(buffer, ctrl);
+    Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
+    phase->register_new_node(log_addr, ctrl);
+    Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
+    phase->register_new_node(log_store, ctrl);
+    // update the index
+    Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
+    phase->register_new_node(index_update, ctrl);
+
+    // Fast-path case
+    region2->init_req(_fast_path, ctrl);
+    phi2->init_req(_fast_path, index_update);
+
+    ctrl = full;
+
+    Node* base = find_bottom_mem(ctrl, phase);
+
+    MergeMemNode* mm = MergeMemNode::make(base);
+    mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
+    phase->register_new_node(mm, ctrl);
+
+    Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
+    call->init_req(TypeFunc::Control, ctrl);
+    call->init_req(TypeFunc::I_O, phase->C->top());
+    call->init_req(TypeFunc::Memory, mm);
+    call->init_req(TypeFunc::FramePtr, phase->C->top());
+    call->init_req(TypeFunc::ReturnAdr, phase->C->top());
+    call->init_req(TypeFunc::Parms, pre_val);
+    call->init_req(TypeFunc::Parms+1, thread);
+    phase->register_control(call, loop, ctrl);
+
+    Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
+    phase->register_control(ctrl_proj, loop, call);
+    Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
+    phase->register_new_node(mem_proj, call);
+
+    // Slow-path case
+    region2->init_req(_slow_path, ctrl_proj);
+    phi2->init_req(_slow_path, mem_proj);
+
+    phase->register_control(region2, loop, reg2_ctrl);
+    phase->register_new_node(phi2, region2);
+
+    region->init_req(_heap_unstable, region2);
+    phi->init_req(_heap_unstable, phi2);
+
+    phase->register_control(region, loop, heap_stable_ctrl->in(0));
+    phase->register_new_node(phi, region);
+
+    fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
+    for(uint next = 0; next < uses.size(); next++ ) {
+      Node *n = uses.at(next);
+      assert(phase->get_ctrl(n) == init_ctrl, "bad control");
+      assert(n != init_raw_mem, "should leave input raw mem above the barrier");
+      phase->set_ctrl(n, region);
+      follow_barrier_uses(n, init_ctrl, uses, phase);
+    }
+    fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
+
+    phase->igvn().replace_node(barrier, pre_val);
+  }
+
+  for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
+    int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
+    ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
+
+    uint last = phase->C->unique();
+    Node* ctrl = phase->get_ctrl(wb);
+    Node* orig_ctrl = ctrl;
+
+    Node* raw_mem = fixer.find_mem(ctrl, wb);
+    Node* init_raw_mem = raw_mem;
+    Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
+    int alias = phase->C->get_alias_index(wb->adr_type());
+    Node* wb_mem =  wb->in(Memory);
+    Node* init_wb_mem = wb_mem;
+
+    Node* val = wb->in(ValueIn);
+    Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj);
+    IdealLoopTree *loop = phase->get_loop(ctrl);
+    if (loop->_head->is_OuterStripMinedLoop()) {
+      outer_lsms.push(loop->_head);
+    }
+
+    assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers");
+
+    CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn());
+    Node* unc_ctrl = NULL;
+    if (unc != NULL) {
+      if (val->in(0) != ctrl) {
+        unc = NULL;
+      } else {
+        unc_ctrl = val->in(0);
+      }
+    }
+
+    Node* uncasted_val = val;
+    if (unc != NULL) {
+      uncasted_val = val->in(1);
+    }
+
+    Node* heap_stable_ctrl = NULL;
+    Node* null_ctrl = NULL;
+
+    assert(val->bottom_type()->make_oopptr(), "need oop");
+    assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
+
+    enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
+    Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
+    Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 };
+    Node* region2 = new RegionNode(PATH_LIMIT2);
+    Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr());
+    Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
+    Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+      // Stable path.
+    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
+
+    // Heap stable case
+    region->init_req(_heap_stable, heap_stable_ctrl);
+    val_phi->init_req(_heap_stable, uncasted_val);
+    mem_phi->init_req(_heap_stable, wb_mem);
+    raw_mem_phi->init_req(_heap_stable, raw_mem);
+
+    Node* reg2_ctrl = NULL;<