changeset 57178:100038cae4e1 lworld

8231497: [lworld] Inline type use of Access API (Runtime) Reviewed-by: fparain
author dsimms
date Wed, 02 Oct 2019 11:51:57 +0200
parents b0a43c82421a
children b592654fd114
files make/autoconf/hotspot.m4 src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp src/hotspot/share/asm/macroAssembler_common.cpp src/hotspot/share/c1/c1_Runtime1.cpp src/hotspot/share/ci/ciInstanceKlass.cpp src/hotspot/share/ci/ciReplay.cpp src/hotspot/share/ci/ciValueKlass.cpp src/hotspot/share/classfile/classFileParser.cpp src/hotspot/share/classfile/classLoaderData.cpp src/hotspot/share/classfile/fieldLayoutBuilder.cpp src/hotspot/share/classfile/javaClasses.cpp src/hotspot/share/gc/shared/barrierSet.hpp src/hotspot/share/gc/shared/modRefBarrierSet.hpp src/hotspot/share/gc/shared/modRefBarrierSet.inline.hpp src/hotspot/share/gc/z/zBarrier.hpp src/hotspot/share/gc/z/zBarrierSet.hpp src/hotspot/share/gc/z/zBarrierSet.inline.hpp src/hotspot/share/interpreter/interpreterRuntime.cpp src/hotspot/share/memory/heapInspection.cpp src/hotspot/share/oops/access.hpp src/hotspot/share/oops/access.inline.hpp src/hotspot/share/oops/accessBackend.cpp src/hotspot/share/oops/accessBackend.hpp src/hotspot/share/oops/accessBackend.inline.hpp src/hotspot/share/oops/method.cpp src/hotspot/share/oops/valueArrayKlass.cpp src/hotspot/share/oops/valueArrayOop.hpp src/hotspot/share/oops/valueArrayOop.inline.hpp src/hotspot/share/oops/valueKlass.cpp src/hotspot/share/oops/valueKlass.hpp src/hotspot/share/oops/valueKlass.inline.hpp src/hotspot/share/opto/runtime.cpp src/hotspot/share/prims/jni.cpp src/hotspot/share/prims/unsafe.cpp src/hotspot/share/runtime/deoptimization.cpp src/hotspot/share/runtime/reflection.cpp src/hotspot/share/runtime/sharedRuntime.cpp src/hotspot/share/runtime/signature.cpp test/hotspot/jtreg/runtime/valhalla/valuetypes/ValueOops.java
diffstat 39 files changed, 386 insertions(+), 279 deletions(-) [+]
line wrap: on
line diff
--- a/make/autoconf/hotspot.m4	Fri Sep 27 15:02:28 2019 +0200
+++ b/make/autoconf/hotspot.m4	Wed Oct 02 11:51:57 2019 +0200
@@ -355,7 +355,7 @@
   # Only enable ZGC on supported platforms
   # Temp diasable for Valhalla, working in Access API
   AC_MSG_CHECKING([if zgc can be built])
-  if $VALHALLA_TEMP && ((test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64") || \
+  if ((test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64") || \
      (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xaarch64")); then
     AC_MSG_RESULT([yes])
   else
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -37,6 +37,7 @@
 #include "gc/shared/cardTableBarrierSet.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "nativeInst_x86.hpp"
+#include "oops/oop.inline.hpp"
 #include "oops/objArrayKlass.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
--- a/src/hotspot/share/asm/macroAssembler_common.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/asm/macroAssembler_common.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -27,7 +27,7 @@
 #include "asm/assembler.hpp"
 #include "asm/assembler.inline.hpp"
 #include "asm/macroAssembler.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/signature_cc.hpp"
 #include "runtime/sharedRuntime.hpp"
 #ifdef COMPILER2
--- a/src/hotspot/share/c1/c1_Runtime1.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -447,22 +447,9 @@
 
 JRT_ENTRY(void, Runtime1::load_flattened_array(JavaThread* thread, valueArrayOopDesc* array, int index))
   NOT_PRODUCT(_load_flattened_array_slowcase_cnt++;)
-  Klass* klass = array->klass();
-  assert(klass->is_valueArray_klass(), "expected value array oop");
   assert(array->length() > 0 && index < array->length(), "already checked");
-
-  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-  ValueKlass* vklass = vaklass->element_klass();
-
-  // We have a non-empty flattened array, so the element type must have been initialized.
-  assert(vklass->is_initialized(), "must be");
-  Handle holder(THREAD, vklass->klass_holder()); // keep the vklass alive
-  valueArrayHandle ha(THREAD, array);
-  oop obj = vklass->allocate_instance(CHECK);
-
-  void* src = ha()->value_at_addr(index, vaklass->layout_helper());
-  vklass->value_store(src, vklass->data_for_oop(obj),
-                      vaklass->element_byte_size(), true, false);
+  valueArrayHandle vah(thread, array);
+  oop obj = valueArrayOopDesc::value_alloc_copy_from_index(vah, index, CHECK);
   thread->set_vm_result(obj);
 JRT_END
 
@@ -472,15 +459,7 @@
   if (value == NULL) {
     SharedRuntime::throw_and_post_jvmti_exception(thread, vmSymbols::java_lang_NullPointerException());
   } else {
-    Klass* klass = array->klass();
-    assert(klass->is_valueArray_klass(), "expected value array");
-    assert(ArrayKlass::cast(klass)->element_klass() == value->klass(), "Store type incorrect");
-
-    ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-    ValueKlass* vklass = vaklass->element_klass();
-    const int lh = vaklass->layout_helper();
-    vklass->value_store(vklass->data_for_oop(value), array->value_at_addr(index, lh),
-                        vaklass->element_byte_size(), true, false);
+    array->value_copy_to_index(value, index);
   }
 JRT_END
 
--- a/src/hotspot/share/ci/ciInstanceKlass.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -34,7 +34,7 @@
 #include "memory/resourceArea.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/fieldStreams.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
--- a/src/hotspot/share/ci/ciReplay.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/ci/ciReplay.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -37,7 +37,7 @@
 #include "oops/constantPool.hpp"
 #include "oops/method.inline.hpp"
 #include "oops/oop.inline.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "utilities/copy.hpp"
--- a/src/hotspot/share/ci/ciValueKlass.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/ci/ciValueKlass.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -26,7 +26,7 @@
 #include "ci/ciField.hpp"
 #include "ci/ciUtilities.inline.hpp"
 #include "ci/ciValueKlass.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 
 int ciValueKlass::compute_nonstatic_fields() {
   int result = ciInstanceKlass::compute_nonstatic_fields();
--- a/src/hotspot/share/classfile/classFileParser.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/classfile/classFileParser.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -58,7 +58,7 @@
 #include "oops/method.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/symbol.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/arguments.hpp"
--- a/src/hotspot/share/classfile/classLoaderData.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/classfile/classLoaderData.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -63,7 +63,7 @@
 #include "oops/access.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/oopHandle.inline.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "oops/weakHandle.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/handles.inline.hpp"
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -29,7 +29,7 @@
 #include "memory/resourceArea.hpp"
 #include "oops/array.hpp"
 #include "oops/instanceMirrorKlass.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 
 RawBlock::RawBlock(Kind kind, int size) {
--- a/src/hotspot/share/classfile/javaClasses.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/classfile/javaClasses.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -52,6 +52,7 @@
 #include "oops/symbol.hpp"
 #include "oops/typeArrayOop.inline.hpp"
 #include "oops/valueArrayKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/resolvedMethodTable.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
--- a/src/hotspot/share/gc/shared/barrierSet.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/shared/barrierSet.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -315,6 +315,10 @@
       Raw::clone(src, dst, size);
     }
 
+    static void value_copy_in_heap(void* src, void* dst, ValueKlass* md) {
+      Raw::value_copy(src, dst, md);
+    }
+
     static oop resolve(oop obj) {
       return Raw::resolve(obj);
     }
--- a/src/hotspot/share/gc/shared/modRefBarrierSet.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/shared/modRefBarrierSet.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -104,6 +104,8 @@
     static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
       return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
     }
+
+    static void value_copy_in_heap(void* src, void* dst, ValueKlass* md);
   };
 };
 
--- a/src/hotspot/share/gc/shared/modRefBarrierSet.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/shared/modRefBarrierSet.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -31,6 +31,7 @@
 #include "oops/klass.inline.hpp"
 #include "oops/objArrayOop.hpp"
 #include "oops/oop.hpp"
+#include "oops/valueKlass.hpp"
 
 // count is number of array elements being written
 void ModRefBarrierSet::write_ref_array(HeapWord* start, size_t count) {
@@ -152,4 +153,36 @@
   bs->write_region(MemRegion((HeapWord*)(void*)dst, size));
 }
 
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ModRefBarrierSet::AccessBarrier<decorators, BarrierSetT>::
+value_copy_in_heap(void* src, void* dst, ValueKlass* md) {
+  if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value || (!md->contains_oops())) {
+    Raw::value_copy(src, dst, md);
+  } else {
+    BarrierSetT* bs = barrier_set_cast<BarrierSetT>(BarrierSet::barrier_set());
+    // src/dst aren't oops, need offset to adjust oop map offset
+    const address dst_oop_addr_offset = ((address) dst) - md->first_field_offset();
+    typedef typename ValueOopType<decorators>::type OopType;
+
+    // Pre-barriers...
+    OopMapBlock* map = md->start_of_nonstatic_oop_maps();
+    OopMapBlock* const end = map + md->nonstatic_oop_map_count();
+    while (map != end) {
+      address doop_address = dst_oop_addr_offset + map->offset();
+      bs->write_ref_array_pre((OopType*) doop_address, map->count(), false);
+      map++;
+    }
+
+    Raw::value_copy(src, dst, md);
+
+    // Post-barriers...
+    map = md->start_of_nonstatic_oop_maps();
+    while (map != end) {
+      address doop_address = dst_oop_addr_offset + map->offset();
+      bs->write_ref_array((HeapWord*) doop_address, map->count());
+      map++;
+    }
+  }
+}
+
 #endif // SHARE_GC_SHARED_MODREFBARRIERSET_INLINE_HPP
--- a/src/hotspot/share/gc/z/zBarrier.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/z/zBarrier.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -26,6 +26,7 @@
 
 #include "memory/allocation.hpp"
 #include "oops/oop.hpp"
+#include "oops/valueKlass.hpp"
 
 typedef bool (*ZBarrierFastPath)(uintptr_t);
 typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t);
--- a/src/hotspot/share/gc/z/zBarrierSet.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/z/zBarrierSet.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -84,6 +84,8 @@
 
     static void clone_in_heap(oop src, oop dst, size_t size);
 
+    static void value_copy_in_heap(void* src, void* dst, ValueKlass* md);
+
     //
     // Not in heap
     //
--- a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -214,6 +214,23 @@
   Raw::clone_in_heap(src, dst, size);
 }
 
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, ValueKlass* md) {
+  if (md->contains_oops()) {
+    // src/dst aren't oops, need offset to adjust oop map offset
+    const address src_oop_addr_offset = ((address) src) - md->first_field_offset();
+
+    OopMapBlock* map = md->start_of_nonstatic_oop_maps();
+    OopMapBlock* const end = map + md->nonstatic_oop_map_count();
+    while (map != end) {
+      address soop_address = src_oop_addr_offset + map->offset();
+      ZBarrier::load_barrier_on_oop_array((oop*) soop_address, map->count());
+      map++;
+    }
+  }
+  Raw::value_copy_in_heap(src, dst, md);
+}
+
 //
 // Not in heap
 //
--- a/src/hotspot/share/interpreter/interpreterRuntime.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -48,10 +48,9 @@
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/symbol.hpp"
-#include "oops/valueKlass.hpp"
 #include "oops/valueArrayKlass.hpp"
-#include "oops/valueArrayOop.hpp"
 #include "oops/valueArrayOop.inline.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/nativeLookup.hpp"
 #include "runtime/atomic.hpp"
@@ -337,9 +336,7 @@
   instanceOop new_value = vklass->allocate_instance(
       CHECK_((type2size[field_type]) * AbstractInterpreter::stackElementSize));
   Handle new_value_h = Handle(THREAD, new_value);
-  int first_offset = vklass->first_field_offset();
-  vklass->value_store(vklass->data_for_oop(old_value_h()),
-      vklass->data_for_oop(new_value_h()), true, false);
+  vklass->value_copy_oop_to_new_oop(old_value_h(), new_value_h());
 
   // Updating the field specified in arguments
   if (field_type == T_ARRAY || field_type == T_OBJECT) {
@@ -357,8 +354,7 @@
       Klass* field_k = vklass->get_value_field_klass(field_index);
       ValueKlass* field_vk = ValueKlass::cast(field_k);
       assert(field_vk == vt_oop->klass(), "Must match");
-      field_vk->value_store(field_vk->data_for_oop(vt_oop),
-          ((char*)(oopDesc*)new_value_h()) + field_offset, false, false);
+      field_vk->value_copy_oop_to_new_payload(vt_oop, ((char*)(oopDesc*)new_value_h()) + field_offset);
     } else { // not flattened
       oop voop = *(oop*)f.interpreter_frame_expression_stack_at(tos_idx);
       if (voop == NULL && cp_entry->is_flattenable()) {
@@ -445,7 +441,7 @@
 
   ValueKlass* vklass = ValueKlass::cast(value->klass());
   if (!vklass->is_empty_value()) {
-    vklass->value_store(vklass->data_for_oop(value), ((char*)(oopDesc*)rcv) + offset, true, true);
+    vklass->value_copy_oop_to_payload(value, ((char*)(oopDesc*)rcv) + offset);
   }
 JRT_END
 
@@ -466,11 +462,8 @@
   if (field_vklass->is_empty_value()) {
     res = (instanceOop)field_vklass->default_value();
   } else {
-    // allocate instance
     res = field_vklass->allocate_instance(CHECK);
-    // copy value
-    field_vklass->value_store(((char*)(oopDesc*)obj_h()) + klass->field_offset(index),
-        field_vklass->data_for_oop(res), true, true);
+    field_vklass->value_copy_payload_to_new_oop(((char*)(oopDesc*)obj_h()) + klass->field_offset(index), res);
   }
   assert(res != NULL, "Must be set in one of two paths above");
   thread->set_vm_result(res);
@@ -496,39 +489,14 @@
 JRT_END
 
 JRT_ENTRY(void, InterpreterRuntime::value_array_load(JavaThread* thread, arrayOopDesc* array, int index))
-  Klass* klass = array->klass();
-  assert(klass->is_valueArray_klass(), "expected value array oop");
-
-  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-  ValueKlass* vklass = vaklass->element_klass();
-  arrayHandle ah(THREAD, array);
-  instanceOop value_holder = NULL;
-  if (vklass->is_empty_value()) {
-    value_holder = (instanceOop)vklass->default_value();
-  } else {
-    value_holder = vklass->allocate_instance(CHECK);
-    void* src = ((valueArrayOop)ah())->value_at_addr(index, vaklass->layout_helper());
-    vklass->value_store(src, vklass->data_for_oop(value_holder),
-        vaklass->element_byte_size(), true, false);
-  }
-  assert(value_holder != NULL, "Must be set in one of two paths above");
+  valueArrayHandle vah(thread, (valueArrayOop)array);
+  oop value_holder = valueArrayOopDesc::value_alloc_copy_from_index(vah, index, CHECK);
   thread->set_vm_result(value_holder);
 JRT_END
 
 JRT_ENTRY(void, InterpreterRuntime::value_array_store(JavaThread* thread, void* val, arrayOopDesc* array, int index))
   assert(val != NULL, "can't store null into flat array");
-  Klass* klass = array->klass();
-  assert(klass->is_valueArray_klass(), "expected value array");
-  assert(ArrayKlass::cast(klass)->element_klass() == ((oop)val)->klass(), "Store type incorrect");
-
-  valueArrayOop varray = (valueArrayOop)array;
-  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-  ValueKlass* vklass = vaklass->element_klass();
-  if (!vklass->is_empty_value()) {
-    const int lh = vaklass->layout_helper();
-    vklass->value_store(vklass->data_for_oop((oop)val), varray->value_at_addr(index, lh),
-        vaklass->element_byte_size(), true, false);
-  }
+  ((valueArrayOop)array)->value_copy_to_index((oop)val, index);
 JRT_END
 
 JRT_ENTRY(void, InterpreterRuntime::multianewarray(JavaThread* thread, jint* first_size_address))
--- a/src/hotspot/share/memory/heapInspection.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/memory/heapInspection.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -33,7 +33,7 @@
 #include "memory/universe.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/reflectionAccessorImplKlassHelper.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/os.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 #include "utilities/globalDefinitions.hpp"
--- a/src/hotspot/share/oops/access.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/access.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -57,6 +57,7 @@
 // * atomic_xchg_at: Atomically swap a new value at an internal pointer address if previous value matched the compared value.
 // * arraycopy: Copy data from one heap array to another heap array. The ArrayAccess class has convenience functions for this.
 // * clone: Clone the contents of an object to a newly allocated object.
+// * value_copy: Copy the contents of a value type from one heap address to another
 // * resolve: Resolve a stable to-space invariant oop that is guaranteed not to relocate its payload until a subsequent thread transition.
 //
 // == IMPLEMENTATION ==
@@ -123,6 +124,12 @@
     verify_decorators<expected_mo_decorators | heap_oop_decorators>();
   }
 
+  template <DecoratorSet expected_mo_decorators>
+  static void verify_heap_value_decorators() {
+    const DecoratorSet heap_value_decorators = IN_HEAP | IS_DEST_UNINITIALIZED;
+    verify_decorators<expected_mo_decorators | heap_value_decorators>();
+  }
+
   static const DecoratorSet load_mo_decorators = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | MO_ACQUIRE | MO_SEQ_CST;
   static const DecoratorSet store_mo_decorators = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | MO_RELEASE | MO_SEQ_CST;
   static const DecoratorSet atomic_xchg_mo_decorators = MO_SEQ_CST;
@@ -213,6 +220,14 @@
     AccessInternal::clone<decorators>(src, dst, size);
   }
 
+  // Value type inline heap access (flattened)...
+
+  // Copy value type data from src to dst
+  static inline void value_copy(void* src, void* dst, ValueKlass* md) {
+    verify_heap_value_decorators<IN_HEAP>();
+    AccessInternal::value_copy<decorators>(src, dst, md);
+  }
+
   // Primitive accesses
   template <typename P>
   static inline P load(P* addr) {
--- a/src/hotspot/share/oops/access.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/access.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -199,6 +199,13 @@
   };
 
   template <class GCBarrierType, DecoratorSet decorators>
+  struct PostRuntimeDispatch<GCBarrierType, BARRIER_VALUE_COPY, decorators>: public AllStatic {
+    static void access_barrier(void* src, void* dst, ValueKlass* md) {
+      GCBarrierType::value_copy_in_heap(src, dst, md);
+    }
+  };
+
+  template <class GCBarrierType, DecoratorSet decorators>
   struct PostRuntimeDispatch<GCBarrierType, BARRIER_RESOLVE, decorators>: public AllStatic {
     static oop access_barrier(oop obj) {
       return GCBarrierType::resolve(obj);
@@ -354,6 +361,13 @@
   }
 
   template <DecoratorSet decorators, typename T>
+  void RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::value_copy_init(void* src, void* dst, ValueKlass* md) {
+    func_t function = BarrierResolver<decorators, func_t, BARRIER_VALUE_COPY>::resolve_barrier();
+    _value_copy_func = function;
+    function(src, dst, md);
+  }
+
+  template <DecoratorSet decorators, typename T>
   oop RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::resolve_init(oop obj) {
     func_t function = BarrierResolver<decorators, func_t, BARRIER_RESOLVE>::resolve_barrier();
     _resolve_func = function;
--- a/src/hotspot/share/oops/accessBackend.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/accessBackend.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -26,6 +26,7 @@
 #include "accessBackend.inline.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "oops/oop.inline.hpp"
+#include "oops/valueKlass.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/vm_version.hpp"
 #include "utilities/copy.hpp"
--- a/src/hotspot/share/oops/accessBackend.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/accessBackend.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,6 +52,14 @@
   typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
 };
 
+// This meta-function returns either oop or narrowOop depending on whether
+// a back-end needs to consider compressed oops types or not.
+template <DecoratorSet decorators>
+struct ValueOopType: AllStatic {
+  static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
+  typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
+};
+
 namespace AccessInternal {
   enum BarrierType {
     BARRIER_STORE,
@@ -64,6 +72,7 @@
     BARRIER_ATOMIC_XCHG_AT,
     BARRIER_ARRAYCOPY,
     BARRIER_CLONE,
+    BARRIER_VALUE_COPY,
     BARRIER_RESOLVE
   };
 
@@ -114,6 +123,7 @@
                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
                                      size_t length);
     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
+    typedef void (*value_copy_func_t)(void* src, void* dst, ValueKlass* md);
     typedef oop (*resolve_func_t)(oop obj);
   };
 
@@ -141,6 +151,7 @@
   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
+  ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
 #undef ACCESS_GENERATE_ACCESS_FUNCTION
 
@@ -406,6 +417,8 @@
 
   static void clone(oop src, oop dst, size_t size);
 
+  static void value_copy(void* src, void* dst, ValueKlass* md);
+
   static oop resolve(oop obj) { return obj; }
 };
 
@@ -589,6 +602,18 @@
   };
 
   template <DecoratorSet decorators, typename T>
+  struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
+    typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
+    static func_t _value_copy_func;
+
+    static void value_copy_init(void* src, void* dst, ValueKlass* md);
+
+    static inline void value_copy(void* src, void* dst, ValueKlass* md) {
+      _value_copy_func(src, dst, md);
+    }
+  };
+
+  template <DecoratorSet decorators, typename T>
   struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
     typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
     static func_t _resolve_func;
@@ -642,6 +667,10 @@
   RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
 
   template <DecoratorSet decorators, typename T>
+  typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
+  RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
+
+  template <DecoratorSet decorators, typename T>
   typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type
   RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init;
 
@@ -963,6 +992,23 @@
 
     template <DecoratorSet decorators>
     inline static typename EnableIf<
+      HasDecorator<decorators, AS_RAW>::value>::type
+    value_copy(void* src, void* dst, ValueKlass* md) {
+      typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
+      Raw::value_copy(src, dst, md);
+    }
+
+    template <DecoratorSet decorators>
+    inline static typename EnableIf<
+      !HasDecorator<decorators, AS_RAW>::value>::type
+      value_copy(void* src, void* dst, ValueKlass* md) {
+      const DecoratorSet expanded_decorators = decorators;
+      RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md);
+    }
+
+
+    template <DecoratorSet decorators>
+    inline static typename EnableIf<
       HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
     resolve(oop obj) {
       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
@@ -1268,6 +1314,12 @@
   }
 
   template <DecoratorSet decorators>
+  inline void value_copy(void* src, void* dst, ValueKlass* md) {
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+    PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md);
+  }
+
+  template <DecoratorSet decorators>
   inline oop resolve(oop obj) {
     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
     return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
--- a/src/hotspot/share/oops/accessBackend.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/accessBackend.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -29,6 +29,7 @@
 #include "oops/accessBackend.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oopsHierarchy.hpp"
+#include "oops/valueKlass.hpp"
 
 template <DecoratorSet decorators>
 template <DecoratorSet idecorators, typename T>
@@ -362,4 +363,9 @@
   dst->init_mark_raw();
 }
 
+template <DecoratorSet decorators>
+inline void RawAccessBarrier<decorators>::value_copy(void* src, void* dst, ValueKlass* md) {
+  assert(is_aligned(src, md->get_alignment()) && is_aligned(dst, md->get_alignment()), "Unalign value_copy");
+  AccessInternal::arraycopy_conjoint_atomic(src, dst, static_cast<size_t>(md->get_exact_size_in_bytes()));
+}
 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
--- a/src/hotspot/share/oops/method.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/method.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -50,7 +50,7 @@
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/symbol.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
 #include "prims/nativeLookup.hpp"
--- a/src/hotspot/share/oops/valueArrayKlass.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueArrayKlass.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -248,12 +248,12 @@
            do {
              src -= elem_incr;
              dst -= elem_incr;
-             s_elem_vklass->value_store(src, dst, element_byte_size(), true, false);
+             HeapAccess<>::value_copy(src, dst, s_elem_vklass);
            } while (src > src_end);
          } else {
            address src_end = src + (length << log2_element_size());
            while (src < src_end) {
-             s_elem_vklass->value_store(src, dst, element_byte_size(), true, false);
+             HeapAccess<>::value_copy(src, dst, s_elem_vklass);
              src += elem_incr;
              dst += elem_incr;
            }
@@ -271,9 +271,7 @@
        valueArrayHandle sh(THREAD, sa);
        int dst_end = dst_pos + length;
        while (dst_pos < dst_end) {
-         oop o = s_elem_vklass->allocate_instance(CHECK);
-         s_elem_vklass->value_store(sh->value_at_addr(src_pos, layout_helper()),
-                                             s_elem_vklass->data_for_oop(o), true, true);
+         oop o = valueArrayOopDesc::value_alloc_copy_from_index(sh, src_pos, CHECK);
          dh->obj_at_put(dst_pos, o);
          dst_pos++;
          src_pos++;
@@ -298,7 +296,7 @@
        if (se->klass() != d_elem_klass) {
          THROW(vmSymbols::java_lang_ArrayStoreException());
        }
-       d_elem_vklass->value_store(d_elem_vklass->data_for_oop(se), dst, true, false);
+       d_elem_vklass->value_copy_oop_to_payload(se, dst);
        dst += delem_incr;
        src_pos++;
      }
--- a/src/hotspot/share/oops/valueArrayOop.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueArrayOop.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -36,6 +36,12 @@
  public:
   void*  base() const;
   void* value_at_addr(int index, jint lh) const;
+
+  // Return a buffered element from index
+  static oop value_alloc_copy_from_index(valueArrayHandle vah, int index, TRAPS);
+  void value_copy_from_index(int index, oop dst) const;
+  void value_copy_to_index(oop src, int index) const;
+
   // Sizing
   static size_t element_size(int lh, int nof_elements) {
     return nof_elements << Klass::layout_helper_log2_element_size(lh);
--- a/src/hotspot/share/oops/valueArrayOop.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueArrayOop.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -28,6 +28,7 @@
 #include "oops/access.inline.hpp"
 #include "oops/arrayOop.inline.hpp"
 #include "oops/valueArrayOop.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/globals.hpp"
 
@@ -45,6 +46,39 @@
   return object_size(klass()->layout_helper(), length());
 }
 
+inline oop valueArrayOopDesc::value_alloc_copy_from_index(valueArrayHandle vah, int index, TRAPS) {
+  ValueArrayKlass* vaklass = ValueArrayKlass::cast(vah->klass());
+  ValueKlass* vklass = vaklass->element_klass();
+  if (vklass->is_empty_value()) {
+    return vklass->default_value();
+  } else {
+    oop buf = vklass->allocate_instance(CHECK_NULL);
+    vklass->value_copy_payload_to_new_oop(vah->value_at_addr(index, vaklass->layout_helper()) ,buf);
+    return buf;
+  }
+}
+
+inline void valueArrayOopDesc::value_copy_from_index(int index, oop dst) const {
+  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass());
+  ValueKlass* vklass = vaklass->element_klass();
+  if (vklass->is_empty_value()) {
+    return; // Assumes dst was a new and clean buffer (OptoRuntime::load_unknown_value())
+  } else {
+    void* src = value_at_addr(index, vaklass->layout_helper());
+    return vklass->value_copy_payload_to_new_oop(src ,dst);
+  }
+}
+
+inline void valueArrayOopDesc::value_copy_to_index(oop src, int index) const {
+  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass());
+  ValueKlass* vklass = vaklass->element_klass();
+  if (vklass->is_empty_value()) {
+    return;
+  }
+  void* dst = value_at_addr(index, vaklass->layout_helper());
+  vklass->value_copy_oop_to_payload(src, dst);
+}
+
 
 
 #endif // SHARE_VM_OOPS_VALUEARRAYOOP_INLINE_HPP
--- a/src/hotspot/share/oops/valueKlass.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueKlass.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -36,7 +36,7 @@
 #include "oops/method.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/objArrayKlass.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "oops/valueArrayKlass.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 #include "runtime/handles.inline.hpp"
@@ -46,6 +46,29 @@
 #include "runtime/thread.inline.hpp"
 #include "utilities/copy.hpp"
 
+  // Constructor
+ValueKlass::ValueKlass(const ClassFileParser& parser)
+    : InstanceKlass(parser, InstanceKlass::_misc_kind_value_type, InstanceKlass::ID) {
+  _adr_valueklass_fixed_block = valueklass_static_block();
+  // Addresses used for value type calling convention
+  *((Array<SigEntry>**)adr_extended_sig()) = NULL;
+  *((Array<VMRegPair>**)adr_return_regs()) = NULL;
+  *((address*)adr_pack_handler()) = NULL;
+  *((address*)adr_unpack_handler()) = NULL;
+  assert(pack_handler() == NULL, "pack handler not null");
+  *((int*)adr_default_value_offset()) = 0;
+  *((Klass**)adr_value_array_klass()) = NULL;
+  set_prototype_header(markWord::always_locked_prototype());
+}
+
+oop ValueKlass::default_value() {
+  oop val = java_mirror()->obj_field_acquire(default_value_offset());
+  assert(oopDesc::is_oop(val), "Sanity check");
+  assert(val->is_value(), "Sanity check");
+  assert(val->klass() == this, "sanity check");
+  return val;
+}
+
 int ValueKlass::first_field_offset_old() {
 #ifdef ASSERT
   int first_offset = INT_MAX;
@@ -191,124 +214,6 @@
     ArrayKlass::cast(get_value_array_klass())->array_klasses_do(f);
 }
 
-void ValueKlass::raw_field_copy(void* src, void* dst, size_t raw_byte_size) {
-  if (!UseNewLayout) {
-    /*
-     * Try not to shear fields even if not an atomic store...
-     *
-     * First 3 cases handle value array store, otherwise works on the same basis
-     * as JVM_Clone, at this size data is aligned. The order of primitive types
-     * is largest to smallest, and it not possible for fields to stradle long
-     * copy boundaries.
-     *
-     * If MT without exclusive access, possible to observe partial value store,
-     * but not partial primitive and reference field values
-     */
-    switch (raw_byte_size) {
-    case 1:
-      *((jbyte*) dst) = *(jbyte*)src;
-      break;
-    case 2:
-      *((jshort*) dst) = *(jshort*)src;
-      break;
-    case 4:
-      *((jint*) dst) = *(jint*) src;
-      break;
-    default:
-      assert(raw_byte_size % sizeof(jlong) == 0, "Unaligned raw_byte_size");
-      Copy::conjoint_jlongs_atomic((jlong*)src, (jlong*)dst, raw_byte_size >> LogBytesPerLong);
-    }
-  } else {
-    int size = this->get_exact_size_in_bytes();
-    int length;
-    switch (this->get_alignment()) {
-    case BytesPerLong:
-      length = size >> LogBytesPerLong;
-      if (length > 0) {
-        Copy::conjoint_jlongs_atomic((jlong*)src, (jlong*)dst, length);
-        size -= length << LogBytesPerLong;
-        src = (jlong*)src + length;
-        dst = (jlong*)dst + length;
-      }
-      // Fallthrough
-    case BytesPerInt:
-      length = size >> LogBytesPerInt;
-      if (length > 0) {
-        Copy::conjoint_jints_atomic((jint*)src, (jint*)dst, length);
-        size -= length << LogBytesPerInt;
-        src = (jint*)src + length;
-        dst = (jint*)dst + length;
-      }
-      // Fallthrough
-    case BytesPerShort:
-      length = size >> LogBytesPerShort;
-      if (length > 0) {
-        Copy::conjoint_jshorts_atomic((jshort*)src, (jshort*)dst, length);
-        size -= length << LogBytesPerShort;
-        src = (jshort*)src + length;
-        dst = (jshort*)dst +length;
-      }
-      // Fallthrough
-    case 1:
-      if (size > 0) Copy::conjoint_jbytes_atomic((jbyte*)src, (jbyte*)dst, size);
-      break;
-    default:
-      fatal("Unsupported alignment");
-    }
-  }
-}
-
-/*
- * Store the value of this klass contained with src into dst.
- *
- * This operation is appropriate for use from vastore, vaload and putfield (for values)
- *
- * GC barriers currently can lock with no safepoint check and allocate c-heap,
- * so raw point is "safe" for now.
- *
- * Going forward, look to use machine generated (stub gen or bc) version for most used klass layouts
- *
- */
-void ValueKlass::value_store(void* src, void* dst, size_t raw_byte_size, bool dst_heap, bool dst_uninitialized) {
-  if (contains_oops()) {
-    if (dst_heap) {
-      // src/dst aren't oops, need offset to adjust oop map offset
-      const address dst_oop_addr = ((address) dst) - first_field_offset();
-
-      ModRefBarrierSet* bs = barrier_set_cast<ModRefBarrierSet>(BarrierSet::barrier_set());
-
-      // Pre-barriers...
-      OopMapBlock* map = start_of_nonstatic_oop_maps();
-      OopMapBlock* const end = map + nonstatic_oop_map_count();
-      while (map != end) {
-        // Shame we can't just use the existing oop iterator...src/dst aren't oop
-        address doop_address = dst_oop_addr + map->offset();
-        // TEMP HACK: barrier code need to migrate to => access API (need own versions of value type ops)
-        if (UseCompressedOops) {
-          bs->write_ref_array_pre((narrowOop*) doop_address, map->count(), dst_uninitialized);
-        } else {
-          bs->write_ref_array_pre((oop*) doop_address, map->count(), dst_uninitialized);
-        }
-        map++;
-      }
-
-      raw_field_copy(src, dst, raw_byte_size);
-
-      // Post-barriers...
-      map = start_of_nonstatic_oop_maps();
-      while (map != end) {
-        address doop_address = dst_oop_addr + map->offset();
-        bs->write_ref_array((HeapWord*) doop_address, map->count());
-        map++;
-      }
-    } else { // Buffered value case
-      raw_field_copy(src, dst, raw_byte_size);
-    }
-  } else {   // Primitive-only case...
-    raw_field_copy(src, dst, raw_byte_size);
-  }
-}
-
 // Value type arguments are not passed by reference, instead each
 // field of the value type is passed as an argument. This helper
 // function collects the fields of the value types (including embedded
--- a/src/hotspot/share/oops/valueKlass.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueKlass.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -28,7 +28,7 @@
 #include "classfile/javaClasses.hpp"
 #include "oops/instanceKlass.hpp"
 #include "oops/method.hpp"
-#include "oops/oop.inline.hpp"
+//#include "oops/oop.inline.hpp"
 
 // A ValueKlass is a specialized InstanceKlass for value types.
 
@@ -40,19 +40,7 @@
  private:
 
   // Constructor
-  ValueKlass(const ClassFileParser& parser)
-    : InstanceKlass(parser, InstanceKlass::_misc_kind_value_type, InstanceKlass::ID) {
-    _adr_valueklass_fixed_block = valueklass_static_block();
-    // Addresses used for value type calling convention
-    *((Array<SigEntry>**)adr_extended_sig()) = NULL;
-    *((Array<VMRegPair>**)adr_return_regs()) = NULL;
-    *((address*)adr_pack_handler()) = NULL;
-    *((address*)adr_unpack_handler()) = NULL;
-    assert(pack_handler() == NULL, "pack handler not null");
-    *((int*)adr_default_value_offset()) = 0;
-    *((Klass**)adr_value_array_klass()) = NULL;
-    set_prototype_header(markWord::always_locked_prototype());
-  }
+  ValueKlass(const ClassFileParser& parser);
 
   ValueKlassFixedBlock* valueklass_static_block() const {
     address adr_jf = adr_value_fields_klasses();
@@ -153,7 +141,7 @@
     *(int*)adr_first_field_offset() = offset;
   }
 
-  int get_exact_size_in_bytes() {
+  int get_exact_size_in_bytes() const {
     return *(int*)adr_exact_size_in_bytes();
   }
 
@@ -188,10 +176,7 @@
   oop indirect_mirror() const { return java_lang_Class::indirect_type_mirror(java_mirror()); }
 
   // Casting from Klass*
-  static ValueKlass* cast(Klass* k) {
-    assert(k->is_value(), "cast to ValueKlass");
-    return (ValueKlass*) k;
-  }
+  static ValueKlass* cast(Klass* k);
 
   // Use this to return the size of an instance in heap words
   // Implementation is currently simple because all value types are allocated
@@ -209,15 +194,8 @@
   // minimum number of bytes occupied by nonstatic fields, HeapWord aligned or pow2
   int raw_value_byte_size();
 
-  address data_for_oop(oop o) const {
-    return ((address) (void*) o) + first_field_offset();
-  }
-
-  oop oop_for_data(address data) const {
-    oop o = (oop) (data - first_field_offset());
-    assert(oopDesc::is_oop(o, false), "Not an oop");
-    return o;
-  }
+  address data_for_oop(oop o) const;
+  oop oop_for_data(address data) const;
 
   // Query if h/w provides atomic load/store
   bool is_atomic();
@@ -227,17 +205,18 @@
   bool contains_oops() const { return nonstatic_oop_map_count() > 0; }
   int nonstatic_oop_count();
 
-  // Prototype general store methods...
+  // General store methods
+  //
+  // Normally loads and store methods would be found in *Oops classes, but since values can be
+  // "in-lined" (flattened) into containing oops, these methods reside here in ValueKlass.
+  //
+  // "value_copy_*_to_new_*" assume new memory (i.e. IS_DEST_UNINITIALIZED for write barriers)
 
-  // copy the fields, with no concern for GC barriers
-  void raw_field_copy(void* src, void* dst, size_t raw_byte_size);
+  void value_copy_payload_to_new_oop(void* src, oop dst);
+  void value_copy_oop_to_new_oop(oop src, oop dst);
+  void value_copy_oop_to_new_payload(oop src, void* dst);
 
-  void value_store(void* src, void* dst, bool dst_is_heap, bool dst_uninitialized) {
-    value_store(src, dst, nonstatic_field_size() << LogBytesPerHeapOop, dst_is_heap, dst_uninitialized);
-  }
-
-  // store the value of this klass contained with src into dst, raw data ptr
-  void value_store(void* src, void* dst, size_t raw_byte_size, bool dst_is_heap, bool dst_uninitialized);
+  void value_copy_oop_to_payload(oop src, void* dst);
 
   // oop iterate raw value type data pointer (where oop_addr may not be an oop, but backing/array-element)
   template <typename T, class OopClosureType>
@@ -298,14 +277,7 @@
     indirect_mirror()->obj_field_put(default_value_offset(), val);
   }
 
-  oop default_value() {
-    oop val = java_mirror()->obj_field_acquire(default_value_offset());
-    assert(oopDesc::is_oop(val), "Sanity check");
-    assert(val->is_value(), "Sanity check");
-    assert(val->klass() == this, "sanity check");
-    return val;
-  }
-
+  oop default_value();
   void deallocate_contents(ClassLoaderData* loader_data);
   static void cleanup(ValueKlass* ik) ;
 
--- a/src/hotspot/share/oops/valueKlass.inline.hpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/oops/valueKlass.inline.hpp	Wed Oct 02 11:51:57 2019 +0200
@@ -31,6 +31,38 @@
 #include "oops/valueKlass.hpp"
 #include "utilities/macros.hpp"
 
+inline ValueKlass* ValueKlass::cast(Klass* k) {
+  assert(k->is_value(), "cast to ValueKlass");
+  return (ValueKlass*) k;
+}
+
+inline address ValueKlass::data_for_oop(oop o) const {
+  return ((address) (void*) o) + first_field_offset();
+}
+
+inline oop ValueKlass::oop_for_data(address data) const {
+  oop o = (oop) (data - first_field_offset());
+  assert(oopDesc::is_oop(o, false), "Not an oop");
+  return o;
+}
+
+inline void ValueKlass::value_copy_payload_to_new_oop(void* src, oop dst) {
+  HeapAccess<IS_DEST_UNINITIALIZED>::value_copy(src, data_for_oop(dst), this);
+}
+
+inline void ValueKlass::value_copy_oop_to_new_oop(oop src, oop dst) {
+  HeapAccess<IS_DEST_UNINITIALIZED>::value_copy(data_for_oop(src), data_for_oop(dst), this);
+}
+
+inline void ValueKlass::value_copy_oop_to_new_payload(oop src, void* dst) {
+  HeapAccess<IS_DEST_UNINITIALIZED>::value_copy(data_for_oop(src), dst, this);
+}
+
+inline void ValueKlass::value_copy_oop_to_payload(oop src, void* dst) {
+  HeapAccess<>::value_copy(data_for_oop(src), dst, this);
+}
+
+
 template <typename T, class OopClosureType>
 void ValueKlass::oop_iterate_specialized(const address oop_addr, OopClosureType* closure) {
   OopMapBlock* map = start_of_nonstatic_oop_maps();
--- a/src/hotspot/share/opto/runtime.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/opto/runtime.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -1744,14 +1744,7 @@
 
 JRT_LEAF(void, OptoRuntime::load_unknown_value(valueArrayOopDesc* array, int index, instanceOopDesc* buffer))
 {
-  Klass* klass = array->klass();
-  assert(klass->is_valueArray_klass(), "expected value array oop");
-
-  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-  ValueKlass* vklass = vaklass->element_klass();
-  void* src = array->value_at_addr(index, vaklass->layout_helper());
-  vklass->value_store(src, vklass->data_for_oop(buffer),
-                        vaklass->element_byte_size(), true, false);
+  array->value_copy_from_index(index, buffer);
 }
 JRT_END
 
@@ -1777,15 +1770,7 @@
 JRT_LEAF(void, OptoRuntime::store_unknown_value(instanceOopDesc* buffer, valueArrayOopDesc* array, int index))
 {
   assert(buffer != NULL, "can't store null into flat array");
-  Klass* klass = array->klass();
-  assert(klass->is_valueArray_klass(), "expected value array");
-  assert(ArrayKlass::cast(klass)->element_klass() == buffer->klass(), "Store type incorrect");
-
-  ValueArrayKlass* vaklass = ValueArrayKlass::cast(klass);
-  ValueKlass* vklass = vaklass->element_klass();
-  const int lh = vaklass->layout_helper();
-  vklass->value_store(vklass->data_for_oop(buffer), array->value_at_addr(index, lh),
-                      vaklass->element_byte_size(), true, false);
+  array->value_copy_to_index(buffer, index);
 }
 JRT_END
 
--- a/src/hotspot/share/prims/jni.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/prims/jni.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -58,7 +58,7 @@
 #include "oops/symbol.hpp"
 #include "oops/typeArrayKlass.hpp"
 #include "oops/typeArrayOop.inline.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/jniCheck.hpp"
 #include "prims/jniExport.hpp"
 #include "prims/jniFastGetField.hpp"
--- a/src/hotspot/share/prims/unsafe.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/prims/unsafe.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -39,8 +39,8 @@
 #include "oops/oop.inline.hpp"
 #include "oops/typeArrayOop.inline.hpp"
 #include "oops/valueArrayKlass.hpp"
-#include "oops/valueArrayOop.hpp"
 #include "oops/valueArrayOop.inline.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/unsafe.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
@@ -368,9 +368,7 @@
   Handle base_h(THREAD, base);
   oop v = vk->allocate_instance(CHECK_NULL); // allocate instance
   vk->initialize(CHECK_NULL); // If field is a default value, value class might not be initialized yet
-  vk->value_store(((address)(oopDesc*)base_h()) + offset,
-                  vk->data_for_oop(v),
-                  true, true);
+  vk->value_copy_payload_to_new_oop(((address)(oopDesc*)base_h()) + offset, v);
   return JNIHandles::make_local(env, v);
 } UNSAFE_END
 
@@ -381,8 +379,7 @@
   assert(!base->is_value() || base->mark().is_larval_state(), "must be an object instance or a larval value");
   assert_and_log_unsafe_value_access(base, offset, vk);
   oop v = JNIHandles::resolve(value);
-  vk->value_store(vk->data_for_oop(v),
-                 ((address)(oopDesc*)base) + offset, true, true);
+  vk->value_copy_oop_to_payload(v, ((address)(oopDesc*)base) + offset);
 } UNSAFE_END
 
 UNSAFE_ENTRY(jobject, Unsafe_MakePrivateBuffer(JNIEnv *env, jobject unsafe, jobject value)) {
@@ -391,7 +388,7 @@
   Handle vh(THREAD, v);
   ValueKlass* vk = ValueKlass::cast(v->klass());
   instanceOop new_value = vk->allocate_instance(CHECK_NULL);
-  vk->value_store(vk->data_for_oop(vh()), vk->data_for_oop(new_value), true, false);
+  vk->value_copy_oop_to_new_oop(vh(),  new_value);
   markWord mark = new_value->mark();
   new_value->set_mark(mark.enter_larval_state());
   return JNIHandles::make_local(env, new_value);
--- a/src/hotspot/share/runtime/deoptimization.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/runtime/deoptimization.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -47,7 +47,7 @@
 #include "oops/typeArrayOop.inline.hpp"
 #include "oops/valueArrayKlass.hpp"
 #include "oops/valueArrayOop.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "oops/verifyOopClosure.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/biasedLocking.hpp"
--- a/src/hotspot/share/runtime/reflection.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/runtime/reflection.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -40,7 +40,7 @@
 #include "oops/objArrayKlass.hpp"
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "oops/typeArrayOop.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "runtime/arguments.hpp"
--- a/src/hotspot/share/runtime/sharedRuntime.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/runtime/sharedRuntime.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -54,7 +54,7 @@
 #include "oops/objArrayKlass.hpp"
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "prims/forte.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
--- a/src/hotspot/share/runtime/signature.cpp	Fri Sep 27 15:02:28 2019 +0200
+++ b/src/hotspot/share/runtime/signature.cpp	Wed Oct 02 11:51:57 2019 +0200
@@ -32,7 +32,7 @@
 #include "oops/oop.inline.hpp"
 #include "oops/symbol.hpp"
 #include "oops/typeArrayKlass.hpp"
-#include "oops/valueKlass.hpp"
+#include "oops/valueKlass.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/signature.hpp"
 
--- a/test/hotspot/jtreg/runtime/valhalla/valuetypes/ValueOops.java	Fri Sep 27 15:02:28 2019 +0200
+++ b/test/hotspot/jtreg/runtime/valhalla/valuetypes/ValueOops.java	Wed Oct 02 11:51:57 2019 +0200
@@ -34,9 +34,8 @@
 import jdk.experimental.value.MethodHandleBuilder;
 
 /**
- * @test ValueOops
- * @requires vm.gc == null
- * @requires vm.opt.ExplicitGCInvokesConcurrent != true
+ * @test ValueOops_int_Serial
+ * @requires vm.gc.Serial
  * @summary Test embedding oops into Value types
  * @modules java.base/jdk.experimental.bytecode
  *          java.base/jdk.experimental.value
@@ -48,24 +47,107 @@
  * @run main/othervm -Xint -XX:+UseSerialGC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops
+ */
+
+/**
+ * @test ValueOops_int_G1
+ * @requires vm.gc.G1
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xint  -XX:+UseG1GC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops 100
+ */
+
+/**
+ * @test ValueOops_int_Parallel
+ * @requires vm.gc.Parallel
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xint -XX:+UseParallelGC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops
+ */
+
+/**
+ * @test ValueOops_int_Z
+ * @requires vm.gc.Z
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
+ * @run main/othervm -Xint -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xmx128m
+ *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
+ *                   runtime.valhalla.valuetypes.ValueOops
+ */
+
+/**
+ * @test ValueOops_comp_serial
+ * @requires vm.gc.Serial
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xcomp -XX:+UseSerialGC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops
+ */
+
+/**
+ * @test ValueOops_comp_G1
+ * @requires vm.gc.G1
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xcomp -XX:+UseG1GC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops 100
+ */
+
+/**
+ * @test ValueOops_comp_Parallel
+ * @requires vm.gc.Parallel
+ * @summary Test embedding oops into Value types
+ * @modules java.base/jdk.experimental.bytecode
+ *          java.base/jdk.experimental.value
+ * @library /test/lib
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator Person.java
+ * @compile -XDemitQtypes -XDenableValueTypes -XDallowWithFieldOperator ValueOops.java
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                   sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xcomp -XX:+UseParallelGC -Xmx128m
  *                   -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
  *                   runtime.valhalla.valuetypes.ValueOops
  */
 public class ValueOops {
 
+    // Xcomp with ZGC missing until C1 and C2 barrier code is in place (JDK-8231498)
+
     // Extra debug: -XX:+VerifyOops -XX:+VerifyStack -XX:+VerifyLastFrame -XX:+VerifyBeforeGC -XX:+VerifyAfterGC -XX:+VerifyDuringGC -XX:VerifySubSet=threads,heap
     // Even more debugging: -XX:+TraceNewOopMapGeneration -Xlog:gc*=info