changeset 58904:3b998574be4b

8234338: ZGC: Improve small heap usage Reviewed-by: eosterlund, stefank
author pliden
date Wed, 20 Nov 2019 10:37:46 +0100
parents 877c000fd688
children 82b2ba888190
files src/hotspot/share/gc/z/zGlobals.cpp src/hotspot/share/gc/z/zGlobals.hpp src/hotspot/share/gc/z/zHeuristics.cpp src/hotspot/share/gc/z/zHeuristics.hpp src/hotspot/share/gc/z/zInitialize.cpp src/hotspot/share/gc/z/zObjectAllocator.cpp src/hotspot/share/gc/z/zObjectAllocator.hpp src/hotspot/share/gc/z/zPage.inline.hpp src/hotspot/share/gc/z/zRelocationSetSelector.cpp src/hotspot/share/gc/z/zWorkers.cpp test/hotspot/jtreg/gc/z/TestSmallHeap.java
diffstat 11 files changed, 222 insertions(+), 50 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/share/gc/z/zGlobals.cpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zGlobals.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -24,27 +24,35 @@
 #include "precompiled.hpp"
 #include "gc/z/zGlobals.hpp"
 
-uint32_t ZGlobalPhase                 = ZPhaseRelocate;
-uint32_t ZGlobalSeqNum                = 1;
+uint32_t   ZGlobalPhase                = ZPhaseRelocate;
+uint32_t   ZGlobalSeqNum               = 1;
 
-const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes;
-const int& ZObjectAlignmentSmall      = MinObjAlignmentInBytes;
+size_t     ZPageSizeMediumShift;
+size_t     ZPageSizeMedium;
 
-uintptr_t ZAddressGoodMask;
-uintptr_t ZAddressBadMask;
-uintptr_t ZAddressWeakBadMask;
+size_t     ZObjectSizeLimitMedium;
 
-uintptr_t ZAddressBase;
+const int& ZObjectAlignmentSmallShift  = LogMinObjAlignmentInBytes;
+int        ZObjectAlignmentMediumShift;
 
-size_t    ZAddressOffsetBits;
-uintptr_t ZAddressOffsetMask;
-size_t    ZAddressOffsetMax;
+const int& ZObjectAlignmentSmall       = MinObjAlignmentInBytes;
+int        ZObjectAlignmentMedium;
 
-size_t    ZAddressMetadataShift;
-uintptr_t ZAddressMetadataMask;
+uintptr_t  ZAddressGoodMask;
+uintptr_t  ZAddressBadMask;
+uintptr_t  ZAddressWeakBadMask;
 
-uintptr_t ZAddressMetadataMarked;
-uintptr_t ZAddressMetadataMarked0;
-uintptr_t ZAddressMetadataMarked1;
-uintptr_t ZAddressMetadataRemapped;
-uintptr_t ZAddressMetadataFinalizable;
+uintptr_t  ZAddressBase;
+
+size_t     ZAddressOffsetBits;
+uintptr_t  ZAddressOffsetMask;
+size_t     ZAddressOffsetMax;
+
+size_t     ZAddressMetadataShift;
+uintptr_t  ZAddressMetadataMask;
+
+uintptr_t  ZAddressMetadataMarked;
+uintptr_t  ZAddressMetadataMarked0;
+uintptr_t  ZAddressMetadataMarked1;
+uintptr_t  ZAddressMetadataRemapped;
+uintptr_t  ZAddressMetadataFinalizable;
--- a/src/hotspot/share/gc/z/zGlobals.hpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zGlobals.hpp	Wed Nov 20 10:37:46 2019 +0100
@@ -50,17 +50,6 @@
 // Virtual memory to physical memory ratio
 const size_t      ZVirtualToPhysicalRatio       = 16; // 16:1
 
-//
-// Page Tiers (assuming ZGranuleSize=2M)
-// -------------------------------------
-//
-//                 Page Size        Object Size      Object Alignment
-//                 --------------------------------------------------
-//  Small          2M               <= 265K          MinObjAlignmentInBytes
-//  Medium         32M              <= 4M            4K
-//  Large          N x 2M           > 4M             2M
-//
-
 // Page types
 const uint8_t     ZPageTypeSmall                = 0;
 const uint8_t     ZPageTypeMedium               = 1;
@@ -68,24 +57,24 @@
 
 // Page size shifts
 const size_t      ZPageSizeSmallShift           = ZGranuleSizeShift;
-const size_t      ZPageSizeMediumShift          = ZPageSizeSmallShift + 4;
+extern size_t     ZPageSizeMediumShift;
 
 // Page sizes
 const size_t      ZPageSizeSmall                = (size_t)1 << ZPageSizeSmallShift;
-const size_t      ZPageSizeMedium               = (size_t)1 << ZPageSizeMediumShift;
+extern size_t     ZPageSizeMedium;
 
 // Object size limits
-const size_t      ZObjectSizeLimitSmall         = (ZPageSizeSmall / 8);  // Allow 12.5% waste
-const size_t      ZObjectSizeLimitMedium        = (ZPageSizeMedium / 8); // Allow 12.5% waste
+const size_t      ZObjectSizeLimitSmall         = ZPageSizeSmall / 8; // 12.5% max waste
+extern size_t     ZObjectSizeLimitMedium;
 
 // Object alignment shifts
 extern const int& ZObjectAlignmentSmallShift;
-const int         ZObjectAlignmentMediumShift   = ZPageSizeMediumShift - 13; // 8192 objects per page
-const int         ZObjectAlignmentLargeShift    = ZPageSizeSmallShift;
+extern int        ZObjectAlignmentMediumShift;
+const int         ZObjectAlignmentLargeShift    = ZGranuleSizeShift;
 
 // Object alignments
 extern const int& ZObjectAlignmentSmall;
-const int         ZObjectAlignmentMedium        = 1 << ZObjectAlignmentMediumShift;
+extern int        ZObjectAlignmentMedium;
 const int         ZObjectAlignmentLarge         = 1 << ZObjectAlignmentLargeShift;
 
 //
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zHeuristics.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zCPU.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zHeuristics.hpp"
+#include "gc/z/zUtils.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+
+void ZHeuristics::set_medium_page_size() {
+  // Set ZPageSizeMedium so that a medium page occupies at most 3.125% of the
+  // max heap size. ZPageSizeMedium is initially set to 0, which means medium
+  // pages are effectively disabled. It is adjusted only if ZPageSizeMedium
+  // becomes larger than ZPageSizeSmall.
+  const size_t min = ZGranuleSize;
+  const size_t max = ZGranuleSize * 16;
+  const size_t unclamped = MaxHeapSize * 0.03125;
+  const size_t clamped = MIN2(MAX2(min, unclamped), max);
+  const size_t size = ZUtils::round_down_power_of_2(clamped);
+
+  if (size > ZPageSizeSmall) {
+    // Enable medium pages
+    ZPageSizeMedium             = size;
+    ZPageSizeMediumShift        = log2_intptr(ZPageSizeMedium);
+    ZObjectSizeLimitMedium      = ZPageSizeMedium / 8;
+    ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13;
+    ZObjectAlignmentMedium      = 1 << ZObjectAlignmentMediumShift;
+
+    log_info(gc, init)("Medium Page Size: " SIZE_FORMAT "M", ZPageSizeMedium / M);
+  } else {
+    log_info(gc, init)("Medium Page Size: N/A");
+  }
+}
+
+bool ZHeuristics::use_per_cpu_shared_small_pages() {
+  // Use per-CPU shared small pages only if these pages occupy at most 3.125%
+  // of the max heap size. Otherwise fall back to using a single shared small
+  // page. This is useful when using small heaps on large machines.
+  const size_t per_cpu_share = (MaxHeapSize * 0.03125) / ZCPU::count();
+  return per_cpu_share >= ZPageSizeSmall;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zHeuristics.hpp	Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZHEURISTICS_HPP
+#define SHARE_GC_Z_ZHEURISTICS_HPP
+
+#include "memory/allocation.hpp"
+
+class ZHeuristics : public AllStatic {
+public:
+  static void set_medium_page_size();
+
+  static bool use_per_cpu_shared_small_pages();
+};
+
+#endif // SHARE_GC_Z_ZHEURISTICS_HPP
--- a/src/hotspot/share/gc/z/zInitialize.cpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zInitialize.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -26,6 +26,7 @@
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zCPU.hpp"
 #include "gc/z/zGlobals.hpp"
+#include "gc/z/zHeuristics.hpp"
 #include "gc/z/zInitialize.hpp"
 #include "gc/z/zLargePages.hpp"
 #include "gc/z/zNUMA.hpp"
@@ -49,6 +50,7 @@
   ZThreadLocalAllocBuffer::initialize();
   ZTracer::initialize();
   ZLargePages::initialize();
+  ZHeuristics::set_medium_page_size();
   ZBarrierSet::set_barrier_set(barrier_set);
 
   initialize_os();
--- a/src/hotspot/share/gc/z/zObjectAllocator.cpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zHeuristics.hpp"
 #include "gc/z/zObjectAllocator.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zStat.hpp"
@@ -43,12 +44,21 @@
 static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);
 
 ZObjectAllocator::ZObjectAllocator() :
+    _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
     _used(0),
     _undone(0),
     _shared_medium_page(NULL),
     _shared_small_page(NULL),
     _worker_small_page(NULL) {}
 
+ZPage** ZObjectAllocator::shared_small_page_addr() {
+  return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
+ZPage* const* ZObjectAllocator::shared_small_page_addr() const {
+  return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
 ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
   ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
   if (page != NULL) {
@@ -72,7 +82,7 @@
                                                         size_t size,
                                                         ZAllocationFlags flags) {
   uintptr_t addr = 0;
-  ZPage* page = *shared_page;
+  ZPage* page = OrderAccess::load_acquire(shared_page);
 
   if (page != NULL) {
     addr = page->alloc_object_atomic(size);
@@ -142,7 +152,7 @@
   // Non-worker small page allocation can never use the reserve
   flags.set_no_reserve();
 
-  return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
+  return alloc_object_in_shared_page(shared_small_page_addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
 }
 
 uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) {
@@ -294,7 +304,7 @@
 size_t ZObjectAllocator::remaining() const {
   assert(ZThread::is_java(), "Should be a Java thread");
 
-  ZPage* page = _shared_small_page.get();
+  const ZPage* const page = OrderAccess::load_acquire(shared_small_page_addr());
   if (page != NULL) {
     return page->remaining();
   }
--- a/src/hotspot/share/gc/z/zObjectAllocator.hpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp	Wed Nov 20 10:37:46 2019 +0100
@@ -31,12 +31,16 @@
 
 class ZObjectAllocator {
 private:
+  const bool         _use_per_cpu_shared_small_pages;
   ZPerCPU<size_t>    _used;
   ZPerCPU<size_t>    _undone;
   ZContended<ZPage*> _shared_medium_page;
   ZPerCPU<ZPage*>    _shared_small_page;
   ZPerWorker<ZPage*> _worker_small_page;
 
+  ZPage** shared_small_page_addr();
+  ZPage* const* shared_small_page_addr() const;
+
   ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
   void undo_alloc_page(ZPage* page);
 
--- a/src/hotspot/share/gc/z/zPage.inline.hpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zPage.inline.hpp	Wed Nov 20 10:37:46 2019 +0100
@@ -39,14 +39,11 @@
 #include "utilities/debug.hpp"
 
 inline uint8_t ZPage::type_from_size(size_t size) const {
-  switch (size) {
-  case ZPageSizeSmall:
+  if (size == ZPageSizeSmall) {
     return ZPageTypeSmall;
-
-  case ZPageSizeMedium:
+  } else if (size == ZPageSizeMedium) {
     return ZPageTypeMedium;
-
-  default:
+  } else {
     return ZPageTypeLarge;
   }
 }
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -97,6 +97,11 @@
 }
 
 void ZRelocationSetSelectorGroup::select() {
+  if (_page_size == 0) {
+    // Page type disabled
+    return;
+  }
+
   // Calculate the number of pages to relocate by successively including pages in
   // a candidate relocation set and calculate the maximum space requirement for
   // their live objects.
--- a/src/hotspot/share/gc/z/zWorkers.cpp	Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zWorkers.cpp	Wed Nov 20 10:37:46 2019 +0100
@@ -35,15 +35,16 @@
 }
 
 static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) {
-  const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall;
+  const int nworkers = (MaxHeapSize * (reserve_share_in_percent / 100.0)) / ZPageSizeSmall;
   return MAX2(nworkers, 1);
 }
 
 static uint calculate_nworkers(double cpu_share_in_percent) {
-  // Cap number of workers so that we never use more than 10% of the max heap
-  // for the reserve. This is useful when using small heaps on large machines.
+  // Cap number of workers so that we don't use more than 2% of the max heap
+  // for the small page reserve. This is useful when using small heaps on
+  // large machines.
   return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent),
-              calculate_nworkers_based_on_heap_size(10.0));
+              calculate_nworkers_based_on_heap_size(2.0));
 }
 
 uint ZWorkers::calculate_nparallel() {
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/gc/z/TestSmallHeap.java	Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.z;
+
+/*
+ * @test TestSmallHeap
+ * @requires vm.gc.Z & !vm.graal.enabled
+ * @summary Test ZGC with small heaps
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx8M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx16M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx32M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx64M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx128M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx256M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx512M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx1024M gc.z.TestSmallHeap
+ */
+
+import java.lang.ref.Reference;
+
+public class TestSmallHeap {
+    public static void main(String[] args) throws Exception {
+        final long maxCapacity = Runtime.getRuntime().maxMemory();
+        System.out.println("Max Capacity " + maxCapacity + " bytes");
+
+        // Allocate byte arrays of increasing length, so that
+        // all allocaion paths (small/medium/large) are tested.
+        for (int length = 16; length <= maxCapacity / 16; length *= 2) {
+            System.out.println("Allocating " + length + " bytes");
+            Reference.reachabilityFence(new byte[length]);
+        }
+
+        System.out.println("Success");
+    }
+}