changeset 54829:2632eb4a510e

ZGC: Generalize ZPageCache::flush()
author pliden
date Fri, 29 Mar 2019 12:46:07 +0100
parents 57cce67f0000
children ffab403eaf14
files src/hotspot/share/gc/z/zPageAllocator.cpp src/hotspot/share/gc/z/zPageAllocator.hpp src/hotspot/share/gc/z/zPageCache.cpp src/hotspot/share/gc/z/zPageCache.hpp
diffstat 4 files changed, 100 insertions(+), 56 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/share/gc/z/zPageAllocator.cpp	Fri Mar 29 12:38:30 2019 +0100
+++ b/src/hotspot/share/gc/z/zPageAllocator.cpp	Fri Mar 29 12:46:07 2019 +0100
@@ -37,6 +37,7 @@
 #include "runtime/init.hpp"
 
 static const ZStatCounter       ZCounterAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond);
+static const ZStatCounter       ZCounterPageCacheEvict("Memory", "Page Cache Evict", ZStatUnitBytesPerSecond);
 static const ZStatCriticalPhase ZCriticalPhaseAllocationStall("Allocation Stall");
 
 class ZPageAllocRequest : public StackObj {
@@ -266,16 +267,6 @@
   pmem.clear();
 }
 
-void ZPageAllocator::flush_cache(size_t size) {
-  ZList<ZPage> list;
-
-  _cache.flush(&list, size);
-
-  for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
-    destroy_page(page);
-  }
-}
-
 void ZPageAllocator::check_out_of_memory_during_initialization() {
   if (!is_init_completed()) {
     vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small");
@@ -308,8 +299,8 @@
   // Try ensure that physical memory is available
   const size_t unused = try_ensure_unused(size, flags.no_reserve());
   if (unused < size) {
-    // Flush cache to free up more physical memory
-    flush_cache(size - unused);
+    // Evict cached pages to free up physical memory
+    evict_cached_pages(size - unused);
   }
 
   // Create new page and allocate physical memory
@@ -463,6 +454,67 @@
   satisfy_alloc_queue();
 }
 
+void ZPageAllocator::flush_cache(ZPageCacheFlushClosure* cl) {
+  ZList<ZPage> list;
+
+  _cache.flush(cl, &list);
+
+  for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
+    destroy_page(page);
+  }
+}
+
+class ZPageCacheEvictClosure : public ZPageCacheFlushClosure {
+private:
+  const size_t _requested;
+  size_t       _flushed;
+
+public:
+  ZPageCacheEvictClosure(size_t requested) :
+      _requested(requested),
+      _flushed(0) {}
+
+  ~ZPageCacheEvictClosure() {
+    assert(_flushed >= _requested, "Should never fail");
+  }
+
+  virtual bool do_page(const ZPage* page) {
+    if (_flushed < _requested) {
+      // Flush page
+      _flushed += page->size();
+      return true;
+    }
+
+    // Don't flush page
+    return false;
+  }
+};
+
+void ZPageAllocator::evict_cached_pages(size_t requested) {
+   const size_t cached_before = _cache.available();
+
+  // Don't evict pages if we can't satisfy the request
+  if (requested > cached_before) {
+    return;
+  }
+
+  // Flush pages to evict
+  ZPageCacheEvictClosure cl(requested);
+  flush_cache(&cl);
+
+  const size_t cached_after = _cache.available();
+  const size_t evicted = cached_before - cached_after;
+
+  log_info(gc, heap)("Page Cache: " SIZE_FORMAT "M(%.0lf%%)->" SIZE_FORMAT "M(%.0lf%%), "
+                     "Requested: " SIZE_FORMAT "M, Evicted: " SIZE_FORMAT "M",
+                     cached_before / M, percent_of(cached_before, max_capacity()),
+                     cached_after / M, percent_of(cached_after, max_capacity()),
+                     requested / M, evicted / M);
+
+  // Update statistics
+  ZStatInc(ZCounterPageCacheEvict, evicted);
+}
+
 void ZPageAllocator::enable_deferred_delete() const {
   _safe_delete.enable_deferred_delete();
 }
--- a/src/hotspot/share/gc/z/zPageAllocator.hpp	Fri Mar 29 12:38:30 2019 +0100
+++ b/src/hotspot/share/gc/z/zPageAllocator.hpp	Fri Mar 29 12:46:07 2019 +0100
@@ -67,7 +67,8 @@
   void destroy_page(ZPage* page);
 
   void flush_pre_mapped();
-  void flush_cache(size_t size);
+  void flush_cache(ZPageCacheFlushClosure* cl);
+  void evict_cached_pages(size_t requested);
 
   void check_out_of_memory_during_initialization();
 
--- a/src/hotspot/share/gc/z/zPageCache.cpp	Fri Mar 29 12:38:30 2019 +0100
+++ b/src/hotspot/share/gc/z/zPageCache.cpp	Fri Mar 29 12:46:07 2019 +0100
@@ -32,7 +32,6 @@
 static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond);
-static const ZStatCounter ZCounterPageCacheFlush("Memory", "Page Cache Flush", ZStatUnitBytesPerSecond);
 
 ZPageCache::ZPageCache() :
     _available(0),
@@ -130,64 +129,52 @@
   _available += page->size();
 }
 
-void ZPageCache::flush_list(ZList<ZPage>* from, size_t requested, ZList<ZPage>* to, size_t* flushed) {
-  while (*flushed < requested) {
-    // Flush least recently used
-    ZPage* const page = from->remove_last();
-    if (page == NULL) {
-      break;
+void ZPageCache::flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
+  for (;;) {
+    ZPage* const page = from->last();
+    if (page == NULL || !cl->do_page(page)) {
+      // Done
+      return;
     }
 
-    *flushed += page->size();
+    // Flush page
+    _available -= page->size();
+    from->remove(page);
     to->insert_last(page);
   }
 }
 
-void ZPageCache::flush_per_numa_lists(ZPerNUMA<ZList<ZPage> >* from, size_t requested, ZList<ZPage>* to, size_t* flushed) {
+void ZPageCache::flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList<ZPage> >* from, ZList<ZPage>* to) {
   const uint32_t numa_count = ZNUMA::count();
-  uint32_t numa_empty = 0;
+  uint32_t numa_done = 0;
   uint32_t numa_next = 0;
 
   // Flush lists round-robin
-  while (*flushed < requested) {
-    ZPage* const page = from->get(numa_next).remove_last();
-
+  while (numa_done < numa_count) {
+    ZList<ZPage>& numa_list = from->get(numa_next);
     if (++numa_next == numa_count) {
       numa_next = 0;
     }
 
-    if (page == NULL) {
-      // List is empty
-      if (++numa_empty == numa_count) {
-        // All lists are empty
-        break;
-      }
-
-      // Try next list
+    ZPage* const page = numa_list.last();
+    if (page == NULL || !cl->do_page(page)) {
+      // Done
+      numa_done++;
       continue;
     }
 
+    numa_done = 0;
+
     // Flush page
-    numa_empty = 0;
-    *flushed += page->size();
+    _available -= page->size();
+    numa_list.remove(page);
     to->insert_last(page);
   }
 }
 
-void ZPageCache::flush(ZList<ZPage>* to, size_t requested) {
-  size_t flushed = 0;
-
+void ZPageCache::flush(ZPageCacheFlushClosure* cl, ZList<ZPage>* to) {
   // Prefer flushing large, then medium and last small pages
-  flush_list(&_large, requested, to, &flushed);
-  flush_list(&_medium, requested, to, &flushed);
-  flush_per_numa_lists(&_small, requested, to, &flushed);
-
-  ZStatInc(ZCounterPageCacheFlush, flushed);
-
-  log_info(gc, heap)("Page Cache Flushed: "
-                     SIZE_FORMAT "M requested, "
-                     SIZE_FORMAT "M(" SIZE_FORMAT "M->" SIZE_FORMAT "M) flushed",
-                     requested / M, flushed / M , _available / M, (_available - flushed) / M);
-
-  _available -= flushed;
+  flush_list(cl, &_large, to);
+  flush_list(cl, &_medium, to);
+  flush_per_numa_lists(cl, &_small, to);
 }
--- a/src/hotspot/share/gc/z/zPageCache.hpp	Fri Mar 29 12:38:30 2019 +0100
+++ b/src/hotspot/share/gc/z/zPageCache.hpp	Fri Mar 29 12:46:07 2019 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,10 +29,14 @@
 #include "gc/z/zValue.hpp"
 #include "memory/allocation.hpp"
 
+class ZPageCacheFlushClosure : public StackObj {
+public:
+  virtual bool do_page(const ZPage* page) = 0;
+};
+
 class ZPageCache {
 private:
   size_t                  _available;
-
   ZPerNUMA<ZList<ZPage> > _small;
   ZList<ZPage>            _medium;
   ZList<ZPage>            _large;
@@ -41,8 +45,8 @@
   ZPage* alloc_medium_page();
   ZPage* alloc_large_page(size_t size);
 
-  void flush_list(ZList<ZPage>* from, size_t requested, ZList<ZPage>* to, size_t* flushed);
-  void flush_per_numa_lists(ZPerNUMA<ZList<ZPage> >* from, size_t requested, ZList<ZPage>* to, size_t* flushed);
+  void flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to);
+  void flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList<ZPage> >* from, ZList<ZPage>* to);
 
 public:
   ZPageCache();
@@ -52,7 +56,7 @@
   ZPage* alloc_page(uint8_t type, size_t size);
   void free_page(ZPage* page);
 
-  void flush(ZList<ZPage>* to, size_t requested);
+  void flush(ZPageCacheFlushClosure* cl, ZList<ZPage>* to);
 };
 
 #endif // SHARE_GC_Z_ZPAGECACHE_HPP