changeset 4666:839bc4071e51

6761744: Hotspot crashes if process size limit is exceeded Summary: Check overflow of total_reserved: object heap + perm gen (+ miscellaneous data structures) before reserving heap space, and VM aborts with appropriate message if overflowed. Reviewed-by: brutisso, johnc, tschatzl Contributed-by: tamao <tao.mao@oracle.com>
author tamao
date Thu, 02 May 2013 10:11:16 -0700
parents 944e87027dc7
children 0285b77cb121
files src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp src/share/vm/gc_interface/collectedHeap.cpp src/share/vm/gc_interface/collectedHeap.hpp src/share/vm/memory/genCollectedHeap.cpp test/TEST.ROOT test/gc/init/TestHandleExceedingProcessSizeLimitIn32BitBuilds.java
diffstat 7 files changed, 112 insertions(+), 25 deletions(-) [+]
line wrap: on
line diff
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Thu May 02 09:56:33 2013 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Thu May 02 10:11:16 2013 -0700
@@ -2018,8 +2018,12 @@
 
   // Since max_byte_size is aligned to the size of a heap region (checked
   // above), we also need to align the perm gen size as it might not be.
-  const size_t total_reserved = max_byte_size +
-                                align_size_up(pgs->max_size(), HeapRegion::GrainBytes);
+  size_t total_reserved = 0;
+
+  total_reserved = add_and_check_overflow(total_reserved, max_byte_size);
+  size_t pg_max_size = (size_t) align_size_up(pgs->max_size(), HeapRegion::GrainBytes);
+  total_reserved = add_and_check_overflow(total_reserved, pg_max_size);
+
   Universe::check_alignment(total_reserved, HeapRegion::GrainBytes, "g1 heap and perm");
 
   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Thu May 02 09:56:33 2013 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Thu May 02 10:11:16 2013 -0700
@@ -132,7 +132,12 @@
                   og_min_size, og_max_size,
                   yg_min_size, yg_max_size);
 
-  const size_t total_reserved = pg_max_size + og_max_size + yg_max_size;
+  size_t total_reserved = 0;
+
+  total_reserved = add_and_check_overflow(total_reserved, pg_max_size);
+  total_reserved = add_and_check_overflow(total_reserved, og_max_size);
+  total_reserved = add_and_check_overflow(total_reserved, yg_max_size);
+
   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);
 
   // The main part of the heap (old gen + young gen) can often use a larger page
--- a/src/share/vm/gc_interface/collectedHeap.cpp	Thu May 02 09:56:33 2013 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.cpp	Thu May 02 10:11:16 2013 -0700
@@ -56,6 +56,9 @@
 
 size_t CollectedHeap::_filler_array_max_size = 0;
 
+const char* CollectedHeap::OverflowMessage
+  = "The size of the object heap + perm gen exceeds the maximum representable size";
+
 template <>
 void EventLogBase<GCMessage>::print(outputStream* st, GCMessage& m) {
   st->print_cr("GC heap %s", m.is_before ? "before" : "after");
@@ -180,6 +183,26 @@
 #endif
 }
 
+size_t CollectedHeap::add_and_check_overflow(size_t total, size_t size) {
+  assert(size >= 0, "must be");
+  size_t result = total + size;
+  if (result < size) {
+    // We must have overflowed
+    vm_exit_during_initialization(CollectedHeap::OverflowMessage);
+  }
+  return result;
+}
+
+size_t CollectedHeap::round_up_and_check_overflow(size_t total, size_t size) {
+  assert(size >= 0, "must be");
+  size_t result = round_to(total, size);
+  if (result < size) {
+    // We must have overflowed
+    vm_exit_during_initialization(CollectedHeap::OverflowMessage);
+  }
+  return result;
+}
+
 #ifndef PRODUCT
 void CollectedHeap::check_for_bad_heap_word_value(HeapWord* addr, size_t size) {
   if (CheckMemoryInitialization && ZapUnusedHeapArea) {
--- a/src/share/vm/gc_interface/collectedHeap.hpp	Thu May 02 09:56:33 2013 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.hpp	Thu May 02 10:11:16 2013 -0700
@@ -92,6 +92,8 @@
   // Used for filler objects (static, but initialized in ctor).
   static size_t _filler_array_max_size;
 
+  const static char* OverflowMessage;
+
   GCHeapLog* _gc_heap_log;
 
   // Used in support of ReduceInitialCardMarks; only consulted if COMPILER2 is being used
@@ -134,6 +136,16 @@
   // Reinitialize tlabs before resuming mutators.
   virtual void resize_all_tlabs();
 
+  // Returns the sum of total and size if the sum does not overflow;
+  // Otherwise, call vm_exit_during_initialization().
+  // The overflow check is performed by comparing the result of the sum against size, which is assumed to be non-zero.
+  size_t add_and_check_overflow(size_t total, size_t size);
+
+  // Round up total against size and return the value, if the result does not overflow;
+  // Otherwise, call vm_exit_during_initialization().
+  // The overflow check is performed by comparing the round-up result against size, which is assumed to be non-zero.
+  size_t round_up_and_check_overflow(size_t total, size_t size);
+
   // Allocate from the current thread's TLAB, with broken-out slow path.
   inline static HeapWord* allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size);
   static HeapWord* allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size);
--- a/src/share/vm/memory/genCollectedHeap.cpp	Thu May 02 09:56:33 2013 -0700
+++ b/src/share/vm/memory/genCollectedHeap.cpp	Thu May 02 10:11:16 2013 -0700
@@ -201,9 +201,6 @@
                                  size_t* _total_reserved,
                                  int* _n_covered_regions,
                                  ReservedSpace* heap_rs){
-  const char overflow_msg[] = "The size of the object heap + VM data exceeds "
-    "the maximum representable size";
-
   // Now figure out the total size.
   size_t total_reserved = 0;
   int n_covered_regions = 0;
@@ -211,41 +208,29 @@
       os::large_page_size() : os::vm_page_size();
 
   for (int i = 0; i < _n_gens; i++) {
-    total_reserved += _gen_specs[i]->max_size();
-    if (total_reserved < _gen_specs[i]->max_size()) {
-      vm_exit_during_initialization(overflow_msg);
-    }
+    total_reserved = add_and_check_overflow(total_reserved, _gen_specs[i]->max_size());
     n_covered_regions += _gen_specs[i]->n_covered_regions();
   }
+
   assert(total_reserved % pageSize == 0,
          err_msg("Gen size; total_reserved=" SIZE_FORMAT ", pageSize="
                  SIZE_FORMAT, total_reserved, pageSize));
-  total_reserved += perm_gen_spec->max_size();
+  total_reserved = add_and_check_overflow(total_reserved, perm_gen_spec->max_size());
   assert(total_reserved % pageSize == 0,
          err_msg("Perm size; total_reserved=" SIZE_FORMAT ", pageSize="
                  SIZE_FORMAT ", perm gen max=" SIZE_FORMAT, total_reserved,
                  pageSize, perm_gen_spec->max_size()));
 
-  if (total_reserved < perm_gen_spec->max_size()) {
-    vm_exit_during_initialization(overflow_msg);
-  }
   n_covered_regions += perm_gen_spec->n_covered_regions();
 
   // Add the size of the data area which shares the same reserved area
   // as the heap, but which is not actually part of the heap.
-  size_t s = perm_gen_spec->misc_data_size() + perm_gen_spec->misc_code_size();
-
-  total_reserved += s;
-  if (total_reserved < s) {
-    vm_exit_during_initialization(overflow_msg);
-  }
+  size_t misc = perm_gen_spec->misc_data_size() + perm_gen_spec->misc_code_size();
+  total_reserved = add_and_check_overflow(total_reserved, misc);
 
   if (UseLargePages) {
     assert(total_reserved != 0, "total_reserved cannot be 0");
-    total_reserved = round_to(total_reserved, os::large_page_size());
-    if (total_reserved < os::large_page_size()) {
-      vm_exit_during_initialization(overflow_msg);
-    }
+    total_reserved = round_up_and_check_overflow(total_reserved, os::large_page_size());
   }
 
   // Calculate the address at which the heap must reside in order for
--- a/test/TEST.ROOT	Thu May 02 09:56:33 2013 -0700
+++ b/test/TEST.ROOT	Thu May 02 10:11:16 2013 -0700
@@ -28,4 +28,4 @@
 # DO NOT EDIT without first contacting hotspot-regtest@sun.com
 
 # The list of keywords supported in this test suite
-keys=cte_test jcmd nmt regression
+keys=cte_test jcmd nmt regression gc
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/gc/init/TestHandleExceedingProcessSizeLimitIn32BitBuilds.java	Thu May 02 10:11:16 2013 -0700
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test TestHandleExceedingProcessSizeLimitIn32BitBuilds.java
+ * @key gc
+ * @bug 6761744
+ * @summary Test run with "-Xmx3072m -XX:MaxPermSize=1024m" to correctly handle VM error (if any)
+ * @library /testlibrary
+ */
+
+import com.oracle.java.testlibrary.OutputAnalyzer;
+import com.oracle.java.testlibrary.ProcessTools;
+import java.util.ArrayList;
+import java.util.Collections;
+
+public class TestHandleExceedingProcessSizeLimitIn32BitBuilds {
+  public static void main(String args[]) throws Exception {
+    ArrayList<String> vmOpts = new ArrayList<>();
+    String testVmOptsStr = System.getProperty("test.java.opts");
+    if (!testVmOptsStr.isEmpty()) {
+      String[] testVmOpts = testVmOptsStr.split(" ");
+      Collections.addAll(vmOpts, testVmOpts);
+    }
+    Collections.addAll(vmOpts, new String[] {"-Xmx3072m", "-XX:MaxPermSize=1024m", "-version"});
+
+    ProcessBuilder pb
+      = ProcessTools.createJavaProcessBuilder(vmOpts.toArray(new String[vmOpts.size()]));
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+    String dataModel = System.getProperty("sun.arch.data.model");
+    if (dataModel.equals("32")) {
+      output.shouldContain("The size of the object heap + perm gen exceeds the maximum representable size");
+      if (output.getExitValue() == 0) {
+        throw new RuntimeException("Not expected to get exit value 0");
+      }
+    }
+  }
+}
\ No newline at end of file