changeset 967:12ea00a010fb

. Merge from main OpenJDK repository.
author glewis@misty.eyesbeyond.com
date Mon, 07 Sep 2009 10:57:43 -0700
parents 4256b1662add 6e6427f797c0
children e57bd1cf5c9a
files agent/src/share/classes/sun/jvm/hotspot/runtime/Threads.java src/cpu/x86/vm/stubGenerator_x86_64.cpp src/share/vm/adlc/adlc.hpp src/share/vm/includeDB_core src/share/vm/interpreter/bytecodeTracer.cpp src/share/vm/runtime/globals.hpp src/share/vm/runtime/vm_version.cpp src/share/vm/utilities/macros.hpp
diffstat 342 files changed, 5101 insertions(+), 2641 deletions(-) [+]
line wrap: on
line diff
--- a/.hgtags	Sat Jul 25 20:54:28 2009 -0700
+++ b/.hgtags	Mon Sep 07 10:57:43 2009 -0700
@@ -41,3 +41,8 @@
 ba36394eb84b949b31212bdb32a518a8f92bab5b jdk7-b64
 ba313800759b678979434d6da8ed3bf49eb8bea4 jdk7-b65
 57c71ad0341b8b64ed20f81151eb7f06324f8894 jdk7-b66
+18f526145aea355a9320b724373386fc2170f183 jdk7-b67
+d07e68298d4e17ebf93d8299e43fcc3ded26472a jdk7-b68
+54fd4d9232969ea6cd3d236e5ad276183bb0d423 jdk7-b69
+0632c3e615a315ff11e2ab1d64f4d82ff9853461 jdk7-b70
+50a95aa4a247f0cbbf66df285a8b1d78ffb153d9 jdk7-b71
--- a/THIRD_PARTY_README	Sat Jul 25 20:54:28 2009 -0700
+++ b/THIRD_PARTY_README	Mon Sep 07 10:57:43 2009 -0700
@@ -32,7 +32,7 @@
 
 --- end of LICENSE file ---
 %% This notice is provided with respect to ASM, which may be included with this software: 
-Copyright (c) 2000-2005 INRIA, France Telecom
+Copyright (c) 2000-2007 INRIA, France Telecom
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
--- a/agent/make/saenv.sh	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/make/saenv.sh	Mon Sep 07 10:57:43 2009 -0700
@@ -48,6 +48,8 @@
      CPU=i386
    fi
 else
+   LD_AUDIT_32=$STARTDIR/../src/os/solaris/proc/`uname -p`/libsaproc_audit.so
+   export LD_AUDIT_32
    SA_LIBPATH=$STARTDIR/../src/os/solaris/proc/`uname -p`:$STARTDIR/solaris/`uname -p`
    OPTIONS="-Dsa.library.path=$SA_LIBPATH -Dsun.jvm.hotspot.debugger.useProcDebugger"
    CPU=sparc
--- a/agent/make/saenv64.sh	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/make/saenv64.sh	Mon Sep 07 10:57:43 2009 -0700
@@ -43,6 +43,8 @@
   fi
 fi
 
+LD_AUDIT_64=$STARTDIR/../src/os/solaris/proc/$CPU/libsaproc_audit.so
+export LD_AUDIT_64
 SA_LIBPATH=$STARTDIR/../src/os/solaris/proc/$CPU:$STARTDIR/solaris/$CPU
 
 OPTIONS="-Dsa.library.path=$SA_LIBPATH -Dsun.jvm.hotspot.debugger.useProcDebugger"
--- a/agent/src/os/linux/Makefile	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/os/linux/Makefile	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2002-2006 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/agent/src/os/solaris/proc/Makefile	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/os/solaris/proc/Makefile	Mon Sep 07 10:57:43 2009 -0700
@@ -56,24 +56,28 @@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 amd64:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=amd64 -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=amd64 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 sparc:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=v8  -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=v8 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 sparcv9:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=v9 -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=v9 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 clean::
 	$(RM) -rf sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal.h
--- a/agent/src/os/solaris/proc/mapfile	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/os/solaris/proc/mapfile	Mon Sep 07 10:57:43 2009 -0700
@@ -45,6 +45,8 @@
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_resume0;
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_suspend0;
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_writeBytesToProcess0;
+                # this is needed by saproc_audit.c to redirect opens in libproc.so
+                libsaproc_open;
 local:
 	*;
 };
--- a/agent/src/os/solaris/proc/saproc.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/os/solaris/proc/saproc.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -214,49 +214,58 @@
   }
 }
 
-static int find_file_hook(const char * name, int elf_checksum) {
-  init_alt_root();
+// This function is a complete substitute for the open system call
+// since it's also used to override open calls from libproc to
+// implement as a pathmap style facility for the SA.  If libproc
+// starts using other interfaces then this might have to extended to
+// cover other calls.
+extern "C" int libsaproc_open(const char * name, int oflag, ...) {
+  if (oflag == O_RDONLY) {
+    init_alt_root();
 
-  if (_libsaproc_debug) {
-    printf("libsaproc DEBUG: find_file_hook %s 0x%x\n", name, elf_checksum);
-  }
-
-  if (alt_root_len > 0) {
-    int fd = -1;
-    char alt_path[PATH_MAX+1];
-
-    strcpy(alt_path, alt_root);
-    strcat(alt_path, name);
-    fd = open(alt_path, O_RDONLY);
-    if (fd >= 0) {
-      if (_libsaproc_debug) {
-        printf("libsaproc DEBUG: find_file_hook substituted %s\n", alt_path);
-      }
-      return fd;
+    if (_libsaproc_debug) {
+      printf("libsaproc DEBUG: libsaproc_open %s\n", name);
     }
 
-    if (strrchr(name, '/')) {
+    if (alt_root_len > 0) {
+      int fd = -1;
+      char alt_path[PATH_MAX+1];
+
       strcpy(alt_path, alt_root);
-      strcat(alt_path, strrchr(name, '/'));
+      strcat(alt_path, name);
       fd = open(alt_path, O_RDONLY);
       if (fd >= 0) {
         if (_libsaproc_debug) {
-          printf("libsaproc DEBUG: find_file_hook substituted %s\n", alt_path);
+          printf("libsaproc DEBUG: libsaproc_open substituted %s\n", alt_path);
         }
         return fd;
       }
+
+      if (strrchr(name, '/')) {
+        strcpy(alt_path, alt_root);
+        strcat(alt_path, strrchr(name, '/'));
+        fd = open(alt_path, O_RDONLY);
+        if (fd >= 0) {
+          if (_libsaproc_debug) {
+            printf("libsaproc DEBUG: libsaproc_open substituted %s\n", alt_path);
+          }
+          return fd;
+        }
+      }
     }
   }
-  return -1;
+
+  {
+    mode_t mode;
+    va_list ap;
+    va_start(ap, oflag);
+    mode = va_arg(ap, mode_t);
+    va_end(ap);
+
+    return open(name, oflag, mode);
+  }
 }
 
-static int pathmap_open(const char* name) {
-  int fd = open(name, O_RDONLY);
-  if (fd < 0) {
-    fd = find_file_hook(name, 0);
-  }
-  return fd;
-}
 
 static void * pathmap_dlopen(const char * name, int mode) {
   init_alt_root();
@@ -608,7 +617,7 @@
   print_debug("looking for %s\n", classes_jsa);
 
   // open the classes[_g].jsa
-  int fd = pathmap_open(classes_jsa);
+  int fd = libsaproc_open(classes_jsa, O_RDONLY);
   if (fd < 0) {
     char errMsg[ERR_MSG_SIZE];
     sprintf(errMsg, "can't open shared archive file %s", classes_jsa);
@@ -1209,8 +1218,6 @@
   return res;
 }
 
-typedef int (*find_file_hook_t)(const char *, int elf_checksum);
-
 /*
  * Class:       sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal
  * Method:      initIDs
@@ -1230,16 +1237,6 @@
   if (libproc_handle == 0)
      THROW_NEW_DEBUGGER_EXCEPTION("can't load libproc.so, if you are using Solaris 5.7 or below, copy libproc.so from 5.8!");
 
-  // If possible, set shared object find file hook.
-  void (*set_hook)(find_file_hook_t) = (void(*)(find_file_hook_t))dlsym(libproc_handle, "Pset_find_file_hook");
-  if (set_hook) {
-    // we found find file hook symbol, set up our hook function.
-    set_hook(find_file_hook);
-  } else if (getenv(SA_ALTROOT)) {
-    printf("libsaproc WARNING: %s set, but can't set file hook. " \
-           "Did you use right version of libproc.so?\n", SA_ALTROOT);
-  }
-
   p_ps_prochandle_ID = env->GetFieldID(clazz, "p_ps_prochandle", "J");
   CHECK_EXCEPTION;
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/src/os/solaris/proc/saproc_audit.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include <link.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <varargs.h>
+
+// This class sets up an interposer on open calls from libproc.so to
+// support a pathmap facility in the SA.
+
+static uintptr_t* libproc_cookie;
+static uintptr_t* libc_cookie;
+static uintptr_t* libsaproc_cookie;
+
+
+uint_t
+la_version(uint_t version)
+{
+  return (LAV_CURRENT);
+}
+
+
+uint_t
+la_objopen(Link_map * lmp, Lmid_t lmid, uintptr_t * cookie)
+{
+  if (strstr(lmp->l_name, "/libproc.so") != NULL) {
+    libproc_cookie = cookie;
+    return LA_FLG_BINDFROM;
+  }
+  if (strstr(lmp->l_name, "/libc.so") != NULL) {
+    libc_cookie = cookie;
+    return LA_FLG_BINDTO;
+  }
+  if (strstr(lmp->l_name, "/libsaproc.so") != NULL) {
+    libsaproc_cookie = cookie;
+    return LA_FLG_BINDTO | LA_FLG_BINDFROM;
+  }
+  return 0;
+}
+
+
+#if     defined(_LP64)
+uintptr_t
+la_symbind64(Elf64_Sym *symp, uint_t symndx, uintptr_t *refcook,
+             uintptr_t *defcook, uint_t *sb_flags, const char *sym_name)
+#else
+uintptr_t
+la_symbind32(Elf32_Sym *symp, uint_t symndx, uintptr_t *refcook,
+             uintptr_t *defcook, uint_t *sb_flags)
+#endif
+{
+#if     !defined(_LP64)
+  const char      *sym_name = (const char *)symp->st_name;
+#endif
+  if (strcmp(sym_name, "open") == 0 && refcook == libproc_cookie) {
+    // redirect all open calls from libproc.so through libsaproc_open which will
+    // try the alternate library locations first.
+    void* handle = dlmopen(LM_ID_BASE, "libsaproc.so", RTLD_NOLOAD);
+    if (handle == NULL) {
+      fprintf(stderr, "libsaproc_audit.so: didn't find libsaproc.so during linking\n");
+    } else {
+      uintptr_t libsaproc_open = (uintptr_t)dlsym(handle, "libsaproc_open");
+      if (libsaproc_open == 0) {
+        fprintf(stderr, "libsaproc_audit.so: didn't find libsaproc_open during linking\n");
+      } else {
+        return libsaproc_open;
+      }
+    }
+  }
+  return symp->st_value;
+}
--- a/agent/src/share/classes/sun/jvm/hotspot/HotSpotTypeDataBase.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/HotSpotTypeDataBase.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,8 +81,4 @@
     Assert.that(false, "should not reach here");
     return null;
   }
-
-  public int readBCI() {
-    return readInt() + InvocationEntryBCI;
-  }
 }
--- a/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/code/PCDesc.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/PCDesc.java	Mon Sep 07 10:57:43 2009 -0700
@@ -82,6 +82,7 @@
       tty.print(" ");
       sd.getMethod().printValueOn(tty);
       tty.print("  @" + sd.getBCI());
+      tty.print("  reexecute=" + sd.getReexecute());
       tty.println();
     }
   }
--- a/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,7 @@
   private NMethod code;
   private Method  method;
   private int     bci;
+  private boolean reexecute;
   /** Decoding offsets */
   private int     decodeOffset;
   private int     senderDecodeOffset;
@@ -61,7 +62,7 @@
 
     senderDecodeOffset = stream.readInt();
     method = (Method) VM.getVM().getObjectHeap().newOop(stream.readOopHandle());
-    bci    = stream.readBCI();
+    setBCIAndReexecute(stream.readInt());
     // Decode offsets for body and sender
     localsDecodeOffset      = stream.readInt();
     expressionsDecodeOffset = stream.readInt();
@@ -78,7 +79,7 @@
 
     senderDecodeOffset = stream.readInt();
     method = (Method) VM.getVM().getObjectHeap().newOop(stream.readOopHandle());
-    bci    = stream.readBCI();
+    setBCIAndReexecute(stream.readInt());
     // Decode offsets for body and sender
     localsDecodeOffset      = stream.readInt();
     expressionsDecodeOffset = stream.readInt();
@@ -88,6 +89,7 @@
   public NMethod getNMethod() { return code; }
   public Method getMethod() { return method; }
   public int    getBCI()    { return bci;    }
+  public boolean getReexecute() {return reexecute;}
 
   /** Returns a List&lt;ScopeValue&gt; */
   public List getLocals() {
@@ -150,6 +152,7 @@
     tty.print("ScopeDesc for ");
     method.printValueOn(tty);
     tty.println(" @bci " + bci);
+    tty.println(" reexecute: " + reexecute);
   }
 
   // FIXME: add more accessors
@@ -157,6 +160,11 @@
   //--------------------------------------------------------------------------------
   // Internals only below this point
   //
+  private void setBCIAndReexecute(int combination) {
+    int InvocationEntryBci = VM.getVM().getInvocationEntryBCI();
+    bci = (combination >> 1) + InvocationEntryBci;
+    reexecute = (combination & 1)==1 ? true : false;
+  }
 
   private DebugInfoReadStream streamAt(int decodeOffset) {
     return new DebugInfoReadStream(code, decodeOffset, objects);
--- a/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2001 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/Debugger.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/Debugger.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/DebuggerBase.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/DebuggerBase.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/JVMDebugger.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/JVMDebugger.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebugger.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebugger.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebuggerClient.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebuggerClient.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebuggerServer.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebuggerServer.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/memory/CompactibleFreeListSpace.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/memory/CompactibleFreeListSpace.java	Mon Sep 07 10:57:43 2009 -0700
@@ -176,19 +176,6 @@
 
       for (; cur.lessThan(limit);) {
          Address klassOop = cur.getAddressAt(addressSize);
-         // FIXME: need to do a better job here.
-         // can I use bitMap here?
-         if (klassOop == null) {
-            //Find the object size using Printezis bits and skip over
-            System.err.println("Finding object size using Printezis bits and skipping over...");
-            long size = collector().blockSizeUsingPrintezisBits(cur);
-            if (size == -1) {
-              System.err.println("Printezis bits not set...");
-              break;
-            }
-            cur = cur.addOffsetTo(adjustObjectSizeInBytes(size));
-         }
-
          if (FreeChunk.indicatesFreeChunk(cur)) {
             if (! cur.equals(regionStart)) {
                res.add(new MemRegion(regionStart, cur));
@@ -200,12 +187,21 @@
             }
             // note that fc.size() gives chunk size in heap words
             cur = cur.addOffsetTo(chunkSize * addressSize);
-            System.err.println("Free chunk in CMS heap, size="+chunkSize * addressSize);
             regionStart = cur;
          } else if (klassOop != null) {
             Oop obj = heap.newOop(cur.addOffsetToAsOopHandle(0));
             long objectSize = obj.getObjectSize();
             cur = cur.addOffsetTo(adjustObjectSizeInBytes(objectSize));
+         } else {
+            // FIXME: need to do a better job here.
+            // can I use bitMap here?
+            //Find the object size using Printezis bits and skip over
+            long size = collector().blockSizeUsingPrintezisBits(cur);
+            if (size == -1) {
+              System.err.println("Printezis bits not set...");
+              break;
+            }
+            cur = cur.addOffsetTo(adjustObjectSizeInBytes(size));
          }
       }
       return res;
--- a/agent/src/share/classes/sun/jvm/hotspot/memory/FreeChunk.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/memory/FreeChunk.java	Mon Sep 07 10:57:43 2009 -0700
@@ -63,7 +63,7 @@
 
    public long size() {
       if (VM.getVM().isCompressedOopsEnabled()) {
-        Mark mark = new Mark(sizeField.getValue(addr));
+        Mark mark = new Mark(addr.addOffsetTo(sizeField.getOffset()));
         return mark.getSize();
       } else {
         Address size = sizeField.getValue(addr);
@@ -83,7 +83,7 @@
 
    public boolean isFree() {
       if (VM.getVM().isCompressedOopsEnabled()) {
-        Mark mark = new Mark(sizeField.getValue(addr));
+        Mark mark = new Mark(addr.addOffsetTo(sizeField.getOffset()));
         return mark.isCmsFreeChunk();
       } else {
         Address prev = prevField.getValue(addr);
--- a/agent/src/share/classes/sun/jvm/hotspot/memory/Universe.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/memory/Universe.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/ClassConstants.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/ClassConstants.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2001 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/JavaThread.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/JavaThread.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2001 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/StubRoutines.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/StubRoutines.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/Thread.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/Thread.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/Threads.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/Threads.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/runtime/VM.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/VM.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ByteCodeRewriter.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ByteCodeRewriter.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ClassDump.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ClassDump.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ClassWriter.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/tools/jcore/ClassWriter.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2004-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/make/hotspot_version	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/hotspot_version	Mon Sep 07 10:57:43 2009 -0700
@@ -35,7 +35,7 @@
 
 HS_MAJOR_VER=16
 HS_MINOR_VER=0
-HS_BUILD_NUMBER=06
+HS_BUILD_NUMBER=08
 
 JDK_MAJOR_VER=1
 JDK_MINOR_VER=7
--- a/make/jprt.properties	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/jprt.properties	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -306,7 +306,6 @@
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCBasher_ParallelGC, \
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCBasher_ParNewGC, \
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCBasher_CMS, \
-    ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCBasher_G1, \
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCBasher_ParOldGC, \
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCOld_default, \
     ${jprt.my.windows.x64}-{product|fastdebug}-c2-GCOld_SerialGC, \
--- a/make/linux/makefiles/jsig.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/linux/makefiles/jsig.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/linux/makefiles/saproc.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/linux/makefiles/saproc.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/solaris/makefiles/optimized.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/solaris/makefiles/optimized.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/solaris/makefiles/sparcWorks.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/solaris/makefiles/sparcWorks.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/build_vm_def.sh	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/build_vm_def.sh	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/create.bat	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/create.bat	Mon Sep 07 10:57:43 2009 -0700
@@ -1,6 +1,6 @@
 @echo off
 REM
-REM Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+REM Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
 REM DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 REM
 REM This code is free software; you can redistribute it and/or modify it
--- a/make/windows/get_msc_ver.sh	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/get_msc_ver.sh	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/adlc.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/adlc.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/compile.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/compile.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/makedeps.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/makedeps.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/rules.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/rules.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/sa.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/sa.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/sanity.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/sanity.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/make/windows/makefiles/vm.make	Sat Jul 25 20:54:28 2009 -0700
+++ b/make/windows/makefiles/vm.make	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/sparc/vm/assembler_sparc.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -4208,6 +4208,7 @@
                   PtrQueue::byte_offset_of_active()),
          tmp);
   }
+
   // Check on whether to annul.
   br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
   delayed() -> nop();
@@ -4215,13 +4216,13 @@
   // satb_log_barrier_work1(tmp, offset);
   if (index == noreg) {
     if (Assembler::is_simm13(offset)) {
-      ld_ptr(obj, offset, tmp);
+      load_heap_oop(obj, offset, tmp);
     } else {
       set(offset, tmp);
-      ld_ptr(obj, tmp, tmp);
+      load_heap_oop(obj, tmp, tmp);
     }
   } else {
-    ld_ptr(obj, index, tmp);
+    load_heap_oop(obj, index, tmp);
   }
 
   // satb_log_barrier_work2(obj, tmp, offset);
--- a/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/sparc/vm/frame_sparc.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/sparc/vm/frame_sparc.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/sparc/vm/globals_sparc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/sparc/vm/globals_sparc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/assembler_x86.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/assembler_x86.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -6805,14 +6805,18 @@
   jcc(Assembler::equal, done);
 
   // if (x.f == NULL) goto done;
-  cmpptr(Address(obj, 0), NULL_WORD);
+#ifdef _LP64
+  load_heap_oop(tmp2, Address(obj, 0));
+#else
+  movptr(tmp2, Address(obj, 0));
+#endif
+  cmpptr(tmp2, (int32_t) NULL_WORD);
   jcc(Assembler::equal, done);
 
   // Can we store original value in the thread's buffer?
 
-  LP64_ONLY(movslq(tmp, index);)
-  movptr(tmp2, Address(obj, 0));
 #ifdef _LP64
+  movslq(tmp, index);
   cmpq(tmp, 0);
 #else
   cmpl(index, 0);
@@ -6834,8 +6838,7 @@
   if(tosca_live) push(rax);
   push(obj);
 #ifdef _LP64
-  movq(c_rarg0, Address(obj, 0));
-  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, r15_thread);
+  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, r15_thread);
 #else
   push(thread);
   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, thread);
@@ -8332,15 +8335,13 @@
   // Cannot assert, unverified entry point counts instructions (see .ad file)
   // vtableStubs also counts instructions in pd_code_size_limit.
   // Also do not verify_oop as this is called by verify_oop.
-  if (Universe::narrow_oop_base() == NULL) {
-    if (Universe::narrow_oop_shift() != 0) {
-      assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
-      shlq(r, LogMinObjAlignmentInBytes);
-    }
+  if (Universe::narrow_oop_shift() != 0) {
+    assert (Address::times_8 == LogMinObjAlignmentInBytes &&
+            Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
+    // Don't use Shift since it modifies flags.
+    leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
   } else {
-      assert (Address::times_8 == LogMinObjAlignmentInBytes &&
-              Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
-    leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
+    assert (Universe::narrow_oop_base() == NULL, "sanity");
   }
 }
 
@@ -8355,6 +8356,7 @@
             Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
     leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
   } else if (dst != src) {
+    assert (Universe::narrow_oop_base() == NULL, "sanity");
     movq(dst, src);
   }
 }
--- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/frame_x86.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/frame_x86.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/globals_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/globals_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/interpreterRT_x86_64.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/interpreterRT_x86_64.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/methodHandles_x86.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/methodHandles_x86.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -269,11 +269,11 @@
 
 #ifndef PRODUCT
 void trace_method_handle_stub(const char* adaptername,
-                              oop mh,
+                              oopDesc* mh,
                               intptr_t* entry_sp,
                               intptr_t* saved_sp) {
   // called as a leaf from native code: do not block the JVM!
-  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, mh, entry_sp, entry_sp - saved_sp);
+  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, (void*)mh, entry_sp, entry_sp - saved_sp);
 }
 #endif //PRODUCT
 
--- a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1302,22 +1302,19 @@
 
   const Register ic_reg = rax;
   const Register receiver = j_rarg0;
-  const Register tmp = rdx;
 
   Label ok;
   Label exception_pending;
 
+  assert_different_registers(ic_reg, receiver, rscratch1);
   __ verify_oop(receiver);
-  __ push(tmp); // spill (any other registers free here???)
-  __ load_klass(tmp, receiver);
-  __ cmpq(ic_reg, tmp);
+  __ load_klass(rscratch1, receiver);
+  __ cmpq(ic_reg, rscratch1);
   __ jcc(Assembler::equal, ok);
 
-  __ pop(tmp);
   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 
   __ bind(ok);
-  __ pop(tmp);
 
   // Verified entry point must be aligned
   __ align(8);
--- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -709,7 +709,7 @@
   //
   //  Input:
   //     start   -  starting address
-  //     end     -  element count
+  //     count   -  element count
   void  gen_write_ref_array_pre_barrier(Register start, Register count) {
     assert_different_registers(start, count);
     BarrierSet* bs = Universe::heap()->barrier_set();
@@ -757,7 +757,6 @@
           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
           __ addptr(rsp, 2*wordSize);
           __ popa();
-
         }
         break;
 
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1207,9 +1207,9 @@
           __ pusha();                      // push registers (overkill)
           // must compute element count unless barrier set interface is changed (other platforms supply count)
           assert_different_registers(start, end, scratch);
-          __ lea(scratch, Address(end, wordSize));
-          __ subptr(scratch, start);
-          __ shrptr(scratch, LogBytesPerWord);
+          __ lea(scratch, Address(end, BytesPerHeapOop));
+          __ subptr(scratch, start);               // subtract start to get #bytes
+          __ shrptr(scratch, LogBytesPerHeapOop);  // convert to element count
           __ mov(c_rarg0, start);
           __ mov(c_rarg1, scratch);
           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
@@ -1225,6 +1225,7 @@
           Label L_loop;
 
            __ shrptr(start, CardTableModRefBS::card_shift);
+           __ addptr(end, BytesPerHeapOop);
            __ shrptr(end, CardTableModRefBS::card_shift);
            __ subptr(end, start); // number of bytes to copy
 
@@ -2251,6 +2252,7 @@
     // and report their number to the caller.
     assert_different_registers(rax, r14_length, count, to, end_to, rcx);
     __ lea(end_to, to_element_addr);
+    __ addptr(end_to, -heapOopSize);      // make an inclusive end pointer
     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
     __ movptr(rax, r14_length);           // original oops
     __ addptr(rax, count);                // K = (original - remaining) oops
@@ -2259,7 +2261,7 @@
 
     // Come here on success only.
     __ BIND(L_do_card_marks);
-    __ addptr(end_to, -wordSize);         // make an inclusive end pointer
+    __ addptr(end_to, -heapOopSize);         // make an inclusive end pointer
     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
     __ xorptr(rax, rax);                  // return 0 on success
 
--- a/src/cpu/x86/vm/templateTable_x86_32.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/templateTable_x86_32.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/templateTable_x86_64.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/templateTable_x86_64.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/vtableStubs_x86_32.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/vtableStubs_x86_32.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/vtableStubs_x86_64.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/cpu/x86/vm/vtableStubs_x86_64.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/linux/vm/os_linux.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/linux/vm/os_linux.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/solaris/dtrace/generateJvmOffsets.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/solaris/dtrace/generateJvmOffsets.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/solaris/dtrace/jhelper.d	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/solaris/dtrace/jhelper.d	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/solaris/dtrace/libjvm_db.c	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/solaris/dtrace/libjvm_db.c	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/solaris/vm/os_solaris.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/solaris/vm/os_solaris.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1643,7 +1643,8 @@
 inline hrtime_t getTimeNanos() {
   if (VM_Version::supports_cx8()) {
     const hrtime_t now = gethrtime();
-    const hrtime_t prev = max_hrtime;
+    // Use atomic long load since 32-bit x86 uses 2 registers to keep long.
+    const hrtime_t prev = Atomic::load((volatile jlong*)&max_hrtime);
     if (now <= prev)  return prev;   // same or retrograde time;
     const hrtime_t obsv = Atomic::cmpxchg(now, (volatile jlong*)&max_hrtime, prev);
     assert(obsv >= prev, "invariant");   // Monotonicity
--- a/src/os/windows/vm/os_windows.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os/windows/vm/os_windows.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -616,12 +616,13 @@
 }
 
 julong os::win32::available_memory() {
-  // FIXME: GlobalMemoryStatus() may return incorrect value if total memory
-  // is larger than 4GB
-  MEMORYSTATUS ms;
-  GlobalMemoryStatus(&ms);
-
-  return (julong)ms.dwAvailPhys;
+  // Use GlobalMemoryStatusEx() because GlobalMemoryStatus() may return incorrect
+  // value if total memory is larger than 4GB
+  MEMORYSTATUSEX ms;
+  ms.dwLength = sizeof(ms);
+  GlobalMemoryStatusEx(&ms);
+
+  return (julong)ms.ullAvailPhys;
 }
 
 julong os::physical_memory() {
@@ -1579,16 +1580,17 @@
   st->print("Memory:");
   st->print(" %dk page", os::vm_page_size()>>10);
 
-  // FIXME: GlobalMemoryStatus() may return incorrect value if total memory
-  // is larger than 4GB
-  MEMORYSTATUS ms;
-  GlobalMemoryStatus(&ms);
+  // Use GlobalMemoryStatusEx() because GlobalMemoryStatus() may return incorrect
+  // value if total memory is larger than 4GB
+  MEMORYSTATUSEX ms;
+  ms.dwLength = sizeof(ms);
+  GlobalMemoryStatusEx(&ms);
 
   st->print(", physical %uk", os::physical_memory() >> 10);
   st->print("(%uk free)", os::available_memory() >> 10);
 
-  st->print(", swap %uk", ms.dwTotalPageFile >> 10);
-  st->print("(%uk free)", ms.dwAvailPageFile >> 10);
+  st->print(", swap %uk", ms.ullTotalPageFile >> 10);
+  st->print("(%uk free)", ms.ullAvailPageFile >> 10);
   st->cr();
 }
 
@@ -3135,11 +3137,13 @@
   _processor_level = si.wProcessorLevel;
   _processor_count = si.dwNumberOfProcessors;
 
-  MEMORYSTATUS ms;
+  MEMORYSTATUSEX ms;
+  ms.dwLength = sizeof(ms);
+
   // also returns dwAvailPhys (free physical memory bytes), dwTotalVirtual, dwAvailVirtual,
   // dwMemoryLoad (% of memory in use)
-  GlobalMemoryStatus(&ms);
-  _physical_memory = ms.dwTotalPhys;
+  GlobalMemoryStatusEx(&ms);
+  _physical_memory = ms.ullTotalPhys;
 
   OSVERSIONINFO oi;
   oi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
--- a/src/os_cpu/linux_sparc/vm/globals_linux_sparc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/linux_sparc/vm/globals_linux_sparc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/linux_sparc/vm/os_linux_sparc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/linux_sparc/vm/os_linux_sparc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/linux_x86/vm/globals_linux_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/linux_x86/vm/globals_linux_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -46,6 +46,8 @@
 inline void Atomic::dec_ptr(volatile intptr_t* dest) { (void)add_ptr(-1, dest); }
 inline void Atomic::dec_ptr(volatile void*     dest) { (void)add_ptr(-1, dest); }
 
+inline jlong Atomic::load(volatile jlong* src) { return *src; }
+
 #ifdef _GNU_SOURCE
 
 inline jint     Atomic::add    (jint     add_value, volatile jint*     dest) {
--- a/src/os_cpu/solaris_sparc/vm/globals_solaris_sparc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_sparc/vm/globals_solaris_sparc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_sparc/vm/os_solaris_sparc.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_sparc/vm/os_solaris_sparc.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_sparc/vm/os_solaris_sparc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_sparc/vm/os_solaris_sparc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -99,6 +99,8 @@
   return (void*)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, (int) os::is_MP());
 }
 
+inline jlong Atomic::load(volatile jlong* src) { return *src; }
+
 #else // !AMD64
 
 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) {
@@ -131,6 +133,15 @@
 inline void*    Atomic::cmpxchg_ptr(void*    exchange_value, volatile void*     dest, void*    compare_value) {
   return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
 }
+
+extern "C" void _Atomic_load_long(volatile jlong* src, volatile jlong* dst);
+
+inline jlong Atomic::load(volatile jlong* src) {
+  volatile jlong dest;
+  _Atomic_load_long(src, &dest);
+  return dest;
+}
+
 #endif // AMD64
 
 #ifdef _GNU_SOURCE
--- a/src/os_cpu/solaris_x86/vm/globals_solaris_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/globals_solaris_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_x86/vm/solaris_x86_32.il	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/solaris_x86_32.il	Mon Sep 07 10:57:43 2009 -0700
@@ -97,6 +97,15 @@
       popl     %ebx
       .end
 
+  // Support for void Atomic::load(volatile jlong* src, volatile jlong* dest).
+      .inline _Atomic_load_long,2
+      movl     0(%esp), %eax   // src
+      fildll    (%eax)
+      movl     4(%esp), %eax   // dest
+      fistpll   (%eax)
+      .end
+
+
   // Support for OrderAccess::acquire()
       .inline _OrderAccess_acquire,0
       movl     0(%esp), %eax
--- a/src/os_cpu/windows_x86/vm/globals_windows_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/windows_x86/vm/globals_windows_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/windows_x86/vm/os_windows_x86.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/windows_x86/vm/os_windows_x86.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/windows_x86/vm/os_windows_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/windows_x86/vm/os_windows_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/windows_x86/vm/unwind_windows_x86.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/os_cpu/windows_x86/vm/unwind_windows_x86.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2004-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/MakeDeps/BuildConfig.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/MakeDeps/BuildConfig.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/MakeDeps/WinGammaPlatformVC7.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/MakeDeps/WinGammaPlatformVC7.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/MakeDeps/WinGammaPlatformVC8.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/MakeDeps/WinGammaPlatformVC8.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/MakeDeps/WinGammaPlatformVC9.java	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/MakeDeps/WinGammaPlatformVC9.java	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/hsdis/Makefile	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/hsdis/Makefile	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 #
-# Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/hsdis/hsdis-demo.c	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/hsdis/hsdis-demo.c	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/tools/hsdis/hsdis.c	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/tools/hsdis/hsdis.c	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/adlc/adlc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/adlc/adlc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/adlc/formssel.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/adlc/formssel.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -420,6 +420,13 @@
   return  _matrule->is_ideal_load();
 }
 
+// Return 'true' if this instruction matches an ideal 'LoadKlass' node
+bool InstructForm::skip_antidep_check() const {
+  if( _matrule == NULL ) return false;
+
+  return  _matrule->skip_antidep_check();
+}
+
 // Return 'true' if this instruction matches an ideal 'Load?' node
 Form::DataType InstructForm::is_ideal_store() const {
   if( _matrule == NULL ) return Form::none;
@@ -567,6 +574,8 @@
 
 // loads from memory, so must check for anti-dependence
 bool InstructForm::needs_anti_dependence_check(FormDict &globals) const {
+  if ( skip_antidep_check() ) return false;
+
   // Machine independent loads must be checked for anti-dependences
   if( is_ideal_load() != Form::none )  return true;
 
@@ -3957,6 +3966,28 @@
 }
 
 
+bool MatchRule::skip_antidep_check() const {
+  // Some loads operate on what is effectively immutable memory so we
+  // should skip the anti dep computations.  For some of these nodes
+  // the rewritable field keeps the anti dep logic from triggering but
+  // for certain kinds of LoadKlass it does not since they are
+  // actually reading memory which could be rewritten by the runtime,
+  // though never by generated code.  This disables it uniformly for
+  // the nodes that behave like this: LoadKlass, LoadNKlass and
+  // LoadRange.
+  if ( _opType && (strcmp(_opType,"Set") == 0) && _rChild ) {
+    const char *opType = _rChild->_opType;
+    if (strcmp("LoadKlass", opType) == 0 ||
+        strcmp("LoadNKlass", opType) == 0 ||
+        strcmp("LoadRange", opType) == 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+
 Form::DataType MatchRule::is_ideal_store() const {
   Form::DataType ideal_store = Form::none;
 
--- a/src/share/vm/adlc/formssel.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/adlc/formssel.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -158,6 +158,9 @@
 
   virtual Form::CallType is_ideal_call() const; // matches ideal 'Call'
   virtual Form::DataType is_ideal_load() const; // node matches ideal 'LoadXNode'
+  // Should antidep checks be disabled for this Instruct
+  // See definition of MatchRule::skip_antidep_check
+  bool skip_antidep_check() const;
   virtual Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'
           bool        is_ideal_mem() const { return is_ideal_load() != Form::none || is_ideal_store() != Form::none; }
   virtual uint        two_address(FormDict &globals); // output reg must match input reg
@@ -1003,6 +1006,9 @@
   bool       is_ideal_loopEnd() const; // node matches ideal 'LoopEnd'
   bool       is_ideal_bool() const;    // node matches ideal 'Bool'
   Form::DataType is_ideal_load() const;// node matches ideal 'LoadXNode'
+  // Should antidep checks be disabled for this rule
+  // See definition of MatchRule::skip_antidep_check
+  bool skip_antidep_check() const;
   Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'
 
   // Check if 'mRule2' is a cisc-spill variant of this MatchRule
--- a/src/share/vm/asm/assembler.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/asm/assembler.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_Compilation.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_Compilation.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_GraphBuilder.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_GraphBuilder.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_IR.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_IR.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -208,6 +208,15 @@
   return scope->caller_bci();
 }
 
+bool IRScopeDebugInfo::should_reexecute() {
+  ciMethod* cur_method = scope()->method();
+  int       cur_bci    = bci();
+  if (cur_method != NULL && cur_bci != SynchronizationEntryBCI) {
+    Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
+    return Interpreter::bytecode_should_reexecute(code);
+  } else
+    return false;
+}
 
 
 // Implementation of CodeEmitInfo
@@ -253,7 +262,7 @@
 void CodeEmitInfo::record_debug_info(DebugInformationRecorder* recorder, int pc_offset) {
   // record the safepoint before recording the debug info for enclosing scopes
   recorder->add_safepoint(pc_offset, _oop_map->deep_copy());
-  _scope_debug_info->record_debug_info(recorder, pc_offset);
+  _scope_debug_info->record_debug_info(recorder, pc_offset, true/*topmost*/);
   recorder->end_safepoint(pc_offset);
 }
 
--- a/src/share/vm/c1/c1_IR.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_IR.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -239,15 +239,20 @@
   GrowableArray<MonitorValue*>* monitors()    { return _monitors;    }
   IRScopeDebugInfo*             caller()      { return _caller;      }
 
-  void record_debug_info(DebugInformationRecorder* recorder, int pc_offset) {
+  //Whether we should reexecute this bytecode for deopt
+  bool should_reexecute();
+
+  void record_debug_info(DebugInformationRecorder* recorder, int pc_offset, bool topmost) {
     if (caller() != NULL) {
       // Order is significant:  Must record caller first.
-      caller()->record_debug_info(recorder, pc_offset);
+      caller()->record_debug_info(recorder, pc_offset, false/*topmost*/);
     }
     DebugToken* locvals = recorder->create_scope_values(locals());
     DebugToken* expvals = recorder->create_scope_values(expressions());
     DebugToken* monvals = recorder->create_monitor_values(monitors());
-    recorder->describe_scope(pc_offset, scope()->method(), bci(), locvals, expvals, monvals);
+    // reexecute allowed only for the topmost frame
+    bool      reexecute = topmost ? should_reexecute() : false;
+    recorder->describe_scope(pc_offset, scope()->method(), bci(), reexecute, locvals, expvals, monvals);
   }
 };
 
--- a/src/share/vm/c1/c1_LIRAssembler.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_LIRAssembler.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -379,7 +379,8 @@
     ValueStack* s = nth_oldest(vstack, n, s_bci);
     if (s == NULL)  break;
     IRScope* scope = s->scope();
-    debug_info->describe_scope(pc_offset, scope->method(), s_bci);
+    //Always pass false for reexecute since these ScopeDescs are never used for deopt
+    debug_info->describe_scope(pc_offset, scope->method(), s_bci, false/*reexecute*/);
   }
 
   debug_info->end_non_safepoint(pc_offset);
--- a/src/share/vm/c1/c1_LinearScan.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/c1/c1_LinearScan.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/bcEscapeAnalyzer.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/bcEscapeAnalyzer.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciEnv.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciEnv.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciEnv.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciEnv.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciMethodBlocks.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciMethodBlocks.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciObjectFactory.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciObjectFactory.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -219,24 +219,27 @@
   ASSERT_IN_VM;
 
 #ifdef ASSERT
-  oop last = NULL;
-  for (int j = 0; j< _ci_objects->length(); j++) {
-    oop o = _ci_objects->at(j)->get_oop();
-    assert(last < o, "out of order");
-    last = o;
+  if (CIObjectFactoryVerify) {
+    oop last = NULL;
+    for (int j = 0; j< _ci_objects->length(); j++) {
+      oop o = _ci_objects->at(j)->get_oop();
+      assert(last < o, "out of order");
+      last = o;
+    }
   }
 #endif // ASSERT
   int len = _ci_objects->length();
   int index = find(key, _ci_objects);
 #ifdef ASSERT
-  for (int i=0; i<_ci_objects->length(); i++) {
-    if (_ci_objects->at(i)->get_oop() == key) {
-      assert(index == i, " bad lookup");
+  if (CIObjectFactoryVerify) {
+    for (int i=0; i<_ci_objects->length(); i++) {
+      if (_ci_objects->at(i)->get_oop() == key) {
+        assert(index == i, " bad lookup");
+      }
     }
   }
 #endif
   if (!is_found_at(index, key, _ci_objects)) {
-
     // Check in the non-perm area before putting it in the list.
     NonPermObject* &bucket = find_non_perm(key);
     if (bucket != NULL) {
@@ -539,11 +542,13 @@
     objects->at_put(index, obj);
   }
 #ifdef ASSERT
-  oop last = NULL;
-  for (int j = 0; j< objects->length(); j++) {
-    oop o = objects->at(j)->get_oop();
-    assert(last < o, "out of order");
-    last = o;
+  if (CIObjectFactoryVerify) {
+    oop last = NULL;
+    for (int j = 0; j< objects->length(); j++) {
+      oop o = objects->at(j)->get_oop();
+      assert(last < o, "out of order");
+      last = o;
+    }
   }
 #endif // ASSERT
 }
--- a/src/share/vm/ci/ciStreams.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciStreams.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciStreams.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciStreams.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/ci/ciTypeFlow.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/ci/ciTypeFlow.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/classFileParser.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/classFileParser.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -547,7 +547,6 @@
                                                  int length,
                                                  Handle class_loader,
                                                  Handle protection_domain,
-                                                 PerfTraceTime* vmtimer,
                                                  symbolHandle class_name,
                                                  TRAPS) {
   ClassFileStream* cfs = stream();
@@ -575,13 +574,11 @@
       guarantee_property(unresolved_klass->byte_at(0) != JVM_SIGNATURE_ARRAY,
                          "Bad interface name in class file %s", CHECK_(nullHandle));
 
-      vmtimer->suspend();  // do not count recursive loading twice
       // Call resolve_super so classcircularity is checked
       klassOop k = SystemDictionary::resolve_super_or_fail(class_name,
                     unresolved_klass, class_loader, protection_domain,
                     false, CHECK_(nullHandle));
       interf = KlassHandle(THREAD, k);
-      vmtimer->resume();
 
       if (LinkWellKnownClasses)  // my super type is well known to me
         cp->klass_at_put(interface_index, interf()); // eagerly resolve
@@ -2558,7 +2555,15 @@
 
   ClassFileStream* cfs = stream();
   // Timing
-  PerfTraceTime vmtimer(ClassLoader::perf_accumulated_time());
+  assert(THREAD->is_Java_thread(), "must be a JavaThread");
+  JavaThread* jt = (JavaThread*) THREAD;
+
+  PerfClassTraceTime ctimer(ClassLoader::perf_class_parse_time(),
+                            ClassLoader::perf_class_parse_selftime(),
+                            NULL,
+                            jt->get_thread_stat()->perf_recursion_counts_addr(),
+                            jt->get_thread_stat()->perf_timers_addr(),
+                            PerfClassTraceTime::PARSE_CLASS);
 
   _has_finalizer = _has_empty_finalizer = _has_vanilla_constructor = false;
 
@@ -2738,7 +2743,7 @@
     if (itfs_len == 0) {
       local_interfaces = objArrayHandle(THREAD, Universe::the_empty_system_obj_array());
     } else {
-      local_interfaces = parse_interfaces(cp, itfs_len, class_loader, protection_domain, &vmtimer, _class_name, CHECK_(nullHandle));
+      local_interfaces = parse_interfaces(cp, itfs_len, class_loader, protection_domain, _class_name, CHECK_(nullHandle));
     }
 
     // Fields (offsets are filled in later)
@@ -2782,6 +2787,7 @@
                                                            protection_domain,
                                                            true,
                                                            CHECK_(nullHandle));
+
       KlassHandle kh (THREAD, k);
       super_klass = instanceKlassHandle(THREAD, kh());
       if (LinkWellKnownClasses)  // my super class is well known to me
@@ -3231,6 +3237,16 @@
     this_klass->set_minor_version(minor_version);
     this_klass->set_major_version(major_version);
 
+    // Set up methodOop::intrinsic_id as soon as we know the names of methods.
+    // (We used to do this lazily, but now we query it in Rewriter,
+    // which is eagerly done for every method, so we might as well do it now,
+    // when everything is fresh in memory.)
+    if (methodOopDesc::klass_id_for_intrinsics(this_klass->as_klassOop()) != vmSymbols::NO_SID) {
+      for (int j = 0; j < methods->length(); j++) {
+        ((methodOop)methods->obj_at(j))->init_intrinsic_id();
+      }
+    }
+
     if (cached_class_file_bytes != NULL) {
       // JVMTI: we have an instanceKlass now, tell it about the cached bytes
       this_klass->set_cached_class_file(cached_class_file_bytes,
--- a/src/share/vm/classfile/classFileParser.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/classFileParser.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -61,7 +61,6 @@
                                   int length,
                                   Handle class_loader,
                                   Handle protection_domain,
-                                  PerfTraceTime* vmtimer,
                                   symbolHandle class_name,
                                   TRAPS);
 
--- a/src/share/vm/classfile/classLoader.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/classLoader.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -48,9 +48,26 @@
 PerfCounter*    ClassLoader::_perf_accumulated_time = NULL;
 PerfCounter*    ClassLoader::_perf_classes_inited = NULL;
 PerfCounter*    ClassLoader::_perf_class_init_time = NULL;
+PerfCounter*    ClassLoader::_perf_class_init_selftime = NULL;
+PerfCounter*    ClassLoader::_perf_classes_verified = NULL;
 PerfCounter*    ClassLoader::_perf_class_verify_time = NULL;
+PerfCounter*    ClassLoader::_perf_class_verify_selftime = NULL;
 PerfCounter*    ClassLoader::_perf_classes_linked = NULL;
 PerfCounter*    ClassLoader::_perf_class_link_time = NULL;
+PerfCounter*    ClassLoader::_perf_class_link_selftime = NULL;
+PerfCounter*    ClassLoader::_perf_class_parse_time = NULL;
+PerfCounter*    ClassLoader::_perf_class_parse_selftime = NULL;
+PerfCounter*    ClassLoader::_perf_sys_class_lookup_time = NULL;
+PerfCounter*    ClassLoader::_perf_shared_classload_time = NULL;
+PerfCounter*    ClassLoader::_perf_sys_classload_time = NULL;
+PerfCounter*    ClassLoader::_perf_app_classload_time = NULL;
+PerfCounter*    ClassLoader::_perf_app_classload_selftime = NULL;
+PerfCounter*    ClassLoader::_perf_app_classload_count = NULL;
+PerfCounter*    ClassLoader::_perf_define_appclasses = NULL;
+PerfCounter*    ClassLoader::_perf_define_appclass_time = NULL;
+PerfCounter*    ClassLoader::_perf_define_appclass_selftime = NULL;
+PerfCounter*    ClassLoader::_perf_app_classfile_bytes_read = NULL;
+PerfCounter*    ClassLoader::_perf_sys_classfile_bytes_read = NULL;
 PerfCounter*    ClassLoader::_sync_systemLoaderLockContentionRate = NULL;
 PerfCounter*    ClassLoader::_sync_nonSystemLoaderLockContentionRate = NULL;
 PerfCounter*    ClassLoader::_sync_JVMFindLoadedClassLockFreeCounter = NULL;
@@ -152,6 +169,9 @@
       hpi::close(file_handle);
       // construct ClassFileStream
       if (num_read == (size_t)st.st_size) {
+        if (UsePerfData) {
+          ClassLoader::perf_sys_classfile_bytes_read()->inc(num_read);
+        }
         return new ClassFileStream(buffer, st.st_size, _dir);    // Resource allocated
       }
     }
@@ -198,6 +218,9 @@
       buffer     = NEW_RESOURCE_ARRAY(u1, filesize);
       if (!(*ReadEntry)(_zip, entry, buffer, filename)) return NULL;
   }
+  if (UsePerfData) {
+    ClassLoader::perf_sys_classfile_bytes_read()->inc(filesize);
+  }
   // return result
   return new ClassFileStream(buffer, filesize, _zip_name);    // Resource allocated
 }
@@ -825,7 +848,9 @@
   ClassFileStream* stream = NULL;
   int classpath_index = 0;
   {
-    PerfTraceTime vmtimer(perf_accumulated_time());
+    PerfClassTraceTime vmtimer(perf_sys_class_lookup_time(),
+                               ((JavaThread*) THREAD)->get_thread_stat()->perf_timers_addr(),
+                               PerfClassTraceTime::CLASS_LOAD);
     ClassPathEntry* e = _first_entry;
     while (e != NULL) {
       stream = e->open_stream(name);
@@ -890,11 +915,29 @@
     // jvmstat performance counters
     NEWPERFTICKCOUNTER(_perf_accumulated_time, SUN_CLS, "time");
     NEWPERFTICKCOUNTER(_perf_class_init_time, SUN_CLS, "classInitTime");
+    NEWPERFTICKCOUNTER(_perf_class_init_selftime, SUN_CLS, "classInitTime.self");
     NEWPERFTICKCOUNTER(_perf_class_verify_time, SUN_CLS, "classVerifyTime");
+    NEWPERFTICKCOUNTER(_perf_class_verify_selftime, SUN_CLS, "classVerifyTime.self");
     NEWPERFTICKCOUNTER(_perf_class_link_time, SUN_CLS, "classLinkedTime");
-
+    NEWPERFTICKCOUNTER(_perf_class_link_selftime, SUN_CLS, "classLinkedTime.self");
     NEWPERFEVENTCOUNTER(_perf_classes_inited, SUN_CLS, "initializedClasses");
     NEWPERFEVENTCOUNTER(_perf_classes_linked, SUN_CLS, "linkedClasses");
+    NEWPERFEVENTCOUNTER(_perf_classes_verified, SUN_CLS, "verifiedClasses");
+
+    NEWPERFTICKCOUNTER(_perf_class_parse_time, SUN_CLS, "parseClassTime");
+    NEWPERFTICKCOUNTER(_perf_class_parse_selftime, SUN_CLS, "parseClassTime.self");
+    NEWPERFTICKCOUNTER(_perf_sys_class_lookup_time, SUN_CLS, "lookupSysClassTime");
+    NEWPERFTICKCOUNTER(_perf_shared_classload_time, SUN_CLS, "sharedClassLoadTime");
+    NEWPERFTICKCOUNTER(_perf_sys_classload_time, SUN_CLS, "sysClassLoadTime");
+    NEWPERFTICKCOUNTER(_perf_app_classload_time, SUN_CLS, "appClassLoadTime");
+    NEWPERFTICKCOUNTER(_perf_app_classload_selftime, SUN_CLS, "appClassLoadTime.self");
+    NEWPERFEVENTCOUNTER(_perf_app_classload_count, SUN_CLS, "appClassLoadCount");
+    NEWPERFTICKCOUNTER(_perf_define_appclasses, SUN_CLS, "defineAppClasses");
+    NEWPERFTICKCOUNTER(_perf_define_appclass_time, SUN_CLS, "defineAppClassTime");
+    NEWPERFTICKCOUNTER(_perf_define_appclass_selftime, SUN_CLS, "defineAppClassTime.self");
+    NEWPERFBYTECOUNTER(_perf_app_classfile_bytes_read, SUN_CLS, "appClassBytes");
+    NEWPERFBYTECOUNTER(_perf_sys_classfile_bytes_read, SUN_CLS, "sysClassBytes");
+
 
     // The following performance counters are added for measuring the impact
     // of the bug fix of 6365597. They are mainly focused on finding out
--- a/src/share/vm/classfile/classLoader.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/classLoader.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -149,9 +149,26 @@
   static PerfCounter* _perf_accumulated_time;
   static PerfCounter* _perf_classes_inited;
   static PerfCounter* _perf_class_init_time;
+  static PerfCounter* _perf_class_init_selftime;
+  static PerfCounter* _perf_classes_verified;
   static PerfCounter* _perf_class_verify_time;
+  static PerfCounter* _perf_class_verify_selftime;
   static PerfCounter* _perf_classes_linked;
   static PerfCounter* _perf_class_link_time;
+  static PerfCounter* _perf_class_link_selftime;
+  static PerfCounter* _perf_class_parse_time;
+  static PerfCounter* _perf_class_parse_selftime;
+  static PerfCounter* _perf_sys_class_lookup_time;
+  static PerfCounter* _perf_shared_classload_time;
+  static PerfCounter* _perf_sys_classload_time;
+  static PerfCounter* _perf_app_classload_time;
+  static PerfCounter* _perf_app_classload_selftime;
+  static PerfCounter* _perf_app_classload_count;
+  static PerfCounter* _perf_define_appclasses;
+  static PerfCounter* _perf_define_appclass_time;
+  static PerfCounter* _perf_define_appclass_selftime;
+  static PerfCounter* _perf_app_classfile_bytes_read;
+  static PerfCounter* _perf_sys_classfile_bytes_read;
 
   static PerfCounter* _sync_systemLoaderLockContentionRate;
   static PerfCounter* _sync_nonSystemLoaderLockContentionRate;
@@ -196,12 +213,29 @@
   static void print_bootclasspath();
 
   // Timing
-  static PerfCounter* perf_accumulated_time()  { return _perf_accumulated_time; }
-  static PerfCounter* perf_classes_inited()    { return _perf_classes_inited; }
-  static PerfCounter* perf_class_init_time()   { return _perf_class_init_time; }
-  static PerfCounter* perf_class_verify_time() { return _perf_class_verify_time; }
-  static PerfCounter* perf_classes_linked()    { return _perf_classes_linked; }
-  static PerfCounter* perf_class_link_time() { return _perf_class_link_time; }
+  static PerfCounter* perf_accumulated_time()         { return _perf_accumulated_time; }
+  static PerfCounter* perf_classes_inited()           { return _perf_classes_inited; }
+  static PerfCounter* perf_class_init_time()          { return _perf_class_init_time; }
+  static PerfCounter* perf_class_init_selftime()      { return _perf_class_init_selftime; }
+  static PerfCounter* perf_classes_verified()         { return _perf_classes_verified; }
+  static PerfCounter* perf_class_verify_time()        { return _perf_class_verify_time; }
+  static PerfCounter* perf_class_verify_selftime()    { return _perf_class_verify_selftime; }
+  static PerfCounter* perf_classes_linked()           { return _perf_classes_linked; }
+  static PerfCounter* perf_class_link_time()          { return _perf_class_link_time; }
+  static PerfCounter* perf_class_link_selftime()      { return _perf_class_link_selftime; }
+  static PerfCounter* perf_class_parse_time()         { return _perf_class_parse_time; }
+  static PerfCounter* perf_class_parse_selftime()     { return _perf_class_parse_selftime; }
+  static PerfCounter* perf_sys_class_lookup_time()    { return _perf_sys_class_lookup_time; }
+  static PerfCounter* perf_shared_classload_time()    { return _perf_shared_classload_time; }
+  static PerfCounter* perf_sys_classload_time()       { return _perf_sys_classload_time; }
+  static PerfCounter* perf_app_classload_time()       { return _perf_app_classload_time; }
+  static PerfCounter* perf_app_classload_selftime()   { return _perf_app_classload_selftime; }
+  static PerfCounter* perf_app_classload_count()      { return _perf_app_classload_count; }
+  static PerfCounter* perf_define_appclasses()        { return _perf_define_appclasses; }
+  static PerfCounter* perf_define_appclass_time()     { return _perf_define_appclass_time; }
+  static PerfCounter* perf_define_appclass_selftime() { return _perf_define_appclass_selftime; }
+  static PerfCounter* perf_app_classfile_bytes_read() { return _perf_app_classfile_bytes_read; }
+  static PerfCounter* perf_sys_classfile_bytes_read() { return _perf_sys_classfile_bytes_read; }
 
   // Record how often system loader lock object is contended
   static PerfCounter* sync_systemLoaderLockContentionRate() {
@@ -307,3 +341,118 @@
   static int  compile_the_world_counter() { return _compile_the_world_counter; }
 #endif //PRODUCT
 };
+
+// PerfClassTraceTime is used to measure time for class loading related events.
+// This class tracks cumulative time and exclusive time for specific event types.
+// During the execution of one event, other event types (e.g. class loading and
+// resolution) as well as recursive calls of the same event type could happen.
+// Only one elapsed timer (cumulative) and one thread-local self timer (exclusive)
+// (i.e. only one event type) are active at a time even multiple PerfClassTraceTime
+// instances have been created as multiple events are happening.
+class PerfClassTraceTime {
+  public:
+    enum {
+       CLASS_LOAD   = 0,
+       PARSE_CLASS  = 1,
+       CLASS_LINK   = 2,
+       CLASS_VERIFY = 3,
+       CLASS_CLINIT = 4,
+       DEFINE_CLASS = 5,
+       EVENT_TYPE_COUNT = 6
+    };
+  protected:
+    // _t tracks time from initialization to destruction of this timer instance
+    // including time for all other event types, and recursive calls of this type.
+    // When a timer is called recursively, the elapsedTimer _t would not be used.
+    elapsedTimer     _t;
+    PerfLongCounter* _timep;
+    PerfLongCounter* _selftimep;
+    PerfLongCounter* _eventp;
+    // pointer to thread-local recursion counter and timer array
+    // The thread_local timers track cumulative time for specific event types
+    // exclusive of time for other event types, but including recursive calls
+    // of the same type.
+    int*             _recursion_counters;
+    elapsedTimer*    _timers;
+    int              _event_type;
+    int              _prev_active_event;
+
+  public:
+
+    inline PerfClassTraceTime(PerfLongCounter* timep,     /* counter incremented with inclusive time */
+                              PerfLongCounter* selftimep, /* counter incremented with exclusive time */
+                              PerfLongCounter* eventp,    /* event counter */
+                              int* recursion_counters,    /* thread-local recursion counter array */
+                              elapsedTimer* timers,       /* thread-local timer array */
+                              int type                    /* event type */ ) :
+        _timep(timep), _selftimep(selftimep), _eventp(eventp), _recursion_counters(recursion_counters), _timers(timers), _event_type(type) {
+      initialize();
+    }
+
+    inline PerfClassTraceTime(PerfLongCounter* timep,     /* counter incremented with inclusive time */
+                              elapsedTimer* timers,       /* thread-local timer array */
+                              int type                    /* event type */ ) :
+        _timep(timep), _selftimep(NULL), _eventp(NULL), _recursion_counters(NULL), _timers(timers), _event_type(type) {
+      initialize();
+    }
+
+    void initialize() {
+      if (!UsePerfData) return;
+
+      if (_eventp != NULL) {
+        // increment the event counter
+        _eventp->inc();
+      }
+
+      // stop the current active thread-local timer to measure inclusive time
+      _prev_active_event = -1;
+      for (int i=0; i < EVENT_TYPE_COUNT; i++) {
+         if (_timers[i].is_active()) {
+           assert(_prev_active_event == -1, "should have only one active timer");
+           _prev_active_event = i;
+           _timers[i].stop();
+         }
+      }
+
+      if (_recursion_counters == NULL || (_recursion_counters[_event_type])++ == 0) {
+        // start the inclusive timer if not recursively called
+        _t.start();
+      }
+
+      // start thread-local timer of the given event type
+      if (!_timers[_event_type].is_active()) {
+        _timers[_event_type].start();
+      }
+    }
+
+    inline void suspend() { _t.stop(); _timers[_event_type].stop(); }
+    inline void resume()  { _t.start(); _timers[_event_type].start(); }
+
+    ~PerfClassTraceTime() {
+      if (!UsePerfData) return;
+
+      // stop the thread-local timer as the event completes
+      // and resume the thread-local timer of the event next on the stack
+      _timers[_event_type].stop();
+      jlong selftime = _timers[_event_type].ticks();
+
+      if (_prev_active_event >= 0) {
+        _timers[_prev_active_event].start();
+      }
+
+      if (_recursion_counters != NULL && --(_recursion_counters[_event_type]) > 0) return;
+
+      // increment the counters only on the leaf call
+      _t.stop();
+      _timep->inc(_t.ticks());
+      if (_selftimep != NULL) {
+        _selftimep->inc(selftime);
+      }
+      // add all class loading related event selftime to the accumulated time counter
+      ClassLoader::perf_accumulated_time()->inc(selftime);
+
+      // reset the timer
+      _timers[_event_type].reset();
+    }
+};
+
--- a/src/share/vm/classfile/javaClasses.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/javaClasses.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1229,10 +1229,13 @@
 
     // Compiled java method case.
     if (decode_offset != 0) {
+      bool dummy_reexecute = false;
       DebugInfoReadStream stream(nm, decode_offset);
       decode_offset = stream.read_int();
       method = (methodOop)nm->oop_at(stream.read_int());
-      bci = stream.read_bci();
+      //fill_in_stack_trace does not need the reexecute information which is designed
+      //for the deopt to reexecute
+      bci = stream.read_bci_and_reexecute(dummy_reexecute);
     } else {
       if (fr.is_first_frame()) break;
       address pc = fr.pc();
--- a/src/share/vm/classfile/systemDictionary.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/systemDictionary.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1306,13 +1306,18 @@
 instanceKlassHandle SystemDictionary::load_instance_class(symbolHandle class_name, Handle class_loader, TRAPS) {
   instanceKlassHandle nh = instanceKlassHandle(); // null Handle
   if (class_loader.is_null()) {
+
     // Search the shared system dictionary for classes preloaded into the
     // shared spaces.
     instanceKlassHandle k;
-    k = load_shared_class(class_name, class_loader, THREAD);
+    {
+      PerfTraceTime vmtimer(ClassLoader::perf_shared_classload_time());
+      k = load_shared_class(class_name, class_loader, THREAD);
+    }
 
     if (k.is_null()) {
       // Use VM class loader
+      PerfTraceTime vmtimer(ClassLoader::perf_sys_classload_time());
       k = ClassLoader::load_classfile(class_name, CHECK_(nh));
     }
 
@@ -1334,6 +1339,16 @@
     // Use user specified class loader to load class. Call loadClass operation on class_loader.
     ResourceMark rm(THREAD);
 
+    assert(THREAD->is_Java_thread(), "must be a JavaThread");
+    JavaThread* jt = (JavaThread*) THREAD;
+
+    PerfClassTraceTime vmtimer(ClassLoader::perf_app_classload_time(),
+                               ClassLoader::perf_app_classload_selftime(),
+                               ClassLoader::perf_app_classload_count(),
+                               jt->get_thread_stat()->perf_recursion_counts_addr(),
+                               jt->get_thread_stat()->perf_timers_addr(),
+                               PerfClassTraceTime::CLASS_LOAD);
+
     Handle s = java_lang_String::create_from_symbol(class_name, CHECK_(nh));
     // Translate to external class name format, i.e., convert '/' chars to '.'
     Handle string = java_lang_String::externalize_classname(s, CHECK_(nh));
--- a/src/share/vm/classfile/verifier.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/verifier.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/vmSymbols.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/classfile/vmSymbols.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -513,9 +513,6 @@
 //
 // for Emacs: (let ((c-backslash-column 120) (c-backslash-max-column 120)) (c-backslash-region (point) (point-max) nil t))
 #define VM_INTRINSICS_DO(do_intrinsic, do_class, do_name, do_signature, do_alias)                                       \
-  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
-  /*    (symbol object_initializer_name defined above) */                                                               \
-                                                                                                                        \
   do_intrinsic(_hashCode,                 java_lang_Object,       hashCode_name, void_int_signature,             F_R)   \
    do_name(     hashCode_name,                                   "hashCode")                                            \
   do_intrinsic(_getClass,                 java_lang_Object,       getClass_name, void_class_signature,           F_R)   \
@@ -635,9 +632,6 @@
   do_intrinsic(_equalsC,                  java_util_Arrays,       equals_name,    equalsC_signature,             F_S)   \
    do_signature(equalsC_signature,                               "([C[C)Z")                                             \
                                                                                                                         \
-  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
-  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
-                                                                                                                        \
   do_intrinsic(_compareTo,                java_lang_String,       compareTo_name, string_int_signature,          F_R)   \
    do_name(     compareTo_name,                                  "compareTo")                                           \
   do_intrinsic(_indexOf,                  java_lang_String,       indexOf_name, string_int_signature,            F_R)   \
@@ -656,8 +650,6 @@
    do_name(     attemptUpdate_name,                                 "attemptUpdate")                                    \
    do_signature(attemptUpdate_signature,                            "(JJ)Z")                                            \
                                                                                                                         \
-  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
-                                                                                                                        \
   /* support for sun.misc.Unsafe */                                                                                     \
   do_class(sun_misc_Unsafe,               "sun/misc/Unsafe")                                                            \
                                                                                                                         \
@@ -819,10 +811,22 @@
    do_name(     prefetchReadStatic_name,                         "prefetchReadStatic")                                  \
   do_intrinsic(_prefetchWriteStatic,      sun_misc_Unsafe,        prefetchWriteStatic_name, prefetch_signature,  F_SN)  \
    do_name(     prefetchWriteStatic_name,                        "prefetchWriteStatic")                                 \
+    /*== LAST_COMPILER_INLINE*/                                                                                         \
+    /*the compiler does have special inlining code for these; bytecode inline is just fine */                           \
+                                                                                                                        \
+  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
+                                                                                                                        \
+  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
+  /*    (symbol object_initializer_name defined above) */                                                               \
+                                                                                                                        \
+  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
+  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
+                                                                                                                        \
     /*end*/
 
 
 
+
 // Class vmSymbols
 
 class vmSymbols: AllStatic {
@@ -935,6 +939,7 @@
     #undef VM_INTRINSIC_ENUM
 
     ID_LIMIT,
+    LAST_COMPILER_INLINE = _prefetchWriteStatic,
     FIRST_ID = _none + 1
   };
 
@@ -972,4 +977,7 @@
   static Flags              flags_for(ID id);
 
   static const char* short_name_as_C_string(ID id, char* buf, int size);
+
+  // Access to intrinsic methods:
+  static methodOop method_for(ID id);
 };
--- a/src/share/vm/code/debugInfo.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/debugInfo.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -255,7 +255,8 @@
   ScopeValue* read_object_value();
   ScopeValue* get_cached_object();
   // BCI encoding is mostly unsigned, but -1 is a distinguished value
-  int read_bci() { return read_int() + InvocationEntryBci; }
+  // Decoding based on encoding: bci = InvocationEntryBci + read_int()/2; reexecute = read_int()%2 == 1 ? true : false;
+  int read_bci_and_reexecute(bool& reexecute) { int i = read_int(); reexecute = (i & 1) ? true : false; return (i >> 1) + InvocationEntryBci; }
 };
 
 // DebugInfoWriteStream specializes CompressedWriteStream for
@@ -268,5 +269,6 @@
  public:
   DebugInfoWriteStream(DebugInformationRecorder* recorder, int initial_size);
   void write_handle(jobject h);
-  void write_bci(int bci) { write_int(bci - InvocationEntryBci); }
+  //Encoding bci and reexecute into one word as (bci - InvocationEntryBci)*2 + reexecute
+  void write_bci_and_reexecute(int bci, bool reexecute) { write_int(((bci - InvocationEntryBci) << 1) + (reexecute ? 1 : 0)); }
 };
--- a/src/share/vm/code/debugInfoRec.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/debugInfoRec.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -280,6 +280,7 @@
 void DebugInformationRecorder::describe_scope(int         pc_offset,
                                               ciMethod*   method,
                                               int         bci,
+                                              bool        reexecute,
                                               DebugToken* locals,
                                               DebugToken* expressions,
                                               DebugToken* monitors) {
@@ -297,7 +298,7 @@
   // serialize scope
   jobject method_enc = (method == NULL)? NULL: method->encoding();
   stream()->write_int(oop_recorder()->find_index(method_enc));
-  stream()->write_bci(bci);
+  stream()->write_bci_and_reexecute(bci, reexecute);
   assert(method == NULL ||
          (method->is_native() && bci == 0) ||
          (!method->is_native() && 0 <= bci && bci < method->code_size()) ||
--- a/src/share/vm/code/debugInfoRec.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/debugInfoRec.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -87,6 +87,7 @@
   void describe_scope(int         pc_offset,
                       ciMethod*   method,
                       int         bci,
+                      bool        reexecute,
                       DebugToken* locals      = NULL,
                       DebugToken* expressions = NULL,
                       DebugToken* monitors    = NULL);
--- a/src/share/vm/code/scopeDesc.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/scopeDesc.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -46,6 +46,7 @@
   _decode_offset = parent->_sender_decode_offset;
   _objects       = parent->_objects;
   decode_body();
+  assert(_reexecute == false, "reexecute not allowed");
 }
 
 
@@ -56,6 +57,7 @@
     _sender_decode_offset = DebugInformationRecorder::serialized_null;
     _method = methodHandle(_code->method());
     _bci = InvocationEntryBci;
+    _reexecute = false;
     _locals_decode_offset = DebugInformationRecorder::serialized_null;
     _expressions_decode_offset = DebugInformationRecorder::serialized_null;
     _monitors_decode_offset = DebugInformationRecorder::serialized_null;
@@ -65,7 +67,8 @@
 
     _sender_decode_offset = stream->read_int();
     _method = methodHandle((methodOop) stream->read_oop());
-    _bci    = stream->read_bci();
+    _bci    = stream->read_bci_and_reexecute(_reexecute);
+
     // decode offsets for body and sender
     _locals_decode_offset      = stream->read_int();
     _expressions_decode_offset = stream->read_int();
@@ -170,6 +173,7 @@
     st->print("ScopeDesc[%d]@" PTR_FORMAT " ", _decode_offset, _code->instructions_begin());
     st->print_cr(" offset:     %d",    _decode_offset);
     st->print_cr(" bci:        %d",    bci());
+    st->print_cr(" reexecute:  %s",    should_reexecute() ? "true" : "false");
     st->print_cr(" locals:     %d",    _locals_decode_offset);
     st->print_cr(" stack:      %d",    _expressions_decode_offset);
     st->print_cr(" monitor:    %d",    _monitors_decode_offset);
--- a/src/share/vm/code/scopeDesc.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/scopeDesc.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -39,7 +39,8 @@
     DebugInfoReadStream buffer(code, pc_desc->scope_decode_offset());
     int ignore_sender = buffer.read_int();
     _method           = methodOop(buffer.read_oop());
-    _bci              = buffer.read_bci();
+    bool dummy_reexecute; //only methodOop and bci are needed!
+    _bci              = buffer.read_bci_and_reexecute(dummy_reexecute);
   }
 
   methodOop method() { return _method; }
@@ -60,8 +61,9 @@
   ScopeDesc(const nmethod* code, int decode_offset);
 
   // JVM state
-  methodHandle method() const { return _method; }
-  int          bci()    const { return _bci;    }
+  methodHandle method()   const { return _method; }
+  int          bci()      const { return _bci;    }
+  bool should_reexecute() const { return _reexecute; }
 
   GrowableArray<ScopeValue*>*   locals();
   GrowableArray<ScopeValue*>*   expressions();
@@ -86,6 +88,7 @@
   // JVM state
   methodHandle  _method;
   int           _bci;
+  bool          _reexecute;
 
   // Decoding offsets
   int _decode_offset;
--- a/src/share/vm/code/vtableStubs.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/code/vtableStubs.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/compiler/compileBroker.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/compiler/compileBroker.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/compiler/oopMap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/compiler/oopMap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -379,7 +379,15 @@
         if ( loc != NULL ) {
           oop *base_loc    = fr->oopmapreg_to_location(omv.content_reg(), reg_map);
           oop *derived_loc = loc;
-          derived_oop_fn(base_loc, derived_loc);
+          oop val = *base_loc;
+          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
+            // Ignore NULL oops and decoded NULL narrow oops which
+            // equal to Universe::narrow_oop_base when a narrow oop
+            // implicit null check is used in compiled code.
+            // The narrow_oop_base could be NULL or be the address
+            // of the page below heap depending on compressed oops mode.
+          } else
+            derived_oop_fn(base_loc, derived_loc);
         }
         oms.next();
       }  while (!oms.is_done());
@@ -394,6 +402,15 @@
       oop* loc = fr->oopmapreg_to_location(omv.reg(),reg_map);
       if ( loc != NULL ) {
         if ( omv.type() == OopMapValue::oop_value ) {
+          oop val = *loc;
+          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
+            // Ignore NULL oops and decoded NULL narrow oops which
+            // equal to Universe::narrow_oop_base when a narrow oop
+            // implicit null check is used in compiled code.
+            // The narrow_oop_base could be NULL or be the address
+            // of the page below heap depending on compressed oops mode.
+            continue;
+          }
 #ifdef ASSERT
           if ((((uintptr_t)loc & (sizeof(*loc)-1)) != 0) ||
              !Universe::heap()->is_in_or_null(*loc)) {
@@ -410,6 +427,8 @@
 #endif // ASSERT
           oop_fn->do_oop(loc);
         } else if ( omv.type() == OopMapValue::value_value ) {
+          assert((*loc) == (oop)NULL || !Universe::is_narrow_oop_base(*loc),
+                 "found invalid value pointer");
           value_fn->do_oop(loc);
         } else if ( omv.type() == OopMapValue::narrowoop_value ) {
           narrowOop *nl = (narrowOop*)loc;
--- a/src/share/vm/compiler/oopMap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/compiler/oopMap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -233,6 +233,10 @@
   int heap_size() const;
   void copy_to(address addr);
 
+  // Methods oops_do() and all_do() filter out NULL oops and
+  // oop == Universe::narrow_oop_base() before passing oops
+  // to closures.
+
   // Iterates through frame for a compiled method
   static void oops_do            (const frame* fr,
                                   const RegisterMap* reg_map, OopClosure* f);
--- a/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -42,35 +42,40 @@
     BufferLength = 1024
   };
 
-  oop          *_buffer[BufferLength];
-  oop         **_buffer_top;
-  oop         **_buffer_curr;
+  StarTask  _buffer[BufferLength];
+  StarTask* _buffer_top;
+  StarTask* _buffer_curr;
 
-  OopClosure  *_oc;
-  double       _closure_app_seconds;
+  OopClosure* _oc;
+  double      _closure_app_seconds;
 
   void process_buffer () {
-
     double start = os::elapsedTime();
-    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
-      _oc->do_oop(*curr);
+    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
+      if (curr->is_narrow()) {
+        assert(UseCompressedOops, "Error");
+        _oc->do_oop((narrowOop*)(*curr));
+      } else {
+        _oc->do_oop((oop*)(*curr));
+      }
     }
     _buffer_curr = _buffer;
     _closure_app_seconds += (os::elapsedTime() - start);
   }
 
-public:
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop *p) {
+  template <class T> inline void do_oop_work(T* p) {
     if (_buffer_curr == _buffer_top) {
       process_buffer();
     }
-
-    *_buffer_curr = p;
+    StarTask new_ref(p);
+    *_buffer_curr = new_ref;
     ++_buffer_curr;
   }
+
+public:
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
+
   void done () {
     if (_buffer_curr > _buffer) {
       process_buffer();
@@ -88,18 +93,17 @@
 class BufferingOopsInGenClosure: public OopsInGenClosure {
   BufferingOopClosure _boc;
   OopsInGenClosure* _oc;
-public:
+ protected:
+  template <class T> inline void do_oop_work(T* p) {
+    assert(generation()->is_in_reserved((void*)p), "Must be in!");
+    _boc.do_oop(p);
+  }
+ public:
   BufferingOopsInGenClosure(OopsInGenClosure *oc) :
     _boc(oc), _oc(oc) {}
 
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  virtual void do_oop(oop* p) {
-    assert(generation()->is_in_reserved(p), "Must be in!");
-    _boc.do_oop(p);
-  }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
 
   void done() {
     _boc.done();
@@ -130,14 +134,14 @@
     BufferLength = 1024
   };
 
-  oop                      *_buffer[BufferLength];
-  oop                     **_buffer_top;
-  oop                     **_buffer_curr;
+  StarTask     _buffer[BufferLength];
+  StarTask*    _buffer_top;
+  StarTask*    _buffer_curr;
 
-  HeapRegion               *_hr_buffer[BufferLength];
-  HeapRegion              **_hr_curr;
+  HeapRegion*  _hr_buffer[BufferLength];
+  HeapRegion** _hr_curr;
 
-  OopsInHeapRegionClosure  *_oc;
+  OopsInHeapRegionClosure*  _oc;
   double                    _closure_app_seconds;
 
   void process_buffer () {
@@ -146,15 +150,20 @@
            "the two lengths should be the same");
 
     double start = os::elapsedTime();
-    HeapRegion **hr_curr = _hr_buffer;
-    HeapRegion *hr_prev = NULL;
-    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
-      HeapRegion *region = *hr_curr;
+    HeapRegion** hr_curr = _hr_buffer;
+    HeapRegion*  hr_prev = NULL;
+    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
+      HeapRegion* region = *hr_curr;
       if (region != hr_prev) {
         _oc->set_region(region);
         hr_prev = region;
       }
-      _oc->do_oop(*curr);
+      if (curr->is_narrow()) {
+        assert(UseCompressedOops, "Error");
+        _oc->do_oop((narrowOop*)(*curr));
+      } else {
+        _oc->do_oop((oop*)(*curr));
+      }
       ++hr_curr;
     }
     _buffer_curr = _buffer;
@@ -163,17 +172,16 @@
   }
 
 public:
-  virtual void do_oop(narrowOop *p) {
-    guarantee(false, "NYI");
-  }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
 
-  virtual void do_oop(oop *p) {
+  template <class T> void do_oop_work(T* p) {
     if (_buffer_curr == _buffer_top) {
       assert(_hr_curr > _hr_buffer, "_hr_curr should be consistent with _buffer_curr");
       process_buffer();
     }
-
-    *_buffer_curr = p;
+    StarTask new_ref(p);
+    *_buffer_curr = new_ref;
     ++_buffer_curr;
     *_hr_curr = _from;
     ++_hr_curr;
--- a/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,21 @@
 #include "incls/_precompiled.incl"
 #include "incls/_concurrentG1Refine.cpp.incl"
 
+// Possible sizes for the card counts cache: odd primes that roughly double in size.
+// (See jvmtiTagMap.cpp).
+int ConcurrentG1Refine::_cc_cache_sizes[] = {
+        16381,    32771,    76831,    150001,   307261,
+       614563,  1228891,  2457733,   4915219,  9830479,
+     19660831, 39321619, 78643219, 157286461,       -1
+  };
+
 ConcurrentG1Refine::ConcurrentG1Refine() :
-  _card_counts(NULL), _cur_card_count_histo(NULL), _cum_card_count_histo(NULL),
+  _card_counts(NULL), _card_epochs(NULL),
+  _n_card_counts(0), _max_n_card_counts(0),
+  _cache_size_index(0), _expand_card_counts(false),
   _hot_cache(NULL),
   _def_use_cache(false), _use_cache(false),
-  _n_periods(0), _total_cards(0), _total_travs(0),
+  _n_periods(0),
   _threads(NULL), _n_threads(0)
 {
   if (G1ConcRefine) {
@@ -57,32 +67,51 @@
 }
 
 void ConcurrentG1Refine::init() {
-  if (G1ConcRSLogCacheSize > 0 || G1ConcRSCountTraversals) {
-    G1CollectedHeap* g1h = G1CollectedHeap::heap();
-    _n_card_counts =
-      (unsigned) (g1h->g1_reserved_obj_bytes() >> CardTableModRefBS::card_shift);
-    _card_counts = NEW_C_HEAP_ARRAY(unsigned char, _n_card_counts);
-    for (size_t i = 0; i < _n_card_counts; i++) _card_counts[i] = 0;
-    ModRefBarrierSet* bs = g1h->mr_bs();
+  if (G1ConcRSLogCacheSize > 0) {
+    _g1h = G1CollectedHeap::heap();
+    _max_n_card_counts =
+      (unsigned) (_g1h->g1_reserved_obj_bytes() >> CardTableModRefBS::card_shift);
+
+    size_t max_card_num = ((size_t)1 << (sizeof(unsigned)*BitsPerByte-1)) - 1;
+    guarantee(_max_n_card_counts < max_card_num, "card_num representation");
+
+    int desired = _max_n_card_counts / InitialCacheFraction;
+    for (_cache_size_index = 0;
+              _cc_cache_sizes[_cache_size_index] >= 0; _cache_size_index++) {
+      if (_cc_cache_sizes[_cache_size_index] >= desired) break;
+    }
+    _cache_size_index = MAX2(0, (_cache_size_index - 1));
+
+    int initial_size = _cc_cache_sizes[_cache_size_index];
+    if (initial_size < 0) initial_size = _max_n_card_counts;
+
+    // Make sure we don't go bigger than we will ever need
+    _n_card_counts = MIN2((unsigned) initial_size, _max_n_card_counts);
+
+    _card_counts = NEW_C_HEAP_ARRAY(CardCountCacheEntry, _n_card_counts);
+    _card_epochs = NEW_C_HEAP_ARRAY(CardEpochCacheEntry, _n_card_counts);
+
+    Copy::fill_to_bytes(&_card_counts[0],
+                        _n_card_counts * sizeof(CardCountCacheEntry));
+    Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry));
+
+    ModRefBarrierSet* bs = _g1h->mr_bs();
     guarantee(bs->is_a(BarrierSet::CardTableModRef), "Precondition");
-    CardTableModRefBS* ctbs = (CardTableModRefBS*)bs;
-    _ct_bot = ctbs->byte_for_const(g1h->reserved_region().start());
-    if (G1ConcRSCountTraversals) {
-      _cur_card_count_histo = NEW_C_HEAP_ARRAY(unsigned, 256);
-      _cum_card_count_histo = NEW_C_HEAP_ARRAY(unsigned, 256);
-      for (int i = 0; i < 256; i++) {
-        _cur_card_count_histo[i] = 0;
-        _cum_card_count_histo[i] = 0;
-      }
-    }
-  }
-  if (G1ConcRSLogCacheSize > 0) {
+    _ct_bs = (CardTableModRefBS*)bs;
+    _ct_bot = _ct_bs->byte_for_const(_g1h->reserved_region().start());
+
     _def_use_cache = true;
     _use_cache = true;
     _hot_cache_size = (1 << G1ConcRSLogCacheSize);
     _hot_cache = NEW_C_HEAP_ARRAY(jbyte*, _hot_cache_size);
     _n_hot = 0;
     _hot_cache_idx = 0;
+
+    // For refining the cards in the hot cache in parallel
+    int n_workers = (ParallelGCThreads > 0 ?
+                        _g1h->workers()->total_workers() : 1);
+    _hot_cache_par_chunk_size = MAX2(1, _hot_cache_size / n_workers);
+    _hot_cache_par_claimed_idx = 0;
   }
 }
 
@@ -95,15 +124,11 @@
 }
 
 ConcurrentG1Refine::~ConcurrentG1Refine() {
-  if (G1ConcRSLogCacheSize > 0 || G1ConcRSCountTraversals) {
+  if (G1ConcRSLogCacheSize > 0) {
     assert(_card_counts != NULL, "Logic");
-    FREE_C_HEAP_ARRAY(unsigned char, _card_counts);
-    assert(_cur_card_count_histo != NULL, "Logic");
-    FREE_C_HEAP_ARRAY(unsigned, _cur_card_count_histo);
-    assert(_cum_card_count_histo != NULL, "Logic");
-    FREE_C_HEAP_ARRAY(unsigned, _cum_card_count_histo);
-  }
-  if (G1ConcRSLogCacheSize > 0) {
+    FREE_C_HEAP_ARRAY(CardCountCacheEntry, _card_counts);
+    assert(_card_epochs != NULL, "Logic");
+    FREE_C_HEAP_ARRAY(CardEpochCacheEntry, _card_epochs);
     assert(_hot_cache != NULL, "Logic");
     FREE_C_HEAP_ARRAY(jbyte*, _hot_cache);
   }
@@ -123,165 +148,232 @@
   }
 }
 
-
-int ConcurrentG1Refine::add_card_count(jbyte* card_ptr) {
-  size_t card_num = (card_ptr - _ct_bot);
-  guarantee(0 <= card_num && card_num < _n_card_counts, "Bounds");
-  unsigned char cnt = _card_counts[card_num];
-  if (cnt < 255) _card_counts[card_num]++;
-  return cnt;
-  _total_travs++;
+bool ConcurrentG1Refine::is_young_card(jbyte* card_ptr) {
+  HeapWord* start = _ct_bs->addr_for(card_ptr);
+  HeapRegion* r = _g1h->heap_region_containing(start);
+  if (r != NULL && r->is_young()) {
+    return true;
+  }
+  // This card is not associated with a heap region
+  // so can't be young.
+  return false;
 }
 
-jbyte* ConcurrentG1Refine::cache_insert(jbyte* card_ptr) {
-  int count = add_card_count(card_ptr);
-  // Count previously unvisited cards.
-  if (count == 0) _total_cards++;
-  // We'll assume a traversal unless we store it in the cache.
+jbyte* ConcurrentG1Refine::add_card_count(jbyte* card_ptr, int* count, bool* defer) {
+  unsigned new_card_num = ptr_2_card_num(card_ptr);
+  unsigned bucket = hash(new_card_num);
+  assert(0 <= bucket && bucket < _n_card_counts, "Bounds");
+
+  CardCountCacheEntry* count_ptr = &_card_counts[bucket];
+  CardEpochCacheEntry* epoch_ptr = &_card_epochs[bucket];
+
+  // We have to construct a new entry if we haven't updated the counts
+  // during the current period, or if the count was updated for a
+  // different card number.
+  unsigned int new_epoch = (unsigned int) _n_periods;
+  julong new_epoch_entry = make_epoch_entry(new_card_num, new_epoch);
+
+  while (true) {
+    // Fetch the previous epoch value
+    julong prev_epoch_entry = epoch_ptr->_value;
+    julong cas_res;
+
+    if (extract_epoch(prev_epoch_entry) != new_epoch) {
+      // This entry has not yet been updated during this period.
+      // Note: we update the epoch value atomically to ensure
+      // that there is only one winner that updates the cached
+      // card_ptr value even though all the refine threads share
+      // the same epoch value.
+
+      cas_res = (julong) Atomic::cmpxchg((jlong) new_epoch_entry,
+                                         (volatile jlong*)&epoch_ptr->_value,
+                                         (jlong) prev_epoch_entry);
+
+      if (cas_res == prev_epoch_entry) {
+        // We have successfully won the race to update the
+        // epoch and card_num value. Make it look like the
+        // count and eviction count were previously cleared.
+        count_ptr->_count = 1;
+        count_ptr->_evict_count = 0;
+        *count = 0;
+        // We can defer the processing of card_ptr
+        *defer = true;
+        return card_ptr;
+      }
+      // We did not win the race to update the epoch field, so some other
+      // thread must have done it. The value that gets returned by CAS
+      // should be the new epoch value.
+      assert(extract_epoch(cas_res) == new_epoch, "unexpected epoch");
+      // We could 'continue' here or just re-read the previous epoch value
+      prev_epoch_entry = epoch_ptr->_value;
+    }
+
+    // The epoch entry for card_ptr has been updated during this period.
+    unsigned old_card_num = extract_card_num(prev_epoch_entry);
+
+    // The card count that will be returned to caller
+    *count = count_ptr->_count;
+
+    // Are we updating the count for the same card?
+    if (new_card_num == old_card_num) {
+      // Same card - just update the count. We could have more than one
+      // thread racing to update count for the current card. It should be
+      // OK not to use a CAS as the only penalty should be some missed
+      // increments of the count which delays identifying the card as "hot".
+
+      if (*count < max_jubyte) count_ptr->_count++;
+      // We can defer the processing of card_ptr
+      *defer = true;
+      return card_ptr;
+    }
+
+    // Different card - evict old card info
+    if (count_ptr->_evict_count < max_jubyte) count_ptr->_evict_count++;
+    if (count_ptr->_evict_count > G1CardCountCacheExpandThreshold) {
+      // Trigger a resize the next time we clear
+      _expand_card_counts = true;
+    }
+
+    cas_res = (julong) Atomic::cmpxchg((jlong) new_epoch_entry,
+                                       (volatile jlong*)&epoch_ptr->_value,
+                                       (jlong) prev_epoch_entry);
+
+    if (cas_res == prev_epoch_entry) {
+      // We successfully updated the card num value in the epoch entry
+      count_ptr->_count = 0; // initialize counter for new card num
+
+      // Even though the region containg the card at old_card_num was not
+      // in the young list when old_card_num was recorded in the epoch
+      // cache it could have been added to the free list and subsequently
+      // added to the young list in the intervening time. If the evicted
+      // card is in a young region just return the card_ptr and the evicted
+      // card will not be cleaned. See CR 6817995.
+
+      jbyte* old_card_ptr = card_num_2_ptr(old_card_num);
+      if (is_young_card(old_card_ptr)) {
+        *count = 0;
+        // We can defer the processing of card_ptr
+        *defer = true;
+        return card_ptr;
+      }
+
+      // We do not want to defer processing of card_ptr in this case
+      // (we need to refine old_card_ptr and card_ptr)
+      *defer = false;
+      return old_card_ptr;
+    }
+    // Someone else beat us - try again.
+  }
+}
+
+jbyte* ConcurrentG1Refine::cache_insert(jbyte* card_ptr, bool* defer) {
+  int count;
+  jbyte* cached_ptr = add_card_count(card_ptr, &count, defer);
+  assert(cached_ptr != NULL, "bad cached card ptr");
+  assert(!is_young_card(cached_ptr), "shouldn't get a card in young region");
+
+  // The card pointer we obtained from card count cache is not hot
+  // so do not store it in the cache; return it for immediate
+  // refining.
   if (count < G1ConcRSHotCardLimit) {
-    _total_travs++;
-    return card_ptr;
+    return cached_ptr;
   }
-  // Otherwise, it's hot.
+
+  // Otherwise, the pointer we got from the _card_counts is hot.
   jbyte* res = NULL;
   MutexLockerEx x(HotCardCache_lock, Mutex::_no_safepoint_check_flag);
   if (_n_hot == _hot_cache_size) {
-    _total_travs++;
     res = _hot_cache[_hot_cache_idx];
     _n_hot--;
   }
   // Now _n_hot < _hot_cache_size, and we can insert at _hot_cache_idx.
-  _hot_cache[_hot_cache_idx] = card_ptr;
+  _hot_cache[_hot_cache_idx] = cached_ptr;
   _hot_cache_idx++;
   if (_hot_cache_idx == _hot_cache_size) _hot_cache_idx = 0;
   _n_hot++;
+
+  if (res != NULL) {
+    // Even though the region containg res was not in the young list
+    // when it was recorded in the hot cache it could have been added
+    // to the free list and subsequently added to the young list in
+    // the intervening time. If res is in a young region, return NULL
+    // so that res is not cleaned. See CR 6817995.
+
+    if (is_young_card(res)) {
+      res = NULL;
+    }
+  }
+
   return res;
 }
 
-
 void ConcurrentG1Refine::clean_up_cache(int worker_i, G1RemSet* g1rs) {
   assert(!use_cache(), "cache should be disabled");
-  int start_ind = _hot_cache_idx-1;
-  for (int i = 0; i < _n_hot; i++) {
-    int ind = start_ind - i;
-    if (ind < 0) ind = ind + _hot_cache_size;
-    jbyte* entry = _hot_cache[ind];
-    if (entry != NULL) {
-      g1rs->concurrentRefineOneCard(entry, worker_i);
+  int start_idx;
+
+  while ((start_idx = _hot_cache_par_claimed_idx) < _n_hot) { // read once
+    int end_idx = start_idx + _hot_cache_par_chunk_size;
+
+    if (start_idx ==
+        Atomic::cmpxchg(end_idx, &_hot_cache_par_claimed_idx, start_idx)) {
+      // The current worker has successfully claimed the chunk [start_idx..end_idx)
+      end_idx = MIN2(end_idx, _n_hot);
+      for (int i = start_idx; i < end_idx; i++) {
+        jbyte* entry = _hot_cache[i];
+        if (entry != NULL) {
+          g1rs->concurrentRefineOneCard(entry, worker_i);
+        }
+      }
     }
   }
-  _n_hot = 0;
-  _hot_cache_idx = 0;
+}
+
+void ConcurrentG1Refine::expand_card_count_cache() {
+  if (_n_card_counts < _max_n_card_counts) {
+    int new_idx = _cache_size_index+1;
+    int new_size = _cc_cache_sizes[new_idx];
+    if (new_size < 0) new_size = _max_n_card_counts;
+
+    // Make sure we don't go bigger than we will ever need
+    new_size = MIN2((unsigned) new_size, _max_n_card_counts);
+
+    // Expand the card count and card epoch tables
+    if (new_size > (int)_n_card_counts) {
+      // We can just free and allocate a new array as we're
+      // not interested in preserving the contents
+      assert(_card_counts != NULL, "Logic!");
+      assert(_card_epochs != NULL, "Logic!");
+      FREE_C_HEAP_ARRAY(CardCountCacheEntry, _card_counts);
+      FREE_C_HEAP_ARRAY(CardEpochCacheEntry, _card_epochs);
+      _n_card_counts = new_size;
+      _card_counts = NEW_C_HEAP_ARRAY(CardCountCacheEntry, _n_card_counts);
+      _card_epochs = NEW_C_HEAP_ARRAY(CardEpochCacheEntry, _n_card_counts);
+      _cache_size_index = new_idx;
+    }
+  }
 }
 
 void ConcurrentG1Refine::clear_and_record_card_counts() {
-  if (G1ConcRSLogCacheSize == 0 && !G1ConcRSCountTraversals) return;
+  if (G1ConcRSLogCacheSize == 0) return;
+
+#ifndef PRODUCT
+  double start = os::elapsedTime();
+#endif
+
+  if (_expand_card_counts) {
+    expand_card_count_cache();
+    _expand_card_counts = false;
+    // Only need to clear the epochs.
+    Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry));
+  }
+
+  int this_epoch = (int) _n_periods;
+  assert((this_epoch+1) <= max_jint, "to many periods");
+  // Update epoch
   _n_periods++;
-  if (G1ConcRSCountTraversals) {
-    for (size_t i = 0; i < _n_card_counts; i++) {
-      unsigned char bucket = _card_counts[i];
-      _cur_card_count_histo[bucket]++;
-      _card_counts[i] = 0;
-    }
-    gclog_or_tty->print_cr("Card counts:");
-    for (int i = 0; i < 256; i++) {
-      if (_cur_card_count_histo[i] > 0) {
-        gclog_or_tty->print_cr("  %3d: %9d", i, _cur_card_count_histo[i]);
-        _cum_card_count_histo[i] += _cur_card_count_histo[i];
-        _cur_card_count_histo[i] = 0;
-      }
-    }
-  } else {
-    assert(G1ConcRSLogCacheSize > 0, "Logic");
-    Copy::fill_to_words((HeapWord*)(&_card_counts[0]),
-                        _n_card_counts / HeapWordSize);
-  }
+
+#ifndef PRODUCT
+  double elapsed = os::elapsedTime() - start;
+  _g1h->g1_policy()->record_cc_clear_time(elapsed * 1000.0);
+#endif
 }
-
-void
-ConcurrentG1Refine::
-print_card_count_histo_range(unsigned* histo, int from, int to,
-                             float& cum_card_pct,
-                             float& cum_travs_pct) {
-  unsigned cards = 0;
-  unsigned travs = 0;
-  guarantee(to <= 256, "Precondition");
-  for (int i = from; i < to-1; i++) {
-    cards += histo[i];
-    travs += histo[i] * i;
-  }
-  if (to == 256) {
-    unsigned histo_card_sum = 0;
-    unsigned histo_trav_sum = 0;
-    for (int i = 1; i < 255; i++) {
-      histo_trav_sum += histo[i] * i;
-    }
-    cards += histo[255];
-    // correct traversals for the last one.
-    unsigned travs_255 = (unsigned) (_total_travs - histo_trav_sum);
-    travs += travs_255;
-
-  } else {
-    cards += histo[to-1];
-    travs += histo[to-1] * (to-1);
-  }
-  float fperiods = (float)_n_periods;
-  float f_tot_cards = (float)_total_cards/fperiods;
-  float f_tot_travs = (float)_total_travs/fperiods;
-  if (cards > 0) {
-    float fcards = (float)cards/fperiods;
-    float ftravs = (float)travs/fperiods;
-    if (to == 256) {
-      gclog_or_tty->print(" %4d-       %10.2f%10.2f", from, fcards, ftravs);
-    } else {
-      gclog_or_tty->print(" %4d-%4d   %10.2f%10.2f", from, to-1, fcards, ftravs);
-    }
-    float pct_cards = fcards*100.0/f_tot_cards;
-    cum_card_pct += pct_cards;
-    float pct_travs = ftravs*100.0/f_tot_travs;
-    cum_travs_pct += pct_travs;
-    gclog_or_tty->print_cr("%10.2f%10.2f%10.2f%10.2f",
-                  pct_cards, cum_card_pct,
-                  pct_travs, cum_travs_pct);
-  }
-}
-
-void ConcurrentG1Refine::print_final_card_counts() {
-  if (!G1ConcRSCountTraversals) return;
-
-  gclog_or_tty->print_cr("Did %d total traversals of %d distinct cards.",
-                _total_travs, _total_cards);
-  float fperiods = (float)_n_periods;
-  gclog_or_tty->print_cr("  This is an average of %8.2f traversals, %8.2f cards, "
-                "per collection.", (float)_total_travs/fperiods,
-                (float)_total_cards/fperiods);
-  gclog_or_tty->print_cr("  This is an average of %8.2f traversals/distinct "
-                "dirty card.\n",
-                _total_cards > 0 ?
-                (float)_total_travs/(float)_total_cards : 0.0);
-
-
-  gclog_or_tty->print_cr("Histogram:\n\n%10s   %10s%10s%10s%10s%10s%10s",
-                "range", "# cards", "# travs", "% cards", "(cum)",
-                "% travs", "(cum)");
-  gclog_or_tty->print_cr("------------------------------------------------------------"
-                "-------------");
-  float cum_cards_pct = 0.0;
-  float cum_travs_pct = 0.0;
-  for (int i = 1; i < 10; i++) {
-    print_card_count_histo_range(_cum_card_count_histo, i, i+1,
-                                 cum_cards_pct, cum_travs_pct);
-  }
-  for (int i = 10; i < 100; i += 10) {
-    print_card_count_histo_range(_cum_card_count_histo, i, i+10,
-                                 cum_cards_pct, cum_travs_pct);
-  }
-  print_card_count_histo_range(_cum_card_count_histo, 100, 150,
-                               cum_cards_pct, cum_travs_pct);
-  print_card_count_histo_range(_cum_card_count_histo, 150, 200,
-                               cum_cards_pct, cum_travs_pct);
-  print_card_count_histo_range(_cum_card_count_histo, 150, 255,
-                               cum_cards_pct, cum_travs_pct);
-  print_card_count_histo_range(_cum_card_count_histo, 255, 256,
-                               cum_cards_pct, cum_travs_pct);
-}
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -29,29 +29,117 @@
 class ConcurrentG1Refine: public CHeapObj {
   ConcurrentG1RefineThread** _threads;
   int _n_threads;
+
   // The cache for card refinement.
-  bool     _use_cache;
-  bool     _def_use_cache;
-  size_t _n_periods;
-  size_t _total_cards;
-  size_t _total_travs;
+  bool   _use_cache;
+  bool   _def_use_cache;
 
-  unsigned char*  _card_counts;
+  size_t _n_periods;    // Used as clearing epoch
+
+  // An evicting cache of the number of times each card
+  // is accessed. Reduces, but does not eliminate, the amount
+  // of duplicated processing of dirty cards.
+
+  enum SomePrivateConstants {
+    epoch_bits           = 32,
+    card_num_shift       = epoch_bits,
+    epoch_mask           = AllBits,
+    card_num_mask        = AllBits,
+
+    // The initial cache size is approximately this fraction
+    // of a maximal cache (i.e. the size needed for all cards
+    // in the heap)
+    InitialCacheFraction = 512
+  };
+
+  const static julong card_num_mask_in_place =
+                        (julong) card_num_mask << card_num_shift;
+
+  typedef struct {
+    julong _value;      // |  card_num   |  epoch   |
+  } CardEpochCacheEntry;
+
+  julong make_epoch_entry(unsigned int card_num, unsigned int epoch) {
+    assert(0 <= card_num && card_num < _max_n_card_counts, "Bounds");
+    assert(0 <= epoch && epoch <= _n_periods, "must be");
+
+    return ((julong) card_num << card_num_shift) | epoch;
+  }
+
+  unsigned int extract_epoch(julong v) {
+    return (v & epoch_mask);
+  }
+
+  unsigned int extract_card_num(julong v) {
+    return (v & card_num_mask_in_place) >> card_num_shift;
+  }
+
+  typedef struct {
+    unsigned char _count;
+    unsigned char _evict_count;
+  } CardCountCacheEntry;
+
+  CardCountCacheEntry* _card_counts;
+  CardEpochCacheEntry* _card_epochs;
+
+  // The current number of buckets in the card count cache
   unsigned _n_card_counts;
+
+  // The max number of buckets required for the number of
+  // cards for the entire reserved heap
+  unsigned _max_n_card_counts;
+
+  // Possible sizes of the cache: odd primes that roughly double in size.
+  // (See jvmtiTagMap.cpp).
+  static int _cc_cache_sizes[];
+
+  // The index in _cc_cache_sizes corresponding to the size of
+  // _card_counts.
+  int _cache_size_index;
+
+  bool _expand_card_counts;
+
   const jbyte* _ct_bot;
-  unsigned* _cur_card_count_histo;
-  unsigned* _cum_card_count_histo;
-  jbyte**  _hot_cache;
-  int      _hot_cache_size;
-  int      _n_hot;
-  int      _hot_cache_idx;
+
+  jbyte**      _hot_cache;
+  int          _hot_cache_size;
+  int          _n_hot;
+  int          _hot_cache_idx;
+
+  int          _hot_cache_par_chunk_size;
+  volatile int _hot_cache_par_claimed_idx;
+
+  // Needed to workaround 6817995
+  CardTableModRefBS* _ct_bs;
+  G1CollectedHeap*   _g1h;
+
+  // Expands the array that holds the card counts to the next size up
+  void expand_card_count_cache();
+
+  // hash a given key (index of card_ptr) with the specified size
+  static unsigned int hash(size_t key, int size) {
+    return (unsigned int) key % size;
+  }
+
+  // hash a given key (index of card_ptr)
+  unsigned int hash(size_t key) {
+    return hash(key, _n_card_counts);
+  }
+
+  unsigned ptr_2_card_num(jbyte* card_ptr) {
+    return (unsigned) (card_ptr - _ct_bot);
+  }
+
+  jbyte* card_num_2_ptr(unsigned card_num) {
+    return (jbyte*) (_ct_bot + card_num);
+  }
 
   // Returns the count of this card after incrementing it.
-  int add_card_count(jbyte* card_ptr);
+  jbyte* add_card_count(jbyte* card_ptr, int* count, bool* defer);
 
-  void print_card_count_histo_range(unsigned* histo, int from, int to,
-                                    float& cum_card_pct,
-                                    float& cum_travs_pct);
+  // Returns true if this card is in a young region
+  bool is_young_card(jbyte* card_ptr);
+
  public:
   ConcurrentG1Refine();
   ~ConcurrentG1Refine();
@@ -65,11 +153,16 @@
   // If this is the first entry for the slot, writes into the cache and
   // returns NULL.  If it causes an eviction, returns the evicted pointer.
   // Otherwise, its a cache hit, and returns NULL.
-  jbyte* cache_insert(jbyte* card_ptr);
+  jbyte* cache_insert(jbyte* card_ptr, bool* defer);
 
   // Process the cached entries.
   void clean_up_cache(int worker_i, G1RemSet* g1rs);
 
+  // Set up for parallel processing of the cards in the hot cache
+  void clear_hot_cache_claimed_index() {
+    _hot_cache_par_claimed_idx = 0;
+  }
+
   // Discard entries in the hot cache.
   void clear_hot_cache() {
     _hot_cache_idx = 0; _n_hot = 0;
@@ -84,7 +177,6 @@
   }
 
   void clear_and_record_card_counts();
-  void print_final_card_counts();
 
   static size_t thread_num();
 };
--- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -104,17 +104,17 @@
     double start_vtime_sec; // only used when G1SmoothConcRefine is on
     int prev_buffer_num; // only used when G1SmoothConcRefine is on
     // This thread activation threshold
-    int threshold = DCQBarrierProcessCompletedThreshold * _worker_id;
+    int threshold = G1UpdateBufferQueueProcessingThreshold * _worker_id;
     // Next thread activation threshold
-    int next_threshold = threshold + DCQBarrierProcessCompletedThreshold;
-    int deactivation_threshold = MAX2<int>(threshold - DCQBarrierProcessCompletedThreshold / 2, 0);
+    int next_threshold = threshold + G1UpdateBufferQueueProcessingThreshold;
+    int deactivation_threshold = MAX2<int>(threshold - G1UpdateBufferQueueProcessingThreshold / 2, 0);
 
     if (G1SmoothConcRefine) {
       lower_limit = 0;
       start_vtime_sec = os::elapsedVTime();
       prev_buffer_num = (int) dcqs.completed_buffers_num();
     } else {
-      lower_limit = DCQBarrierProcessCompletedThreshold / 4; // For now.
+      lower_limit = G1UpdateBufferQueueProcessingThreshold / 4; // For now.
     }
     while (dcqs.apply_closure_to_completed_buffer(_worker_id + _worker_id_offset, lower_limit)) {
       double end_vtime_sec;
--- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -452,13 +452,10 @@
   _regionStack.allocate(G1MarkRegionStackSize);
 
   // Create & start a ConcurrentMark thread.
-  if (G1ConcMark) {
-    _cmThread = new ConcurrentMarkThread(this);
-    assert(cmThread() != NULL, "CM Thread should have been created");
-    assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
-  } else {
-    _cmThread = NULL;
-  }
+  _cmThread = new ConcurrentMarkThread(this);
+  assert(cmThread() != NULL, "CM Thread should have been created");
+  assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
+
   _g1h = G1CollectedHeap::heap();
   assert(CGC_lock != NULL, "Where's the CGC_lock?");
   assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
@@ -783,18 +780,18 @@
                      bool do_barrier) : _cm(cm), _g1h(g1h),
                                         _do_barrier(do_barrier) { }
 
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  virtual void do_oop(oop* p) {
-    oop thisOop = *p;
-    if (thisOop != NULL) {
-      assert(thisOop->is_oop() || thisOop->mark() == NULL,
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      assert(obj->is_oop() || obj->mark() == NULL,
              "expected an oop, possibly with mark word displaced");
-      HeapWord* addr = (HeapWord*)thisOop;
+      HeapWord* addr = (HeapWord*)obj;
       if (_g1h->is_in_g1_reserved(addr)) {
-        _cm->grayRoot(thisOop);
+        _cm->grayRoot(obj);
       }
     }
     if (_do_barrier) {
@@ -850,16 +847,6 @@
   double start = os::elapsedTime();
   GCOverheadReporter::recordSTWStart(start);
 
-  // If there has not been a GC[n-1] since last GC[n] cycle completed,
-  // precede our marking with a collection of all
-  // younger generations to keep floating garbage to a minimum.
-  // YSR: we won't do this for now -- it's an optimization to be
-  // done post-beta.
-
-  // YSR:    ignoring weak refs for now; will do at bug fixing stage
-  // EVM:    assert(discoveredRefsAreClear());
-
-
   G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
   g1p->record_concurrent_mark_init_start();
   checkpointRootsInitialPre();
@@ -1135,6 +1122,13 @@
     return;
   }
 
+  if (VerifyDuringGC) {
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(before)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(true, false, true);
+  }
+
   G1CollectorPolicy* g1p = g1h->g1_policy();
   g1p->record_concurrent_mark_remark_start();
 
@@ -1159,10 +1153,12 @@
     JavaThread::satb_mark_queue_set().set_active_all_threads(false);
 
     if (VerifyDuringGC) {
-      g1h->prepare_for_verify();
-      g1h->verify(/* allow_dirty */      true,
-                  /* silent */           false,
-                  /* use_prev_marking */ false);
+      HandleMark hm;  // handle scope
+      gclog_or_tty->print(" VerifyDuringGC:(after)");
+      Universe::heap()->prepare_for_verify();
+      Universe::heap()->verify(/* allow_dirty */      true,
+                               /* silent */           false,
+                               /* use_prev_marking */ false);
     }
   }
 
@@ -1658,6 +1654,15 @@
     return;
   }
 
+  if (VerifyDuringGC) {
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(before)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(/* allow dirty  */ true,
+                     /* silent       */ false,
+                     /* prev marking */ true);
+  }
+
   _cleanup_co_tracker.disable();
 
   G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
@@ -1790,10 +1795,12 @@
   g1h->increment_total_collections();
 
   if (VerifyDuringGC) {
-    g1h->prepare_for_verify();
-    g1h->verify(/* allow_dirty */      true,
-                /* silent */           false,
-                /* use_prev_marking */ true);
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(after)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(/* allow dirty  */ true,
+                     /* silent       */ false,
+                     /* prev marking */ true);
   }
 }
 
@@ -1852,12 +1859,11 @@
     _g1(g1), _cm(cm),
     _bitMap(bitMap) {}
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop thisOop = *p;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    oop thisOop = oopDesc::load_decode_heap_oop(p);
     HeapWord* addr = (HeapWord*)thisOop;
     if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
       _bitMap->mark(addr);
@@ -2016,12 +2022,11 @@
   ReachablePrinterOopClosure(CMBitMapRO* bitmap, outputStream* out) :
     _bitmap(bitmap), _g1h(G1CollectedHeap::heap()), _out(out) { }
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop         obj = *p;
+  void do_oop(narrowOop* p) { do_oop_work(p); }
+  void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    oop         obj = oopDesc::load_decode_heap_oop(p);
     const char* str = NULL;
     const char* str2 = "";
 
@@ -2163,6 +2168,7 @@
 
 
   HeapWord* objAddr = (HeapWord*) obj;
+  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
   if (_g1h->is_in_g1_reserved(objAddr)) {
     tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
     HeapRegion* hr = _g1h->heap_region_containing(obj);
@@ -2380,7 +2386,7 @@
     }
   }
 
-  bool drain() {
+  template <class T> bool drain() {
     while (_ms_ind > 0) {
       oop obj = pop();
       assert(obj != NULL, "Since index was non-zero.");
@@ -2394,9 +2400,8 @@
         }
         // Now process this portion of this one.
         int lim = MIN2(next_arr_ind, len);
-        assert(!UseCompressedOops, "This needs to be fixed");
         for (int j = arr_ind; j < lim; j++) {
-          do_oop(aobj->obj_at_addr<oop>(j));
+          do_oop(aobj->objArrayOopDesc::obj_at_addr<T>(j));
         }
 
       } else {
@@ -2423,13 +2428,13 @@
     FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
   }
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop obj = *p;
-    if (obj == NULL) return;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (oopDesc::is_null(heap_oop)) return;
+    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
     if (obj->is_forwarded()) {
       // If the object has already been forwarded, we have to make sure
       // that it's marked.  So follow the forwarding pointer.  Note that
@@ -2478,7 +2483,11 @@
     oop obj = oop(addr);
     if (!obj->is_forwarded()) {
       if (!_oop_cl.push(obj)) return false;
-      if (!_oop_cl.drain()) return false;
+      if (UseCompressedOops) {
+        if (!_oop_cl.drain<narrowOop>()) return false;
+      } else {
+        if (!_oop_cl.drain<oop>()) return false;
+      }
     }
     // Otherwise...
     return true;
@@ -2636,9 +2645,6 @@
 
 // abandon current marking iteration due to a Full GC
 void ConcurrentMark::abort() {
-  // If we're not marking, nothing to do.
-  if (!G1ConcMark) return;
-
   // Clear all marks to force marking thread to do nothing
   _nextMarkBitMap->clearAll();
   // Empty mark stack
@@ -2814,14 +2820,14 @@
   CMTask*            _task;
 
 public:
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
     tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant" );
-
-    oop obj = *p;
+    tmp_guarantee_CM( !_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(), "invariant" );
+
+    oop obj = oopDesc::load_decode_heap_oop(p);
     if (_cm->verbose_high())
       gclog_or_tty->print_cr("[%d] we're looking at location "
                              "*"PTR_FORMAT" = "PTR_FORMAT,
@@ -2967,6 +2973,7 @@
   ++_refs_reached;
 
   HeapWord* objAddr = (HeapWord*) obj;
+  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
   if (_g1h->is_in_g1_reserved(objAddr)) {
     tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
     HeapRegion* hr =  _g1h->heap_region_containing(obj);
@@ -3030,6 +3037,7 @@
 void CMTask::push(oop obj) {
   HeapWord* objAddr = (HeapWord*) obj;
   tmp_guarantee_CM( _g1h->is_in_g1_reserved(objAddr), "invariant" );
+  tmp_guarantee_CM( !_g1h->heap_region_containing(objAddr)->is_on_free_list(), "invariant" );
   tmp_guarantee_CM( !_g1h->is_obj_ill(obj), "invariant" );
   tmp_guarantee_CM( _nextMarkBitMap->isMarked(objAddr), "invariant" );
 
@@ -3275,6 +3283,8 @@
 
       tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) obj),
                         "invariant" );
+      tmp_guarantee_CM( !_g1h->heap_region_containing(obj)->is_on_free_list(),
+                        "invariant" );
 
       scan_object(obj);
 
--- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -763,6 +763,7 @@
   CMBitMap*                   _nextMarkBitMap;
   // the task queue of this task
   CMTaskQueue*                _task_queue;
+private:
   // the task queue set---needed for stealing
   CMTaskQueueSet*             _task_queues;
   // indicates whether the task has been claimed---this is only  for
--- a/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -80,8 +80,8 @@
                                    int max_completed_queue,
                                    Mutex* lock, PtrQueueSet* fl_owner) {
   PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue, fl_owner);
-  set_buffer_size(DCQBarrierQueueBufferSize);
-  set_process_completed_threshold(DCQBarrierProcessCompletedThreshold);
+  set_buffer_size(G1UpdateBufferSize);
+  set_process_completed_threshold(G1UpdateBufferQueueProcessingThreshold);
 
   _shared_dirty_card_queue.set_lock(lock);
   _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon);
--- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -424,7 +424,7 @@
       while (n <= next_boundary) {
         q = n;
         oop obj = oop(q);
-        if (obj->klass() == NULL) return q;
+        if (obj->klass_or_null() == NULL) return q;
         n += obj->size();
       }
       assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
@@ -436,7 +436,7 @@
       while (n <= next_boundary) {
         q = n;
         oop obj = oop(q);
-        if (obj->klass() == NULL) return q;
+        if (obj->klass_or_null() == NULL) return q;
         n += _sp->block_size(q);
       }
       assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
--- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -96,14 +96,14 @@
     while (n <= addr) {
       q = n;
       oop obj = oop(q);
-      if (obj->klass() == NULL) return q;
+      if (obj->klass_or_null() == NULL) return q;
       n += obj->size();
     }
   } else {
     while (n <= addr) {
       q = n;
       oop obj = oop(q);
-      if (obj->klass() == NULL) return q;
+      if (obj->klass_or_null() == NULL) return q;
       n += _sp->block_size(q);
     }
   }
@@ -115,7 +115,7 @@
 inline HeapWord*
 G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q,
                                                      const void* addr) {
-  if (oop(q)->klass() == NULL) return q;
+  if (oop(q)->klass_or_null() == NULL) return q;
   HeapWord* n = q + _sp->block_size(q);
   // In the normal case, where the query "addr" is a card boundary, and the
   // offset table chunks are the same size as cards, the block starting at
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1591,7 +1591,7 @@
 
   JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,
                                                 DirtyCardQ_FL_lock,
-                                                G1DirtyCardQueueMax,
+                                                G1UpdateBufferQueueMaxLength,
                                                 Shared_DirtyCardQ_lock);
 
   if (G1DeferredRSUpdate) {
@@ -1637,6 +1637,9 @@
 
 void G1CollectedHeap::iterate_dirty_card_closure(bool concurrent,
                                                  int worker_i) {
+  // Clean cards in the hot card cache
+  concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set());
+
   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
   int n_completed_buffers = 0;
   while (dcqs.apply_closure_to_completed_buffer(worker_i, 0, true)) {
@@ -1645,9 +1648,6 @@
   g1_policy()->record_update_rs_processed_buffers(worker_i,
                                                   (double) n_completed_buffers);
   dcqs.clear_n_completed_buffers();
-  // Finish up the queue...
-  if (worker_i == 0) concurrent_g1_refine()->clean_up_cache(worker_i,
-                                                            g1_rem_set());
   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");
 }
 
@@ -1658,8 +1658,15 @@
   assert(Heap_lock->owner() != NULL,
          "Should be owned on this thread's behalf.");
   size_t result = _summary_bytes_used;
-  if (_cur_alloc_region != NULL)
-    result += _cur_alloc_region->used();
+  // Read only once in case it is set to NULL concurrently
+  HeapRegion* hr = _cur_alloc_region;
+  if (hr != NULL)
+    result += hr->used();
+  return result;
+}
+
+size_t G1CollectedHeap::used_unlocked() const {
+  size_t result = _summary_bytes_used;
   return result;
 }
 
@@ -2133,13 +2140,13 @@
   VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {
     g1h = _g1h;
   }
-  void do_oop(narrowOop *p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop *p) {
-    oop obj = *p;
-    assert(obj == NULL || !g1h->is_obj_dead(obj),
-           "Dead object referenced by a not dead object");
+  void do_oop(narrowOop *p) { do_oop_work(p); }
+  void do_oop(      oop *p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T *p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
+    guarantee(obj == NULL || !g1h->is_obj_dead(obj),
+              "Dead object referenced by a not dead object");
   }
 };
 
@@ -2206,8 +2213,10 @@
   // use_prev_marking == true  -> use "prev" marking information,
   // use_prev_marking == false -> use "next" marking information
   VerifyRegionClosure(bool allow_dirty, bool par, bool use_prev_marking)
-    : _allow_dirty(allow_dirty), _par(par),
+    : _allow_dirty(allow_dirty),
+      _par(par),
       _use_prev_marking(use_prev_marking) {}
+
   bool doHeapRegion(HeapRegion* r) {
     guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
               "Should be unclaimed at verify points.");
@@ -2231,18 +2240,16 @@
   // use_prev_marking == true  -> use "prev" marking information,
   // use_prev_marking == false -> use "next" marking information
   VerifyRootsClosure(bool use_prev_marking) :
-    _g1h(G1CollectedHeap::heap()), _failures(false),
+    _g1h(G1CollectedHeap::heap()),
+    _failures(false),
     _use_prev_marking(use_prev_marking) { }
 
   bool failures() { return _failures; }
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop obj = *p;
-    if (obj != NULL) {
+  template <class T> void do_oop_nv(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
       if (_g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
         gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
                                "points to dead obj "PTR_FORMAT, p, (void*) obj);
@@ -2251,6 +2258,9 @@
       }
     }
   }
+
+  void do_oop(oop* p)       { do_oop_nv(p); }
+  void do_oop(narrowOop* p) { do_oop_nv(p); }
 };
 
 // This is the task used for parallel heap verification.
@@ -2267,7 +2277,8 @@
   G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty,
                   bool use_prev_marking) :
     AbstractGangTask("Parallel verify task"),
-    _g1h(g1h), _allow_dirty(allow_dirty),
+    _g1h(g1h),
+    _allow_dirty(allow_dirty),
     _use_prev_marking(use_prev_marking) { }
 
   void work(int worker_i) {
@@ -2342,7 +2353,7 @@
 void G1CollectedHeap::print_on(outputStream* st, bool extended) const {
   st->print(" %-20s", "garbage-first heap");
   st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",
-            capacity()/K, used()/K);
+            capacity()/K, used_unlocked()/K);
   st->print(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")",
             _g1_storage.low_boundary(),
             _g1_storage.high(),
@@ -2403,8 +2414,6 @@
 }
 
 void G1CollectedHeap::print_tracing_info() const {
-  concurrent_g1_refine()->print_final_card_counts();
-
   // We'll overload this to mean "trace GC pause statistics."
   if (TraceGen0Time || TraceGen1Time) {
     // The "G1CollectorPolicy" is keeping track of these stats, so delegate
@@ -2479,12 +2488,10 @@
 
 void
 G1CollectedHeap::doConcurrentMark() {
-  if (G1ConcMark) {
-    MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
-    if (!_cmThread->in_progress()) {
-      _cmThread->set_started();
-      CGC_lock->notify();
-    }
+  MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+  if (!_cmThread->in_progress()) {
+    _cmThread->set_started();
+    CGC_lock->notify();
   }
 }
 
@@ -2561,9 +2568,11 @@
                           "Not enough space for young surv words summary.");
   }
   memset(_surviving_young_words, 0, array_length * sizeof(size_t));
+#ifdef ASSERT
   for (size_t i = 0;  i < array_length; ++i) {
-    guarantee( _surviving_young_words[i] == 0, "invariant" );
-  }
+    assert( _surviving_young_words[i] == 0, "memset above" );
+  }
+#endif // !ASSERT
 }
 
 void
@@ -2649,7 +2658,7 @@
       COMPILER2_PRESENT(DerivedPointerTable::clear());
 
       // We want to turn off ref discovery, if necessary, and turn it back on
-      // on again later if we do.
+      // on again later if we do. XXX Dubious: why is discovery disabled?
       bool was_enabled = ref_processor()->discovery_enabled();
       if (was_enabled) ref_processor()->disable_discovery();
 
@@ -2662,9 +2671,6 @@
       double start_time_sec = os::elapsedTime();
       GCOverheadReporter::recordSTWStart(start_time_sec);
       size_t start_used_bytes = used();
-      if (!G1ConcMark) {
-        do_sync_mark();
-      }
 
       g1_policy()->record_collection_pause_start(start_time_sec,
                                                  start_used_bytes);
@@ -2775,6 +2781,13 @@
           g1_policy()->should_initiate_conc_mark()) {
         concurrent_mark()->checkpointRootsInitialPost();
         set_marking_started();
+        // CAUTION: after the doConcurrentMark() call below,
+        // the concurrent marking thread(s) could be running
+        // concurrently with us. Make sure that anything after
+        // this point does not assume that we are the only GC thread
+        // running. Note: of course, the actual marking work will
+        // not start until the safepoint itself is released in
+        // ConcurrentGCThread::safepoint_desynchronize().
         doConcurrentMark();
       }
 
@@ -2830,6 +2843,11 @@
   if (PrintHeapAtGC) {
     Universe::print_heap_after_gc();
   }
+  if (G1SummarizeRSetStats &&
+      (G1SummarizeRSetStatsPeriod > 0) &&
+      (total_collections() % G1SummarizeRSetStatsPeriod == 0)) {
+    g1_rem_set()->print_summary_info();
+  }
 }
 
 void G1CollectedHeap::set_gc_alloc_region(int purpose, HeapRegion* r) {
@@ -2965,6 +2983,7 @@
 
   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
     assert(_gc_alloc_regions[ap] == NULL, "invariant");
+    assert(_gc_alloc_region_counts[ap] == 0, "invariant");
 
     // Create new GC alloc regions.
     HeapRegion* alloc_region = _retained_gc_alloc_regions[ap];
@@ -2993,6 +3012,9 @@
     if (alloc_region == NULL) {
       // we will get a new GC alloc region
       alloc_region = newAllocRegionWithExpansion(ap, 0);
+    } else {
+      // the region was retained from the last collection
+      ++_gc_alloc_region_counts[ap];
     }
 
     if (alloc_region != NULL) {
@@ -3031,11 +3053,11 @@
   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
     HeapRegion* r = _gc_alloc_regions[ap];
     _retained_gc_alloc_regions[ap] = NULL;
+    _gc_alloc_region_counts[ap] = 0;
 
     if (r != NULL) {
       // we retain nothing on _gc_alloc_regions between GCs
       set_gc_alloc_region(ap, NULL);
-      _gc_alloc_region_counts[ap] = 0;
 
       if (r->is_empty()) {
         // we didn't actually allocate anything in it; let's just put
@@ -3123,10 +3145,8 @@
   G1CollectedHeap* _g1;
 public:
   G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
+  void do_oop(      oop* p) {
     oop obj = *p;
 #ifdef G1_DEBUG
     if (PrintGC && Verbose) {
@@ -3138,7 +3158,6 @@
     if (_g1->obj_in_cs(obj)) {
       assert( obj->is_forwarded(), "invariant" );
       *p = obj->forwardee();
-
 #ifdef G1_DEBUG
       gclog_or_tty->print_cr("     in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
                              (void*) obj, (void*) *p);
@@ -3155,12 +3174,12 @@
   UpdateRSetImmediate(G1CollectedHeap* g1) :
     _g1(g1), _g1_rem_set(g1->g1_rem_set()) {}
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
     assert(_from->is_in_reserved(p), "paranoia");
-    if (*p != NULL && !_from->is_survivor()) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) {
       _g1_rem_set->par_write_ref(_from, p, 0);
     }
   }
@@ -3176,12 +3195,12 @@
   UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
     _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
 
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
     assert(_from->is_in_reserved(p), "paranoia");
-    if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
+    if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
+        !_from->is_survivor()) {
       size_t card_index = _ct_bs->index_for(p);
       if (_ct_bs->mark_card_deferred(card_index)) {
         _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
@@ -3536,614 +3555,66 @@
   fill_with_object(block, free_words);
 }
 
-#define use_local_bitmaps         1
-#define verify_local_bitmaps      0
-
 #ifndef PRODUCT
-
-class GCLabBitMap;
-class GCLabBitMapClosure: public BitMapClosure {
-private:
-  ConcurrentMark* _cm;
-  GCLabBitMap*    _bitmap;
-
-public:
-  GCLabBitMapClosure(ConcurrentMark* cm,
-                     GCLabBitMap* bitmap) {
-    _cm     = cm;
-    _bitmap = bitmap;
-  }
-
-  virtual bool do_bit(size_t offset);
-};
-
-#endif // PRODUCT
-
-#define oop_buffer_length 256
-
-class GCLabBitMap: public BitMap {
-private:
-  ConcurrentMark* _cm;
-
-  int       _shifter;
-  size_t    _bitmap_word_covers_words;
-
-  // beginning of the heap
-  HeapWord* _heap_start;
-
-  // this is the actual start of the GCLab
-  HeapWord* _real_start_word;
-
-  // this is the actual end of the GCLab
-  HeapWord* _real_end_word;
-
-  // this is the first word, possibly located before the actual start
-  // of the GCLab, that corresponds to the first bit of the bitmap
-  HeapWord* _start_word;
-
-  // size of a GCLab in words
-  size_t _gclab_word_size;
-
-  static int shifter() {
-    return MinObjAlignment - 1;
-  }
-
-  // how many heap words does a single bitmap word corresponds to?
-  static size_t bitmap_word_covers_words() {
-    return BitsPerWord << shifter();
-  }
-
-  static size_t gclab_word_size() {
-    return G1ParallelGCAllocBufferSize / HeapWordSize;
-  }
-
-  static size_t bitmap_size_in_bits() {
-    size_t bits_in_bitmap = gclab_word_size() >> shifter();
-    // We are going to ensure that the beginning of a word in this
-    // bitmap also corresponds to the beginning of a word in the
-    // global marking bitmap. To handle the case where a GCLab
-    // starts from the middle of the bitmap, we need to add enough
-    // space (i.e. up to a bitmap word) to ensure that we have
-    // enough bits in the bitmap.
-    return bits_in_bitmap + BitsPerWord - 1;
-  }
-public:
-  GCLabBitMap(HeapWord* heap_start)
-    : BitMap(bitmap_size_in_bits()),
-      _cm(G1CollectedHeap::heap()->concurrent_mark()),
-      _shifter(shifter()),
-      _bitmap_word_covers_words(bitmap_word_covers_words()),
-      _heap_start(heap_start),
-      _gclab_word_size(gclab_word_size()),
-      _real_start_word(NULL),
-      _real_end_word(NULL),
-      _start_word(NULL)
-  {
-    guarantee( size_in_words() >= bitmap_size_in_words(),
-               "just making sure");
-  }
-
-  inline unsigned heapWordToOffset(HeapWord* addr) {
-    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
-    assert(offset < size(), "offset should be within bounds");
-    return offset;
-  }
-
-  inline HeapWord* offsetToHeapWord(size_t offset) {
-    HeapWord* addr =  _start_word + (offset << _shifter);
-    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
-    return addr;
-  }
-
-  bool fields_well_formed() {
-    bool ret1 = (_real_start_word == NULL) &&
-                (_real_end_word == NULL) &&
-                (_start_word == NULL);
-    if (ret1)
-      return true;
-
-    bool ret2 = _real_start_word >= _start_word &&
-      _start_word < _real_end_word &&
-      (_real_start_word + _gclab_word_size) == _real_end_word &&
-      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
-                                                              > _real_end_word;
-    return ret2;
-  }
-
-  inline bool mark(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    if (addr >= _real_start_word && addr < _real_end_word) {
-      assert(!isMarked(addr), "should not have already been marked");
-
-      // first mark it on the bitmap
-      at_put(heapWordToOffset(addr), true);
-
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  inline bool isMarked(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    return at(heapWordToOffset(addr));
-  }
-
-  void set_buffer(HeapWord* start) {
-    guarantee(use_local_bitmaps, "invariant");
-    clear();
-
-    assert(start != NULL, "invariant");
-    _real_start_word = start;
-    _real_end_word   = start + _gclab_word_size;
-
-    size_t diff =
-      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
-    _start_word = start - diff;
-
-    assert(fields_well_formed(), "invariant");
-  }
-
-#ifndef PRODUCT
-  void verify() {
-    // verify that the marks have been propagated
-    GCLabBitMapClosure cl(_cm, this);
-    iterate(&cl);
-  }
-#endif // PRODUCT
-
-  void retire() {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    if (_start_word != NULL) {
-      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
-
-      // this means that the bitmap was set up for the GCLab
-      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
-
-      mark_bitmap->mostly_disjoint_range_union(this,
-                                0, // always start from the start of the bitmap
-                                _start_word,
-                                size_in_words());
-      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
-
-#ifndef PRODUCT
-      if (use_local_bitmaps && verify_local_bitmaps)
-        verify();
-#endif // PRODUCT
-    } else {
-      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
-    }
-  }
-
-  static size_t bitmap_size_in_words() {
-    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
-  }
-};
-
-#ifndef PRODUCT
-
 bool GCLabBitMapClosure::do_bit(size_t offset) {
   HeapWord* addr = _bitmap->offsetToHeapWord(offset);
   guarantee(_cm->isMarked(oop(addr)), "it should be!");
   return true;
 }
-
 #endif // PRODUCT
 
-class G1ParGCAllocBuffer: public ParGCAllocBuffer {
-private:
-  bool        _retired;
-  bool        _during_marking;
-  GCLabBitMap _bitmap;
-
-public:
-  G1ParGCAllocBuffer() :
-    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
-    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
-    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
-    _retired(false)
-  { }
-
-  inline bool mark(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(_during_marking, "invariant");
-    return _bitmap.mark(addr);
-  }
-
-  inline void set_buf(HeapWord* buf) {
-    if (use_local_bitmaps && _during_marking)
-      _bitmap.set_buffer(buf);
-    ParGCAllocBuffer::set_buf(buf);
-    _retired = false;
-  }
-
-  inline void retire(bool end_of_gc, bool retain) {
-    if (_retired)
-      return;
-    if (use_local_bitmaps && _during_marking) {
-      _bitmap.retire();
-    }
-    ParGCAllocBuffer::retire(end_of_gc, retain);
-    _retired = true;
-  }
-};
-
-
-class G1ParScanThreadState : public StackObj {
-protected:
-  G1CollectedHeap* _g1h;
-  RefToScanQueue*  _refs;
-  DirtyCardQueue   _dcq;
-  CardTableModRefBS* _ct_bs;
-  G1RemSet* _g1_rem;
-
-  typedef GrowableArray<oop*> OverflowQueue;
-  OverflowQueue* _overflowed_refs;
-
-  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
-  ageTable           _age_table;
-
-  size_t           _alloc_buffer_waste;
-  size_t           _undo_waste;
-
-  OopsInHeapRegionClosure*      _evac_failure_cl;
-  G1ParScanHeapEvacClosure*     _evac_cl;
-  G1ParScanPartialArrayClosure* _partial_scan_cl;
-
-  int _hash_seed;
-  int _queue_num;
-
-  int _term_attempts;
+G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
+  : _g1h(g1h),
+    _refs(g1h->task_queue(queue_num)),
+    _dcq(&g1h->dirty_card_queue_set()),
+    _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
+    _g1_rem(g1h->g1_rem_set()),
+    _hash_seed(17), _queue_num(queue_num),
+    _term_attempts(0),
+    _age_table(false),
 #if G1_DETAILED_STATS
-  int _pushes, _pops, _steals, _steal_attempts;
-  int _overflow_pushes;
+    _pushes(0), _pops(0), _steals(0),
+    _steal_attempts(0),  _overflow_pushes(0),
 #endif
-
-  double _start;
-  double _start_strong_roots;
-  double _strong_roots_time;
-  double _start_term;
-  double _term_time;
-
-  // Map from young-age-index (0 == not young, 1 is youngest) to
-  // surviving words. base is what we get back from the malloc call
-  size_t* _surviving_young_words_base;
-  // this points into the array, as we use the first few entries for padding
-  size_t* _surviving_young_words;
-
-#define PADDING_ELEM_NUM (64 / sizeof(size_t))
-
-  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
-
-  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
-
-  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
-  CardTableModRefBS* ctbs()                      { return _ct_bs; }
-
-  void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
-    if (!from->is_survivor()) {
-      _g1_rem->par_write_ref(from, p, tid);
-    }
-  }
-
-  void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
-    // If the new value of the field points to the same region or
-    // is the to-space, we don't need to include it in the Rset updates.
-    if (!from->is_in_reserved(*p) && !from->is_survivor()) {
-      size_t card_index = ctbs()->index_for(p);
-      // If the card hasn't been added to the buffer, do it.
-      if (ctbs()->mark_card_deferred(card_index)) {
-        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
-      }
-    }
-  }
-
-public:
-  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
-    : _g1h(g1h),
-      _refs(g1h->task_queue(queue_num)),
-      _dcq(&g1h->dirty_card_queue_set()),
-      _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
-      _g1_rem(g1h->g1_rem_set()),
-      _hash_seed(17), _queue_num(queue_num),
-      _term_attempts(0),
-      _age_table(false),
-#if G1_DETAILED_STATS
-      _pushes(0), _pops(0), _steals(0),
-      _steal_attempts(0),  _overflow_pushes(0),
-#endif
-      _strong_roots_time(0), _term_time(0),
-      _alloc_buffer_waste(0), _undo_waste(0)
-  {
-    // we allocate G1YoungSurvRateNumRegions plus one entries, since
-    // we "sacrifice" entry 0 to keep track of surviving bytes for
-    // non-young regions (where the age is -1)
-    // We also add a few elements at the beginning and at the end in
-    // an attempt to eliminate cache contention
-    size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
-    size_t array_length = PADDING_ELEM_NUM +
-                          real_length +
-                          PADDING_ELEM_NUM;
-    _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
-    if (_surviving_young_words_base == NULL)
-      vm_exit_out_of_memory(array_length * sizeof(size_t),
-                            "Not enough space for young surv histo.");
-    _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
-    memset(_surviving_young_words, 0, real_length * sizeof(size_t));
-
-    _overflowed_refs = new OverflowQueue(10);
-
-    _start = os::elapsedTime();
-  }
-
-  ~G1ParScanThreadState() {
-    FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
-  }
-
-  RefToScanQueue*   refs()            { return _refs;             }
-  OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
-  ageTable*         age_table()       { return &_age_table;       }
-
-  G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
-    return &_alloc_buffers[purpose];
-  }
-
-  size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
-  size_t undo_waste()                            { return _undo_waste; }
-
-  void push_on_queue(oop* ref) {
-    assert(ref != NULL, "invariant");
-    assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref), "invariant");
-
-    if (!refs()->push(ref)) {
-      overflowed_refs()->push(ref);
-      IF_G1_DETAILED_STATS(note_overflow_push());
-    } else {
-      IF_G1_DETAILED_STATS(note_push());
-    }
-  }
-
-  void pop_from_queue(oop*& ref) {
-    if (!refs()->pop_local(ref)) {
-      ref = NULL;
-    } else {
-      assert(ref != NULL, "invariant");
-      assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref),
-             "invariant");
-
-      IF_G1_DETAILED_STATS(note_pop());
-    }
-  }
-
-  void pop_from_overflow_queue(oop*& ref) {
-    ref = overflowed_refs()->pop();
-  }
-
-  int refs_to_scan()                             { return refs()->size();                 }
-  int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
-
-  void update_rs(HeapRegion* from, oop* p, int tid) {
-    if (G1DeferredRSUpdate) {
-      deferred_rs_update(from, p, tid);
-    } else {
-      immediate_rs_update(from, p, tid);
-    }
-  }
-
-  HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
-
-    HeapWord* obj = NULL;
-    if (word_sz * 100 <
-        (size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
-                                                  ParallelGCBufferWastePct) {
-      G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
-      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
-      alloc_buf->retire(false, false);
-
-      HeapWord* buf =
-        _g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
-      if (buf == NULL) return NULL; // Let caller handle allocation failure.
-      // Otherwise.
-      alloc_buf->set_buf(buf);
-
-      obj = alloc_buf->allocate(word_sz);
-      assert(obj != NULL, "buffer was definitely big enough...");
-    } else {
-      obj = _g1h->par_allocate_during_gc(purpose, word_sz);
-    }
-    return obj;
-  }
-
-  HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
-    HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
-    if (obj != NULL) return obj;
-    return allocate_slow(purpose, word_sz);
-  }
-
-  void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
-    if (alloc_buffer(purpose)->contains(obj)) {
-      guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1),
-                "should contain whole object");
-      alloc_buffer(purpose)->undo_allocation(obj, word_sz);
-    } else {
-      CollectedHeap::fill_with_object(obj, word_sz);
-      add_to_undo_waste(word_sz);
-    }
-  }
-
-  void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
-    _evac_failure_cl = evac_failure_cl;
-  }
-  OopsInHeapRegionClosure* evac_failure_closure() {
-    return _evac_failure_cl;
-  }
-
-  void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
-    _evac_cl = evac_cl;
-  }
-
-  void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
-    _partial_scan_cl = partial_scan_cl;
-  }
-
-  int* hash_seed() { return &_hash_seed; }
-  int  queue_num() { return _queue_num; }
-
-  int term_attempts()   { return _term_attempts; }
-  void note_term_attempt()  { _term_attempts++; }
-
-#if G1_DETAILED_STATS
-  int pushes()          { return _pushes; }
-  int pops()            { return _pops; }
-  int steals()          { return _steals; }
-  int steal_attempts()  { return _steal_attempts; }
-  int overflow_pushes() { return _overflow_pushes; }
-
-  void note_push()          { _pushes++; }
-  void note_pop()           { _pops++; }
-  void note_steal()         { _steals++; }
-  void note_steal_attempt() { _steal_attempts++; }
-  void note_overflow_push() { _overflow_pushes++; }
-#endif
-
-  void start_strong_roots() {
-    _start_strong_roots = os::elapsedTime();
-  }
-  void end_strong_roots() {
-    _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
-  }
-  double strong_roots_time() { return _strong_roots_time; }
-
-  void start_term_time() {
-    note_term_attempt();
-    _start_term = os::elapsedTime();
-  }
-  void end_term_time() {
-    _term_time += (os::elapsedTime() - _start_term);
-  }
-  double term_time() { return _term_time; }
-
-  double elapsed() {
-    return os::elapsedTime() - _start;
-  }
-
-  size_t* surviving_young_words() {
-    // We add on to hide entry 0 which accumulates surviving words for
-    // age -1 regions (i.e. non-young ones)
-    return _surviving_young_words;
-  }
-
-  void retire_alloc_buffers() {
-    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
-      size_t waste = _alloc_buffers[ap].words_remaining();
-      add_to_alloc_buffer_waste(waste);
-      _alloc_buffers[ap].retire(true, false);
-    }
-  }
-
-private:
-  void deal_with_reference(oop* ref_to_scan) {
-    if (has_partial_array_mask(ref_to_scan)) {
-      _partial_scan_cl->do_oop_nv(ref_to_scan);
-    } else {
-      // Note: we can use "raw" versions of "region_containing" because
-      // "obj_to_scan" is definitely in the heap, and is not in a
-      // humongous region.
-      HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
-      _evac_cl->set_region(r);
-      _evac_cl->do_oop_nv(ref_to_scan);
-    }
-  }
-
-public:
-  void trim_queue() {
-    // I've replicated the loop twice, first to drain the overflow
-    // queue, second to drain the task queue. This is better than
-    // having a single loop, which checks both conditions and, inside
-    // it, either pops the overflow queue or the task queue, as each
-    // loop is tighter. Also, the decision to drain the overflow queue
-    // first is not arbitrary, as the overflow queue is not visible
-    // to the other workers, whereas the task queue is. So, we want to
-    // drain the "invisible" entries first, while allowing the other
-    // workers to potentially steal the "visible" entries.
-
-    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
-      while (overflowed_refs_to_scan() > 0) {
-        oop *ref_to_scan = NULL;
-        pop_from_overflow_queue(ref_to_scan);
-        assert(ref_to_scan != NULL, "invariant");
-        // We shouldn't have pushed it on the queue if it was not
-        // pointing into the CSet.
-        assert(ref_to_scan != NULL, "sanity");
-        assert(has_partial_array_mask(ref_to_scan) ||
-                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
-
-        deal_with_reference(ref_to_scan);
-      }
-
-      while (refs_to_scan() > 0) {
-        oop *ref_to_scan = NULL;
-        pop_from_queue(ref_to_scan);
-
-        if (ref_to_scan != NULL) {
-          // We shouldn't have pushed it on the queue if it was not
-          // pointing into the CSet.
-          assert(has_partial_array_mask(ref_to_scan) ||
-                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
-
-          deal_with_reference(ref_to_scan);
-        }
-      }
-    }
-  }
-};
+    _strong_roots_time(0), _term_time(0),
+    _alloc_buffer_waste(0), _undo_waste(0)
+{
+  // we allocate G1YoungSurvRateNumRegions plus one entries, since
+  // we "sacrifice" entry 0 to keep track of surviving bytes for
+  // non-young regions (where the age is -1)
+  // We also add a few elements at the beginning and at the end in
+  // an attempt to eliminate cache contention
+  size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
+  size_t array_length = PADDING_ELEM_NUM +
+                        real_length +
+                        PADDING_ELEM_NUM;
+  _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
+  if (_surviving_young_words_base == NULL)
+    vm_exit_out_of_memory(array_length * sizeof(size_t),
+                          "Not enough space for young surv histo.");
+  _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
+  memset(_surviving_young_words, 0, real_length * sizeof(size_t));
+
+  _overflowed_refs = new OverflowQueue(10);
+
+  _start = os::elapsedTime();
+}
 
 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   _par_scan_state(par_scan_state) { }
 
-// This closure is applied to the fields of the objects that have just been copied.
-// Should probably be made inline and moved in g1OopClosures.inline.hpp.
-void G1ParScanClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-
-  if (obj != NULL) {
-    if (_g1->in_cset_fast_test(obj)) {
-      // We're not going to even bother checking whether the object is
-      // already forwarded or not, as this usually causes an immediate
-      // stall. We'll try to prefetch the object (for write, given that
-      // we might need to install the forwarding reference) and we'll
-      // get back to it when pop it from the queue
-      Prefetch::write(obj->mark_addr(), 0);
-      Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
-
-      // slightly paranoid test; I'm trying to catch potential
-      // problems before we go into push_on_queue to know where the
-      // problem is coming from
-      assert(obj == *p, "the value of *p should not have changed");
-      _par_scan_state->push_on_queue(p);
-    } else {
-      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
-    }
-  }
-}
-
-void G1ParCopyHelper::mark_forwardee(oop* p) {
+template <class T> void G1ParCopyHelper::mark_forwardee(T* p) {
   // This is called _after_ do_oop_work has been called, hence after
   // the object has been relocated to its new location and *p points
   // to its new location.
 
-  oop thisOop = *p;
-  if (thisOop != NULL) {
-    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)),
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop)) {
+    oop obj = oopDesc::decode_heap_oop(heap_oop);
+    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(obj)),
            "shouldn't still be in the CSet if evacuation didn't fail.");
-    HeapWord* addr = (HeapWord*)thisOop;
+    HeapWord* addr = (HeapWord*)obj;
     if (_g1->is_in_g1_reserved(addr))
       _cm->grayRoot(oop(addr));
   }
@@ -4226,7 +3697,8 @@
 
     if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
       arrayOop(old)->set_length(0);
-      _par_scan_state->push_on_queue(set_partial_array_mask(old));
+      oop* old_p = set_partial_array_mask(old);
+      _par_scan_state->push_on_queue(old_p);
     } else {
       // No point in using the slower heap_region_containing() method,
       // given that we know obj is in the heap.
@@ -4240,11 +3712,11 @@
   return obj;
 }
 
-template<bool do_gen_barrier, G1Barrier barrier,
-         bool do_mark_forwardee, bool skip_cset_test>
-void G1ParCopyClosure<do_gen_barrier, barrier,
-                      do_mark_forwardee, skip_cset_test>::do_oop_work(oop* p) {
-  oop obj = *p;
+template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee, bool skip_cset_test>
+template <class T>
+void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_test>
+::do_oop_work(T* p) {
+  oop obj = oopDesc::load_decode_heap_oop(p);
   assert(barrier != G1BarrierRS || obj != NULL,
          "Precondition: G1BarrierRS implies obj is nonNull");
 
@@ -4261,9 +3733,10 @@
                            "into CS.", p, (void*) obj);
 #endif
     if (obj->is_forwarded()) {
-      *p = obj->forwardee();
+      oopDesc::encode_store_heap_oop(p, obj->forwardee());
     } else {
-      *p = copy_to_survivor_space(obj);
+      oop copy_oop = copy_to_survivor_space(obj);
+      oopDesc::encode_store_heap_oop(p, copy_oop);
     }
     // When scanning the RS, we only care about objs in CS.
     if (barrier == G1BarrierRS) {
@@ -4282,21 +3755,9 @@
 }
 
 template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p);
-
-template<class T> void G1ParScanPartialArrayClosure::process_array_chunk(
-  oop obj, int start, int end) {
-  // process our set of indices (include header in first chunk)
-  assert(start < end, "invariant");
-  T* const base      = (T*)objArrayOop(obj)->base();
-  T* const start_addr = (start == 0) ? (T*) obj : base + start;
-  T* const end_addr   = base + end;
-  MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
-  _scanner.set_region(_g1->heap_region_containing(obj));
-  obj->oop_iterate(&_scanner, mr);
-}
-
-void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
-  assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
+template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(narrowOop* p);
+
+template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
   assert(has_partial_array_mask(p), "invariant");
   oop old = clear_partial_array_mask(p);
   assert(old->is_objArray(), "must be obj array");
@@ -4316,19 +3777,19 @@
     end = start + ParGCArrayScanChunk;
     arrayOop(old)->set_length(end);
     // Push remainder.
-    _par_scan_state->push_on_queue(set_partial_array_mask(old));
+    oop* old_p = set_partial_array_mask(old);
+    assert(arrayOop(old)->length() < obj->length(), "Empty push?");
+    _par_scan_state->push_on_queue(old_p);
   } else {
     // Restore length so that the heap remains parsable in
     // case of evacuation failure.
     arrayOop(old)->set_length(end);
   }
-
+  _scanner.set_region(_g1->heap_region_containing_raw(obj));
   // process our set of indices (include header in first chunk)
-  process_array_chunk<oop>(obj, start, end);
+  obj->oop_iterate_range(&_scanner, start, end);
 }
 
-int G1ScanAndBalanceClosure::_nq = 0;
-
 class G1ParEvacuateFollowersClosure : public VoidClosure {
 protected:
   G1CollectedHeap*              _g1h;
@@ -4351,21 +3812,28 @@
   void do_void() {
     G1ParScanThreadState* pss = par_scan_state();
     while (true) {
-      oop* ref_to_scan;
       pss->trim_queue();
       IF_G1_DETAILED_STATS(pss->note_steal_attempt());
-      if (queues()->steal(pss->queue_num(),
-                          pss->hash_seed(),
-                          ref_to_scan)) {
+
+      StarTask stolen_task;
+      if (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
         IF_G1_DETAILED_STATS(pss->note_steal());
 
         // slightly paranoid tests; I'm trying to catch potential
         // problems before we go into push_on_queue to know where the
         // problem is coming from
-        assert(ref_to_scan != NULL, "invariant");
-        assert(has_partial_array_mask(ref_to_scan) ||
-                                   _g1h->obj_in_cs(*ref_to_scan), "invariant");
-        pss->push_on_queue(ref_to_scan);
+        assert((oop*)stolen_task != NULL, "Error");
+        if (stolen_task.is_narrow()) {
+          assert(UseCompressedOops, "Error");
+          narrowOop* p = (narrowOop*) stolen_task;
+          assert(has_partial_array_mask(p) ||
+                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error");
+          pss->push_on_queue(p);
+        } else {
+          oop* p = (oop*) stolen_task;
+          assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error");
+          pss->push_on_queue(p);
+        }
         continue;
       }
       pss->start_term_time();
@@ -4382,6 +3850,7 @@
   G1CollectedHeap*       _g1h;
   RefToScanQueueSet      *_queues;
   ParallelTaskTerminator _terminator;
+  int _n_workers;
 
   Mutex _stats_lock;
   Mutex* stats_lock() { return &_stats_lock; }
@@ -4397,7 +3866,8 @@
       _g1h(g1h),
       _queues(task_queues),
       _terminator(workers, _queues),
-      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true)
+      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true),
+      _n_workers(workers)
   {}
 
   RefToScanQueueSet* queues() { return _queues; }
@@ -4407,6 +3877,7 @@
   }
 
   void work(int i) {
+    if (i >= _n_workers) return;  // no work needed this round
     ResourceMark rm;
     HandleMark   hm;
 
@@ -4504,23 +3975,6 @@
 
 // *** Common G1 Evacuation Stuff
 
-class G1CountClosure: public OopsInHeapRegionClosure {
-public:
-  int n;
-  G1CountClosure() : n(0) {}
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
-    oop obj = *p;
-    assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj),
-           "Rem set closure called on non-rem-set pointer.");
-    n++;
-  }
-};
-
-static G1CountClosure count_closure;
-
 void
 G1CollectedHeap::
 g1_process_strong_roots(bool collecting_perm_gen,
@@ -4655,6 +4109,8 @@
 
   g1_rem_set()->prepare_for_oops_into_collection_set_do();
   concurrent_g1_refine()->set_use_cache(false);
+  concurrent_g1_refine()->clear_hot_cache_claimed_index();
+
   int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
   set_par_threads(n_workers);
   G1ParTask g1_par_task(this, n_workers, _task_queues);
@@ -4687,6 +4143,7 @@
   }
   g1_rem_set()->cleanup_after_oops_into_collection_set_do();
 
+  concurrent_g1_refine()->clear_hot_cache();
   concurrent_g1_refine()->set_use_cache(true);
 
   finalize_for_evac_failure();
@@ -5570,8 +5027,3 @@
 void G1CollectedHeap::g1_unimplemented() {
   // Unimplemented();
 }
-
-
-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -56,8 +56,8 @@
 #  define IF_G1_DETAILED_STATS(code)
 #endif
 
-typedef GenericTaskQueue<oop*>    RefToScanQueue;
-typedef GenericTaskQueueSet<oop*> RefToScanQueueSet;
+typedef GenericTaskQueue<StarTask>    RefToScanQueue;
+typedef GenericTaskQueueSet<StarTask> RefToScanQueueSet;
 
 typedef int RegionIdx_t;   // needs to hold [ 0..max_regions() )
 typedef int CardIdx_t;     // needs to hold [ 0..CardsPerRegion )
@@ -700,6 +700,9 @@
   size_t g1_reserved_obj_bytes() { return _g1_reserved.byte_size(); }
   virtual size_t capacity() const;
   virtual size_t used() const;
+  // This should be called when we're not holding the heap lock. The
+  // result might be a bit inaccurate.
+  size_t used_unlocked() const;
   size_t recalculate_used() const;
 #ifndef PRODUCT
   size_t recalculate_used_regions() const;
@@ -1271,6 +1274,552 @@
 
 };
 
-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***
+#define use_local_bitmaps         1
+#define verify_local_bitmaps      0
+#define oop_buffer_length       256
+
+#ifndef PRODUCT
+class GCLabBitMap;
+class GCLabBitMapClosure: public BitMapClosure {
+private:
+  ConcurrentMark* _cm;
+  GCLabBitMap*    _bitmap;
+
+public:
+  GCLabBitMapClosure(ConcurrentMark* cm,
+                     GCLabBitMap* bitmap) {
+    _cm     = cm;
+    _bitmap = bitmap;
+  }
+
+  virtual bool do_bit(size_t offset);
+};
+#endif // !PRODUCT
+
+class GCLabBitMap: public BitMap {
+private:
+  ConcurrentMark* _cm;
+
+  int       _shifter;
+  size_t    _bitmap_word_covers_words;
+
+  // beginning of the heap
+  HeapWord* _heap_start;
+
+  // this is the actual start of the GCLab
+  HeapWord* _real_start_word;
+
+  // this is the actual end of the GCLab
+  HeapWord* _real_end_word;
+
+  // this is the first word, possibly located before the actual start
+  // of the GCLab, that corresponds to the first bit of the bitmap
+  HeapWord* _start_word;
+
+  // size of a GCLab in words
+  size_t _gclab_word_size;
+
+  static int shifter() {
+    return MinObjAlignment - 1;
+  }
+
+  // how many heap words does a single bitmap word corresponds to?
+  static size_t bitmap_word_covers_words() {
+    return BitsPerWord << shifter();
+  }
+
+  static size_t gclab_word_size() {
+    return G1ParallelGCAllocBufferSize / HeapWordSize;
+  }
+
+  static size_t bitmap_size_in_bits() {
+    size_t bits_in_bitmap = gclab_word_size() >> shifter();
+    // We are going to ensure that the beginning of a word in this
+    // bitmap also corresponds to the beginning of a word in the
+    // global marking bitmap. To handle the case where a GCLab
+    // starts from the middle of the bitmap, we need to add enough
+    // space (i.e. up to a bitmap word) to ensure that we have
+    // enough bits in the bitmap.
+    return bits_in_bitmap + BitsPerWord - 1;
+  }
+public:
+  GCLabBitMap(HeapWord* heap_start)
+    : BitMap(bitmap_size_in_bits()),
+      _cm(G1CollectedHeap::heap()->concurrent_mark()),
+      _shifter(shifter()),
+      _bitmap_word_covers_words(bitmap_word_covers_words()),
+      _heap_start(heap_start),
+      _gclab_word_size(gclab_word_size()),
+      _real_start_word(NULL),
+      _real_end_word(NULL),
+      _start_word(NULL)
+  {
+    guarantee( size_in_words() >= bitmap_size_in_words(),
+               "just making sure");
+  }
+
+  inline unsigned heapWordToOffset(HeapWord* addr) {
+    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
+    assert(offset < size(), "offset should be within bounds");
+    return offset;
+  }
+
+  inline HeapWord* offsetToHeapWord(size_t offset) {
+    HeapWord* addr =  _start_word + (offset << _shifter);
+    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
+    return addr;
+  }
+
+  bool fields_well_formed() {
+    bool ret1 = (_real_start_word == NULL) &&
+                (_real_end_word == NULL) &&
+                (_start_word == NULL);
+    if (ret1)
+      return true;
+
+    bool ret2 = _real_start_word >= _start_word &&
+      _start_word < _real_end_word &&
+      (_real_start_word + _gclab_word_size) == _real_end_word &&
+      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
+                                                              > _real_end_word;
+    return ret2;
+  }
+
+  inline bool mark(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    if (addr >= _real_start_word && addr < _real_end_word) {
+      assert(!isMarked(addr), "should not have already been marked");
+
+      // first mark it on the bitmap
+      at_put(heapWordToOffset(addr), true);
+
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  inline bool isMarked(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    return at(heapWordToOffset(addr));
+  }
+
+  void set_buffer(HeapWord* start) {
+    guarantee(use_local_bitmaps, "invariant");
+    clear();
+
+    assert(start != NULL, "invariant");
+    _real_start_word = start;
+    _real_end_word   = start + _gclab_word_size;
+
+    size_t diff =
+      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
+    _start_word = start - diff;
+
+    assert(fields_well_formed(), "invariant");
+  }
+
+#ifndef PRODUCT
+  void verify() {
+    // verify that the marks have been propagated
+    GCLabBitMapClosure cl(_cm, this);
+    iterate(&cl);
+  }
+#endif // PRODUCT
+
+  void retire() {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    if (_start_word != NULL) {
+      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
+
+      // this means that the bitmap was set up for the GCLab
+      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
+
+      mark_bitmap->mostly_disjoint_range_union(this,
+                                0, // always start from the start of the bitmap
+                                _start_word,
+                                size_in_words());
+      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
+
+#ifndef PRODUCT
+      if (use_local_bitmaps && verify_local_bitmaps)
+        verify();
+#endif // PRODUCT
+    } else {
+      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
+    }
+  }
+
+  static size_t bitmap_size_in_words() {
+    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
+  }
+};
+
+class G1ParGCAllocBuffer: public ParGCAllocBuffer {
+private:
+  bool        _retired;
+  bool        _during_marking;
+  GCLabBitMap _bitmap;
+
+public:
+  G1ParGCAllocBuffer() :
+    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
+    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
+    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
+    _retired(false)
+  { }
+
+  inline bool mark(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(_during_marking, "invariant");
+    return _bitmap.mark(addr);
+  }
+
+  inline void set_buf(HeapWord* buf) {
+    if (use_local_bitmaps && _during_marking)
+      _bitmap.set_buffer(buf);
+    ParGCAllocBuffer::set_buf(buf);
+    _retired = false;
+  }
+
+  inline void retire(bool end_of_gc, bool retain) {
+    if (_retired)
+      return;
+    if (use_local_bitmaps && _during_marking) {
+      _bitmap.retire();
+    }
+    ParGCAllocBuffer::retire(end_of_gc, retain);
+    _retired = true;
+  }
+};
+
+class G1ParScanThreadState : public StackObj {
+protected:
+  G1CollectedHeap* _g1h;
+  RefToScanQueue*  _refs;
+  DirtyCardQueue   _dcq;
+  CardTableModRefBS* _ct_bs;
+  G1RemSet* _g1_rem;
+
+  typedef GrowableArray<StarTask> OverflowQueue;
+  OverflowQueue* _overflowed_refs;
+
+  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
+  ageTable           _age_table;
+
+  size_t           _alloc_buffer_waste;
+  size_t           _undo_waste;
+
+  OopsInHeapRegionClosure*      _evac_failure_cl;
+  G1ParScanHeapEvacClosure*     _evac_cl;
+  G1ParScanPartialArrayClosure* _partial_scan_cl;
+
+  int _hash_seed;
+  int _queue_num;
+
+  int _term_attempts;
+#if G1_DETAILED_STATS
+  int _pushes, _pops, _steals, _steal_attempts;
+  int _overflow_pushes;
+#endif
+
+  double _start;
+  double _start_strong_roots;
+  double _strong_roots_time;
+  double _start_term;
+  double _term_time;
+
+  // Map from young-age-index (0 == not young, 1 is youngest) to
+  // surviving words. base is what we get back from the malloc call
+  size_t* _surviving_young_words_base;
+  // this points into the array, as we use the first few entries for padding
+  size_t* _surviving_young_words;
+
+#define PADDING_ELEM_NUM (64 / sizeof(size_t))
+
+  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
+
+  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
+
+  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
+  CardTableModRefBS* ctbs()                      { return _ct_bs; }
+
+  template <class T> void immediate_rs_update(HeapRegion* from, T* p, int tid) {
+    if (!from->is_survivor()) {
+      _g1_rem->par_write_ref(from, p, tid);
+    }
+  }
+
+  template <class T> void deferred_rs_update(HeapRegion* from, T* p, int tid) {
+    // If the new value of the field points to the same region or
+    // is the to-space, we don't need to include it in the Rset updates.
+    if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) {
+      size_t card_index = ctbs()->index_for(p);
+      // If the card hasn't been added to the buffer, do it.
+      if (ctbs()->mark_card_deferred(card_index)) {
+        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
+      }
+    }
+  }
+
+public:
+  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num);
+
+  ~G1ParScanThreadState() {
+    FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
+  }
+
+  RefToScanQueue*   refs()            { return _refs;             }
+  OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
+  ageTable*         age_table()       { return &_age_table;       }
+
+  G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
+    return &_alloc_buffers[purpose];
+  }
+
+  size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
+  size_t undo_waste()                            { return _undo_waste; }
+
+  template <class T> void push_on_queue(T* ref) {
+    assert(ref != NULL, "invariant");
+    assert(has_partial_array_mask(ref) ||
+           _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(ref)), "invariant");
+#ifdef ASSERT
+    if (has_partial_array_mask(ref)) {
+      oop p = clear_partial_array_mask(ref);
+      // Verify that we point into the CS
+      assert(_g1h->obj_in_cs(p), "Should be in CS");
+    }
+#endif
+    if (!refs()->push(ref)) {
+      overflowed_refs()->push(ref);
+      IF_G1_DETAILED_STATS(note_overflow_push());
+    } else {
+      IF_G1_DETAILED_STATS(note_push());
+    }
+  }
+
+  void pop_from_queue(StarTask& ref) {
+    if (refs()->pop_local(ref)) {
+      assert((oop*)ref != NULL, "pop_local() returned true");
+      assert(UseCompressedOops || !ref.is_narrow(), "Error");
+      assert(has_partial_array_mask((oop*)ref) ||
+             _g1h->obj_in_cs(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref)
+                                             : oopDesc::load_decode_heap_oop((oop*)ref)),
+             "invariant");
+      IF_G1_DETAILED_STATS(note_pop());
+    } else {
+      StarTask null_task;
+      ref = null_task;
+    }
+  }
+
+  void pop_from_overflow_queue(StarTask& ref) {
+    StarTask new_ref = overflowed_refs()->pop();
+    assert((oop*)new_ref != NULL, "pop() from a local non-empty stack");
+    assert(UseCompressedOops || !new_ref.is_narrow(), "Error");
+    assert(has_partial_array_mask((oop*)new_ref) ||
+           _g1h->obj_in_cs(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref)
+                                               : oopDesc::load_decode_heap_oop((oop*)new_ref)),
+             "invariant");
+    ref = new_ref;
+  }
+
+  int refs_to_scan()                             { return refs()->size();                 }
+  int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
+
+  template <class T> void update_rs(HeapRegion* from, T* p, int tid) {
+    if (G1DeferredRSUpdate) {
+      deferred_rs_update(from, p, tid);
+    } else {
+      immediate_rs_update(from, p, tid);
+    }
+  }
+
+  HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
+
+    HeapWord* obj = NULL;
+    if (word_sz * 100 <
+        (size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
+                                                  ParallelGCBufferWastePct) {
+      G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
+      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
+      alloc_buf->retire(false, false);
+
+      HeapWord* buf =
+        _g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
+      if (buf == NULL) return NULL; // Let caller handle allocation failure.
+      // Otherwise.
+      alloc_buf->set_buf(buf);
+
+      obj = alloc_buf->allocate(word_sz);
+      assert(obj != NULL, "buffer was definitely big enough...");
+    } else {
+      obj = _g1h->par_allocate_during_gc(purpose, word_sz);
+    }
+    return obj;
+  }
+
+  HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
+    HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
+    if (obj != NULL) return obj;
+    return allocate_slow(purpose, word_sz);
+  }
+
+  void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
+    if (alloc_buffer(purpose)->contains(obj)) {
+      assert(alloc_buffer(purpose)->contains(obj + word_sz - 1),
+             "should contain whole object");
+      alloc_buffer(purpose)->undo_allocation(obj, word_sz);
+    } else {
+      CollectedHeap::fill_with_object(obj, word_sz);
+      add_to_undo_waste(word_sz);
+    }
+  }
+
+  void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
+    _evac_failure_cl = evac_failure_cl;
+  }
+  OopsInHeapRegionClosure* evac_failure_closure() {
+    return _evac_failure_cl;
+  }
+
+  void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
+    _evac_cl = evac_cl;
+  }
+
+  void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
+    _partial_scan_cl = partial_scan_cl;
+  }
+
+  int* hash_seed() { return &_hash_seed; }
+  int  queue_num() { return _queue_num; }
+
+  int term_attempts()   { return _term_attempts; }
+  void note_term_attempt()  { _term_attempts++; }
+
+#if G1_DETAILED_STATS
+  int pushes()          { return _pushes; }
+  int pops()            { return _pops; }
+  int steals()          { return _steals; }
+  int steal_attempts()  { return _steal_attempts; }
+  int overflow_pushes() { return _overflow_pushes; }
+
+  void note_push()          { _pushes++; }
+  void note_pop()           { _pops++; }
+  void note_steal()         { _steals++; }
+  void note_steal_attempt() { _steal_attempts++; }
+  void note_overflow_push() { _overflow_pushes++; }
+#endif
+
+  void start_strong_roots() {
+    _start_strong_roots = os::elapsedTime();
+  }
+  void end_strong_roots() {
+    _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
+  }
+  double strong_roots_time() { return _strong_roots_time; }
+
+  void start_term_time() {
+    note_term_attempt();
+    _start_term = os::elapsedTime();
+  }
+  void end_term_time() {
+    _term_time += (os::elapsedTime() - _start_term);
+  }
+  double term_time() { return _term_time; }
+
+  double elapsed() {
+    return os::elapsedTime() - _start;
+  }
+
+  size_t* surviving_young_words() {
+    // We add on to hide entry 0 which accumulates surviving words for
+    // age -1 regions (i.e. non-young ones)
+    return _surviving_young_words;
+  }
+
+  void retire_alloc_buffers() {
+    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+      size_t waste = _alloc_buffers[ap].words_remaining();
+      add_to_alloc_buffer_waste(waste);
+      _alloc_buffers[ap].retire(true, false);
+    }
+  }
+
+private:
+  template <class T> void deal_with_reference(T* ref_to_scan) {
+    if (has_partial_array_mask(ref_to_scan)) {
+      _partial_scan_cl->do_oop_nv(ref_to_scan);
+    } else {
+      // Note: we can use "raw" versions of "region_containing" because
+      // "obj_to_scan" is definitely in the heap, and is not in a
+      // humongous region.
+      HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
+      _evac_cl->set_region(r);
+      _evac_cl->do_oop_nv(ref_to_scan);
+    }
+  }
+
+public:
+  void trim_queue() {
+    // I've replicated the loop twice, first to drain the overflow
+    // queue, second to drain the task queue. This is better than
+    // having a single loop, which checks both conditions and, inside
+    // it, either pops the overflow queue or the task queue, as each
+    // loop is tighter. Also, the decision to drain the overflow queue
+    // first is not arbitrary, as the overflow queue is not visible
+    // to the other workers, whereas the task queue is. So, we want to
+    // drain the "invisible" entries first, while allowing the other
+    // workers to potentially steal the "visible" entries.
+
+    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
+      while (overflowed_refs_to_scan() > 0) {
+        StarTask ref_to_scan;
+        assert((oop*)ref_to_scan == NULL, "Constructed above");
+        pop_from_overflow_queue(ref_to_scan);
+        // We shouldn't have pushed it on the queue if it was not
+        // pointing into the CSet.
+        assert((oop*)ref_to_scan != NULL, "Follows from inner loop invariant");
+        if (ref_to_scan.is_narrow()) {
+          assert(UseCompressedOops, "Error");
+          narrowOop* p = (narrowOop*)ref_to_scan;
+          assert(!has_partial_array_mask(p) &&
+                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
+          deal_with_reference(p);
+        } else {
+          oop* p = (oop*)ref_to_scan;
+          assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
+                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
+          deal_with_reference(p);
+        }
+      }
+
+      while (refs_to_scan() > 0) {
+        StarTask ref_to_scan;
+        assert((oop*)ref_to_scan == NULL, "Constructed above");
+        pop_from_queue(ref_to_scan);
+        if ((oop*)ref_to_scan != NULL) {
+          if (ref_to_scan.is_narrow()) {
+            assert(UseCompressedOops, "Error");
+            narrowOop* p = (narrowOop*)ref_to_scan;
+            assert(!has_partial_array_mask(p) &&
+                   _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
+            deal_with_reference(p);
+          } else {
+            oop* p = (oop*)ref_to_scan;
+            assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
+                  _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
+            deal_with_reference(p);
+          }
+        }
+      }
+    }
+  }
+};
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -94,7 +94,14 @@
   _summary(new Summary()),
   _abandoned_summary(new AbandonedSummary()),
 
+#ifndef PRODUCT
   _cur_clear_ct_time_ms(0.0),
+  _min_clear_cc_time_ms(-1.0),
+  _max_clear_cc_time_ms(-1.0),
+  _cur_clear_cc_time_ms(0.0),
+  _cum_clear_cc_time_ms(0.0),
+  _num_cc_clears(0L),
+#endif
 
   _region_num_young(0),
   _region_num_tenured(0),
@@ -293,10 +300,6 @@
   if (G1SteadyStateUsed < 50) {
     vm_exit_during_initialization("G1SteadyStateUsed must be at least 50%.");
   }
-  if (UseConcMarkSweepGC) {
-    vm_exit_during_initialization("-XX:+UseG1GC is incompatible with "
-                                  "-XX:+UseConcMarkSweepGC.");
-  }
 
   initialize_gc_policy_counters();
 
@@ -1652,6 +1655,15 @@
         print_stats(1, "Object Copying", obj_copy_time);
       }
     }
+#ifndef PRODUCT
+    print_stats(1, "Cur Clear CC", _cur_clear_cc_time_ms);
+    print_stats(1, "Cum Clear CC", _cum_clear_cc_time_ms);
+    print_stats(1, "Min Clear CC", _min_clear_cc_time_ms);
+    print_stats(1, "Max Clear CC", _max_clear_cc_time_ms);
+    if (_num_cc_clears > 0) {
+      print_stats(1, "Avg Clear CC", _cum_clear_cc_time_ms / ((double)_num_cc_clears));
+    }
+#endif
     print_stats(1, "Other", other_time_ms);
     for (int i = 0; i < _aux_num; ++i) {
       if (_cur_aux_times_set[i]) {
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -112,7 +112,6 @@
     return 8*M;
   }
 
-
   double _cur_collection_start_sec;
   size_t _cur_collection_pause_used_at_start_bytes;
   size_t _cur_collection_pause_used_regions_at_start;
@@ -122,6 +121,15 @@
   double _cur_clear_ct_time_ms;
   bool   _satb_drain_time_set;
 
+#ifndef PRODUCT
+  // Card Table Count Cache stats
+  double _min_clear_cc_time_ms;         // min
+  double _max_clear_cc_time_ms;         // max
+  double _cur_clear_cc_time_ms;         // clearing time during current pause
+  double _cum_clear_cc_time_ms;         // cummulative clearing time
+  jlong  _num_cc_clears;                // number of times the card count cache has been cleared
+#endif
+
   double _cur_CH_strong_roots_end_sec;
   double _cur_CH_strong_roots_dur_ms;
   double _cur_G1_strong_roots_end_sec;
@@ -931,6 +939,18 @@
     _cur_aux_times_ms[i] += ms;
   }
 
+#ifndef PRODUCT
+  void record_cc_clear_time(double ms) {
+    if (_min_clear_cc_time_ms < 0.0 || ms <= _min_clear_cc_time_ms)
+      _min_clear_cc_time_ms = ms;
+    if (_max_clear_cc_time_ms < 0.0 || ms >= _max_clear_cc_time_ms)
+      _max_clear_cc_time_ms = ms;
+    _cur_clear_cc_time_ms = ms;
+    _cum_clear_cc_time_ms += ms;
+    _num_cc_clears++;
+  }
+#endif
+
   // Record the fact that "bytes" bytes allocated in a region.
   void record_before_bytes(size_t bytes);
   void record_after_bytes(size_t bytes);
--- a/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -42,18 +42,6 @@
   virtual void set_region(HeapRegion* from) { _from = from; }
 };
 
-
-class G1ScanAndBalanceClosure : public OopClosure {
-  G1CollectedHeap* _g1;
-  static int _nq;
-public:
-  G1ScanAndBalanceClosure(G1CollectedHeap* g1) : _g1(g1) { }
-  inline  void do_oop_nv(oop* p);
-  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
-};
-
 class G1ParClosureSuper : public OopsInHeapRegionClosure {
 protected:
   G1CollectedHeap* _g1;
@@ -69,34 +57,32 @@
 public:
   G1ParScanClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
     G1ParClosureSuper(g1, par_scan_state) { }
-  void do_oop_nv(oop* p);   // should be made inline
-  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
   virtual void do_oop(oop* p)          { do_oop_nv(p); }
   virtual void do_oop(narrowOop* p)    { do_oop_nv(p); }
 };
 
-#define G1_PARTIAL_ARRAY_MASK 1
+#define G1_PARTIAL_ARRAY_MASK 0x2
 
-inline bool has_partial_array_mask(oop* ref) {
-  return (intptr_t) ref & G1_PARTIAL_ARRAY_MASK;
+template <class T> inline bool has_partial_array_mask(T* ref) {
+  return ((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) == G1_PARTIAL_ARRAY_MASK;
 }
 
-inline oop* set_partial_array_mask(oop obj) {
-  return (oop*) ((intptr_t) obj | G1_PARTIAL_ARRAY_MASK);
+template <class T> inline T* set_partial_array_mask(T obj) {
+  assert(((uintptr_t)obj & G1_PARTIAL_ARRAY_MASK) == 0, "Information loss!");
+  return (T*) ((uintptr_t)obj | G1_PARTIAL_ARRAY_MASK);
 }
 
-inline oop clear_partial_array_mask(oop* ref) {
-  return oop((intptr_t) ref & ~G1_PARTIAL_ARRAY_MASK);
+template <class T> inline oop clear_partial_array_mask(T* ref) {
+  return oop((intptr_t)ref & ~G1_PARTIAL_ARRAY_MASK);
 }
 
 class G1ParScanPartialArrayClosure : public G1ParClosureSuper {
   G1ParScanClosure _scanner;
-  template <class T> void process_array_chunk(oop obj, int start, int end);
 public:
   G1ParScanPartialArrayClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
     G1ParClosureSuper(g1, par_scan_state), _scanner(g1, par_scan_state) { }
-  void do_oop_nv(oop* p);
-  void do_oop_nv(narrowOop* p)      { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
   virtual void do_oop(oop* p)       { do_oop_nv(p); }
   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
 };
@@ -105,7 +91,7 @@
 class G1ParCopyHelper : public G1ParClosureSuper {
   G1ParScanClosure *_scanner;
 protected:
-  void mark_forwardee(oop* p);
+  template <class T> void mark_forwardee(T* p);
   oop copy_to_survivor_space(oop obj);
 public:
   G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state,
@@ -117,36 +103,35 @@
          bool do_mark_forwardee, bool skip_cset_test>
 class G1ParCopyClosure : public G1ParCopyHelper {
   G1ParScanClosure _scanner;
-  void do_oop_work(oop* p);
-  void do_oop_work(narrowOop* p) { guarantee(false, "NYI"); }
+  template <class T> void do_oop_work(T* p);
 public:
   G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
     _scanner(g1, par_scan_state), G1ParCopyHelper(g1, par_scan_state, &_scanner) { }
-  inline void do_oop_nv(oop* p) {
+  template <class T> void do_oop_nv(T* p) {
     do_oop_work(p);
     if (do_mark_forwardee)
       mark_forwardee(p);
   }
-  inline void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
   virtual void do_oop(oop* p)       { do_oop_nv(p); }
   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
 };
 
 typedef G1ParCopyClosure<false, G1BarrierNone, false, false> G1ParScanExtRootClosure;
 typedef G1ParCopyClosure<true,  G1BarrierNone, false, false> G1ParScanPermClosure;
+typedef G1ParCopyClosure<false, G1BarrierRS,   false, false> G1ParScanHeapRSClosure;
 typedef G1ParCopyClosure<false, G1BarrierNone, true,  false> G1ParScanAndMarkExtRootClosure;
 typedef G1ParCopyClosure<true,  G1BarrierNone, true,  false> G1ParScanAndMarkPermClosure;
-typedef G1ParCopyClosure<false, G1BarrierRS,   false, false> G1ParScanHeapRSClosure;
 typedef G1ParCopyClosure<false, G1BarrierRS,   true,  false> G1ParScanAndMarkHeapRSClosure;
 // This is the only case when we set skip_cset_test. Basically, this
 // closure is (should?) only be called directly while we're draining
 // the overflow and task queues. In that case we know that the
 // reference in question points into the collection set, otherwise we
-// would not have pushed it on the queue.
-typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
+// would not have pushed it on the queue. The following is defined in
+// g1_specialized_oop_closures.hpp.
+// typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
 // We need a separate closure to handle references during evacuation
-// failure processing, as it cannot asume that the reference already
- // points to the collection set (like G1ParScanHeapEvacClosure does).
+// failure processing, as we cannot asume that the reference already
+// points into the collection set (like G1ParScanHeapEvacClosure does).
 typedef G1ParCopyClosure<false, G1BarrierEvac, false, false> G1ParScanHeapEvacFailureClosure;
 
 class FilterIntoCSClosure: public OopClosure {
@@ -158,10 +143,9 @@
                         G1CollectedHeap* g1, OopClosure* oc) :
     _dcto_cl(dcto_cl), _g1(g1), _oc(oc)
   {}
-  inline void do_oop_nv(oop* p);
-  inline void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p)   { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
+  virtual void do_oop(oop* p)        { do_oop_nv(p); }
+  virtual void do_oop(narrowOop* p)  { do_oop_nv(p); }
   bool apply_to_weak_ref_discovered_field() { return true; }
   bool do_header() { return false; }
 };
@@ -174,10 +158,9 @@
                                      OopsInHeapRegionClosure* oc) :
     _g1(g1), _oc(oc)
   {}
-  inline  void do_oop_nv(oop* p);
-  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
+  virtual void do_oop(oop* p) { do_oop_nv(p); }
+  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   bool apply_to_weak_ref_discovered_field() { return true; }
   bool do_header() { return false; }
   void set_region(HeapRegion* from) {
@@ -195,10 +178,9 @@
                                             ConcurrentMark* cm)
   : _g1(g1), _oc(oc), _cm(cm) { }
 
-  inline  void do_oop_nv(oop* p);
-  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
+  virtual void do_oop(oop* p) { do_oop_nv(p); }
+  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   bool apply_to_weak_ref_discovered_field() { return true; }
   bool do_header() { return false; }
   void set_region(HeapRegion* from) {
@@ -213,10 +195,9 @@
   int _out_of_region;
 public:
   FilterOutOfRegionClosure(HeapRegion* r, OopClosure* oc);
-  inline  void do_oop_nv(oop* p);
-  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p)   { guarantee(false, "NYI"); }
+  template <class T> void do_oop_nv(T* p);
+  virtual void do_oop(oop* p) { do_oop_nv(p); }
+  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   bool apply_to_weak_ref_discovered_field() { return true; }
   bool do_header() { return false; }
   int out_of_region() { return _out_of_region; }
--- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -31,9 +31,10 @@
 // perf-critical inner loop.
 #define FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT 0
 
-inline void FilterIntoCSClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-  if (obj != NULL && _g1->obj_in_cs(obj)) {
+template <class T> inline void FilterIntoCSClosure::do_oop_nv(T* p) {
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop) &&
+      _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) {
     _oc->do_oop(p);
 #if FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT
     _dcto_cl->incr_count();
@@ -41,44 +42,32 @@
   }
 }
 
-inline void FilterIntoCSClosure::do_oop(oop* p)
-{
-  do_oop_nv(p);
-}
-
 #define FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT 0
 
-inline void FilterOutOfRegionClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-  HeapWord* obj_hw = (HeapWord*)obj;
-  if (obj_hw != NULL && (obj_hw < _r_bottom || obj_hw >= _r_end)) {
-    _oc->do_oop(p);
+template <class T> inline void FilterOutOfRegionClosure::do_oop_nv(T* p) {
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop)) {
+    HeapWord* obj_hw = (HeapWord*)oopDesc::decode_heap_oop_not_null(heap_oop);
+    if (obj_hw < _r_bottom || obj_hw >= _r_end) {
+      _oc->do_oop(p);
 #if FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT
-    _out_of_region++;
+      _out_of_region++;
 #endif
+    }
   }
 }
 
-inline void FilterOutOfRegionClosure::do_oop(oop* p)
-{
-  do_oop_nv(p);
-}
-
-inline void FilterInHeapRegionAndIntoCSClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-  if (obj != NULL && _g1->obj_in_cs(obj))
+template <class T> inline void FilterInHeapRegionAndIntoCSClosure::do_oop_nv(T* p) {
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop) &&
+      _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop)))
     _oc->do_oop(p);
 }
 
-inline void FilterInHeapRegionAndIntoCSClosure::do_oop(oop* p)
-{
-  do_oop_nv(p);
-}
-
-
-inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-  if (obj != NULL) {
+template <class T> inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop_nv(T* p) {
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop)) {
+    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
     HeapRegion* hr = _g1->heap_region_containing((HeapWord*) obj);
     if (hr != NULL) {
       if (hr->in_collection_set())
@@ -89,24 +78,29 @@
   }
 }
 
-inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop(oop* p)
-{
-  do_oop_nv(p);
+// This closure is applied to the fields of the objects that have just been copied.
+template <class T> inline void G1ParScanClosure::do_oop_nv(T* p) {
+  T heap_oop = oopDesc::load_heap_oop(p);
+
+  if (!oopDesc::is_null(heap_oop)) {
+    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+    if (_g1->in_cset_fast_test(obj)) {
+      // We're not going to even bother checking whether the object is
+      // already forwarded or not, as this usually causes an immediate
+      // stall. We'll try to prefetch the object (for write, given that
+      // we might need to install the forwarding reference) and we'll
+      // get back to it when pop it from the queue
+      Prefetch::write(obj->mark_addr(), 0);
+      Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
+
+      // slightly paranoid test; I'm trying to catch potential
+      // problems before we go into push_on_queue to know where the
+      // problem is coming from
+      assert(obj == oopDesc::load_decode_heap_oop(p),
+             "p should still be pointing to obj");
+      _par_scan_state->push_on_queue(p);
+    } else {
+      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
+    }
+  }
 }
-
-inline void G1ScanAndBalanceClosure::do_oop_nv(oop* p) {
-  RefToScanQueue* q;
-  if (ParallelGCThreads > 0) {
-    // Deal the work out equally.
-    _nq = (_nq + 1) % ParallelGCThreads;
-    q = _g1->task_queue(_nq);
-  } else {
-    q = _g1->task_queue(0);
-  }
-  bool nooverflow = q->push(p);
-  guarantee(nooverflow, "Overflow during poplularity region processing");
-}
-
-inline void G1ScanAndBalanceClosure::do_oop(oop* p) {
-  do_oop_nv(p);
-}
--- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -65,11 +65,10 @@
   void set_region(HeapRegion* from) {
     _blk->set_region(from);
   }
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop* p) {
-    oop obj = *p;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
     if (_g1->obj_in_cs(obj)) _blk->do_oop(p);
   }
   bool apply_to_weak_ref_discovered_field() { return true; }
@@ -110,11 +109,10 @@
 public:
   VerifyRSCleanCardOopClosure(G1CollectedHeap* g1) : _g1(g1) {}
 
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop* p) {
-    oop obj = *p;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
     HeapRegion* to = _g1->heap_region_containing(obj);
     guarantee(to == NULL || !to->in_collection_set(),
               "Missed a rem set member.");
@@ -129,9 +127,9 @@
 {
   _seq_task = new SubTasksDone(NumSeqTasks);
   guarantee(n_workers() > 0, "There should be some workers");
-  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, n_workers());
+  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers());
   for (uint i = 0; i < n_workers(); i++) {
-    _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
+    _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true);
   }
 }
 
@@ -140,7 +138,7 @@
   for (uint i = 0; i < n_workers(); i++) {
     delete _new_refs[i];
   }
-  FREE_C_HEAP_ARRAY(GrowableArray<oop*>*, _new_refs);
+  FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs);
 }
 
 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
@@ -428,15 +426,15 @@
   }
 };
 
-void
-HRInto_G1RemSet::scanNewRefsRS(OopsInHeapRegionClosure* oc,
-                                             int worker_i) {
+template <class T> void
+HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc,
+                                    int worker_i) {
   double scan_new_refs_start_sec = os::elapsedTime();
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
   for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
-    oop* p = _new_refs[worker_i]->at(i);
-    oop obj = *p;
+    T* p = (T*) _new_refs[worker_i]->at(i);
+    oop obj = oopDesc::load_decode_heap_oop(p);
     // *p was in the collection set when p was pushed on "_new_refs", but
     // another thread may have processed this location from an RS, so it
     // might not point into the CS any longer.  If so, it's obviously been
@@ -549,11 +547,10 @@
   G1CollectedHeap* _g1;
 public:
   UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { }
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop* p) {
-    HeapRegion* to = _g1->heap_region_containing(*p);
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
+    HeapRegion* to = _g1->heap_region_containing(oopDesc::load_decode_heap_oop(p));
     if (to->in_collection_set()) {
       to->rem_set()->add_reference(p, 0);
     }
@@ -567,11 +564,10 @@
 public:
   UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
     _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { }
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop* p) {
-    oop obj = *p;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
     if (_g1->obj_in_cs(obj)) {
       size_t card_index = _ct_bs->index_for(p);
       if (_ct_bs->mark_card_deferred(card_index)) {
@@ -581,10 +577,10 @@
   }
 };
 
-void HRInto_G1RemSet::new_refs_iterate(OopClosure* cl) {
+template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) {
   for (size_t i = 0; i < n_workers(); i++) {
     for (int j = 0; j < _new_refs[i]->length(); j++) {
-      oop* p = _new_refs[i]->at(j);
+      T* p = (T*) _new_refs[i]->at(j);
       cl->do_oop(p);
     }
   }
@@ -680,6 +676,55 @@
 
 static IntHistogram out_of_histo(50, 50);
 
+void HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i) {
+  // Construct the region representing the card.
+  HeapWord* start = _ct_bs->addr_for(card_ptr);
+  // And find the region containing it.
+  HeapRegion* r = _g1->heap_region_containing(start);
+  assert(r != NULL, "unexpected null");
+
+  HeapWord* end   = _ct_bs->addr_for(card_ptr + 1);
+  MemRegion dirtyRegion(start, end);
+
+#if CARD_REPEAT_HISTO
+  init_ct_freq_table(_g1->g1_reserved_obj_bytes());
+  ct_freq_note_card(_ct_bs->index_for(start));
+#endif
+
+  UpdateRSOopClosure update_rs_oop_cl(this, worker_i);
+  update_rs_oop_cl.set_from(r);
+  FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl);
+
+  // Undirty the card.
+  *card_ptr = CardTableModRefBS::clean_card_val();
+  // We must complete this write before we do any of the reads below.
+  OrderAccess::storeload();
+  // And process it, being careful of unallocated portions of TLAB's.
+  HeapWord* stop_point =
+    r->oops_on_card_seq_iterate_careful(dirtyRegion,
+                                        &filter_then_update_rs_oop_cl);
+  // If stop_point is non-null, then we encountered an unallocated region
+  // (perhaps the unfilled portion of a TLAB.)  For now, we'll dirty the
+  // card and re-enqueue: if we put off the card until a GC pause, then the
+  // unallocated portion will be filled in.  Alternatively, we might try
+  // the full complexity of the technique used in "regular" precleaning.
+  if (stop_point != NULL) {
+    // The card might have gotten re-dirtied and re-enqueued while we
+    // worked.  (In fact, it's pretty likely.)
+    if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
+      *card_ptr = CardTableModRefBS::dirty_card_val();
+      MutexLockerEx x(Shared_DirtyCardQ_lock,
+                      Mutex::_no_safepoint_check_flag);
+      DirtyCardQueue* sdcq =
+        JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
+      sdcq->enqueue(card_ptr);
+    }
+  } else {
+    out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region());
+    _conc_refine_cards++;
+  }
+}
+
 void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) {
   // If the card is no longer dirty, nothing to do.
   if (*card_ptr != CardTableModRefBS::dirty_card_val()) return;
@@ -720,61 +765,63 @@
     return;
   }
 
-  // Should we defer it?
+  // Should we defer processing the card?
+  //
+  // Previously the result from the insert_cache call would be
+  // either card_ptr (implying that card_ptr was currently "cold"),
+  // null (meaning we had inserted the card ptr into the "hot"
+  // cache, which had some headroom), or a "hot" card ptr
+  // extracted from the "hot" cache.
+  //
+  // Now that the _card_counts cache in the ConcurrentG1Refine
+  // instance is an evicting hash table, the result we get back
+  // could be from evicting the card ptr in an already occupied
+  // bucket (in which case we have replaced the card ptr in the
+  // bucket with card_ptr and "defer" is set to false). To avoid
+  // having a data structure (updates to which would need a lock)
+  // to hold these unprocessed dirty cards, we need to immediately
+  // process card_ptr. The actions needed to be taken on return
+  // from cache_insert are summarized in the following table:
+  //
+  // res      defer   action
+  // --------------------------------------------------------------
+  // null     false   card evicted from _card_counts & replaced with
+  //                  card_ptr; evicted ptr added to hot cache.
+  //                  No need to process res; immediately process card_ptr
+  //
+  // null     true    card not evicted from _card_counts; card_ptr added
+  //                  to hot cache.
+  //                  Nothing to do.
+  //
+  // non-null false   card evicted from _card_counts & replaced with
+  //                  card_ptr; evicted ptr is currently "cold" or
+  //                  caused an eviction from the hot cache.
+  //                  Immediately process res; process card_ptr.
+  //
+  // non-null true    card not evicted from _card_counts; card_ptr is
+  //                  currently cold, or caused an eviction from hot
+  //                  cache.
+  //                  Immediately process res; no need to process card_ptr.
+
+  jbyte* res = card_ptr;
+  bool defer = false;
   if (_cg1r->use_cache()) {
-    card_ptr = _cg1r->cache_insert(card_ptr);
-    // If it was not an eviction, nothing to do.
-    if (card_ptr == NULL) return;
-
-    // OK, we have to reset the card start, region, etc.
-    start = _ct_bs->addr_for(card_ptr);
-    r = _g1->heap_region_containing(start);
-    if (r == NULL) {
-      guarantee(_g1->is_in_permanent(start), "Or else where?");
-      return;  // Not in the G1 heap (might be in perm, for example.)
+    jbyte* res = _cg1r->cache_insert(card_ptr, &defer);
+    if (res != NULL && (res != card_ptr || defer)) {
+      start = _ct_bs->addr_for(res);
+      r = _g1->heap_region_containing(start);
+      if (r == NULL) {
+        assert(_g1->is_in_permanent(start), "Or else where?");
+      } else {
+        guarantee(!r->is_young(), "It was evicted in the current minor cycle.");
+        // Process card pointer we get back from the hot card cache
+        concurrentRefineOneCard_impl(res, worker_i);
+      }
     }
-    guarantee(!r->is_young(), "It was evicted in the current minor cycle.");
   }
 
-  HeapWord* end   = _ct_bs->addr_for(card_ptr + 1);
-  MemRegion dirtyRegion(start, end);
-
-#if CARD_REPEAT_HISTO
-  init_ct_freq_table(_g1->g1_reserved_obj_bytes());
-  ct_freq_note_card(_ct_bs->index_for(start));
-#endif
-
-  UpdateRSOopClosure update_rs_oop_cl(this, worker_i);
-  update_rs_oop_cl.set_from(r);
-  FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl);
-
-  // Undirty the card.
-  *card_ptr = CardTableModRefBS::clean_card_val();
-  // We must complete this write before we do any of the reads below.
-  OrderAccess::storeload();
-  // And process it, being careful of unallocated portions of TLAB's.
-  HeapWord* stop_point =
-    r->oops_on_card_seq_iterate_careful(dirtyRegion,
-                                        &filter_then_update_rs_oop_cl);
-  // If stop_point is non-null, then we encountered an unallocated region
-  // (perhaps the unfilled portion of a TLAB.)  For now, we'll dirty the
-  // card and re-enqueue: if we put off the card until a GC pause, then the
-  // unallocated portion will be filled in.  Alternatively, we might try
-  // the full complexity of the technique used in "regular" precleaning.
-  if (stop_point != NULL) {
-    // The card might have gotten re-dirtied and re-enqueued while we
-    // worked.  (In fact, it's pretty likely.)
-    if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
-      *card_ptr = CardTableModRefBS::dirty_card_val();
-      MutexLockerEx x(Shared_DirtyCardQ_lock,
-                      Mutex::_no_safepoint_check_flag);
-      DirtyCardQueue* sdcq =
-        JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
-      sdcq->enqueue(card_ptr);
-    }
-  } else {
-    out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region());
-    _conc_refine_cards++;
+  if (!defer) {
+    concurrentRefineOneCard_impl(card_ptr, worker_i);
   }
 }
 
--- a/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -62,10 +62,12 @@
   // If "this" is of the given subtype, return "this", else "NULL".
   virtual HRInto_G1RemSet* as_HRInto_G1RemSet() { return NULL; }
 
-  // Record, if necessary, the fact that *p (where "p" is in region "from")
-  // has changed to its new value.
+  // Record, if necessary, the fact that *p (where "p" is in region "from",
+  // and is, a fortiori, required to be non-NULL) has changed to its new value.
   virtual void write_ref(HeapRegion* from, oop* p) = 0;
+  virtual void write_ref(HeapRegion* from, narrowOop* p) = 0;
   virtual void par_write_ref(HeapRegion* from, oop* p, int tid) = 0;
+  virtual void par_write_ref(HeapRegion* from, narrowOop* p, int tid) = 0;
 
   // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region
   // or card, respectively, such that a region or card with a corresponding
@@ -105,7 +107,9 @@
 
   // Nothing is necessary in the version below.
   void write_ref(HeapRegion* from, oop* p) {}
+  void write_ref(HeapRegion* from, narrowOop* p) {}
   void par_write_ref(HeapRegion* from, oop* p, int tid) {}
+  void par_write_ref(HeapRegion* from, narrowOop* p, int tid) {}
 
   void scrub(BitMap* region_bm, BitMap* card_bm) {}
   void scrub_par(BitMap* region_bm, BitMap* card_bm,
@@ -143,8 +147,23 @@
   // their references into the collection summarized in "_new_refs".
   bool _par_traversal_in_progress;
   void set_par_traversal(bool b) { _par_traversal_in_progress = b; }
-  GrowableArray<oop*>** _new_refs;
-  void new_refs_iterate(OopClosure* cl);
+  GrowableArray<OopOrNarrowOopStar>** _new_refs;
+  template <class T> void new_refs_iterate_work(OopClosure* cl);
+  void new_refs_iterate(OopClosure* cl) {
+    if (UseCompressedOops) {
+      new_refs_iterate_work<narrowOop>(cl);
+    } else {
+      new_refs_iterate_work<oop>(cl);
+    }
+  }
+
+  // The routine that performs the actual work of refining a dirty
+  // card.
+  void concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i);
+
+protected:
+  template <class T> void write_ref_nv(HeapRegion* from, T* p);
+  template <class T> void par_write_ref_nv(HeapRegion* from, T* p, int tid);
 
 public:
   // This is called to reset dual hash tables after the gc pause
@@ -161,7 +180,14 @@
   void prepare_for_oops_into_collection_set_do();
   void cleanup_after_oops_into_collection_set_do();
   void scanRS(OopsInHeapRegionClosure* oc, int worker_i);
-  void scanNewRefsRS(OopsInHeapRegionClosure* oc, int worker_i);
+  template <class T> void scanNewRefsRS_work(OopsInHeapRegionClosure* oc, int worker_i);
+  void scanNewRefsRS(OopsInHeapRegionClosure* oc, int worker_i) {
+    if (UseCompressedOops) {
+      scanNewRefsRS_work<narrowOop>(oc, worker_i);
+    } else {
+      scanNewRefsRS_work<oop>(oc, worker_i);
+    }
+  }
   void updateRS(int worker_i);
   HeapRegion* calculateStartRegion(int i);
 
@@ -172,12 +198,22 @@
 
   // Record, if necessary, the fact that *p (where "p" is in region "from",
   // which is required to be non-NULL) has changed to a new non-NULL value.
-  inline void write_ref(HeapRegion* from, oop* p);
-  // The "_nv" version is the same; it exists just so that it is not virtual.
-  inline void write_ref_nv(HeapRegion* from, oop* p);
+  // [Below the virtual version calls a non-virtual protected
+  // workhorse that is templatified for narrow vs wide oop.]
+  inline void write_ref(HeapRegion* from, oop* p) {
+    write_ref_nv(from, p);
+  }
+  inline void write_ref(HeapRegion* from, narrowOop* p) {
+    write_ref_nv(from, p);
+  }
+  inline void par_write_ref(HeapRegion* from, oop* p, int tid) {
+    par_write_ref_nv(from, p, tid);
+  }
+  inline void par_write_ref(HeapRegion* from, narrowOop* p, int tid) {
+    par_write_ref_nv(from, p, tid);
+  }
 
-  inline bool self_forwarded(oop obj);
-  inline void par_write_ref(HeapRegion* from, oop* p, int tid);
+  bool self_forwarded(oop obj);
 
   void scrub(BitMap* region_bm, BitMap* card_bm);
   void scrub_par(BitMap* region_bm, BitMap* card_bm,
@@ -208,6 +244,9 @@
   HeapRegion* _from;
   HRInto_G1RemSet* _rs;
   int _worker_i;
+
+  template <class T> void do_oop_work(T* p);
+
 public:
   UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
     _from(NULL), _rs(rs), _worker_i(worker_i) {
@@ -219,11 +258,10 @@
     _from = from;
   }
 
-  virtual void do_oop(narrowOop* p);
-  virtual void do_oop(oop* p);
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
 
   // Override: this closure is idempotent.
   //  bool idempotent() { return true; }
   bool apply_to_weak_ref_discovered_field() { return true; }
 };
-
--- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,12 +30,8 @@
   }
 }
 
-inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, oop* p) {
-  par_write_ref(from, p, 0);
-}
-
-inline void HRInto_G1RemSet::write_ref(HeapRegion* from, oop* p) {
-  write_ref_nv(from, p);
+template <class T> inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, T* p) {
+  par_write_ref_nv(from, p, 0);
 }
 
 inline bool HRInto_G1RemSet::self_forwarded(oop obj) {
@@ -43,8 +39,8 @@
   return result;
 }
 
-inline void HRInto_G1RemSet::par_write_ref(HeapRegion* from, oop* p, int tid) {
-  oop obj = *p;
+template <class T> inline void HRInto_G1RemSet::par_write_ref_nv(HeapRegion* from, T* p, int tid) {
+  oop obj = oopDesc::load_decode_heap_oop(p);
 #ifdef ASSERT
   // can't do because of races
   // assert(obj == NULL || obj->is_oop(), "expected an oop");
@@ -71,7 +67,7 @@
     // false during the evacuation failure handing.
     if (_par_traversal_in_progress &&
         to->in_collection_set() && !self_forwarded(obj)) {
-      _new_refs[tid]->push(p);
+      _new_refs[tid]->push((void*)p);
       // Deferred updates to the Cset are either discarded (in the normal case),
       // or processed (if an evacuation failure occurs) at the end
       // of the collection.
@@ -89,11 +85,7 @@
   }
 }
 
-inline void UpdateRSOopClosure::do_oop(narrowOop* p) {
-  guarantee(false, "NYI");
-}
-
-inline void UpdateRSOopClosure::do_oop(oop* p) {
+template <class T> inline void UpdateRSOopClosure::do_oop_work(T* p) {
   assert(_from != NULL, "from region must be non-NULL");
   _rs->par_write_ref(_from, p, _worker_i);
 }
--- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -34,6 +34,7 @@
 
 
 void G1SATBCardTableModRefBS::enqueue(oop pre_val) {
+  assert(pre_val->is_oop_or_null(true), "Error");
   if (!JavaThread::satb_mark_queue_set().active()) return;
   Thread* thr = Thread::current();
   if (thr->is_Java_thread()) {
@@ -46,32 +47,31 @@
 }
 
 // When we know the current java thread:
-void
-G1SATBCardTableModRefBS::write_ref_field_pre_static(void* field,
-                                                    oop newVal,
+template <class T> void
+G1SATBCardTableModRefBS::write_ref_field_pre_static(T* field,
+                                                    oop new_val,
                                                     JavaThread* jt) {
   if (!JavaThread::satb_mark_queue_set().active()) return;
-  assert(!UseCompressedOops, "Else will need to modify this to deal with narrowOop");
-  oop preVal = *(oop*)field;
-  if (preVal != NULL) {
-    jt->satb_mark_queue().enqueue(preVal);
+  T heap_oop = oopDesc::load_heap_oop(field);
+  if (!oopDesc::is_null(heap_oop)) {
+    oop pre_val = oopDesc::decode_heap_oop_not_null(heap_oop);
+    assert(pre_val->is_oop(true /* ignore mark word */), "Error");
+    jt->satb_mark_queue().enqueue(pre_val);
   }
 }
 
-void
-G1SATBCardTableModRefBS::write_ref_array_pre(MemRegion mr) {
+template <class T> void
+G1SATBCardTableModRefBS::write_ref_array_pre_work(T* dst, int count) {
   if (!JavaThread::satb_mark_queue_set().active()) return;
-  assert(!UseCompressedOops, "Else will need to modify this to deal with narrowOop");
-  oop* elem_ptr = (oop*)mr.start();
-  while ((HeapWord*)elem_ptr < mr.end()) {
-    oop elem = *elem_ptr;
-    if (elem != NULL) enqueue(elem);
-    elem_ptr++;
+  T* elem_ptr = dst;
+  for (int i = 0; i < count; i++, elem_ptr++) {
+    T heap_oop = oopDesc::load_heap_oop(elem_ptr);
+    if (!oopDesc::is_null(heap_oop)) {
+      enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
+    }
   }
 }
 
-
-
 G1SATBCardTableLoggingModRefBS::
 G1SATBCardTableLoggingModRefBS(MemRegion whole_heap,
                                int max_covered_regions) :
--- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -47,31 +47,41 @@
 
   // This notes that we don't need to access any BarrierSet data
   // structures, so this can be called from a static context.
-  static void write_ref_field_pre_static(void* field, oop newVal) {
-    assert(!UseCompressedOops, "Else needs to be templatized");
-    oop preVal = *((oop*)field);
-    if (preVal != NULL) {
-      enqueue(preVal);
+  template <class T> static void write_ref_field_pre_static(T* field, oop newVal) {
+    T heap_oop = oopDesc::load_heap_oop(field);
+    if (!oopDesc::is_null(heap_oop)) {
+      enqueue(oopDesc::decode_heap_oop(heap_oop));
     }
   }
 
   // When we know the current java thread:
-  static void write_ref_field_pre_static(void* field, oop newVal,
-                                         JavaThread* jt);
+  template <class T> static void write_ref_field_pre_static(T* field, oop newVal,
+                                                            JavaThread* jt);
 
   // We export this to make it available in cases where the static
   // type of the barrier set is known.  Note that it is non-virtual.
-  inline void inline_write_ref_field_pre(void* field, oop newVal) {
+  template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal) {
     write_ref_field_pre_static(field, newVal);
   }
 
-  // This is the more general virtual version.
-  void write_ref_field_pre_work(void* field, oop new_val) {
+  // These are the more general virtual versions.
+  virtual void write_ref_field_pre_work(oop* field, oop new_val) {
     inline_write_ref_field_pre(field, new_val);
   }
+  virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {
+    inline_write_ref_field_pre(field, new_val);
+  }
+  virtual void write_ref_field_pre_work(void* field, oop new_val) {
+    guarantee(false, "Not needed");
+  }
 
-  virtual void write_ref_array_pre(MemRegion mr);
-
+  template <class T> void write_ref_array_pre_work(T* dst, int count);
+  virtual void write_ref_array_pre(oop* dst, int count) {
+    write_ref_array_pre_work(dst, count);
+  }
+  virtual void write_ref_array_pre(narrowOop* dst, int count) {
+    write_ref_array_pre_work(dst, count);
+  }
 };
 
 // Adds card-table logging to the post-barrier.
--- a/src/share/vm/gc_implementation/g1/g1_globals.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -34,7 +34,7 @@
   product(intx, G1ConfidencePercent, 50,                                    \
           "Confidence level for MMU/pause predictions")                     \
                                                                             \
-  develop(intx, G1MarkingOverheadPercent, 0,                                   \
+  develop(intx, G1MarkingOverheadPercent, 0,                                \
           "Overhead of concurrent marking")                                 \
                                                                             \
   develop(bool, G1AccountConcurrentOverhead, false,                         \
@@ -47,7 +47,7 @@
   develop(bool, G1Gen, true,                                                \
           "If true, it will enable the generational G1")                    \
                                                                             \
-  develop(intx, G1GCPercent, 10,                                                \
+  develop(intx, G1GCPercent, 10,                                            \
           "The desired percent time spent on GC")                           \
                                                                             \
   develop(intx, G1PolicyVerbose, 0,                                         \
@@ -74,15 +74,18 @@
   diagnostic(bool, G1SummarizeRSetStats, false,                             \
           "Summarize remembered set processing info")                       \
                                                                             \
+  diagnostic(intx, G1SummarizeRSetStatsPeriod, 0,                           \
+          "The period (in number of GCs) at which we will generate "        \
+          "update buffer processing info "                                  \
+          "(0 means do not periodically generate this info); "              \
+          "it also requires -XX:+G1SummarizeRSetStats")                     \
+                                                                            \
   diagnostic(bool, G1SummarizeZFStats, false,                               \
           "Summarize zero-filling info")                                    \
                                                                             \
   develop(bool, G1TraceConcurrentRefinement, false,                         \
           "Trace G1 concurrent refinement")                                 \
                                                                             \
-  develop(bool, G1ConcMark, true,                                           \
-          "If true, run concurrent marking for G1")                         \
-                                                                            \
   product(intx, G1MarkStackSize, 2 * 1024 * 1024,                           \
           "Size of the mark stack for concurrent marking.")                 \
                                                                             \
@@ -170,17 +173,20 @@
   develop(bool, G1DisablePostBarrier, false,                                \
           "Disable generation of post-barrier (i.e., RS barrier)   ")       \
                                                                             \
-  product(intx, G1DirtyCardQueueMax, 30,                                    \
-          "Maximum number of completed RS buffers before mutator threads "  \
-          "start processing them.")                                         \
+  product(intx, G1UpdateBufferSize, 256,                                    \
+          "Size of an update buffer")                                       \
+                                                                            \
+  product(intx, G1UpdateBufferQueueProcessingThreshold, 5,                  \
+          "Number of enqueued update buffers that will "                    \
+          "trigger concurrent processing")                                  \
+                                                                            \
+  product(intx, G1UpdateBufferQueueMaxLength, 30,                           \
+          "Maximum number of enqueued update buffers before mutator "       \
+          "threads start processing new ones instead of enqueueing them")   \
                                                                             \
   develop(intx, G1ConcRSLogCacheSize, 10,                                   \
           "Log base 2 of the length of conc RS hot-card cache.")            \
                                                                             \
-  develop(bool, G1ConcRSCountTraversals, false,                             \
-          "If true, gather data about the number of times CR traverses "    \
-          "cards ")                                                         \
-                                                                            \
   develop(intx, G1ConcRSHotCardLimit, 4,                                    \
           "The threshold that defines (>=) a hot card.")                    \
                                                                             \
@@ -254,6 +260,10 @@
                                                                             \
   product(uintx, G1ParallelRSetThreads, 0,                                  \
           "If non-0 is the number of parallel rem set update threads, "     \
-          "otherwise the value is determined ergonomically.")
+          "otherwise the value is determined ergonomically.")               \
+                                                                            \
+  develop(intx, G1CardCountCacheExpandThreshold, 16,                        \
+          "Expand the card count cache if the number of collisions for "    \
+          "a particular entry exceeds this value.")
 
 G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG)
--- a/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -37,14 +37,12 @@
 class G1ParCopyClosure;
 class G1ParScanClosure;
 
-typedef G1ParCopyClosure<false, G1BarrierEvac, false, true>
-                                                      G1ParScanHeapEvacClosure;
+typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
 
 class FilterIntoCSClosure;
 class FilterOutOfRegionClosure;
 class FilterInHeapRegionAndIntoCSClosure;
 class FilterAndMarkInHeapRegionAndIntoCSClosure;
-class G1ScanAndBalanceClosure;
 
 #ifdef FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES
 #error "FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES already defined."
@@ -56,8 +54,7 @@
       f(FilterIntoCSClosure,_nv)                        \
       f(FilterOutOfRegionClosure,_nv)                   \
       f(FilterInHeapRegionAndIntoCSClosure,_nv)         \
-      f(FilterAndMarkInHeapRegionAndIntoCSClosure,_nv)  \
-      f(G1ScanAndBalanceClosure,_nv)
+      f(FilterAndMarkInHeapRegionAndIntoCSClosure,_nv)
 
 #ifdef FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES
 #error "FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES already defined."
--- a/src/share/vm/gc_implementation/g1/heapRegion.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -66,16 +66,16 @@
   bool failures() { return _failures; }
   int n_failures() { return _n_failures; }
 
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
 
-  void do_oop(oop* p) {
+  template <class T> void do_oop_work(T* p) {
     assert(_containing_obj != NULL, "Precondition");
     assert(!_g1h->is_obj_dead_cond(_containing_obj, _use_prev_marking),
            "Precondition");
-    oop obj = *p;
-    if (obj != NULL) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
       bool failed = false;
       if (!_g1h->is_in_closed_subset(obj) ||
           _g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
@@ -106,8 +106,8 @@
       }
 
       if (!_g1h->full_collection()) {
-        HeapRegion* from = _g1h->heap_region_containing(p);
-        HeapRegion* to   = _g1h->heap_region_containing(*p);
+        HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
+        HeapRegion* to   = _g1h->heap_region_containing(obj);
         if (from != NULL && to != NULL &&
             from != to &&
             !to->isHumongous()) {
@@ -534,13 +534,13 @@
   // Otherwise, find the obj that extends onto mr.start().
 
   assert(cur <= mr.start()
-         && (oop(cur)->klass() == NULL ||
+         && (oop(cur)->klass_or_null() == NULL ||
              cur + oop(cur)->size() > mr.start()),
          "postcondition of block_start");
   oop obj;
   while (cur < mr.end()) {
     obj = oop(cur);
-    if (obj->klass() == NULL) {
+    if (obj->klass_or_null() == NULL) {
       // Ran into an unparseable point.
       return cur;
     } else if (!g1h->is_obj_dead(obj)) {
@@ -577,7 +577,7 @@
   assert(cur <= mr.start(), "Postcondition");
 
   while (cur <= mr.start()) {
-    if (oop(cur)->klass() == NULL) {
+    if (oop(cur)->klass_or_null() == NULL) {
       // Ran into an unparseable point.
       return cur;
     }
@@ -591,7 +591,7 @@
   obj = oop(cur);
   // If we finish this loop...
   assert(cur <= mr.start()
-         && obj->klass() != NULL
+         && obj->klass_or_null() != NULL
          && cur + obj->size() > mr.start(),
          "Loop postcondition");
   if (!g1h->is_obj_dead(obj)) {
@@ -601,7 +601,7 @@
   HeapWord* next;
   while (cur < mr.end()) {
     obj = oop(cur);
-    if (obj->klass() == NULL) {
+    if (obj->klass_or_null() == NULL) {
       // Ran into an unparseable point.
       return cur;
     };
@@ -781,8 +781,13 @@
     // will pick up the right saved_mark_word() as the high water mark
     // of the region. Either way, the behaviour will be correct.
     ContiguousSpace::set_saved_mark();
+    OrderAccess::storestore();
     _gc_time_stamp = curr_gc_time_stamp;
-    OrderAccess::fence();
+    // The following fence is to force a flush of the writes above, but
+    // is strictly not needed because when an allocating worker thread
+    // calls set_saved_mark() it does so under the ParGCRareEvent_lock;
+    // when the lock is released, the write will be flushed.
+    // OrderAccess::fence();
   }
 }
 
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -126,7 +126,7 @@
     }
   }
 
-  void add_reference_work(oop* from, bool par) {
+  void add_reference_work(OopOrNarrowOopStar from, bool par) {
     // Must make this robust in case "from" is not in "_hr", because of
     // concurrency.
 
@@ -173,11 +173,11 @@
     _bm.clear();
   }
 
-  void add_reference(oop* from) {
+  void add_reference(OopOrNarrowOopStar from) {
     add_reference_work(from, /*parallel*/ true);
   }
 
-  void seq_add_reference(oop* from) {
+  void seq_add_reference(OopOrNarrowOopStar from) {
     add_reference_work(from, /*parallel*/ false);
   }
 
@@ -220,7 +220,7 @@
   }
 
   // Requires "from" to be in "hr()".
-  bool contains_reference(oop* from) const {
+  bool contains_reference(OopOrNarrowOopStar from) const {
     assert(hr()->is_in_reserved(from), "Precondition.");
     size_t card_ind = pointer_delta(from, hr()->bottom(),
                                     CardTableModRefBS::card_size);
@@ -394,7 +394,7 @@
   void set_next(PosParPRT* nxt) { _next = nxt; }
   PosParPRT** next_addr() { return &_next; }
 
-  void add_reference(oop* from, int tid) {
+  void add_reference(OopOrNarrowOopStar from, int tid) {
     // Expand if necessary.
     PerRegionTable** pt = par_tables();
     if (par_tables() == NULL && tid > 0 && hr()->is_gc_alloc_region()) {
@@ -447,7 +447,7 @@
     return res;
   }
 
-  bool contains_reference(oop* from) const {
+  bool contains_reference(OopOrNarrowOopStar from) const {
     if (PerRegionTable::contains_reference(from)) return true;
     if (_par_tables != NULL) {
       for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) {
@@ -564,12 +564,15 @@
 }
 #endif
 
-void OtherRegionsTable::add_reference(oop* from, int tid) {
+void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
   size_t cur_hrs_ind = hr()->hrs_index();
 
 #if HRRS_VERBOSE
   gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").",
-                                                  from, *from);
+                                                  from,
+                                                  UseCompressedOops
+                                                  ? oopDesc::load_decode_heap_oop((narrowOop*)from)
+                                                  : oopDesc::load_decode_heap_oop((oop*)from));
 #endif
 
   int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);
@@ -1021,13 +1024,13 @@
   }
 }
 
-bool OtherRegionsTable::contains_reference(oop* from) const {
+bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const {
   // Cast away const in this case.
   MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag);
   return contains_reference_locked(from);
 }
 
-bool OtherRegionsTable::contains_reference_locked(oop* from) const {
+bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const {
   HeapRegion* hr = _g1h->heap_region_containing_raw(from);
   if (hr == NULL) return false;
   RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index();
@@ -1288,24 +1291,24 @@
 
 
 
-oop**        HeapRegionRemSet::_recorded_oops = NULL;
-HeapWord**   HeapRegionRemSet::_recorded_cards = NULL;
-HeapRegion** HeapRegionRemSet::_recorded_regions = NULL;
-int          HeapRegionRemSet::_n_recorded = 0;
+OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL;
+HeapWord**          HeapRegionRemSet::_recorded_cards = NULL;
+HeapRegion**        HeapRegionRemSet::_recorded_regions = NULL;
+int                 HeapRegionRemSet::_n_recorded = 0;
 
 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL;
 int*         HeapRegionRemSet::_recorded_event_index = NULL;
 int          HeapRegionRemSet::_n_recorded_events = 0;
 
-void HeapRegionRemSet::record(HeapRegion* hr, oop* f) {
+void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) {
   if (_recorded_oops == NULL) {
     assert(_n_recorded == 0
            && _recorded_cards == NULL
            && _recorded_regions == NULL,
            "Inv");
-    _recorded_oops = NEW_C_HEAP_ARRAY(oop*, MaxRecorded);
-    _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded);
-    _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded);
+    _recorded_oops    = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded);
+    _recorded_cards   = NEW_C_HEAP_ARRAY(HeapWord*,          MaxRecorded);
+    _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*,        MaxRecorded);
   }
   if (_n_recorded == MaxRecorded) {
     gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded);
@@ -1408,21 +1411,21 @@
   HeapRegionRemSet* hrrs = hr0->rem_set();
 
   // Make three references from region 0x101...
-  hrrs->add_reference((oop*)hr1_start);
-  hrrs->add_reference((oop*)hr1_mid);
-  hrrs->add_reference((oop*)hr1_last);
+  hrrs->add_reference((OopOrNarrowOopStar)hr1_start);
+  hrrs->add_reference((OopOrNarrowOopStar)hr1_mid);
+  hrrs->add_reference((OopOrNarrowOopStar)hr1_last);
 
-  hrrs->add_reference((oop*)hr2_start);
-  hrrs->add_reference((oop*)hr2_mid);
-  hrrs->add_reference((oop*)hr2_last);
+  hrrs->add_reference((OopOrNarrowOopStar)hr2_start);
+  hrrs->add_reference((OopOrNarrowOopStar)hr2_mid);
+  hrrs->add_reference((OopOrNarrowOopStar)hr2_last);
 
-  hrrs->add_reference((oop*)hr3_start);
-  hrrs->add_reference((oop*)hr3_mid);
-  hrrs->add_reference((oop*)hr3_last);
+  hrrs->add_reference((OopOrNarrowOopStar)hr3_start);
+  hrrs->add_reference((OopOrNarrowOopStar)hr3_mid);
+  hrrs->add_reference((OopOrNarrowOopStar)hr3_last);
 
   // Now cause a coarsening.
-  hrrs->add_reference((oop*)hr4->bottom());
-  hrrs->add_reference((oop*)hr5->bottom());
+  hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom());
+  hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom());
 
   // Now, does iteration yield these three?
   HeapRegionRemSetIterator iter;
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -116,9 +116,9 @@
 
   // For now.  Could "expand" some tables in the future, so that this made
   // sense.
-  void add_reference(oop* from, int tid);
+  void add_reference(OopOrNarrowOopStar from, int tid);
 
-  void add_reference(oop* from) {
+  void add_reference(OopOrNarrowOopStar from) {
     return add_reference(from, 0);
   }
 
@@ -140,8 +140,8 @@
   static size_t static_mem_size();
   static size_t fl_mem_size();
 
-  bool contains_reference(oop* from) const;
-  bool contains_reference_locked(oop* from) const;
+  bool contains_reference(OopOrNarrowOopStar from) const;
+  bool contains_reference_locked(OopOrNarrowOopStar from) const;
 
   void clear();
 
@@ -192,10 +192,10 @@
   // Unused unless G1RecordHRRSOops is true.
 
   static const int MaxRecorded = 1000000;
-  static oop**        _recorded_oops;
-  static HeapWord**   _recorded_cards;
-  static HeapRegion** _recorded_regions;
-  static int          _n_recorded;
+  static OopOrNarrowOopStar* _recorded_oops;
+  static HeapWord**          _recorded_cards;
+  static HeapRegion**        _recorded_regions;
+  static int                 _n_recorded;
 
   static const int MaxRecordedEvents = 1000;
   static Event*       _recorded_events;
@@ -231,13 +231,13 @@
 
   /* Used in the sequential case.  Returns "true" iff this addition causes
      the size limit to be reached. */
-  void add_reference(oop* from) {
+  void add_reference(OopOrNarrowOopStar from) {
     _other_regions.add_reference(from);
   }
 
   /* Used in the parallel case.  Returns "true" iff this addition causes
      the size limit to be reached. */
-  void add_reference(oop* from, int tid) {
+  void add_reference(OopOrNarrowOopStar from, int tid) {
     _other_regions.add_reference(from, tid);
   }
 
@@ -301,7 +301,7 @@
     return OtherRegionsTable::fl_mem_size();
   }
 
-  bool contains_reference(oop* from) const {
+  bool contains_reference(OopOrNarrowOopStar from) const {
     return _other_regions.contains_reference(from);
   }
   void print() const;
@@ -329,7 +329,7 @@
   }
 #endif
 
-  static void record(HeapRegion* hr, oop* f);
+  static void record(HeapRegion* hr, OopOrNarrowOopStar f);
   static void print_recorded();
   static void record_event(Event evnt);
 
--- a/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/satbQueue.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -43,6 +43,18 @@
     }
   }
 }
+
+#ifdef ASSERT
+void ObjPtrQueue::verify_oops_in_buffer() {
+  if (_buf == NULL) return;
+  for (size_t i = _index; i < _sz; i += oopSize) {
+    oop obj = (oop)_buf[byte_index_to_index((int)i)];
+    assert(obj != NULL && obj->is_oop(true /* ignore mark word */),
+           "Not an oop");
+  }
+}
+#endif
+
 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 #endif // _MSC_VER
@@ -66,6 +78,7 @@
 
 
 void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) {
+  DEBUG_ONLY(t->satb_mark_queue().verify_oops_in_buffer();)
   t->satb_mark_queue().handle_zero_index();
 }
 
@@ -143,7 +156,7 @@
     }
     _completed_buffers_tail = NULL;
     _n_completed_buffers = 0;
-    debug_only(assert_completed_buffer_list_len_correct_locked());
+    DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked());
   }
   while (buffers_to_delete != NULL) {
     CompletedBufferNode* nd = buffers_to_delete;
--- a/src/share/vm/gc_implementation/g1/satbQueue.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -39,6 +39,7 @@
   static void apply_closure_to_buffer(ObjectClosure* cl,
                                       void** buf, size_t index, size_t sz);
 
+  void verify_oops_in_buffer() NOT_DEBUG_RETURN;
 };
 
 
--- a/src/share/vm/gc_implementation/g1/sparsePRT.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/sparsePRT.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/vm_operations_g1.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/vm_operations_g1.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/includeDB_gc_g1	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/includeDB_gc_g1	Mon Sep 07 10:57:43 2009 -0700
@@ -27,6 +27,7 @@
 bufferingOopClosure.hpp			genOopClosures.hpp
 bufferingOopClosure.hpp			generation.hpp
 bufferingOopClosure.hpp			os.hpp
+bufferingOopClosure.hpp			taskqueue.hpp
 
 cardTableRS.cpp				concurrentMark.hpp
 cardTableRS.cpp				g1SATBCardTableModRefBS.hpp
@@ -44,11 +45,14 @@
 concurrentG1Refine.cpp			concurrentG1RefineThread.hpp
 concurrentG1Refine.cpp			copy.hpp
 concurrentG1Refine.cpp			g1CollectedHeap.inline.hpp
+concurrentG1Refine.cpp                  g1CollectorPolicy.hpp
 concurrentG1Refine.cpp			g1RemSet.hpp
 concurrentG1Refine.cpp			space.inline.hpp
+concurrentG1Refine.cpp                  heapRegionSeq.inline.hpp
 
 concurrentG1Refine.hpp			globalDefinitions.hpp
 concurrentG1Refine.hpp			allocation.hpp
+concurrentG1Refine.hpp                  cardTableModRefBS.hpp
 concurrentG1Refine.hpp			thread.hpp
 
 concurrentG1RefineThread.cpp		concurrentG1Refine.hpp
@@ -139,7 +143,7 @@
 g1CollectedHeap.cpp                     g1CollectedHeap.inline.hpp
 g1CollectedHeap.cpp                     g1CollectorPolicy.hpp
 g1CollectedHeap.cpp                     g1MarkSweep.hpp
-g1CollectedHeap.cpp                     g1RemSet.hpp
+g1CollectedHeap.cpp                     g1RemSet.inline.hpp
 g1CollectedHeap.cpp                     g1OopClosures.inline.hpp
 g1CollectedHeap.cpp                     genOopClosures.inline.hpp
 g1CollectedHeap.cpp                     gcLocker.inline.hpp
@@ -151,13 +155,14 @@
 g1CollectedHeap.cpp                     isGCActiveMark.hpp
 g1CollectedHeap.cpp			oop.inline.hpp
 g1CollectedHeap.cpp			oop.pcgc.inline.hpp
-g1CollectedHeap.cpp			parGCAllocBuffer.hpp
 g1CollectedHeap.cpp                     vm_operations_g1.hpp
 g1CollectedHeap.cpp                     vmThread.hpp
 
 g1CollectedHeap.hpp                     barrierSet.hpp
+g1CollectedHeap.hpp                     g1RemSet.hpp
 g1CollectedHeap.hpp                     heapRegion.hpp
 g1CollectedHeap.hpp                     memRegion.hpp
+g1CollectedHeap.hpp			parGCAllocBuffer.hpp
 g1CollectedHeap.hpp                     sharedHeap.hpp
 
 g1CollectedHeap.inline.hpp              concurrentMark.hpp
@@ -245,6 +250,7 @@
 g1RemSet.cpp				iterator.hpp
 g1RemSet.cpp				oop.inline.hpp
 
+g1RemSet.inline.hpp			oop.inline.hpp
 g1RemSet.inline.hpp			g1RemSet.hpp
 g1RemSet.inline.hpp			heapRegionRemSet.hpp
 
@@ -255,6 +261,7 @@
 g1SATBCardTableModRefBS.cpp		thread_<os_family>.inline.hpp
 g1SATBCardTableModRefBS.cpp		satbQueue.hpp
 
+g1SATBCardTableModRefBS.hpp		oop.inline.hpp
 g1SATBCardTableModRefBS.hpp		cardTableModRefBS.hpp
 g1SATBCardTableModRefBS.hpp		memRegion.hpp
 
--- a/src/share/vm/gc_implementation/parNew/parCardTableModRefBS.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parNew/parCardTableModRefBS.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -31,9 +31,10 @@
                                                         bool clear,
                                                         int n_threads) {
   if (n_threads > 0) {
-    assert(n_threads == (int)ParallelGCThreads, "# worker threads != # requested!");
-
-      // Make sure the LNC array is valid for the space.
+    assert((n_threads == 1 && ParallelGCThreads == 0) ||
+           n_threads <= (int)ParallelGCThreads,
+           "# worker threads != # requested!");
+    // Make sure the LNC array is valid for the space.
     jbyte**   lowest_non_clean;
     uintptr_t lowest_non_clean_base_chunk_index;
     size_t    lowest_non_clean_chunk_size;
--- a/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -885,7 +885,7 @@
 }
 
 
-void ParallelScavengeHeap::verify(bool allow_dirty, bool silent) {
+void ParallelScavengeHeap::verify(bool allow_dirty, bool silent, bool option /* ignored */) {
   // Why do we need the total_collections()-filter below?
   if (total_collections() > 0) {
     if (!silent) {
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -217,7 +217,7 @@
   virtual void gc_threads_do(ThreadClosure* tc) const;
   virtual void print_tracing_info() const;
 
-  void verify(bool allow_dirty, bool silent);
+  void verify(bool allow_dirty, bool silent, bool /* option */);
 
   void print_heap_change(size_t prev_used);
 
--- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -117,6 +117,7 @@
     process_array_chunk(old);
   } else {
     if (p.is_narrow()) {
+      assert(UseCompressedOops, "Error");
       PSScavenge::copy_and_push_safe_barrier(this, (narrowOop*)p);
     } else {
       PSScavenge::copy_and_push_safe_barrier(this, (oop*)p);
--- a/src/share/vm/gc_implementation/shared/concurrentGCThread.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/concurrentGCThread.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/concurrentGCThread.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/concurrentGCThread.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_interface/collectedHeap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -533,7 +533,7 @@
   virtual void print_tracing_info() const = 0;
 
   // Heap verification
-  virtual void verify(bool allow_dirty, bool silent) = 0;
+  virtual void verify(bool allow_dirty, bool silent, bool option) = 0;
 
   // Non product verification and debugging.
 #ifndef PRODUCT
--- a/src/share/vm/gc_interface/gcCause.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/gc_interface/gcCause.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/includeDB_compiler1	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/includeDB_compiler1	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 //
-// Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/includeDB_core	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/includeDB_core	Mon Sep 07 10:57:43 2009 -0700
@@ -554,7 +554,6 @@
 ciEnv.cpp                               linkResolver.hpp
 ciEnv.cpp                               methodDataOop.hpp
 ciEnv.cpp                               objArrayKlass.hpp
-ciEnv.cpp                               oop.hpp
 ciEnv.cpp                               oop.inline.hpp
 ciEnv.cpp                               oop.inline2.hpp
 ciEnv.cpp                               oopFactory.hpp
@@ -785,7 +784,6 @@
 ciSignature.cpp                         allocation.inline.hpp
 ciSignature.cpp                         ciSignature.hpp
 ciSignature.cpp                         ciUtilities.hpp
-ciSignature.cpp                         oop.hpp
 ciSignature.cpp                         oop.inline.hpp
 ciSignature.cpp                         signature.hpp
 
@@ -874,6 +872,7 @@
 classFileParser.cpp                     symbolOop.hpp
 classFileParser.cpp                     symbolTable.hpp
 classFileParser.cpp                     systemDictionary.hpp
+classFileParser.cpp                     threadService.hpp
 classFileParser.cpp                     timer.hpp
 classFileParser.cpp                     universe.inline.hpp
 classFileParser.cpp                     verificationType.hpp
@@ -926,6 +925,7 @@
 classLoader.cpp                         symbolOop.hpp
 classLoader.cpp                         systemDictionary.hpp
 classLoader.cpp                         threadCritical.hpp
+classLoader.cpp                         threadService.hpp
 classLoader.cpp                         timer.hpp
 classLoader.cpp                         universe.inline.hpp
 classLoader.cpp                         vmSymbols.hpp
@@ -950,7 +950,6 @@
 classify.cpp                            classify.hpp
 classify.cpp                            systemDictionary.hpp
 
-classify.hpp                            oop.hpp
 classify.hpp                            oop.inline.hpp
 
 codeBlob.cpp                            allocation.inline.hpp
@@ -1185,7 +1184,6 @@
 compilerOracle.cpp                      jniHandles.hpp
 compilerOracle.cpp                      klass.hpp
 compilerOracle.cpp                      methodOop.hpp
-compilerOracle.cpp                      oop.hpp
 compilerOracle.cpp                      oop.inline.hpp
 compilerOracle.cpp                      oopFactory.hpp
 compilerOracle.cpp                      resourceArea.hpp
@@ -1629,7 +1627,6 @@
 frame.cpp                               methodOop.hpp
 frame.cpp                               monitorChunk.hpp
 frame.cpp                               nativeInst_<arch>.hpp
-frame.cpp                               oop.hpp
 frame.cpp                               oop.inline.hpp
 frame.cpp                               oop.inline2.hpp
 frame.cpp                               oopMapCache.hpp
@@ -1797,7 +1794,6 @@
 generation.cpp                          generation.hpp
 generation.cpp                          generation.inline.hpp
 generation.cpp                          java.hpp
-generation.cpp                          oop.hpp
 generation.cpp                          oop.inline.hpp
 generation.cpp                          spaceDecorator.hpp
 generation.cpp                          space.inline.hpp
@@ -2270,7 +2266,6 @@
 java.cpp                                memprofiler.hpp
 java.cpp                                methodOop.hpp
 java.cpp                                objArrayOop.hpp
-java.cpp                                oop.hpp
 java.cpp                                oop.inline.hpp
 java.cpp                                oopFactory.hpp
 java.cpp                                sharedRuntime.hpp
@@ -2947,7 +2942,7 @@
 nativeInst_<arch>.cpp                   assembler_<arch>.inline.hpp
 nativeInst_<arch>.cpp                   handles.hpp
 nativeInst_<arch>.cpp                   nativeInst_<arch>.hpp
-nativeInst_<arch>.cpp                   oop.hpp
+nativeInst_<arch>.cpp                   oop.inline.hpp
 nativeInst_<arch>.cpp                   ostream.hpp
 nativeInst_<arch>.cpp                   resourceArea.hpp
 nativeInst_<arch>.cpp                   sharedRuntime.hpp
@@ -3842,7 +3837,7 @@
 stackValue.cpp                          debugInfo.hpp
 stackValue.cpp                          frame.inline.hpp
 stackValue.cpp                          handles.inline.hpp
-stackValue.cpp                          oop.hpp
+stackValue.cpp                          oop.inline.hpp
 stackValue.cpp                          stackValue.hpp
 
 stackValue.hpp                          handles.hpp
@@ -4026,6 +4021,7 @@
 systemDictionary.cpp                    resolutionErrors.hpp
 systemDictionary.cpp                    signature.hpp
 systemDictionary.cpp                    systemDictionary.hpp
+systemDictionary.cpp                    threadService.hpp
 systemDictionary.cpp                    typeArrayKlass.hpp
 systemDictionary.cpp                    vmSymbols.hpp
 
@@ -4329,7 +4325,6 @@
 unhandledOops.cpp                       collectedHeap.hpp
 unhandledOops.cpp                       gcLocker.inline.hpp
 unhandledOops.cpp                       globalDefinitions.hpp
-unhandledOops.cpp                       oop.hpp
 unhandledOops.cpp                       oop.inline.hpp
 unhandledOops.cpp                       thread.hpp
 unhandledOops.cpp                       unhandledOops.hpp
@@ -4465,7 +4460,6 @@
 vframe.cpp                              nmethod.hpp
 vframe.cpp                              objectMonitor.hpp
 vframe.cpp                              objectMonitor.inline.hpp
-vframe.cpp                              oop.hpp
 vframe.cpp                              oop.inline.hpp
 vframe.cpp                              oopMapCache.hpp
 vframe.cpp                              pcDesc.hpp
@@ -4577,7 +4571,6 @@
 vmThread.cpp                            interfaceSupport.hpp
 vmThread.cpp                            methodOop.hpp
 vmThread.cpp                            mutexLocker.hpp
-vmThread.cpp                            oop.hpp
 vmThread.cpp                            oop.inline.hpp
 vmThread.cpp                            os.hpp
 vmThread.cpp                            resourceArea.hpp
--- a/src/share/vm/includeDB_features	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/includeDB_features	Mon Sep 07 10:57:43 2009 -0700
@@ -47,7 +47,7 @@
 dump.cpp                                javaClasses.hpp
 dump.cpp                                loaderConstraints.hpp
 dump.cpp                                methodDataOop.hpp
-dump.cpp                                oop.hpp
+dump.cpp                                oop.inline.hpp
 dump.cpp                                oopFactory.hpp
 dump.cpp                                resourceArea.hpp
 dump.cpp                                signature.hpp
@@ -237,7 +237,7 @@
 serialize.cpp                           compiledICHolderOop.hpp
 serialize.cpp                           methodDataOop.hpp
 serialize.cpp                           objArrayOop.hpp
-serialize.cpp                           oop.hpp
+serialize.cpp                           oop.inline.hpp
 serialize.cpp                           symbolTable.hpp
 serialize.cpp                           systemDictionary.hpp
 
@@ -295,7 +295,7 @@
 vmStructs.cpp                           objArrayKlass.hpp
 vmStructs.cpp                           objArrayKlassKlass.hpp
 vmStructs.cpp                           objArrayOop.hpp
-vmStructs.cpp                           oop.hpp
+vmStructs.cpp                           oop.inline.hpp
 vmStructs.cpp                           oopMap.hpp
 vmStructs.cpp                           pcDesc.hpp
 vmStructs.cpp                           perfMemory.hpp
--- a/src/share/vm/includeDB_jvmti	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/includeDB_jvmti	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 //
-// Copyright 2007 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/abstractInterpreter.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/abstractInterpreter.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -122,11 +122,15 @@
   static int        size_top_interpreter_activation(methodOop method);
 
   // Deoptimization support
-  static address    continuation_for(methodOop method,
-                                     address bcp,
-                                     int callee_parameters,
-                                     bool is_top_frame,
-                                     bool& use_next_mdp);
+  // Compute the entry address for continuation after
+  static address deopt_continue_after_entry(methodOop method,
+                                            address bcp,
+                                            int callee_parameters,
+                                            bool is_top_frame);
+  // Compute the entry address for reexecution
+  static address deopt_reexecute_entry(methodOop method, address bcp);
+  // Deoptimization should reexecute this bytecode
+  static bool    bytecode_should_reexecute(Bytecodes::Code code);
 
   // share implementation of size_activation and layout_activation:
   static int        size_activation(methodOop method,
--- a/src/share/vm/interpreter/bytecode.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecode.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2002 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecode.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecode.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2002 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecodeStream.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecodeStream.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecodeTracer.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecodeTracer.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecodes.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecodes.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecodes.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/bytecodes.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/interpreter.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/interpreter.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -284,76 +284,19 @@
 //------------------------------------------------------------------------------------------------------------------------
 // Deoptimization support
 
-// If deoptimization happens, this method returns the point where to continue in
-// interpreter. For calls (invokexxxx, newxxxx) the continuation is at next
-// bci and the top of stack is in eax/edx/FPU tos.
-// For putfield/getfield, put/getstatic, the continuation is at the same
-// bci and the TOS is on stack.
-
-// Note: deopt_entry(type, 0) means reexecute bytecode
-//       deopt_entry(type, length) means continue at next bytecode
-
-address AbstractInterpreter::continuation_for(methodOop method, address bcp, int callee_parameters, bool is_top_frame, bool& use_next_mdp) {
+// If deoptimization happens, this function returns the point of next bytecode to continue execution
+address AbstractInterpreter::deopt_continue_after_entry(methodOop method, address bcp, int callee_parameters, bool is_top_frame) {
   assert(method->contains(bcp), "just checkin'");
   Bytecodes::Code code   = Bytecodes::java_code_at(bcp);
+  assert(!Interpreter::bytecode_should_reexecute(code), "should not reexecute");
   int             bci    = method->bci_from(bcp);
   int             length = -1; // initial value for debugging
   // compute continuation length
   length = Bytecodes::length_at(bcp);
   // compute result type
   BasicType type = T_ILLEGAL;
-  // when continuing after a compiler safepoint, re-execute the bytecode
-  // (an invoke is continued after the safepoint)
-  use_next_mdp = true;
+
   switch (code) {
-    case Bytecodes::_lookupswitch:
-    case Bytecodes::_tableswitch:
-    case Bytecodes::_fast_binaryswitch:
-    case Bytecodes::_fast_linearswitch:
-    // recompute condtional expression folded into _if<cond>
-    case Bytecodes::_lcmp      :
-    case Bytecodes::_fcmpl     :
-    case Bytecodes::_fcmpg     :
-    case Bytecodes::_dcmpl     :
-    case Bytecodes::_dcmpg     :
-    case Bytecodes::_ifnull    :
-    case Bytecodes::_ifnonnull :
-    case Bytecodes::_goto      :
-    case Bytecodes::_goto_w    :
-    case Bytecodes::_ifeq      :
-    case Bytecodes::_ifne      :
-    case Bytecodes::_iflt      :
-    case Bytecodes::_ifge      :
-    case Bytecodes::_ifgt      :
-    case Bytecodes::_ifle      :
-    case Bytecodes::_if_icmpeq :
-    case Bytecodes::_if_icmpne :
-    case Bytecodes::_if_icmplt :
-    case Bytecodes::_if_icmpge :
-    case Bytecodes::_if_icmpgt :
-    case Bytecodes::_if_icmple :
-    case Bytecodes::_if_acmpeq :
-    case Bytecodes::_if_acmpne :
-    // special cases
-    case Bytecodes::_getfield  :
-    case Bytecodes::_putfield  :
-    case Bytecodes::_getstatic :
-    case Bytecodes::_putstatic :
-    case Bytecodes::_aastore   :
-      // reexecute the operation and TOS value is on stack
-      assert(is_top_frame, "must be top frame");
-      use_next_mdp = false;
-      return Interpreter::deopt_entry(vtos, 0);
-      break;
-
-#ifdef COMPILER1
-    case Bytecodes::_athrow    :
-      assert(is_top_frame, "must be top frame");
-      use_next_mdp = false;
-      return Interpreter::rethrow_exception_entry();
-      break;
-#endif /* COMPILER1 */
-
     case Bytecodes::_invokevirtual  :
     case Bytecodes::_invokespecial  :
     case Bytecodes::_invokestatic   :
@@ -392,6 +335,70 @@
     : Interpreter::return_entry(as_TosState(type), length);
 }
 
+// If deoptimization happens, this function returns the point where the interpreter reexecutes
+// the bytecode.
+// Note: Bytecodes::_athrow is a special case in that it does not return
+//       Interpreter::deopt_entry(vtos, 0) like others
+address AbstractInterpreter::deopt_reexecute_entry(methodOop method, address bcp) {
+  assert(method->contains(bcp), "just checkin'");
+  Bytecodes::Code code   = Bytecodes::java_code_at(bcp);
+#ifdef COMPILER1
+  if(code == Bytecodes::_athrow ) {
+    return Interpreter::rethrow_exception_entry();
+  }
+#endif /* COMPILER1 */
+  return Interpreter::deopt_entry(vtos, 0);
+}
+
+// If deoptimization happens, the interpreter should reexecute these bytecodes.
+// This function mainly helps the compilers to set up the reexecute bit.
+bool AbstractInterpreter::bytecode_should_reexecute(Bytecodes::Code code) {
+  switch (code) {
+    case Bytecodes::_lookupswitch:
+    case Bytecodes::_tableswitch:
+    case Bytecodes::_fast_binaryswitch:
+    case Bytecodes::_fast_linearswitch:
+    // recompute condtional expression folded into _if<cond>
+    case Bytecodes::_lcmp      :
+    case Bytecodes::_fcmpl     :
+    case Bytecodes::_fcmpg     :
+    case Bytecodes::_dcmpl     :
+    case Bytecodes::_dcmpg     :
+    case Bytecodes::_ifnull    :
+    case Bytecodes::_ifnonnull :
+    case Bytecodes::_goto      :
+    case Bytecodes::_goto_w    :
+    case Bytecodes::_ifeq      :
+    case Bytecodes::_ifne      :
+    case Bytecodes::_iflt      :
+    case Bytecodes::_ifge      :
+    case Bytecodes::_ifgt      :
+    case Bytecodes::_ifle      :
+    case Bytecodes::_if_icmpeq :
+    case Bytecodes::_if_icmpne :
+    case Bytecodes::_if_icmplt :
+    case Bytecodes::_if_icmpge :
+    case Bytecodes::_if_icmpgt :
+    case Bytecodes::_if_icmple :
+    case Bytecodes::_if_acmpeq :
+    case Bytecodes::_if_acmpne :
+    // special cases
+    case Bytecodes::_getfield  :
+    case Bytecodes::_putfield  :
+    case Bytecodes::_getstatic :
+    case Bytecodes::_putstatic :
+    case Bytecodes::_aastore   :
+#ifdef COMPILER1
+    //special case of reexecution
+    case Bytecodes::_athrow    :
+#endif
+      return true;
+
+    default:
+      return false;
+  }
+}
+
 void AbstractInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
   // Quick & dirty stack overflow checking: bang the stack & handle trap.
   // Note that we do the banging after the frame is setup, since the exception
--- a/src/share/vm/interpreter/invocationCounter.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/invocationCounter.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/rewriter.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/rewriter.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -273,6 +273,7 @@
   compute_index_maps();
 
   if (RegisterFinalizersAtInit && _klass->name() == vmSymbols::java_lang_Object()) {
+    bool did_rewrite = false;
     int i = _methods->length();
     while (i-- > 0) {
       methodOop method = (methodOop)_methods->obj_at(i);
@@ -281,9 +282,11 @@
         // object for finalization if needed.
         methodHandle m(THREAD, method);
         rewrite_Object_init(m, CHECK);
+        did_rewrite = true;
         break;
       }
     }
+    assert(did_rewrite, "must find Object::<init> to rewrite it");
   }
 
   // rewrite methods, in two passes
--- a/src/share/vm/interpreter/rewriter.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/rewriter.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/templateInterpreter.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/templateInterpreter.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -605,28 +605,41 @@
   }
 }
 
-// If deoptimization happens, this method returns the point where to continue in
-// interpreter. For calls (invokexxxx, newxxxx) the continuation is at next
-// bci and the top of stack is in eax/edx/FPU tos.
-// For putfield/getfield, put/getstatic, the continuation is at the same
-// bci and the TOS is on stack.
+//------------------------------------------------------------------------------------------------------------------------
+// Deoptimization support
 
-// Note: deopt_entry(type, 0) means reexecute bytecode
-//       deopt_entry(type, length) means continue at next bytecode
+// If deoptimization happens, this function returns the point of next bytecode to continue execution
+address TemplateInterpreter::deopt_continue_after_entry(methodOop method, address bcp, int callee_parameters, bool is_top_frame) {
+  return AbstractInterpreter::deopt_continue_after_entry(method, bcp, callee_parameters, is_top_frame);
+}
 
-address TemplateInterpreter::continuation_for(methodOop method, address bcp, int callee_parameters, bool is_top_frame, bool& use_next_mdp) {
+// If deoptimization happens, this function returns the point where the interpreter reexecutes
+// the bytecode.
+// Note: Bytecodes::_athrow (C1 only) and Bytecodes::_return are the special cases
+//       that do not return "Interpreter::deopt_entry(vtos, 0)"
+address TemplateInterpreter::deopt_reexecute_entry(methodOop method, address bcp) {
   assert(method->contains(bcp), "just checkin'");
   Bytecodes::Code code   = Bytecodes::java_code_at(bcp);
   if (code == Bytecodes::_return) {
-      // This is used for deopt during registration of finalizers
-      // during Object.<init>.  We simply need to resume execution at
-      // the standard return vtos bytecode to pop the frame normally.
-      // reexecuting the real bytecode would cause double registration
-      // of the finalizable object.
-      assert(is_top_frame, "must be on top");
-      return _normal_table.entry(Bytecodes::_return).entry(vtos);
+    // This is used for deopt during registration of finalizers
+    // during Object.<init>.  We simply need to resume execution at
+    // the standard return vtos bytecode to pop the frame normally.
+    // reexecuting the real bytecode would cause double registration
+    // of the finalizable object.
+    return _normal_table.entry(Bytecodes::_return).entry(vtos);
   } else {
-    return AbstractInterpreter::continuation_for(method, bcp, callee_parameters, is_top_frame, use_next_mdp);
+    return AbstractInterpreter::deopt_reexecute_entry(method, bcp);
+  }
+}
+
+// If deoptimization happens, the interpreter should reexecute this bytecode.
+// This function mainly helps the compilers to set up the reexecute bit.
+bool TemplateInterpreter::bytecode_should_reexecute(Bytecodes::Code code) {
+  if (code == Bytecodes::_return) {
+    //Yes, we consider Bytecodes::_return as a special case of reexecution
+    return true;
+  } else {
+    return AbstractInterpreter::bytecode_should_reexecute(code);
   }
 }
 
--- a/src/share/vm/interpreter/templateInterpreter.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/templateInterpreter.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -171,11 +171,15 @@
   static void       ignore_safepoints();                        // ignores safepoints
 
   // Deoptimization support
-  static address    continuation_for(methodOop method,
-                                     address bcp,
-                                     int callee_parameters,
-                                     bool is_top_frame,
-                                     bool& use_next_mdp);
+  // Compute the entry address for continuation after
+  static address deopt_continue_after_entry(methodOop method,
+                                            address bcp,
+                                            int callee_parameters,
+                                            bool is_top_frame);
+  // Deoptimization should reexecute this bytecode
+  static bool    bytecode_should_reexecute(Bytecodes::Code code);
+  // Compute the address for reexecution
+  static address deopt_reexecute_entry(methodOop method, address bcp);
 
 #include "incls/_templateInterpreter_pd.hpp.incl"
 
--- a/src/share/vm/interpreter/templateTable.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/templateTable.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/templateTable.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/interpreter/templateTable.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/barrierSet.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/barrierSet.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -25,12 +25,27 @@
 # include "incls/_precompiled.incl"
 # include "incls/_barrierSet.cpp.incl"
 
-// count is in HeapWord's
+// count is number of array elements being written
 void BarrierSet::static_write_ref_array_pre(HeapWord* start, size_t count) {
-   Universe::heap()->barrier_set()->write_ref_array_pre(MemRegion(start, start + count));
+  assert(count <= (size_t)max_intx, "count too large");
+#if 0
+  warning("Pre: \t" INTPTR_FORMAT "[" SIZE_FORMAT "]\t",
+                   start,            count);
+#endif
+  if (UseCompressedOops) {
+    Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count);
+  } else {
+    Universe::heap()->barrier_set()->write_ref_array_pre(      (oop*)start, (int)count);
+  }
 }
 
-// count is in HeapWord's
+// count is number of array elements being written
 void BarrierSet::static_write_ref_array_post(HeapWord* start, size_t count) {
-   Universe::heap()->barrier_set()->write_ref_array_work(MemRegion(start, start + count));
+  assert(count <= (size_t)max_intx, "count too large");
+  HeapWord* end = start + objArrayOopDesc::array_size((int)count);
+#if 0
+  warning("Post:\t" INTPTR_FORMAT "[" SIZE_FORMAT "] : [" INTPTR_FORMAT","INTPTR_FORMAT")\t",
+                   start,            count,              start,          end);
+#endif
+  Universe::heap()->barrier_set()->write_ref_array_work(MemRegion(start, end));
 }
--- a/src/share/vm/memory/barrierSet.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/barrierSet.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -81,9 +81,13 @@
   // barrier types.  Semantically, it should be thought of as a call to the
   // virtual "_work" function below, which must implement the barrier.)
   // First the pre-write versions...
-  inline void write_ref_field_pre(void* field, oop new_val);
+  template <class T> inline void write_ref_field_pre(T* field, oop new_val);
+private:
+  // Keep this private so as to catch violations at build time.
+  virtual void write_ref_field_pre_work(     void* field, oop new_val) { guarantee(false, "Not needed"); };
 protected:
-  virtual void write_ref_field_pre_work(void* field, oop new_val) {};
+  virtual void write_ref_field_pre_work(      oop* field, oop new_val) {};
+  virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {};
 public:
 
   // ...then the post-write version.
@@ -117,12 +121,17 @@
   virtual void read_ref_array(MemRegion mr) = 0;
   virtual void read_prim_array(MemRegion mr) = 0;
 
-  virtual void write_ref_array_pre(MemRegion mr) {}
+  virtual void write_ref_array_pre(      oop* dst, int length) {}
+  virtual void write_ref_array_pre(narrowOop* dst, int length) {}
   inline void write_ref_array(MemRegion mr);
 
   // Static versions, suitable for calling from generated code.
   static void static_write_ref_array_pre(HeapWord* start, size_t count);
   static void static_write_ref_array_post(HeapWord* start, size_t count);
+  // Narrow oop versions of the above; count is # of array elements being written,
+  // starting with "start", which is HeapWord-aligned.
+  static void static_write_ref_array_pre_narrow(HeapWord* start, size_t count);
+  static void static_write_ref_array_post_narrow(HeapWord* start, size_t count);
 
 protected:
   virtual void write_ref_array_work(MemRegion mr) = 0;
--- a/src/share/vm/memory/barrierSet.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/barrierSet.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -23,10 +23,10 @@
  */
 
 // Inline functions of BarrierSet, which de-virtualize certain
-// performance-critical calls when when the barrier is the most common
+// performance-critical calls when the barrier is the most common
 // card-table kind.
 
-void BarrierSet::write_ref_field_pre(void* field, oop new_val) {
+template <class T> void BarrierSet::write_ref_field_pre(T* field, oop new_val) {
   if (kind() == CardTableModRef) {
     ((CardTableModRefBS*)this)->inline_write_ref_field_pre(field, new_val);
   } else {
--- a/src/share/vm/memory/blockOffsetTable.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/blockOffsetTable.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/cardTableModRefBS.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/cardTableModRefBS.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -253,8 +253,16 @@
     }
 #endif
     // The guard page is always committed and should not be committed over.
-    HeapWord* const new_end_for_commit = MIN2(new_end_aligned,
-                                              _guard_region.start());
+    // "guarded" is used for assertion checking below and recalls the fact
+    // that the would-be end of the new committed region would have
+    // penetrated the guard page.
+    HeapWord* new_end_for_commit = new_end_aligned;
+
+    DEBUG_ONLY(bool guarded = false;)
+    if (new_end_for_commit > _guard_region.start()) {
+      new_end_for_commit = _guard_region.start();
+      DEBUG_ONLY(guarded = true;)
+    }
 
     if (new_end_for_commit > cur_committed.end()) {
       // Must commit new pages.
@@ -302,7 +310,7 @@
     // not the aligned up expanded region.
     // jbyte* const end = byte_after(new_region.last());
     jbyte* const end = (jbyte*) new_end_for_commit;
-    assert((end >= byte_after(new_region.last())) || collided,
+    assert((end >= byte_after(new_region.last())) || collided || guarded,
       "Expect to be beyond new region unless impacting another region");
     // do nothing if we resized downward.
 #ifdef ASSERT
--- a/src/share/vm/memory/cardTableModRefBS.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/cardTableModRefBS.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -287,7 +287,7 @@
   // these functions here for performance.
 protected:
   void write_ref_field_work(oop obj, size_t offset, oop newVal);
-  void write_ref_field_work(void* field, oop newVal);
+  virtual void write_ref_field_work(void* field, oop newVal);
 public:
 
   bool has_write_ref_array_opt() { return true; }
@@ -317,10 +317,10 @@
 
   // *** Card-table-barrier-specific things.
 
-  inline void inline_write_ref_field_pre(void* field, oop newVal) {}
+  template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal) {}
 
-  inline void inline_write_ref_field(void* field, oop newVal) {
-    jbyte* byte = byte_for(field);
+  template <class T> inline void inline_write_ref_field(T* field, oop newVal) {
+    jbyte* byte = byte_for((void*)field);
     *byte = dirty_card;
   }
 
--- a/src/share/vm/memory/cardTableRS.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/cardTableRS.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/gcLocker.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/gcLocker.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/genCollectedHeap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/genCollectedHeap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -482,6 +482,10 @@
     for (int i = starting_level; i <= max_level; i++) {
       if (_gens[i]->should_collect(full, size, is_tlab)) {
         if (i == n_gens() - 1) {  // a major collection is to happen
+          if (!complete) {
+            // The full_collections increment was missed above.
+            increment_total_full_collections();
+          }
           pre_full_gc_dump();    // do any pre full gc dumps
         }
         // Timer for individual generations. Last argument is false: no CR
@@ -1194,7 +1198,7 @@
   return _gens[level]->gc_stats();
 }
 
-void GenCollectedHeap::verify(bool allow_dirty, bool silent) {
+void GenCollectedHeap::verify(bool allow_dirty, bool silent, bool option /* ignored */) {
   if (!silent) {
     gclog_or_tty->print("permgen ");
   }
--- a/src/share/vm/memory/genCollectedHeap.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/genCollectedHeap.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -325,7 +325,7 @@
   void prepare_for_verify();
 
   // Override.
-  void verify(bool allow_dirty, bool silent);
+  void verify(bool allow_dirty, bool silent, bool /* option */);
 
   // Override.
   void print() const;
--- a/src/share/vm/memory/genOopClosures.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/genOopClosures.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -57,7 +57,7 @@
   template <class T> void do_barrier(T* p);
 
   // Version for use by closures that may be called in parallel code.
-  void par_do_barrier(oop* p);
+  template <class T> void par_do_barrier(T* p);
 
  public:
   OopsInGenClosure() : OopClosure(NULL),
--- a/src/share/vm/memory/genOopClosures.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/genOopClosures.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -40,18 +40,20 @@
 
 template <class T> inline void OopsInGenClosure::do_barrier(T* p) {
   assert(generation()->is_in_reserved(p), "expected ref in generation");
-  assert(!oopDesc::is_null(*p), "expected non-null object");
-  oop obj = oopDesc::load_decode_heap_oop_not_null(p);
+  T heap_oop = oopDesc::load_heap_oop(p);
+  assert(!oopDesc::is_null(heap_oop), "expected non-null oop");
+  oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   // If p points to a younger generation, mark the card.
   if ((HeapWord*)obj < _gen_boundary) {
     _rs->inline_write_ref_field_gc(p, obj);
   }
 }
 
-inline void OopsInGenClosure::par_do_barrier(oop* p) {
+template <class T> inline void OopsInGenClosure::par_do_barrier(T* p) {
   assert(generation()->is_in_reserved(p), "expected ref in generation");
-  oop obj = *p;
-  assert(obj != NULL, "expected non-null object");
+  T heap_oop = oopDesc::load_heap_oop(p);
+  assert(!oopDesc::is_null(heap_oop), "expected non-null oop");
+  oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   // If p points to a younger generation, mark the card.
   if ((HeapWord*)obj < gen_boundary()) {
     rs()->write_ref_field_gc_par(p, obj);
--- a/src/share/vm/memory/heap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/heap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/referenceProcessor.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/referenceProcessor.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1013,12 +1013,19 @@
   // discovered_addr.
   oop current_head = refs_list.head();
 
-  // Note: In the case of G1, this pre-barrier is strictly
+  // Note: In the case of G1, this specific pre-barrier is strictly
   // not necessary because the only case we are interested in
-  // here is when *discovered_addr is NULL, so this will expand to
-  // nothing. As a result, I am just manually eliding this out for G1.
+  // here is when *discovered_addr is NULL (see the CAS further below),
+  // so this will expand to nothing. As a result, we have manually
+  // elided this out for G1, but left in the test for some future
+  // collector that might have need for a pre-barrier here.
   if (_discovered_list_needs_barrier && !UseG1GC) {
-    _bs->write_ref_field_pre((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
+    if (UseCompressedOops) {
+      _bs->write_ref_field_pre((narrowOop*)discovered_addr, current_head);
+    } else {
+      _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
+    }
+    guarantee(false, "Need to check non-G1 collector");
   }
   oop retest = oopDesc::atomic_compare_exchange_oop(current_head, discovered_addr,
                                                     NULL);
@@ -1029,9 +1036,8 @@
     refs_list.set_head(obj);
     refs_list.inc_length(1);
     if (_discovered_list_needs_barrier) {
-      _bs->write_ref_field((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
+      _bs->write_ref_field((void*)discovered_addr, current_head);
     }
-
   } else {
     // If retest was non NULL, another thread beat us to it:
     // The reference has already been discovered...
@@ -1177,11 +1183,16 @@
     // pre-value, we can safely elide the pre-barrier here for the case of G1.
     assert(discovered == NULL, "control point invariant");
     if (_discovered_list_needs_barrier && !UseG1GC) { // safe to elide for G1
-      _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
+      if (UseCompressedOops) {
+        _bs->write_ref_field_pre((narrowOop*)discovered_addr, current_head);
+      } else {
+        _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
+      }
+      guarantee(false, "Need to check non-G1 collector");
     }
     oop_store_raw(discovered_addr, current_head);
     if (_discovered_list_needs_barrier) {
-      _bs->write_ref_field((oop*)discovered_addr, current_head);
+      _bs->write_ref_field((void*)discovered_addr, current_head);
     }
     list->set_head(obj);
     list->inc_length(1);
--- a/src/share/vm/memory/serialize.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/serialize.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -51,7 +51,7 @@
   soc->do_tag(arrayOopDesc::base_offset_in_bytes(T_BYTE));
   soc->do_tag(sizeof(constantPoolOopDesc));
   soc->do_tag(sizeof(constantPoolCacheOopDesc));
-  soc->do_tag(objArrayOopDesc::base_offset_in_bytes(T_BYTE));
+  soc->do_tag(objArrayOopDesc::base_offset_in_bytes());
   soc->do_tag(typeArrayOopDesc::base_offset_in_bytes(T_BYTE));
   soc->do_tag(sizeof(symbolOopDesc));
   soc->do_tag(sizeof(klassOopDesc));
--- a/src/share/vm/memory/space.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/space.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -106,6 +106,7 @@
   virtual void set_end(HeapWord* value)    { _end = value; }
 
   virtual HeapWord* saved_mark_word() const  { return _saved_mark_word; }
+
   void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; }
 
   MemRegionClosure* preconsumptionDirtyCardClosure() const {
--- a/src/share/vm/memory/universe.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/universe.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1170,7 +1170,7 @@
   st->print_cr("}");
 }
 
-void Universe::verify(bool allow_dirty, bool silent) {
+void Universe::verify(bool allow_dirty, bool silent, bool option) {
   if (SharedSkipVerify) {
     return;
   }
@@ -1194,7 +1194,7 @@
   if (!silent) gclog_or_tty->print("[Verifying ");
   if (!silent) gclog_or_tty->print("threads ");
   Threads::verify();
-  heap()->verify(allow_dirty, silent);
+  heap()->verify(allow_dirty, silent, option);
 
   if (!silent) gclog_or_tty->print("syms ");
   SymbolTable::verify();
--- a/src/share/vm/memory/universe.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/memory/universe.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -343,6 +343,7 @@
   // For UseCompressedOops
   static address* narrow_oop_base_addr()              { return &_narrow_oop._base; }
   static address  narrow_oop_base()                   { return  _narrow_oop._base; }
+  static bool  is_narrow_oop_base(void* addr)         { return (narrow_oop_base() == (address)addr); }
   static int      narrow_oop_shift()                  { return  _narrow_oop._shift; }
   static void     set_narrow_oop_base(address base)   { _narrow_oop._base  = base; }
   static void     set_narrow_oop_shift(int shift)     { _narrow_oop._shift = shift; }
@@ -398,7 +399,7 @@
 
   // Debugging
   static bool verify_in_progress() { return _verify_in_progress; }
-  static void verify(bool allow_dirty = true, bool silent = false);
+  static void verify(bool allow_dirty = true, bool silent = false, bool option = true);
   static int  verify_count()                  { return _verify_count; }
   static void print();
   static void print_on(outputStream* st);
--- a/src/share/vm/oops/arrayKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/arrayKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -140,6 +140,7 @@
     THROW_0(vmSymbols::java_lang_NegativeArraySizeException());
   }
   if (length > arrayOopDesc::max_array_length(T_ARRAY)) {
+    report_java_out_of_memory("Requested array size exceeds VM limit");
     THROW_OOP_0(Universe::out_of_memory_error_array_size());
   }
   int size = objArrayOopDesc::object_size(length);
--- a/src/share/vm/oops/cpCacheOop.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/cpCacheOop.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/generateOopMap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/generateOopMap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/instanceKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/instanceKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -158,9 +158,6 @@
   // timer handles recursion
   assert(THREAD->is_Java_thread(), "non-JavaThread in link_class_impl");
   JavaThread* jt = (JavaThread*)THREAD;
-  PerfTraceTimedEvent vmtimer(ClassLoader::perf_class_link_time(),
-                        ClassLoader::perf_classes_linked(),
-                        jt->get_thread_stat()->class_link_recursion_count_addr());
 
   // link super class before linking this class
   instanceKlassHandle super(THREAD, this_oop->super());
@@ -194,6 +191,15 @@
     return true;
   }
 
+  // trace only the link time for this klass that includes
+  // the verification time
+  PerfClassTraceTime vmtimer(ClassLoader::perf_class_link_time(),
+                             ClassLoader::perf_class_link_selftime(),
+                             ClassLoader::perf_classes_linked(),
+                             jt->get_thread_stat()->perf_recursion_counts_addr(),
+                             jt->get_thread_stat()->perf_timers_addr(),
+                             PerfClassTraceTime::CLASS_LINK);
+
   // verification & rewriting
   {
     ObjectLocker ol(this_oop, THREAD);
@@ -203,12 +209,14 @@
     if (!this_oop->is_linked()) {
       if (!this_oop->is_rewritten()) {
         {
-          assert(THREAD->is_Java_thread(), "non-JavaThread in link_class_impl");
-          JavaThread* jt = (JavaThread*)THREAD;
           // Timer includes any side effects of class verification (resolution,
           // etc), but not recursive entry into verify_code().
-          PerfTraceTime timer(ClassLoader::perf_class_verify_time(),
-                            jt->get_thread_stat()->class_verify_recursion_count_addr());
+          PerfClassTraceTime timer(ClassLoader::perf_class_verify_time(),
+                                   ClassLoader::perf_class_verify_selftime(),
+                                   ClassLoader::perf_classes_verified(),
+                                   jt->get_thread_stat()->perf_recursion_counts_addr(),
+                                   jt->get_thread_stat()->perf_timers_addr(),
+                                   PerfClassTraceTime::CLASS_VERIFY);
           bool verify_ok = verify_code(this_oop, throw_verifyerror, THREAD);
           if (!verify_ok) {
             return false;
@@ -350,9 +358,12 @@
     JavaThread* jt = (JavaThread*)THREAD;
     // Timer includes any side effects of class initialization (resolution,
     // etc), but not recursive entry into call_class_initializer().
-    PerfTraceTimedEvent timer(ClassLoader::perf_class_init_time(),
-                              ClassLoader::perf_classes_inited(),
-                              jt->get_thread_stat()->class_init_recursion_count_addr());
+    PerfClassTraceTime timer(ClassLoader::perf_class_init_time(),
+                             ClassLoader::perf_class_init_selftime(),
+                             ClassLoader::perf_classes_inited(),
+                             jt->get_thread_stat()->perf_recursion_counts_addr(),
+                             jt->get_thread_stat()->perf_timers_addr(),
+                             PerfClassTraceTime::CLASS_CLINIT);
     this_oop->call_class_initializer(THREAD);
   }
 
@@ -497,6 +508,7 @@
 objArrayOop instanceKlass::allocate_objArray(int n, int length, TRAPS) {
   if (length < 0) THROW_0(vmSymbols::java_lang_NegativeArraySizeException());
   if (length > arrayOopDesc::max_array_length(T_OBJECT)) {
+    report_java_out_of_memory("Requested array size exceeds VM limit");
     THROW_OOP_0(Universe::out_of_memory_error_array_size());
   }
   int size = objArrayOopDesc::object_size(length);
--- a/src/share/vm/oops/instanceRefKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/instanceRefKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -28,13 +28,14 @@
 template <class T>
 static void specialized_oop_follow_contents(instanceRefKlass* ref, oop obj) {
   T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
-  oop referent = oopDesc::load_decode_heap_oop(referent_addr);
+  T heap_oop = oopDesc::load_heap_oop(referent_addr);
   debug_only(
     if(TraceReferenceGC && PrintGCDetails) {
       gclog_or_tty->print_cr("instanceRefKlass::oop_follow_contents " INTPTR_FORMAT, obj);
     }
   )
-  if (referent != NULL) {
+  if (!oopDesc::is_null(heap_oop)) {
+    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
     if (!referent->is_gc_marked() &&
         MarkSweep::ref_processor()->
           discover_reference(obj, ref->reference_type())) {
@@ -81,13 +82,14 @@
                                             ParCompactionManager* cm,
                                             oop obj) {
   T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
-  oop referent = oopDesc::load_decode_heap_oop(referent_addr);
+  T heap_oop = oopDesc::load_heap_oop(referent_addr);
   debug_only(
     if(TraceReferenceGC && PrintGCDetails) {
       gclog_or_tty->print_cr("instanceRefKlass::oop_follow_contents " INTPTR_FORMAT, obj);
     }
   )
-  if (referent != NULL) {
+  if (!oopDesc::is_null(heap_oop)) {
+    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
     if (PSParallelCompact::mark_bitmap()->is_unmarked(referent) &&
         PSParallelCompact::ref_processor()->
           discover_reference(obj, ref->reference_type())) {
@@ -182,9 +184,10 @@
   }                                                                             \
                                                                                 \
   T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);           \
-  oop referent = oopDesc::load_decode_heap_oop(referent_addr);                  \
-  if (referent != NULL && contains(referent_addr)) {                            \
+  T heap_oop = oopDesc::load_heap_oop(referent_addr);                           \
+  if (!oopDesc::is_null(heap_oop) && contains(referent_addr)) {                 \
     ReferenceProcessor* rp = closure->_ref_processor;                           \
+    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);                 \
     if (!referent->is_gc_marked() && (rp != NULL) &&                            \
         rp->discover_reference(obj, reference_type())) {                        \
       return size;                                                              \
--- a/src/share/vm/oops/methodDataOop.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/methodDataOop.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/methodKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/methodKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -68,7 +68,7 @@
   m->set_constants(NULL);
   m->set_max_stack(0);
   m->set_max_locals(0);
-  m->clear_intrinsic_id_cache();
+  m->set_intrinsic_id(vmIntrinsics::_none);
   m->set_method_data(NULL);
   m->set_interpreter_throwout_count(0);
   m->set_vtable_index(methodOopDesc::garbage_vtable_index);
--- a/src/share/vm/oops/methodOop.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/methodOop.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -962,26 +962,39 @@
   return newm;
 }
 
-vmIntrinsics::ID methodOopDesc::compute_intrinsic_id() const {
-  assert(vmIntrinsics::_none == 0, "correct coding of default case");
-  const uintptr_t max_cache_uint = right_n_bits((int)(sizeof(_intrinsic_id_cache) * BitsPerByte));
-  assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_cache_uint, "else fix cache size");
+vmSymbols::SID methodOopDesc::klass_id_for_intrinsics(klassOop holder) {
   // if loader is not the default loader (i.e., != NULL), we can't know the intrinsics
   // because we are not loading from core libraries
-  if (instanceKlass::cast(method_holder())->class_loader() != NULL) return vmIntrinsics::_none;
+  if (instanceKlass::cast(holder)->class_loader() != NULL)
+    return vmSymbols::NO_SID;   // regardless of name, no intrinsics here
 
   // see if the klass name is well-known:
-  symbolOop klass_name    = instanceKlass::cast(method_holder())->name();
-  vmSymbols::SID klass_id = vmSymbols::find_sid(klass_name);
-  if (klass_id == vmSymbols::NO_SID)  return vmIntrinsics::_none;
+  symbolOop klass_name = instanceKlass::cast(holder)->name();
+  return vmSymbols::find_sid(klass_name);
+}
+
+void methodOopDesc::init_intrinsic_id() {
+  assert(_intrinsic_id == vmIntrinsics::_none, "do this just once");
+  const uintptr_t max_id_uint = right_n_bits((int)(sizeof(_intrinsic_id) * BitsPerByte));
+  assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_id_uint, "else fix size");
+
+  // the klass name is well-known:
+  vmSymbols::SID klass_id = klass_id_for_intrinsics(method_holder());
+  assert(klass_id != vmSymbols::NO_SID, "caller responsibility");
 
   // ditto for method and signature:
   vmSymbols::SID  name_id = vmSymbols::find_sid(name());
-  if (name_id  == vmSymbols::NO_SID)  return vmIntrinsics::_none;
+  if (name_id  == vmSymbols::NO_SID)  return;
   vmSymbols::SID   sig_id = vmSymbols::find_sid(signature());
-  if (sig_id   == vmSymbols::NO_SID)  return vmIntrinsics::_none;
+  if (sig_id   == vmSymbols::NO_SID)  return;
   jshort flags = access_flags().as_short();
 
+  vmIntrinsics::ID id = vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
+  if (id != vmIntrinsics::_none) {
+    set_intrinsic_id(id);
+    return;
+  }
+
   // A few slightly irregular cases:
   switch (klass_id) {
   case vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_StrictMath):
@@ -992,15 +1005,18 @@
     case vmSymbols::VM_SYMBOL_ENUM_NAME(sqrt_name):
       // pretend it is the corresponding method in the non-strict class:
       klass_id = vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_Math);
+      id = vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
       break;
     }
   }
 
-  // return intrinsic id if any
-  return vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
+  if (id != vmIntrinsics::_none) {
+    // Set up its iid.  It is an alias method.
+    set_intrinsic_id(id);
+    return;
+  }
 }
 
-
 // These two methods are static since a GC may move the methodOopDesc
 bool methodOopDesc::load_signature_classes(methodHandle m, TRAPS) {
   bool sig_is_loaded = true;
--- a/src/share/vm/oops/methodOop.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/methodOop.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -104,7 +104,7 @@
   u2                _max_stack;                  // Maximum number of entries on the expression stack
   u2                _max_locals;                 // Number of local variables used by this method
   u2                _size_of_parameters;         // size of the parameter block (receiver + arguments) in words
-  u1                _intrinsic_id_cache;         // Cache for intrinsic_id; 0 or 1+vmInt::ID
+  u1                _intrinsic_id;               // vmSymbols::intrinsic_id (0 == _none)
   u1                _highest_tier_compile;       // Highest compile level this method has ever seen.
   u2                _interpreter_throwout_count; // Count of times method was exited via exception while interpreting
   u2                _number_of_breakpoints;      // fullspeed debugging support
@@ -224,8 +224,6 @@
   int highest_tier_compile()                     { return _highest_tier_compile;}
   void set_highest_tier_compile(int level)      { _highest_tier_compile = level;}
 
-  void clear_intrinsic_id_cache() { _intrinsic_id_cache = 0; }
-
   // Count of times method was exited via exception while interpreting
   void interpreter_throwout_increment() {
     if (_interpreter_throwout_count < 65534) {
@@ -571,18 +569,12 @@
   void set_cached_itable_index(int index)           { instanceKlass::cast(method_holder())->set_cached_itable_index(method_idnum(), index); }
 
   // Support for inlining of intrinsic methods
-  vmIntrinsics::ID intrinsic_id() const { // returns zero if not an intrinsic
-    const u1& cache = _intrinsic_id_cache;
-    if (cache != 0) {
-      return (vmIntrinsics::ID)(cache - 1);
-    } else {
-      vmIntrinsics::ID id = compute_intrinsic_id();
-      *(u1*)&cache = ((u1) id) + 1;   // force the cache to be non-const
-      vmIntrinsics::verify_method(id, (methodOop) this);
-      assert((vmIntrinsics::ID)(cache - 1) == id, "proper conversion");
-      return id;
-    }
-  }
+  vmIntrinsics::ID intrinsic_id() const          { return (vmIntrinsics::ID) _intrinsic_id;           }
+  void     set_intrinsic_id(vmIntrinsics::ID id) {                           _intrinsic_id = (u1) id; }
+
+  // Helper routines for intrinsic_id() and vmIntrinsics::method().
+  void init_intrinsic_id();     // updates from _none if a match
+  static vmSymbols::SID klass_id_for_intrinsics(klassOop holder);
 
   // On-stack replacement support
   bool has_osr_nmethod()                         { return instanceKlass::cast(method_holder())->lookup_osr_nmethod(this, InvocationEntryBci) != NULL; }
@@ -635,9 +627,6 @@
   void set_size_of_parameters(int size)          { _size_of_parameters = size; }
  private:
 
-  // Helper routine for intrinsic_id().
-  vmIntrinsics::ID compute_intrinsic_id() const;
-
   // Inlined elements
   address* native_function_addr() const          { assert(is_native(), "must be native"); return (address*) (this+1); }
   address* signature_handler_addr() const        { return native_function_addr() + 1; }
--- a/src/share/vm/oops/objArrayKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/objArrayKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -39,6 +39,7 @@
       assert(a->is_parsable(), "Can't publish unless parsable");
       return a;
     } else {
+      report_java_out_of_memory("Requested array size exceeds VM limit");
       THROW_OOP_0(Universe::out_of_memory_error_array_size());
     }
   } else {
@@ -84,8 +85,6 @@
 template <class T> void objArrayKlass::do_copy(arrayOop s, T* src,
                                arrayOop d, T* dst, int length, TRAPS) {
 
-  const size_t word_len = objArrayOopDesc::array_size(length);
-
   BarrierSet* bs = Universe::heap()->barrier_set();
   // For performance reasons, we assume we are that the write barrier we
   // are using has optimized modes for arrays of references.  At least one
@@ -93,11 +92,10 @@
   assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt");
   assert(bs->has_write_ref_array_pre_opt(), "For pre-barrier as well.");
 
-  MemRegion dst_mr = MemRegion((HeapWord*)dst, word_len);
   if (s == d) {
     // since source and destination are equal we do not need conversion checks.
     assert(length > 0, "sanity check");
-    bs->write_ref_array_pre(dst_mr);
+    bs->write_ref_array_pre(dst, length);
     Copy::conjoint_oops_atomic(src, dst, length);
   } else {
     // We have to make sure all elements conform to the destination array
@@ -105,7 +103,7 @@
     klassOop stype = objArrayKlass::cast(s->klass())->element_klass();
     if (stype == bound || Klass::cast(stype)->is_subtype_of(bound)) {
       // elements are guaranteed to be subtypes, so no check necessary
-      bs->write_ref_array_pre(dst_mr);
+      bs->write_ref_array_pre(dst, length);
       Copy::conjoint_oops_atomic(src, dst, length);
     } else {
       // slow case: need individual subtype checks
@@ -137,6 +135,7 @@
       }
     }
   }
+  const size_t word_len = objArrayOopDesc::array_size(length);
   bs->write_ref_array(MemRegion((HeapWord*)dst, word_len));
 }
 
--- a/src/share/vm/oops/objArrayOop.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/objArrayOop.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -38,6 +38,11 @@
   }
 
  public:
+  // Returns the offset of the first element.
+  static int base_offset_in_bytes() {
+    return arrayOopDesc::base_offset_in_bytes(T_OBJECT);
+  }
+
   // base is the address following the header.
   HeapWord* base() const      { return (HeapWord*) arrayOopDesc::base(T_OBJECT); }
 
--- a/src/share/vm/oops/oop.inline.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/oop.inline.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -148,12 +148,14 @@
 
 inline narrowOop oopDesc::encode_heap_oop_not_null(oop v) {
   assert(!is_null(v), "oop value can never be zero");
+  assert(Universe::heap()->is_in_reserved(v), "Address not in heap");
   address base = Universe::narrow_oop_base();
   int    shift = Universe::narrow_oop_shift();
   uint64_t  pd = (uint64_t)(pointer_delta((void*)v, (void*)base, 1));
   assert(OopEncodingHeapMax > pd, "change encoding max if new encoding");
   uint64_t result = pd >> shift;
   assert((result & CONST64(0xffffffff00000000)) == 0, "narrow oop overflow");
+  assert(decode_heap_oop(result) == v, "reversibility");
   return (narrowOop)result;
 }
 
@@ -449,7 +451,7 @@
   oopDesc::bs()->write_ref_field(p, v);
 }
 
-inline void update_barrier_set_pre(void* p, oop v) {
+template <class T> inline void update_barrier_set_pre(T* p, oop v) {
   oopDesc::bs()->write_ref_field_pre(p, v);
 }
 
@@ -459,15 +461,15 @@
   } else {
     update_barrier_set_pre(p, v);
     oopDesc::encode_store_heap_oop(p, v);
-    update_barrier_set(p, v);
+    update_barrier_set((void*)p, v);  // cast away type
   }
 }
 
 template <class T> inline void oop_store(volatile T* p, oop v) {
-  update_barrier_set_pre((void*)p, v);
+  update_barrier_set_pre((T*)p, v);   // cast away volatile
   // Used by release_obj_field_put, so use release_store_ptr.
   oopDesc::release_encode_store_heap_oop(p, v);
-  update_barrier_set((void*)p, v);
+  update_barrier_set((void*)p, v);    // cast away type
 }
 
 template <class T> inline void oop_store_without_check(T* p, oop v) {
--- a/src/share/vm/oops/oopsHierarchy.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/oopsHierarchy.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -29,6 +29,7 @@
 typedef juint narrowOop; // Offset instead of address for an oop within a java object
 typedef class klassOopDesc* wideKlassOop; // to keep SA happy and unhandled oop
                                           // detector happy.
+typedef void* OopOrNarrowOopStar;
 
 #ifndef CHECK_UNHANDLED_OOPS
 
--- a/src/share/vm/oops/typeArrayKlass.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/oops/typeArrayKlass.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -80,6 +80,7 @@
       assert(t->is_parsable(), "Don't publish unless parsable");
       return t;
     } else {
+      report_java_out_of_memory("Requested array size exceeds VM limit");
       THROW_OOP_0(Universe::out_of_memory_error_array_size());
     }
   } else {
--- a/src/share/vm/opto/addnode.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/addnode.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/block.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/block.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -910,7 +910,16 @@
               !(b->head()->is_Loop() && n->is_Phi()) &&
               // See (+++) comment in reg_split.cpp
               !(n->jvms() != NULL && n->jvms()->is_monitor_use(k)) ) {
-            assert( b->find_node(def) < j, "uses must follow definitions" );
+            bool is_loop = false;
+            if (n->is_Phi()) {
+              for( uint l = 1; l < def->req(); l++ ) {
+                if (n == def->in(l)) {
+                  is_loop = true;
+                  break; // Some kind of loop
+                }
+              }
+            }
+            assert( is_loop || b->find_node(def) < j, "uses must follow definitions" );
           }
           if( def->is_SafePointScalarObject() ) {
             assert(_bbs[def->_idx] == b, "SafePointScalarObject Node should be at the same block as its SafePoint node");
--- a/src/share/vm/opto/block.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/block.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/buildOopMap.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/buildOopMap.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/bytecodeInfo.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/bytecodeInfo.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,7 @@
     // Keep a private copy of the caller_jvms:
     _caller_jvms = new (C) JVMState(caller_jvms->method(), caller_tree->caller_jvms());
     _caller_jvms->set_bci(caller_jvms->bci());
+    assert(!caller_jvms->should_reexecute(), "there should be no reexecute bytecode with inlining");
   }
   assert(_caller_jvms->same_calls_as(caller_jvms), "consistent JVMS");
   assert((caller_tree == NULL ? 0 : caller_tree->inline_depth() + 1) == inline_depth(), "correct (redundant) depth parameter");
--- a/src/share/vm/opto/c2compiler.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/c2compiler.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/callnode.cpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/callnode.cpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -223,6 +223,7 @@
 JVMState::JVMState(ciMethod* method, JVMState* caller) {
   assert(method != NULL, "must be valid call site");
   _method = method;
+  _reexecute = Reexecute_Undefined;
   debug_only(_bci = -99);  // random garbage value
   debug_only(_map = (SafePointNode*)-1);
   _caller = caller;
@@ -237,6 +238,7 @@
 JVMState::JVMState(int stack_size) {
   _method = NULL;
   _bci = InvocationEntryBci;
+  _reexecute = Reexecute_Undefined;
   debug_only(_map = (SafePointNode*)-1);
   _caller = NULL;
   _depth  = 1;
@@ -269,6 +271,7 @@
     if (p->_method != q->_method)    return false;
     if (p->_method == NULL)          return true;   // bci is irrelevant
     if (p->_bci    != q->_bci)       return false;
+    if (p->_reexecute != q->_reexecute)  return false;
     p = p->caller();
     q = q->caller();
     if (p == q)                      return true;
@@ -490,6 +493,7 @@
     if (!printed)
       _method->print_short_name(st);
     st->print(" @ bci:%d",_bci);
+    st->print(" reexecute:%s", _reexecute==Reexecute_True?"true":"false");
   } else {
     st->print(" runtime stub");
   }
@@ -509,8 +513,8 @@
     }
     _map->dump(2);
   }
-  st->print("JVMS depth=%d loc=%d stk=%d mon=%d scalar=%d end=%d mondepth=%d sp=%d bci=%d method=",
-             depth(), locoff(), stkoff(), monoff(), scloff(), endoff(), monitor_depth(), sp(), bci());
+  st->print("JVMS depth=%d loc=%d stk=%d mon=%d scalar=%d end=%d mondepth=%d sp=%d bci=%d reexecute=%s method=",
+             depth(), locoff(), stkoff(), monoff(), scloff(), endoff(), monitor_depth(), sp(), bci(), should_reexecute()?"true":"false");
   if (_method == NULL) {
     st->print_cr("(none)");
   } else {
@@ -537,6 +541,7 @@
 JVMState* JVMState::clone_shallow(Compile* C) const {
   JVMState* n = has_method() ? new (C) JVMState(_method, _caller) : new (C) JVMState(0);
   n->set_bci(_bci);
+  n->_reexecute = _reexecute;
   n->set_locoff(_locoff);
   n->set_stkoff(_stkoff);
   n->set_monoff(_monoff);
--- a/src/share/vm/opto/callnode.hpp	Sat Jul 25 20:54:28 2009 -0700
+++ b/src/share/vm/opto/callnode.hpp	Mon Sep 07 10:57:43 2009 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -178,6 +178,13 @@
 // This provides a way to map the optimized program back into the interpreter,
 // or to let the GC mark the stack.
 class JVMState : public ResourceObj {
+public:
+  typedef enum {
+    Reexecute_Undefined = -1, // not defined -- will be translated into false later
+    Reexecute_False     =  0, // false       -- do not reexecute
+    Reexecute_True      =  1  // true        -- reexecute the bytecode
+  } ReexecuteState; //Reexecute State
+
 private:
   JVMState*         _caller;    // List pointer for forming scope chains
   uint              _depth;     // One mroe than caller depth, or one.
@@ -188,10 +195,12 @@
   uint              _endoff;    // Offset to end of input edge mapping
   uint              _sp;        // Jave Expression Stack Pointer for this state
   int               _bci;       // Byte Code Index of this JVM point
+  ReexecuteState    _reexecute; // Whether this bytecode need to be re-executed
   ciMethod*         _method;    // Method Pointer
   SafePointNode*    _map;       // Map node associated with this scope
 public:
   friend class Compile;
+  friend class PreserveReexecuteState;
 
   // Because JVMState objects live over the entire lifetime of the
   // Compile object, they are allocated into the comp_arena, which
@@ -222,16 +231,18 @@
   bool        is_mon(uint i) const { return i >= _monoff && i < _scloff; }
   bool        is_scl(uint i) const { return i >= _scloff && i < _endoff; }
 
-  uint              sp()     const { return _sp; }
-  int               bci()    const { return _bci; }
-  bool          has_method() const { return _method != NULL; }
-  ciMethod*         method() const { assert(has_method(), ""); return _method; }
-  JVMState*         caller() const { return _caller; }
-  SafePointNode*    map()    const { return _map; }
-  uint              depth()  const { return _depth; }
-  uint        debug_start()  const; // returns locoff of root caller
-  uint        debug_end()    const; // returns endoff of self
-  uint        debug_size()   const {
+  uint                      sp() const { return _sp; }
+  int                      bci() const { return _bci; }
+  bool        should_reexecute() const { return _reexecute==Reexecute_True; }
+  bool  is_reexecute_undefined() const { return _reexecute==Reexecute_Undefined; }
+  bool              has_method() const { return _method != NULL; }
+  ciMethod*             method() const { assert(has_method(), ""); return _method; }
+  JVMState*             caller() const { return _caller; }
+  S