OpenJDK / loom / loom

--- a/.hgtags	Mon Jul 27 09:06:22 2009 -0700
+++ b/.hgtags	Mon Jul 27 17:23:52 2009 -0400
@@ -35,3 +35,7 @@
 d60a9ce3c3eabf28f5d50ae839d18be04a551bc2 jdk7-b58
 c33e7d38c9210741dbc285507403a4b20bd802a0 jdk7-b59
 5a10e4d0b14d7beac53a7b2213ae6864afe1fd3e jdk7-b60
+dbb955b1ee59b876dd1f133952b557b48b1d7732 jdk7-b61
+6107cbff3130c747d243c25a7874cd59db5744a8 jdk7-b62
+dfd8506f74c3731bb169ce93c72612d78ee0413b jdk7-b63
+d22867c5f1b295a0a2b3b4bc8999a2676f6e20c3 jdk7-b64
--- a/.hgtags-top-repo	Mon Jul 27 09:06:22 2009 -0700
+++ b/.hgtags-top-repo	Mon Jul 27 17:23:52 2009 -0400
@@ -35,3 +35,7 @@
 59b497130f82ec809c245ffb5e521e3a5fabf8af jdk7-b58
 030142474602b4a067662fffc0c8e541de5a78df jdk7-b59
 39565502682c7085369bd09e51640919dc741097 jdk7-b60
+472c21584cfd7e9c0229ad6a100366a5c03d2976 jdk7-b61
+c7ed15ab92ce36a09d264a5e34025884b2d7607f jdk7-b62
+57f7e028c7ad1806500ae89eb3f4cd9a51b10e18 jdk7-b63
+269c1ec4435dfb7b452ae6e3bdde005d55c5c830 jdk7-b64
--- a/README-builds.html	Mon Jul 27 09:06:22 2009 -0700
+++ b/README-builds.html	Mon Jul 27 17:23:52 2009 -0400
@@ -72,6 +72,7 @@
                             <li>Linux and Solaris:
                                 <ul>
                                     <li><a href="#cups">CUPS Include files</a> </li>
+                                    <li><a href="#xrender">XRender Include files</a></li>
                                 </ul>
                             </li>
                             <li>Linux only:
@@ -119,6 +120,7 @@
                         <th>Base OS and Architecture</th>
                         <th>OS</th>
                         <th>C/C++ Compiler</th>
+                        <th>BOOT JDK</th>
                     </tr>
                 </thead>
                 <tbody>
@@ -126,57 +128,65 @@
                         <td>Linux X86 (32-bit)</td>
                         <td>Fedora 9</td>
                         <td>gcc 4 </td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Linux X64 (64-bit)</td>
                         <td>Fedora 9</td>
                         <td>gcc 4 </td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Solaris SPARC (32-bit)</td>
-                        <td>Solaris 10 + patches
+                        <td>Solaris 10u2 + patches
                             <br>
                             See <a href="http://sunsolve.sun.com/pub-cgi/show.pl?target=patches/JavaSE" target="_blank">
                             SunSolve</a> for patch downloads.
                         </td>
                         <td>Sun Studio 12</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Solaris SPARCV9 (64-bit)</td>
-                        <td>Solaris 10 + patches
+                        <td>Solaris 10u2 + patches
                             <br>
                             See <a href="http://sunsolve.sun.com/pub-cgi/show.pl?target=patches/JavaSE" target="_blank">
                             SunSolve</a> for patch downloads.
                         </td>
                         <td>Sun Studio 12</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Solaris X86 (32-bit)</td>
-                        <td>Solaris 10 + patches
+                        <td>Solaris 10u2 + patches
                             <br>
                             See <a href="http://sunsolve.sun.com/pub-cgi/show.pl?target=patches/JavaSE" target="_blank">
                             SunSolve</a> for patch downloads.
                         </td>
                         <td>Sun Studio 12</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Solaris X64 (64-bit)</td>
-                        <td>Solaris 10 + patches
+                        <td>Solaris 10u2 + patches
                             <br>
                             See <a href="http://sunsolve.sun.com/pub-cgi/show.pl?target=patches/JavaSE" target="_blank">
                             SunSolve</a> for patch downloads.
                         </td>
                         <td>Sun Studio 12</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Windows X86 (32-bit)</td>
                         <td>Windows XP</td>
                         <td>Microsoft Visual Studio C++ 2008 Standard Edition</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                     <tr>
                         <td>Windows X64 (64-bit)</td>
                         <td>Windows Server 2003 - Enterprise x64 Edition</td>
                         <td>Microsoft Platform SDK - April 2005</td>
+                        <td>JDK 6u14 FCS </td>
                     </tr>
                 </tbody>
             </table>
@@ -657,6 +667,9 @@
                     <tt><a href="#ALT_CUPS_HEADERS_PATH">ALT_CUPS_HEADERS_PATH</a></tt>.
                 </li>
                 <li>
+                    Install the <a href="#xrender">XRender Include files</a>.
+                </li>
+                <li>
                     Install the
                     <a href="#jibx">JIBX Libraries</a>, set
                     <tt><a href="#ALT_JIBX_LIBS_PATH">ALT_JIBX_LIBS_PATH</a></tt>.
@@ -1032,6 +1045,27 @@
                 CUPS Header files.
             </blockquote>
             <!-- ------------------------------------------------------ -->
+            <h4><a name="xrender">XRender Extension Headers (Solaris &amp; Linux)</a></h4>
+            <blockquote>
+                <p>
+                <strong>Solaris:</strong>
+                XRender header files are required for building the
+                OpenJDK on Solaris.
+                The XRender header file is included with the other X11 header files
+                in the package <strong>SFWxwinc</strong> on new enough versions of
+                Solaris and will be installed in
+	        <tt>/usr/X11/include/X11/extensions/Xrender.h</tt>
+                </p><p>
+                <strong>Linux:</strong>
+                XRender header files are required for building the
+                OpenJDK on Linux.
+                The Linux header files are usually available from a "Xrender"
+                development package, it's recommended that you try and use
+                the package provided by the particular distribution of Linux that
+                you are using.
+	        </p>
+            </blockquote>
+            <!-- ------------------------------------------------------ -->
             <h4><a name="freetype">FreeType 2</a></h4>
             <blockquote>
                 Version 2.3 or newer of FreeType is required for building the OpenJDK.
--- a/corba/.hgtags	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/.hgtags	Mon Jul 27 17:23:52 2009 -0400
@@ -35,3 +35,7 @@
 2e3b8edab3ef55406494d3dd562e06882e6fc15e jdk7-b58
 7e6b2b55c00cc523b468272353ada3979adbbf16 jdk7-b59
 f1e1cccbd13aa96d2d8bd872782ff764010bc22c jdk7-b60
+e906b16a12a9a63b615898afa5d9673cbd1c5ab8 jdk7-b61
+65b66117dbd70a493e9644aeb4033cf95a4e3c99 jdk7-b62
+d20e45cd539f20405ff843652069cfd7550c5ab3 jdk7-b63
+047dd27fddb607f8135296b3754131f6e13cb8c7 jdk7-b64
--- a/corba/make/Makefile	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/make/Makefile	Mon Jul 27 17:23:52 2009 -0400
@@ -106,7 +106,12 @@

 SOURCE_LEVEL = 5
 BOOTSTRAP_TARGET_LEVEL = 5
-TARGET_LEVEL = 6
+
+ifdef TARGET_CLASS_VERSION
+    TARGET_LEVEL = $(TARGET_CLASS_VERSION)
+else
+    TARGET_LEVEL = 6
+endif

 ifndef TARGET_JAVA
   TARGET_JAVA = java
--- a/corba/make/common/Defs-windows.gmk	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/make/common/Defs-windows.gmk	Mon Jul 27 17:23:52 2009 -0400
@@ -365,10 +365,10 @@
     COPYRIGHT_YEAR = 2007
 endif

-RC_FLAGS += -d "J2SE_BUILD_ID=$(FULL_VERSION)" \
-            -d "J2SE_COMPANY=$(COMPANY_NAME)" \
-            -d "J2SE_COMPONENT=$(PRODUCT_NAME) Platform SE binary" \
-            -d "J2SE_VER=$(JDK_MINOR_VERSION).$(JDK_MICRO_VERSION).$(J2SE_UPDATE_VER).$(COOKED_BUILD_NUMBER)" \
-            -d "J2SE_COPYRIGHT=Copyright \xA9 $(COPYRIGHT_YEAR)" \
-            -d "J2SE_NAME=$(PRODUCT_NAME) Platform SE $(JDK_MINOR_VERSION) $(J2SE_UPDATE_META_TAG)" \
-            -d "J2SE_FVER=$(JDK_VERSION)"
+RC_FLAGS += -d "JDK_BUILD_ID=$(FULL_VERSION)" \
+            -d "JDK_COMPANY=$(COMPANY_NAME)" \
+            -d "JDK_COMPONENT=$(PRODUCT_NAME) Platform SE binary" \
+            -d "JDK_VER=$(JDK_MINOR_VERSION).$(JDK_MICRO_VERSION).$(JDK_UPDATE_VER).$(COOKED_BUILD_NUMBER)" \
+            -d "JDK_COPYRIGHT=Copyright \xA9 $(COPYRIGHT_YEAR)" \
+            -d "JDK_NAME=$(PRODUCT_NAME) Platform SE $(JDK_MINOR_VERSION) $(JDK_UPDATE_META_TAG)" \
+            -d "JDK_FVER=$(JDK_VERSION)"
--- a/corba/make/common/Library.gmk	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/make/common/Library.gmk	Mon Jul 27 17:23:52 2009 -0400
@@ -158,9 +158,9 @@
 endif
 	@$(ECHO) Created $@

-RC_FLAGS += /D "J2SE_FNAME=$(LIBRARY).dll" \
-            /D "J2SE_INTERNAL_NAME=$(LIBRARY)" \
-            /D "J2SE_FTYPE=0x2L"
+RC_FLAGS += /D "JDK_FNAME=$(LIBRARY).dll" \
+            /D "JDK_INTERNAL_NAME=$(LIBRARY)" \
+            /D "JDK_FTYPE=0x2L"

 $(OBJDIR)/$(LIBRARY).res: $(VERSIONINFO_RESOURCE)
 ifndef LOCAL_RESOURCE_FILE
--- a/corba/make/org/omg/idl/Makefile	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/make/org/omg/idl/Makefile	Mon Jul 27 17:23:52 2009 -0400
@@ -32,6 +32,12 @@
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

+# This program must contain a manifest that defines the execution level
+# needed to follow standard Vista User Access Control Guidelines
+# This must be set before Program.gmk is included
+#
+BUILD_MANIFEST=true
+
 #
 # Files
 #
--- a/corba/src/windows/resource/version.rc	Mon Jul 27 09:06:22 2009 -0700
+++ b/corba/src/windows/resource/version.rc	Mon Jul 27 17:23:52 2009 -0400
@@ -37,8 +37,8 @@
 //

 VS_VERSION_INFO VERSIONINFO
- FILEVERSION    J2SE_FVER
- PRODUCTVERSION J2SE_FVER
+ FILEVERSION    JDK_FVER
+ PRODUCTVERSION JDK_FVER
  FILEFLAGSMASK 0x3fL
 #ifdef _DEBUG
  FILEFLAGS 0x1L
@@ -48,22 +48,22 @@
  // FILEOS 0x4 is Win32, 0x40004 is Win32 NT only
  FILEOS 0x4L
  // FILETYPE should be 0x1 for .exe and 0x2 for .dll
- FILETYPE J2SE_FTYPE
+ FILETYPE JDK_FTYPE
  FILESUBTYPE 0x0L
 BEGIN
     BLOCK "StringFileInfo"
     BEGIN
         BLOCK "000004b0"
         BEGIN
-            VALUE "CompanyName",      XSTR(J2SE_COMPANY)       "\0"
-            VALUE "FileDescription",  XSTR(J2SE_COMPONENT)     "\0"
-            VALUE "FileVersion",      XSTR(J2SE_VER)           "\0"
-            VALUE "Full Version",     XSTR(J2SE_BUILD_ID)      "\0"
-	    VALUE "InternalName",     XSTR(J2SE_INTERNAL_NAME) "\0"
-            VALUE "LegalCopyright",   XSTR(J2SE_COPYRIGHT)     "\0"
-            VALUE "OriginalFilename", XSTR(J2SE_FNAME)         "\0"
-            VALUE "ProductName",      XSTR(J2SE_NAME)          "\0"
-            VALUE "ProductVersion",   XSTR(J2SE_VER)           "\0"
+            VALUE "CompanyName",      XSTR(JDK_COMPANY)       "\0"
+            VALUE "FileDescription",  XSTR(JDK_COMPONENT)     "\0"
+            VALUE "FileVersion",      XSTR(JDK_VER)           "\0"
+            VALUE "Full Version",     XSTR(JDK_BUILD_ID)      "\0"
+	    VALUE "InternalName",     XSTR(JDK_INTERNAL_NAME) "\0"
+            VALUE "LegalCopyright",   XSTR(JDK_COPYRIGHT)     "\0"
+            VALUE "OriginalFilename", XSTR(JDK_FNAME)         "\0"
+            VALUE "ProductName",      XSTR(JDK_NAME)          "\0"
+            VALUE "ProductVersion",   XSTR(JDK_VER)           "\0"
         END
     END
     BLOCK "VarFileInfo"
--- a/hotspot/.hgtags	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/.hgtags	Mon Jul 27 17:23:52 2009 -0400
@@ -35,3 +35,7 @@
 53d9bf689e80fcc76b221bbe6c5d58e08b80cbc6 jdk7-b58
 c55be0c7bd32c016c52218eb4c8b5da8a75450b5 jdk7-b59
 a77eddcd510c3972717c025cfcef9a60bfa4ecac jdk7-b60
+27b728fd1281ab62e9d7e4424f8bbb6ca438d803 jdk7-b61
+a88386380bdaaa5ab4ffbedf22c57bac5dbec034 jdk7-b62
+32c83fb84370a35344676991a48440378e6b6c8a jdk7-b63
+ba36394eb84b949b31212bdb32a518a8f92bab5b jdk7-b64
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Mon Jul 27 17:23:52 2009 -0400
@@ -24,23 +24,64 @@

 package sun.jvm.hotspot.code;

+import java.util.*;
+
 import sun.jvm.hotspot.debugger.*;
 import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.utilities.*;

 public class DebugInfoReadStream extends CompressedReadStream {
   private NMethod code;
   private int InvocationEntryBCI;
+  private List objectPool; // ArrayList<ObjectValue>

   public DebugInfoReadStream(NMethod code, int offset) {
     super(code.scopesDataBegin(), offset);
     InvocationEntryBCI = VM.getVM().getInvocationEntryBCI();
     this.code = code;
+    this.objectPool = null;
+  }
+
+  public DebugInfoReadStream(NMethod code, int offset, List objectPool) {
+    super(code.scopesDataBegin(), offset);
+    InvocationEntryBCI = VM.getVM().getInvocationEntryBCI();
+    this.code = code;
+    this.objectPool = objectPool;
   }

   public OopHandle readOopHandle() {
     return code.getOopAt(readInt());
   }

+  ScopeValue readObjectValue() {
+    int id = readInt();
+    if (Assert.ASSERTS_ENABLED) {
+      Assert.that(objectPool != null, "object pool does not exist");
+      for (Iterator itr = objectPool.iterator(); itr.hasNext();) {
+        ObjectValue ov = (ObjectValue) itr.next();
+        Assert.that(ov.id() != id, "should not be read twice");
+      }
+    }
+    ObjectValue result = new ObjectValue(id);
+    // Cache the object since an object field could reference it.
+    objectPool.add(result);
+    result.readObject(this);
+    return result;
+  }
+
+  ScopeValue getCachedObject() {
+    int id = readInt();
+    Assert.that(objectPool != null, "object pool does not exist");
+    for (Iterator itr = objectPool.iterator(); itr.hasNext();) {
+      ObjectValue ov = (ObjectValue) itr.next();
+      if (ov.id() == id) {
+        return ov;
+      }
+    }
+    Assert.that(false, "should not reach here");
+    return null;
+  }
+
   public int readBCI() {
     return readInt() + InvocationEntryBCI;
   }
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Mon Jul 27 17:23:52 2009 -0400
@@ -29,6 +29,7 @@
 public class MonitorValue {
   private ScopeValue owner;
   private Location   basicLock;
+  private boolean    eliminated;

   // FIXME: not useful yet
   //  MonitorValue(ScopeValue* owner, Location basic_lock);
@@ -36,10 +37,12 @@
   public MonitorValue(DebugInfoReadStream stream) {
     basicLock = new Location(stream);
     owner     = ScopeValue.readFrom(stream);
+    eliminated= stream.readBoolean();
   }

   public ScopeValue owner()     { return owner; }
   public Location   basicLock() { return basicLock; }
+  public boolean   eliminated() { return eliminated; }

   // FIXME: not yet implementable
   //  void write_on(DebugInfoWriteStream* stream);
@@ -50,5 +53,8 @@
     tty.print(",");
     basicLock().printOn(tty);
     tty.print("}");
+    if (eliminated) {
+      tty.print(" (eliminated)");
+    }
   }
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/ObjectValue.java	Mon Jul 27 17:23:52 2009 -0400
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+package sun.jvm.hotspot.code;
+
+import java.io.*;
+import java.util.*;
+
+import sun.jvm.hotspot.debugger.*;
+import sun.jvm.hotspot.utilities.*;
+
+/** An ObjectValue describes an object eliminated by escape analysis. */
+
+public class ObjectValue extends ScopeValue {
+  private int        id;
+  private ScopeValue klass;
+  private List       fieldsValue; // ArrayList<ScopeValue>
+
+  // Field "boolean visited" is not implemented here since
+  // it is used only a during debug info creation.
+
+  public ObjectValue(int id) {
+    this.id = id;
+    klass   = null;
+    fieldsValue = new ArrayList();
+  }
+
+  public boolean isObject() { return true; }
+  public int id() { return id; }
+  public ScopeValue getKlass() { return klass; }
+  public List getFieldsValue() { return fieldsValue; }
+  public ScopeValue getFieldAt(int i) { return (ScopeValue)fieldsValue.get(i); }
+  public int fieldsSize() { return fieldsValue.size(); }
+
+  // Field "value" is always NULL here since it is used
+  // only during deoptimization of a compiled frame
+  // pointing to reallocated object.
+  public OopHandle getValue() { return null; }
+
+  /** Serialization of debugging information */
+
+  void readObject(DebugInfoReadStream stream) {
+    klass = readFrom(stream);
+    Assert.that(klass.isConstantOop(), "should be constant klass oop");
+    int length = stream.readInt();
+    for (int i = 0; i < length; i++) {
+      ScopeValue val = readFrom(stream);
+      fieldsValue.add(val);
+    }
+  }
+
+  // Printing
+
+  public void print() {
+    printOn(System.out);
+  }
+
+  public void printOn(PrintStream tty) {
+    tty.print("scalarObj[" + id + "]");
+  }
+
+  void printFieldsOn(PrintStream tty) {
+    if (fieldsValue.size() > 0) {
+      ((ScopeValue)fieldsValue.get(0)).printOn(tty);
+    }
+    for (int i = 1; i < fieldsValue.size(); i++) {
+      tty.print(", ");
+      ((ScopeValue)fieldsValue.get(i)).printOn(tty);
+    }
+  }
+
+};
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Mon Jul 27 17:23:52 2009 -0400
@@ -27,8 +27,10 @@
 import java.io.*;
 import java.util.*;

+import sun.jvm.hotspot.debugger.*;
 import sun.jvm.hotspot.oops.*;
 import sun.jvm.hotspot.runtime.*;
+import sun.jvm.hotspot.utilities.*;

 /** ScopeDescs contain the information that makes source-level
     debugging of nmethods possible; each scopeDesc describes a method
@@ -45,10 +47,31 @@
   private int     localsDecodeOffset;
   private int     expressionsDecodeOffset;
   private int     monitorsDecodeOffset;
+  /** Scalar replaced bjects pool */
+  private List    objects; // ArrayList<ScopeValue>
+

   public ScopeDesc(NMethod code, int decodeOffset) {
     this.code = code;
     this.decodeOffset = decodeOffset;
+    this.objects      = decodeObjectValues(DebugInformationRecorder.SERIALIZED_NULL);
+
+    // Decode header
+    DebugInfoReadStream stream  = streamAt(decodeOffset);
+
+    senderDecodeOffset = stream.readInt();
+    method = (Method) VM.getVM().getObjectHeap().newOop(stream.readOopHandle());
+    bci    = stream.readBCI();
+    // Decode offsets for body and sender
+    localsDecodeOffset      = stream.readInt();
+    expressionsDecodeOffset = stream.readInt();
+    monitorsDecodeOffset    = stream.readInt();
+  }
+
+  public ScopeDesc(NMethod code, int decodeOffset, int objectDecodeOffset) {
+    this.code = code;
+    this.decodeOffset = decodeOffset;
+    this.objects      = decodeObjectValues(objectDecodeOffset);

     // Decode header
     DebugInfoReadStream stream  = streamAt(decodeOffset);
@@ -81,6 +104,11 @@
     return decodeMonitorValues(monitorsDecodeOffset);
   }

+  /** Returns a List&lt;MonitorValue&gt; */
+  public List getObjects() {
+    return objects;
+  }
+
   /** Stack walking. Returns null if this is the outermost scope. */
   public ScopeDesc sender() {
     if (isTop()) {
@@ -131,7 +159,7 @@
   //

   private DebugInfoReadStream streamAt(int decodeOffset) {
-    return new DebugInfoReadStream(code, decodeOffset);
+    return new DebugInfoReadStream(code, decodeOffset, objects);
   }

   /** Returns a List&lt;ScopeValue&gt; or null if no values were present */
@@ -161,4 +189,22 @@
     }
     return res;
   }
+
+  /** Returns a List&lt;ObjectValue&gt; or null if no values were present */
+  private List decodeObjectValues(int decodeOffset) {
+    if (decodeOffset == DebugInformationRecorder.SERIALIZED_NULL) {
+      return null;
+    }
+    List res = new ArrayList();
+    DebugInfoReadStream stream = new DebugInfoReadStream(code, decodeOffset, res);
+    int length = stream.readInt();
+    for (int i = 0; i < length; i++) {
+      // Objects values are pushed to 'res' array during read so that
+      // object's fields could reference it (OBJECT_ID_CODE).
+      ScopeValue.readFrom(stream);
+      // res.add(ScopeValue.readFrom(stream));
+    }
+    Assert.that(res.size() == length, "inconsistent debug information");
+    return res;
+  }
 }
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Mon Jul 27 17:23:52 2009 -0400
@@ -49,12 +49,15 @@
   static final int CONSTANT_OOP_CODE    = 2;
   static final int CONSTANT_LONG_CODE   = 3;
   static final int CONSTANT_DOUBLE_CODE = 4;
+  static final int CONSTANT_OBJECT_CODE = 5;
+  static final int CONSTANT_OBJECT_ID_CODE = 6;

   public boolean isLocation()       { return false; }
   public boolean isConstantInt()    { return false; }
   public boolean isConstantDouble() { return false; }
   public boolean isConstantLong()   { return false; }
   public boolean isConstantOop()    { return false; }
+  public boolean isObject()         { return false; }

   public static ScopeValue readFrom(DebugInfoReadStream stream) {
     switch (stream.readInt()) {
@@ -68,6 +71,10 @@
       return new ConstantLongValue(stream);
     case CONSTANT_DOUBLE_CODE:
       return new ConstantDoubleValue(stream);
+    case CONSTANT_OBJECT_CODE:
+      return stream.readObjectValue();
+    case CONSTANT_OBJECT_ID_CODE:
+      return stream.getCachedObject();
     default:
       Assert.that(false, "should not reach here");
       return null;
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Mon Jul 27 17:23:52 2009 -0400
@@ -249,6 +249,7 @@
             OopHandle givenHandle = obj.getHandle();
             for (Iterator itr = monitors.iterator(); itr.hasNext();) {
                 MonitorInfo mi = (MonitorInfo) itr.next();
+                if (mi.eliminated() && frame.isCompiledFrame()) continue; // skip eliminated monitor
                 if (givenHandle.equals(mi.owner())) {
                     res++;
                 }
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Mon Jul 27 17:23:52 2009 -0400
@@ -301,6 +301,9 @@
             List frameMonitors = frame.getMonitors();  // List<MonitorInfo>
             for (Iterator miItr = frameMonitors.iterator(); miItr.hasNext(); ) {
                 sun.jvm.hotspot.runtime.MonitorInfo mi = (sun.jvm.hotspot.runtime.MonitorInfo) miItr.next();
+                if (mi.eliminated() && frame.isCompiledFrame()) {
+                  continue; // skip eliminated monitor
+                }
                 OopHandle obj = mi.owner();
                 if (obj == null) {
                     // this monitor doesn't have an owning object so skip it
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Mon Jul 27 17:23:52 2009 -0400
@@ -131,8 +131,18 @@
     List result = new ArrayList(monitors.size());
     for (int i = 0; i < monitors.size(); i++) {
       MonitorValue mv = (MonitorValue) monitors.get(i);
-      StackValue ownerSV = createStackValue(mv.owner()); // it is an oop
-      result.add(new MonitorInfo(ownerSV.getObject(), resolveMonitorLock(mv.basicLock())));
+      ScopeValue ov = mv.owner();
+      StackValue ownerSV = createStackValue(ov); // it is an oop
+      if (ov.isObject()) { // The owner object was scalar replaced
+        Assert.that(mv.eliminated() && ownerSV.objIsScalarReplaced(), "monitor should be eliminated for scalar replaced object");
+        // Put klass for scalar replaced object.
+        ScopeValue kv = ((ObjectValue)ov).getKlass();
+        Assert.that(kv.isConstantOop(), "klass should be oop constant for scalar replaced object");
+        OopHandle k = ((ConstantOopReadValue)kv).getValue();
+        result.add(new MonitorInfo(k, resolveMonitorLock(mv.basicLock()), mv.eliminated(), true));
+      } else {
+        result.add(new MonitorInfo(ownerSV.getObject(), resolveMonitorLock(mv.basicLock()), mv.eliminated(), false));
+      }
     }
     return result;
   }
@@ -212,12 +222,12 @@
           // long or is unused.  He always saves a long.  Here we know
           // a long was saved, but we only want an narrow oop back.  Narrow the
           // saved long to the narrow oop that the JVM wants.
-          return new StackValue(valueAddr.getCompOopHandleAt(VM.getVM().getIntSize()));
+          return new StackValue(valueAddr.getCompOopHandleAt(VM.getVM().getIntSize()), 0);
         } else {
-          return new StackValue(valueAddr.getCompOopHandleAt(0));
+          return new StackValue(valueAddr.getCompOopHandleAt(0), 0);
         }
       } else if( loc.holdsOop() ) {  // Holds an oop?
-        return new StackValue(valueAddr.getOopHandleAt(0));
+        return new StackValue(valueAddr.getOopHandleAt(0), 0);
       } else if( loc.holdsDouble() ) {
         // Double value in a single stack slot
         return new StackValue(valueAddr.getJIntAt(0) & 0xFFFFFFFF);
@@ -277,7 +287,7 @@
       return new StackValue(((ConstantIntValue) sv).getValue() & 0xFFFFFFFF);
     } else if (sv.isConstantOop()) {
       // constant oop
-      return new StackValue(((ConstantOopReadValue) sv).getValue());
+      return new StackValue(((ConstantOopReadValue) sv).getValue(), 0);
     } else if (sv.isConstantDouble()) {
       // Constant double in a single stack slot
       double d = ((ConstantDoubleValue) sv).getValue();
@@ -285,6 +295,9 @@
     } else if (VM.getVM().isLP64() && sv.isConstantLong()) {
       // Constant long in a single stack slot
       return new StackValue(((ConstantLongValue) sv).getValue() & 0xFFFFFFFF);
+    } else if (sv.isObject()) {
+      // Scalar replaced object in compiled frame
+      return new StackValue(((ObjectValue)sv).getValue(), 1);
     }

     // Unknown ScopeValue type
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Mon Jul 27 17:23:52 2009 -0400
@@ -61,7 +61,7 @@
       StackValue sv;
       if (oopMask.isOop(i)) {
         // oop value
-        sv = new StackValue(addr.getOopHandleAt(0));
+        sv = new StackValue(addr.getOopHandleAt(0), 0);
       } else {
         // integer
         // Fetch a signed integer the size of a stack slot
@@ -95,7 +95,7 @@
       StackValue sv;
       if (oopMask.isOop(i + nofLocals)) {
         // oop value
-        sv = new StackValue(addr.getOopHandleAt(0));
+        sv = new StackValue(addr.getOopHandleAt(0), 0);
       } else {
         // integer
         // Fetch a signed integer the size of a stack slot
@@ -113,7 +113,7 @@
     for (BasicObjectLock current = getFrame().interpreterFrameMonitorEnd();
          current.address().lessThan(getFrame().interpreterFrameMonitorBegin().address());
          current = getFrame().nextMonitorInInterpreterFrame(current)) {
-      result.add(new MonitorInfo(current.obj(), current.lock()));
+      result.add(new MonitorInfo(current.obj(), current.lock(), false, false));
     }
     return result;
   }
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Mon Jul 27 17:23:52 2009 -0400
@@ -25,16 +25,39 @@
 package sun.jvm.hotspot.runtime;

 import sun.jvm.hotspot.debugger.*;
+import sun.jvm.hotspot.utilities.*;

 public class MonitorInfo {
   private OopHandle owner;
   private BasicLock lock;
+  private OopHandle ownerKlass;
+  private boolean eliminated;
+  private boolean ownerIsScalarReplaced;

-  public MonitorInfo(OopHandle owner, BasicLock lock) {
-    this.owner = owner;
-    this.lock  = lock;
+  public MonitorInfo(OopHandle owner, BasicLock lock, boolean eliminated, boolean ownerIsScalarReplaced) {
+    if (!ownerIsScalarReplaced) {
+      this.owner = owner;
+      this.ownerKlass = null;
+    } else {
+      Assert.that(eliminated, "monitor should be eliminated for scalar replaced object");
+      this.owner = null;
+      this.ownerKlass = owner;
+    }
+    this.eliminated = eliminated;
+    this.ownerIsScalarReplaced = ownerIsScalarReplaced;
   }

-  public OopHandle owner() { return owner; }
+  public OopHandle owner() {
+   Assert.that(!ownerIsScalarReplaced, "should not be called for scalar replaced object");
+   return owner;
+  }
+
+  public OopHandle ownerKlass() {
+   Assert.that(ownerIsScalarReplaced, "should not be called for not scalar replaced object");
+   return ownerKlass;
+  }
+
   public BasicLock lock()  { return lock; }
+  public boolean eliminated() { return eliminated; }
+  public boolean ownerIsScalarReplaced() { return ownerIsScalarReplaced; }
 }
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Mon Jul 27 17:23:52 2009 -0400
@@ -37,9 +37,11 @@
     type = BasicType.getTConflict();
   }

-  public StackValue(OopHandle h) {
+  public StackValue(OopHandle h, long scalar_replaced) {
     handleValue = h;
     type = BasicType.getTObject();
+    integerValue = scalar_replaced;
+    Assert.that(integerValue == 0 || handleValue == null, "not null object should not be marked as scalar replaced");
   }

   public StackValue(long i) {
@@ -59,6 +61,13 @@
     return handleValue;
   }

+  boolean objIsScalarReplaced() {
+    if (Assert.ASSERTS_ENABLED) {
+      Assert.that(type == BasicType.getTObject(), "type check");
+    }
+    return integerValue != 0;
+  }
+
   public long getInteger() {
     if (Assert.ASSERTS_ENABLED) {
       Assert.that(type == BasicType.getTInt(), "type check");
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Mon Jul 27 17:23:52 2009 -0400
@@ -161,6 +161,8 @@
           child = new OopTreeNodeAdapter(field.getValue(getObj()), field.getID(), getTreeTableMode());
         } catch (AddressException e) {
           child = new BadOopTreeNodeAdapter(field.getValueAsOopHandle(getObj()), field.getID(), getTreeTableMode());
+        } catch (UnknownOopException e) {
+          child = new BadOopTreeNodeAdapter(field.getValueAsOopHandle(getObj()), field.getID(), getTreeTableMode());
         }
       }
       ++curField;
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Mon Jul 27 17:23:52 2009 -0400
@@ -135,6 +135,10 @@
                     List frameMonitors = frame.getMonitors();  // List<MonitorInfo>
                     for (Iterator miItr = frameMonitors.iterator(); miItr.hasNext(); ) {
                         MonitorInfo mi = (MonitorInfo) miItr.next();
+
+                        if (mi.eliminated() && frame.isCompiledFrame()) {
+                          continue; // skip eliminated monitor
+                        }
                         OopHandle obj = mi.owner();
                         if (obj == null) {
                             // this monitor doesn't have an owning object so skip it
--- a/hotspot/make/hotspot_version	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/make/hotspot_version	Mon Jul 27 17:23:52 2009 -0400
@@ -35,7 +35,7 @@

 HS_MAJOR_VER=16
 HS_MINOR_VER=0
-HS_BUILD_NUMBER=04
+HS_BUILD_NUMBER=06

 JDK_MAJOR_VER=1
 JDK_MINOR_VER=7
--- a/hotspot/make/solaris/makefiles/adlc.make	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/make/solaris/makefiles/adlc.make	Mon Jul 27 17:23:52 2009 -0400
@@ -68,7 +68,9 @@

 # CFLAGS_WARN holds compiler options to suppress/enable warnings.
 # Compiler warnings are treated as errors
-CFLAGS_WARN = +w -errwarn
+ifeq ($(shell expr $(COMPILER_REV_NUMERIC) \>= 509), 1)
+  CFLAGS_WARN = +w -errwarn
+endif
 CFLAGS += $(CFLAGS_WARN)

 ifeq ("${Platform_compiler}", "sparcWorks")
--- a/hotspot/make/solaris/makefiles/optimized.make	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/make/solaris/makefiles/optimized.make	Mon Jul 27 17:23:52 2009 -0400
@@ -41,7 +41,7 @@
 endif

 # Workaround SS11 bug 6345274 (all platforms) (Fixed in SS11 patch and SS12)
-ifeq ($(COMPILER_REV_NUMERIC),508))
+ifeq ($(COMPILER_REV_NUMERIC),508)
 OPT_CFLAGS/ciTypeFlow.o = $(OPT_CFLAGS/O2)
 endif # COMPILER_REV_NUMERIC == 508
--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -4208,6 +4208,7 @@
                   PtrQueue::byte_offset_of_active()),
          tmp);
   }
+
   // Check on whether to annul.
   br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
   delayed() -> nop();
@@ -4215,13 +4216,13 @@
   // satb_log_barrier_work1(tmp, offset);
   if (index == noreg) {
     if (Assembler::is_simm13(offset)) {
-      ld_ptr(obj, offset, tmp);
+      load_heap_oop(obj, offset, tmp);
     } else {
       set(offset, tmp);
-      ld_ptr(obj, tmp, tmp);
+      load_heap_oop(obj, tmp, tmp);
     }
   } else {
-    ld_ptr(obj, index, tmp);
+    load_heap_oop(obj, index, tmp);
   }

   // satb_log_barrier_work2(obj, tmp, offset);
--- a/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -371,7 +371,7 @@
   }
   __ move(value.result(), array_addr, null_check_info);
   if (obj_store) {
-    // Is this precise?
+    // Precise card mark
     post_barrier(LIR_OprFact::address(array_addr), value.result());
   }
 }
@@ -685,11 +685,8 @@
   LIR_Opr result = rlock_result(x);
   __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result);
   if (type == objectType) {  // Write-barrier needed for Object fields.
-#ifdef PRECISE_CARDMARK
+    // Precise card mark since could either be object or array
     post_barrier(addr, val.result());
-#else
-    post_barrier(obj.result(), val.result());
-#endif // PRECISE_CARDMARK
   }
 }
--- a/hotspot/src/cpu/sparc/vm/sparc.ad	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/sparc/vm/sparc.ad	Mon Jul 27 17:23:52 2009 -0400
@@ -1891,17 +1891,17 @@
 // The intptr_t operand types, defined by textual substitution.
 // (Cf. opto/type.hpp.  This lets us avoid many, many other ifdefs.)
 #ifdef _LP64
-#define immX     immL
-#define immX13   immL13
-#define immX13m7 immL13m7
-#define iRegX    iRegL
-#define g1RegX   g1RegL
+#define immX      immL
+#define immX13    immL13
+#define immX13m7  immL13m7
+#define iRegX     iRegL
+#define g1RegX    g1RegL
 #else
-#define immX     immI
-#define immX13   immI13
-#define immX13m7 immI13m7
-#define iRegX    iRegI
-#define g1RegX   g1RegI
+#define immX      immI
+#define immX13    immI13
+#define immX13m7  immI13m7
+#define iRegX     iRegI
+#define g1RegX    g1RegI
 #endif

 //----------ENCODING BLOCK-----------------------------------------------------
@@ -3446,6 +3446,15 @@
   interface(CONST_INTER);
 %}

+// Integer Immediate: 8-bit
+operand immI8() %{
+  predicate(Assembler::is_simm(n->get_int(), 8));
+  match(ConI);
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // Integer Immediate: 13-bit
 operand immI13() %{
   predicate(Assembler::is_simm13(n->get_int()));
@@ -3466,6 +3475,15 @@
   interface(CONST_INTER);
 %}

+// Integer Immediate: 16-bit
+operand immI16() %{
+  predicate(Assembler::is_simm(n->get_int(), 16));
+  match(ConI);
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // Unsigned (positive) Integer Immediate: 13-bit
 operand immU13() %{
   predicate((0 <= n->get_int()) && Assembler::is_simm13(n->get_int()));
@@ -5544,7 +5562,7 @@
   ins_encode %{
     __ ldub($mem$$Address, $dst$$Register);
   %}
-  ins_pipe(iload_mask_mem);
+  ins_pipe(iload_mem);
 %}

 // Load Unsigned Byte (8bit UNsigned) into a Long Register
@@ -5557,7 +5575,22 @@
   ins_encode %{
     __ ldub($mem$$Address, $dst$$Register);
   %}
-  ins_pipe(iload_mask_mem);
+  ins_pipe(iload_mem);
+%}
+
+// Load Unsigned Byte (8 bit UNsigned) with 8-bit mask into Long Register
+instruct loadUB2L_immI8(iRegL dst, memory mem, immI8 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
+  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+
+  size(2*4);
+  format %{ "LDUB   $mem,$dst\t# ubyte & 8-bit mask -> long\n\t"
+            "AND    $dst,$mask,$dst" %}
+  ins_encode %{
+    __ ldub($mem$$Address, $dst$$Register);
+    __ and3($dst$$Register, $mask$$constant, $dst$$Register);
+  %}
+  ins_pipe(iload_mem);
 %}

 // Load Short (16bit signed)
@@ -5610,7 +5643,7 @@
   ins_encode %{
     __ lduh($mem$$Address, $dst$$Register);
   %}
-  ins_pipe(iload_mask_mem);
+  ins_pipe(iload_mem);
 %}

 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
@@ -5636,7 +5669,56 @@
   ins_encode %{
     __ lduh($mem$$Address, $dst$$Register);
   %}
-  ins_pipe(iload_mask_mem);
+  ins_pipe(iload_mem);
+%}
+
+// Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
+instruct loadUS2L_immI_255(iRegL dst, indOffset13m7 mem, immI_255 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUB   $mem+1,$dst\t! ushort/char & 0xFF -> long" %}
+  ins_encode %{
+    __ ldub($mem$$Address, $dst$$Register, 1);  // LSB is index+1 on BE
+  %}
+  ins_pipe(iload_mem);
+%}
+
+// Load Unsigned Short/Char (16bit UNsigned) with a 13-bit mask into a Long Register
+instruct loadUS2L_immI13(iRegL dst, memory mem, immI13 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+
+  size(2*4);
+  format %{ "LDUH   $mem,$dst\t! ushort/char & 13-bit mask -> long\n\t"
+            "AND    $dst,$mask,$dst" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ lduh($mem$$Address, Rdst);
+    __ and3(Rdst, $mask$$constant, Rdst);
+  %}
+  ins_pipe(iload_mem);
+%}
+
+// Load Unsigned Short/Char (16bit UNsigned) with a 16-bit mask into a Long Register
+instruct loadUS2L_immI16(iRegL dst, memory mem, immI16 mask, iRegL tmp) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  effect(TEMP dst, TEMP tmp);
+  ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
+
+  size(3*4);
+  format %{ "LDUH   $mem,$dst\t! ushort/char & 16-bit mask -> long\n\t"
+            "SET    $mask,$tmp\n\t"
+            "AND    $dst,$tmp,$dst" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    Register Rtmp = $tmp$$Register;
+    __ lduh($mem$$Address, Rdst);
+    __ set($mask$$constant, Rtmp);
+    __ and3(Rdst, Rtmp, Rdst);
+  %}
+  ins_pipe(iload_mem);
 %}

 // Load Integer
@@ -5718,6 +5800,68 @@
   ins_encode %{
     __ ldsw($mem$$Address, $dst$$Register);
   %}
+  ins_pipe(iload_mask_mem);
+%}
+
+// Load Integer with mask 0xFF into a Long Register
+instruct loadI2L_immI_255(iRegL dst, indOffset13m7 mem, immI_255 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUB   $mem+3,$dst\t! int & 0xFF -> long" %}
+  ins_encode %{
+    __ ldub($mem$$Address, $dst$$Register, 3);  // LSB is index+3 on BE
+  %}
+  ins_pipe(iload_mem);
+%}
+
+// Load Integer with mask 0xFFFF into a Long Register
+instruct loadI2L_immI_65535(iRegL dst, indOffset13m7 mem, immI_65535 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUH   $mem+2,$dst\t! int & 0xFFFF -> long" %}
+  ins_encode %{
+    __ lduh($mem$$Address, $dst$$Register, 2);  // LSW is index+2 on BE
+  %}
+  ins_pipe(iload_mem);
+%}
+
+// Load Integer with a 13-bit mask into a Long Register
+instruct loadI2L_immI13(iRegL dst, memory mem, immI13 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+
+  size(2*4);
+  format %{ "LDUW   $mem,$dst\t! int & 13-bit mask -> long\n\t"
+            "AND    $dst,$mask,$dst" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ lduw($mem$$Address, Rdst);
+    __ and3(Rdst, $mask$$constant, Rdst);
+  %}
+  ins_pipe(iload_mem);
+%}
+
+// Load Integer with a 32-bit mask into a Long Register
+instruct loadI2L_immI(iRegL dst, memory mem, immI mask, iRegL tmp) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  effect(TEMP dst, TEMP tmp);
+  ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
+
+  size(3*4);
+  format %{ "LDUW   $mem,$dst\t! int & 32-bit mask -> long\n\t"
+            "SET    $mask,$tmp\n\t"
+            "AND    $dst,$tmp,$dst" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    Register Rtmp = $tmp$$Register;
+    __ lduw($mem$$Address, Rdst);
+    __ set($mask$$constant, Rtmp);
+    __ and3(Rdst, Rtmp, Rdst);
+  %}
   ins_pipe(iload_mem);
 %}
--- a/hotspot/src/cpu/x86/vm/assembler_x86.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/assembler_x86.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -6805,14 +6805,18 @@
   jcc(Assembler::equal, done);

   // if (x.f == NULL) goto done;
-  cmpptr(Address(obj, 0), NULL_WORD);
+#ifdef _LP64
+  load_heap_oop(tmp2, Address(obj, 0));
+#else
+  movptr(tmp2, Address(obj, 0));
+#endif
+  cmpptr(tmp2, (int32_t) NULL_WORD);
   jcc(Assembler::equal, done);

   // Can we store original value in the thread's buffer?

-  LP64_ONLY(movslq(tmp, index);)
-  movptr(tmp2, Address(obj, 0));
 #ifdef _LP64
+  movslq(tmp, index);
   cmpq(tmp, 0);
 #else
   cmpl(index, 0);
@@ -6834,8 +6838,7 @@
   if(tosca_live) push(rax);
   push(obj);
 #ifdef _LP64
-  movq(c_rarg0, Address(obj, 0));
-  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, r15_thread);
+  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, r15_thread);
 #else
   push(thread);
   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, thread);
--- a/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1372,6 +1372,8 @@
     // If no method data exists, go to profile_continue.
     test_method_data_pointer(mdp, profile_continue);

+    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
+
     // The method data pointer needs to be updated.
     int mdp_delta = in_bytes(BitData::bit_data_size());
     if (TypeProfileCasts) {
--- a/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1409,6 +1409,8 @@
     // If no method data exists, go to profile_continue.
     test_method_data_pointer(mdp, profile_continue);

+    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
+
     // The method data pointer needs to be updated.
     int mdp_delta = in_bytes(BitData::bit_data_size());
     if (TypeProfileCasts) {
--- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -269,11 +269,11 @@

 #ifndef PRODUCT
 void trace_method_handle_stub(const char* adaptername,
-                              oop mh,
+                              oopDesc* mh,
                               intptr_t* entry_sp,
                               intptr_t* saved_sp) {
   // called as a leaf from native code: do not block the JVM!
-  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, mh, entry_sp, entry_sp - saved_sp);
+  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, (void*)mh, entry_sp, entry_sp - saved_sp);
 }
 #endif //PRODUCT
--- a/hotspot/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1302,22 +1302,19 @@

   const Register ic_reg = rax;
   const Register receiver = j_rarg0;
-  const Register tmp = rdx;

   Label ok;
   Label exception_pending;

+  assert_different_registers(ic_reg, receiver, rscratch1);
   __ verify_oop(receiver);
-  __ push(tmp); // spill (any other registers free here???)
-  __ load_klass(tmp, receiver);
-  __ cmpq(ic_reg, tmp);
+  __ load_klass(rscratch1, receiver);
+  __ cmpq(ic_reg, rscratch1);
   __ jcc(Assembler::equal, ok);

-  __ pop(tmp);
   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));

   __ bind(ok);
-  __ pop(tmp);

   // Verified entry point must be aligned
   __ align(8);
--- a/hotspot/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -709,7 +709,7 @@
   //
   //  Input:
   //     start   -  starting address
-  //     end     -  element count
+  //     count   -  element count
   void  gen_write_ref_array_pre_barrier(Register start, Register count) {
     assert_different_registers(start, count);
     BarrierSet* bs = Universe::heap()->barrier_set();
@@ -757,7 +757,6 @@
           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
           __ addptr(rsp, 2*wordSize);
           __ popa();
-
         }
         break;
--- a/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1207,9 +1207,9 @@
           __ pusha();                      // push registers (overkill)
           // must compute element count unless barrier set interface is changed (other platforms supply count)
           assert_different_registers(start, end, scratch);
-          __ lea(scratch, Address(end, wordSize));
-          __ subptr(scratch, start);
-          __ shrptr(scratch, LogBytesPerWord);
+          __ lea(scratch, Address(end, BytesPerHeapOop));
+          __ subptr(scratch, start);               // subtract start to get #bytes
+          __ shrptr(scratch, LogBytesPerHeapOop);  // convert to element count
           __ mov(c_rarg0, start);
           __ mov(c_rarg1, scratch);
           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
@@ -1225,6 +1225,7 @@
           Label L_loop;

            __ shrptr(start, CardTableModRefBS::card_shift);
+           __ addptr(end, BytesPerHeapOop);
            __ shrptr(end, CardTableModRefBS::card_shift);
            __ subptr(end, start); // number of bytes to copy

@@ -2251,6 +2252,7 @@
     // and report their number to the caller.
     assert_different_registers(rax, r14_length, count, to, end_to, rcx);
     __ lea(end_to, to_element_addr);
+    __ addptr(end_to, -heapOopSize);      // make an inclusive end pointer
     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
     __ movptr(rax, r14_length);           // original oops
     __ addptr(rax, count);                // K = (original - remaining) oops
@@ -2259,7 +2261,7 @@

     // Come here on success only.
     __ BIND(L_do_card_marks);
-    __ addptr(end_to, -wordSize);         // make an inclusive end pointer
+    __ addptr(end_to, -heapOopSize);         // make an inclusive end pointer
     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
     __ xorptr(rax, rax);                  // return 0 on success
--- a/hotspot/src/cpu/x86/vm/x86_32.ad	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/x86_32.ad	Mon Jul 27 17:23:52 2009 -0400
@@ -6885,8 +6885,9 @@
 %}

 // Load Byte (8bit signed) into Long Register
-instruct loadB2L(eRegL dst, memory mem) %{
+instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (ConvI2L (LoadB mem)));
+  effect(KILL cr);

   ins_cost(375);
   format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
@@ -6917,19 +6918,37 @@
 %}

 // Load Unsigned Byte (8 bit UNsigned) into Long Register
-instruct loadUB2L(eRegL dst, memory mem)
-%{
+instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (ConvI2L (LoadUB mem)));
+  effect(KILL cr);

   ins_cost(250);
   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
             "XOR    $dst.hi,$dst.hi" %}

   ins_encode %{
-    __ movzbl($dst$$Register, $mem$$Address);
-    __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
-  %}
-
+    Register Rdst = $dst$$Register;
+    __ movzbl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
+instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
+            "XOR    $dst.hi,$dst.hi\n\t"
+            "AND    $dst.lo,$mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzbl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+    __ andl(Rdst, $mask$$constant);
+  %}
   ins_pipe(ialu_reg_mem);
 %}

@@ -6960,8 +6979,9 @@
 %}

 // Load Short (16bit signed) into Long Register
-instruct loadS2L(eRegL dst, memory mem) %{
+instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (ConvI2L (LoadS mem)));
+  effect(KILL cr);

   ins_cost(375);
   format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
@@ -7004,8 +7024,9 @@
 %}

 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
-instruct loadUS2L(eRegL dst, memory mem) %{
+instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (ConvI2L (LoadUS mem)));
+  effect(KILL cr);

   ins_cost(250);
   format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
@@ -7019,6 +7040,38 @@
   ins_pipe(ialu_reg_mem);
 %}

+// Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
+instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzbl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
+instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
+            "XOR    $dst.hi,$dst.hi\n\t"
+            "AND    $dst.lo,$mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzwl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+    __ andl(Rdst, $mask$$constant);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
 // Load Integer
 instruct loadI(eRegI dst, memory mem) %{
   match(Set dst (LoadI mem));
@@ -7082,8 +7135,9 @@
 %}

 // Load Integer into Long Register
-instruct loadI2L(eRegL dst, memory mem) %{
+instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (ConvI2L (LoadI mem)));
+  effect(KILL cr);

   ins_cost(375);
   format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
@@ -7099,9 +7153,57 @@
   ins_pipe(ialu_reg_mem);
 %}

+// Load Integer with mask 0xFF into Long Register
+instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzbl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer with mask 0xFFFF into Long Register
+instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzwl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer with 32-bit mask into Long Register
+instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  effect(KILL cr);
+
+  format %{ "MOV    $dst.lo,$mem\t# int & 32-bit mask -> long\n\t"
+            "XOR    $dst.hi,$dst.hi\n\t"
+            "AND    $dst.lo,$mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movl(Rdst, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
+    __ andl(Rdst, $mask$$constant);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
 // Load Unsigned Integer into Long Register
-instruct loadUI2L(eRegL dst, memory mem) %{
+instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
   match(Set dst (LoadUI2L mem));
+  effect(KILL cr);

   ins_cost(250);
   format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
@@ -7695,6 +7797,17 @@
   ins_pipe( ialu_mem_long_reg );
 %}

+// Store Long to Integer
+instruct storeL2I(memory mem, eRegL src) %{
+  match(Set mem (StoreI mem (ConvL2I src)));
+
+  format %{ "MOV    $mem,$src.lo\t# long -> int" %}
+  ins_encode %{
+    __ movl($mem$$Address, $src$$Register);
+  %}
+  ins_pipe(ialu_mem_reg);
+%}
+
 // Volatile Store Long.  Must be atomic, so move it into
 // the FP TOS and then do a 64-bit FIST.  Has to probe the
 // target address before the store (for null-ptr checks)
--- a/hotspot/src/cpu/x86/vm/x86_64.ad	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/cpu/x86/vm/x86_64.ad	Mon Jul 27 17:23:52 2009 -0400
@@ -6444,6 +6444,21 @@
   ins_pipe(ialu_reg_mem);
 %}

+// Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
+instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
+  effect(KILL cr);
+
+  format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
+            "andl    $dst, $mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzbq(Rdst, $mem$$Address);
+    __ andl(Rdst, $mask$$constant);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
 // Load Short (16 bit signed)
 instruct loadS(rRegI dst, memory mem)
 %{
@@ -6528,6 +6543,32 @@
   ins_pipe(ialu_reg_mem);
 %}

+// Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
+instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+
+  format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
+  ins_encode %{
+    __ movzbq($dst$$Register, $mem$$Address);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
+instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+  effect(KILL cr);
+
+  format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
+            "andl    $dst, $mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movzwq(Rdst, $mem$$Address);
+    __ andl(Rdst, $mask$$constant);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
 // Load Integer
 instruct loadI(rRegI dst, memory mem)
 %{
@@ -6606,6 +6647,43 @@
   ins_pipe(ialu_reg_mem);
 %}

+// Load Integer with mask 0xFF into Long Register
+instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+
+  format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
+  ins_encode %{
+    __ movzbq($dst$$Register, $mem$$Address);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer with mask 0xFFFF into Long Register
+instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+
+  format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
+  ins_encode %{
+    __ movzwq($dst$$Register, $mem$$Address);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer with a 32-bit mask into Long Register
+instruct loadI2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
+  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+  effect(KILL cr);
+
+  format %{ "movl    $dst, $mem\t# int & 32-bit mask -> long\n\t"
+            "andl    $dst, $mask" %}
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    __ movl(Rdst, $mem$$Address);
+    __ andl(Rdst, $mask$$constant);
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
 // Load Unsigned Integer into Long Register
 instruct loadUI2L(rRegL dst, memory mem)
 %{
@@ -11673,8 +11751,9 @@

   ins_cost(125);
   format %{ "movslq  $dst, $src\t# i2l" %}
-  opcode(0x63); // needs REX.W
-  ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
+  ins_encode %{
+    __ movslq($dst$$Register, $src$$Register);
+  %}
   ins_pipe(ialu_reg_reg);
 %}
--- a/hotspot/src/share/tools/MakeDeps/Database.java	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/tools/MakeDeps/Database.java	Mon Jul 27 17:23:52 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,7 +39,6 @@
   private HashMap<String,String> platformDepFiles;
   private long threshold;
   private int nOuterFiles;
-  private int nPrecompiledFiles;
   private boolean missingOk;
   private Platform plat;
   /** These allow you to specify files not in the include database
@@ -62,7 +61,6 @@

     threshold = t;
     nOuterFiles = 0;
-    nPrecompiledFiles = 0;
     missingOk = false;
     firstFile = null;
     lastFile = null;
@@ -343,7 +341,6 @@
             plat.getGIFileTemplate().getInvDir() +
             list.getName() +
             "\"");
-        nPrecompiledFiles += 1;
       }
     }
     inclFile.println();
@@ -408,22 +405,22 @@
       gd.println();
     }

-    if (nPrecompiledFiles > 0) {
-      // write Precompiled_Files = ...
-      gd.println("Precompiled_Files = \\");
-      for (Iterator iter = grandInclude.iterator(); iter.hasNext(); ) {
-        FileList list = (FileList) iter.next();
+    // write Precompiled_Files = ...
+    gd.println("Precompiled_Files = \\");
+    for (Iterator iter = grandInclude.iterator(); iter.hasNext(); ) {
+      FileList list = (FileList) iter.next();
+      if (list.getCount() >= threshold) {
         gd.println(list.getName() + " \\");
         String platformDep = platformDepFiles.get(list.getName());
         if (platformDep != null) {
-            // make sure changes to the platform dependent file will
-            // cause regeneration of the pch file.
-            gd.println(platformDep + " \\");
+          // make sure changes to the platform dependent file will
+          // cause regeneration of the pch file.
+          gd.println(platformDep + " \\");
         }
       }
-      gd.println();
-      gd.println();
     }
+    gd.println();
+    gd.println();

     gd.println("DTraced_Files = \\");
     for (Iterator iter = outerFiles.iterator(); iter.hasNext(); ) {
@@ -483,7 +480,6 @@
         }

         if (plat.includeGIDependencies()
-            && nPrecompiledFiles > 0
             && anII.getUseGrandInclude()) {
           gd.println("    $(Precompiled_Files) \\");
         }
--- a/hotspot/src/share/vm/adlc/formssel.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/adlc/formssel.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -420,6 +420,13 @@
   return  _matrule->is_ideal_load();
 }

+// Return 'true' if this instruction matches an ideal 'LoadKlass' node
+bool InstructForm::skip_antidep_check() const {
+  if( _matrule == NULL ) return false;
+
+  return  _matrule->skip_antidep_check();
+}
+
 // Return 'true' if this instruction matches an ideal 'Load?' node
 Form::DataType InstructForm::is_ideal_store() const {
   if( _matrule == NULL ) return Form::none;
@@ -567,6 +574,8 @@

 // loads from memory, so must check for anti-dependence
 bool InstructForm::needs_anti_dependence_check(FormDict &globals) const {
+  if ( skip_antidep_check() ) return false;
+
   // Machine independent loads must be checked for anti-dependences
   if( is_ideal_load() != Form::none )  return true;

@@ -3957,6 +3966,28 @@
 }


+bool MatchRule::skip_antidep_check() const {
+  // Some loads operate on what is effectively immutable memory so we
+  // should skip the anti dep computations.  For some of these nodes
+  // the rewritable field keeps the anti dep logic from triggering but
+  // for certain kinds of LoadKlass it does not since they are
+  // actually reading memory which could be rewritten by the runtime,
+  // though never by generated code.  This disables it uniformly for
+  // the nodes that behave like this: LoadKlass, LoadNKlass and
+  // LoadRange.
+  if ( _opType && (strcmp(_opType,"Set") == 0) && _rChild ) {
+    const char *opType = _rChild->_opType;
+    if (strcmp("LoadKlass", opType) == 0 ||
+        strcmp("LoadNKlass", opType) == 0 ||
+        strcmp("LoadRange", opType) == 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+
 Form::DataType MatchRule::is_ideal_store() const {
   Form::DataType ideal_store = Form::none;
--- a/hotspot/src/share/vm/adlc/formssel.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/adlc/formssel.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -158,6 +158,9 @@

   virtual Form::CallType is_ideal_call() const; // matches ideal 'Call'
   virtual Form::DataType is_ideal_load() const; // node matches ideal 'LoadXNode'
+  // Should antidep checks be disabled for this Instruct
+  // See definition of MatchRule::skip_antidep_check
+  bool skip_antidep_check() const;
   virtual Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'
           bool        is_ideal_mem() const { return is_ideal_load() != Form::none || is_ideal_store() != Form::none; }
   virtual uint        two_address(FormDict &globals); // output reg must match input reg
@@ -1003,6 +1006,9 @@
   bool       is_ideal_loopEnd() const; // node matches ideal 'LoopEnd'
   bool       is_ideal_bool() const;    // node matches ideal 'Bool'
   Form::DataType is_ideal_load() const;// node matches ideal 'LoadXNode'
+  // Should antidep checks be disabled for this rule
+  // See definition of MatchRule::skip_antidep_check
+  bool skip_antidep_check() const;
   Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'

   // Check if 'mRule2' is a cisc-spill variant of this MatchRule
--- a/hotspot/src/share/vm/adlc/output_h.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/adlc/output_h.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1367,11 +1367,11 @@
         else if (!strcmp(oper->ideal_type(_globalNames), "ConN")) {
           // Access the locally stored constant
           fprintf(fp,"  virtual intptr_t       constant() const {");
-          fprintf(fp,   " return _c0->make_oopptr()->get_con();");
+          fprintf(fp,   " return _c0->get_ptrtype()->get_con();");
           fprintf(fp, " }\n");
           // Generate query to determine if this pointer is an oop
           fprintf(fp,"  virtual bool           constant_is_oop() const {");
-          fprintf(fp,   " return _c0->make_oopptr()->isa_oop_ptr();");
+          fprintf(fp,   " return _c0->get_ptrtype()->isa_oop_ptr();");
           fprintf(fp, " }\n");
         }
         else if (!strcmp(oper->ideal_type(_globalNames), "ConL")) {
--- a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -1534,12 +1534,8 @@
   }

   if (is_oop) {
-#ifdef PRECISE_CARDMARK
-    // Precise cardmarks don't work
-    post_barrier(LIR_OprFact::address(address), value.result());
-#else
+    // Store to object so mark the card of the header
     post_barrier(object.result(), value.result());
-#endif // PRECISE_CARDMARK
   }

   if (is_volatile && os::is_MP()) {
--- a/hotspot/src/share/vm/classfile/classFileParser.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/classfile/classFileParser.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -3237,6 +3237,16 @@
     this_klass->set_minor_version(minor_version);
     this_klass->set_major_version(major_version);

+    // Set up methodOop::intrinsic_id as soon as we know the names of methods.
+    // (We used to do this lazily, but now we query it in Rewriter,
+    // which is eagerly done for every method, so we might as well do it now,
+    // when everything is fresh in memory.)
+    if (methodOopDesc::klass_id_for_intrinsics(this_klass->as_klassOop()) != vmSymbols::NO_SID) {
+      for (int j = 0; j < methods->length(); j++) {
+        ((methodOop)methods->obj_at(j))->init_intrinsic_id();
+      }
+    }
+
     if (cached_class_file_bytes != NULL) {
       // JVMTI: we have an instanceKlass now, tell it about the cached bytes
       this_klass->set_cached_class_file(cached_class_file_bytes,
--- a/hotspot/src/share/vm/classfile/vmSymbols.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/classfile/vmSymbols.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -513,9 +513,6 @@
 //
 // for Emacs: (let ((c-backslash-column 120) (c-backslash-max-column 120)) (c-backslash-region (point) (point-max) nil t))
 #define VM_INTRINSICS_DO(do_intrinsic, do_class, do_name, do_signature, do_alias)                                       \
-  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
-  /*    (symbol object_initializer_name defined above) */                                                               \
-                                                                                                                        \
   do_intrinsic(_hashCode,                 java_lang_Object,       hashCode_name, void_int_signature,             F_R)   \
    do_name(     hashCode_name,                                   "hashCode")                                            \
   do_intrinsic(_getClass,                 java_lang_Object,       getClass_name, void_class_signature,           F_R)   \
@@ -635,9 +632,6 @@
   do_intrinsic(_equalsC,                  java_util_Arrays,       equals_name,    equalsC_signature,             F_S)   \
    do_signature(equalsC_signature,                               "([C[C)Z")                                             \
                                                                                                                         \
-  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
-  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
-                                                                                                                        \
   do_intrinsic(_compareTo,                java_lang_String,       compareTo_name, string_int_signature,          F_R)   \
    do_name(     compareTo_name,                                  "compareTo")                                           \
   do_intrinsic(_indexOf,                  java_lang_String,       indexOf_name, string_int_signature,            F_R)   \
@@ -656,8 +650,6 @@
    do_name(     attemptUpdate_name,                                 "attemptUpdate")                                    \
    do_signature(attemptUpdate_signature,                            "(JJ)Z")                                            \
                                                                                                                         \
-  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
-                                                                                                                        \
   /* support for sun.misc.Unsafe */                                                                                     \
   do_class(sun_misc_Unsafe,               "sun/misc/Unsafe")                                                            \
                                                                                                                         \
@@ -819,10 +811,22 @@
    do_name(     prefetchReadStatic_name,                         "prefetchReadStatic")                                  \
   do_intrinsic(_prefetchWriteStatic,      sun_misc_Unsafe,        prefetchWriteStatic_name, prefetch_signature,  F_SN)  \
    do_name(     prefetchWriteStatic_name,                        "prefetchWriteStatic")                                 \
+    /*== LAST_COMPILER_INLINE*/                                                                                         \
+    /*the compiler does have special inlining code for these; bytecode inline is just fine */                           \
+                                                                                                                        \
+  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
+                                                                                                                        \
+  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
+  /*    (symbol object_initializer_name defined above) */                                                               \
+                                                                                                                        \
+  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
+  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
+                                                                                                                        \
     /*end*/


+
 // Class vmSymbols

 class vmSymbols: AllStatic {
@@ -935,6 +939,7 @@
     #undef VM_INTRINSIC_ENUM

     ID_LIMIT,
+    LAST_COMPILER_INLINE = _prefetchWriteStatic,
     FIRST_ID = _none + 1
   };

@@ -972,4 +977,7 @@
   static Flags              flags_for(ID id);

   static const char* short_name_as_C_string(ID id, char* buf, int size);
+
+  // Access to intrinsic methods:
+  static methodOop method_for(ID id);
 };
--- a/hotspot/src/share/vm/compiler/oopMap.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/compiler/oopMap.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -379,7 +379,15 @@
         if ( loc != NULL ) {
           oop *base_loc    = fr->oopmapreg_to_location(omv.content_reg(), reg_map);
           oop *derived_loc = loc;
-          derived_oop_fn(base_loc, derived_loc);
+          oop val = *base_loc;
+          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
+            // Ignore NULL oops and decoded NULL narrow oops which
+            // equal to Universe::narrow_oop_base when a narrow oop
+            // implicit null check is used in compiled code.
+            // The narrow_oop_base could be NULL or be the address
+            // of the page below heap depending on compressed oops mode.
+          } else
+            derived_oop_fn(base_loc, derived_loc);
         }
         oms.next();
       }  while (!oms.is_done());
@@ -394,6 +402,15 @@
       oop* loc = fr->oopmapreg_to_location(omv.reg(),reg_map);
       if ( loc != NULL ) {
         if ( omv.type() == OopMapValue::oop_value ) {
+          oop val = *loc;
+          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
+            // Ignore NULL oops and decoded NULL narrow oops which
+            // equal to Universe::narrow_oop_base when a narrow oop
+            // implicit null check is used in compiled code.
+            // The narrow_oop_base could be NULL or be the address
+            // of the page below heap depending on compressed oops mode.
+            continue;
+          }
 #ifdef ASSERT
           if ((((uintptr_t)loc & (sizeof(*loc)-1)) != 0) ||
              !Universe::heap()->is_in_or_null(*loc)) {
@@ -410,6 +427,8 @@
 #endif // ASSERT
           oop_fn->do_oop(loc);
         } else if ( omv.type() == OopMapValue::value_value ) {
+          assert((*loc) == (oop)NULL || !Universe::is_narrow_oop_base(*loc),
+                 "found invalid value pointer");
           value_fn->do_oop(loc);
         } else if ( omv.type() == OopMapValue::narrowoop_value ) {
           narrowOop *nl = (narrowOop*)loc;
--- a/hotspot/src/share/vm/compiler/oopMap.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/compiler/oopMap.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -233,6 +233,10 @@
   int heap_size() const;
   void copy_to(address addr);

+  // Methods oops_do() and all_do() filter out NULL oops and
+  // oop == Universe::narrow_oop_base() before passing oops
+  // to closures.
+
   // Iterates through frame for a compiled method
   static void oops_do            (const frame* fr,
                                   const RegisterMap* reg_map, OopClosure* f);
--- a/hotspot/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -42,35 +42,40 @@
     BufferLength = 1024
   };

-  oop          *_buffer[BufferLength];
-  oop         **_buffer_top;
-  oop         **_buffer_curr;
+  StarTask  _buffer[BufferLength];
+  StarTask* _buffer_top;
+  StarTask* _buffer_curr;

-  OopClosure  *_oc;
-  double       _closure_app_seconds;
+  OopClosure* _oc;
+  double      _closure_app_seconds;

   void process_buffer () {
-
     double start = os::elapsedTime();
-    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
-      _oc->do_oop(*curr);
+    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
+      if (curr->is_narrow()) {
+        assert(UseCompressedOops, "Error");
+        _oc->do_oop((narrowOop*)(*curr));
+      } else {
+        _oc->do_oop((oop*)(*curr));
+      }
     }
     _buffer_curr = _buffer;
     _closure_app_seconds += (os::elapsedTime() - start);
   }

-public:
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop *p) {
+  template <class T> inline void do_oop_work(T* p) {
     if (_buffer_curr == _buffer_top) {
       process_buffer();
     }
-
-    *_buffer_curr = p;
+    StarTask new_ref(p);
+    *_buffer_curr = new_ref;
     ++_buffer_curr;
   }
+
+public:
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
+
   void done () {
     if (_buffer_curr > _buffer) {
       process_buffer();
@@ -88,18 +93,17 @@
 class BufferingOopsInGenClosure: public OopsInGenClosure {
   BufferingOopClosure _boc;
   OopsInGenClosure* _oc;
-public:
+ protected:
+  template <class T> inline void do_oop_work(T* p) {
+    assert(generation()->is_in_reserved((void*)p), "Must be in!");
+    _boc.do_oop(p);
+  }
+ public:
   BufferingOopsInGenClosure(OopsInGenClosure *oc) :
     _boc(oc), _oc(oc) {}

-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  virtual void do_oop(oop* p) {
-    assert(generation()->is_in_reserved(p), "Must be in!");
-    _boc.do_oop(p);
-  }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }

   void done() {
     _boc.done();
@@ -130,14 +134,14 @@
     BufferLength = 1024
   };

-  oop                      *_buffer[BufferLength];
-  oop                     **_buffer_top;
-  oop                     **_buffer_curr;
+  StarTask     _buffer[BufferLength];
+  StarTask*    _buffer_top;
+  StarTask*    _buffer_curr;

-  HeapRegion               *_hr_buffer[BufferLength];
-  HeapRegion              **_hr_curr;
+  HeapRegion*  _hr_buffer[BufferLength];
+  HeapRegion** _hr_curr;

-  OopsInHeapRegionClosure  *_oc;
+  OopsInHeapRegionClosure*  _oc;
   double                    _closure_app_seconds;

   void process_buffer () {
@@ -146,15 +150,20 @@
            "the two lengths should be the same");

     double start = os::elapsedTime();
-    HeapRegion **hr_curr = _hr_buffer;
-    HeapRegion *hr_prev = NULL;
-    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
-      HeapRegion *region = *hr_curr;
+    HeapRegion** hr_curr = _hr_buffer;
+    HeapRegion*  hr_prev = NULL;
+    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
+      HeapRegion* region = *hr_curr;
       if (region != hr_prev) {
         _oc->set_region(region);
         hr_prev = region;
       }
-      _oc->do_oop(*curr);
+      if (curr->is_narrow()) {
+        assert(UseCompressedOops, "Error");
+        _oc->do_oop((narrowOop*)(*curr));
+      } else {
+        _oc->do_oop((oop*)(*curr));
+      }
       ++hr_curr;
     }
     _buffer_curr = _buffer;
@@ -163,17 +172,16 @@
   }

 public:
-  virtual void do_oop(narrowOop *p) {
-    guarantee(false, "NYI");
-  }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }

-  virtual void do_oop(oop *p) {
+  template <class T> void do_oop_work(T* p) {
     if (_buffer_curr == _buffer_top) {
       assert(_hr_curr > _hr_buffer, "_hr_curr should be consistent with _buffer_curr");
       process_buffer();
     }
-
-    *_buffer_curr = p;
+    StarTask new_ref(p);
+    *_buffer_curr = new_ref;
     ++_buffer_curr;
     *_hr_curr = _from;
     ++_hr_curr;
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -452,13 +452,10 @@
   _regionStack.allocate(G1MarkRegionStackSize);

   // Create & start a ConcurrentMark thread.
-  if (G1ConcMark) {
-    _cmThread = new ConcurrentMarkThread(this);
-    assert(cmThread() != NULL, "CM Thread should have been created");
-    assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
-  } else {
-    _cmThread = NULL;
-  }
+  _cmThread = new ConcurrentMarkThread(this);
+  assert(cmThread() != NULL, "CM Thread should have been created");
+  assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
+
   _g1h = G1CollectedHeap::heap();
   assert(CGC_lock != NULL, "Where's the CGC_lock?");
   assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
@@ -783,18 +780,18 @@
                      bool do_barrier) : _cm(cm), _g1h(g1h),
                                         _do_barrier(do_barrier) { }

-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  virtual void do_oop(oop* p) {
-    oop thisOop = *p;
-    if (thisOop != NULL) {
-      assert(thisOop->is_oop() || thisOop->mark() == NULL,
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      assert(obj->is_oop() || obj->mark() == NULL,
              "expected an oop, possibly with mark word displaced");
-      HeapWord* addr = (HeapWord*)thisOop;
+      HeapWord* addr = (HeapWord*)obj;
       if (_g1h->is_in_g1_reserved(addr)) {
-        _cm->grayRoot(thisOop);
+        _cm->grayRoot(obj);
       }
     }
     if (_do_barrier) {
@@ -850,16 +847,6 @@
   double start = os::elapsedTime();
   GCOverheadReporter::recordSTWStart(start);

-  // If there has not been a GC[n-1] since last GC[n] cycle completed,
-  // precede our marking with a collection of all
-  // younger generations to keep floating garbage to a minimum.
-  // YSR: we won't do this for now -- it's an optimization to be
-  // done post-beta.
-
-  // YSR:    ignoring weak refs for now; will do at bug fixing stage
-  // EVM:    assert(discoveredRefsAreClear());
-
-
   G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
   g1p->record_concurrent_mark_init_start();
   checkpointRootsInitialPre();
@@ -1135,6 +1122,13 @@
     return;
   }

+  if (VerifyDuringGC) {
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(before)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(true, false, true);
+  }
+
   G1CollectorPolicy* g1p = g1h->g1_policy();
   g1p->record_concurrent_mark_remark_start();

@@ -1159,10 +1153,12 @@
     JavaThread::satb_mark_queue_set().set_active_all_threads(false);

     if (VerifyDuringGC) {
-      g1h->prepare_for_verify();
-      g1h->verify(/* allow_dirty */      true,
-                  /* silent */           false,
-                  /* use_prev_marking */ false);
+      HandleMark hm;  // handle scope
+      gclog_or_tty->print(" VerifyDuringGC:(after)");
+      Universe::heap()->prepare_for_verify();
+      Universe::heap()->verify(/* allow_dirty */      true,
+                               /* silent */           false,
+                               /* use_prev_marking */ false);
     }
   }

@@ -1233,6 +1229,41 @@
                CardTableModRefBS::card_shift);
   }

+  // It takes a region that's not empty (i.e., it has at least one
+  // live object in it and sets its corresponding bit on the region
+  // bitmap to 1. If the region is "starts humongous" it will also set
+  // to 1 the bits on the region bitmap that correspond to its
+  // associated "continues humongous" regions.
+  void set_bit_for_region(HeapRegion* hr) {
+    assert(!hr->continuesHumongous(), "should have filtered those out");
+
+    size_t index = hr->hrs_index();
+    if (!hr->startsHumongous()) {
+      // Normal (non-humongous) case: just set the bit.
+      _region_bm->par_at_put((BitMap::idx_t) index, true);
+    } else {
+      // Starts humongous case: calculate how many regions are part of
+      // this humongous region and then set the bit range. It might
+      // have been a bit more efficient to look at the object that
+      // spans these humongous regions to calculate their number from
+      // the object's size. However, it's a good idea to calculate
+      // this based on the metadata itself, and not the region
+      // contents, so that this code is not aware of what goes into
+      // the humongous regions (in case this changes in the future).
+      G1CollectedHeap* g1h = G1CollectedHeap::heap();
+      size_t end_index = index + 1;
+      while (end_index < g1h->n_regions()) {
+        HeapRegion* chr = g1h->region_at(end_index);
+        if (!chr->continuesHumongous()) {
+          break;
+        }
+        end_index += 1;
+      }
+      _region_bm->par_at_put_range((BitMap::idx_t) index,
+                                   (BitMap::idx_t) end_index, true);
+    }
+  }
+
   bool doHeapRegion(HeapRegion* hr) {
     if (_co_tracker != NULL)
       _co_tracker->update();
@@ -1241,13 +1272,13 @@
       _start_vtime_sec = os::elapsedVTime();

     if (hr->continuesHumongous()) {
-      HeapRegion* hum_start = hr->humongous_start_region();
-      // If the head region of the humongous region has been determined
-      // to be alive, then all the tail regions should be marked
-      // such as well.
-      if (_region_bm->at(hum_start->hrs_index())) {
-        _region_bm->par_at_put(hr->hrs_index(), 1);
-      }
+      // We will ignore these here and process them when their
+      // associated "starts humongous" region is processed (see
+      // set_bit_for_heap_region()). Note that we cannot rely on their
+      // associated "starts humongous" region to have their bit set to
+      // 1 since, due to the region chunking in the parallel region
+      // iteration, a "continues humongous" region might be visited
+      // before its associated "starts humongous".
       return false;
     }

@@ -1343,14 +1374,14 @@
           intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
         mark_card_num_range(start_card_num, last_card_num);
         // This definitely means the region has live objects.
-        _region_bm->par_at_put(hr->hrs_index(), 1);
+        set_bit_for_region(hr);
       }
     }

     hr->add_to_marked_bytes(marked_bytes);
     // Update the live region bitmap.
     if (marked_bytes > 0) {
-      _region_bm->par_at_put(hr->hrs_index(), 1);
+      set_bit_for_region(hr);
     }
     hr->set_top_at_conc_mark_count(nextTop);
     _tot_live += hr->next_live_bytes();
@@ -1623,6 +1654,15 @@
     return;
   }

+  if (VerifyDuringGC) {
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(before)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(/* allow dirty  */ true,
+                     /* silent       */ false,
+                     /* prev marking */ true);
+  }
+
   _cleanup_co_tracker.disable();

   G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
@@ -1755,10 +1795,12 @@
   g1h->increment_total_collections();

   if (VerifyDuringGC) {
-    g1h->prepare_for_verify();
-    g1h->verify(/* allow_dirty */      true,
-                /* silent */           false,
-                /* use_prev_marking */ true);
+    HandleMark hm;  // handle scope
+    gclog_or_tty->print(" VerifyDuringGC:(after)");
+    Universe::heap()->prepare_for_verify();
+    Universe::verify(/* allow dirty  */ true,
+                     /* silent       */ false,
+                     /* prev marking */ true);
   }
 }

@@ -1817,12 +1859,11 @@
     _g1(g1), _cm(cm),
     _bitMap(bitMap) {}

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop thisOop = *p;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    oop thisOop = oopDesc::load_decode_heap_oop(p);
     HeapWord* addr = (HeapWord*)thisOop;
     if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
       _bitMap->mark(addr);
@@ -1981,12 +2022,11 @@
   ReachablePrinterOopClosure(CMBitMapRO* bitmap, outputStream* out) :
     _bitmap(bitmap), _g1h(G1CollectedHeap::heap()), _out(out) { }

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop         obj = *p;
+  void do_oop(narrowOop* p) { do_oop_work(p); }
+  void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    oop         obj = oopDesc::load_decode_heap_oop(p);
     const char* str = NULL;
     const char* str2 = "";

@@ -2128,6 +2168,7 @@


   HeapWord* objAddr = (HeapWord*) obj;
+  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
   if (_g1h->is_in_g1_reserved(objAddr)) {
     tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
     HeapRegion* hr = _g1h->heap_region_containing(obj);
@@ -2345,7 +2386,7 @@
     }
   }

-  bool drain() {
+  template <class T> bool drain() {
     while (_ms_ind > 0) {
       oop obj = pop();
       assert(obj != NULL, "Since index was non-zero.");
@@ -2359,9 +2400,8 @@
         }
         // Now process this portion of this one.
         int lim = MIN2(next_arr_ind, len);
-        assert(!UseCompressedOops, "This needs to be fixed");
         for (int j = arr_ind; j < lim; j++) {
-          do_oop(aobj->obj_at_addr<oop>(j));
+          do_oop(aobj->obj_at_addr<T>(j));
         }

       } else {
@@ -2388,13 +2428,13 @@
     FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
   }

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop obj = *p;
-    if (obj == NULL) return;
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (oopDesc::is_null(heap_oop)) return;
+    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
     if (obj->is_forwarded()) {
       // If the object has already been forwarded, we have to make sure
       // that it's marked.  So follow the forwarding pointer.  Note that
@@ -2443,7 +2483,11 @@
     oop obj = oop(addr);
     if (!obj->is_forwarded()) {
       if (!_oop_cl.push(obj)) return false;
-      if (!_oop_cl.drain()) return false;
+      if (UseCompressedOops) {
+        if (!_oop_cl.drain<narrowOop>()) return false;
+      } else {
+        if (!_oop_cl.drain<oop>()) return false;
+      }
     }
     // Otherwise...
     return true;
@@ -2601,9 +2645,6 @@

 // abandon current marking iteration due to a Full GC
 void ConcurrentMark::abort() {
-  // If we're not marking, nothing to do.
-  if (!G1ConcMark) return;
-
   // Clear all marks to force marking thread to do nothing
   _nextMarkBitMap->clearAll();
   // Empty mark stack
@@ -2779,14 +2820,14 @@
   CMTask*            _task;

 public:
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T* p) {
     tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant" );
-
-    oop obj = *p;
+    tmp_guarantee_CM( !_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(), "invariant" );
+
+    oop obj = oopDesc::load_decode_heap_oop(p);
     if (_cm->verbose_high())
       gclog_or_tty->print_cr("[%d] we're looking at location "
                              "*"PTR_FORMAT" = "PTR_FORMAT,
@@ -2932,6 +2973,7 @@
   ++_refs_reached;

   HeapWord* objAddr = (HeapWord*) obj;
+  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
   if (_g1h->is_in_g1_reserved(objAddr)) {
     tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
     HeapRegion* hr =  _g1h->heap_region_containing(obj);
@@ -2995,6 +3037,7 @@
 void CMTask::push(oop obj) {
   HeapWord* objAddr = (HeapWord*) obj;
   tmp_guarantee_CM( _g1h->is_in_g1_reserved(objAddr), "invariant" );
+  tmp_guarantee_CM( !_g1h->heap_region_containing(objAddr)->is_on_free_list(), "invariant" );
   tmp_guarantee_CM( !_g1h->is_obj_ill(obj), "invariant" );
   tmp_guarantee_CM( _nextMarkBitMap->isMarked(objAddr), "invariant" );

@@ -3240,6 +3283,8 @@

       tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) obj),
                         "invariant" );
+      tmp_guarantee_CM( !_g1h->heap_region_containing(obj)->is_on_free_list(),
+                        "invariant" );

       scan_object(obj);
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -763,6 +763,7 @@
   CMBitMap*                   _nextMarkBitMap;
   // the task queue of this task
   CMTaskQueue*                _task_queue;
+private:
   // the task queue set---needed for stealing
   CMTaskQueueSet*             _task_queues;
   // indicates whether the task has been claimed---this is only  for
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -424,7 +424,7 @@
       while (n <= next_boundary) {
         q = n;
         oop obj = oop(q);
-        if (obj->klass() == NULL) return q;
+        if (obj->klass_or_null() == NULL) return q;
         n += obj->size();
       }
       assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
@@ -436,7 +436,7 @@
       while (n <= next_boundary) {
         q = n;
         oop obj = oop(q);
-        if (obj->klass() == NULL) return q;
+        if (obj->klass_or_null() == NULL) return q;
         n += _sp->block_size(q);
       }
       assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -96,14 +96,14 @@
     while (n <= addr) {
       q = n;
       oop obj = oop(q);
-      if (obj->klass() == NULL) return q;
+      if (obj->klass_or_null() == NULL) return q;
       n += obj->size();
     }
   } else {
     while (n <= addr) {
       q = n;
       oop obj = oop(q);
-      if (obj->klass() == NULL) return q;
+      if (obj->klass_or_null() == NULL) return q;
       n += _sp->block_size(q);
     }
   }
@@ -115,7 +115,7 @@
 inline HeapWord*
 G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q,
                                                      const void* addr) {
-  if (oop(q)->klass() == NULL) return q;
+  if (oop(q)->klass_or_null() == NULL) return q;
   HeapWord* n = q + _sp->block_size(q);
   // In the normal case, where the query "addr" is a card boundary, and the
   // offset table chunks are the same size as cards, the block starting at
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Jul 27 17:23:52 2009 -0400
@@ -902,6 +902,10 @@
                                     size_t word_size) {
   ResourceMark rm;

+  if (PrintHeapAtGC) {
+    Universe::print_heap_before_gc();
+  }
+
   if (full && DisableExplicitGC) {
     gclog_or_tty->print("\n\n\nDisabling Explicit GC\n\n\n");
     return;
@@ -927,7 +931,7 @@
     g1_policy()->record_full_collection_start();

     gc_prologue(true);
-    increment_total_collections();
+    increment_total_collections(true /* full gc */);

     size_t g1h_prev_used = used();
     assert(used() == recalculate_used(), "Should be equal");
@@ -1066,6 +1070,10 @@
     assert( check_young_list_empty(false, false),
             "young list should be empty at this point");
   }
+
+  if (PrintHeapAtGC) {
+    Universe::print_heap_after_gc();
+  }
 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
@@ -1650,8 +1658,15 @@
   assert(Heap_lock->owner() != NULL,
          "Should be owned on this thread's behalf.");
   size_t result = _summary_bytes_used;
-  if (_cur_alloc_region != NULL)
-    result += _cur_alloc_region->used();
+  // Read only once in case it is set to NULL concurrently
+  HeapRegion* hr = _cur_alloc_region;
+  if (hr != NULL)
+    result += hr->used();
+  return result;
+}
+
+size_t G1CollectedHeap::used_unlocked() const {
+  size_t result = _summary_bytes_used;
   return result;
 }

@@ -2125,13 +2140,13 @@
   VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {
     g1h = _g1h;
   }
-  void do_oop(narrowOop *p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop *p) {
-    oop obj = *p;
-    assert(obj == NULL || !g1h->is_obj_dead(obj),
-           "Dead object referenced by a not dead object");
+  void do_oop(narrowOop *p) { do_oop_work(p); }
+  void do_oop(      oop *p) { do_oop_work(p); }
+
+  template <class T> void do_oop_work(T *p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
+    guarantee(obj == NULL || !g1h->is_obj_dead(obj),
+              "Dead object referenced by a not dead object");
   }
 };

@@ -2198,8 +2213,10 @@
   // use_prev_marking == true  -> use "prev" marking information,
   // use_prev_marking == false -> use "next" marking information
   VerifyRegionClosure(bool allow_dirty, bool par, bool use_prev_marking)
-    : _allow_dirty(allow_dirty), _par(par),
+    : _allow_dirty(allow_dirty),
+      _par(par),
       _use_prev_marking(use_prev_marking) {}
+
   bool doHeapRegion(HeapRegion* r) {
     guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
               "Should be unclaimed at verify points.");
@@ -2223,18 +2240,16 @@
   // use_prev_marking == true  -> use "prev" marking information,
   // use_prev_marking == false -> use "next" marking information
   VerifyRootsClosure(bool use_prev_marking) :
-    _g1h(G1CollectedHeap::heap()), _failures(false),
+    _g1h(G1CollectedHeap::heap()),
+    _failures(false),
     _use_prev_marking(use_prev_marking) { }

   bool failures() { return _failures; }

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-
-  void do_oop(oop* p) {
-    oop obj = *p;
-    if (obj != NULL) {
+  template <class T> void do_oop_nv(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
       if (_g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
         gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
                                "points to dead obj "PTR_FORMAT, p, (void*) obj);
@@ -2243,6 +2258,9 @@
       }
     }
   }
+
+  void do_oop(oop* p)       { do_oop_nv(p); }
+  void do_oop(narrowOop* p) { do_oop_nv(p); }
 };

 // This is the task used for parallel heap verification.
@@ -2259,7 +2277,8 @@
   G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty,
                   bool use_prev_marking) :
     AbstractGangTask("Parallel verify task"),
-    _g1h(g1h), _allow_dirty(allow_dirty),
+    _g1h(g1h),
+    _allow_dirty(allow_dirty),
     _use_prev_marking(use_prev_marking) { }

   void work(int worker_i) {
@@ -2325,9 +2344,37 @@
   }
 };

-void G1CollectedHeap::print() const { print_on(gclog_or_tty); }
+void G1CollectedHeap::print() const { print_on(tty); }

 void G1CollectedHeap::print_on(outputStream* st) const {
+  print_on(st, PrintHeapAtGCExtended);
+}
+
+void G1CollectedHeap::print_on(outputStream* st, bool extended) const {
+  st->print(" %-20s", "garbage-first heap");
+  st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",
+            capacity()/K, used_unlocked()/K);
+  st->print(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")",
+            _g1_storage.low_boundary(),
+            _g1_storage.high(),
+            _g1_storage.high_boundary());
+  st->cr();
+  st->print("  region size " SIZE_FORMAT "K, ",
+            HeapRegion::GrainBytes/K);
+  size_t young_regions = _young_list->length();
+  st->print(SIZE_FORMAT " young (" SIZE_FORMAT "K), ",
+            young_regions, young_regions * HeapRegion::GrainBytes / K);
+  size_t survivor_regions = g1_policy()->recorded_survivor_regions();
+  st->print(SIZE_FORMAT " survivors (" SIZE_FORMAT "K)",
+            survivor_regions, survivor_regions * HeapRegion::GrainBytes / K);
+  st->cr();
+  perm()->as_gen()->print_on(st);
+  if (extended) {
+    print_on_extended(st);
+  }
+}
+
+void G1CollectedHeap::print_on_extended(outputStream* st) const {
   PrintRegionClosure blk(st);
   _hrs->iterate(&blk);
 }
@@ -2408,10 +2455,6 @@
 }

 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
-  if (PrintHeapAtGC){
-    gclog_or_tty->print_cr(" {Heap before GC collections=%d:", total_collections());
-    Universe::print();
-  }
   assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
   // Call allocation profiler
   AllocationProfiler::iterate_since_last_gc();
@@ -2425,12 +2468,6 @@
   // is set.
   COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),
                         "derived pointer present"));
-
-  if (PrintHeapAtGC){
-    gclog_or_tty->print_cr(" Heap after GC collections=%d:", total_collections());
-    Universe::print();
-    gclog_or_tty->print("} ");
-  }
 }

 void G1CollectedHeap::do_collection_pause() {
@@ -2453,12 +2490,10 @@

 void
 G1CollectedHeap::doConcurrentMark() {
-  if (G1ConcMark) {
-    MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
-    if (!_cmThread->in_progress()) {
-      _cmThread->set_started();
-      CGC_lock->notify();
-    }
+  MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+  if (!_cmThread->in_progress()) {
+    _cmThread->set_started();
+    CGC_lock->notify();
   }
 }

@@ -2535,9 +2570,11 @@
                           "Not enough space for young surv words summary.");
   }
   memset(_surviving_young_words, 0, array_length * sizeof(size_t));
+#ifdef ASSERT
   for (size_t i = 0;  i < array_length; ++i) {
-    guarantee( _surviving_young_words[i] == 0, "invariant" );
-  }
+    assert( _surviving_young_words[i] == 0, "memset above" );
+  }
+#endif // !ASSERT
 }

 void
@@ -2559,241 +2596,254 @@

 void
 G1CollectedHeap::do_collection_pause_at_safepoint() {
-  char verbose_str[128];
-  sprintf(verbose_str, "GC pause ");
-  if (g1_policy()->in_young_gc_mode()) {
-    if (g1_policy()->full_young_gcs())
-      strcat(verbose_str, "(young)");
-    else
-      strcat(verbose_str, "(partial)");
-  }
-  if (g1_policy()->should_initiate_conc_mark())
-    strcat(verbose_str, " (initial-mark)");
-
-  GCCauseSetter x(this, GCCause::_g1_inc_collection_pause);
-
-  // if PrintGCDetails is on, we'll print long statistics information
-  // in the collector policy code, so let's not print this as the output
-  // is messy if we do.
-  gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
-  TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
-  TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);
-
-  ResourceMark rm;
-  assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
-  assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
-  guarantee(!is_gc_active(), "collection is not reentrant");
-  assert(regions_accounted_for(), "Region leakage!");
-
-  increment_gc_time_stamp();
-
-  if (g1_policy()->in_young_gc_mode()) {
-    assert(check_young_list_well_formed(),
-                "young list should be well formed");
-  }
-
-  if (GC_locker::is_active()) {
-    return; // GC is disabled (e.g. JNI GetXXXCritical operation)
-  }
-
-  bool abandoned = false;
-  { // Call to jvmpi::post_class_unload_events must occur outside of active GC
-    IsGCActiveMark x;
-
-    gc_prologue(false);
-    increment_total_collections();
+  if (PrintHeapAtGC) {
+    Universe::print_heap_before_gc();
+  }
+
+  {
+    char verbose_str[128];
+    sprintf(verbose_str, "GC pause ");
+    if (g1_policy()->in_young_gc_mode()) {
+      if (g1_policy()->full_young_gcs())
+        strcat(verbose_str, "(young)");
+      else
+        strcat(verbose_str, "(partial)");
+    }
+    if (g1_policy()->should_initiate_conc_mark())
+      strcat(verbose_str, " (initial-mark)");
+
+    GCCauseSetter x(this, GCCause::_g1_inc_collection_pause);
+
+    // if PrintGCDetails is on, we'll print long statistics information
+    // in the collector policy code, so let's not print this as the output
+    // is messy if we do.
+    gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
+    TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
+    TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);
+
+    ResourceMark rm;
+    assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+    assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
+    guarantee(!is_gc_active(), "collection is not reentrant");
+    assert(regions_accounted_for(), "Region leakage!");
+
+    increment_gc_time_stamp();
+
+    if (g1_policy()->in_young_gc_mode()) {
+      assert(check_young_list_well_formed(),
+             "young list should be well formed");
+    }
+
+    if (GC_locker::is_active()) {
+      return; // GC is disabled (e.g. JNI GetXXXCritical operation)
+    }
+
+    bool abandoned = false;
+    { // Call to jvmpi::post_class_unload_events must occur outside of active GC
+      IsGCActiveMark x;
+
+      gc_prologue(false);
+      increment_total_collections(false /* full gc */);

 #if G1_REM_SET_LOGGING
-    gclog_or_tty->print_cr("\nJust chose CS, heap:");
-    print();
-#endif
-
-    if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
-      HandleMark hm;  // Discard invalid handles created during verification
-      prepare_for_verify();
-      gclog_or_tty->print(" VerifyBeforeGC:");
-      Universe::verify(false);
-    }
-
-    COMPILER2_PRESENT(DerivedPointerTable::clear());
-
-    // We want to turn off ref discovery, if necessary, and turn it back on
-    // on again later if we do.
-    bool was_enabled = ref_processor()->discovery_enabled();
-    if (was_enabled) ref_processor()->disable_discovery();
-
-    // Forget the current alloc region (we might even choose it to be part
-    // of the collection set!).
-    abandon_cur_alloc_region();
-
-    // The elapsed time induced by the start time below deliberately elides
-    // the possible verification above.
-    double start_time_sec = os::elapsedTime();
-    GCOverheadReporter::recordSTWStart(start_time_sec);
-    size_t start_used_bytes = used();
-    if (!G1ConcMark) {
-      do_sync_mark();
-    }
-
-    g1_policy()->record_collection_pause_start(start_time_sec,
-                                               start_used_bytes);
-
-    guarantee(_in_cset_fast_test == NULL, "invariant");
-    guarantee(_in_cset_fast_test_base == NULL, "invariant");
-    _in_cset_fast_test_length = max_regions();
-    _in_cset_fast_test_base =
-                             NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
-    memset(_in_cset_fast_test_base, false,
-                                     _in_cset_fast_test_length * sizeof(bool));
-    // We're biasing _in_cset_fast_test to avoid subtracting the
-    // beginning of the heap every time we want to index; basically
-    // it's the same with what we do with the card table.
-    _in_cset_fast_test = _in_cset_fast_test_base -
-              ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
-
-#if SCAN_ONLY_VERBOSE
-    _young_list->print();
-#endif // SCAN_ONLY_VERBOSE
-
-    if (g1_policy()->should_initiate_conc_mark()) {
-      concurrent_mark()->checkpointRootsInitialPre();
-    }
-    save_marks();
-
-    // We must do this before any possible evacuation that should propagate
-    // marks.
-    if (mark_in_progress()) {
-      double start_time_sec = os::elapsedTime();
-
-      _cm->drainAllSATBBuffers();
-      double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
-      g1_policy()->record_satb_drain_time(finish_mark_ms);
-
-    }
-    // Record the number of elements currently on the mark stack, so we
-    // only iterate over these.  (Since evacuation may add to the mark
-    // stack, doing more exposes race conditions.)  If no mark is in
-    // progress, this will be zero.
-    _cm->set_oops_do_bound();
-
-    assert(regions_accounted_for(), "Region leakage.");
-
-    if (mark_in_progress())
-      concurrent_mark()->newCSet();
-
-    // Now choose the CS.
-    g1_policy()->choose_collection_set();
-
-    // We may abandon a pause if we find no region that will fit in the MMU
-    // pause.
-    bool abandoned = (g1_policy()->collection_set() == NULL);
-
-    // Nothing to do if we were unable to choose a collection set.
-    if (!abandoned) {
-#if G1_REM_SET_LOGGING
-      gclog_or_tty->print_cr("\nAfter pause, heap:");
+      gclog_or_tty->print_cr("\nJust chose CS, heap:");
       print();
 #endif

-      setup_surviving_young_words();
-
-      // Set up the gc allocation regions.
-      get_gc_alloc_regions();
-
-      // Actually do the work...
-      evacuate_collection_set();
-      free_collection_set(g1_policy()->collection_set());
-      g1_policy()->clear_collection_set();
-
-      FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
-      // this is more for peace of mind; we're nulling them here and
-      // we're expecting them to be null at the beginning of the next GC
-      _in_cset_fast_test = NULL;
-      _in_cset_fast_test_base = NULL;
-
-      release_gc_alloc_regions(false /* totally */);
-
-      cleanup_surviving_young_words();
-
-      if (g1_policy()->in_young_gc_mode()) {
-        _young_list->reset_sampled_info();
-        assert(check_young_list_empty(true),
-               "young list should be empty");
+      if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
+        HandleMark hm;  // Discard invalid handles created during verification
+        prepare_for_verify();
+        gclog_or_tty->print(" VerifyBeforeGC:");
+        Universe::verify(false);
+      }
+
+      COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+      // We want to turn off ref discovery, if necessary, and turn it back on
+      // on again later if we do. XXX Dubious: why is discovery disabled?
+      bool was_enabled = ref_processor()->discovery_enabled();
+      if (was_enabled) ref_processor()->disable_discovery();
+
+      // Forget the current alloc region (we might even choose it to be part
+      // of the collection set!).
+      abandon_cur_alloc_region();
+
+      // The elapsed time induced by the start time below deliberately elides
+      // the possible verification above.
+      double start_time_sec = os::elapsedTime();
+      GCOverheadReporter::recordSTWStart(start_time_sec);
+      size_t start_used_bytes = used();
+
+      g1_policy()->record_collection_pause_start(start_time_sec,
+                                                 start_used_bytes);
+
+      guarantee(_in_cset_fast_test == NULL, "invariant");
+      guarantee(_in_cset_fast_test_base == NULL, "invariant");
+      _in_cset_fast_test_length = max_regions();
+      _in_cset_fast_test_base =
+                             NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
+      memset(_in_cset_fast_test_base, false,
+                                     _in_cset_fast_test_length * sizeof(bool));
+      // We're biasing _in_cset_fast_test to avoid subtracting the
+      // beginning of the heap every time we want to index; basically
+      // it's the same with what we do with the card table.
+      _in_cset_fast_test = _in_cset_fast_test_base -
+              ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

 #if SCAN_ONLY_VERBOSE
-        _young_list->print();
+      _young_list->print();
 #endif // SCAN_ONLY_VERBOSE

-        g1_policy()->record_survivor_regions(_young_list->survivor_length(),
-                                             _young_list->first_survivor_region(),
-                                             _young_list->last_survivor_region());
-        _young_list->reset_auxilary_lists();
+      if (g1_policy()->should_initiate_conc_mark()) {
+        concurrent_mark()->checkpointRootsInitialPre();
       }
-    } else {
-      COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+      save_marks();
+
+      // We must do this before any possible evacuation that should propagate
+      // marks.
+      if (mark_in_progress()) {
+        double start_time_sec = os::elapsedTime();
+
+        _cm->drainAllSATBBuffers();
+        double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
+        g1_policy()->record_satb_drain_time(finish_mark_ms);
+      }
+      // Record the number of elements currently on the mark stack, so we
+      // only iterate over these.  (Since evacuation may add to the mark
+      // stack, doing more exposes race conditions.)  If no mark is in
+      // progress, this will be zero.
+      _cm->set_oops_do_bound();
+
+      assert(regions_accounted_for(), "Region leakage.");
+
+      if (mark_in_progress())
+        concurrent_mark()->newCSet();
+
+      // Now choose the CS.
+      g1_policy()->choose_collection_set();
+
+      // We may abandon a pause if we find no region that will fit in the MMU
+      // pause.
+      bool abandoned = (g1_policy()->collection_set() == NULL);
+
+      // Nothing to do if we were unable to choose a collection set.
+      if (!abandoned) {
+#if G1_REM_SET_LOGGING
+        gclog_or_tty->print_cr("\nAfter pause, heap:");
+        print();
+#endif
+
+        setup_surviving_young_words();
+
+        // Set up the gc allocation regions.
+        get_gc_alloc_regions();
+
+        // Actually do the work...
+        evacuate_collection_set();
+        free_collection_set(g1_policy()->collection_set());
+        g1_policy()->clear_collection_set();
+
+        FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
+        // this is more for peace of mind; we're nulling them here and
+        // we're expecting them to be null at the beginning of the next GC
+        _in_cset_fast_test = NULL;
+        _in_cset_fast_test_base = NULL;
+
+        release_gc_alloc_regions(false /* totally */);
+
+        cleanup_surviving_young_words();
+
+        if (g1_policy()->in_young_gc_mode()) {
+          _young_list->reset_sampled_info();
+          assert(check_young_list_empty(true),
+                 "young list should be empty");
+
+#if SCAN_ONLY_VERBOSE
+          _young_list->print();
+#endif // SCAN_ONLY_VERBOSE
+
+          g1_policy()->record_survivor_regions(_young_list->survivor_length(),
+                                          _young_list->first_survivor_region(),
+                                          _young_list->last_survivor_region());
+          _young_list->reset_auxilary_lists();
+        }
+      } else {
+        COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+      }
+
+      if (evacuation_failed()) {
+        _summary_bytes_used = recalculate_used();
+      } else {
+        // The "used" of the the collection set have already been subtracted
+        // when they were freed.  Add in the bytes evacuated.
+        _summary_bytes_used += g1_policy()->bytes_in_to_space();
+      }
+
+      if (g1_policy()->in_young_gc_mode() &&
+          g1_policy()->should_initiate_conc_mark()) {
+        concurrent_mark()->checkpointRootsInitialPost();
+        set_marking_started();
+        // CAUTION: after the doConcurrentMark() call below,
+        // the concurrent marking thread(s) could be running
+        // concurrently with us. Make sure that anything after
+        // this point does not assume that we are the only GC thread
+        // running. Note: of course, the actual marking work will
+        // not start until the safepoint itself is released in
+        // ConcurrentGCThread::safepoint_desynchronize().
+        doConcurrentMark();
+      }
+
+#if SCAN_ONLY_VERBOSE
+      _young_list->print();
+#endif // SCAN_ONLY_VERBOSE
+
+      double end_time_sec = os::elapsedTime();
+      double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
+      g1_policy()->record_pause_time_ms(pause_time_ms);
+      GCOverheadReporter::recordSTWEnd(end_time_sec);
+      g1_policy()->record_collection_pause_end(abandoned);
+
+      assert(regions_accounted_for(), "Region leakage.");
+
+      if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
+        HandleMark hm;  // Discard invalid handles created during verification
+        gclog_or_tty->print(" VerifyAfterGC:");
+        prepare_for_verify();
+        Universe::verify(false);
+      }
+
+      if (was_enabled) ref_processor()->enable_discovery();
+
+      {
+        size_t expand_bytes = g1_policy()->expansion_amount();
+        if (expand_bytes > 0) {
+          size_t bytes_before = capacity();
+          expand(expand_bytes);
+        }
+      }
+
+      if (mark_in_progress()) {
+        concurrent_mark()->update_g1_committed();
+      }
+
+#ifdef TRACESPINNING
+      ParallelTaskTerminator::print_termination_counts();
+#endif
+
+      gc_epilogue(false);
     }

-    if (evacuation_failed()) {
-      _summary_bytes_used = recalculate_used();
-    } else {
-      // The "used" of the the collection set have already been subtracted
-      // when they were freed.  Add in the bytes evacuated.
-      _summary_bytes_used += g1_policy()->bytes_in_to_space();
+    assert(verify_region_lists(), "Bad region lists.");
+
+    if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
+      gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
+      print_tracing_info();
+      vm_exit(-1);
     }
-
-    if (g1_policy()->in_young_gc_mode() &&
-        g1_policy()->should_initiate_conc_mark()) {
-      concurrent_mark()->checkpointRootsInitialPost();
-      set_marking_started();
-      doConcurrentMark();
-    }
-
-#if SCAN_ONLY_VERBOSE
-    _young_list->print();
-#endif // SCAN_ONLY_VERBOSE
-
-    double end_time_sec = os::elapsedTime();
-    double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
-    g1_policy()->record_pause_time_ms(pause_time_ms);
-    GCOverheadReporter::recordSTWEnd(end_time_sec);
-    g1_policy()->record_collection_pause_end(abandoned);
-
-    assert(regions_accounted_for(), "Region leakage.");
-
-    if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
-      HandleMark hm;  // Discard invalid handles created during verification
-      gclog_or_tty->print(" VerifyAfterGC:");
-      prepare_for_verify();
-      Universe::verify(false);
-    }
-
-    if (was_enabled) ref_processor()->enable_discovery();
-
-    {
-      size_t expand_bytes = g1_policy()->expansion_amount();
-      if (expand_bytes > 0) {
-        size_t bytes_before = capacity();
-        expand(expand_bytes);
-      }
-    }
-
-    if (mark_in_progress()) {
-      concurrent_mark()->update_g1_committed();
-    }
-
-#ifdef TRACESPINNING
-    ParallelTaskTerminator::print_termination_counts();
-#endif
-
-    gc_epilogue(false);
-  }
-
-  assert(verify_region_lists(), "Bad region lists.");
-
-  if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
-    gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
-    print_tracing_info();
-    vm_exit(-1);
+  }
+
+  if (PrintHeapAtGC) {
+    Universe::print_heap_after_gc();
   }
 }

@@ -2930,6 +2980,7 @@

   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
     assert(_gc_alloc_regions[ap] == NULL, "invariant");
+    assert(_gc_alloc_region_counts[ap] == 0, "invariant");

     // Create new GC alloc regions.
     HeapRegion* alloc_region = _retained_gc_alloc_regions[ap];
@@ -2958,6 +3009,9 @@
     if (alloc_region == NULL) {
       // we will get a new GC alloc region
       alloc_region = newAllocRegionWithExpansion(ap, 0);
+    } else {
+      // the region was retained from the last collection
+      ++_gc_alloc_region_counts[ap];
     }

     if (alloc_region != NULL) {
@@ -2996,11 +3050,11 @@
   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
     HeapRegion* r = _gc_alloc_regions[ap];
     _retained_gc_alloc_regions[ap] = NULL;
+    _gc_alloc_region_counts[ap] = 0;

     if (r != NULL) {
       // we retain nothing on _gc_alloc_regions between GCs
       set_gc_alloc_region(ap, NULL);
-      _gc_alloc_region_counts[ap] = 0;

       if (r->is_empty()) {
         // we didn't actually allocate anything in it; let's just put
@@ -3088,10 +3142,8 @@
   G1CollectedHeap* _g1;
 public:
   G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
+  void do_oop(      oop* p) {
     oop obj = *p;
 #ifdef G1_DEBUG
     if (PrintGC && Verbose) {
@@ -3103,7 +3155,6 @@
     if (_g1->obj_in_cs(obj)) {
       assert( obj->is_forwarded(), "invariant" );
       *p = obj->forwardee();
-
 #ifdef G1_DEBUG
       gclog_or_tty->print_cr("     in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
                              (void*) obj, (void*) *p);
@@ -3120,12 +3171,12 @@
   UpdateRSetImmediate(G1CollectedHeap* g1) :
     _g1(g1), _g1_rem_set(g1->g1_rem_set()) {}

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
     assert(_from->is_in_reserved(p), "paranoia");
-    if (*p != NULL && !_from->is_survivor()) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) {
       _g1_rem_set->par_write_ref(_from, p, 0);
     }
   }
@@ -3141,12 +3192,12 @@
   UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
     _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}

-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(      oop* p) { do_oop_work(p); }
+  template <class T> void do_oop_work(T* p) {
     assert(_from->is_in_reserved(p), "paranoia");
-    if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
+    if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
+        !_from->is_survivor()) {
       size_t card_index = _ct_bs->index_for(p);
       if (_ct_bs->mark_card_deferred(card_index)) {
         _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
@@ -3501,614 +3552,66 @@
   fill_with_object(block, free_words);
 }

-#define use_local_bitmaps         1
-#define verify_local_bitmaps      0
-
 #ifndef PRODUCT
-
-class GCLabBitMap;
-class GCLabBitMapClosure: public BitMapClosure {
-private:
-  ConcurrentMark* _cm;
-  GCLabBitMap*    _bitmap;
-
-public:
-  GCLabBitMapClosure(ConcurrentMark* cm,
-                     GCLabBitMap* bitmap) {
-    _cm     = cm;
-    _bitmap = bitmap;
-  }
-
-  virtual bool do_bit(size_t offset);
-};
-
-#endif // PRODUCT
-
-#define oop_buffer_length 256
-
-class GCLabBitMap: public BitMap {
-private:
-  ConcurrentMark* _cm;
-
-  int       _shifter;
-  size_t    _bitmap_word_covers_words;
-
-  // beginning of the heap
-  HeapWord* _heap_start;
-
-  // this is the actual start of the GCLab
-  HeapWord* _real_start_word;
-
-  // this is the actual end of the GCLab
-  HeapWord* _real_end_word;
-
-  // this is the first word, possibly located before the actual start
-  // of the GCLab, that corresponds to the first bit of the bitmap
-  HeapWord* _start_word;
-
-  // size of a GCLab in words
-  size_t _gclab_word_size;
-
-  static int shifter() {
-    return MinObjAlignment - 1;
-  }
-
-  // how many heap words does a single bitmap word corresponds to?
-  static size_t bitmap_word_covers_words() {
-    return BitsPerWord << shifter();
-  }
-
-  static size_t gclab_word_size() {
-    return G1ParallelGCAllocBufferSize / HeapWordSize;
-  }
-
-  static size_t bitmap_size_in_bits() {
-    size_t bits_in_bitmap = gclab_word_size() >> shifter();
-    // We are going to ensure that the beginning of a word in this
-    // bitmap also corresponds to the beginning of a word in the
-    // global marking bitmap. To handle the case where a GCLab
-    // starts from the middle of the bitmap, we need to add enough
-    // space (i.e. up to a bitmap word) to ensure that we have
-    // enough bits in the bitmap.
-    return bits_in_bitmap + BitsPerWord - 1;
-  }
-public:
-  GCLabBitMap(HeapWord* heap_start)
-    : BitMap(bitmap_size_in_bits()),
-      _cm(G1CollectedHeap::heap()->concurrent_mark()),
-      _shifter(shifter()),
-      _bitmap_word_covers_words(bitmap_word_covers_words()),
-      _heap_start(heap_start),
-      _gclab_word_size(gclab_word_size()),
-      _real_start_word(NULL),
-      _real_end_word(NULL),
-      _start_word(NULL)
-  {
-    guarantee( size_in_words() >= bitmap_size_in_words(),
-               "just making sure");
-  }
-
-  inline unsigned heapWordToOffset(HeapWord* addr) {
-    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
-    assert(offset < size(), "offset should be within bounds");
-    return offset;
-  }
-
-  inline HeapWord* offsetToHeapWord(size_t offset) {
-    HeapWord* addr =  _start_word + (offset << _shifter);
-    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
-    return addr;
-  }
-
-  bool fields_well_formed() {
-    bool ret1 = (_real_start_word == NULL) &&
-                (_real_end_word == NULL) &&
-                (_start_word == NULL);
-    if (ret1)
-      return true;
-
-    bool ret2 = _real_start_word >= _start_word &&
-      _start_word < _real_end_word &&
-      (_real_start_word + _gclab_word_size) == _real_end_word &&
-      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
-                                                              > _real_end_word;
-    return ret2;
-  }
-
-  inline bool mark(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    if (addr >= _real_start_word && addr < _real_end_word) {
-      assert(!isMarked(addr), "should not have already been marked");
-
-      // first mark it on the bitmap
-      at_put(heapWordToOffset(addr), true);
-
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  inline bool isMarked(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    return at(heapWordToOffset(addr));
-  }
-
-  void set_buffer(HeapWord* start) {
-    guarantee(use_local_bitmaps, "invariant");
-    clear();
-
-    assert(start != NULL, "invariant");
-    _real_start_word = start;
-    _real_end_word   = start + _gclab_word_size;
-
-    size_t diff =
-      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
-    _start_word = start - diff;
-
-    assert(fields_well_formed(), "invariant");
-  }
-
-#ifndef PRODUCT
-  void verify() {
-    // verify that the marks have been propagated
-    GCLabBitMapClosure cl(_cm, this);
-    iterate(&cl);
-  }
-#endif // PRODUCT
-
-  void retire() {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    if (_start_word != NULL) {
-      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
-
-      // this means that the bitmap was set up for the GCLab
-      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
-
-      mark_bitmap->mostly_disjoint_range_union(this,
-                                0, // always start from the start of the bitmap
-                                _start_word,
-                                size_in_words());
-      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
-
-#ifndef PRODUCT
-      if (use_local_bitmaps && verify_local_bitmaps)
-        verify();
-#endif // PRODUCT
-    } else {
-      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
-    }
-  }
-
-  static size_t bitmap_size_in_words() {
-    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
-  }
-};
-
-#ifndef PRODUCT
-
 bool GCLabBitMapClosure::do_bit(size_t offset) {
   HeapWord* addr = _bitmap->offsetToHeapWord(offset);
   guarantee(_cm->isMarked(oop(addr)), "it should be!");
   return true;
 }
-
 #endif // PRODUCT

-class G1ParGCAllocBuffer: public ParGCAllocBuffer {
-private:
-  bool        _retired;
-  bool        _during_marking;
-  GCLabBitMap _bitmap;
-
-public:
-  G1ParGCAllocBuffer() :
-    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
-    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
-    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
-    _retired(false)
-  { }
-
-  inline bool mark(HeapWord* addr) {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(_during_marking, "invariant");
-    return _bitmap.mark(addr);
-  }
-
-  inline void set_buf(HeapWord* buf) {
-    if (use_local_bitmaps && _during_marking)
-      _bitmap.set_buffer(buf);
-    ParGCAllocBuffer::set_buf(buf);
-    _retired = false;
-  }
-
-  inline void retire(bool end_of_gc, bool retain) {
-    if (_retired)
-      return;
-    if (use_local_bitmaps && _during_marking) {
-      _bitmap.retire();
-    }
-    ParGCAllocBuffer::retire(end_of_gc, retain);
-    _retired = true;
-  }
-};
-
-
-class G1ParScanThreadState : public StackObj {
-protected:
-  G1CollectedHeap* _g1h;
-  RefToScanQueue*  _refs;
-  DirtyCardQueue   _dcq;
-  CardTableModRefBS* _ct_bs;
-  G1RemSet* _g1_rem;
-
-  typedef GrowableArray<oop*> OverflowQueue;
-  OverflowQueue* _overflowed_refs;
-
-  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
-  ageTable           _age_table;
-
-  size_t           _alloc_buffer_waste;
-  size_t           _undo_waste;
-
-  OopsInHeapRegionClosure*      _evac_failure_cl;
-  G1ParScanHeapEvacClosure*     _evac_cl;
-  G1ParScanPartialArrayClosure* _partial_scan_cl;
-
-  int _hash_seed;
-  int _queue_num;
-
-  int _term_attempts;
+G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
+  : _g1h(g1h),
+    _refs(g1h->task_queue(queue_num)),
+    _dcq(&g1h->dirty_card_queue_set()),
+    _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
+    _g1_rem(g1h->g1_rem_set()),
+    _hash_seed(17), _queue_num(queue_num),
+    _term_attempts(0),
+    _age_table(false),
 #if G1_DETAILED_STATS
-  int _pushes, _pops, _steals, _steal_attempts;
-  int _overflow_pushes;
+    _pushes(0), _pops(0), _steals(0),
+    _steal_attempts(0),  _overflow_pushes(0),
 #endif
-
-  double _start;
-  double _start_strong_roots;
-  double _strong_roots_time;
-  double _start_term;
-  double _term_time;
-
-  // Map from young-age-index (0 == not young, 1 is youngest) to
-  // surviving words. base is what we get back from the malloc call
-  size_t* _surviving_young_words_base;
-  // this points into the array, as we use the first few entries for padding
-  size_t* _surviving_young_words;
-
-#define PADDING_ELEM_NUM (64 / sizeof(size_t))
-
-  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
-
-  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
-
-  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
-  CardTableModRefBS* ctbs()                      { return _ct_bs; }
-
-  void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
-    if (!from->is_survivor()) {
-      _g1_rem->par_write_ref(from, p, tid);
-    }
-  }
-
-  void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
-    // If the new value of the field points to the same region or
-    // is the to-space, we don't need to include it in the Rset updates.
-    if (!from->is_in_reserved(*p) && !from->is_survivor()) {
-      size_t card_index = ctbs()->index_for(p);
-      // If the card hasn't been added to the buffer, do it.
-      if (ctbs()->mark_card_deferred(card_index)) {
-        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
-      }
-    }
-  }
-
-public:
-  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
-    : _g1h(g1h),
-      _refs(g1h->task_queue(queue_num)),
-      _dcq(&g1h->dirty_card_queue_set()),
-      _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
-      _g1_rem(g1h->g1_rem_set()),
-      _hash_seed(17), _queue_num(queue_num),
-      _term_attempts(0),
-      _age_table(false),
-#if G1_DETAILED_STATS
-      _pushes(0), _pops(0), _steals(0),
-      _steal_attempts(0),  _overflow_pushes(0),
-#endif
-      _strong_roots_time(0), _term_time(0),
-      _alloc_buffer_waste(0), _undo_waste(0)
-  {
-    // we allocate G1YoungSurvRateNumRegions plus one entries, since
-    // we "sacrifice" entry 0 to keep track of surviving bytes for
-    // non-young regions (where the age is -1)
-    // We also add a few elements at the beginning and at the end in
-    // an attempt to eliminate cache contention
-    size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
-    size_t array_length = PADDING_ELEM_NUM +
-                          real_length +
-                          PADDING_ELEM_NUM;
-    _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
-    if (_surviving_young_words_base == NULL)
-      vm_exit_out_of_memory(array_length * sizeof(size_t),
-                            "Not enough space for young surv histo.");
-    _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
-    memset(_surviving_young_words, 0, real_length * sizeof(size_t));
-
-    _overflowed_refs = new OverflowQueue(10);
-
-    _start = os::elapsedTime();
-  }
-
-  ~G1ParScanThreadState() {
-    FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
-  }
-
-  RefToScanQueue*   refs()            { return _refs;             }
-  OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
-  ageTable*         age_table()       { return &_age_table;       }
-
-  G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
-    return &_alloc_buffers[purpose];
-  }
-
-  size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
-  size_t undo_waste()                            { return _undo_waste; }
-
-  void push_on_queue(oop* ref) {
-    assert(ref != NULL, "invariant");
-    assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref), "invariant");
-
-    if (!refs()->push(ref)) {
-      overflowed_refs()->push(ref);
-      IF_G1_DETAILED_STATS(note_overflow_push());
-    } else {
-      IF_G1_DETAILED_STATS(note_push());
-    }
-  }
-
-  void pop_from_queue(oop*& ref) {
-    if (!refs()->pop_local(ref)) {
-      ref = NULL;
-    } else {
-      assert(ref != NULL, "invariant");
-      assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref),
-             "invariant");
-
-      IF_G1_DETAILED_STATS(note_pop());
-    }
-  }
-
-  void pop_from_overflow_queue(oop*& ref) {
-    ref = overflowed_refs()->pop();
-  }
-
-  int refs_to_scan()                             { return refs()->size();                 }
-  int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
-
-  void update_rs(HeapRegion* from, oop* p, int tid) {
-    if (G1DeferredRSUpdate) {
-      deferred_rs_update(from, p, tid);
-    } else {
-      immediate_rs_update(from, p, tid);
-    }
-  }
-
-  HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
-
-    HeapWord* obj = NULL;
-    if (word_sz * 100 <
-        (size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
-                                                  ParallelGCBufferWastePct) {
-      G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
-      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
-      alloc_buf->retire(false, false);
-
-      HeapWord* buf =
-        _g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
-      if (buf == NULL) return NULL; // Let caller handle allocation failure.
-      // Otherwise.
-      alloc_buf->set_buf(buf);
-
-      obj = alloc_buf->allocate(word_sz);
-      assert(obj != NULL, "buffer was definitely big enough...");
-    } else {
-      obj = _g1h->par_allocate_during_gc(purpose, word_sz);
-    }
-    return obj;
-  }
-
-  HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
-    HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
-    if (obj != NULL) return obj;
-    return allocate_slow(purpose, word_sz);
-  }
-
-  void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
-    if (alloc_buffer(purpose)->contains(obj)) {
-      guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1),
-                "should contain whole object");
-      alloc_buffer(purpose)->undo_allocation(obj, word_sz);
-    } else {
-      CollectedHeap::fill_with_object(obj, word_sz);
-      add_to_undo_waste(word_sz);
-    }
-  }
-
-  void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
-    _evac_failure_cl = evac_failure_cl;
-  }
-  OopsInHeapRegionClosure* evac_failure_closure() {
-    return _evac_failure_cl;
-  }
-
-  void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
-    _evac_cl = evac_cl;
-  }
-
-  void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
-    _partial_scan_cl = partial_scan_cl;
-  }
-
-  int* hash_seed() { return &_hash_seed; }
-  int  queue_num() { return _queue_num; }
-
-  int term_attempts()   { return _term_attempts; }
-  void note_term_attempt()  { _term_attempts++; }
-
-#if G1_DETAILED_STATS
-  int pushes()          { return _pushes; }
-  int pops()            { return _pops; }
-  int steals()          { return _steals; }
-  int steal_attempts()  { return _steal_attempts; }
-  int overflow_pushes() { return _overflow_pushes; }
-
-  void note_push()          { _pushes++; }
-  void note_pop()           { _pops++; }
-  void note_steal()         { _steals++; }
-  void note_steal_attempt() { _steal_attempts++; }
-  void note_overflow_push() { _overflow_pushes++; }
-#endif
-
-  void start_strong_roots() {
-    _start_strong_roots = os::elapsedTime();
-  }
-  void end_strong_roots() {
-    _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
-  }
-  double strong_roots_time() { return _strong_roots_time; }
-
-  void start_term_time() {
-    note_term_attempt();
-    _start_term = os::elapsedTime();
-  }
-  void end_term_time() {
-    _term_time += (os::elapsedTime() - _start_term);
-  }
-  double term_time() { return _term_time; }
-
-  double elapsed() {
-    return os::elapsedTime() - _start;
-  }
-
-  size_t* surviving_young_words() {
-    // We add on to hide entry 0 which accumulates surviving words for
-    // age -1 regions (i.e. non-young ones)
-    return _surviving_young_words;
-  }
-
-  void retire_alloc_buffers() {
-    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
-      size_t waste = _alloc_buffers[ap].words_remaining();
-      add_to_alloc_buffer_waste(waste);
-      _alloc_buffers[ap].retire(true, false);
-    }
-  }
-
-private:
-  void deal_with_reference(oop* ref_to_scan) {
-    if (has_partial_array_mask(ref_to_scan)) {
-      _partial_scan_cl->do_oop_nv(ref_to_scan);
-    } else {
-      // Note: we can use "raw" versions of "region_containing" because
-      // "obj_to_scan" is definitely in the heap, and is not in a
-      // humongous region.
-      HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
-      _evac_cl->set_region(r);
-      _evac_cl->do_oop_nv(ref_to_scan);
-    }
-  }
-
-public:
-  void trim_queue() {
-    // I've replicated the loop twice, first to drain the overflow
-    // queue, second to drain the task queue. This is better than
-    // having a single loop, which checks both conditions and, inside
-    // it, either pops the overflow queue or the task queue, as each
-    // loop is tighter. Also, the decision to drain the overflow queue
-    // first is not arbitrary, as the overflow queue is not visible
-    // to the other workers, whereas the task queue is. So, we want to
-    // drain the "invisible" entries first, while allowing the other
-    // workers to potentially steal the "visible" entries.
-
-    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
-      while (overflowed_refs_to_scan() > 0) {
-        oop *ref_to_scan = NULL;
-        pop_from_overflow_queue(ref_to_scan);
-        assert(ref_to_scan != NULL, "invariant");
-        // We shouldn't have pushed it on the queue if it was not
-        // pointing into the CSet.
-        assert(ref_to_scan != NULL, "sanity");
-        assert(has_partial_array_mask(ref_to_scan) ||
-                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
-
-        deal_with_reference(ref_to_scan);
-      }
-
-      while (refs_to_scan() > 0) {
-        oop *ref_to_scan = NULL;
-        pop_from_queue(ref_to_scan);
-
-        if (ref_to_scan != NULL) {
-          // We shouldn't have pushed it on the queue if it was not
-          // pointing into the CSet.
-          assert(has_partial_array_mask(ref_to_scan) ||
-                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
-
-          deal_with_reference(ref_to_scan);
-        }
-      }
-    }
-  }
-};
+    _strong_roots_time(0), _term_time(0),
+    _alloc_buffer_waste(0), _undo_waste(0)
+{
+  // we allocate G1YoungSurvRateNumRegions plus one entries, since
+  // we "sacrifice" entry 0 to keep track of surviving bytes for
+  // non-young regions (where the age is -1)
+  // We also add a few elements at the beginning and at the end in
+  // an attempt to eliminate cache contention
+  size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
+  size_t array_length = PADDING_ELEM_NUM +
+                        real_length +
+                        PADDING_ELEM_NUM;
+  _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
+  if (_surviving_young_words_base == NULL)
+    vm_exit_out_of_memory(array_length * sizeof(size_t),
+                          "Not enough space for young surv histo.");
+  _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
+  memset(_surviving_young_words, 0, real_length * sizeof(size_t));
+
+  _overflowed_refs = new OverflowQueue(10);
+
+  _start = os::elapsedTime();
+}

 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   _par_scan_state(par_scan_state) { }

-// This closure is applied to the fields of the objects that have just been copied.
-// Should probably be made inline and moved in g1OopClosures.inline.hpp.
-void G1ParScanClosure::do_oop_nv(oop* p) {
-  oop obj = *p;
-
-  if (obj != NULL) {
-    if (_g1->in_cset_fast_test(obj)) {
-      // We're not going to even bother checking whether the object is
-      // already forwarded or not, as this usually causes an immediate
-      // stall. We'll try to prefetch the object (for write, given that
-      // we might need to install the forwarding reference) and we'll
-      // get back to it when pop it from the queue
-      Prefetch::write(obj->mark_addr(), 0);
-      Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
-
-      // slightly paranoid test; I'm trying to catch potential
-      // problems before we go into push_on_queue to know where the
-      // problem is coming from
-      assert(obj == *p, "the value of *p should not have changed");
-      _par_scan_state->push_on_queue(p);
-    } else {
-      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
-    }
-  }
-}
-
-void G1ParCopyHelper::mark_forwardee(oop* p) {
+template <class T> void G1ParCopyHelper::mark_forwardee(T* p) {
   // This is called _after_ do_oop_work has been called, hence after
   // the object has been relocated to its new location and *p points
   // to its new location.

-  oop thisOop = *p;
-  if (thisOop != NULL) {
-    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)),
+  T heap_oop = oopDesc::load_heap_oop(p);
+  if (!oopDesc::is_null(heap_oop)) {
+    oop obj = oopDesc::decode_heap_oop(heap_oop);
+    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(obj)),
            "shouldn't still be in the CSet if evacuation didn't fail.");
-    HeapWord* addr = (HeapWord*)thisOop;
+    HeapWord* addr = (HeapWord*)obj;
     if (_g1->is_in_g1_reserved(addr))
       _cm->grayRoot(oop(addr));
   }
@@ -4191,7 +3694,8 @@

     if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
       arrayOop(old)->set_length(0);
-      _par_scan_state->push_on_queue(set_partial_array_mask(old));
+      oop* old_p = set_partial_array_mask(old);
+      _par_scan_state->push_on_queue(old_p);
     } else {
       // No point in using the slower heap_region_containing() method,
       // given that we know obj is in the heap.
@@ -4205,11 +3709,11 @@
   return obj;
 }

-template<bool do_gen_barrier, G1Barrier barrier,
-         bool do_mark_forwardee, bool skip_cset_test>
-void G1ParCopyClosure<do_gen_barrier, barrier,
-                      do_mark_forwardee, skip_cset_test>::do_oop_work(oop* p) {
-  oop obj = *p;
+template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee, bool skip_cset_test>
+template <class T>
+void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_test>
+::do_oop_work(T* p) {
+  oop obj = oopDesc::load_decode_heap_oop(p);
   assert(barrier != G1BarrierRS || obj != NULL,
          "Precondition: G1BarrierRS implies obj is nonNull");

@@ -4226,9 +3730,10 @@
                            "into CS.", p, (void*) obj);
 #endif
     if (obj->is_forwarded()) {
-      *p = obj->forwardee();
+      oopDesc::encode_store_heap_oop(p, obj->forwardee());
     } else {
-      *p = copy_to_survivor_space(obj);
+      oop copy_oop = copy_to_survivor_space(obj);
+      oopDesc::encode_store_heap_oop(p, copy_oop);
     }
     // When scanning the RS, we only care about objs in CS.
     if (barrier == G1BarrierRS) {
@@ -4247,21 +3752,9 @@
 }

 template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p);
-
-template<class T> void G1ParScanPartialArrayClosure::process_array_chunk(
-  oop obj, int start, int end) {
-  // process our set of indices (include header in first chunk)
-  assert(start < end, "invariant");
-  T* const base      = (T*)objArrayOop(obj)->base();
-  T* const start_addr = (start == 0) ? (T*) obj : base + start;
-  T* const end_addr   = base + end;
-  MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
-  _scanner.set_region(_g1->heap_region_containing(obj));
-  obj->oop_iterate(&_scanner, mr);
-}
-
-void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
-  assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
+template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(narrowOop* p);
+
+template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
   assert(has_partial_array_mask(p), "invariant");
   oop old = clear_partial_array_mask(p);
   assert(old->is_objArray(), "must be obj array");
@@ -4281,19 +3774,19 @@
     end = start + ParGCArrayScanChunk;
     arrayOop(old)->set_length(end);
     // Push remainder.
-    _par_scan_state->push_on_queue(set_partial_array_mask(old));
+    oop* old_p = set_partial_array_mask(old);
+    assert(arrayOop(old)->length() < obj->length(), "Empty push?");
+    _par_scan_state->push_on_queue(old_p);
   } else {
     // Restore length so that the heap remains parsable in
     // case of evacuation failure.
     arrayOop(old)->set_length(end);
   }
-
+  _scanner.set_region(_g1->heap_region_containing_raw(obj));
   // process our set of indices (include header in first chunk)
-  process_array_chunk<oop>(obj, start, end);
+  obj->oop_iterate_range(&_scanner, start, end);
 }

-int G1ScanAndBalanceClosure::_nq = 0;
-
 class G1ParEvacuateFollowersClosure : public VoidClosure {
 protected:
   G1CollectedHeap*              _g1h;
@@ -4316,21 +3809,28 @@
   void do_void() {
     G1ParScanThreadState* pss = par_scan_state();
     while (true) {
-      oop* ref_to_scan;
       pss->trim_queue();
       IF_G1_DETAILED_STATS(pss->note_steal_attempt());
-      if (queues()->steal(pss->queue_num(),
-                          pss->hash_seed(),
-                          ref_to_scan)) {
+
+      StarTask stolen_task;
+      if (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
         IF_G1_DETAILED_STATS(pss->note_steal());

         // slightly paranoid tests; I'm trying to catch potential
         // problems before we go into push_on_queue to know where the
         // problem is coming from
-        assert(ref_to_scan != NULL, "invariant");
-        assert(has_partial_array_mask(ref_to_scan) ||
-                                   _g1h->obj_in_cs(*ref_to_scan), "invariant");
-        pss->push_on_queue(ref_to_scan);
+        assert((oop*)stolen_task != NULL, "Error");
+        if (stolen_task.is_narrow()) {
+          assert(UseCompressedOops, "Error");
+          narrowOop* p = (narrowOop*) stolen_task;
+          assert(has_partial_array_mask(p) ||
+                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error");
+          pss->push_on_queue(p);
+        } else {
+          oop* p = (oop*) stolen_task;
+          assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error");
+          pss->push_on_queue(p);
+        }
         continue;
       }
       pss->start_term_time();
@@ -4347,6 +3847,7 @@
   G1CollectedHeap*       _g1h;
   RefToScanQueueSet      *_queues;
   ParallelTaskTerminator _terminator;
+  int _n_workers;

   Mutex _stats_lock;
   Mutex* stats_lock() { return &_stats_lock; }
@@ -4362,7 +3863,8 @@
       _g1h(g1h),
       _queues(task_queues),
       _terminator(workers, _queues),
-      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true)
+      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true),
+      _n_workers(workers)
   {}

   RefToScanQueueSet* queues() { return _queues; }
@@ -4372,6 +3874,7 @@
   }

   void work(int i) {
+    if (i >= _n_workers) return;  // no work needed this round
     ResourceMark rm;
     HandleMark   hm;

@@ -4469,23 +3972,6 @@

 // *** Common G1 Evacuation Stuff

-class G1CountClosure: public OopsInHeapRegionClosure {
-public:
-  int n;
-  G1CountClosure() : n(0) {}
-  void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  void do_oop(oop* p) {
-    oop obj = *p;
-    assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj),
-           "Rem set closure called on non-rem-set pointer.");
-    n++;
-  }
-};
-
-static G1CountClosure count_closure;
-
 void
 G1CollectedHeap::
 g1_process_strong_roots(bool collecting_perm_gen,
@@ -5357,7 +4843,7 @@
   assert(_free_region_list == NULL, "Postcondition of loop.");
   if (_free_region_list_size != 0) {
     gclog_or_tty->print_cr("Size is %d.", _free_region_list_size);
-    print();
+    print_on(gclog_or_tty, true /* extended */);
   }
   assert(_free_region_list_size == 0, "Postconditions of loop.");
 }
@@ -5535,8 +5021,3 @@
 void G1CollectedHeap::g1_unimplemented() {
   // Unimplemented();
 }
-
-
-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Jul 27 09:06:22 2009 -0700
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Jul 27 17:23:52 2009 -0400
@@ -56,8 +56,8 @@
 #  define IF_G1_DETAILED_STATS(code)
 #endif

-typedef GenericTaskQueue<oop*>    RefToScanQueue;
-typedef GenericTaskQueueSet<oop*> RefToScanQueueSet;
+typedef GenericTaskQueue<StarTask>    RefToScanQueue;
+typedef GenericTaskQueueSet<StarTask> RefToScanQueueSet;

 typedef int RegionIdx_t;   // needs to hold [ 0..max_regions() )
 typedef int CardIdx_t;     // needs to hold [ 0..CardsPerRegion )
@@ -700,6 +700,9 @@
   size_t g1_reserved_obj_bytes() { return _g1_reserved.byte_size(); }
   virtual size_t capacity() const;
   virtual size_t used() const;
+  // This should be called when we're not holding the heap lock. The
+  // result might be a bit inaccurate.
+  size_t used_unlocked() const;
   size_t recalculate_used() const;
 #ifndef PRODUCT
   size_t recalculate_used_regions() const;
@@ -1061,8 +1064,14 @@

   // Override; it uses the "prev" marking information
   virtual void verify(bool allow_dirty, bool silent);
+  // Default behavior by calling print(tty);
   virtual void print() const;
+  // This calls print_on(st, PrintHeapAtGCExtended).
   virtual void print_on(outputStream* st) const;
+  // If extended is true, it will print out information for all
+  // regions in the heap by calling print_on_extended(st).
+  virtual void print_on(outputStream* st, bool extended) const;
+  virtual void print_on_extended(outputStream* st) const;

   virtual void print_gc_threads_on(outputStream* st) const;
   virtual void gc_threads_do(ThreadClosure* tc) const;
@@ -1265,6 +1274,552 @@

 };

-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***
+#define use_local_bitmaps         1
+#define verify_local_bitmaps      0
+#define oop_buffer_length       256
+
+#ifndef PRODUCT
+class GCLabBitMap;
+class GCLabBitMapClosure: public BitMapClosure {
+private:
+  ConcurrentMark* _cm;
+  GCLabBitMap*    _bitmap;
+
+public:
+  GCLabBitMapClosure(ConcurrentMark* cm,
+                     GCLabBitMap* bitmap) {
+    _cm     = cm;
+    _bitmap = bitmap;
+  }
+
+  virtual bool do_bit(size_t offset);
+};
+#endif // !PRODUCT
+
+class GCLabBitMap: public BitMap {
+private:
+  ConcurrentMark* _cm;
+
+  int       _shifter;
+  size_t    _bitmap_word_covers_words;
+
+  // beginning of the heap
+  HeapWord* _heap_start;
+
+  // this is the actual start of the GCLab
+  HeapWord* _real_start_word;
+
+  // this is the actual end of the GCLab
+  HeapWord* _real_end_word;
+
+  // this is the first word, possibly located before the actual start
+  // of the GCLab, that corresponds to the first bit of the bitmap
+  HeapWord* _start_word;
+
+  // size of a GCLab in words
+  size_t _gclab_word_size;
+
+  static int shifter() {
+    return MinObjAlignment - 1;
+  }
+
+  // how many heap words does a single bitmap word corresponds to?
+  static size_t bitmap_word_covers_words() {
+    return BitsPerWord << shifter();
+  }
+
+  static size_t gclab_word_size() {
+    return G1ParallelGCAllocBufferSize / HeapWordSize;
+  }
+
+  static size_t bitmap_size_in_bits() {
+    size_t bits_in_bitmap = gclab_word_size() >> shifter();
+    // We are going to ensure that the beginning of a word in this
+    // bitmap also corresponds to the beginning of a word in the
+    // global marking bitmap. To handle the case where a GCLab
+    // starts from the middle of the bitmap, we need to add enough
+    // space (i.e. up to a bitmap word) to ensure that we have
+    // enough bits in the bitmap.
+    return bits_in_bitmap + BitsPerWord - 1;
+  }
+public:
+  GCLabBitMap(HeapWord* heap_start)
+    : BitMap(bitmap_size_in_bits()),
+      _cm(G1CollectedHeap::heap()->concurrent_mark()),
+      _shifter(shifter()),
+      _bitmap_word_covers_words(bitmap_word_covers_words()),
+      _heap_start(heap_start),
+      _gclab_word_size(gclab_word_size()),
+      _real_start_word(NULL),
+      _real_end_word(NULL),
+      _start_word(NULL)
+  {
+    guarantee( size_in_words() >= bitmap_size_in_words(),
+               "just making sure");
+  }
+
+  inline unsigned heapWordToOffset(HeapWord* addr) {
+    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
+    assert(offset < size(), "offset should be within bounds");
+    return offset;
+  }
+
+  inline HeapWord* offsetToHeapWord(size_t offset) {
+    HeapWord* addr =  _start_word + (offset << _shifter);
+    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
+    return addr;
+  }
+
+  bool fields_well_formed() {
+    bool ret1 = (_real_start_word == NULL) &&
+                (_real_end_word == NULL) &&
+                (_start_word == NULL);
+    if (ret1)
+      return true;
+
+    bool ret2 = _real_start_word >= _start_word &&
+      _start_word < _real_end_word &&
+      (_real_start_word + _gclab_word_size) == _real_end_word &&
+      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
+                                                              > _real_end_word;
+    return ret2;
+  }
+
+  inline bool mark(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    if (addr >= _real_start_word && addr < _real_end_word) {
+      assert(!isMarked(addr), "should not have already been marked");
+
+      // first mark it on the bitmap
+      at_put(heapWordToOffset(addr), true);
+
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  inline bool isMarked(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    return at(heapWordToOffset(addr));
+  }
+
+  void set_buffer(HeapWord* start) {
+    guarantee(use_local_bitmaps, "invariant");
+    clear();
+
+    assert(start != NULL, "invariant");
+    _real_start_word = start;
+    _real_end_word   = start + _gclab_word_size;
+
+    size_t diff =
+      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
+    _start_word = start - diff;
+
+    assert(fields_well_formed(), "invariant");
+  }
+
+#ifndef PRODUCT
+  void verify() {
+    // verify that the marks have been propagated
+    GCLabBitMapClosure cl(_cm, this);
+    iterate(&cl);
+  }
+#endif // PRODUCT
+
+  void retire() {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(fields_well_formed(), "invariant");
+
+    if (_start_word != NULL) {
+      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
+
+      // this means that the bitmap was set up for the GCLab
+      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
+
+      mark_bitmap->mostly_disjoint_range_union(this,
+                                0, // always start from the start of the bitmap
+                                _start_word,
+                                size_in_words());
+      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
+
+#ifndef PRODUCT
+      if (use_local_bitmaps && verify_local_bitmaps)
+        verify();
+#endif // PRODUCT
+    } else {
+      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
+    }
+  }
+
+  static size_t bitmap_size_in_words() {
+    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
+  }
+};
+
+class G1ParGCAllocBuffer: public ParGCAllocBuffer {
+private:
+  bool        _retired;
+  bool        _during_marking;
+  GCLabBitMap _bitmap;
+
+public:
+  G1ParGCAllocBuffer() :
+    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
+    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
+    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
+    _retired(false)
+  { }
+
+  inline bool mark(HeapWord* addr) {
+    guarantee(use_local_bitmaps, "invariant");
+    assert(_during_marking, "invariant");
+    return _bitmap.mark(addr);
+  }
+
+  inline void set_buf(HeapWord* buf) {
+    if (use_local_bitmaps && _during_marking)
+      _bitmap.set_buffer(buf);
+    ParGCAllocBuffer::set_buf(buf);
+    _retired = false;
+  }
+
+  inline void retire(bool end_of_gc, bool retain) {
+    if (_retired)
+      return;
+    if (use_local_bitmaps && _during_marking) {
+      _bitmap.retire();
+    }
+    ParGCAllocBuffer::retire(end_of_gc, retain);
+    _retired = true;
+  }
+};
+
+class G1ParScanThreadState : public StackObj {
+protected:
+  G1CollectedHeap* _g1h;
+  RefToScanQueue*  _refs;
+  DirtyCardQueue   _dcq;
+  CardTableModRefBS* _ct_bs;
+  G1RemSet* _g1_rem;
+
+  typedef GrowableArray<StarTask> OverflowQueue;
+  OverflowQueue* _overflowed_refs;
+
+  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
+  ageTable           _age_table;
+
+  size_t           _alloc_buffer_waste;
+  size_t           _undo_waste;
+
+  OopsInHeapRegionClosure*      _evac_failure_cl;
+  G1ParScanHeapEvacClosure*     _evac_cl;
+  G1ParScanPartialArrayClosure* _partial_scan_cl;
+
+  int _hash_seed;
+  int _queue_num;
+
+  int _term_attempts;
+#if G1_DETAILED_STATS
+  int _pushes, _pops, _steals, _steal_attempts;
+  int _overflow_pushes;
+#endif
+
+  double _start;
+  double _start_strong_roots;
+  double _strong_roots_time;
+  double _start_term;
+  double _term_time;
+
+  // Map from young-age-index (0 == not young, 1 is youngest) to
+  // surviving words. base is what we get back from the malloc call
+  size_t* _surviving_young_words_base;
+  // this points into the array, as we use the first few entries for padding
+  size_t* _surviving_young_words;
+
+#define PADDING_ELEM_NUM (64 / sizeof(size_t))
+
+  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
+
+  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
+
+  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
+  CardTableModRefBS* ctbs()                      { return _ct_bs; }
+
+  template <class T> void immediate_rs_update(HeapRegion* from, T* p, int tid) {
+    if (!from->is_survivor()) {
+      _g1_rem->par_write_ref(from, p, tid);
+    }
+  }
+
+  template <class T> void deferred_rs_update(HeapRegion* from, T* p, int tid) {
+    // If the new value of the field points to the same region or
+    // is the to-space, we don't need to include it in the Rset updates.
+    if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) {
+      size_t card_index = ctbs()->index_for(p);
+      // If the card hasn't been added to the buffer, do it.
+      if (ctbs()->mark_card_deferred(card_index)) {
+        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
+      }
+    }
+  }
+
+public:
+  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num);
+
+  ~G1ParScanThreadState() {