changeset 6941:190b0e9d0ad1

7173584: Implement arraycopy as a macro node Summary: delay the conversion of arraycopy to stub calls to macro expansion Reviewed-by: kvn, iveresov
author roland
date Mon, 11 Aug 2014 14:12:51 +0200
parents 6618133bfaf7
children db7d2f27bcb6
files src/share/vm/opto/callnode.cpp src/share/vm/opto/callnode.hpp src/share/vm/opto/classes.hpp src/share/vm/opto/compile.cpp src/share/vm/opto/compile.hpp src/share/vm/opto/graphKit.cpp src/share/vm/opto/graphKit.hpp src/share/vm/opto/library_call.cpp src/share/vm/opto/loopPredicate.cpp src/share/vm/opto/macro.cpp src/share/vm/opto/macro.hpp src/share/vm/opto/macroArrayCopy.cpp src/share/vm/opto/node.hpp src/share/vm/opto/phase.hpp src/share/vm/opto/phaseX.hpp
diffstat 15 files changed, 1795 insertions(+), 1087 deletions(-) [+]
line wrap: on
line diff
--- a/src/share/vm/opto/callnode.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/callnode.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -1815,3 +1815,49 @@
   }
   return result;
 }
+
+ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled)
+  : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), _alloc_tightly_coupled(alloc_tightly_coupled), _kind(ArrayCopy) {
+  init_class_id(Class_ArrayCopy);
+  init_flags(Flag_is_macro);
+  C->add_macro_node(this);
+}
+
+uint ArrayCopyNode::size_of() const { return sizeof(*this); }
+
+ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw,
+                                   Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* length,
+                                   bool alloc_tightly_coupled) {
+
+  ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled);
+  Node* prev_mem = kit->set_predefined_input_for_runtime_call(ac);
+
+  ac->init_req( ArrayCopyNode::Src, src);
+  ac->init_req( ArrayCopyNode::SrcPos, src_offset);
+  ac->init_req( ArrayCopyNode::Dest, dest);
+  ac->init_req( ArrayCopyNode::DestPos, dest_offset);
+  ac->init_req( ArrayCopyNode::Length, length);
+
+  if (may_throw) {
+    ac->set_req(TypeFunc::I_O , kit->i_o());
+    kit->add_safepoint_edges(ac, false);
+  }
+
+  return ac;
+}
+
+void ArrayCopyNode::connect_outputs(GraphKit* kit) {
+  kit->set_all_memory_call(this, true);
+  kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));
+  kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));
+  kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);
+  kit->set_all_memory_call(this);
+}
+
+#ifndef PRODUCT
+const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};
+void ArrayCopyNode::dump_spec(outputStream *st) const {
+  CallNode::dump_spec(st);
+  st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");
+}
+#endif
--- a/src/share/vm/opto/callnode.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/callnode.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -1063,4 +1063,96 @@
   virtual bool        guaranteed_safepoint()  { return false; }
 };
 
+class GraphKit;
+
+class ArrayCopyNode : public CallNode {
+private:
+
+  // What kind of arraycopy variant is this?
+  enum {
+    ArrayCopy,       // System.arraycopy()
+    ArrayCopyNoTest, // System.arraycopy(), all arguments validated
+    CloneBasic,      // A clone that can be copied by 64 bit chunks
+    CloneOop,        // An oop array clone
+    CopyOf,          // Arrays.copyOf()
+    CopyOfRange      // Arrays.copyOfRange()
+  } _kind;
+
+#ifndef PRODUCT
+  static const char* _kind_names[CopyOfRange+1];
+#endif
+  // Is the alloc obtained with
+  // AllocateArrayNode::Ideal_array_allocation() tighly coupled
+  // (arraycopy follows immediately the allocation)?
+  // We cache the result of LibraryCallKit::tightly_coupled_allocation
+  // here because it's much easier to find whether there's a tightly
+  // couple allocation at parse time than at macro expansion time. At
+  // macro expansion time, for every use of the allocation node we
+  // would need to figure out whether it happens after the arraycopy (and
+  // can be ignored) or between the allocation and the arraycopy. At
+  // parse time, it's straightforward because whatever happens after
+  // the arraycopy is not parsed yet so doesn't exist when
+  // LibraryCallKit::tightly_coupled_allocation() is called.
+  bool _alloc_tightly_coupled;
+
+  static const TypeFunc* arraycopy_type() {
+    const Type** fields = TypeTuple::fields(ParmLimit - TypeFunc::Parms);
+    fields[Src]     = TypeInstPtr::BOTTOM;
+    fields[SrcPos]  = TypeInt::INT;
+    fields[Dest]    = TypeInstPtr::BOTTOM;
+    fields[DestPos] = TypeInt::INT;
+    fields[Length]  = TypeInt::INT;
+    const TypeTuple *domain = TypeTuple::make(ParmLimit, fields);
+
+    // create result type (range)
+    fields = TypeTuple::fields(0);
+
+    const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+    return TypeFunc::make(domain, range);
+  }
+
+  ArrayCopyNode(Compile* C, bool alloc_tightly_coupled);
+
+public:
+
+  enum {
+    Src   = TypeFunc::Parms,
+    SrcPos,
+    Dest,
+    DestPos,
+    Length,
+    ParmLimit
+  };
+
+  static ArrayCopyNode* make(GraphKit* kit, bool may_throw,
+                             Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* length,
+                             bool alloc_tightly_coupled);
+
+  void connect_outputs(GraphKit* kit);
+
+  bool is_arraycopy()         const { return _kind == ArrayCopy; }
+  bool is_arraycopy_notest()  const { return _kind == ArrayCopyNoTest; }
+  bool is_clonebasic()        const { return _kind == CloneBasic; }
+  bool is_cloneoop()          const { return _kind == CloneOop; }
+  bool is_copyof()            const { return _kind == CopyOf; }
+  bool is_copyofrange()       const { return _kind == CopyOfRange; }
+
+  void set_arraycopy()         { _kind = ArrayCopy; }
+  void set_arraycopy_notest()  { _kind = ArrayCopyNoTest; }
+  void set_clonebasic()        { _kind = CloneBasic; }
+  void set_cloneoop()          { _kind = CloneOop; }
+  void set_copyof()            { _kind = CopyOf; }
+  void set_copyofrange()       { _kind = CopyOfRange; }
+
+  virtual int Opcode() const;
+  virtual uint size_of() const; // Size is bigger
+  virtual bool guaranteed_safepoint()  { return false; }
+
+  bool is_alloc_tightly_coupled() const { return _alloc_tightly_coupled; }
+
+#ifndef PRODUCT
+  virtual void dump_spec(outputStream *st) const;
+#endif
+};
 #endif // SHARE_VM_OPTO_CALLNODE_HPP
--- a/src/share/vm/opto/classes.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/classes.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -37,6 +37,7 @@
 macro(AllocateArray)
 macro(AndI)
 macro(AndL)
+macro(ArrayCopy)
 macro(AryEq)
 macro(AtanD)
 macro(Binary)
--- a/src/share/vm/opto/compile.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/compile.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -3787,6 +3787,56 @@
   }
 }
 
+//----------------------------static_subtype_check-----------------------------
+// Shortcut important common cases when superklass is exact:
+// (0) superklass is java.lang.Object (can occur in reflective code)
+// (1) subklass is already limited to a subtype of superklass => always ok
+// (2) subklass does not overlap with superklass => always fail
+// (3) superklass has NO subtypes and we can check with a simple compare.
+int Compile::static_subtype_check(ciKlass* superk, ciKlass* subk) {
+  if (StressReflectiveCode) {
+    return SSC_full_test;       // Let caller generate the general case.
+  }
+
+  if (superk == env()->Object_klass()) {
+    return SSC_always_true;     // (0) this test cannot fail
+  }
+
+  ciType* superelem = superk;
+  if (superelem->is_array_klass())
+    superelem = superelem->as_array_klass()->base_element_type();
+
+  if (!subk->is_interface()) {  // cannot trust static interface types yet
+    if (subk->is_subtype_of(superk)) {
+      return SSC_always_true;   // (1) false path dead; no dynamic test needed
+    }
+    if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
+        !superk->is_subtype_of(subk)) {
+      return SSC_always_false;
+    }
+  }
+
+  // If casting to an instance klass, it must have no subtypes
+  if (superk->is_interface()) {
+    // Cannot trust interfaces yet.
+    // %%% S.B. superk->nof_implementors() == 1
+  } else if (superelem->is_instance_klass()) {
+    ciInstanceKlass* ik = superelem->as_instance_klass();
+    if (!ik->has_subklass() && !ik->is_interface()) {
+      if (!ik->is_final()) {
+        // Add a dependency if there is a chance of a later subclass.
+        dependencies()->assert_leaf_type(ik);
+      }
+      return SSC_easy_test;     // (3) caller can do a simple ptr comparison
+    }
+  } else {
+    // A primitive array type has no subtypes.
+    return SSC_easy_test;       // (3) caller can do a simple ptr comparison
+  }
+
+  return SSC_full_test;
+}
+
 // The message about the current inlining is accumulated in
 // _print_inlining_stream and transfered into the _print_inlining_list
 // once we know whether inlining succeeds or not. For regular
--- a/src/share/vm/opto/compile.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/compile.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -1200,6 +1200,10 @@
   // Definitions of pd methods
   static void pd_compiler2_init();
 
+  // Static parse-time type checking logic for gen_subtype_check:
+  enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
+  int static_subtype_check(ciKlass* superk, ciKlass* subk);
+
   // Auxiliary method for randomized fuzzing/stressing
   static bool randomized_select(int count);
 };
--- a/src/share/vm/opto/graphKit.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/graphKit.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -2520,6 +2520,21 @@
   set_control(norm);
 }
 
+static IfNode* gen_subtype_check_compare(Node* ctrl, Node* in1, Node* in2, BoolTest::mask test, float p, PhaseGVN* gvn, BasicType bt) {
+  Node* cmp = NULL;
+  switch(bt) {
+  case T_INT: cmp = new CmpINode(in1, in2); break;
+  case T_ADDRESS: cmp = new CmpPNode(in1, in2); break;
+  default: fatal(err_msg("unexpected comparison type %s", type2name(bt)));
+  }
+  gvn->transform(cmp);
+  Node* bol = gvn->transform(new BoolNode(cmp, test));
+  IfNode* iff = new IfNode(ctrl, bol, p, COUNT_UNKNOWN);
+  gvn->transform(iff);
+  if (!bol->is_Con()) gvn->record_for_igvn(iff);
+  return iff;
+}
+
 
 //-------------------------------gen_subtype_check-----------------------------
 // Generate a subtyping check.  Takes as input the subtype and supertype.
@@ -2529,16 +2544,17 @@
 // but that's not exposed to the optimizer.  This call also doesn't take in an
 // Object; if you wish to check an Object you need to load the Object's class
 // prior to coming here.
-Node* GraphKit::gen_subtype_check(Node* subklass, Node* superklass) {
+Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn) {
+  Compile* C = gvn->C;
   // Fast check for identical types, perhaps identical constants.
   // The types can even be identical non-constants, in cases
   // involving Array.newInstance, Object.clone, etc.
   if (subklass == superklass)
-    return top();             // false path is dead; no test needed.
-
-  if (_gvn.type(superklass)->singleton()) {
-    ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
-    ciKlass* subk   = _gvn.type(subklass)->is_klassptr()->klass();
+    return C->top();             // false path is dead; no test needed.
+
+  if (gvn->type(superklass)->singleton()) {
+    ciKlass* superk = gvn->type(superklass)->is_klassptr()->klass();
+    ciKlass* subk   = gvn->type(subklass)->is_klassptr()->klass();
 
     // In the common case of an exact superklass, try to fold up the
     // test before generating code.  You may ask, why not just generate
@@ -2549,25 +2565,23 @@
     //    Foo[] fa = blah(); Foo x = fa[0]; fa[1] = x;
     // Here, the type of 'fa' is often exact, so the store check
     // of fa[1]=x will fold up, without testing the nullness of x.
-    switch (static_subtype_check(superk, subk)) {
-    case SSC_always_false:
+    switch (C->static_subtype_check(superk, subk)) {
+    case Compile::SSC_always_false:
       {
-        Node* always_fail = control();
-        set_control(top());
+        Node* always_fail = *ctrl;
+        *ctrl = gvn->C->top();
         return always_fail;
       }
-    case SSC_always_true:
-      return top();
-    case SSC_easy_test:
+    case Compile::SSC_always_true:
+      return C->top();
+    case Compile::SSC_easy_test:
       {
         // Just do a direct pointer compare and be done.
-        Node* cmp = _gvn.transform( new CmpPNode(subklass, superklass) );
-        Node* bol = _gvn.transform( new BoolNode(cmp, BoolTest::eq) );
-        IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
-        set_control( _gvn.transform( new IfTrueNode (iff) ) );
-        return       _gvn.transform( new IfFalseNode(iff) );
+        IfNode* iff = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_STATIC_FREQUENT, gvn, T_ADDRESS);
+        *ctrl = gvn->transform(new IfTrueNode(iff));
+        return gvn->transform(new IfFalseNode(iff));
       }
-    case SSC_full_test:
+    case Compile::SSC_full_test:
       break;
     default:
       ShouldNotReachHere();
@@ -2579,11 +2593,11 @@
   // will always succeed.  We could leave a dependency behind to ensure this.
 
   // First load the super-klass's check-offset
-  Node *p1 = basic_plus_adr( superklass, superklass, in_bytes(Klass::super_check_offset_offset()) );
-  Node *chk_off = _gvn.transform(new LoadINode(NULL, memory(p1), p1, _gvn.type(p1)->is_ptr(),
-                                                   TypeInt::INT, MemNode::unordered));
+  Node *p1 = gvn->transform(new AddPNode(superklass, superklass, gvn->MakeConX(in_bytes(Klass::super_check_offset_offset()))));
+  Node* m = mem->memory_at(C->get_alias_index(gvn->type(p1)->is_ptr()));
+  Node *chk_off = gvn->transform(new LoadINode(NULL, m, p1, gvn->type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered));
   int cacheoff_con = in_bytes(Klass::secondary_super_cache_offset());
-  bool might_be_cache = (find_int_con(chk_off, cacheoff_con) == cacheoff_con);
+  bool might_be_cache = (gvn->find_int_con(chk_off, cacheoff_con) == cacheoff_con);
 
   // Load from the sub-klass's super-class display list, or a 1-word cache of
   // the secondary superclass list, or a failing value with a sentinel offset
@@ -2591,42 +2605,44 @@
   // hierarchy and we have to scan the secondary superclass list the hard way.
   // Worst-case type is a little odd: NULL is allowed as a result (usually
   // klass loads can never produce a NULL).
-  Node *chk_off_X = ConvI2X(chk_off);
-  Node *p2 = _gvn.transform( new AddPNode(subklass,subklass,chk_off_X) );
+  Node *chk_off_X = chk_off;
+#ifdef _LP64
+  chk_off_X = gvn->transform(new ConvI2LNode(chk_off_X));
+#endif
+  Node *p2 = gvn->transform(new AddPNode(subklass,subklass,chk_off_X));
   // For some types like interfaces the following loadKlass is from a 1-word
   // cache which is mutable so can't use immutable memory.  Other
   // types load from the super-class display table which is immutable.
-  Node *kmem = might_be_cache ? memory(p2) : immutable_memory();
-  Node *nkls = _gvn.transform( LoadKlassNode::make( _gvn, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
+  m = mem->memory_at(C->get_alias_index(gvn->type(p2)->is_ptr()));
+  Node *kmem = might_be_cache ? m : C->immutable_memory();
+  Node *nkls = gvn->transform(LoadKlassNode::make(*gvn, kmem, p2, gvn->type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL));
 
   // Compile speed common case: ARE a subtype and we canNOT fail
   if( superklass == nkls )
-    return top();             // false path is dead; no test needed.
+    return C->top();             // false path is dead; no test needed.
 
   // See if we get an immediate positive hit.  Happens roughly 83% of the
   // time.  Test to see if the value loaded just previously from the subklass
   // is exactly the superklass.
-  Node *cmp1 = _gvn.transform( new CmpPNode( superklass, nkls ) );
-  Node *bol1 = _gvn.transform( new BoolNode( cmp1, BoolTest::eq ) );
-  IfNode *iff1 = create_and_xform_if( control(), bol1, PROB_LIKELY(0.83f), COUNT_UNKNOWN );
-  Node *iftrue1 = _gvn.transform( new IfTrueNode ( iff1 ) );
-  set_control(    _gvn.transform( new IfFalseNode( iff1 ) ) );
+  IfNode *iff1 = gen_subtype_check_compare(*ctrl, superklass, nkls, BoolTest::eq, PROB_LIKELY(0.83f), gvn, T_ADDRESS);
+  Node *iftrue1 = gvn->transform( new IfTrueNode (iff1));
+  *ctrl = gvn->transform(new IfFalseNode(iff1));
 
   // Compile speed common case: Check for being deterministic right now.  If
   // chk_off is a constant and not equal to cacheoff then we are NOT a
   // subklass.  In this case we need exactly the 1 test above and we can
   // return those results immediately.
   if (!might_be_cache) {
-    Node* not_subtype_ctrl = control();
-    set_control(iftrue1); // We need exactly the 1 test above
+    Node* not_subtype_ctrl = *ctrl;
+    *ctrl = iftrue1; // We need exactly the 1 test above
     return not_subtype_ctrl;
   }
 
   // Gather the various success & failures here
   RegionNode *r_ok_subtype = new RegionNode(4);
-  record_for_igvn(r_ok_subtype);
+  gvn->record_for_igvn(r_ok_subtype);
   RegionNode *r_not_subtype = new RegionNode(3);
-  record_for_igvn(r_not_subtype);
+  gvn->record_for_igvn(r_not_subtype);
 
   r_ok_subtype->init_req(1, iftrue1);
 
@@ -2635,21 +2651,17 @@
   // check-offset points into the subklass display list or the 1-element
   // cache.  If it points to the display (and NOT the cache) and the display
   // missed then it's not a subtype.
-  Node *cacheoff = _gvn.intcon(cacheoff_con);
-  Node *cmp2 = _gvn.transform( new CmpINode( chk_off, cacheoff ) );
-  Node *bol2 = _gvn.transform( new BoolNode( cmp2, BoolTest::ne ) );
-  IfNode *iff2 = create_and_xform_if( control(), bol2, PROB_LIKELY(0.63f), COUNT_UNKNOWN );
-  r_not_subtype->init_req(1, _gvn.transform( new IfTrueNode (iff2) ) );
-  set_control(                _gvn.transform( new IfFalseNode(iff2) ) );
+  Node *cacheoff = gvn->intcon(cacheoff_con);
+  IfNode *iff2 = gen_subtype_check_compare(*ctrl, chk_off, cacheoff, BoolTest::ne, PROB_LIKELY(0.63f), gvn, T_INT);
+  r_not_subtype->init_req(1, gvn->transform(new IfTrueNode (iff2)));
+  *ctrl = gvn->transform(new IfFalseNode(iff2));
 
   // Check for self.  Very rare to get here, but it is taken 1/3 the time.
   // No performance impact (too rare) but allows sharing of secondary arrays
   // which has some footprint reduction.
-  Node *cmp3 = _gvn.transform( new CmpPNode( subklass, superklass ) );
-  Node *bol3 = _gvn.transform( new BoolNode( cmp3, BoolTest::eq ) );
-  IfNode *iff3 = create_and_xform_if( control(), bol3, PROB_LIKELY(0.36f), COUNT_UNKNOWN );
-  r_ok_subtype->init_req(2, _gvn.transform( new IfTrueNode ( iff3 ) ) );
-  set_control(               _gvn.transform( new IfFalseNode( iff3 ) ) );
+  IfNode *iff3 = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_LIKELY(0.36f), gvn, T_ADDRESS);
+  r_ok_subtype->init_req(2, gvn->transform(new IfTrueNode(iff3)));
+  *ctrl = gvn->transform(new IfFalseNode(iff3));
 
   // -- Roads not taken here: --
   // We could also have chosen to perform the self-check at the beginning
@@ -2672,68 +2684,16 @@
   // out of line, and it can only improve I-cache density.
   // The decision to inline or out-of-line this final check is platform
   // dependent, and is found in the AD file definition of PartialSubtypeCheck.
-  Node* psc = _gvn.transform(
-    new PartialSubtypeCheckNode(control(), subklass, superklass) );
-
-  Node *cmp4 = _gvn.transform( new CmpPNode( psc, null() ) );
-  Node *bol4 = _gvn.transform( new BoolNode( cmp4, BoolTest::ne ) );
-  IfNode *iff4 = create_and_xform_if( control(), bol4, PROB_FAIR, COUNT_UNKNOWN );
-  r_not_subtype->init_req(2, _gvn.transform( new IfTrueNode (iff4) ) );
-  r_ok_subtype ->init_req(3, _gvn.transform( new IfFalseNode(iff4) ) );
+  Node* psc = gvn->transform(
+    new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
+
+  IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn->zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
+  r_not_subtype->init_req(2, gvn->transform(new IfTrueNode (iff4)));
+  r_ok_subtype ->init_req(3, gvn->transform(new IfFalseNode(iff4)));
 
   // Return false path; set default control to true path.
-  set_control( _gvn.transform(r_ok_subtype) );
-  return _gvn.transform(r_not_subtype);
-}
-
-//----------------------------static_subtype_check-----------------------------
-// Shortcut important common cases when superklass is exact:
-// (0) superklass is java.lang.Object (can occur in reflective code)
-// (1) subklass is already limited to a subtype of superklass => always ok
-// (2) subklass does not overlap with superklass => always fail
-// (3) superklass has NO subtypes and we can check with a simple compare.
-int GraphKit::static_subtype_check(ciKlass* superk, ciKlass* subk) {
-  if (StressReflectiveCode) {
-    return SSC_full_test;       // Let caller generate the general case.
-  }
-
-  if (superk == env()->Object_klass()) {
-    return SSC_always_true;     // (0) this test cannot fail
-  }
-
-  ciType* superelem = superk;
-  if (superelem->is_array_klass())
-    superelem = superelem->as_array_klass()->base_element_type();
-
-  if (!subk->is_interface()) {  // cannot trust static interface types yet
-    if (subk->is_subtype_of(superk)) {
-      return SSC_always_true;   // (1) false path dead; no dynamic test needed
-    }
-    if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
-        !superk->is_subtype_of(subk)) {
-      return SSC_always_false;
-    }
-  }
-
-  // If casting to an instance klass, it must have no subtypes
-  if (superk->is_interface()) {
-    // Cannot trust interfaces yet.
-    // %%% S.B. superk->nof_implementors() == 1
-  } else if (superelem->is_instance_klass()) {
-    ciInstanceKlass* ik = superelem->as_instance_klass();
-    if (!ik->has_subklass() && !ik->is_interface()) {
-      if (!ik->is_final()) {
-        // Add a dependency if there is a chance of a later subclass.
-        C->dependencies()->assert_leaf_type(ik);
-      }
-      return SSC_easy_test;     // (3) caller can do a simple ptr comparison
-    }
-  } else {
-    // A primitive array type has no subtypes.
-    return SSC_easy_test;       // (3) caller can do a simple ptr comparison
-  }
-
-  return SSC_full_test;
+  *ctrl = gvn->transform(r_ok_subtype);
+  return gvn->transform(r_not_subtype);
 }
 
 // Profile-driven exact type check:
@@ -2813,7 +2773,7 @@
   ciKlass* exact_kls = spec_klass == NULL ? profile_has_unique_klass() : spec_klass;
   if (exact_kls != NULL) {// no cast failures here
     if (require_klass == NULL ||
-        static_subtype_check(require_klass, exact_kls) == SSC_always_true) {
+        C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) {
       // If we narrow the type to match what the type profile sees or
       // the speculative type, we can then remove the rest of the
       // cast.
@@ -2833,7 +2793,7 @@
       }
       return exact_obj;
     }
-    // assert(ssc == SSC_always_true)... except maybe the profile lied to us.
+    // assert(ssc == Compile::SSC_always_true)... except maybe the profile lied to us.
   }
 
   return NULL;
@@ -2938,8 +2898,8 @@
     ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
     ciKlass* subk = _gvn.type(obj)->is_oopptr()->klass();
     if (subk != NULL && subk->is_loaded()) {
-      int static_res = static_subtype_check(superk, subk);
-      known_statically = (static_res == SSC_always_true || static_res == SSC_always_false);
+      int static_res = C->static_subtype_check(superk, subk);
+      known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
     }
   }
 
@@ -3007,13 +2967,13 @@
   if (tk->singleton()) {
     const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
     if (objtp != NULL && objtp->klass() != NULL) {
-      switch (static_subtype_check(tk->klass(), objtp->klass())) {
-      case SSC_always_true:
+      switch (C->static_subtype_check(tk->klass(), objtp->klass())) {
+      case Compile::SSC_always_true:
         // If we know the type check always succeed then we don't use
         // the profiling data at this bytecode. Don't lose it, feed it
         // to the type system as a speculative type.
         return record_profiled_receiver_for_speculation(obj);
-      case SSC_always_false:
+      case Compile::SSC_always_false:
         // It needs a null check because a null will *pass* the cast check.
         // A non-null value will always produce an exception.
         return null_assert(obj);
--- a/src/share/vm/opto/graphKit.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/graphKit.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -829,17 +829,13 @@
   Node* gen_checkcast( Node *subobj, Node* superkls,
                        Node* *failure_control = NULL );
 
-  // Generate a subtyping check.  Takes as input the subtype and supertype.
-  // Returns 2 values: sets the default control() to the true path and
-  // returns the false path.  Only reads from constant memory taken from the
-  // default memory; does not write anything.  It also doesn't take in an
-  // Object; if you wish to check an Object you need to load the Object's
-  // class prior to coming here.
-  Node* gen_subtype_check(Node* subklass, Node* superklass);
-
-  // Static parse-time type checking logic for gen_subtype_check:
-  enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
-  int static_subtype_check(ciKlass* superk, ciKlass* subk);
+  Node* gen_subtype_check(Node* subklass, Node* superklass) {
+    MergeMemNode* mem = merged_memory();
+    Node* ctrl = control();
+    Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, &_gvn);
+    set_control(ctrl);
+    return n;
+  }
 
   // Exact type check used for predicted calls and casts.
   // Rewrites (*casted_receiver) to be casted to the stronger type.
--- a/src/share/vm/opto/library_call.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/library_call.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -146,15 +146,10 @@
   Node* generate_negative_guard(Node* index, RegionNode* region,
                                 // resulting CastII of index:
                                 Node* *pos_index = NULL);
-  Node* generate_nonpositive_guard(Node* index, bool never_negative,
-                                   // resulting CastII of index:
-                                   Node* *pos_index = NULL);
   Node* generate_limit_guard(Node* offset, Node* subseq_length,
                              Node* array_length,
                              RegionNode* region);
   Node* generate_current_thread(Node* &tls_output);
-  address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
-                              bool disjoint_bases, const char* &name, bool dest_uninitialized);
   Node* load_mirror_from_klass(Node* klass);
   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
                                       RegionNode* region, int null_path,
@@ -264,47 +259,8 @@
 
   // Helper functions for inlining arraycopy
   bool inline_arraycopy();
-  void generate_arraycopy(const TypePtr* adr_type,
-                          BasicType basic_elem_type,
-                          Node* src,  Node* src_offset,
-                          Node* dest, Node* dest_offset,
-                          Node* copy_length,
-                          bool disjoint_bases = false,
-                          bool length_never_negative = false,
-                          RegionNode* slow_region = NULL);
   AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
                                                 RegionNode* slow_region);
-  void generate_clear_array(const TypePtr* adr_type,
-                            Node* dest,
-                            BasicType basic_elem_type,
-                            Node* slice_off,
-                            Node* slice_len,
-                            Node* slice_end);
-  bool generate_block_arraycopy(const TypePtr* adr_type,
-                                BasicType basic_elem_type,
-                                AllocateNode* alloc,
-                                Node* src,  Node* src_offset,
-                                Node* dest, Node* dest_offset,
-                                Node* dest_size, bool dest_uninitialized);
-  void generate_slow_arraycopy(const TypePtr* adr_type,
-                               Node* src,  Node* src_offset,
-                               Node* dest, Node* dest_offset,
-                               Node* copy_length, bool dest_uninitialized);
-  Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
-                                     Node* dest_elem_klass,
-                                     Node* src,  Node* src_offset,
-                                     Node* dest, Node* dest_offset,
-                                     Node* copy_length, bool dest_uninitialized);
-  Node* generate_generic_arraycopy(const TypePtr* adr_type,
-                                   Node* src,  Node* src_offset,
-                                   Node* dest, Node* dest_offset,
-                                   Node* copy_length, bool dest_uninitialized);
-  void generate_unchecked_arraycopy(const TypePtr* adr_type,
-                                    BasicType basic_elem_type,
-                                    bool disjoint_bases,
-                                    Node* src,  Node* src_offset,
-                                    Node* dest, Node* dest_offset,
-                                    Node* copy_length, bool dest_uninitialized);
   typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
   bool inline_unsafe_ordered_store(BasicType type);
@@ -1049,25 +1005,6 @@
   return is_neg;
 }
 
-inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
-                                                        Node* *pos_index) {
-  if (stopped())
-    return NULL;                // already stopped
-  if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
-    return NULL;                // index is already adequately typed
-  Node* cmp_le = _gvn.transform(new CmpINode(index, intcon(0)));
-  BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
-  Node* bol_le = _gvn.transform(new BoolNode(cmp_le, le_or_eq));
-  Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
-  if (is_notp != NULL && pos_index != NULL) {
-    // Emulate effect of Parse::adjust_map_after_if.
-    Node* ccast = new CastIINode(index, TypeInt::POS1);
-    ccast->set_req(0, control());
-    (*pos_index) = _gvn.transform(ccast);
-  }
-  return is_notp;
-}
-
 // Make sure that 'position' is a valid limit index, in [0..length].
 // There are two equivalent plans for checking this:
 //   A. (offset + copyLength)  unsigned<=  arrayLength
@@ -3928,13 +3865,18 @@
       // oop stores need checking.
       // Extreme case:  Arrays.copyOf((Integer[])x, 10, String[].class).
       // This will fail a store-check if x contains any non-nulls.
-      bool disjoint_bases = true;
-      // if start > orig_length then the length of the copy may be
-      // negative.
-      bool length_never_negative = !is_copyOfRange;
-      generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
-                         original, start, newcopy, intcon(0), moved,
-                         disjoint_bases, length_never_negative);
+
+      Node* alloc = tightly_coupled_allocation(newcopy, NULL);
+
+      ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, alloc != NULL);
+      if (!is_copyOfRange) {
+        ac->set_copyof();
+      } else {
+        ac->set_copyofrange();
+      }
+      Node* n = _gvn.transform(ac);
+      assert(n == ac, "cannot disappear");
+      ac->connect_outputs(this);
     }
   } // original reexecute is set back here
 
@@ -4445,10 +4387,12 @@
   countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
 
   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-  bool disjoint_bases = true;
-  generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
-                               src, NULL, dest, NULL, countx,
-                               /*dest_uninitialized*/true);
+
+  ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
+  ac->set_clonebasic();
+  Node* n = _gvn.transform(ac);
+  assert(n == ac, "cannot disappear");
+  set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
 
   // If necessary, emit some card marks afterwards.  (Non-arrays only.)
   if (card_mark) {
@@ -4557,12 +4501,13 @@
           PreserveJVMState pjvms2(this);
           set_control(is_obja);
           // Generate a direct call to the right arraycopy function(s).
-          bool disjoint_bases = true;
-          bool length_never_negative = true;
-          generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
-                             obj, intcon(0), alloc_obj, intcon(0),
-                             obj_length,
-                             disjoint_bases, length_never_negative);
+          Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
+          ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
+          ac->set_cloneoop();
+          Node* n = _gvn.transform(ac);
+          assert(n == ac, "cannot disappear");
+          ac->connect_outputs(this);
+
           result_reg->init_req(_objArray_path, control());
           result_val->init_req(_objArray_path, alloc_obj);
           result_i_o ->set_req(_objArray_path, i_o());
@@ -4656,42 +4601,6 @@
   return true;
 }
 
-//------------------------------basictype2arraycopy----------------------------
-address LibraryCallKit::basictype2arraycopy(BasicType t,
-                                            Node* src_offset,
-                                            Node* dest_offset,
-                                            bool disjoint_bases,
-                                            const char* &name,
-                                            bool dest_uninitialized) {
-  const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
-  const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
-
-  bool aligned = false;
-  bool disjoint = disjoint_bases;
-
-  // if the offsets are the same, we can treat the memory regions as
-  // disjoint, because either the memory regions are in different arrays,
-  // or they are identical (which we can treat as disjoint.)  We can also
-  // treat a copy with a destination index  less that the source index
-  // as disjoint since a low->high copy will work correctly in this case.
-  if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
-      dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
-    // both indices are constants
-    int s_offs = src_offset_inttype->get_con();
-    int d_offs = dest_offset_inttype->get_con();
-    int element_size = type2aelembytes(t);
-    aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
-              ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
-    if (s_offs >= d_offs)  disjoint = true;
-  } else if (src_offset == dest_offset && src_offset != NULL) {
-    // This can occur if the offsets are identical non-constants.
-    disjoint = true;
-  }
-
-  return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
-}
-
-
 //------------------------------inline_arraycopy-----------------------
 // public static native void java.lang.System.arraycopy(Object src,  int  srcPos,
 //                                                      Object dest, int destPos,
@@ -4704,13 +4613,26 @@
   Node* dest_offset = argument(3);  // type: int
   Node* length      = argument(4);  // type: int
 
-  // Compile time checks.  If any of these checks cannot be verified at compile time,
-  // we do not make a fast path for this call.  Instead, we let the call remain as it
-  // is.  The checks we choose to mandate at compile time are:
-  //
+  // The following tests must be performed
   // (1) src and dest are arrays.
-  const Type* src_type  = src->Value(&_gvn);
-  const Type* dest_type = dest->Value(&_gvn);
+  // (2) src and dest arrays must have elements of the same BasicType
+  // (3) src and dest must not be null.
+  // (4) src_offset must not be negative.
+  // (5) dest_offset must not be negative.
+  // (6) length must not be negative.
+  // (7) src_offset + length must not exceed length of src.
+  // (8) dest_offset + length must not exceed length of dest.
+  // (9) each element of an oop array must be assignable
+
+  // (3) src and dest must not be null.
+  // always do this here because we need the JVM state for uncommon traps
+  src  = null_check(src,  T_ARRAY);
+  dest = null_check(dest, T_ARRAY);
+
+  bool notest = false;
+
+  const Type* src_type  = _gvn.type(src);
+  const Type* dest_type = _gvn.type(dest);
   const TypeAryPtr* top_src  = src_type->isa_aryptr();
   const TypeAryPtr* top_dest = dest_type->isa_aryptr();
 
@@ -4768,556 +4690,114 @@
     }
   }
 
-  if (!has_src || !has_dest) {
-    // Conservatively insert a memory barrier on all memory slices.
-    // Do not let writes into the source float below the arraycopy.
-    insert_mem_bar(Op_MemBarCPUOrder);
-
-    // Call StubRoutines::generic_arraycopy stub.
-    generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
-                       src, src_offset, dest, dest_offset, length);
-
-    // Do not let reads from the destination float above the arraycopy.
-    // Since we cannot type the arrays, we don't know which slices
-    // might be affected.  We could restrict this barrier only to those
-    // memory slices which pertain to array elements--but don't bother.
-    if (!InsertMemBarAfterArraycopy)
-      // (If InsertMemBarAfterArraycopy, there is already one in place.)
-      insert_mem_bar(Op_MemBarCPUOrder);
-    return true;
-  }
-
-  // (2) src and dest arrays must have elements of the same BasicType
-  // Figure out the size and type of the elements we will be copying.
-  BasicType src_elem  =  top_src->klass()->as_array_klass()->element_type()->basic_type();
-  BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
-  if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
-  if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
-
-  if (src_elem != dest_elem || dest_elem == T_VOID) {
-    // The component types are not the same or are not recognized.  Punt.
-    // (But, avoid the native method wrapper to JVM_ArrayCopy.)
-    generate_slow_arraycopy(TypePtr::BOTTOM,
-                            src, src_offset, dest, dest_offset, length,
-                            /*dest_uninitialized*/false);
-    return true;
-  }
-
-  if (src_elem == T_OBJECT) {
-    // If both arrays are object arrays then having the exact types
-    // for both will remove the need for a subtype check at runtime
-    // before the call and may make it possible to pick a faster copy
-    // routine (without a subtype check on every element)
-    // Do we have the exact type of src?
-    bool could_have_src = src_spec;
-    // Do we have the exact type of dest?
-    bool could_have_dest = dest_spec;
-    ciKlass* src_k = top_src->klass();
-    ciKlass* dest_k = top_dest->klass();
-    if (!src_spec) {
-      src_k = src_type->speculative_type_not_null();
-      if (src_k != NULL && src_k->is_array_klass()) {
+  if (has_src && has_dest) {
+    BasicType src_elem  = top_src->klass()->as_array_klass()->element_type()->basic_type();
+    BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+    if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
+    if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
+
+    if (src_elem == dest_elem && src_elem == T_OBJECT) {
+      // If both arrays are object arrays then having the exact types
+      // for both will remove the need for a subtype check at runtime
+      // before the call and may make it possible to pick a faster copy
+      // routine (without a subtype check on every element)
+      // Do we have the exact type of src?
+      bool could_have_src = src_spec;
+      // Do we have the exact type of dest?
+      bool could_have_dest = dest_spec;
+      ciKlass* src_k = top_src->klass();
+      ciKlass* dest_k = top_dest->klass();
+      if (!src_spec) {
+        src_k = src_type->speculative_type_not_null();
+        if (src_k != NULL && src_k->is_array_klass()) {
           could_have_src = true;
+        }
       }
-    }
-    if (!dest_spec) {
-      dest_k = dest_type->speculative_type_not_null();
-      if (dest_k != NULL && dest_k->is_array_klass()) {
-        could_have_dest = true;
+      if (!dest_spec) {
+        dest_k = dest_type->speculative_type_not_null();
+        if (dest_k != NULL && dest_k->is_array_klass()) {
+          could_have_dest = true;
+        }
       }
-    }
-    if (could_have_src && could_have_dest) {
-      // If we can have both exact types, emit the missing guards
-      if (could_have_src && !src_spec) {
-        src = maybe_cast_profiled_obj(src, src_k);
-      }
-      if (could_have_dest && !dest_spec) {
-        dest = maybe_cast_profiled_obj(dest, dest_k);
+      if (could_have_src && could_have_dest) {
+        // If we can have both exact types, emit the missing guards
+        if (could_have_src && !src_spec) {
+          src = maybe_cast_profiled_obj(src, src_k);
+        }
+        if (could_have_dest && !dest_spec) {
+          dest = maybe_cast_profiled_obj(dest, dest_k);
+        }
       }
     }
   }
 
-  //---------------------------------------------------------------------------
-  // We will make a fast path for this call to arraycopy.
-
-  // We have the following tests left to perform:
-  //
-  // (3) src and dest must not be null.
-  // (4) src_offset must not be negative.
-  // (5) dest_offset must not be negative.
-  // (6) length must not be negative.
-  // (7) src_offset + length must not exceed length of src.
-  // (8) dest_offset + length must not exceed length of dest.
-  // (9) each element of an oop array must be assignable
-
-  RegionNode* slow_region = new RegionNode(1);
-  record_for_igvn(slow_region);
-
-  // (3) operands must not be null
-  // We currently perform our null checks with the null_check routine.
-  // This means that the null exceptions will be reported in the caller
-  // rather than (correctly) reported inside of the native arraycopy call.
-  // This should be corrected, given time.  We do our null check with the
-  // stack pointer restored.
-  src  = null_check(src,  T_ARRAY);
-  dest = null_check(dest, T_ARRAY);
-
-  // (4) src_offset must not be negative.
-  generate_negative_guard(src_offset, slow_region);
-
-  // (5) dest_offset must not be negative.
-  generate_negative_guard(dest_offset, slow_region);
-
-  // (6) length must not be negative (moved to generate_arraycopy()).
-  // generate_negative_guard(length, slow_region);
-
-  // (7) src_offset + length must not exceed length of src.
-  generate_limit_guard(src_offset, length,
-                       load_array_length(src),
-                       slow_region);
-
-  // (8) dest_offset + length must not exceed length of dest.
-  generate_limit_guard(dest_offset, length,
-                       load_array_length(dest),
-                       slow_region);
-
-  // (9) each element of an oop array must be assignable
-  // The generate_arraycopy subroutine checks this.
-
-  // This is where the memory effects are placed:
-  const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
-  generate_arraycopy(adr_type, dest_elem,
-                     src, src_offset, dest, dest_offset, length,
-                     false, false, slow_region);
-
-  return true;
-}
-
-//-----------------------------generate_arraycopy----------------------
-// Generate an optimized call to arraycopy.
-// Caller must guard against non-arrays.
-// Caller must determine a common array basic-type for both arrays.
-// Caller must validate offsets against array bounds.
-// The slow_region has already collected guard failure paths
-// (such as out of bounds length or non-conformable array types).
-// The generated code has this shape, in general:
-//
-//     if (length == 0)  return   // via zero_path
-//     slowval = -1
-//     if (types unknown) {
-//       slowval = call generic copy loop
-//       if (slowval == 0)  return  // via checked_path
-//     } else if (indexes in bounds) {
-//       if ((is object array) && !(array type check)) {
-//         slowval = call checked copy loop
-//         if (slowval == 0)  return  // via checked_path
-//       } else {
-//         call bulk copy loop
-//         return  // via fast_path
-//       }
-//     }
-//     // adjust params for remaining work:
-//     if (slowval != -1) {
-//       n = -1^slowval; src_offset += n; dest_offset += n; length -= n
-//     }
-//   slow_region:
-//     call slow arraycopy(src, src_offset, dest, dest_offset, length)
-//     return  // via slow_call_path
-//
-// This routine is used from several intrinsics:  System.arraycopy,
-// Object.clone (the array subcase), and Arrays.copyOf[Range].
-//
-void
-LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
-                                   BasicType basic_elem_type,
-                                   Node* src,  Node* src_offset,
-                                   Node* dest, Node* dest_offset,
-                                   Node* copy_length,
-                                   bool disjoint_bases,
-                                   bool length_never_negative,
-                                   RegionNode* slow_region) {
-
-  if (slow_region == NULL) {
-    slow_region = new RegionNode(1);
+  if (!too_many_traps(Deoptimization::Reason_intrinsic) && !src->is_top() && !dest->is_top()) {
+    // validate arguments: enables transformation the ArrayCopyNode
+    notest = true;
+
+    RegionNode* slow_region = new RegionNode(1);
     record_for_igvn(slow_region);
-  }
-
-  Node* original_dest      = dest;
-  AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
-  bool  dest_uninitialized = false;
-
-  // See if this is the initialization of a newly-allocated array.
-  // If so, we will take responsibility here for initializing it to zero.
-  // (Note:  Because tightly_coupled_allocation performs checks on the
-  // out-edges of the dest, we need to avoid making derived pointers
-  // from it until we have checked its uses.)
-  if (ReduceBulkZeroing
-      && !ZeroTLAB              // pointless if already zeroed
-      && basic_elem_type != T_CONFLICT // avoid corner case
-      && !src->eqv_uncast(dest)
-      && ((alloc = tightly_coupled_allocation(dest, slow_region))
-          != NULL)
-      && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
-      && alloc->maybe_set_complete(&_gvn)) {
-    // "You break it, you buy it."
-    InitializeNode* init = alloc->initialization();
-    assert(init->is_complete(), "we just did this");
-    init->set_complete_with_arraycopy();
-    assert(dest->is_CheckCastPP(), "sanity");
-    assert(dest->in(0)->in(0) == init, "dest pinned");
-    adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
-    // From this point on, every exit path is responsible for
-    // initializing any non-copied parts of the object to zero.
-    // Also, if this flag is set we make sure that arraycopy interacts properly
-    // with G1, eliding pre-barriers. See CR 6627983.
-    dest_uninitialized = true;
-  } else {
-    // No zeroing elimination here.
-    alloc             = NULL;
-    //original_dest   = dest;
-    //dest_uninitialized = false;
-  }
-
-  // Results are placed here:
-  enum { fast_path        = 1,  // normal void-returning assembly stub
-         checked_path     = 2,  // special assembly stub with cleanup
-         slow_call_path   = 3,  // something went wrong; call the VM
-         zero_path        = 4,  // bypass when length of copy is zero
-         bcopy_path       = 5,  // copy primitive array by 64-bit blocks
-         PATH_LIMIT       = 6
-  };
-  RegionNode* result_region = new RegionNode(PATH_LIMIT);
-  PhiNode*    result_i_o    = new PhiNode(result_region, Type::ABIO);
-  PhiNode*    result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
-  record_for_igvn(result_region);
-  _gvn.set_type_bottom(result_i_o);
-  _gvn.set_type_bottom(result_memory);
-  assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
-
-  // The slow_control path:
-  Node* slow_control;
-  Node* slow_i_o = i_o();
-  Node* slow_mem = memory(adr_type);
-  debug_only(slow_control = (Node*) badAddress);
-
-  // Checked control path:
-  Node* checked_control = top();
-  Node* checked_mem     = NULL;
-  Node* checked_i_o     = NULL;
-  Node* checked_value   = NULL;
-
-  if (basic_elem_type == T_CONFLICT) {
-    assert(!dest_uninitialized, "");
-    Node* cv = generate_generic_arraycopy(adr_type,
-                                          src, src_offset, dest, dest_offset,
-                                          copy_length, dest_uninitialized);
-    if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
-    checked_control = control();
-    checked_i_o     = i_o();
-    checked_mem     = memory(adr_type);
-    checked_value   = cv;
-    set_control(top());         // no fast path
-  }
-
-  Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
-  if (not_pos != NULL) {
-    PreserveJVMState pjvms(this);
-    set_control(not_pos);
-
-    // (6) length must not be negative.
-    if (!length_never_negative) {
-      generate_negative_guard(copy_length, slow_region);
-    }
-
-    // copy_length is 0.
-    if (!stopped() && dest_uninitialized) {
-      Node* dest_length = alloc->in(AllocateNode::ALength);
-      if (copy_length->eqv_uncast(dest_length)
-          || _gvn.find_int_con(dest_length, 1) <= 0) {
-        // There is no zeroing to do. No need for a secondary raw memory barrier.
-      } else {
-        // Clear the whole thing since there are no source elements to copy.
-        generate_clear_array(adr_type, dest, basic_elem_type,
-                             intcon(0), NULL,
-                             alloc->in(AllocateNode::AllocSize));
-        // Use a secondary InitializeNode as raw memory barrier.
-        // Currently it is needed only on this path since other
-        // paths have stub or runtime calls as raw memory barriers.
-        InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
-                                                       Compile::AliasIdxRaw,
-                                                       top())->as_Initialize();
-        init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
-      }
-    }
-
-    // Present the results of the fast call.
-    result_region->init_req(zero_path, control());
-    result_i_o   ->init_req(zero_path, i_o());
-    result_memory->init_req(zero_path, memory(adr_type));
-  }
-
-  if (!stopped() && dest_uninitialized) {
-    // We have to initialize the *uncopied* part of the array to zero.
-    // The copy destination is the slice dest[off..off+len].  The other slices
-    // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
-    Node* dest_size   = alloc->in(AllocateNode::AllocSize);
-    Node* dest_length = alloc->in(AllocateNode::ALength);
-    Node* dest_tail   = _gvn.transform(new AddINode(dest_offset, copy_length));
-
-    // If there is a head section that needs zeroing, do it now.
-    if (find_int_con(dest_offset, -1) != 0) {
-      generate_clear_array(adr_type, dest, basic_elem_type,
-                           intcon(0), dest_offset,
-                           NULL);
-    }
-
-    // Next, perform a dynamic check on the tail length.
-    // It is often zero, and we can win big if we prove this.
-    // There are two wins:  Avoid generating the ClearArray
-    // with its attendant messy index arithmetic, and upgrade
-    // the copy to a more hardware-friendly word size of 64 bits.
-    Node* tail_ctl = NULL;
-    if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
-      Node* cmp_lt   = _gvn.transform(new CmpINode(dest_tail, dest_length));
-      Node* bol_lt   = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
-      tail_ctl = generate_slow_guard(bol_lt, NULL);
-      assert(tail_ctl != NULL || !stopped(), "must be an outcome");
-    }
-
-    // At this point, let's assume there is no tail.
-    if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
-      // There is no tail.  Try an upgrade to a 64-bit copy.
-      bool didit = false;
-      { PreserveJVMState pjvms(this);
-        didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
-                                         src, src_offset, dest, dest_offset,
-                                         dest_size, dest_uninitialized);
-        if (didit) {
-          // Present the results of the block-copying fast call.
-          result_region->init_req(bcopy_path, control());
-          result_i_o   ->init_req(bcopy_path, i_o());
-          result_memory->init_req(bcopy_path, memory(adr_type));
-        }
-      }
-      if (didit)
-        set_control(top());     // no regular fast path
-    }
-
-    // Clear the tail, if any.
-    if (tail_ctl != NULL) {
-      Node* notail_ctl = stopped() ? NULL : control();
-      set_control(tail_ctl);
-      if (notail_ctl == NULL) {
-        generate_clear_array(adr_type, dest, basic_elem_type,
-                             dest_tail, NULL,
-                             dest_size);
-      } else {
-        // Make a local merge.
-        Node* done_ctl = new RegionNode(3);
-        Node* done_mem = new PhiNode(done_ctl, Type::MEMORY, adr_type);
-        done_ctl->init_req(1, notail_ctl);
-        done_mem->init_req(1, memory(adr_type));
-        generate_clear_array(adr_type, dest, basic_elem_type,
-                             dest_tail, NULL,
-                             dest_size);
-        done_ctl->init_req(2, control());
-        done_mem->init_req(2, memory(adr_type));
-        set_control( _gvn.transform(done_ctl));
-        set_memory(  _gvn.transform(done_mem), adr_type );
-      }
-    }
-  }
-
-  BasicType copy_type = basic_elem_type;
-  assert(basic_elem_type != T_ARRAY, "caller must fix this");
-  if (!stopped() && copy_type == T_OBJECT) {
-    // If src and dest have compatible element types, we can copy bits.
-    // Types S[] and D[] are compatible if D is a supertype of S.
-    //
-    // If they are not, we will use checked_oop_disjoint_arraycopy,
-    // which performs a fast optimistic per-oop check, and backs off
-    // further to JVM_ArrayCopy on the first per-oop check that fails.
-    // (Actually, we don't move raw bits only; the GC requires card marks.)
-
-    // Get the Klass* for both src and dest
+
+    // (1) src and dest are arrays.
+    generate_non_array_guard(load_object_klass(src), slow_region);
+    generate_non_array_guard(load_object_klass(dest), slow_region);
+
+    // (2) src and dest arrays must have elements of the same BasicType
+    // done at macro expansion or at Ideal transformation time
+
+    // (4) src_offset must not be negative.
+    generate_negative_guard(src_offset, slow_region);
+
+    // (5) dest_offset must not be negative.
+    generate_negative_guard(dest_offset, slow_region);
+
+    // (7) src_offset + length must not exceed length of src.
+    generate_limit_guard(src_offset, length,
+                         load_array_length(src),
+                         slow_region);
+
+    // (8) dest_offset + length must not exceed length of dest.
+    generate_limit_guard(dest_offset, length,
+                         load_array_length(dest),
+                         slow_region);
+
+    // (9) each element of an oop array must be assignable
     Node* src_klass  = load_object_klass(src);
     Node* dest_klass = load_object_klass(dest);
-
-    // Generate the subtype check.
-    // This might fold up statically, or then again it might not.
-    //
-    // Non-static example:  Copying List<String>.elements to a new String[].
-    // The backing store for a List<String> is always an Object[],
-    // but its elements are always type String, if the generic types
-    // are correct at the source level.
-    //
-    // Test S[] against D[], not S against D, because (probably)
-    // the secondary supertype cache is less busy for S[] than S.
-    // This usually only matters when D is an interface.
     Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
-    // Plug failing path into checked_oop_disjoint_arraycopy
+
     if (not_subtype_ctrl != top()) {
       PreserveJVMState pjvms(this);
       set_control(not_subtype_ctrl);
-      // (At this point we can assume disjoint_bases, since types differ.)
-      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
-      Node* p1 = basic_plus_adr(dest_klass, ek_offset);
-      Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
-      Node* dest_elem_klass = _gvn.transform(n1);
-      Node* cv = generate_checkcast_arraycopy(adr_type,
-                                              dest_elem_klass,
-                                              src, src_offset, dest, dest_offset,
-                                              ConvI2X(copy_length), dest_uninitialized);
-      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
-      checked_control = control();
-      checked_i_o     = i_o();
-      checked_mem     = memory(adr_type);
-      checked_value   = cv;
+      uncommon_trap(Deoptimization::Reason_intrinsic,
+                    Deoptimization::Action_make_not_entrant);
+      assert(stopped(), "Should be stopped");
     }
-    // At this point we know we do not need type checks on oop stores.
-
-    // Let's see if we need card marks:
-    if (alloc != NULL && use_ReduceInitialCardMarks()) {
-      // If we do not need card marks, copy using the jint or jlong stub.
-      copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
-      assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
-             "sizes agree");
+    {
+      PreserveJVMState pjvms(this);
+      set_control(_gvn.transform(slow_region));
+      uncommon_trap(Deoptimization::Reason_intrinsic,
+                    Deoptimization::Action_make_not_entrant);
+      assert(stopped(), "Should be stopped");
     }
   }
 
-  if (!stopped()) {
-    // Generate the fast path, if possible.
-    PreserveJVMState pjvms(this);
-    generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
-                                 src, src_offset, dest, dest_offset,
-                                 ConvI2X(copy_length), dest_uninitialized);
-
-    // Present the results of the fast call.
-    result_region->init_req(fast_path, control());
-    result_i_o   ->init_req(fast_path, i_o());
-    result_memory->init_req(fast_path, memory(adr_type));
+  if (stopped()) {
+    return true;
   }
 
-  // Here are all the slow paths up to this point, in one bundle:
-  slow_control = top();
-  if (slow_region != NULL)
-    slow_control = _gvn.transform(slow_region);
-  DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
-
-  set_control(checked_control);
-  if (!stopped()) {
-    // Clean up after the checked call.
-    // The returned value is either 0 or -1^K,
-    // where K = number of partially transferred array elements.
-    Node* cmp = _gvn.transform(new CmpINode(checked_value, intcon(0)));
-    Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
-    IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
-
-    // If it is 0, we are done, so transfer to the end.
-    Node* checks_done = _gvn.transform(new IfTrueNode(iff));
-    result_region->init_req(checked_path, checks_done);
-    result_i_o   ->init_req(checked_path, checked_i_o);
-    result_memory->init_req(checked_path, checked_mem);
-
-    // If it is not zero, merge into the slow call.
-    set_control( _gvn.transform(new IfFalseNode(iff) ));
-    RegionNode* slow_reg2 = new RegionNode(3);
-    PhiNode*    slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
-    PhiNode*    slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
-    record_for_igvn(slow_reg2);
-    slow_reg2  ->init_req(1, slow_control);
-    slow_i_o2  ->init_req(1, slow_i_o);
-    slow_mem2  ->init_req(1, slow_mem);
-    slow_reg2  ->init_req(2, control());
-    slow_i_o2  ->init_req(2, checked_i_o);
-    slow_mem2  ->init_req(2, checked_mem);
-
-    slow_control = _gvn.transform(slow_reg2);
-    slow_i_o     = _gvn.transform(slow_i_o2);
-    slow_mem     = _gvn.transform(slow_mem2);
-
-    if (alloc != NULL) {
-      // We'll restart from the very beginning, after zeroing the whole thing.
-      // This can cause double writes, but that's OK since dest is brand new.
-      // So we ignore the low 31 bits of the value returned from the stub.
-    } else {
-      // We must continue the copy exactly where it failed, or else
-      // another thread might see the wrong number of writes to dest.
-      Node* checked_offset = _gvn.transform(new XorINode(checked_value, intcon(-1)));
-      Node* slow_offset    = new PhiNode(slow_reg2, TypeInt::INT);
-      slow_offset->init_req(1, intcon(0));
-      slow_offset->init_req(2, checked_offset);
-      slow_offset  = _gvn.transform(slow_offset);
-
-      // Adjust the arguments by the conditionally incoming offset.
-      Node* src_off_plus  = _gvn.transform(new AddINode(src_offset,  slow_offset));
-      Node* dest_off_plus = _gvn.transform(new AddINode(dest_offset, slow_offset));
-      Node* length_minus  = _gvn.transform(new SubINode(copy_length, slow_offset));
-
-      // Tweak the node variables to adjust the code produced below:
-      src_offset  = src_off_plus;
-      dest_offset = dest_off_plus;
-      copy_length = length_minus;
-    }
+  AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
+  ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL);
+
+  if (notest) {
+    ac->set_arraycopy_notest();
   }
 
-  set_control(slow_control);
-  if (!stopped()) {
-    // Generate the slow path, if needed.
-    PreserveJVMState pjvms(this);   // replace_in_map may trash the map
-
-    set_memory(slow_mem, adr_type);
-    set_i_o(slow_i_o);
-
-    if (dest_uninitialized) {
-      generate_clear_array(adr_type, dest, basic_elem_type,
-                           intcon(0), NULL,
-                           alloc->in(AllocateNode::AllocSize));
-    }
-
-    generate_slow_arraycopy(adr_type,
-                            src, src_offset, dest, dest_offset,
-                            copy_length, /*dest_uninitialized*/false);
-
-    result_region->init_req(slow_call_path, control());
-    result_i_o   ->init_req(slow_call_path, i_o());
-    result_memory->init_req(slow_call_path, memory(adr_type));
-  }
-
-  // Remove unused edges.
-  for (uint i = 1; i < result_region->req(); i++) {
-    if (result_region->in(i) == NULL)
-      result_region->init_req(i, top());
-  }
-
-  // Finished; return the combined state.
-  set_control( _gvn.transform(result_region));
-  set_i_o(     _gvn.transform(result_i_o)    );
-  set_memory(  _gvn.transform(result_memory), adr_type );
-
-  // The memory edges above are precise in order to model effects around
-  // array copies accurately to allow value numbering of field loads around
-  // arraycopy.  Such field loads, both before and after, are common in Java
-  // collections and similar classes involving header/array data structures.
-  //
-  // But with low number of register or when some registers are used or killed
-  // by arraycopy calls it causes registers spilling on stack. See 6544710.
-  // The next memory barrier is added to avoid it. If the arraycopy can be
-  // optimized away (which it can, sometimes) then we can manually remove
-  // the membar also.
-  //
-  // Do not let reads from the cloned object float above the arraycopy.
-  if (alloc != NULL) {
-    // Do not let stores that initialize this object be reordered with
-    // a subsequent store that would make this object accessible by
-    // other threads.
-    // Record what AllocateNode this StoreStore protects so that
-    // escape analysis can go from the MemBarStoreStoreNode to the
-    // AllocateNode and eliminate the MemBarStoreStoreNode if possible
-    // based on the escape status of the AllocateNode.
-    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
-  } else if (InsertMemBarAfterArraycopy)
-    insert_mem_bar(Op_MemBarCPUOrder);
+  Node* n = _gvn.transform(ac);
+  assert(n == ac, "cannot disappear");
+  ac->connect_outputs(this);
+
+  return true;
 }
 
 
@@ -5397,305 +4877,6 @@
   return alloc;
 }
 
-// Helper for initialization of arrays, creating a ClearArray.
-// It writes zero bits in [start..end), within the body of an array object.
-// The memory effects are all chained onto the 'adr_type' alias category.
-//
-// Since the object is otherwise uninitialized, we are free
-// to put a little "slop" around the edges of the cleared area,
-// as long as it does not go back into the array's header,
-// or beyond the array end within the heap.
-//
-// The lower edge can be rounded down to the nearest jint and the
-// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
-//
-// Arguments:
-//   adr_type           memory slice where writes are generated
-//   dest               oop of the destination array
-//   basic_elem_type    element type of the destination
-//   slice_idx          array index of first element to store
-//   slice_len          number of elements to store (or NULL)
-//   dest_size          total size in bytes of the array object
-//
-// Exactly one of slice_len or dest_size must be non-NULL.
-// If dest_size is non-NULL, zeroing extends to the end of the object.
-// If slice_len is non-NULL, the slice_idx value must be a constant.
-void
-LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
-                                     Node* dest,
-                                     BasicType basic_elem_type,
-                                     Node* slice_idx,
-                                     Node* slice_len,
-                                     Node* dest_size) {
-  // one or the other but not both of slice_len and dest_size:
-  assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
-  if (slice_len == NULL)  slice_len = top();
-  if (dest_size == NULL)  dest_size = top();
-
-  // operate on this memory slice:
-  Node* mem = memory(adr_type); // memory slice to operate on
-
-  // scaling and rounding of indexes:
-  int scale = exact_log2(type2aelembytes(basic_elem_type));
-  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
-  int clear_low = (-1 << scale) & (BytesPerInt  - 1);
-  int bump_bit  = (-1 << scale) & BytesPerInt;
-
-  // determine constant starts and ends
-  const intptr_t BIG_NEG = -128;
-  assert(BIG_NEG + 2*abase < 0, "neg enough");
-  intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
-  intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
-  if (slice_len_con == 0) {
-    return;                     // nothing to do here
-  }
-  intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
-  intptr_t end_con   = find_intptr_t_con(dest_size, -1);
-  if (slice_idx_con >= 0 && slice_len_con >= 0) {
-    assert(end_con < 0, "not two cons");
-    end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
-                       BytesPerLong);
-  }
-
-  if (start_con >= 0 && end_con >= 0) {
-    // Constant start and end.  Simple.
-    mem = ClearArrayNode::clear_memory(control(), mem, dest,
-                                       start_con, end_con, &_gvn);
-  } else if (start_con >= 0 && dest_size != top()) {
-    // Constant start, pre-rounded end after the tail of the array.
-    Node* end = dest_size;
-    mem = ClearArrayNode::clear_memory(control(), mem, dest,
-                                       start_con, end, &_gvn);
-  } else if (start_con >= 0 && slice_len != top()) {
-    // Constant start, non-constant end.  End needs rounding up.
-    // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
-    intptr_t end_base  = abase + (slice_idx_con << scale);
-    int      end_round = (-1 << scale) & (BytesPerLong  - 1);
-    Node*    end       = ConvI2X(slice_len);
-    if (scale != 0)
-      end = _gvn.transform(new LShiftXNode(end, intcon(scale) ));
-    end_base += end_round;
-    end = _gvn.transform(new AddXNode(end, MakeConX(end_base)));
-    end = _gvn.transform(new AndXNode(end, MakeConX(~end_round)));
-    mem = ClearArrayNode::clear_memory(control(), mem, dest,
-                                       start_con, end, &_gvn);
-  } else if (start_con < 0 && dest_size != top()) {
-    // Non-constant start, pre-rounded end after the tail of the array.
-    // This is almost certainly a "round-to-end" operation.
-    Node* start = slice_idx;
-    start = ConvI2X(start);
-    if (scale != 0)
-      start = _gvn.transform(new LShiftXNode( start, intcon(scale) ));
-    start = _gvn.transform(new AddXNode(start, MakeConX(abase)));
-    if ((bump_bit | clear_low) != 0) {
-      int to_clear = (bump_bit | clear_low);
-      // Align up mod 8, then store a jint zero unconditionally
-      // just before the mod-8 boundary.
-      if (((abase + bump_bit) & ~to_clear) - bump_bit
-          < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
-        bump_bit = 0;
-        assert((abase & to_clear) == 0, "array base must be long-aligned");
-      } else {
-        // Bump 'start' up to (or past) the next jint boundary:
-        start = _gvn.transform(new AddXNode(start, MakeConX(bump_bit)));
-        assert((abase & clear_low) == 0, "array base must be int-aligned");
-      }
-      // Round bumped 'start' down to jlong boundary in body of array.
-      start = _gvn.transform(new AndXNode(start, MakeConX(~to_clear)));
-      if (bump_bit != 0) {
-        // Store a zero to the immediately preceding jint:
-        Node* x1 = _gvn.transform(new AddXNode(start, MakeConX(-bump_bit)));
-        Node* p1 = basic_plus_adr(dest, x1);
-        mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
-        mem = _gvn.transform(mem);
-      }
-    }
-    Node* end = dest_size; // pre-rounded
-    mem = ClearArrayNode::clear_memory(control(), mem, dest,
-                                       start, end, &_gvn);
-  } else {
-    // Non-constant start, unrounded non-constant end.
-    // (Nobody zeroes a random midsection of an array using this routine.)
-    ShouldNotReachHere();       // fix caller
-  }
-
-  // Done.
-  set_memory(mem, adr_type);
-}
-
-
-bool
-LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
-                                         BasicType basic_elem_type,
-                                         AllocateNode* alloc,
-                                         Node* src,  Node* src_offset,
-                                         Node* dest, Node* dest_offset,
-                                         Node* dest_size, bool dest_uninitialized) {
-  // See if there is an advantage from block transfer.
-  int scale = exact_log2(type2aelembytes(basic_elem_type));
-  if (scale >= LogBytesPerLong)
-    return false;               // it is already a block transfer
-
-  // Look at the alignment of the starting offsets.
-  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
-
-  intptr_t src_off_con  = (intptr_t) find_int_con(src_offset, -1);
-  intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
-  if (src_off_con < 0 || dest_off_con < 0)
-    // At present, we can only understand constants.
-    return false;
-
-  intptr_t src_off  = abase + (src_off_con  << scale);
-  intptr_t dest_off = abase + (dest_off_con << scale);
-
-  if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
-    // Non-aligned; too bad.
-    // One more chance:  Pick off an initial 32-bit word.
-    // This is a common case, since abase can be odd mod 8.
-    if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
-        ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
-      Node* sptr = basic_plus_adr(src,  src_off);
-      Node* dptr = basic_plus_adr(dest, dest_off);
-      Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
-      store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
-      src_off += BytesPerInt;
-      dest_off += BytesPerInt;
-    } else {
-      return false;
-    }
-  }
-  assert(src_off % BytesPerLong == 0, "");
-  assert(dest_off % BytesPerLong == 0, "");
-
-  // Do this copy by giant steps.
-  Node* sptr  = basic_plus_adr(src,  src_off);
-  Node* dptr  = basic_plus_adr(dest, dest_off);
-  Node* countx = dest_size;
-  countx = _gvn.transform(new SubXNode(countx, MakeConX(dest_off)));
-  countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong)));
-
-  bool disjoint_bases = true;   // since alloc != NULL
-  generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
-                               sptr, NULL, dptr, NULL, countx, dest_uninitialized);
-
-  return true;
-}
-
-
-// Helper function; generates code for the slow case.
-// We make a call to a runtime method which emulates the native method,
-// but without the native wrapper overhead.
-void
-LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
-                                        Node* src,  Node* src_offset,
-                                        Node* dest, Node* dest_offset,
-                                        Node* copy_length, bool dest_uninitialized) {
-  assert(!dest_uninitialized, "Invariant");
-  Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
-                                 OptoRuntime::slow_arraycopy_Type(),
-                                 OptoRuntime::slow_arraycopy_Java(),
-                                 "slow_arraycopy", adr_type,
-                                 src, src_offset, dest, dest_offset,
-                                 copy_length);
-
-  // Handle exceptions thrown by this fellow:
-  make_slow_call_ex(call, env()->Throwable_klass(), false);
-}
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
-                                             Node* dest_elem_klass,
-                                             Node* src,  Node* src_offset,
-                                             Node* dest, Node* dest_offset,
-                                             Node* copy_length, bool dest_uninitialized) {
-  if (stopped())  return NULL;
-
-  address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
-  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
-    return NULL;
-  }
-
-  // Pick out the parameters required to perform a store-check
-  // for the target array.  This is an optimistic check.  It will
-  // look in each non-null element's class, at the desired klass's
-  // super_check_offset, for the desired klass.
-  int sco_offset = in_bytes(Klass::super_check_offset_offset());
-  Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
-  Node* n3 = new LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
-  Node* check_offset = ConvI2X(_gvn.transform(n3));
-  Node* check_value  = dest_elem_klass;
-
-  Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
-  Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
-
-  // (We know the arrays are never conjoint, because their types differ.)
-  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
-                                 OptoRuntime::checkcast_arraycopy_Type(),
-                                 copyfunc_addr, "checkcast_arraycopy", adr_type,
-                                 // five arguments, of which two are
-                                 // intptr_t (jlong in LP64)
-                                 src_start, dest_start,
-                                 copy_length XTOP,
-                                 check_offset XTOP,
-                                 check_value);
-
-  return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
-                                           Node* src,  Node* src_offset,
-                                           Node* dest, Node* dest_offset,
-                                           Node* copy_length, bool dest_uninitialized) {
-  assert(!dest_uninitialized, "Invariant");
-  if (stopped())  return NULL;
-  address copyfunc_addr = StubRoutines::generic_arraycopy();
-  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
-    return NULL;
-  }
-
-  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
-                    OptoRuntime::generic_arraycopy_Type(),
-                    copyfunc_addr, "generic_arraycopy", adr_type,
-                    src, src_offset, dest, dest_offset, copy_length);
-
-  return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-// Helper function; generates the fast out-of-line call to an arraycopy stub.
-void
-LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
-                                             BasicType basic_elem_type,
-                                             bool disjoint_bases,
-                                             Node* src,  Node* src_offset,
-                                             Node* dest, Node* dest_offset,
-                                             Node* copy_length, bool dest_uninitialized) {
-  if (stopped())  return;               // nothing to do
-
-  Node* src_start  = src;
-  Node* dest_start = dest;
-  if (src_offset != NULL || dest_offset != NULL) {
-    assert(src_offset != NULL && dest_offset != NULL, "");
-    src_start  = array_element_address(src,  src_offset,  basic_elem_type);
-    dest_start = array_element_address(dest, dest_offset, basic_elem_type);
-  }
-
-  // Figure out which arraycopy runtime method to call.
-  const char* copyfunc_name = "arraycopy";
-  address     copyfunc_addr =
-      basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
-                          disjoint_bases, copyfunc_name, dest_uninitialized);
-
-  // Call it.  Note that the count_ix value is not scaled to a byte-size.
-  make_runtime_call(RC_LEAF|RC_NO_FP,
-                    OptoRuntime::fast_arraycopy_Type(),
-                    copyfunc_addr, copyfunc_name, adr_type,
-                    src_start, dest_start, copy_length XTOP);
-}
-
 //-------------inline_encodeISOArray-----------------------------------
 // encode char[] to byte[] in ISO_8859_1
 bool LibraryCallKit::inline_encodeISOArray() {
--- a/src/share/vm/opto/loopPredicate.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/loopPredicate.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -763,9 +763,7 @@
         loop->dump_head();
       }
 #endif
-    } else if ((cl != NULL) && (proj->_con == predicate_proj->_con) &&
-               loop->is_range_check_if(iff, this, invar)) {
-
+    } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
       // Range check for counted loops
       const Node*    cmp    = bol->in(1)->as_Cmp();
       Node*          idx    = cmp->in(1);
@@ -800,18 +798,31 @@
       }
 
       // Test the lower bound
-      Node*  lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+      BoolNode*  lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+      // Negate test if necessary
+      bool negated = false;
+      if (proj->_con != predicate_proj->_con) {
+        lower_bound_bol = new BoolNode(lower_bound_bol->in(1), lower_bound_bol->_test.negate());
+        register_new_node(lower_bound_bol, ctrl);
+        negated = true;
+      }
       IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
       _igvn.hash_delete(lower_bound_iff);
       lower_bound_iff->set_req(1, lower_bound_bol);
-      if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
+      if (TraceLoopPredicate) tty->print_cr("lower bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
 
       // Test the upper bound
-      Node* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+      BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+      negated = false;
+      if (proj->_con != predicate_proj->_con) {
+        upper_bound_bol = new BoolNode(upper_bound_bol->in(1), upper_bound_bol->_test.negate());
+        register_new_node(upper_bound_bol, ctrl);
+        negated = true;
+      }
       IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
       _igvn.hash_delete(upper_bound_iff);
       upper_bound_iff->set_req(1, upper_bound_bol);
-      if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
+      if (TraceLoopPredicate) tty->print_cr("upper bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
 
       // Fall through into rest of the clean up code which will move
       // any dependent nodes onto the upper bound test.
--- a/src/share/vm/opto/macro.cpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/macro.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -2469,6 +2469,8 @@
         assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
         _has_locks = true;
         break;
+      case Node::Class_ArrayCopy:
+        break;
       default:
         assert(n->Opcode() == Op_LoopLimit ||
                n->Opcode() == Op_Opaque1   ||
@@ -2544,6 +2546,25 @@
     }
   }
 
+  // expand arraycopy "macro" nodes first
+  // For ReduceBulkZeroing, we must first process all arraycopy nodes
+  // before the allocate nodes are expanded.
+  int macro_idx = C->macro_count() - 1;
+  while (macro_idx >= 0) {
+    Node * n = C->macro_node(macro_idx);
+    assert(n->is_macro(), "only macro nodes expected here");
+    if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) {
+      // node is unreachable, so don't try to expand it
+      C->remove_macro_node(n);
+    } else if (n->is_ArrayCopy()){
+      int macro_count = C->macro_count();
+      expand_arraycopy_node(n->as_ArrayCopy());
+      assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+    }
+    if (C->failing())  return true;
+    macro_idx --;
+  }
+
   // expand "macro" nodes
   // nodes are removed from the macro list as they are processed
   while (C->macro_count() > 0) {
--- a/src/share/vm/opto/macro.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/macro.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -37,7 +37,7 @@
 private:
   PhaseIterGVN &_igvn;
 
-  // Helper methods roughly modelled after GraphKit:
+  // Helper methods roughly modeled after GraphKit:
   Node* top()                   const { return C->top(); }
   Node* intcon(jint con)        const { return _igvn.intcon(con); }
   Node* longcon(jlong con)      const { return _igvn.longcon(con); }
@@ -101,6 +101,86 @@
   void expand_lock_node(LockNode *lock);
   void expand_unlock_node(UnlockNode *unlock);
 
+  // More helper methods modeled after GraphKit for array copy
+  void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL);
+  Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
+  Node* ConvI2L(Node* offset);
+  Node* make_leaf_call(Node* ctrl, Node* mem,
+                       const TypeFunc* call_type, address call_addr,
+                       const char* call_name,
+                       const TypePtr* adr_type,
+                       Node* parm0 = NULL, Node* parm1 = NULL,
+                       Node* parm2 = NULL, Node* parm3 = NULL,
+                       Node* parm4 = NULL, Node* parm5 = NULL,
+                       Node* parm6 = NULL, Node* parm7 = NULL);
+
+  // helper methods modeled after LibraryCallKit for array copy
+  Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
+  Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
+  void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
+  void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
+
+  // More helper methods for array copy
+  Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
+  void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
+  address basictype2arraycopy(BasicType t,
+                              Node* src_offset,
+                              Node* dest_offset,
+                              bool disjoint_bases,
+                              const char* &name,
+                              bool dest_uninitialized);
+  Node* generate_arraycopy(ArrayCopyNode *ac,
+                           AllocateArrayNode* alloc,
+                           Node** ctrl, MergeMemNode* mem, Node** io,
+                           const TypePtr* adr_type,
+                           BasicType basic_elem_type,
+                           Node* src,  Node* src_offset,
+                           Node* dest, Node* dest_offset,
+                           Node* copy_length,
+                           bool disjoint_bases = false,
+                           bool length_never_negative = false,
+                           RegionNode* slow_region = NULL);
+  void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+                            const TypePtr* adr_type,
+                            Node* dest,
+                            BasicType basic_elem_type,
+                            Node* slice_idx,
+                            Node* slice_len,
+                            Node* dest_size);
+  bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+                                const TypePtr* adr_type,
+                                BasicType basic_elem_type,
+                                AllocateNode* alloc,
+                                Node* src,  Node* src_offset,
+                                Node* dest, Node* dest_offset,
+                                Node* dest_size, bool dest_uninitialized);
+  MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
+                                        Node** ctrl, Node* mem, Node** io,
+                                        const TypePtr* adr_type,
+                                        Node* src,  Node* src_offset,
+                                        Node* dest, Node* dest_offset,
+                                        Node* copy_length, bool dest_uninitialized);
+  Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                     const TypePtr* adr_type,
+                                     Node* dest_elem_klass,
+                                     Node* src,  Node* src_offset,
+                                     Node* dest, Node* dest_offset,
+                                     Node* copy_length, bool dest_uninitialized);
+  Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                   const TypePtr* adr_type,
+                                   Node* src,  Node* src_offset,
+                                   Node* dest, Node* dest_offset,
+                                   Node* copy_length, bool dest_uninitialized);
+  void generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                    const TypePtr* adr_type,
+                                    BasicType basic_elem_type,
+                                    bool disjoint_bases,
+                                    Node* src,  Node* src_offset,
+                                    Node* dest, Node* dest_offset,
+                                    Node* copy_length, bool dest_uninitialized);
+
+  void expand_arraycopy_node(ArrayCopyNode *ac);
+
   int replace_input(Node *use, Node *oldref, Node *newref);
   void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
   Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/opto/macroArrayCopy.cpp	Mon Aug 11 14:12:51 2014 +0200
@@ -0,0 +1,1245 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "opto/convertnode.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/macro.hpp"
+#include "opto/runtime.hpp"
+
+
+void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
+  MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
+  mb->init_req(TypeFunc::Control, *ctrl);
+  mb->init_req(TypeFunc::Memory, *mem);
+  transform_later(mb);
+  *ctrl = new ProjNode(mb,TypeFunc::Control);
+  transform_later(*ctrl);
+  Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
+  transform_later(mem_proj);
+  *mem = mem_proj;
+}
+
+Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
+  uint shift  = exact_log2(type2aelembytes(elembt));
+  uint header = arrayOopDesc::base_offset_in_bytes(elembt);
+  Node* base =  basic_plus_adr(ary, header);
+#ifdef _LP64
+  // see comment in GraphKit::array_element_address
+  int index_max = max_jint - 1;  // array size is max_jint, index is one less
+  const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
+  idx = transform_later( new ConvI2LNode(idx, lidxtype) );
+#endif
+  Node* scale = new LShiftXNode(idx, intcon(shift));
+  transform_later(scale);
+  return basic_plus_adr(ary, base, scale);
+}
+
+Node* PhaseMacroExpand::ConvI2L(Node* offset) {
+  return transform_later(new ConvI2LNode(offset));
+}
+
+Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem,
+                                       const TypeFunc* call_type, address call_addr,
+                                       const char* call_name,
+                                       const TypePtr* adr_type,
+                                       Node* parm0, Node* parm1,
+                                       Node* parm2, Node* parm3,
+                                       Node* parm4, Node* parm5,
+                                       Node* parm6, Node* parm7) {
+  int size = call_type->domain()->cnt();
+  Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
+  call->init_req(TypeFunc::Control, ctrl);
+  call->init_req(TypeFunc::I_O    , top());
+  call->init_req(TypeFunc::Memory , mem);
+  call->init_req(TypeFunc::ReturnAdr, top());
+  call->init_req(TypeFunc::FramePtr, top());
+
+  // Hook each parm in order.  Stop looking at the first NULL.
+  if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0);
+  if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1);
+  if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2);
+  if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3);
+  if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4);
+  if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5);
+  if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6);
+  if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7);
+    /* close each nested if ===> */  } } } } } } } }
+  assert(call->in(call->req()-1) != NULL, "must initialize all parms");
+
+  return call;
+}
+
+
+//------------------------------generate_guard---------------------------
+// Helper function for generating guarded fast-slow graph structures.
+// The given 'test', if true, guards a slow path.  If the test fails
+// then a fast path can be taken.  (We generally hope it fails.)
+// In all cases, GraphKit::control() is updated to the fast path.
+// The returned value represents the control for the slow path.
+// The return value is never 'top'; it is either a valid control
+// or NULL if it is obvious that the slow path can never be taken.
+// Also, if region and the slow control are not NULL, the slow edge
+// is appended to the region.
+Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) {
+  if ((*ctrl)->is_top()) {
+    // Already short circuited.
+    return NULL;
+  }
+  // Build an if node and its projections.
+  // If test is true we take the slow path, which we assume is uncommon.
+  if (_igvn.type(test) == TypeInt::ZERO) {
+    // The slow branch is never taken.  No need to build this guard.
+    return NULL;
+  }
+
+  IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN);
+  transform_later(iff);
+
+  Node* if_slow = new IfTrueNode(iff);
+  transform_later(if_slow);
+
+  if (region != NULL) {
+    region->add_req(if_slow);
+  }
+
+  Node* if_fast = new IfFalseNode(iff);
+  transform_later(if_fast);
+
+  *ctrl = if_fast;
+
+  return if_slow;
+}
+
+inline Node* PhaseMacroExpand::generate_slow_guard(Node** ctrl, Node* test, RegionNode* region) {
+  return generate_guard(ctrl, test, region, PROB_UNLIKELY_MAG(3));
+}
+
+void PhaseMacroExpand::generate_negative_guard(Node** ctrl, Node* index, RegionNode* region) {
+  if ((*ctrl)->is_top())
+    return;                // already stopped
+  if (_igvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
+    return;                // index is already adequately typed
+  Node* cmp_lt = new CmpINode(index, intcon(0));
+  transform_later(cmp_lt);
+  Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+  transform_later(bol_lt);
+  generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+void PhaseMacroExpand::generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) {
+  if ((*ctrl)->is_top())
+    return;                // already stopped
+  bool zero_offset = _igvn.type(offset) == TypeInt::ZERO;
+  if (zero_offset && subseq_length->eqv_uncast(array_length))
+    return;                // common case of whole-array copy
+  Node* last = subseq_length;
+  if (!zero_offset) {            // last += offset
+    last = new AddINode(last, offset);
+    transform_later(last);
+  }
+  Node* cmp_lt = new CmpUNode(array_length, last);
+  transform_later(cmp_lt);
+  Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+  transform_later(bol_lt);
+  generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
+  if ((*ctrl)->is_top())  return NULL;
+
+  if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
+    return NULL;                // index is already adequately typed
+  Node* cmp_le = new CmpINode(index, intcon(0));
+  transform_later(cmp_le);
+  BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
+  Node* bol_le = new BoolNode(cmp_le, le_or_eq);
+  transform_later(bol_le);
+  Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
+
+  return is_notp;
+}
+
+void PhaseMacroExpand::finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type) {
+  transform_later(call);
+
+  *ctrl = new ProjNode(call,TypeFunc::Control);
+  transform_later(*ctrl);
+  Node* newmem = new ProjNode(call, TypeFunc::Memory);
+  transform_later(newmem);
+
+  uint alias_idx = C->get_alias_index(adr_type);
+  if (alias_idx != Compile::AliasIdxBot) {
+    *mem = MergeMemNode::make(*mem);
+    (*mem)->set_memory_at(alias_idx, newmem);
+  } else {
+    *mem = MergeMemNode::make(newmem);
+  }
+  transform_later(*mem);
+}
+
+address PhaseMacroExpand::basictype2arraycopy(BasicType t,
+                                              Node* src_offset,
+                                              Node* dest_offset,
+                                              bool disjoint_bases,
+                                              const char* &name,
+                                              bool dest_uninitialized) {
+  const TypeInt* src_offset_inttype  = _igvn.find_int_type(src_offset);;
+  const TypeInt* dest_offset_inttype = _igvn.find_int_type(dest_offset);;
+
+  bool aligned = false;
+  bool disjoint = disjoint_bases;
+
+  // if the offsets are the same, we can treat the memory regions as
+  // disjoint, because either the memory regions are in different arrays,
+  // or they are identical (which we can treat as disjoint.)  We can also
+  // treat a copy with a destination index  less that the source index
+  // as disjoint since a low->high copy will work correctly in this case.
+  if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
+      dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
+    // both indices are constants
+    int s_offs = src_offset_inttype->get_con();
+    int d_offs = dest_offset_inttype->get_con();
+    int element_size = type2aelembytes(t);
+    aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
+              ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
+    if (s_offs >= d_offs)  disjoint = true;
+  } else if (src_offset == dest_offset && src_offset != NULL) {
+    // This can occur if the offsets are identical non-constants.
+    disjoint = true;
+  }
+
+  return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
+}
+
+#define COMMA ,
+#define XTOP LP64_ONLY(COMMA top())
+
+// Generate an optimized call to arraycopy.
+// Caller must guard against non-arrays.
+// Caller must determine a common array basic-type for both arrays.
+// Caller must validate offsets against array bounds.
+// The slow_region has already collected guard failure paths
+// (such as out of bounds length or non-conformable array types).
+// The generated code has this shape, in general:
+//
+//     if (length == 0)  return   // via zero_path
+//     slowval = -1
+//     if (types unknown) {
+//       slowval = call generic copy loop
+//       if (slowval == 0)  return  // via checked_path
+//     } else if (indexes in bounds) {
+//       if ((is object array) && !(array type check)) {
+//         slowval = call checked copy loop
+//         if (slowval == 0)  return  // via checked_path
+//       } else {
+//         call bulk copy loop
+//         return  // via fast_path
+//       }
+//     }
+//     // adjust params for remaining work:
+//     if (slowval != -1) {
+//       n = -1^slowval; src_offset += n; dest_offset += n; length -= n
+//     }
+//   slow_region:
+//     call slow arraycopy(src, src_offset, dest, dest_offset, length)
+//     return  // via slow_call_path
+//
+// This routine is used from several intrinsics:  System.arraycopy,
+// Object.clone (the array subcase), and Arrays.copyOf[Range].
+//
+Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* alloc,
+                                           Node** ctrl, MergeMemNode* mem, Node** io,
+                                           const TypePtr* adr_type,
+                                           BasicType basic_elem_type,
+                                           Node* src,  Node* src_offset,
+                                           Node* dest, Node* dest_offset,
+                                           Node* copy_length,
+                                           bool disjoint_bases,
+                                           bool length_never_negative,
+                                           RegionNode* slow_region) {
+  if (slow_region == NULL) {
+    slow_region = new RegionNode(1);
+    transform_later(slow_region);
+  }
+
+  Node* original_dest      = dest;
+  bool  dest_uninitialized = false;
+
+  // See if this is the initialization of a newly-allocated array.
+  // If so, we will take responsibility here for initializing it to zero.
+  // (Note:  Because tightly_coupled_allocation performs checks on the
+  // out-edges of the dest, we need to avoid making derived pointers
+  // from it until we have checked its uses.)
+  if (ReduceBulkZeroing
+      && !ZeroTLAB              // pointless if already zeroed
+      && basic_elem_type != T_CONFLICT // avoid corner case
+      && !src->eqv_uncast(dest)
+      && alloc != NULL
+      && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
+      && alloc->maybe_set_complete(&_igvn)) {
+    // "You break it, you buy it."
+    InitializeNode* init = alloc->initialization();
+    assert(init->is_complete(), "we just did this");
+    init->set_complete_with_arraycopy();
+    assert(dest->is_CheckCastPP(), "sanity");
+    assert(dest->in(0)->in(0) == init, "dest pinned");
+    adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
+    // From this point on, every exit path is responsible for
+    // initializing any non-copied parts of the object to zero.
+    // Also, if this flag is set we make sure that arraycopy interacts properly
+    // with G1, eliding pre-barriers. See CR 6627983.
+    dest_uninitialized = true;
+  } else {
+    // No zeroing elimination here.
+    alloc             = NULL;
+    //original_dest   = dest;
+    //dest_uninitialized = false;
+  }
+
+  uint alias_idx = C->get_alias_index(adr_type);
+
+  // Results are placed here:
+  enum { fast_path        = 1,  // normal void-returning assembly stub
+         checked_path     = 2,  // special assembly stub with cleanup
+         slow_call_path   = 3,  // something went wrong; call the VM
+         zero_path        = 4,  // bypass when length of copy is zero
+         bcopy_path       = 5,  // copy primitive array by 64-bit blocks
+         PATH_LIMIT       = 6
+  };
+  RegionNode* result_region = new RegionNode(PATH_LIMIT);
+  PhiNode*    result_i_o    = new PhiNode(result_region, Type::ABIO);
+  PhiNode*    result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
+  assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
+  transform_later(result_region);
+  transform_later(result_i_o);
+  transform_later(result_memory);
+
+  // The slow_control path:
+  Node* slow_control;
+  Node* slow_i_o = *io;
+  Node* slow_mem = mem->memory_at(alias_idx);
+  DEBUG_ONLY(slow_control = (Node*) badAddress);
+
+  // Checked control path:
+  Node* checked_control = top();
+  Node* checked_mem     = NULL;
+  Node* checked_i_o     = NULL;
+  Node* checked_value   = NULL;
+
+  if (basic_elem_type == T_CONFLICT) {
+    assert(!dest_uninitialized, "");
+    Node* cv = generate_generic_arraycopy(ctrl, &mem,
+                                          adr_type,
+                                          src, src_offset, dest, dest_offset,
+                                          copy_length, dest_uninitialized);
+    if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
+    checked_control = *ctrl;
+    checked_i_o     = *io;
+    checked_mem     = mem->memory_at(alias_idx);
+    checked_value   = cv;
+    *ctrl = top();
+  }
+
+  Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative);
+  if (not_pos != NULL) {
+    Node* local_ctrl = not_pos, *local_io = *io;
+    MergeMemNode* local_mem = MergeMemNode::make(mem);
+    transform_later(local_mem);
+
+    // (6) length must not be negative.
+    if (!length_never_negative) {
+      generate_negative_guard(&local_ctrl, copy_length, slow_region);
+    }
+
+    // copy_length is 0.
+    if (dest_uninitialized) {
+      assert(!local_ctrl->is_top(), "no ctrl?");
+      Node* dest_length = alloc->in(AllocateNode::ALength);
+      if (copy_length->eqv_uncast(dest_length)
+          || _igvn.find_int_con(dest_length, 1) <= 0) {
+        // There is no zeroing to do. No need for a secondary raw memory barrier.
+      } else {
+        // Clear the whole thing since there are no source elements to copy.
+        generate_clear_array(local_ctrl, local_mem,
+                             adr_type, dest, basic_elem_type,
+                             intcon(0), NULL,
+                             alloc->in(AllocateNode::AllocSize));
+        // Use a secondary InitializeNode as raw memory barrier.
+        // Currently it is needed only on this path since other
+        // paths have stub or runtime calls as raw memory barriers.
+        MemBarNode* mb = MemBarNode::make(C, Op_Initialize,
+                                          Compile::AliasIdxRaw,
+                                          top());
+        transform_later(mb);
+        mb->set_req(TypeFunc::Control,local_ctrl);
+        mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
+        local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
+        local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
+
+        InitializeNode* init = mb->as_Initialize();
+        init->set_complete(&_igvn);  // (there is no corresponding AllocateNode)
+      }
+    }
+
+    // Present the results of the fast call.
+    result_region->init_req(zero_path, local_ctrl);
+    result_i_o   ->init_req(zero_path, local_io);
+    result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
+  }
+
+  if (!(*ctrl)->is_top() && dest_uninitialized) {
+    // We have to initialize the *uncopied* part of the array to zero.
+    // The copy destination is the slice dest[off..off+len].  The other slices
+    // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
+    Node* dest_size   = alloc->in(AllocateNode::AllocSize);
+    Node* dest_length = alloc->in(AllocateNode::ALength);
+    Node* dest_tail   = transform_later( new AddINode(dest_offset, copy_length));
+
+    // If there is a head section that needs zeroing, do it now.
+    if (_igvn.find_int_con(dest_offset, -1) != 0) {
+      generate_clear_array(*ctrl, mem,
+                           adr_type, dest, basic_elem_type,
+                           intcon(0), dest_offset,
+                           NULL);
+    }
+
+    // Next, perform a dynamic check on the tail length.
+    // It is often zero, and we can win big if we prove this.
+    // There are two wins:  Avoid generating the ClearArray
+    // with its attendant messy index arithmetic, and upgrade
+    // the copy to a more hardware-friendly word size of 64 bits.
+    Node* tail_ctl = NULL;
+    if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) {
+      Node* cmp_lt   = transform_later( new CmpINode(dest_tail, dest_length) );
+      Node* bol_lt   = transform_later( new BoolNode(cmp_lt, BoolTest::lt) );
+      tail_ctl = generate_slow_guard(ctrl, bol_lt, NULL);
+      assert(tail_ctl != NULL || !(*ctrl)->is_top(), "must be an outcome");
+    }
+
+    // At this point, let's assume there is no tail.
+    if (!(*ctrl)->is_top() && alloc != NULL && basic_elem_type != T_OBJECT) {
+      // There is no tail.  Try an upgrade to a 64-bit copy.
+      bool didit = false;
+      {
+        Node* local_ctrl = *ctrl, *local_io = *io;
+        MergeMemNode* local_mem = MergeMemNode::make(mem);
+        transform_later(local_mem);
+
+        didit = generate_block_arraycopy(&local_ctrl, &local_mem, local_io,
+                                         adr_type, basic_elem_type, alloc,
+                                         src, src_offset, dest, dest_offset,
+                                         dest_size, dest_uninitialized);
+        if (didit) {
+          // Present the results of the block-copying fast call.
+          result_region->init_req(bcopy_path, local_ctrl);
+          result_i_o   ->init_req(bcopy_path, local_io);
+          result_memory->init_req(bcopy_path, local_mem->memory_at(alias_idx));
+        }
+      }
+      if (didit) {
+        *ctrl = top();     // no regular fast path
+      }
+    }
+
+    // Clear the tail, if any.
+    if (tail_ctl != NULL) {
+      Node* notail_ctl = (*ctrl)->is_top() ? NULL : *ctrl;
+      *ctrl = tail_ctl;
+      if (notail_ctl == NULL) {
+        generate_clear_array(*ctrl, mem,
+                             adr_type, dest, basic_elem_type,
+                             dest_tail, NULL,
+                             dest_size);
+      } else {
+        // Make a local merge.
+        Node* done_ctl = transform_later(new RegionNode(3));
+        Node* done_mem = transform_later(new PhiNode(done_ctl, Type::MEMORY, adr_type));
+        done_ctl->init_req(1, notail_ctl);
+        done_mem->init_req(1, mem->memory_at(alias_idx));
+        generate_clear_array(*ctrl, mem,
+                             adr_type, dest, basic_elem_type,
+                             dest_tail, NULL,
+                             dest_size);
+        done_ctl->init_req(2, *ctrl);
+        done_mem->init_req(2, mem->memory_at(alias_idx));
+        *ctrl = done_ctl;
+        mem->set_memory_at(alias_idx, done_mem);
+      }
+    }
+  }
+
+  BasicType copy_type = basic_elem_type;
+  assert(basic_elem_type != T_ARRAY, "caller must fix this");
+  if (!(*ctrl)->is_top() && copy_type == T_OBJECT) {
+    // If src and dest have compatible element types, we can copy bits.
+    // Types S[] and D[] are compatible if D is a supertype of S.
+    //
+    // If they are not, we will use checked_oop_disjoint_arraycopy,
+    // which performs a fast optimistic per-oop check, and backs off
+    // further to JVM_ArrayCopy on the first per-oop check that fails.
+    // (Actually, we don't move raw bits only; the GC requires card marks.)
+
+    // Get the klass* for both src and dest
+    Node* k_adr =  new AddPNode(src, src, MakeConX(oopDesc::klass_offset_in_bytes()));
+    transform_later(k_adr);
+    Node* src_klass  = LoadKlassNode::make(_igvn, C->immutable_memory(), k_adr, TypeInstPtr::KLASS);
+    transform_later(src_klass);
+    k_adr =  new AddPNode(dest, dest, MakeConX(oopDesc::klass_offset_in_bytes()));
+    transform_later(k_adr);
+    Node* dest_klass  = LoadKlassNode::make(_igvn, C->immutable_memory(), k_adr, TypeInstPtr::KLASS);
+    transform_later(dest_klass);
+
+    // Generate the subtype check.
+    // This might fold up statically, or then again it might not.
+    //
+    // Non-static example:  Copying List<String>.elements to a new String[].
+    // The backing store for a List<String> is always an Object[],
+    // but its elements are always type String, if the generic types
+    // are correct at the source level.
+    //
+    // Test S[] against D[], not S against D, because (probably)
+    // the secondary supertype cache is less busy for S[] than S.
+    // This usually only matters when D is an interface.
+    Node* not_subtype_ctrl = ac->is_arraycopy_notest() ? top() : Phase::gen_subtype_check(src_klass, dest_klass, ctrl, mem, &_igvn);
+    // Plug failing path into checked_oop_disjoint_arraycopy
+    if (not_subtype_ctrl != top()) {
+      Node* local_ctrl = not_subtype_ctrl;
+      MergeMemNode* local_mem = MergeMemNode::make(mem);
+      transform_later(local_mem);
+
+      // (At this point we can assume disjoint_bases, since types differ.)
+      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
+      Node* p1 = basic_plus_adr(dest_klass, ek_offset);
+      Node* n1 = LoadKlassNode::make(_igvn, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
+      Node* dest_elem_klass = transform_later(n1);
+      Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
+                                              adr_type,
+                                              dest_elem_klass,
+                                              src, src_offset, dest, dest_offset,
+                                              ConvI2X(copy_length), dest_uninitialized);
+      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
+      checked_control = local_ctrl;
+      checked_i_o     = *io;
+      checked_mem     = local_mem->memory_at(alias_idx);
+      checked_value   = cv;
+    }
+    // At this point we know we do not need type checks on oop stores.
+
+    // Let's see if we need card marks:
+    if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks()) {
+      // If we do not need card marks, copy using the jint or jlong stub.
+      copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
+      assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
+             "sizes agree");
+    }
+  }
+
+  if (!(*ctrl)->is_top()) {
+    // Generate the fast path, if possible.
+    Node* local_ctrl = *ctrl;
+    MergeMemNode* local_mem = MergeMemNode::make(mem);
+    transform_later(local_mem);
+
+    generate_unchecked_arraycopy(&local_ctrl, &local_mem,
+                                 adr_type, copy_type, disjoint_bases,
+                                 src, src_offset, dest, dest_offset,
+                                 ConvI2X(copy_length), dest_uninitialized);
+
+    // Present the results of the fast call.
+    result_region->init_req(fast_path, local_ctrl);
+    result_i_o   ->init_req(fast_path, *io);
+    result_memory->init_req(fast_path, local_mem->memory_at(alias_idx));
+  }
+
+  // Here are all the slow paths up to this point, in one bundle:
+  assert(slow_region != NULL, "allocated on entry");
+  slow_control = slow_region;
+  DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
+
+  *ctrl = checked_control;
+  if (!(*ctrl)->is_top()) {
+    // Clean up after the checked call.
+    // The returned value is either 0 or -1^K,
+    // where K = number of partially transferred array elements.
+    Node* cmp = new CmpINode(checked_value, intcon(0));
+    transform_later(cmp);
+    Node* bol = new BoolNode(cmp, BoolTest::eq);
+    transform_later(bol);
+    IfNode* iff = new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN);
+    transform_later(iff);
+
+    // If it is 0, we are done, so transfer to the end.
+    Node* checks_done = new IfTrueNode(iff);
+    transform_later(checks_done);
+    result_region->init_req(checked_path, checks_done);
+    result_i_o   ->init_req(checked_path, checked_i_o);
+    result_memory->init_req(checked_path, checked_mem);
+
+    // If it is not zero, merge into the slow call.
+    *ctrl = new IfFalseNode(iff);
+    transform_later(*ctrl);
+    RegionNode* slow_reg2 = new RegionNode(3);
+    PhiNode*    slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
+    PhiNode*    slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
+    transform_later(slow_reg2);
+    transform_later(slow_i_o2);
+    transform_later(slow_mem2);
+    slow_reg2  ->init_req(1, slow_control);
+    slow_i_o2  ->init_req(1, slow_i_o);
+    slow_mem2  ->init_req(1, slow_mem);
+    slow_reg2  ->init_req(2, *ctrl);
+    slow_i_o2  ->init_req(2, checked_i_o);
+    slow_mem2  ->init_req(2, checked_mem);
+
+    slow_control = slow_reg2;
+    slow_i_o     = slow_i_o2;
+    slow_mem     = slow_mem2;
+
+    if (alloc != NULL) {
+      // We'll restart from the very beginning, after zeroing the whole thing.
+      // This can cause double writes, but that's OK since dest is brand new.
+      // So we ignore the low 31 bits of the value returned from the stub.
+    } else {
+      // We must continue the copy exactly where it failed, or else
+      // another thread might see the wrong number of writes to dest.
+      Node* checked_offset = new XorINode(checked_value, intcon(-1));
+      Node* slow_offset    = new PhiNode(slow_reg2, TypeInt::INT);
+      transform_later(checked_offset);
+      transform_later(slow_offset);
+      slow_offset->init_req(1, intcon(0));
+      slow_offset->init_req(2, checked_offset);
+
+      // Adjust the arguments by the conditionally incoming offset.
+      Node* src_off_plus  = new AddINode(src_offset,  slow_offset);
+      transform_later(src_off_plus);
+      Node* dest_off_plus = new AddINode(dest_offset, slow_offset);
+      transform_later(dest_off_plus);
+      Node* length_minus  = new SubINode(copy_length, slow_offset);
+      transform_later(length_minus);
+
+      // Tweak the node variables to adjust the code produced below:
+      src_offset  = src_off_plus;
+      dest_offset = dest_off_plus;
+      copy_length = length_minus;
+    }
+  }
+  *ctrl = slow_control;
+  if (!(*ctrl)->is_top()) {
+    Node* local_ctrl = *ctrl, *local_io = slow_i_o;
+    MergeMemNode* local_mem = MergeMemNode::make(mem);
+    transform_later(local_mem);
+
+    // Generate the slow path, if needed.
+    local_mem->set_memory_at(alias_idx, slow_mem);
+
+    if (dest_uninitialized) {
+      generate_clear_array(local_ctrl, local_mem,
+                           adr_type, dest, basic_elem_type,
+                           intcon(0), NULL,
+                           alloc->in(AllocateNode::AllocSize));
+    }
+
+    local_mem = generate_slow_arraycopy(ac,
+                                        &local_ctrl, local_mem, &local_io,
+                                        adr_type,
+                                        src, src_offset, dest, dest_offset,
+                                        copy_length, /*dest_uninitialized*/false);
+
+    result_region->init_req(slow_call_path, local_ctrl);
+    result_i_o   ->init_req(slow_call_path, local_io);
+    result_memory->init_req(slow_call_path, local_mem->memory_at(alias_idx));
+  } else {
+    ShouldNotReachHere(); // no call to generate_slow_arraycopy:
+                          // projections were not extracted
+  }
+
+  // Remove unused edges.
+  for (uint i = 1; i < result_region->req(); i++) {
+    if (result_region->in(i) == NULL) {
+      result_region->init_req(i, top());
+    }
+  }
+
+  // Finished; return the combined state.
+  *ctrl = result_region;
+  *io = result_i_o;
+  mem->set_memory_at(alias_idx, result_memory);
+
+  // mem no longer guaranteed to stay a MergeMemNode
+  Node* out_mem = mem;
+  DEBUG_ONLY(mem = NULL);
+
+  // The memory edges above are precise in order to model effects around
+  // array copies accurately to allow value numbering of field loads around
+  // arraycopy.  Such field loads, both before and after, are common in Java
+  // collections and similar classes involving header/array data structures.
+  //
+  // But with low number of register or when some registers are used or killed
+  // by arraycopy calls it causes registers spilling on stack. See 6544710.
+  // The next memory barrier is added to avoid it. If the arraycopy can be
+  // optimized away (which it can, sometimes) then we can manually remove
+  // the membar also.
+  //
+  // Do not let reads from the cloned object float above the arraycopy.
+  if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
+    // Do not let stores that initialize this object be reordered with
+    // a subsequent store that would make this object accessible by
+    // other threads.
+    insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore);
+  } else if (InsertMemBarAfterArraycopy) {
+    insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder);
+  }
+
+  _igvn.replace_node(_memproj_fallthrough, out_mem);
+  _igvn.replace_node(_ioproj_fallthrough, *io);
+  _igvn.replace_node(_fallthroughcatchproj, *ctrl);
+
+  return out_mem;
+}
+
+// Helper for initialization of arrays, creating a ClearArray.
+// It writes zero bits in [start..end), within the body of an array object.
+// The memory effects are all chained onto the 'adr_type' alias category.
+//
+// Since the object is otherwise uninitialized, we are free
+// to put a little "slop" around the edges of the cleared area,
+// as long as it does not go back into the array's header,
+// or beyond the array end within the heap.
+//
+// The lower edge can be rounded down to the nearest jint and the
+// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
+//
+// Arguments:
+//   adr_type           memory slice where writes are generated
+//   dest               oop of the destination array
+//   basic_elem_type    element type of the destination
+//   slice_idx          array index of first element to store
+//   slice_len          number of elements to store (or NULL)
+//   dest_size          total size in bytes of the array object
+//
+// Exactly one of slice_len or dest_size must be non-NULL.
+// If dest_size is non-NULL, zeroing extends to the end of the object.
+// If slice_len is non-NULL, the slice_idx value must be a constant.
+void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+                                            const TypePtr* adr_type,
+                                            Node* dest,
+                                            BasicType basic_elem_type,
+                                            Node* slice_idx,
+                                            Node* slice_len,
+                                            Node* dest_size) {
+  // one or the other but not both of slice_len and dest_size:
+  assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
+  if (slice_len == NULL)  slice_len = top();
+  if (dest_size == NULL)  dest_size = top();
+
+  uint alias_idx = C->get_alias_index(adr_type);
+
+  // operate on this memory slice:
+  Node* mem = merge_mem->memory_at(alias_idx); // memory slice to operate on
+
+  // scaling and rounding of indexes:
+  int scale = exact_log2(type2aelembytes(basic_elem_type));
+  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+  int clear_low = (-1 << scale) & (BytesPerInt  - 1);
+  int bump_bit  = (-1 << scale) & BytesPerInt;
+
+  // determine constant starts and ends
+  const intptr_t BIG_NEG = -128;
+  assert(BIG_NEG + 2*abase < 0, "neg enough");
+  intptr_t slice_idx_con = (intptr_t) _igvn.find_int_con(slice_idx, BIG_NEG);
+  intptr_t slice_len_con = (intptr_t) _igvn.find_int_con(slice_len, BIG_NEG);
+  if (slice_len_con == 0) {
+    return;                     // nothing to do here
+  }
+  intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
+  intptr_t end_con   = _igvn.find_intptr_t_con(dest_size, -1);
+  if (slice_idx_con >= 0 && slice_len_con >= 0) {
+    assert(end_con < 0, "not two cons");
+    end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
+                       BytesPerLong);
+  }
+
+  if (start_con >= 0 && end_con >= 0) {
+    // Constant start and end.  Simple.
+    mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+                                       start_con, end_con, &_igvn);
+  } else if (start_con >= 0 && dest_size != top()) {
+    // Constant start, pre-rounded end after the tail of the array.
+    Node* end = dest_size;
+    mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+                                       start_con, end, &_igvn);
+  } else if (start_con >= 0 && slice_len != top()) {
+    // Constant start, non-constant end.  End needs rounding up.
+    // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
+    intptr_t end_base  = abase + (slice_idx_con << scale);
+    int      end_round = (-1 << scale) & (BytesPerLong  - 1);
+    Node*    end       = ConvI2X(slice_len);
+    if (scale != 0)
+      end = transform_later(new LShiftXNode(end, intcon(scale) ));
+    end_base += end_round;
+    end = transform_later(new AddXNode(end, MakeConX(end_base)) );
+    end = transform_later(new AndXNode(end, MakeConX(~end_round)) );
+    mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+                                       start_con, end, &_igvn);
+  } else if (start_con < 0 && dest_size != top()) {
+    // Non-constant start, pre-rounded end after the tail of the array.
+    // This is almost certainly a "round-to-end" operation.
+    Node* start = slice_idx;
+    start = ConvI2X(start);
+    if (scale != 0)
+      start = transform_later(new LShiftXNode( start, intcon(scale) ));
+    start = transform_later(new AddXNode(start, MakeConX(abase)) );
+    if ((bump_bit | clear_low) != 0) {
+      int to_clear = (bump_bit | clear_low);
+      // Align up mod 8, then store a jint zero unconditionally
+      // just before the mod-8 boundary.
+      if (((abase + bump_bit) & ~to_clear) - bump_bit
+          < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
+        bump_bit = 0;
+        assert((abase & to_clear) == 0, "array base must be long-aligned");
+      } else {
+        // Bump 'start' up to (or past) the next jint boundary:
+        start = transform_later( new AddXNode(start, MakeConX(bump_bit)) );
+        assert((abase & clear_low) == 0, "array base must be int-aligned");
+      }
+      // Round bumped 'start' down to jlong boundary in body of array.
+      start = transform_later(new AndXNode(start, MakeConX(~to_clear)) );
+      if (bump_bit != 0) {
+        // Store a zero to the immediately preceding jint:
+        Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) );
+        Node* p1 = basic_plus_adr(dest, x1);
+        mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
+        mem = transform_later(mem);
+      }
+    }
+    Node* end = dest_size; // pre-rounded
+    mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+                                       start, end, &_igvn);
+  } else {
+    // Non-constant start, unrounded non-constant end.
+    // (Nobody zeroes a random midsection of an array using this routine.)
+    ShouldNotReachHere();       // fix caller
+  }
+
+  // Done.
+  merge_mem->set_memory_at(alias_idx, mem);
+}
+
+bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+                                                const TypePtr* adr_type,
+                                                BasicType basic_elem_type,
+                                                AllocateNode* alloc,
+                                                Node* src,  Node* src_offset,
+                                                Node* dest, Node* dest_offset,
+                                                Node* dest_size, bool dest_uninitialized) {
+  // See if there is an advantage from block transfer.
+  int scale = exact_log2(type2aelembytes(basic_elem_type));
+  if (scale >= LogBytesPerLong)
+    return false;               // it is already a block transfer
+
+  // Look at the alignment of the starting offsets.
+  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+
+  intptr_t src_off_con  = (intptr_t) _igvn.find_int_con(src_offset, -1);
+  intptr_t dest_off_con = (intptr_t) _igvn.find_int_con(dest_offset, -1);
+  if (src_off_con < 0 || dest_off_con < 0) {
+    // At present, we can only understand constants.
+    return false;
+  }
+
+  intptr_t src_off  = abase + (src_off_con  << scale);
+  intptr_t dest_off = abase + (dest_off_con << scale);
+
+  if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
+    // Non-aligned; too bad.
+    // One more chance:  Pick off an initial 32-bit word.
+    // This is a common case, since abase can be odd mod 8.
+    if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
+        ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
+      Node* sptr = basic_plus_adr(src,  src_off);
+      Node* dptr = basic_plus_adr(dest, dest_off);
+      uint alias_idx = C->get_alias_index(adr_type);
+      Node* sval = transform_later(LoadNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), sptr, adr_type, TypeInt::INT, T_INT, MemNode::unordered));
+      Node* st = transform_later(StoreNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), dptr, adr_type, sval, T_INT, MemNode::unordered));
+      (*mem)->set_memory_at(alias_idx, st);
+      src_off += BytesPerInt;
+      dest_off += BytesPerInt;
+    } else {
+      return false;
+    }
+  }
+  assert(src_off % BytesPerLong == 0, "");
+  assert(dest_off % BytesPerLong == 0, "");
+
+  // Do this copy by giant steps.
+  Node* sptr  = basic_plus_adr(src,  src_off);
+  Node* dptr  = basic_plus_adr(dest, dest_off);
+  Node* countx = dest_size;
+  countx = transform_later(new SubXNode(countx, MakeConX(dest_off)));
+  countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong)));
+
+  bool disjoint_bases = true;   // since alloc != NULL
+  generate_unchecked_arraycopy(ctrl, mem,
+                               adr_type, T_LONG, disjoint_bases,
+                               sptr, NULL, dptr, NULL, countx, dest_uninitialized);
+
+  return true;
+}
+
+// Helper function; generates code for the slow case.
+// We make a call to a runtime method which emulates the native method,
+// but without the native wrapper overhead.
+MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac,
+                                                        Node** ctrl, Node* mem, Node** io,
+                                                        const TypePtr* adr_type,
+                                                        Node* src,  Node* src_offset,
+                                                        Node* dest, Node* dest_offset,
+                                                        Node* copy_length, bool dest_uninitialized) {
+  assert(!dest_uninitialized, "Invariant");
+
+  const TypeFunc* call_type = OptoRuntime::slow_arraycopy_Type();
+  CallNode* call = new CallStaticJavaNode(call_type, OptoRuntime::slow_arraycopy_Java(),
+                                          "slow_arraycopy",
+                                          ac->jvms()->bci(), TypePtr::BOTTOM);
+
+  call->init_req(TypeFunc::Control, *ctrl);
+  call->init_req(TypeFunc::I_O    , *io);
+  call->init_req(TypeFunc::Memory , mem);
+  call->init_req(TypeFunc::ReturnAdr, top());
+  call->init_req(TypeFunc::FramePtr, top());
+  call->init_req(TypeFunc::Parms+0, src);
+  call->init_req(TypeFunc::Parms+1, src_offset);
+  call->init_req(TypeFunc::Parms+2, dest);
+  call->init_req(TypeFunc::Parms+3, dest_offset);
+  call->init_req(TypeFunc::Parms+4, copy_length);
+  copy_call_debug_info(ac, call);
+
+  call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
+  _igvn.replace_node(ac, call);
+  transform_later(call);
+
+  extract_call_projections(call);
+  *ctrl = _fallthroughcatchproj->clone();
+  transform_later(*ctrl);
+
+  Node* m = _memproj_fallthrough->clone();
+  transform_later(m);
+
+  uint alias_idx = C->get_alias_index(adr_type);
+  MergeMemNode* out_mem;
+  if (alias_idx != Compile::AliasIdxBot) {
+    out_mem = MergeMemNode::make(mem);
+    out_mem->set_memory_at(alias_idx, m);
+  } else {
+    out_mem = MergeMemNode::make(m);
+  }
+  transform_later(out_mem);
+
+  *io = _ioproj_fallthrough->clone();
+  transform_later(*io);
+
+  return out_mem;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                                     const TypePtr* adr_type,
+                                                     Node* dest_elem_klass,
+                                                     Node* src,  Node* src_offset,
+                                                     Node* dest, Node* dest_offset,
+                                                     Node* copy_length, bool dest_uninitialized) {
+  if ((*ctrl)->is_top())  return NULL;
+
+  address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
+  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+    return NULL;
+  }
+
+  // Pick out the parameters required to perform a store-check
+  // for the target array.  This is an optimistic check.  It will
+  // look in each non-null element's class, at the desired klass's
+  // super_check_offset, for the desired klass.
+  int sco_offset = in_bytes(Klass::super_check_offset_offset());
+  Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
+  Node* n3 = new LoadINode(NULL, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
+  Node* check_offset = ConvI2X(transform_later(n3));
+  Node* check_value  = dest_elem_klass;
+
+  Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
+  Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
+
+  const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type();
+  Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type,
+                              src_start, dest_start, copy_length XTOP, check_offset XTOP, check_value);
+
+  finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+  Node* proj =  new ProjNode(call, TypeFunc::Parms);
+  transform_later(proj);
+
+  return proj;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                                   const TypePtr* adr_type,
+                                                   Node* src,  Node* src_offset,
+                                                   Node* dest, Node* dest_offset,
+                                                   Node* copy_length, bool dest_uninitialized) {
+  if ((*ctrl)->is_top()) return NULL;
+  assert(!dest_uninitialized, "Invariant");
+
+  address copyfunc_addr = StubRoutines::generic_arraycopy();
+  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+    return NULL;
+  }
+
+  const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type();
+  Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "generic_arraycopy", adr_type,
+                              src, src_offset, dest, dest_offset, copy_length);
+
+  finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+  Node* proj =  new ProjNode(call, TypeFunc::Parms);
+  transform_later(proj);
+
+  return proj;
+}
+
+// Helper function; generates the fast out-of-line call to an arraycopy stub.
+void PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+                                                    const TypePtr* adr_type,
+                                                    BasicType basic_elem_type,
+                                                    bool disjoint_bases,
+                                                    Node* src,  Node* src_offset,
+                                                    Node* dest, Node* dest_offset,
+                                                    Node* copy_length, bool dest_uninitialized) {
+  if ((*ctrl)->is_top()) return;
+
+  Node* src_start  = src;
+  Node* dest_start = dest;
+  if (src_offset != NULL || dest_offset != NULL) {
+    src_start =  array_element_address(src, src_offset, basic_elem_type);
+    dest_start = array_element_address(dest, dest_offset, basic_elem_type);
+  }
+
+  // Figure out which arraycopy runtime method to call.
+  const char* copyfunc_name = "arraycopy";
+  address     copyfunc_addr =
+      basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
+                          disjoint_bases, copyfunc_name, dest_uninitialized);
+
+  const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+  Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
+                              src_start, dest_start, copy_length XTOP);
+
+  finish_arraycopy_call(call, ctrl, mem, adr_type);
+}
+
+void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
+  Node* ctrl = ac->in(TypeFunc::Control);
+  Node* io = ac->in(TypeFunc::I_O);
+  Node* src = ac->in(ArrayCopyNode::Src);
+  Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+  Node* dest = ac->in(ArrayCopyNode::Dest);
+  Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+  Node* length = ac->in(ArrayCopyNode::Length);
+  MergeMemNode* merge_mem = NULL;
+
+  if (ac->is_clonebasic()) {
+    assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+    Node* mem = ac->in(TypeFunc::Memory);
+    const char* copyfunc_name = "arraycopy";
+    address     copyfunc_addr =
+      basictype2arraycopy(T_LONG, NULL, NULL,
+                          true, copyfunc_name, true);
+
+    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+    const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+
+    Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
+    transform_later(call);
+
+    _igvn.replace_node(ac, call);
+    return;
+  } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
+    Node* mem = ac->in(TypeFunc::Memory);
+    merge_mem = MergeMemNode::make(mem);
+    transform_later(merge_mem);
+
+    RegionNode* slow_region = new RegionNode(1);
+    transform_later(slow_region);
+
+    AllocateArrayNode* alloc = NULL;
+    if (ac->is_alloc_tightly_coupled()) {
+      alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+      assert(alloc != NULL, "expect alloc");
+    }
+
+    generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+                       TypeAryPtr::OOPS, T_OBJECT,
+                       src, src_offset, dest, dest_offset, length,
+                       true, !ac->is_copyofrange());
+
+    return;
+  }
+
+  AllocateArrayNode* alloc = NULL;
+  if (ac->is_alloc_tightly_coupled()) {
+    alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+    assert(alloc != NULL, "expect alloc");
+  }
+
+  assert(ac->is_arraycopy() || ac->is_arraycopy_notest(), "should be an arraycopy");
+
+  // Compile time checks.  If any of these checks cannot be verified at compile time,
+  // we do not make a fast path for this call.  Instead, we let the call remain as it
+  // is.  The checks we choose to mandate at compile time are:
+  //
+  // (1) src and dest are arrays.
+  const Type* src_type = src->Value(&_igvn);
+  const Type* dest_type = dest->Value(&_igvn);
+  const TypeAryPtr* top_src = src_type->isa_aryptr();
+  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
+
+  if (top_src  == NULL || top_src->klass()  == NULL ||
+      top_dest == NULL || top_dest->klass() == NULL) {
+    // Conservatively insert a memory barrier on all memory slices.
+    // Do not let writes into the source float below the arraycopy.
+    {
+      Node* mem = ac->in(TypeFunc::Memory);
+      insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+
+      merge_mem = MergeMemNode::make(mem);
+      transform_later(merge_mem);
+    }
+
+    // Call StubRoutines::generic_arraycopy stub.
+    Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
+                                   TypeRawPtr::BOTTOM, T_CONFLICT,
+                                   src, src_offset, dest, dest_offset, length);
+
+    // Do not let reads from the destination float above the arraycopy.
+    // Since we cannot type the arrays, we don't know which slices
+    // might be affected.  We could restrict this barrier only to those
+    // memory slices which pertain to array elements--but don't bother.
+    if (!InsertMemBarAfterArraycopy) {
+      // (If InsertMemBarAfterArraycopy, there is already one in place.)
+      insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+    }
+    return;
+  }
+  // (2) src and dest arrays must have elements of the same BasicType
+  // Figure out the size and type of the elements we will be copying.
+  BasicType src_elem  =  top_src->klass()->as_array_klass()->element_type()->basic_type();
+  BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+  if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
+  if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
+
+  if (src_elem != dest_elem || dest_elem == T_VOID) {
+    // The component types are not the same or are not recognized.  Punt.
+    // (But, avoid the native method wrapper to JVM_ArrayCopy.)
+    {
+      Node* mem = ac->in(TypeFunc::Memory);
+      merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
+    }
+
+    _igvn.replace_node(_memproj_fallthrough, merge_mem);
+    _igvn.replace_node(_ioproj_fallthrough, io);
+    _igvn.replace_node(_fallthroughcatchproj, ctrl);
+    return;
+  }
+
+  //---------------------------------------------------------------------------
+  // We will make a fast path for this call to arraycopy.
+
+  // We have the following tests left to perform:
+  //
+  // (3) src and dest must not be null.
+  // (4) src_offset must not be negative.
+  // (5) dest_offset must not be negative.
+  // (6) length must not be negative.
+  // (7) src_offset + length must not exceed length of src.
+  // (8) dest_offset + length must not exceed length of dest.
+  // (9) each element of an oop array must be assignable
+
+  {
+    Node* mem = ac->in(TypeFunc::Memory);
+    merge_mem = MergeMemNode::make(mem);
+    transform_later(merge_mem);
+  }
+
+  RegionNode* slow_region = new RegionNode(1);
+  transform_later(slow_region);
+
+  if (!ac->is_arraycopy_notest()) {
+    // (3) operands must not be null
+    // We currently perform our null checks with the null_check routine.
+    // This means that the null exceptions will be reported in the caller
+    // rather than (correctly) reported inside of the native arraycopy call.
+    // This should be corrected, given time.  We do our null check with the
+    // stack pointer restored.
+    // null checks done library_call.cpp
+
+    // (4) src_offset must not be negative.
+    generate_negative_guard(&ctrl, src_offset, slow_region);
+
+    // (5) dest_offset must not be negative.
+    generate_negative_guard(&ctrl, dest_offset, slow_region);
+
+    // (6) length must not be negative (moved to generate_arraycopy()).
+    // generate_negative_guard(length, slow_region);
+
+    // (7) src_offset + length must not exceed length of src.
+    Node* r_adr =  new AddPNode(src, src, MakeConX(arrayOopDesc::length_offset_in_bytes()));
+    transform_later(r_adr);
+    Node* alen = new LoadRangeNode(0, C->immutable_memory(), r_adr, TypeInt::POS);
+    transform_later(alen);
+    generate_limit_guard(&ctrl,
+                         src_offset, length,
+                         alen,
+                         slow_region);
+
+    // (8) dest_offset + length must not exceed length of dest.
+    r_adr =  new AddPNode(dest, dest, MakeConX(arrayOopDesc::length_offset_in_bytes()));
+    transform_later(r_adr);
+    alen = new LoadRangeNode(0, C->immutable_memory(), r_adr, TypeInt::POS);
+    transform_later(alen);
+    generate_limit_guard(&ctrl,
+                         dest_offset, length,
+                         alen,
+                         slow_region);
+
+    // (9) each element of an oop array must be assignable
+    // The generate_arraycopy subroutine checks this.
+  }
+  // This is where the memory effects are placed:
+  const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
+  generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+                     adr_type, dest_elem,
+                     src, src_offset, dest, dest_offset, length,
+                     false, false, slow_region);
+}
--- a/src/share/vm/opto/node.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/node.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -40,6 +40,7 @@
 class AliasInfo;
 class AllocateArrayNode;
 class AllocateNode;
+class ArrayCopyNode;
 class Block;
 class BoolNode;
 class BoxLockNode;
@@ -561,6 +562,7 @@
           DEFINE_CLASS_ID(AbstractLock,     Call, 3)
             DEFINE_CLASS_ID(Lock,             AbstractLock, 0)
             DEFINE_CLASS_ID(Unlock,           AbstractLock, 1)
+          DEFINE_CLASS_ID(ArrayCopy,        Call, 4)
       DEFINE_CLASS_ID(MultiBranch, Multi, 1)
         DEFINE_CLASS_ID(PCTable,     MultiBranch, 0)
           DEFINE_CLASS_ID(Catch,       PCTable, 0)
@@ -707,6 +709,7 @@
   DEFINE_CLASS_QUERY(AddP)
   DEFINE_CLASS_QUERY(Allocate)
   DEFINE_CLASS_QUERY(AllocateArray)
+  DEFINE_CLASS_QUERY(ArrayCopy)
   DEFINE_CLASS_QUERY(Bool)
   DEFINE_CLASS_QUERY(BoxLock)
   DEFINE_CLASS_QUERY(Call)
--- a/src/share/vm/opto/phase.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/phase.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -27,7 +27,10 @@
 
 #include "runtime/timer.hpp"
 
-class Compile;
+class IfNode;
+class MergeMemNode;
+class Node;
+class PhaseGVN;
 
 //------------------------------Phase------------------------------------------
 // Most optimizations are done in Phases.  Creating a phase does any long
@@ -114,9 +117,20 @@
   static elapsedTimer   _t_instrSched;
   static elapsedTimer   _t_buildOopMaps;
 #endif
+
+  // Generate a subtyping check.  Takes as input the subtype and supertype.
+  // Returns 2 values: sets the default control() to the true path and
+  // returns the false path.  Only reads from constant memory taken from the
+  // default memory; does not write anything.  It also doesn't take in an
+  // Object; if you wish to check an Object you need to load the Object's
+  // class prior to coming here.
+  // Used in GraphKit and PhaseMacroExpand
+  static Node* gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn);
+
 public:
   Compile * C;
   Phase( PhaseNumber pnum );
+
 #ifndef PRODUCT
   static void print_timers();
 #endif
--- a/src/share/vm/opto/phaseX.hpp	Fri Aug 08 11:36:48 2014 -0700
+++ b/src/share/vm/opto/phaseX.hpp	Mon Aug 11 14:12:51 2014 +0200
@@ -390,6 +390,9 @@
   // in a faster or cheaper fashion.
   Node  *transform( Node *n );
   Node  *transform_no_reclaim( Node *n );
+  virtual void record_for_igvn(Node *n) {
+    C->record_for_igvn(n);
+  }
 
   void replace_with(PhaseGVN* gvn) {
     _table.replace_with(&gvn->_table);
@@ -418,9 +421,6 @@
 
 protected:
 
-  // Idealize new Node 'n' with respect to its inputs and its value
-  virtual Node *transform( Node *a_node );
-
   // Warm up hash table, type table and initial worklist
   void init_worklist( Node *a_root );
 
@@ -434,6 +434,10 @@
   PhaseIterGVN( PhaseGVN *gvn ); // Used after Parser
   PhaseIterGVN( PhaseIterGVN *igvn, const char *dummy ); // Used after +VerifyOpto
 
+  // Idealize new Node 'n' with respect to its inputs and its value
+  virtual Node *transform( Node *a_node );
+  virtual void record_for_igvn(Node *n) { }
+
   virtual PhaseIterGVN *is_IterGVN() { return this; }
 
   Unique_Node_List _worklist;       // Iterative worklist