changeset 13060:927522e04185

Shenandoah matrix update barriers should be conditional.
author shade
date Thu, 09 Mar 2017 18:50:48 +0100
parents e4a4414151f0
children d068ee404af1
files src/cpu/x86/vm/macroAssembler_x86.cpp src/cpu/x86/vm/macroAssembler_x86.hpp src/share/vm/c1/c1_LIRGenerator.cpp src/share/vm/opto/compile.cpp src/share/vm/opto/graphKit.cpp
diffstat 5 files changed, 49 insertions(+), 10 deletions(-) [+]
line wrap: on
line diff
--- a/src/cpu/x86/vm/macroAssembler_x86.cpp	Thu Mar 09 15:50:19 2017 +0100
+++ b/src/cpu/x86/vm/macroAssembler_x86.cpp	Thu Mar 09 18:50:48 2017 +0100
@@ -5379,8 +5379,12 @@
   // Compute matrix index
   imulq(tmp, tmp, matrix->stride());
   addq(tmp, tmp2);
+  // Address is _matrix[from * stride + to]
   movptr(rscratch1, (intptr_t) matrix_addr);
-  // Store true at _matrix[from * stride + to]
+  // Test if the element is already set.
+  testbool(Address(rscratch1, tmp, Address::times_1));
+  jcc(Assembler::notZero, done);
+  // Store true, if not yet set.
   movbool(Address(rscratch1, tmp, Address::times_1), true);
   bind(done);
 }
@@ -5588,6 +5592,15 @@
     ShouldNotReachHere();
 }
 
+void MacroAssembler::testbool(Address dst) {
+  if(sizeof(bool) == 1) {
+    testb(dst, 0xff);
+  } else {
+    // unsupported
+    ShouldNotReachHere();
+  }
+}
+
 void MacroAssembler::testptr(Register dst, Register src) {
   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 }
--- a/src/cpu/x86/vm/macroAssembler_x86.hpp	Thu Mar 09 15:50:19 2017 +0100
+++ b/src/cpu/x86/vm/macroAssembler_x86.hpp	Thu Mar 09 18:50:48 2017 +0100
@@ -331,6 +331,7 @@
   void movbool(Address dst, bool boolconst);
   void movbool(Address dst, Register src);
   void testbool(Register dst);
+  void testbool(Address dst);
 
   void load_mirror(Register mirror, Register method);
 
--- a/src/share/vm/c1/c1_LIRGenerator.cpp	Thu Mar 09 15:50:19 2017 +0100
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp	Thu Mar 09 18:50:48 2017 +0100
@@ -1683,6 +1683,11 @@
   LIR_Opr tmp4 = new_pointer_register();
   __ move(LIR_OprFact::intptrConst((intptr_t) matrix->matrix_addr()), tmp4);
   LIR_Address* matrix_elem_addr = new LIR_Address(tmp4, tmp1, T_BOOLEAN);
+
+  LIR_Opr tmp5 = new_register(T_INT);
+  __ move(matrix_elem_addr, tmp5);
+  __ cmp(lir_cond_notEqual, tmp5, LIR_OprFact::intConst(0));
+  __ branch(lir_cond_notEqual, T_BOOLEAN, L_done->label());
   __ move(LIR_OprFact::intConst((int) true), matrix_elem_addr);
   __ branch_destination(L_done->label());
 }
--- a/src/share/vm/opto/compile.cpp	Thu Mar 09 15:50:19 2017 +0100
+++ b/src/share/vm/opto/compile.cpp	Thu Mar 09 18:50:48 2017 +0100
@@ -4666,13 +4666,13 @@
     Node *n = wq.at(next);
     if (n->is_Store()) {
       igvn->replace_node(n, n->in(MemNode::Memory));
-    } else {
+    } else if (!n->is_Load()) {
       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
         Node* u = n->fast_out(i);
         wq.push(u);
       }
     }
-    }
+  }
   igvn->replace_node(p2x, C->top());
 }
 
--- a/src/share/vm/opto/graphKit.cpp	Thu Mar 09 15:50:19 2017 +0100
+++ b/src/share/vm/opto/graphKit.cpp	Thu Mar 09 18:50:48 2017 +0100
@@ -4285,7 +4285,7 @@
 
   ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
 
-  enum { _not_null_path = 1, _null_path, PATH_LIMIT };
+  enum { _set_path = 1, _already_set_path, _val_null_path, PATH_LIMIT };
   RegionNode* region = new RegionNode(PATH_LIMIT);
   Node* prev_mem = memory(Compile::AliasIdxRaw);
   Node* memphi    = PhiNode::make(region, prev_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
@@ -4293,32 +4293,52 @@
   Node* not_null_val = null_check_oop(val, &null_ctrl);
 
   // Null path: nothing to do.
-  region->init_req(_null_path, null_ctrl);
-  memphi->init_req(_null_path, prev_mem);
+  region->init_req(_val_null_path, null_ctrl);
+  memphi->init_req(_val_null_path, prev_mem);
 
   // Not null path. Update the matrix.
   Node* heapbase = MakeConX((intx) ShenandoahHeap::heap()->first_region_bottom());
   Node* region_size_shift = intcon((jint) ShenandoahHeapRegion::RegionSizeShift);
   Node* stride = MakeConX((intx) matrix->stride());
   Node* matrix_base = makecon(TypeRawPtr::make(matrix->matrix_addr()));
+
   // Compute region index for adr.
   Node* adr_idx = _gvn.transform(new CastP2XNode(control(), adr));
   adr_idx = _gvn.transform(new SubXNode(adr_idx, heapbase));
   adr_idx = _gvn.transform(new URShiftXNode(adr_idx, region_size_shift));
+
   // Compute region index for val.
   Node* val_idx = _gvn.transform(new CastP2XNode(control(), not_null_val));
   val_idx = _gvn.transform(new SubXNode(val_idx, heapbase));
   val_idx = _gvn.transform(new URShiftXNode(val_idx, region_size_shift));
+
   // Compute matrix index & address.
   Node* matrix_idx = _gvn.transform(new MulXNode(adr_idx, stride));
   matrix_idx = _gvn.transform(new AddXNode(matrix_idx, val_idx));
   Node* matrix_adr = basic_plus_adr(top(), matrix_base, matrix_idx);
-  // Do the store.
+
+  // Load current value
   const TypePtr* adr_type = TypeRawPtr::BOTTOM;
+  Node* current = _gvn.transform(LoadNode::make(_gvn, control(), memory(Compile::AliasIdxRaw), matrix_adr, adr_type, TypeInt::INT, T_BOOLEAN, MemNode::unordered));
+
+  // Check if already set
+  Node* cmp_set = _gvn.transform(new CmpINode(current, intcon(0)));
+  Node* cmp_set_bool = _gvn.transform(new BoolNode(cmp_set, BoolTest::eq));
+  IfNode* cmp_iff = create_and_map_if(control(), cmp_set_bool, PROB_FAIR, COUNT_UNKNOWN);
+
+  Node* if_not_set = _gvn.transform(new IfTrueNode(cmp_iff));
+  Node* if_set = _gvn.transform(new IfFalseNode(cmp_iff));
+
+  // Already set, exit
+  set_control(if_set);
+  region->init_req(_already_set_path, control());
+  memphi->init_req(_already_set_path, prev_mem);
+
+  // Not set: do the store, and finish up
+  set_control(if_not_set);
   Node* store = _gvn.transform(StoreNode::make(_gvn, control(), memory(Compile::AliasIdxRaw), matrix_adr, adr_type, intcon(1), T_BOOLEAN, MemNode::unordered));
-
-  region->init_req(_not_null_path, control());
-  memphi->init_req(_not_null_path, store);
+  region->init_req(_set_path, control());
+  memphi->init_req(_set_path, store);
 
   // Merge control flows and memory.
   set_control(_gvn.transform(region));