comparison src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @ 54423:6c0ab8bd8da5

8221766: Load-reference barriers for Shenandoah Reviewed-by: kvn, erikj, aph, shade
author rkennke
date Tue, 02 Apr 2019 23:00:22 +0200
parents 744dc9c33676
children 1321f8cf9de5
comparison
equal deleted inserted replaced
4:a8ab8e2a303e 5:311abfb52739
41 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() { 41 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
42 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2()); 42 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
43 } 43 }
44 44
45 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) 45 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
46 : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8, 0, NULL)) { 46 : _enqueue_barriers(new (comp_arena) GrowableArray<ShenandoahEnqueueBarrierNode*>(comp_arena, 8, 0, NULL)),
47 } 47 _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8, 0, NULL)) {
48 48 }
49 int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { 49
50 return _shenandoah_barriers->length(); 50 int ShenandoahBarrierSetC2State::enqueue_barriers_count() const {
51 } 51 return _enqueue_barriers->length();
52 52 }
53 ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { 53
54 return _shenandoah_barriers->at(idx); 54 ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const {
55 } 55 return _enqueue_barriers->at(idx);
56 56 }
57 void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { 57
58 assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); 58 void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
59 _shenandoah_barriers->append(n); 59 assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
60 } 60 _enqueue_barriers->append(n);
61 61 }
62 void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { 62
63 if (_shenandoah_barriers->contains(n)) { 63 void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
64 _shenandoah_barriers->remove(n); 64 if (_enqueue_barriers->contains(n)) {
65 } 65 _enqueue_barriers->remove(n);
66 } 66 }
67 67 }
68 #define __ kit-> 68
69 69 int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
70 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { 70 return _load_reference_barriers->length();
71 if (ShenandoahReadBarrier) { 71 }
72 obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); 72
73 } 73 ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const {
74 return obj; 74 return _load_reference_barriers->at(idx);
75 }
76
77 void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
78 assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list");
79 _load_reference_barriers->append(n);
80 }
81
82 void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
83 if (_load_reference_barriers->contains(n)) {
84 _load_reference_barriers->remove(n);
85 }
75 } 86 }
76 87
77 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { 88 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
78 if (ShenandoahStoreValEnqueueBarrier) { 89 if (ShenandoahStoreValEnqueueBarrier) {
79 obj = shenandoah_write_barrier(kit, obj);
80 obj = shenandoah_enqueue_barrier(kit, obj); 90 obj = shenandoah_enqueue_barrier(kit, obj);
81 } 91 }
82 if (ShenandoahStoreValReadBarrier) {
83 obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
84 }
85 return obj; 92 return obj;
86 } 93 }
87 94
88 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { 95 #define __ kit->
89 const Type* obj_type = obj->bottom_type();
90 if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
91 return obj;
92 }
93 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
94 Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
95
96 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
97 // We know it is null, no barrier needed.
98 return obj;
99 }
100
101 if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
102
103 // We don't know if it's null or not. Need null-check.
104 enum { _not_null_path = 1, _null_path, PATH_LIMIT };
105 RegionNode* region = new RegionNode(PATH_LIMIT);
106 Node* phi = new PhiNode(region, obj_type);
107 Node* null_ctrl = __ top();
108 Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
109
110 region->init_req(_null_path, null_ctrl);
111 phi ->init_req(_null_path, __ zerocon(T_OBJECT));
112
113 Node* ctrl = use_ctrl ? __ control() : NULL;
114 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
115 Node* n = __ gvn().transform(rb);
116
117 region->init_req(_not_null_path, __ control());
118 phi ->init_req(_not_null_path, n);
119
120 __ set_control(__ gvn().transform(region));
121 __ record_for_igvn(region);
122 return __ gvn().transform(phi);
123
124 } else {
125 // We know it is not null. Simple barrier is sufficient.
126 Node* ctrl = use_ctrl ? __ control() : NULL;
127 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
128 Node* n = __ gvn().transform(rb);
129 __ record_for_igvn(n);
130 return n;
131 }
132 }
133
134 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
135 ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
136 Node* n = __ gvn().transform(wb);
137 if (n == wb) { // New barrier needs memory projection.
138 Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
139 __ set_memory(proj, adr_type);
140 }
141 return n;
142 }
143
144 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
145 if (ShenandoahWriteBarrier) {
146 obj = shenandoah_write_barrier_impl(kit, obj);
147 }
148 return obj;
149 }
150
151 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
152 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
153 return obj;
154 }
155 const Type* obj_type = obj->bottom_type();
156 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
157 Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
158 __ record_for_igvn(n);
159 return n;
160 }
161 96
162 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, 97 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
163 BasicType bt, uint adr_idx) const { 98 BasicType bt, uint adr_idx) const {
164 intptr_t offset = 0; 99 intptr_t offset = 0;
165 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); 100 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
302 // Now some of the values 237 // Now some of the values
303 Node* marking; 238 Node* marking;
304 Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); 239 Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
305 Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); 240 Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
306 marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); 241 marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
307 assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); 242 assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
308 243
309 // if (!marking) 244 // if (!marking)
310 __ if_then(marking, BoolTest::ne, zero, unlikely); { 245 __ if_then(marking, BoolTest::ne, zero, unlikely); {
311 BasicType index_bt = TypeX_X->basic_type(); 246 BasicType index_bt = TypeX_X->basic_type();
312 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); 247 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
359 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry); 294 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
360 } 295 }
361 296
362 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) { 297 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
363 return call->is_CallLeaf() && 298 return call->is_CallLeaf() &&
364 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT); 299 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT);
365 } 300 }
366 301
367 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { 302 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
368 if (n->Opcode() != Op_If) { 303 if (n->Opcode() != Op_If) {
369 return false; 304 return false;
545 fields = TypeTuple::fields(1); 480 fields = TypeTuple::fields(1);
546 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; 481 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
547 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); 482 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
548 483
549 return TypeFunc::make(domain, range); 484 return TypeFunc::make(domain, range);
550 }
551
552 void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
553 const TypePtr* adr_type = access.addr().type();
554
555 if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
556 int off = adr_type->is_ptr()->offset();
557 int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
558 arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
559 assert(off != Type::OffsetTop, "unexpected offset");
560 if (off == Type::OffsetBot || off >= base_off) {
561 DecoratorSet decorators = access.decorators();
562 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
563 GraphKit* kit = NULL;
564 if (access.is_parse_access()) {
565 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
566 kit = parse_access.kit();
567 }
568 Node* adr = access.addr().node();
569 assert(adr->is_AddP(), "unexpected address shape");
570 Node* base = adr->in(AddPNode::Base);
571
572 if (is_write) {
573 if (kit != NULL) {
574 base = shenandoah_write_barrier(kit, base);
575 } else {
576 assert(access.is_opt_access(), "either parse or opt access");
577 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
578 }
579 } else {
580 if (adr_type->isa_instptr()) {
581 Compile* C = access.gvn().C;
582 ciField* field = C->alias_type(adr_type)->field();
583
584 // Insert read barrier for Shenandoah.
585 if (field != NULL &&
586 ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) ||
587 (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
588 (ShenandoahOptimizeStableFinals && field->is_stable()))) {
589 // Skip the barrier for special fields
590 } else {
591 if (kit != NULL) {
592 base = shenandoah_read_barrier(kit, base);
593 } else {
594 assert(access.is_opt_access(), "either parse or opt access");
595 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
596 }
597 }
598 } else {
599 if (kit != NULL) {
600 base = shenandoah_read_barrier(kit, base);
601 } else {
602 assert(access.is_opt_access(), "either parse or opt access");
603 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
604 }
605 }
606 }
607 if (base != adr->in(AddPNode::Base)) {
608 assert(kit != NULL, "no barrier should have been added");
609
610 Node* address = adr->in(AddPNode::Address);
611
612 if (address->is_AddP()) {
613 assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
614 assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
615 assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
616 address = address->clone();
617 address->set_req(AddPNode::Base, base);
618 address->set_req(AddPNode::Address, base);
619 address = kit->gvn().transform(address);
620 } else {
621 assert(address == adr->in(AddPNode::Base), "unexpected address shape");
622 address = base;
623 }
624 adr = adr->clone();
625 adr->set_req(AddPNode::Base, base);
626 adr->set_req(AddPNode::Address, address);
627 adr = kit->gvn().transform(adr);
628 access.addr().set_node(adr);
629 }
630 }
631 }
632 } 485 }
633 486
634 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 487 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
635 DecoratorSet decorators = access.decorators(); 488 DecoratorSet decorators = access.decorators();
636 489
660 assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code"); 513 assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
661 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 514 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
662 PhaseGVN& gvn = opt_access.gvn(); 515 PhaseGVN& gvn = opt_access.gvn();
663 MergeMemNode* mm = opt_access.mem(); 516 MergeMemNode* mm = opt_access.mem();
664 517
665 if (ShenandoahStoreValReadBarrier) {
666 RegionNode* region = new RegionNode(3);
667 const Type* v_t = gvn.type(val.node());
668 Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
669 Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
670 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
671 IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
672
673 gvn.transform(iff);
674 if (gvn.is_IterGVN()) {
675 gvn.is_IterGVN()->_worklist.push(iff);
676 } else {
677 gvn.record_for_igvn(iff);
678 }
679
680 Node* null_true = gvn.transform(new IfFalseNode(iff));
681 Node* null_false = gvn.transform(new IfTrueNode(iff));
682 region->init_req(1, null_true);
683 region->init_req(2, null_false);
684 phi->init_req(1, gvn.zerocon(T_OBJECT));
685 Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
686 cast->set_req(0, null_false);
687 cast = gvn.transform(cast);
688 Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
689 phi->init_req(2, rb);
690 opt_access.set_ctl(gvn.transform(region));
691 val.set_node(gvn.transform(phi));
692 }
693 if (ShenandoahStoreValEnqueueBarrier) { 518 if (ShenandoahStoreValEnqueueBarrier) {
694 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node())); 519 Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node()));
695 int alias = gvn.C->get_alias_index(adr_type);
696 Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
697 Node* wb_transformed = gvn.transform(wb);
698 Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
699 if (wb_transformed == wb) {
700 Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
701 mm->set_memory_at(alias, proj);
702 }
703 val.set_node(enqueue); 520 val.set_node(enqueue);
704 } 521 }
705 } 522 }
706 return BarrierSetC2::store_at_resolved(access, val); 523 return BarrierSetC2::store_at_resolved(access, val);
707 } 524 }
721 538
722 Node* top = Compile::current()->top(); 539 Node* top = Compile::current()->top();
723 540
724 Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; 541 Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
725 Node* load = BarrierSetC2::load_at_resolved(access, val_type); 542 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
543
544 if (access.is_oop()) {
545 if (ShenandoahLoadRefBarrier) {
546 load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
547 if (access.is_parse_access()) {
548 load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
549 } else {
550 load = static_cast<C2OptAccess &>(access).gvn().transform(load);
551 }
552 }
553 }
726 554
727 // If we are reading the value of the referent field of a Reference 555 // If we are reading the value of the referent field of a Reference
728 // object (either by using Unsafe directly or through reflection) 556 // object (either by using Unsafe directly or through reflection)
729 // then, if SATB is enabled, we need to record the referent in an 557 // then, if SATB is enabled, we need to record the referent in an
730 // SATB log buffer using the pre-barrier mechanism. 558 // SATB log buffer using the pre-barrier mechanism.
795 access.set_raw_access(load_store); 623 access.set_raw_access(load_store);
796 pin_atomic_op(access); 624 pin_atomic_op(access);
797 625
798 #ifdef _LP64 626 #ifdef _LP64
799 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 627 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
800 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 628 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
801 } 629 }
802 #endif 630 #endif
631 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
803 return load_store; 632 return load_store;
804 } 633 }
805 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 634 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
806 } 635 }
807 636
865 if (access.is_oop()) { 694 if (access.is_oop()) {
866 val = shenandoah_storeval_barrier(kit, val); 695 val = shenandoah_storeval_barrier(kit, val);
867 } 696 }
868 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); 697 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
869 if (access.is_oop()) { 698 if (access.is_oop()) {
699 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
870 shenandoah_write_barrier_pre(kit, false /* do_load */, 700 shenandoah_write_barrier_pre(kit, false /* do_load */,
871 NULL, NULL, max_juint, NULL, NULL, 701 NULL, NULL, max_juint, NULL, NULL,
872 result /* pre_val */, T_OBJECT); 702 result /* pre_val */, T_OBJECT);
873 } 703 }
874 return result; 704 return result;
875 } 705 }
876 706
877 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { 707 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
878 assert(!src->is_AddP(), "unexpected input"); 708 assert(!src->is_AddP(), "unexpected input");
879 src = shenandoah_read_barrier(kit, src);
880 BarrierSetC2::clone(kit, src, dst, size, is_array); 709 BarrierSetC2::clone(kit, src, dst, size, is_array);
881 }
882
883 Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
884 bool is_write = decorators & ACCESS_WRITE;
885 if (is_write) {
886 return shenandoah_write_barrier(kit, n);
887 } else {
888 return shenandoah_read_barrier(kit, n);
889 }
890 } 710 }
891 711
892 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, 712 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
893 Node*& i_o, Node*& needgc_ctrl, 713 Node*& i_o, Node*& needgc_ctrl,
894 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 714 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
913 return fast_oop; 733 return fast_oop;
914 } 734 }
915 735
916 // Support for GC barriers emitted during parsing 736 // Support for GC barriers emitted during parsing
917 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { 737 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
738 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
918 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) { 739 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
919 return false; 740 return false;
920 } 741 }
921 CallLeafNode *call = node->as_CallLeaf(); 742 CallLeafNode *call = node->as_CallLeaf();
922 if (call->_name == NULL) { 743 if (call->_name == NULL) {
927 strcmp(call->_name, "shenandoah_cas_obj") == 0 || 748 strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
928 strcmp(call->_name, "shenandoah_wb_pre") == 0; 749 strcmp(call->_name, "shenandoah_wb_pre") == 0;
929 } 750 }
930 751
931 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const { 752 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
932 return ShenandoahBarrierNode::skip_through_barrier(c); 753 if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
754 return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
755 }
756 if (c->Opcode() == Op_ShenandoahEnqueueBarrier) {
757 c = c->in(1);
758 }
759 return c;
933 } 760 }
934 761
935 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { 762 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
936 return !ShenandoahWriteBarrierNode::expand(C, igvn); 763 return !ShenandoahBarrierC2Support::expand(C, igvn);
937 } 764 }
938 765
939 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { 766 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
940 if (mode == LoopOptsShenandoahExpand) { 767 if (mode == LoopOptsShenandoahExpand) {
941 assert(UseShenandoahGC, "only for shenandoah"); 768 assert(UseShenandoahGC, "only for shenandoah");
942 ShenandoahWriteBarrierNode::pin_and_expand(phase); 769 ShenandoahBarrierC2Support::pin_and_expand(phase);
943 return true; 770 return true;
944 } else if (mode == LoopOptsShenandoahPostExpand) { 771 } else if (mode == LoopOptsShenandoahPostExpand) {
945 assert(UseShenandoahGC, "only for shenandoah"); 772 assert(UseShenandoahGC, "only for shenandoah");
946 visited.Clear(); 773 visited.Clear();
947 ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase); 774 ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase);
948 return true; 775 return true;
949 } 776 }
950 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
951 ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);
952 return false; 777 return false;
953 } 778 }
954 779
955 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { 780 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
956 bool is_oop = type == T_OBJECT || type == T_ARRAY; 781 bool is_oop = type == T_OBJECT || type == T_ARRAY;
957 if (!is_oop) { 782 if (!is_oop) {
958 return false; 783 return false;
959 } 784 }
960
961 if (tightly_coupled_alloc) { 785 if (tightly_coupled_alloc) {
962 if (phase == Optimization) { 786 if (phase == Optimization) {
963 return false; 787 return false;
964 } 788 }
965 return !is_clone; 789 return !is_clone;
983 igvn.C->dependencies()->assert_leaf_type(ik); 807 igvn.C->dependencies()->assert_leaf_type(ik);
984 } 808 }
985 } 809 }
986 } else { 810 } else {
987 return true; 811 return true;
988 } 812 }
989 } else if (src_type->isa_aryptr()) { 813 } else if (src_type->isa_aryptr()) {
990 BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); 814 BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
991 if (src_elem == T_OBJECT || src_elem == T_ARRAY) { 815 if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
992 return true; 816 return true;
993 } 817 }
1036 } 860 }
1037 861
1038 862
1039 // Support for macro expanded GC barriers 863 // Support for macro expanded GC barriers
1040 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { 864 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
1041 if (node->Opcode() == Op_ShenandoahWriteBarrier) { 865 if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
1042 state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); 866 state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
867 }
868 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
869 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1043 } 870 }
1044 } 871 }
1045 872
1046 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { 873 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1047 if (node->Opcode() == Op_ShenandoahWriteBarrier) { 874 if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
1048 state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); 875 state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
876 }
877 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
878 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1049 } 879 }
1050 } 880 }
1051 881
1052 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const { 882 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
1053 if (is_shenandoah_wb_pre_call(n)) { 883 if (is_shenandoah_wb_pre_call(n)) {
1089 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 919 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1090 C->record_for_igvn(n->fast_out(i)); 920 C->record_for_igvn(n->fast_out(i));
1091 } 921 }
1092 } 922 }
1093 } 923 }
1094 for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { 924 for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) {
1095 ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); 925 ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i);
1096 if (!useful.member(n)) { 926 if (!useful.member(n)) {
1097 state()->remove_shenandoah_barrier(n); 927 state()->remove_enqueue_barrier(n);
1098 } 928 }
1099 } 929 }
1100 930 for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
1101 } 931 ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
1102 932 if (!useful.member(n)) {
1103 bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const { 933 state()->remove_load_reference_barrier(n);
1104 assert(node->outcnt() == 1, "match only for unique out"); 934 }
1105 Node* n = node->unique_out(); 935 }
1106 return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
1107 } 936 }
1108 937
1109 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} 938 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
1110 939
1111 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const { 940 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
1121 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } 950 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
1122 951
1123 #ifdef ASSERT 952 #ifdef ASSERT
1124 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { 953 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1125 if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) { 954 if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
1126 ShenandoahBarrierNode::verify(Compile::current()->root()); 955 ShenandoahBarrierC2Support::verify(Compile::current()->root());
1127 } else if (phase == BarrierSetC2::BeforeCodeGen) { 956 } else if (phase == BarrierSetC2::BeforeCodeGen) {
1128 // Verify G1 pre-barriers 957 // Verify G1 pre-barriers
1129 const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()); 958 const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
1130 959
1131 ResourceArea *area = Thread::current()->resource_area(); 960 ResourceArea *area = Thread::current()->resource_area();
1227 } 1056 }
1228 return n; 1057 return n;
1229 } 1058 }
1230 } else if (can_reshape && 1059 } else if (can_reshape &&
1231 n->Opcode() == Op_If && 1060 n->Opcode() == Op_If &&
1232 ShenandoahWriteBarrierNode::is_heap_stable_test(n) && 1061 ShenandoahBarrierC2Support::is_heap_stable_test(n) &&
1233 n->in(0) != NULL) { 1062 n->in(0) != NULL) {
1234 Node* dom = n->in(0); 1063 Node* dom = n->in(0);
1235 Node* prev_dom = n; 1064 Node* prev_dom = n;
1236 int op = n->Opcode(); 1065 int op = n->Opcode();
1237 int dist = 16; 1066 int dist = 16;
1238 // Search up the dominator tree for another heap stable test 1067 // Search up the dominator tree for another heap stable test
1239 while (dom->Opcode() != op || // Not same opcode? 1068 while (dom->Opcode() != op || // Not same opcode?
1240 !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1? 1069 !ShenandoahBarrierC2Support::is_heap_stable_test(dom) || // Not same input 1?
1241 prev_dom->in(0) != dom) { // One path of test does not dominate? 1070 prev_dom->in(0) != dom) { // One path of test does not dominate?
1242 if (dist < 0) return NULL; 1071 if (dist < 0) return NULL;
1243 1072
1244 dist--; 1073 dist--;
1245 prev_dom = dom; 1074 prev_dom = dom;
1254 1083
1255 return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN()); 1084 return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN());
1256 } 1085 }
1257 1086
1258 return NULL; 1087 return NULL;
1259 }
1260
1261 Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
1262 if (n->is_Load()) {
1263 Node *mem = n->in(MemNode::Memory);
1264 Node *value = n->as_Load()->can_see_stored_value(mem, phase);
1265 if (value) {
1266 PhaseIterGVN *igvn = phase->is_IterGVN();
1267 if (igvn != NULL &&
1268 value->is_Phi() &&
1269 value->req() > 2 &&
1270 value->in(1) != NULL &&
1271 value->in(1)->is_ShenandoahBarrier()) {
1272 if (igvn->_worklist.member(value) ||
1273 igvn->_worklist.member(value->in(0)) ||
1274 (value->in(0)->in(1) != NULL &&
1275 value->in(0)->in(1)->is_IfProj() &&
1276 (igvn->_worklist.member(value->in(0)->in(1)) ||
1277 (value->in(0)->in(1)->in(0) != NULL &&
1278 igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
1279 igvn->_worklist.push(n);
1280 return n;
1281 }
1282 }
1283 // (This works even when value is a Con, but LoadNode::Value
1284 // usually runs first, producing the singleton type of the Con.)
1285 Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
1286 if (value->Opcode() == Op_EncodeP) {
1287 if (value_no_barrier != value->in(1)) {
1288 Node *encode = value->clone();
1289 encode->set_req(1, value_no_barrier);
1290 encode = phase->transform(encode);
1291 return encode;
1292 }
1293 } else {
1294 return value_no_barrier;
1295 }
1296 }
1297 }
1298 return n;
1299 } 1088 }
1300 1089
1301 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { 1090 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1302 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 1091 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1303 Node* u = n->fast_out(i); 1092 Node* u = n->fast_out(i);
1304 if (!is_shenandoah_wb_pre_call(u)) { 1093 if (!is_shenandoah_wb_pre_call(u)) {
1305 return false; 1094 return false;
1306 } 1095 }
1307 } 1096 }
1308 return n->outcnt() > 0; 1097 return n->outcnt() > 0;
1309 }
1310
1311 bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
1312 int offset = adr_type->offset();
1313 if (offset == ShenandoahBrooksPointer::byte_offset()) {
1314 if (adr_type->isa_aryptr()) {
1315 adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
1316 } else if (adr_type->isa_instptr()) {
1317 adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
1318 }
1319 return true;
1320 } else {
1321 return false;
1322 }
1323 } 1098 }
1324 1099
1325 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { 1100 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1326 switch (opcode) { 1101 switch (opcode) {
1327 case Op_CallLeaf: 1102 case Op_CallLeaf:
1354 bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); 1129 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1355 assert( !tp || oop_offset_is_sane, "" ); 1130 assert( !tp || oop_offset_is_sane, "" );
1356 } 1131 }
1357 #endif 1132 #endif
1358 return true; 1133 return true;
1359 case Op_ShenandoahReadBarrier: 1134 case Op_ShenandoahLoadReferenceBarrier:
1360 return true;
1361 case Op_ShenandoahWriteBarrier:
1362 assert(false, "should have been expanded already"); 1135 assert(false, "should have been expanded already");
1363 return true; 1136 return true;
1364 default: 1137 default:
1365 return false; 1138 return false;
1366 } 1139 }
1367 } 1140 }
1368
1369 #ifdef ASSERT
1370 bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
1371 if (offset == ShenandoahBrooksPointer::byte_offset() &&
1372 (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
1373 return true;
1374 } else {
1375 return false;
1376 }
1377 }
1378 #endif
1379 1141
1380 bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { 1142 bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1381 switch (opcode) { 1143 switch (opcode) {
1382 case Op_ShenandoahCompareAndExchangeP: 1144 case Op_ShenandoahCompareAndExchangeP:
1383 case Op_ShenandoahCompareAndExchangeN: 1145 case Op_ShenandoahCompareAndExchangeN:
1410 } 1172 }
1411 } 1173 }
1412 } 1174 }
1413 return false; 1175 return false;
1414 } 1176 }
1415 case Op_ShenandoahReadBarrier:
1416 case Op_ShenandoahWriteBarrier:
1417 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1418 // It doesn't escape.
1419 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
1420 break;
1421 case Op_ShenandoahEnqueueBarrier: 1177 case Op_ShenandoahEnqueueBarrier:
1422 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist); 1178 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
1423 break; 1179 break;
1180 case Op_ShenandoahLoadReferenceBarrier:
1181 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist);
1182 return true;
1424 default: 1183 default:
1425 // Nothing 1184 // Nothing
1426 break; 1185 break;
1427 } 1186 }
1428 return false; 1187 return false;
1439 case Op_ShenandoahCompareAndSwapP: 1198 case Op_ShenandoahCompareAndSwapP:
1440 case Op_ShenandoahCompareAndSwapN: 1199 case Op_ShenandoahCompareAndSwapN:
1441 case Op_ShenandoahWeakCompareAndSwapP: 1200 case Op_ShenandoahWeakCompareAndSwapP:
1442 case Op_ShenandoahWeakCompareAndSwapN: 1201 case Op_ShenandoahWeakCompareAndSwapN:
1443 return conn_graph->add_final_edges_unsafe_access(n, opcode); 1202 return conn_graph->add_final_edges_unsafe_access(n, opcode);
1444 case Op_ShenandoahReadBarrier:
1445 case Op_ShenandoahWriteBarrier:
1446 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1447 // It doesn't escape.
1448 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
1449 return true;
1450 case Op_ShenandoahEnqueueBarrier: 1203 case Op_ShenandoahEnqueueBarrier:
1451 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL); 1204 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
1205 return true;
1206 case Op_ShenandoahLoadReferenceBarrier:
1207 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL);
1452 return true; 1208 return true;
1453 default: 1209 default:
1454 // Nothing 1210 // Nothing
1455 break; 1211 break;
1456 } 1212 }
1462 n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN); 1218 n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN);
1463 1219
1464 } 1220 }
1465 1221
1466 bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { 1222 bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
1467 return n->is_ShenandoahBarrier(); 1223 return n->Opcode() == Op_ShenandoahLoadReferenceBarrier;
1468 }
1469
1470 bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1471 switch (opcode) {
1472 case Op_ShenandoahReadBarrier:
1473 if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
1474 matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
1475 }
1476 matcher->set_shared(n);
1477 return true;
1478 default:
1479 break;
1480 }
1481 return false;
1482 } 1224 }
1483 1225
1484 bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { 1226 bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
1485 switch (opcode) { 1227 switch (opcode) {
1486 case Op_ShenandoahCompareAndExchangeP: 1228 case Op_ShenandoahCompareAndExchangeP:
1508 xop == Op_ShenandoahWeakCompareAndSwapP || 1250 xop == Op_ShenandoahWeakCompareAndSwapP ||
1509 xop == Op_ShenandoahWeakCompareAndSwapN || 1251 xop == Op_ShenandoahWeakCompareAndSwapN ||
1510 xop == Op_ShenandoahCompareAndSwapN || 1252 xop == Op_ShenandoahCompareAndSwapN ||
1511 xop == Op_ShenandoahCompareAndSwapP; 1253 xop == Op_ShenandoahCompareAndSwapP;
1512 } 1254 }
1513
1514 void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
1515 if (use->is_ShenandoahBarrier()) {
1516 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1517 Node* u = use->fast_out(i2);
1518 Node* cmp = use->find_out_with(Op_CmpP);
1519 if (u->Opcode() == Op_CmpP) {
1520 igvn->_worklist.push(cmp);
1521 }
1522 }
1523 }
1524 }
1525
1526 void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
1527 if (use->is_ShenandoahBarrier()) {
1528 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1529 Node* p = use->fast_out(i2);
1530 if (p->Opcode() == Op_AddP) {
1531 for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
1532 Node* q = p->fast_out(i3);
1533 if (q->is_Load()) {
1534 if(q->bottom_type() != ccp->type(q)) {
1535 worklist.push(q);
1536 }
1537 }
1538 }
1539 }
1540 }
1541 }
1542 }
1543
1544 Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const {
1545 if (n->Opcode() == Op_ShenandoahReadBarrier) {
1546 ((ShenandoahReadBarrierNode*)n)->try_move(phase);
1547 } else if (n->Opcode() == Op_ShenandoahWriteBarrier) {
1548 return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase);
1549 }
1550
1551 return NULL;
1552 }
1553
1554 bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const {
1555 return ShenandoahBarrierNode::build_loop_late_post(phase, n);
1556 }
1557
1558 bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const {
1559 if (n->is_ShenandoahBarrier()) {
1560 return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl);
1561 }
1562 if (n->is_MergeMem()) {
1563 // PhaseIdealLoop::split_if_with_blocks_post() would:
1564 // _igvn._worklist.yank(x);
1565 // which sometimes causes chains of MergeMem which some of
1566 // shenandoah specific code doesn't support
1567 phase->register_new_node(x, x_ctrl);
1568 return true;
1569 }
1570 return false;
1571 }