annotate src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @ 56675:ff3eab78554f

8232674: Fix build and rename ShenandoahBarrierSet::oop_load_from_native_barrier Reviewed-by: zgu
author shade
date Mon, 21 Oct 2019 15:11:42 +0200
parents 3f35a9efd7de
children 4cbfa5077d68
rev   line source
rkennke@52925 1 /*
rkennke@52925 2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
rkennke@52925 3 *
rkennke@52925 4 * This code is free software; you can redistribute it and/or modify it
rkennke@52925 5 * under the terms of the GNU General Public License version 2 only, as
rkennke@52925 6 * published by the Free Software Foundation.
rkennke@52925 7 *
rkennke@52925 8 * This code is distributed in the hope that it will be useful, but WITHOUT
rkennke@52925 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
rkennke@52925 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
rkennke@52925 11 * version 2 for more details (a copy is included in the LICENSE file that
rkennke@52925 12 * accompanied this code).
rkennke@52925 13 *
rkennke@52925 14 * You should have received a copy of the GNU General Public License version
rkennke@52925 15 * 2 along with this work; if not, write to the Free Software Foundation,
rkennke@52925 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
rkennke@52925 17 *
rkennke@52925 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
rkennke@52925 19 * or visit www.oracle.com if you need additional information or have any
rkennke@52925 20 * questions.
rkennke@52925 21 *
rkennke@52925 22 */
rkennke@52925 23
coleenp@53244 24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
coleenp@53244 25 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
rkennke@52925 26
rkennke@52925 27 #include "gc/shared/barrierSet.hpp"
zgu@55681 28 #include "gc/shenandoah/shenandoahAsserts.hpp"
rkennke@52925 29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
rkennke@56304 30 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
rkennke@54766 31 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
rkennke@52925 32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
rkennke@54383 33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
rkennke@54383 34 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
rkennke@54383 35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
rkennke@56304 36 #include "memory/iterator.inline.hpp"
rkennke@56304 37 #include "oops/oop.inline.hpp"
rkennke@52925 38
rkennke@52925 39 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
rkennke@54766 40 return ShenandoahForwarding::get_forwardee(p);
rkennke@52925 41 }
rkennke@52925 42
rkennke@52925 43 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
rkennke@52925 44 if (((HeapWord*) p) != NULL) {
rkennke@52925 45 return resolve_forwarded_not_null(p);
rkennke@52925 46 } else {
rkennke@52925 47 return p;
rkennke@52925 48 }
rkennke@52925 49 }
rkennke@52925 50
rkennke@52925 51 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@52925 52 template <typename T>
rkennke@54423 53 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
rkennke@54423 54 oop value = Raw::oop_load_in_heap(addr);
rkennke@54423 55 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
rkennke@54423 56 keep_alive_if_weak(decorators, value);
rkennke@54423 57 return value;
rkennke@54423 58 }
rkennke@54423 59
rkennke@54423 60 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 61 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
rkennke@54423 62 oop value = Raw::oop_load_in_heap_at(base, offset);
rkennke@54423 63 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
rkennke@54423 64 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
rkennke@54423 65 return value;
rkennke@54423 66 }
rkennke@54423 67
rkennke@54423 68 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 69 template <typename T>
rkennke@54423 70 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
rkennke@54423 71 oop value = Raw::oop_load_not_in_heap(addr);
shade@56675 72 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_native(value, addr);
rkennke@54423 73 keep_alive_if_weak(decorators, value);
rkennke@54423 74 return value;
rkennke@54423 75 }
rkennke@54423 76
rkennke@54423 77 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 78 template <typename T>
rkennke@54423 79 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
rkennke@54423 80 ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
rkennke@54423 81 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
rkennke@54423 82 if (keep_alive) {
rkennke@54423 83 ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
rkennke@54423 84 }
rkennke@54423 85 Raw::oop_store_in_heap(addr, value);
rkennke@54423 86 }
rkennke@54423 87
rkennke@54423 88 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 89 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
rkennke@54423 90 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
rkennke@54423 91 }
rkennke@54423 92
rkennke@54423 93 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 94 template <typename T>
zgu@55681 95 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) {
zgu@55681 96 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress());
zgu@55681 97 Raw::oop_store(addr, value);
zgu@55681 98 }
zgu@55681 99
zgu@55681 100 template <DecoratorSet decorators, typename BarrierSetT>
zgu@55681 101 template <typename T>
rkennke@54423 102 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
rkennke@52925 103 oop res;
rkennke@52925 104 oop expected = compare_value;
rkennke@52925 105 do {
rkennke@52925 106 compare_value = expected;
rkennke@52925 107 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
rkennke@52925 108 expected = res;
pliden@56286 109 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected)));
rkennke@54423 110 if (res != NULL) {
rkennke@54423 111 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res);
rkennke@54423 112 } else {
rkennke@54423 113 return res;
rkennke@52925 114 }
rkennke@54423 115 }
rkennke@54423 116
rkennke@54423 117 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 118 template <typename T>
rkennke@54423 119 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) {
rkennke@54423 120 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
rkennke@54423 121 oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
rkennke@54423 122 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
rkennke@54423 123 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
pliden@56286 124 (result == compare_value) &&
rkennke@54423 125 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
rkennke@54423 126 ShenandoahBarrierSet::barrier_set()->enqueue(result);
rkennke@54423 127 }
rkennke@54423 128 return result;
rkennke@54423 129 }
rkennke@54423 130
rkennke@54423 131 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 132 template <typename T>
rkennke@54423 133 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
rkennke@54423 134 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value);
rkennke@54423 135 keep_alive_if_weak(decorators, result);
rkennke@54423 136 return result;
rkennke@54423 137 }
rkennke@54423 138
rkennke@54423 139 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 140 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
rkennke@54423 141 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
rkennke@54423 142 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
rkennke@54423 143 return result;
rkennke@54423 144 }
rkennke@54423 145
rkennke@54423 146 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 147 template <typename T>
rkennke@54423 148 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
rkennke@54423 149 oop previous = Raw::oop_atomic_xchg(new_value, addr);
rkennke@54423 150 if (previous != NULL) {
rkennke@54423 151 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous);
rkennke@54423 152 } else {
rkennke@54423 153 return previous;
rkennke@54423 154 }
rkennke@54423 155 }
rkennke@54423 156
rkennke@54423 157 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 158 template <typename T>
rkennke@54423 159 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) {
rkennke@54423 160 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
rkennke@54423 161 oop result = oop_atomic_xchg_not_in_heap(new_value, addr);
rkennke@54423 162 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
rkennke@54423 163 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
rkennke@54423 164 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
rkennke@54423 165 ShenandoahBarrierSet::barrier_set()->enqueue(result);
rkennke@54423 166 }
rkennke@54423 167 return result;
rkennke@52925 168 }
rkennke@52925 169
rkennke@52925 170 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@52925 171 template <typename T>
rkennke@52925 172 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
rkennke@54423 173 oop result = oop_atomic_xchg_in_heap_impl(new_value, addr);
rkennke@54423 174 keep_alive_if_weak(addr, result);
rkennke@54423 175 return result;
rkennke@52925 176 }
rkennke@52925 177
rkennke@52925 178 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@54423 179 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
rkennke@54423 180 oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
rkennke@54423 181 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
rkennke@54423 182 return result;
rkennke@52925 183 }
rkennke@52925 184
rkennke@52925 185 // Clone barrier support
rkennke@52925 186 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@52925 187 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
rkennke@56304 188 if (ShenandoahCloneBarrier) {
rkennke@56397 189 ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src);
rkennke@56304 190 }
rkennke@52925 191 Raw::clone(src, dst, size);
rkennke@52925 192 }
rkennke@52925 193
rkennke@52925 194 template <DecoratorSet decorators, typename BarrierSetT>
rkennke@52925 195 template <typename T>
rkennke@52925 196 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
rkennke@52925 197 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
rkennke@52925 198 size_t length) {
rkennke@56304 199 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
rkennke@56304 200 bs->arraycopy_pre(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
rkennke@56304 201 arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
rkennke@56304 202 length);
rkennke@56304 203 return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
rkennke@56304 204 }
rkennke@56304 205
rkennke@56304 206 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
rkennke@56304 207 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
rkennke@56304 208 Thread* thread = Thread::current();
rkennke@56304 209 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
rkennke@56304 210 ShenandoahMarkingContext* ctx = _heap->marking_context();
rkennke@56304 211 const ShenandoahCollectionSet* const cset = _heap->collection_set();
rkennke@56304 212 T* end = src + count;
rkennke@56304 213 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
rkennke@56304 214 T o = RawAccess<>::oop_load(elem_ptr);
rkennke@56304 215 if (!CompressedOops::is_null(o)) {
rkennke@56304 216 oop obj = CompressedOops::decode_not_null(o);
rkennke@56304 217 if (HAS_FWD && cset->is_in((HeapWord *) obj)) {
rkennke@56304 218 assert(_heap->has_forwarded_objects(), "only get here with forwarded objects");
rkennke@56304 219 oop fwd = resolve_forwarded_not_null(obj);
rkennke@56304 220 if (EVAC && obj == fwd) {
rkennke@56304 221 fwd = _heap->evacuate_object(obj, thread);
rkennke@56304 222 }
rkennke@56304 223 assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
rkennke@56304 224 oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o);
rkennke@56304 225 obj = fwd;
rkennke@56304 226 }
rkennke@56304 227 if (ENQUEUE && !ctx->is_marked(obj)) {
rkennke@56304 228 queue.enqueue_known_active(obj);
rkennke@56304 229 }
rkennke@56304 230 }
rkennke@56304 231 }
rkennke@56304 232 }
rkennke@56304 233
rkennke@56304 234 template <class T>
rkennke@56304 235 void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) {
rkennke@56304 236 if (_heap->is_concurrent_mark_in_progress()) {
rkennke@56304 237 if (_heap->has_forwarded_objects()) {
rkennke@56304 238 arraycopy_work<T, true, false, true>(dst, count);
rkennke@52925 239 } else {
rkennke@56304 240 arraycopy_work<T, false, false, true>(dst, count);
rkennke@52925 241 }
rkennke@52925 242 }
rkennke@52925 243
rkennke@56304 244 arraycopy_update_impl(src, count);
rkennke@56304 245 }
rkennke@56304 246
rkennke@56304 247 void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) {
rkennke@56304 248 arraycopy_pre_work(src, dst, count);
rkennke@56304 249 }
rkennke@56304 250
rkennke@56304 251 void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) {
rkennke@56304 252 arraycopy_pre_work(src, dst, count);
rkennke@56304 253 }
rkennke@56304 254
rkennke@56304 255 template <class T>
rkennke@56304 256 void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) {
rkennke@56304 257 if (_heap->is_evacuation_in_progress()) {
rkennke@56304 258 ShenandoahEvacOOMScope oom_evac;
rkennke@56304 259 arraycopy_work<T, true, true, false>(src, count);
rkennke@56304 260 } else if (_heap->is_concurrent_traversal_in_progress()){
rkennke@56304 261 ShenandoahEvacOOMScope oom_evac;
rkennke@56304 262 arraycopy_work<T, true, true, true>(src, count);
rkennke@56304 263 } else if (_heap->has_forwarded_objects()) {
rkennke@56304 264 arraycopy_work<T, true, false, false>(src, count);
rkennke@56304 265 }
rkennke@56304 266 }
rkennke@56304 267
rkennke@56304 268 void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) {
rkennke@56304 269 arraycopy_update_impl(src, count);
rkennke@56304 270 }
rkennke@56304 271
rkennke@56304 272 void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) {
rkennke@56304 273 arraycopy_update_impl(src, count);
rkennke@56304 274 }
rkennke@56304 275
coleenp@53244 276 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP