annotate src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp @ 54536:fb53a1c25903

8222425: Shenandoah: Move commonly used closures to separate files Reviewed-by: shade
author zgu
date Mon, 15 Apr 2019 13:07:06 -0400
parents a84fefde0543
children 5e5535d25467
rev   line source
rkennke@52925 1 /*
zgu@53383 2 * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
rkennke@52925 3 *
rkennke@52925 4 * This code is free software; you can redistribute it and/or modify it
rkennke@52925 5 * under the terms of the GNU General Public License version 2 only, as
rkennke@52925 6 * published by the Free Software Foundation.
rkennke@52925 7 *
rkennke@52925 8 * This code is distributed in the hope that it will be useful, but WITHOUT
rkennke@52925 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
rkennke@52925 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
rkennke@52925 11 * version 2 for more details (a copy is included in the LICENSE file that
rkennke@52925 12 * accompanied this code).
rkennke@52925 13 *
rkennke@52925 14 * You should have received a copy of the GNU General Public License version
rkennke@52925 15 * 2 along with this work; if not, write to the Free Software Foundation,
rkennke@52925 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
rkennke@52925 17 *
rkennke@52925 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
rkennke@52925 19 * or visit www.oracle.com if you need additional information or have any
rkennke@52925 20 * questions.
rkennke@52925 21 *
rkennke@52925 22 */
rkennke@52925 23
rkennke@52925 24 #include "precompiled.hpp"
rkennke@52925 25
rkennke@52925 26 #include "classfile/classLoaderData.hpp"
rkennke@52925 27 #include "classfile/classLoaderDataGraph.hpp"
rkennke@52925 28 #include "gc/shared/referenceProcessor.hpp"
rkennke@52925 29 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
rkennke@52925 30 #include "gc/shared/workgroup.hpp"
rkennke@52925 31 #include "gc/shared/weakProcessor.inline.hpp"
rkennke@52925 32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
zgu@54536 33 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
rkennke@52925 34 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
rkennke@52925 35 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
rkennke@52925 36 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
rkennke@52925 37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
rkennke@52925 38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
rkennke@52925 39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
rkennke@52925 40 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
rkennke@52925 41 #include "gc/shenandoah/shenandoahHeuristics.hpp"
rkennke@52925 42 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
rkennke@52925 43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
zgu@54344 44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
rkennke@52925 45 #include "gc/shenandoah/shenandoahStringDedup.hpp"
rkennke@52925 46 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
rkennke@52925 47 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
rkennke@52925 48 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
rkennke@52925 49 #include "gc/shenandoah/shenandoahUtils.hpp"
rkennke@52925 50 #include "gc/shenandoah/shenandoahVerifier.hpp"
rkennke@52925 51
rkennke@52925 52 #include "memory/iterator.hpp"
rkennke@52925 53 #include "memory/metaspace.hpp"
rkennke@52925 54 #include "memory/resourceArea.hpp"
rkennke@52925 55
rkennke@52925 56 /**
rkennke@52925 57 * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
rkennke@52925 58 * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
rkennke@52925 59 * is incremental-update-based.
rkennke@52925 60 *
rkennke@52925 61 * NOTE on interaction with TAMS: we want to avoid traversing new objects for
rkennke@52925 62 * several reasons:
rkennke@52925 63 * - We will not reclaim them in this cycle anyway, because they are not in the
rkennke@52925 64 * cset
rkennke@52925 65 * - It makes up for the bulk of work during final-pause
rkennke@52925 66 * - It also shortens the concurrent cycle because we don't need to
rkennke@52925 67 * pointlessly traverse through newly allocated objects.
rkennke@52925 68 * - As a nice side-effect, it solves the I-U termination problem (mutators
rkennke@52925 69 * cannot outrun the GC by allocating like crazy)
rkennke@52925 70 * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
rkennke@52925 71 * target object of stores if it's new. Treating new objects live implicitely
rkennke@52925 72 * achieves the same, but without extra barriers. I think the effect of
rkennke@52925 73 * shortened final-pause (mentioned above) is the main advantage of MWF. In
rkennke@52925 74 * particular, we will not see the head of a completely new long linked list
rkennke@52925 75 * in final-pause and end up traversing huge chunks of the heap there.
rkennke@52925 76 * - We don't need to see/update the fields of new objects either, because they
rkennke@52925 77 * are either still null, or anything that's been stored into them has been
rkennke@52925 78 * evacuated+enqueued before (and will thus be treated later).
rkennke@52925 79 *
rkennke@52925 80 * We achieve this by setting TAMS for each region, and everything allocated
rkennke@52925 81 * beyond TAMS will be 'implicitely marked'.
rkennke@52925 82 *
rkennke@52925 83 * Gotchas:
rkennke@52925 84 * - While we want new objects to be implicitely marked, we don't want to count
rkennke@52925 85 * them alive. Otherwise the next cycle wouldn't pick them up and consider
rkennke@52925 86 * them for cset. This means that we need to protect such regions from
rkennke@52925 87 * getting accidentally thrashed at the end of traversal cycle. This is why I
rkennke@52925 88 * keep track of alloc-regions and check is_alloc_region() in the trashing
rkennke@52925 89 * code.
rkennke@52925 90 * - We *need* to traverse through evacuated objects. Those objects are
rkennke@52925 91 * pre-existing, and any references in them point to interesting objects that
rkennke@52925 92 * we need to see. We also want to count them as live, because we just
rkennke@52925 93 * determined that they are alive :-) I achieve this by upping TAMS
rkennke@52925 94 * concurrently for every gclab/gc-shared alloc before publishing the
rkennke@52925 95 * evacuated object. This way, the GC threads will not consider such objects
rkennke@52925 96 * implictely marked, and traverse through them as normal.
rkennke@52925 97 */
rkennke@52925 98 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
rkennke@52925 99 private:
rkennke@52925 100 ShenandoahObjToScanQueue* _queue;
rkennke@52925 101 ShenandoahTraversalGC* _traversal_gc;
rkennke@52925 102 ShenandoahHeap* const _heap;
rkennke@52925 103
rkennke@52925 104 public:
rkennke@52925 105 ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
rkennke@52925 106 _queue(q),
rkennke@52925 107 _heap(ShenandoahHeap::heap())
rkennke@52925 108 { }
rkennke@52925 109
rkennke@52925 110 void do_buffer(void** buffer, size_t size) {
rkennke@52925 111 for (size_t i = 0; i < size; ++i) {
rkennke@52925 112 oop* p = (oop*) &buffer[i];
rkennke@52925 113 oop obj = RawAccess<>::oop_load(p);
rkennke@52925 114 shenandoah_assert_not_forwarded(p, obj);
rkennke@52925 115 if (_heap->marking_context()->mark(obj)) {
rkennke@52925 116 _queue->push(ShenandoahMarkTask(obj));
rkennke@52925 117 }
rkennke@52925 118 }
rkennke@52925 119 }
rkennke@52925 120 };
rkennke@52925 121
rkennke@52925 122 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
rkennke@52925 123 private:
rkennke@52925 124 ShenandoahTraversalSATBBufferClosure* _satb_cl;
rkennke@52925 125
rkennke@52925 126 public:
rkennke@52925 127 ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
rkennke@52925 128 _satb_cl(satb_cl) {}
rkennke@52925 129
rkennke@52925 130 void do_thread(Thread* thread) {
kbarrett@54006 131 ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
rkennke@52925 132 }
rkennke@52925 133 };
rkennke@52925 134
rkennke@52925 135 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
rkennke@52925 136 // and remark them later during final-traversal.
rkennke@52925 137 class ShenandoahMarkCLDClosure : public CLDClosure {
rkennke@52925 138 private:
rkennke@52925 139 OopClosure* _cl;
rkennke@52925 140 public:
rkennke@52925 141 ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
rkennke@52925 142 void do_cld(ClassLoaderData* cld) {
rkennke@52925 143 cld->oops_do(_cl, true, true);
rkennke@52925 144 }
rkennke@52925 145 };
rkennke@52925 146
rkennke@52925 147 // Like CLDToOopClosure, but only process modified CLDs
rkennke@52925 148 class ShenandoahRemarkCLDClosure : public CLDClosure {
rkennke@52925 149 private:
rkennke@52925 150 OopClosure* _cl;
rkennke@52925 151 public:
rkennke@52925 152 ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
rkennke@52925 153 void do_cld(ClassLoaderData* cld) {
rkennke@52925 154 if (cld->has_modified_oops()) {
rkennke@52925 155 cld->oops_do(_cl, true, true);
rkennke@52925 156 }
rkennke@52925 157 }
rkennke@52925 158 };
rkennke@52925 159
rkennke@52925 160 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
rkennke@52925 161 private:
rkennke@52925 162 ShenandoahRootProcessor* _rp;
rkennke@52925 163 ShenandoahHeap* _heap;
rkennke@52925 164 ShenandoahCsetCodeRootsIterator* _cset_coderoots;
rkennke@52925 165 public:
rkennke@52925 166 ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahCsetCodeRootsIterator* cset_coderoots) :
rkennke@52925 167 AbstractGangTask("Shenandoah Init Traversal Collection"),
rkennke@52925 168 _rp(rp),
rkennke@52925 169 _heap(ShenandoahHeap::heap()),
rkennke@52925 170 _cset_coderoots(cset_coderoots) {}
rkennke@52925 171
rkennke@52925 172 void work(uint worker_id) {
rkennke@52925 173 ShenandoahParallelWorkerSession worker_session(worker_id);
rkennke@52925 174
rkennke@52925 175 ShenandoahEvacOOMScope oom_evac_scope;
rkennke@52925 176 ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
rkennke@52925 177 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
rkennke@52925 178
rkennke@52925 179 bool process_refs = _heap->process_references();
rkennke@52925 180 bool unload_classes = _heap->unload_classes();
rkennke@52925 181 ReferenceProcessor* rp = NULL;
rkennke@52925 182 if (process_refs) {
rkennke@52925 183 rp = _heap->ref_processor();
rkennke@52925 184 }
rkennke@52925 185
rkennke@52925 186 // Step 1: Process ordinary GC roots.
rkennke@52925 187 {
rkennke@52925 188 ShenandoahTraversalClosure roots_cl(q, rp);
rkennke@52925 189 ShenandoahMarkCLDClosure cld_cl(&roots_cl);
rkennke@52925 190 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
rkennke@52925 191 if (unload_classes) {
zgu@54344 192 _rp->process_strong_roots(&roots_cl, &cld_cl, NULL, NULL, worker_id);
rkennke@52925 193 // Need to pre-evac code roots here. Otherwise we might see from-space constants.
rkennke@52925 194 ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times();
rkennke@52925 195 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
rkennke@52925 196 _cset_coderoots->possibly_parallel_blobs_do(&code_cl);
rkennke@52925 197 } else {
zgu@54338 198 _rp->process_all_roots(&roots_cl, &cld_cl, &code_cl, NULL, worker_id);
rkennke@52925 199 }
rkennke@54489 200 if (ShenandoahStringDedup::is_enabled()) {
rkennke@54489 201 AlwaysTrueClosure is_alive;
rkennke@54489 202 ShenandoahStringDedup::parallel_oops_do(&is_alive, &roots_cl, worker_id);
rkennke@54489 203 }
rkennke@52925 204 }
rkennke@52925 205 }
rkennke@52925 206 };
rkennke@52925 207
rkennke@52925 208 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
rkennke@52925 209 private:
rkennke@52925 210 ShenandoahTaskTerminator* _terminator;
rkennke@52925 211 ShenandoahHeap* _heap;
rkennke@52925 212 public:
rkennke@52925 213 ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) :
rkennke@52925 214 AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
rkennke@52925 215 _terminator(terminator),
rkennke@52925 216 _heap(ShenandoahHeap::heap()) {}
rkennke@52925 217
rkennke@52925 218 void work(uint worker_id) {
rkennke@52925 219 ShenandoahConcurrentWorkerSession worker_session(worker_id);
rkennke@52925 220 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
rkennke@52925 221 ShenandoahEvacOOMScope oom_evac_scope;
rkennke@52925 222 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
rkennke@52925 223
rkennke@52925 224 // Drain all outstanding work in queues.
rkennke@52925 225 traversal_gc->main_loop(worker_id, _terminator, true);
rkennke@52925 226 }
rkennke@52925 227 };
rkennke@52925 228
rkennke@52925 229 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
rkennke@52925 230 private:
rkennke@52925 231 ShenandoahRootProcessor* _rp;
rkennke@52925 232 ShenandoahTaskTerminator* _terminator;
rkennke@52925 233 ShenandoahHeap* _heap;
rkennke@52925 234 public:
rkennke@52925 235 ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahTaskTerminator* terminator) :
rkennke@52925 236 AbstractGangTask("Shenandoah Final Traversal Collection"),
rkennke@52925 237 _rp(rp),
rkennke@52925 238 _terminator(terminator),
rkennke@52925 239 _heap(ShenandoahHeap::heap()) {}
rkennke@52925 240
rkennke@52925 241 void work(uint worker_id) {
rkennke@52925 242 ShenandoahParallelWorkerSession worker_session(worker_id);
rkennke@52925 243
rkennke@52925 244 ShenandoahEvacOOMScope oom_evac_scope;
rkennke@52925 245 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
rkennke@52925 246
rkennke@52925 247 ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
rkennke@52925 248 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
rkennke@52925 249
rkennke@52925 250 bool process_refs = _heap->process_references();
rkennke@52925 251 bool unload_classes = _heap->unload_classes();
rkennke@52925 252 ReferenceProcessor* rp = NULL;
rkennke@52925 253 if (process_refs) {
rkennke@52925 254 rp = _heap->ref_processor();
rkennke@52925 255 }
rkennke@52925 256
rkennke@52925 257 // Step 0: Drain outstanding SATB queues.
rkennke@52925 258 // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
rkennke@52925 259 ShenandoahTraversalSATBBufferClosure satb_cl(q);
rkennke@52925 260 {
rkennke@52925 261 // Process remaining finished SATB buffers.
rkennke@52925 262 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
rkennke@52925 263 while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
rkennke@52925 264 // Process remaining threads SATB buffers below.
rkennke@52925 265 }
rkennke@52925 266
rkennke@52925 267 // Step 1: Process GC roots.
rkennke@52925 268 // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
rkennke@52925 269 // and the references to the oops are updated during init pause. New nmethods are handled
rkennke@52925 270 // in similar way during nmethod-register process. Therefore, we don't need to rescan code
rkennke@52925 271 // roots here.
rkennke@52925 272 if (!_heap->is_degenerated_gc_in_progress()) {
rkennke@52925 273 ShenandoahTraversalClosure roots_cl(q, rp);
rkennke@52925 274 CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
rkennke@52925 275 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
rkennke@52925 276 if (unload_classes) {
zgu@54344 277 ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
zgu@54344 278 _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
rkennke@52925 279 } else {
zgu@54338 280 _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
rkennke@52925 281 }
rkennke@52925 282 } else {
rkennke@52925 283 ShenandoahTraversalDegenClosure roots_cl(q, rp);
rkennke@52925 284 CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
rkennke@52925 285 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
rkennke@52925 286 if (unload_classes) {
zgu@54344 287 ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
zgu@54344 288 _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
rkennke@52925 289 } else {
zgu@54338 290 _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
rkennke@52925 291 }
rkennke@52925 292 }
rkennke@52925 293
rkennke@52925 294 {
rkennke@52925 295 ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
rkennke@52925 296 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
rkennke@52925 297
rkennke@52925 298 // Step 3: Finally drain all outstanding work in queues.
rkennke@52925 299 traversal_gc->main_loop(worker_id, _terminator, false);
rkennke@52925 300 }
rkennke@52925 301
rkennke@52925 302 }
rkennke@52925 303 };
rkennke@52925 304
rkennke@52925 305 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
rkennke@52925 306 _heap(heap),
rkennke@52925 307 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
rkennke@52925 308 _traversal_set(ShenandoahHeapRegionSet()) {
rkennke@52925 309
rkennke@52925 310 uint num_queues = heap->max_workers();
rkennke@52925 311 for (uint i = 0; i < num_queues; ++i) {
rkennke@52925 312 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
rkennke@52925 313 task_queue->initialize();
rkennke@52925 314 _task_queues->register_queue(i, task_queue);
rkennke@52925 315 }
rkennke@52925 316 }
rkennke@52925 317
rkennke@52925 318 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
rkennke@52925 319 }
rkennke@52925 320
rkennke@52925 321 void ShenandoahTraversalGC::prepare_regions() {
rkennke@52925 322 size_t num_regions = _heap->num_regions();
rkennke@52925 323 ShenandoahMarkingContext* const ctx = _heap->marking_context();
rkennke@52925 324 for (size_t i = 0; i < num_regions; i++) {
rkennke@52925 325 ShenandoahHeapRegion* region = _heap->get_region(i);
rkennke@52925 326 if (_heap->is_bitmap_slice_committed(region)) {
rkennke@52925 327 if (_traversal_set.is_in(i)) {
rkennke@52925 328 ctx->capture_top_at_mark_start(region);
rkennke@52925 329 region->clear_live_data();
rkennke@52925 330 assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared");
rkennke@52925 331 } else {
rkennke@52925 332 // Everything outside the traversal set is always considered live.
rkennke@52925 333 ctx->reset_top_at_mark_start(region);
rkennke@52925 334 }
rkennke@52925 335 } else {
rkennke@52925 336 // FreeSet may contain uncommitted empty regions, once they are recommitted,
rkennke@52925 337 // their TAMS may have old values, so reset them here.
rkennke@52925 338 ctx->reset_top_at_mark_start(region);
rkennke@52925 339 }
rkennke@52925 340 }
rkennke@52925 341 }
rkennke@52925 342
rkennke@52925 343 void ShenandoahTraversalGC::prepare() {
rkennke@52925 344 _heap->collection_set()->clear();
rkennke@52925 345 assert(_heap->collection_set()->count() == 0, "collection set not clear");
rkennke@52925 346
rkennke@52925 347 {
rkennke@52925 348 ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
rkennke@52925 349 _heap->make_parsable(true);
rkennke@52925 350 }
rkennke@52925 351
rkennke@52925 352 if (UseTLAB) {
rkennke@52925 353 ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs);
rkennke@52925 354 _heap->resize_tlabs();
rkennke@52925 355 }
rkennke@52925 356
rkennke@52925 357 assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
rkennke@52925 358 assert(!_heap->marking_context()->is_complete(), "should not be complete");
rkennke@52925 359
rkennke@52925 360 ShenandoahFreeSet* free_set = _heap->free_set();
rkennke@52925 361 ShenandoahCollectionSet* collection_set = _heap->collection_set();
rkennke@52925 362
rkennke@52925 363 // Find collection set
rkennke@52925 364 _heap->heuristics()->choose_collection_set(collection_set);
rkennke@52925 365 prepare_regions();
rkennke@52925 366
rkennke@52925 367 // Rebuild free set
rkennke@52925 368 free_set->rebuild();
rkennke@52925 369
rkennke@52925 370 log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "M, " SIZE_FORMAT "M CSet, " SIZE_FORMAT " CSet regions",
rkennke@52925 371 collection_set->garbage() / M, collection_set->live_data() / M, collection_set->count());
rkennke@52925 372 }
rkennke@52925 373
rkennke@52925 374 void ShenandoahTraversalGC::init_traversal_collection() {
rkennke@52925 375 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
rkennke@52925 376
rkennke@52925 377 if (ShenandoahVerify) {
rkennke@52925 378 _heap->verifier()->verify_before_traversal();
rkennke@52925 379 }
rkennke@52925 380
rkennke@52925 381 if (VerifyBeforeGC) {
rkennke@52925 382 Universe::verify();
rkennke@52925 383 }
rkennke@52925 384
rkennke@52925 385 {
rkennke@52925 386 ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
rkennke@52925 387 ShenandoahHeapLocker lock(_heap->lock());
rkennke@52925 388 prepare();
rkennke@52925 389 }
rkennke@52925 390
rkennke@52925 391 _heap->set_concurrent_traversal_in_progress(true);
rkennke@52925 392
rkennke@52925 393 bool process_refs = _heap->process_references();
rkennke@52925 394 if (process_refs) {
rkennke@52925 395 ReferenceProcessor* rp = _heap->ref_processor();
rkennke@52925 396 rp->enable_discovery(true /*verify_no_refs*/);
rkennke@52925 397 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
rkennke@52925 398 }
rkennke@52925 399
rkennke@52925 400 {
rkennke@52925 401 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
rkennke@52925 402 assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
rkennke@52925 403 TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
rkennke@52925 404
rkennke@52925 405 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 406 DerivedPointerTable::clear();
rkennke@52925 407 #endif
rkennke@52925 408
rkennke@52925 409 {
rkennke@52925 410 uint nworkers = _heap->workers()->active_workers();
rkennke@52925 411 task_queues()->reserve(nworkers);
rkennke@52925 412 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
rkennke@52925 413
rkennke@52925 414 ShenandoahCsetCodeRootsIterator cset_coderoots = ShenandoahCodeRoots::cset_iterator();
rkennke@52925 415
rkennke@52925 416 ShenandoahInitTraversalCollectionTask traversal_task(&rp, &cset_coderoots);
rkennke@52925 417 _heap->workers()->run_task(&traversal_task);
rkennke@52925 418 }
rkennke@52925 419
rkennke@52925 420 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 421 DerivedPointerTable::update_pointers();
rkennke@52925 422 #endif
rkennke@52925 423 }
rkennke@52925 424
rkennke@52925 425 if (ShenandoahPacing) {
rkennke@52925 426 _heap->pacer()->setup_for_traversal();
rkennke@52925 427 }
rkennke@52925 428 }
rkennke@52925 429
rkennke@52925 430 void ShenandoahTraversalGC::main_loop(uint w, ShenandoahTaskTerminator* t, bool sts_yield) {
rkennke@52925 431 ShenandoahObjToScanQueue* q = task_queues()->queue(w);
rkennke@52925 432
rkennke@52925 433 // Initialize live data.
rkennke@52925 434 jushort* ld = _heap->get_liveness_cache(w);
rkennke@52925 435
rkennke@52925 436 ReferenceProcessor* rp = NULL;
rkennke@52925 437 if (_heap->process_references()) {
rkennke@52925 438 rp = _heap->ref_processor();
rkennke@52925 439 }
rkennke@52925 440 {
rkennke@52925 441 if (!_heap->is_degenerated_gc_in_progress()) {
rkennke@52925 442 if (_heap->unload_classes()) {
rkennke@52925 443 if (ShenandoahStringDedup::is_enabled()) {
rkennke@52925 444 ShenandoahTraversalMetadataDedupClosure cl(q, rp);
rkennke@52925 445 main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 446 } else {
rkennke@52925 447 ShenandoahTraversalMetadataClosure cl(q, rp);
rkennke@52925 448 main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 449 }
rkennke@52925 450 } else {
rkennke@52925 451 if (ShenandoahStringDedup::is_enabled()) {
rkennke@52925 452 ShenandoahTraversalDedupClosure cl(q, rp);
rkennke@52925 453 main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 454 } else {
rkennke@52925 455 ShenandoahTraversalClosure cl(q, rp);
rkennke@52925 456 main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 457 }
rkennke@52925 458 }
rkennke@52925 459 } else {
rkennke@52925 460 if (_heap->unload_classes()) {
rkennke@52925 461 if (ShenandoahStringDedup::is_enabled()) {
rkennke@52925 462 ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp);
rkennke@52925 463 main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 464 } else {
rkennke@52925 465 ShenandoahTraversalMetadataDegenClosure cl(q, rp);
rkennke@52925 466 main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 467 }
rkennke@52925 468 } else {
rkennke@52925 469 if (ShenandoahStringDedup::is_enabled()) {
rkennke@52925 470 ShenandoahTraversalDedupDegenClosure cl(q, rp);
rkennke@52925 471 main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 472 } else {
rkennke@52925 473 ShenandoahTraversalDegenClosure cl(q, rp);
rkennke@52925 474 main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t, sts_yield);
rkennke@52925 475 }
rkennke@52925 476 }
rkennke@52925 477 }
rkennke@52925 478 }
rkennke@52925 479
rkennke@52925 480 _heap->flush_liveness_cache(w);
rkennke@52925 481 }
rkennke@52925 482
rkennke@52925 483 template <class T>
rkennke@52925 484 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator* terminator, bool sts_yield) {
rkennke@52925 485 ShenandoahObjToScanQueueSet* queues = task_queues();
rkennke@52925 486 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
rkennke@52925 487 ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark();
rkennke@52925 488
rkennke@52925 489 uintx stride = ShenandoahMarkLoopStride;
rkennke@52925 490
rkennke@52925 491 ShenandoahMarkTask task;
rkennke@52925 492
rkennke@52925 493 // Process outstanding queues, if any.
rkennke@52925 494 q = queues->claim_next();
rkennke@52925 495 while (q != NULL) {
rkennke@52925 496 if (_heap->check_cancelled_gc_and_yield(sts_yield)) {
rkennke@52925 497 return;
rkennke@52925 498 }
rkennke@52925 499
rkennke@52925 500 for (uint i = 0; i < stride; i++) {
rkennke@52925 501 if (q->pop(task)) {
rkennke@52925 502 conc_mark->do_task<T>(q, cl, live_data, &task);
rkennke@52925 503 } else {
rkennke@52925 504 assert(q->is_empty(), "Must be empty");
rkennke@52925 505 q = queues->claim_next();
rkennke@52925 506 break;
rkennke@52925 507 }
rkennke@52925 508 }
rkennke@52925 509 }
rkennke@52925 510
rkennke@52925 511 if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
rkennke@52925 512
rkennke@52925 513 // Normal loop.
rkennke@52925 514 q = queues->queue(worker_id);
rkennke@52925 515
rkennke@52925 516 ShenandoahTraversalSATBBufferClosure drain_satb(q);
rkennke@52925 517 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
rkennke@52925 518
rkennke@52925 519 while (true) {
rkennke@52925 520 if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
rkennke@52925 521
rkennke@52925 522 while (satb_mq_set.completed_buffers_num() > 0) {
rkennke@52925 523 satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
rkennke@52925 524 }
rkennke@52925 525
rkennke@52925 526 uint work = 0;
rkennke@52925 527 for (uint i = 0; i < stride; i++) {
rkennke@52925 528 if (q->pop(task) ||
rkennke@52925 529 queues->steal(worker_id, task)) {
rkennke@52925 530 conc_mark->do_task<T>(q, cl, live_data, &task);
rkennke@52925 531 work++;
rkennke@52925 532 } else {
rkennke@52925 533 break;
rkennke@52925 534 }
rkennke@52925 535 }
rkennke@52925 536
rkennke@52925 537 if (work == 0) {
rkennke@52925 538 // No more work, try to terminate
rkennke@52925 539 ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
rkennke@52925 540 ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
rkennke@52925 541 ShenandoahTerminationTimingsTracker term_tracker(worker_id);
zgu@53015 542 ShenandoahTerminatorTerminator tt(_heap);
zgu@53015 543
zgu@53015 544 if (terminator->offer_termination(&tt)) return;
rkennke@52925 545 }
rkennke@52925 546 }
rkennke@52925 547 }
rkennke@52925 548
rkennke@52925 549 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ShenandoahTaskTerminator* terminator, bool sts_yield) {
rkennke@52925 550 if (_heap->cancelled_gc()) {
rkennke@52925 551 return true;
rkennke@52925 552 }
rkennke@52925 553 return false;
rkennke@52925 554 }
rkennke@52925 555
rkennke@52925 556 void ShenandoahTraversalGC::concurrent_traversal_collection() {
rkennke@52925 557 ClassLoaderDataGraph::clear_claimed_marks();
rkennke@52925 558
rkennke@52925 559 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
rkennke@52925 560 if (!_heap->cancelled_gc()) {
rkennke@52925 561 uint nworkers = _heap->workers()->active_workers();
rkennke@52925 562 task_queues()->reserve(nworkers);
rkennke@52925 563 ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination);
rkennke@52925 564
rkennke@52925 565 ShenandoahTaskTerminator terminator(nworkers, task_queues());
rkennke@52925 566 ShenandoahConcurrentTraversalCollectionTask task(&terminator);
rkennke@52925 567 _heap->workers()->run_task(&task);
rkennke@52925 568 }
rkennke@52925 569
rkennke@52925 570 if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) {
rkennke@52925 571 preclean_weak_refs();
rkennke@52925 572 }
rkennke@52925 573 }
rkennke@52925 574
rkennke@52925 575 void ShenandoahTraversalGC::final_traversal_collection() {
rkennke@52925 576 _heap->make_parsable(true);
rkennke@52925 577
rkennke@52925 578 if (!_heap->cancelled_gc()) {
rkennke@52925 579 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 580 DerivedPointerTable::clear();
rkennke@52925 581 #endif
rkennke@52925 582 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
rkennke@52925 583 uint nworkers = _heap->workers()->active_workers();
rkennke@52925 584 task_queues()->reserve(nworkers);
rkennke@52925 585
rkennke@52925 586 // Finish traversal
rkennke@52925 587 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
rkennke@52925 588 ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
rkennke@52925 589
rkennke@52925 590 ShenandoahTaskTerminator terminator(nworkers, task_queues());
rkennke@52925 591 ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
rkennke@52925 592 _heap->workers()->run_task(&task);
rkennke@52925 593 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 594 DerivedPointerTable::update_pointers();
rkennke@52925 595 #endif
rkennke@52925 596 }
rkennke@52925 597
rkennke@52925 598 if (!_heap->cancelled_gc() && _heap->process_references()) {
rkennke@52925 599 weak_refs_work();
rkennke@52925 600 }
rkennke@52925 601
zgu@54338 602 if (!_heap->cancelled_gc()) {
rkennke@54479 603 fixup_roots();
zgu@54338 604 if (_heap->unload_classes()) {
zgu@54338 605 _heap->unload_classes_and_cleanup_tables(false);
zgu@54338 606 }
rkennke@52925 607 }
rkennke@52925 608
rkennke@52925 609 if (!_heap->cancelled_gc()) {
rkennke@52925 610 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
rkennke@52925 611 TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
rkennke@52925 612 TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
rkennke@52925 613
rkennke@52925 614 // No more marking expected
rkennke@52925 615 _heap->mark_complete_marking_context();
rkennke@52925 616
rkennke@52925 617 // Resize metaspace
rkennke@52925 618 MetaspaceGC::compute_new_size();
rkennke@52925 619
rkennke@52925 620 // Still good? We can now trash the cset, and make final verification
rkennke@52925 621 {
rkennke@52925 622 ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
rkennke@52925 623 ShenandoahHeapLocker lock(_heap->lock());
rkennke@52925 624
rkennke@52925 625 // Trash everything
rkennke@52925 626 // Clear immediate garbage regions.
rkennke@52925 627 size_t num_regions = _heap->num_regions();
rkennke@52925 628
rkennke@52925 629 ShenandoahHeapRegionSet* traversal_regions = traversal_set();
rkennke@52925 630 ShenandoahFreeSet* free_regions = _heap->free_set();
rkennke@52925 631 ShenandoahMarkingContext* const ctx = _heap->marking_context();
rkennke@52925 632 free_regions->clear();
rkennke@52925 633 for (size_t i = 0; i < num_regions; i++) {
rkennke@52925 634 ShenandoahHeapRegion* r = _heap->get_region(i);
rkennke@52925 635 bool not_allocated = ctx->top_at_mark_start(r) == r->top();
rkennke@52925 636
rkennke@52925 637 bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
rkennke@52925 638 if (r->is_humongous_start() && candidate) {
rkennke@52925 639 // Trash humongous.
rkennke@52925 640 HeapWord* humongous_obj = r->bottom() + ShenandoahBrooksPointer::word_size();
rkennke@52925 641 assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
rkennke@52925 642 r->make_trash_immediate();
rkennke@52925 643 while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
rkennke@52925 644 i++;
rkennke@52925 645 r = _heap->get_region(i);
rkennke@52925 646 assert(r->is_humongous_continuation(), "must be humongous continuation");
rkennke@52925 647 r->make_trash_immediate();
rkennke@52925 648 }
rkennke@52925 649 } else if (!r->is_empty() && candidate) {
rkennke@52925 650 // Trash regular.
rkennke@52925 651 assert(!r->is_humongous(), "handled above");
rkennke@52925 652 assert(!r->is_trash(), "must not already be trashed");
rkennke@52925 653 r->make_trash_immediate();
rkennke@52925 654 }
rkennke@52925 655 }
rkennke@52925 656 _heap->collection_set()->clear();
rkennke@52925 657 _heap->free_set()->rebuild();
rkennke@52925 658 reset();
rkennke@52925 659 }
rkennke@52925 660
rkennke@52925 661 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
rkennke@52925 662 _heap->set_concurrent_traversal_in_progress(false);
rkennke@52925 663 assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here");
rkennke@52925 664
rkennke@52925 665 if (ShenandoahVerify) {
rkennke@52925 666 _heap->verifier()->verify_after_traversal();
rkennke@52925 667 }
rkennke@52925 668
rkennke@52925 669 if (VerifyAfterGC) {
rkennke@52925 670 Universe::verify();
rkennke@52925 671 }
rkennke@52925 672 }
rkennke@52925 673 }
rkennke@52925 674
rkennke@52925 675 class ShenandoahTraversalFixRootsClosure : public OopClosure {
rkennke@52925 676 private:
rkennke@52925 677 template <class T>
rkennke@52925 678 inline void do_oop_work(T* p) {
rkennke@52925 679 T o = RawAccess<>::oop_load(p);
rkennke@52925 680 if (!CompressedOops::is_null(o)) {
rkennke@52925 681 oop obj = CompressedOops::decode_not_null(o);
rkennke@52925 682 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
rkennke@52925 683 if (!oopDesc::equals_raw(obj, forw)) {
rkennke@52925 684 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
rkennke@52925 685 }
rkennke@52925 686 }
rkennke@52925 687 }
rkennke@52925 688
rkennke@52925 689 public:
rkennke@52925 690 inline void do_oop(oop* p) { do_oop_work(p); }
rkennke@52925 691 inline void do_oop(narrowOop* p) { do_oop_work(p); }
rkennke@52925 692 };
rkennke@52925 693
rkennke@52925 694 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
rkennke@52925 695 private:
rkennke@52925 696 ShenandoahRootProcessor* _rp;
rkennke@52925 697
rkennke@52925 698 public:
rkennke@52925 699 ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
rkennke@52925 700 AbstractGangTask("Shenandoah traversal fix roots"),
zgu@54344 701 _rp(rp) {
zgu@54344 702 assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
zgu@54344 703 }
rkennke@52925 704
rkennke@52925 705 void work(uint worker_id) {
rkennke@52925 706 ShenandoahParallelWorkerSession worker_session(worker_id);
rkennke@52925 707 ShenandoahTraversalFixRootsClosure cl;
rkennke@52925 708 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
rkennke@52925 709 CLDToOopClosure cldCl(&cl, ClassLoaderData::_claim_strong);
zgu@54344 710 _rp->update_all_roots<ShenandoahForwardedIsAliveClosure>(&cl, &cldCl, &blobsCl, NULL, worker_id);
rkennke@52925 711 }
rkennke@52925 712 };
rkennke@52925 713
rkennke@52925 714 void ShenandoahTraversalGC::fixup_roots() {
rkennke@52925 715 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 716 DerivedPointerTable::clear();
rkennke@52925 717 #endif
rkennke@52925 718 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
rkennke@52925 719 ShenandoahTraversalFixRootsTask update_roots_task(&rp);
rkennke@52925 720 _heap->workers()->run_task(&update_roots_task);
rkennke@52925 721 #if defined(COMPILER2) || INCLUDE_JVMCI
rkennke@52925 722 DerivedPointerTable::update_pointers();
rkennke@52925 723 #endif
rkennke@52925 724 }
rkennke@52925 725
rkennke@52925 726 void ShenandoahTraversalGC::reset() {
rkennke@52925 727 _task_queues->clear();
rkennke@52925 728 }
rkennke@52925 729
rkennke@52925 730 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
rkennke@52925 731 return _task_queues;
rkennke@52925 732 }
rkennke@52925 733
rkennke@52925 734 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
rkennke@52925 735 private:
rkennke@52925 736 ShenandoahHeap* const _heap;
rkennke@52925 737 public:
rkennke@52925 738 ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
rkennke@52925 739 virtual bool should_return() { return _heap->cancelled_gc(); }
rkennke@52925 740 };
rkennke@52925 741
rkennke@52925 742 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
rkennke@52925 743 public:
rkennke@52925 744 void do_void() {
rkennke@52925 745 ShenandoahHeap* sh = ShenandoahHeap::heap();
rkennke@52925 746 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
rkennke@52925 747 assert(sh->process_references(), "why else would we be here?");
rkennke@52925 748 ShenandoahTaskTerminator terminator(1, traversal_gc->task_queues());
rkennke@52925 749 shenandoah_assert_rp_isalive_installed();
rkennke@52925 750 traversal_gc->main_loop((uint) 0, &terminator, true);
rkennke@52925 751 }
rkennke@52925 752 };
rkennke@52925 753
rkennke@52925 754 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
rkennke@52925 755 private:
rkennke@52925 756 ShenandoahObjToScanQueue* _queue;
rkennke@52925 757 Thread* _thread;
rkennke@52925 758 ShenandoahTraversalGC* _traversal_gc;
rkennke@52925 759 ShenandoahMarkingContext* const _mark_context;
rkennke@52925 760
rkennke@52925 761 template <class T>
rkennke@52925 762 inline void do_oop_work(T* p) {
rkennke@52925 763 _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
rkennke@52925 764 }
rkennke@52925 765
rkennke@52925 766 public:
rkennke@52925 767 ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
rkennke@52925 768 _queue(q), _thread(Thread::current()),
rkennke@52925 769 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
rkennke@52925 770 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
rkennke@52925 771
rkennke@52925 772 void do_oop(narrowOop* p) { do_oop_work(p); }
rkennke@52925 773 void do_oop(oop* p) { do_oop_work(p); }
rkennke@52925 774 };
rkennke@52925 775
rkennke@52925 776 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
rkennke@52925 777 private:
rkennke@52925 778 ShenandoahObjToScanQueue* _queue;
rkennke@52925 779 Thread* _thread;
rkennke@52925 780 ShenandoahTraversalGC* _traversal_gc;
rkennke@52925 781 ShenandoahMarkingContext* const _mark_context;
rkennke@52925 782
rkennke@52925 783 template <class T>
rkennke@52925 784 inline void do_oop_work(T* p) {
rkennke@52925 785 _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
rkennke@52925 786 }
rkennke@52925 787
rkennke@52925 788 public:
rkennke@52925 789 ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
rkennke@52925 790 _queue(q), _thread(Thread::current()),
rkennke@52925 791 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
rkennke@52925 792 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
rkennke@52925 793
rkennke@52925 794 void do_oop(narrowOop* p) { do_oop_work(p); }
rkennke@52925 795 void do_oop(oop* p) { do_oop_work(p); }
rkennke@52925 796 };
rkennke@52925 797
rkennke@52925 798 class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure {
rkennke@52925 799 private:
rkennke@52925 800 ShenandoahObjToScanQueue* _queue;
rkennke@52925 801 Thread* _thread;
rkennke@52925 802 ShenandoahTraversalGC* _traversal_gc;
rkennke@52925 803 ShenandoahMarkingContext* const _mark_context;
rkennke@52925 804
rkennke@52925 805 template <class T>
rkennke@52925 806 inline void do_oop_work(T* p) {
rkennke@52925 807 ShenandoahEvacOOMScope evac_scope;
rkennke@52925 808 _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
rkennke@52925 809 }
rkennke@52925 810
rkennke@52925 811 public:
rkennke@52925 812 ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
rkennke@52925 813 _queue(q), _thread(Thread::current()),
rkennke@52925 814 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
rkennke@52925 815 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
rkennke@52925 816
rkennke@52925 817 void do_oop(narrowOop* p) { do_oop_work(p); }
rkennke@52925 818 void do_oop(oop* p) { do_oop_work(p); }
rkennke@52925 819 };
rkennke@52925 820
rkennke@52925 821 class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure {
rkennke@52925 822 private:
rkennke@52925 823 ShenandoahObjToScanQueue* _queue;
rkennke@52925 824 Thread* _thread;
rkennke@52925 825 ShenandoahTraversalGC* _traversal_gc;
rkennke@52925 826 ShenandoahMarkingContext* const _mark_context;
rkennke@52925 827
rkennke@52925 828 template <class T>
rkennke@52925 829 inline void do_oop_work(T* p) {
rkennke@52925 830 ShenandoahEvacOOMScope evac_scope;
rkennke@52925 831 _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
rkennke@52925 832 }
rkennke@52925 833
rkennke@52925 834 public:
rkennke@52925 835 ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
rkennke@52925 836 _queue(q), _thread(Thread::current()),
rkennke@52925 837 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
rkennke@52925 838 _mark_context(ShenandoahHeap::heap()->marking_context()) {}
rkennke@52925 839
rkennke@52925 840 void do_oop(narrowOop* p) { do_oop_work(p); }
rkennke@52925 841 void do_oop(oop* p) { do_oop_work(p); }
rkennke@52925 842 };
rkennke@52925 843
rkennke@52925 844 class ShenandoahTraversalPrecleanTask : public AbstractGangTask {
rkennke@52925 845 private:
rkennke@52925 846 ReferenceProcessor* _rp;
rkennke@52925 847
rkennke@52925 848 public:
rkennke@52925 849 ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) :
rkennke@52925 850 AbstractGangTask("Precleaning task"),
rkennke@52925 851 _rp(rp) {}
rkennke@52925 852
rkennke@52925 853 void work(uint worker_id) {
rkennke@52925 854 assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
rkennke@52925 855 ShenandoahParallelWorkerSession worker_session(worker_id);
rkennke@52925 856 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
rkennke@52925 857 ShenandoahEvacOOMScope oom_evac_scope;
rkennke@52925 858
rkennke@52925 859 ShenandoahHeap* sh = ShenandoahHeap::heap();
rkennke@52925 860
rkennke@52925 861 ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id);
rkennke@52925 862
rkennke@52925 863 ShenandoahForwardedIsAliveClosure is_alive;
rkennke@52925 864 ShenandoahTraversalCancelledGCYieldClosure yield;
rkennke@52925 865 ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
rkennke@52925 866 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q);
rkennke@52925 867 ResourceMark rm;
rkennke@52925 868 _rp->preclean_discovered_references(&is_alive, &keep_alive,
rkennke@52925 869 &complete_gc, &yield,
rkennke@52925 870 NULL);
rkennke@52925 871 }
rkennke@52925 872 };
rkennke@52925 873
rkennke@52925 874 void ShenandoahTraversalGC::preclean_weak_refs() {
rkennke@52925 875 // Pre-cleaning weak references before diving into STW makes sense at the
rkennke@52925 876 // end of concurrent mark. This will filter out the references which referents
rkennke@52925 877 // are alive. Note that ReferenceProcessor already filters out these on reference
rkennke@52925 878 // discovery, and the bulk of work is done here. This phase processes leftovers
rkennke@52925 879 // that missed the initial filtering, i.e. when referent was marked alive after
rkennke@52925 880 // reference was discovered by RP.
rkennke@52925 881
rkennke@52925 882 assert(_heap->process_references(), "sanity");
rkennke@52925 883 assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
rkennke@52925 884
rkennke@52925 885 // Shortcut if no references were discovered to avoid winding up threads.
rkennke@52925 886 ReferenceProcessor* rp = _heap->ref_processor();
rkennke@52925 887 if (!rp->has_discovered_references()) {
rkennke@52925 888 return;
rkennke@52925 889 }
rkennke@52925 890
rkennke@52925 891 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
rkennke@52925 892
rkennke@52925 893 shenandoah_assert_rp_isalive_not_installed();
rkennke@52925 894 ShenandoahForwardedIsAliveClosure is_alive;
rkennke@52925 895 ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
rkennke@52925 896
rkennke@52925 897 assert(task_queues()->is_empty(), "Should be empty");
rkennke@52925 898
rkennke@52925 899 // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
rkennke@52925 900 // queues and other goodies. When upstream ReferenceProcessor starts supporting
rkennke@52925 901 // parallel precleans, we can extend this to more threads.
rkennke@52925 902 ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false);
rkennke@52925 903
rkennke@52925 904 WorkGang* workers = _heap->workers();
rkennke@52925 905 uint nworkers = workers->active_workers();
rkennke@52925 906 assert(nworkers == 1, "This code uses only a single worker");
rkennke@52925 907 task_queues()->reserve(nworkers);
rkennke@52925 908
rkennke@52925 909 ShenandoahTraversalPrecleanTask task(rp);
rkennke@52925 910 workers->run_task(&task);
rkennke@52925 911
rkennke@52925 912 assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty");
rkennke@52925 913 }
rkennke@52925 914
rkennke@52925 915 // Weak Reference Closures
rkennke@52925 916 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
rkennke@52925 917 uint _worker_id;
rkennke@52925 918 ShenandoahTaskTerminator* _terminator;
rkennke@52925 919 bool _reset_terminator;
rkennke@52925 920
rkennke@52925 921 public:
rkennke@52925 922 ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
rkennke@52925 923 _worker_id(worker_id),
rkennke@52925 924 _terminator(t),
rkennke@52925 925 _reset_terminator(reset_terminator) {
rkennke@52925 926 }
rkennke@52925 927
rkennke@52925 928 void do_void() {
rkennke@52925 929 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
rkennke@52925 930
rkennke@52925 931 ShenandoahHeap* sh = ShenandoahHeap::heap();
rkennke@52925 932 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
rkennke@52925 933 assert(sh->process_references(), "why else would we be here?");
rkennke@52925 934 shenandoah_assert_rp_isalive_installed();
rkennke@52925 935
rkennke@52925 936 traversal_gc->main_loop(_worker_id, _terminator, false);
rkennke@52925 937
rkennke@52925 938 if (_reset_terminator) {
rkennke@52925 939 _terminator->reset_for_reuse();
rkennke@52925 940 }
rkennke@52925 941 }
rkennke@52925 942 };
rkennke@52925 943
rkennke@52925 944 class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure {
rkennke@52925 945 uint _worker_id;
rkennke@52925 946 ShenandoahTaskTerminator* _terminator;
rkennke@52925 947 bool _reset_terminator;
rkennke@52925 948
rkennke@52925 949 public:
rkennke@52925 950 ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
rkennke@52925 951 _worker_id(worker_id),
rkennke@52925 952 _terminator(t),
rkennke@52925 953 _reset_terminator(reset_terminator) {
rkennke@52925 954 }
rkennke@52925 955
rkennke@52925 956 void do_void() {
rkennke@52925 957 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
rkennke@52925 958
rkennke@52925 959 ShenandoahHeap* sh = ShenandoahHeap::heap();
rkennke@52925 960 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
rkennke@52925 961 assert(sh->process_references(), "why else would we be here?");
rkennke@52925 962 shenandoah_assert_rp_isalive_installed();
rkennke@52925 963
rkennke@52925 964 ShenandoahEvacOOMScope evac_scope;
rkennke@52925 965 traversal_gc->main_loop(_worker_id, _terminator, false);
rkennke@52925 966
rkennke@52925 967 if (_reset_terminator) {
rkennke@52925 968 _terminator->reset_for_reuse();
rkennke@52925 969 }
rkennke@52925 970 }
rkennke@52925 971 };
rkennke@52925 972
rkennke@52925 973 void ShenandoahTraversalGC::weak_refs_work() {
rkennke@52925 974 assert(_heap->process_references(), "sanity");
rkennke@52925 975
rkennke@52925 976 ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
rkennke@52925 977
rkennke@52925 978 ShenandoahGCPhase phase(phase_root);
rkennke@52925 979
rkennke@52925 980 ReferenceProcessor* rp = _heap->ref_processor();
rkennke@52925 981
rkennke@52925 982 // NOTE: We cannot shortcut on has_discovered_references() here, because
rkennke@52925 983 // we will miss marking JNI Weak refs then, see implementation in
rkennke@52925 984 // ReferenceProcessor::process_discovered_references.
rkennke@52925 985 weak_refs_work_doit();
rkennke@52925 986
rkennke@52925 987 rp->verify_no_references_recorded();
rkennke@52925 988 assert(!rp->discovery_enabled(), "Post condition");
rkennke@52925 989
rkennke@52925 990 }
rkennke@52925 991
rkennke@52925 992 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
rkennke@52925 993 private:
rkennke@52925 994 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
rkennke@52925 995 ShenandoahTaskTerminator* _terminator;
rkennke@52925 996
rkennke@52925 997 public:
rkennke@52925 998 ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
rkennke@52925 999 ShenandoahTaskTerminator* t) :
rkennke@52925 1000 AbstractGangTask("Process reference objects in parallel"),
rkennke@52925 1001 _proc_task(proc_task),
rkennke@52925 1002 _terminator(t) {
rkennke@52925 1003 }
rkennke@52925 1004
rkennke@52925 1005 void work(uint worker_id) {
rkennke@52925 1006 ShenandoahEvacOOMScope oom_evac_scope;
rkennke@52925 1007 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
rkennke@52925 1008 ShenandoahHeap* heap = ShenandoahHeap::heap();
rkennke@52925 1009 ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
rkennke@52925 1010
rkennke@52925 1011 ShenandoahForwardedIsAliveClosure is_alive;
rkennke@52925 1012 if (!heap->is_degenerated_gc_in_progress()) {
rkennke@52925 1013 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
rkennke@52925 1014 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
rkennke@52925 1015 } else {
rkennke@52925 1016 ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
rkennke@52925 1017 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
rkennke@52925 1018 }
rkennke@52925 1019 }
rkennke@52925 1020 };
rkennke@52925 1021
rkennke@52925 1022 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
rkennke@52925 1023 private:
rkennke@52925 1024 WorkGang* _workers;
rkennke@52925 1025
rkennke@52925 1026 public:
rkennke@52925 1027 ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {}
rkennke@52925 1028
rkennke@52925 1029 // Executes a task using worker threads.
rkennke@52925 1030 void execute(ProcessTask& task, uint ergo_workers) {
rkennke@52925 1031 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
rkennke@52925 1032
rkennke@52925 1033 ShenandoahHeap* heap = ShenandoahHeap::heap();
rkennke@52925 1034 ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
rkennke@52925 1035 ShenandoahPushWorkerQueuesScope scope(_workers,
rkennke@52925 1036 traversal_gc->task_queues(),
rkennke@52925 1037 ergo_workers,
rkennke@52925 1038 /* do_check = */ false);
rkennke@52925 1039 uint nworkers = _workers->active_workers();
rkennke@52925 1040 traversal_gc->task_queues()->reserve(nworkers);
rkennke@52925 1041 ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues());
rkennke@52925 1042 ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
rkennke@52925 1043 _workers->run_task(&proc_task_proxy);
rkennke@52925 1044 }
rkennke@52925 1045 };
rkennke@52925 1046
rkennke@52925 1047 void ShenandoahTraversalGC::weak_refs_work_doit() {
rkennke@52925 1048 ReferenceProcessor* rp = _heap->ref_processor();
rkennke@52925 1049
rkennke@52925 1050 ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
rkennke@52925 1051
rkennke@52925 1052 shenandoah_assert_rp_isalive_not_installed();
rkennke@52925 1053 ShenandoahForwardedIsAliveClosure is_alive;
rkennke@52925 1054 ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
rkennke@52925 1055
rkennke@52925 1056 WorkGang* workers = _heap->workers();
rkennke@52925 1057 uint nworkers = workers->active_workers();
rkennke@52925 1058
rkennke@52925 1059 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
rkennke@52925 1060 rp->set_active_mt_degree(nworkers);
rkennke@52925 1061
rkennke@52925 1062 assert(task_queues()->is_empty(), "Should be empty");
rkennke@52925 1063
rkennke@52925 1064 // complete_gc and keep_alive closures instantiated here are only needed for
rkennke@52925 1065 // single-threaded path in RP. They share the queue 0 for tracking work, which
rkennke@52925 1066 // simplifies implementation. Since RP may decide to call complete_gc several
rkennke@52925 1067 // times, we need to be able to reuse the terminator.
rkennke@52925 1068 uint serial_worker_id = 0;
rkennke@52925 1069 ShenandoahTaskTerminator terminator(1, task_queues());
rkennke@52925 1070 ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
rkennke@52925 1071 ShenandoahPushWorkerQueuesScope scope(workers, task_queues(), 1, /* do_check = */ false);
rkennke@52925 1072
rkennke@52925 1073 ShenandoahTraversalRefProcTaskExecutor executor(workers);
rkennke@52925 1074
rkennke@52925 1075 ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
rkennke@52925 1076 if (!_heap->is_degenerated_gc_in_progress()) {
rkennke@52925 1077 ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
rkennke@52925 1078 rp->process_discovered_references(&is_alive, &keep_alive,
rkennke@52925 1079 &complete_gc, &executor,
rkennke@52925 1080 &pt);
rkennke@52925 1081 } else {
rkennke@52925 1082 ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id));
rkennke@52925 1083 rp->process_discovered_references(&is_alive, &keep_alive,
rkennke@52925 1084 &complete_gc, &executor,
rkennke@52925 1085 &pt);
rkennke@52925 1086 }
rkennke@52925 1087
rkennke@54479 1088 pt.print_all_references();
rkennke@54479 1089 assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
rkennke@52925 1090 }