annotate src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @ 54197:ddfb658c8ce3

8147502: Digest is incorrectly truncated for ECDSA signatures when the bit length of n is less than the field size Summary: Truncate the digest according to the group order, not the field size Reviewed-by: jnimeh
author apetcher
date Tue, 19 Mar 2019 15:18:35 -0400
parents 0da5c17f5e43
children 1f9ad92e337b
rev   line source
ysr@1374 1 /*
zgu@53348 2 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
ysr@1374 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
ysr@1374 4 *
ysr@1374 5 * This code is free software; you can redistribute it and/or modify it
ysr@1374 6 * under the terms of the GNU General Public License version 2 only, as
ysr@1374 7 * published by the Free Software Foundation.
ysr@1374 8 *
ysr@1374 9 * This code is distributed in the hope that it will be useful, but WITHOUT
ysr@1374 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
ysr@1374 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
ysr@1374 12 * version 2 for more details (a copy is included in the LICENSE file that
ysr@1374 13 * accompanied this code).
ysr@1374 14 *
ysr@1374 15 * You should have received a copy of the GNU General Public License version
ysr@1374 16 * 2 along with this work; if not, write to the Free Software Foundation,
ysr@1374 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
ysr@1374 18 *
trims@5547 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@5547 20 * or visit www.oracle.com if you need additional information or have any
trims@5547 21 * questions.
ysr@1374 22 *
ysr@1374 23 */
ysr@1374 24
stefank@7397 25 #include "precompiled.hpp"
coleenp@51959 26 #include "classfile/classLoaderDataGraph.hpp"
stefank@25492 27 #include "code/codeCache.hpp"
pliden@49751 28 #include "gc/g1/g1BarrierSet.hpp"
pliden@30764 29 #include "gc/g1/g1CollectedHeap.inline.hpp"
drwhite@31331 30 #include "gc/g1/g1CollectorState.hpp"
ehelin@35943 31 #include "gc/g1/g1ConcurrentMark.inline.hpp"
lkorinth@49659 32 #include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
kbarrett@53747 33 #include "gc/g1/g1DirtyCardQueue.hpp"
david@35851 34 #include "gc/g1/g1HeapVerifier.hpp"
pliden@30764 35 #include "gc/g1/g1OopClosures.inline.hpp"
mgerdin@37985 36 #include "gc/g1/g1Policy.hpp"
tschatzl@49606 37 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
pliden@30764 38 #include "gc/g1/g1StringDedup.hpp"
pliden@49752 39 #include "gc/g1/g1ThreadLocalData.hpp"
pliden@30764 40 #include "gc/g1/heapRegion.inline.hpp"
pliden@30764 41 #include "gc/g1/heapRegionRemSet.hpp"
pliden@30764 42 #include "gc/g1/heapRegionSet.inline.hpp"
brutisso@33107 43 #include "gc/shared/gcId.hpp"
pliden@30764 44 #include "gc/shared/gcTimer.hpp"
pliden@30764 45 #include "gc/shared/gcTrace.hpp"
brutisso@35061 46 #include "gc/shared/gcTraceTime.inline.hpp"
tschatzl@52876 47 #include "gc/shared/gcVMOperations.hpp"
pliden@30764 48 #include "gc/shared/genOopClosures.inline.hpp"
pliden@30764 49 #include "gc/shared/referencePolicy.hpp"
pliden@30764 50 #include "gc/shared/strongRootsScope.hpp"
rkennke@47647 51 #include "gc/shared/suspendibleThreadSet.hpp"
pliden@30764 52 #include "gc/shared/taskqueue.inline.hpp"
kbarrett@51546 53 #include "gc/shared/weakProcessor.inline.hpp"
manc@52904 54 #include "gc/shared/workerPolicy.hpp"
tschatzl@49664 55 #include "include/jvm.h"
brutisso@35061 56 #include "logging/log.hpp"
stefank@25492 57 #include "memory/allocation.hpp"
stefank@7397 58 #include "memory/resourceArea.hpp"
stefank@49592 59 #include "oops/access.inline.hpp"
stefank@7397 60 #include "oops/oop.inline.hpp"
dholmes@40655 61 #include "runtime/atomic.hpp"
stefank@7397 62 #include "runtime/handles.inline.hpp"
stefank@7397 63 #include "runtime/java.hpp"
goetz@24487 64 #include "runtime/prefetch.inline.hpp"
zgu@13195 65 #include "services/memTracker.hpp"
stefank@46625 66 #include "utilities/align.hpp"
mgerdin@38162 67 #include "utilities/growableArray.hpp"
ysr@1374 68
tschatzl@46751 69 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
tschatzl@46751 70 assert(addr < _cm->finger(), "invariant");
tschatzl@46751 71 assert(addr >= _task->finger(), "invariant");
tschatzl@46751 72
tschatzl@46751 73 // We move that task's local finger along.
tschatzl@46751 74 _task->move_finger_to(addr);
tschatzl@46751 75
tschatzl@46751 76 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
tschatzl@46751 77 // we only partially drain the local queue and global stack
tschatzl@46751 78 _task->drain_local_queue(true);
tschatzl@46751 79 _task->drain_global_stack(true);
tschatzl@46751 80
tschatzl@46751 81 // if the has_aborted flag has been raised, we need to bail out of
tschatzl@46751 82 // the iteration
tschatzl@46751 83 return !_task->has_aborted();
ysr@1374 84 }
ysr@1374 85
tschatzl@39966 86 G1CMMarkStack::G1CMMarkStack() :
tschatzl@41176 87 _max_chunk_capacity(0),
tschatzl@39966 88 _base(NULL),
aharlap@46447 89 _chunk_capacity(0) {
tschatzl@39966 90 set_empty();
tschatzl@39966 91 }
tschatzl@39966 92
tschatzl@39966 93 bool G1CMMarkStack::resize(size_t new_capacity) {
tschatzl@39966 94 assert(is_empty(), "Only resize when stack is empty.");
tschatzl@41176 95 assert(new_capacity <= _max_chunk_capacity,
tschatzl@41176 96 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
tschatzl@41176 97
kbarrett@46704 98 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
tschatzl@41176 99
tschatzl@41176 100 if (new_base == NULL) {
tschatzl@46328 101 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
johnc@14740 102 return false;
tonyp@9999 103 }
tschatzl@41176 104 // Release old mapping.
tschatzl@41176 105 if (_base != NULL) {
kbarrett@46704 106 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
johnc@14740 107 }
tschatzl@39966 108
tschatzl@41176 109 _base = new_base;
tschatzl@41176 110 _chunk_capacity = new_capacity;
tschatzl@39966 111 set_empty();
tschatzl@39966 112
johnc@14740 113 return true;
johnc@14740 114 }
johnc@14740 115
tschatzl@41176 116 size_t G1CMMarkStack::capacity_alignment() {
tschatzl@46328 117 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
tschatzl@41176 118 }
tschatzl@41176 119
tschatzl@41176 120 bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
tschatzl@41176 121 guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
tschatzl@41176 122
tschatzl@46328 123 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
tschatzl@46328 124
stefank@46619 125 _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
stefank@46619 126 size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
tschatzl@41176 127
tschatzl@41176 128 guarantee(initial_chunk_capacity <= _max_chunk_capacity,
tschatzl@41176 129 "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
tschatzl@41176 130 _max_chunk_capacity,
tschatzl@41176 131 initial_chunk_capacity);
tschatzl@41176 132
tschatzl@41176 133 log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
tschatzl@41176 134 initial_chunk_capacity, _max_chunk_capacity);
tschatzl@41176 135
tschatzl@41176 136 return resize(initial_chunk_capacity);
tschatzl@39966 137 }
tschatzl@39966 138
ehelin@35943 139 void G1CMMarkStack::expand() {
tschatzl@41176 140 if (_chunk_capacity == _max_chunk_capacity) {
tschatzl@41176 141 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
johnc@14740 142 return;
johnc@14740 143 }
tschatzl@41176 144 size_t old_capacity = _chunk_capacity;
johnc@14740 145 // Double capacity if possible
tschatzl@41176 146 size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
tschatzl@39966 147
tschatzl@39966 148 if (resize(new_capacity)) {
tschatzl@41176 149 log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
tschatzl@39966 150 old_capacity, new_capacity);
johnc@14740 151 } else {
tschatzl@41176 152 log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
tschatzl@39966 153 old_capacity, new_capacity);
johnc@14740 154 }
johnc@14740 155 }
johnc@14740 156
ehelin@35943 157 G1CMMarkStack::~G1CMMarkStack() {
tonyp@9999 158 if (_base != NULL) {
kbarrett@46704 159 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
tonyp@9999 160 }
ysr@1374 161 }
ysr@1374 162
tschatzl@46328 163 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
tschatzl@41176 164 elem->next = *list;
tschatzl@41176 165 *list = elem;
tschatzl@41176 166 }
tschatzl@41176 167
tschatzl@46328 168 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) {
tschatzl@41176 169 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
tschatzl@41176 170 add_chunk_to_list(&_chunk_list, elem);
tschatzl@41176 171 _chunks_in_chunk_list++;
tschatzl@41176 172 }
tschatzl@41176 173
tschatzl@46328 174 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) {
tschatzl@41176 175 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
tschatzl@41176 176 add_chunk_to_list(&_free_list, elem);
tschatzl@41176 177 }
tschatzl@41176 178
tschatzl@46328 179 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(TaskQueueEntryChunk* volatile* list) {
tschatzl@46328 180 TaskQueueEntryChunk* result = *list;
tschatzl@41176 181 if (result != NULL) {
tschatzl@41176 182 *list = (*list)->next;
ysr@1374 183 }
tschatzl@41176 184 return result;
tschatzl@41176 185 }
tschatzl@41176 186
tschatzl@46328 187 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() {
tschatzl@41176 188 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
tschatzl@46328 189 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list);
tschatzl@41176 190 if (result != NULL) {
tschatzl@41176 191 _chunks_in_chunk_list--;
ysr@1374 192 }
tschatzl@41176 193 return result;
ysr@1374 194 }
ysr@1374 195
tschatzl@46328 196 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() {
tschatzl@41176 197 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
tschatzl@41176 198 return remove_chunk_from_list(&_free_list);
tschatzl@41176 199 }
tschatzl@41176 200
tschatzl@46328 201 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
tschatzl@41176 202 // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
tschatzl@41176 203 // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
tschatzl@41176 204 // wraparound of _hwm.
tschatzl@41176 205 if (_hwm >= _chunk_capacity) {
tschatzl@41176 206 return NULL;
tschatzl@41176 207 }
tschatzl@41176 208
eosterlund@46993 209 size_t cur_idx = Atomic::add(1u, &_hwm) - 1;
tschatzl@41176 210 if (cur_idx >= _chunk_capacity) {
tschatzl@41176 211 return NULL;
tschatzl@41176 212 }
tschatzl@41176 213
tschatzl@46328 214 TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
tschatzl@41176 215 result->next = NULL;
tschatzl@41176 216 return result;
tschatzl@41176 217 }
tschatzl@41176 218
tschatzl@46328 219 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
tschatzl@41176 220 // Get a new chunk.
tschatzl@46328 221 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
tschatzl@41176 222
tschatzl@41176 223 if (new_chunk == NULL) {
tschatzl@41176 224 // Did not get a chunk from the free list. Allocate from backing memory.
tschatzl@41176 225 new_chunk = allocate_new_chunk();
aharlap@46319 226
aharlap@46319 227 if (new_chunk == NULL) {
aharlap@46319 228 return false;
aharlap@46319 229 }
ysr@1374 230 }
tschatzl@41176 231
tschatzl@46328 232 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry));
tschatzl@41176 233
tschatzl@41176 234 add_chunk_to_chunk_list(new_chunk);
tschatzl@41176 235
tschatzl@41176 236 return true;
ysr@1374 237 }
ysr@1374 238
tschatzl@46328 239 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
tschatzl@46328 240 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list();
tschatzl@41176 241
tschatzl@41176 242 if (cur == NULL) {
tschatzl@41176 243 return false;
tschatzl@41176 244 }
tschatzl@41176 245
tschatzl@46328 246 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry));
tschatzl@41176 247
tschatzl@41176 248 add_chunk_to_free_list(cur);
tschatzl@41176 249 return true;
tonyp@11455 250 }
tonyp@11455 251
tschatzl@41176 252 void G1CMMarkStack::set_empty() {
tschatzl@41176 253 _chunks_in_chunk_list = 0;
tschatzl@41176 254 _hwm = 0;
tschatzl@41176 255 _chunk_list = NULL;
tschatzl@41176 256 _free_list = NULL;
tonyp@11455 257 }
tonyp@11455 258
tschatzl@52875 259 G1CMRootRegions::G1CMRootRegions(uint const max_regions) :
tschatzl@52875 260 _root_regions(NEW_C_HEAP_ARRAY(HeapRegion*, max_regions, mtGC)),
tschatzl@52875 261 _max_regions(max_regions),
tschatzl@52875 262 _num_root_regions(0),
tschatzl@52875 263 _claimed_root_regions(0),
tschatzl@52875 264 _scan_in_progress(false),
tschatzl@52875 265 _should_abort(false) { }
tschatzl@52875 266
tschatzl@52875 267 G1CMRootRegions::~G1CMRootRegions() {
tschatzl@52875 268 FREE_C_HEAP_ARRAY(HeapRegion*, _max_regions);
tschatzl@52875 269 }
tschatzl@52875 270
tschatzl@52875 271 void G1CMRootRegions::reset() {
tschatzl@52875 272 _num_root_regions = 0;
tschatzl@52875 273 }
tschatzl@52875 274
tschatzl@52875 275 void G1CMRootRegions::add(HeapRegion* hr) {
tschatzl@52875 276 assert_at_safepoint();
tschatzl@52875 277 size_t idx = Atomic::add((size_t)1, &_num_root_regions) - 1;
tschatzl@52875 278 assert(idx < _max_regions, "Trying to add more root regions than there is space " SIZE_FORMAT, _max_regions);
tschatzl@52875 279 _root_regions[idx] = hr;
tonyp@11584 280 }
tonyp@11584 281
ehelin@35943 282 void G1CMRootRegions::prepare_for_scan() {
tonyp@11584 283 assert(!scan_in_progress(), "pre-condition");
tonyp@11584 284
tschatzl@52875 285 _scan_in_progress = _num_root_regions > 0;
tschatzl@52875 286
tschatzl@52875 287 _claimed_root_regions = 0;
tonyp@11584 288 _should_abort = false;
tonyp@11584 289 }
tonyp@11584 290
ehelin@35943 291 HeapRegion* G1CMRootRegions::claim_next() {
tonyp@11584 292 if (_should_abort) {
tonyp@11584 293 // If someone has set the should_abort flag, we return NULL to
tonyp@11584 294 // force the caller to bail out of their loop.
tonyp@11584 295 return NULL;
tonyp@11584 296 }
tonyp@11584 297
tschatzl@52875 298 if (_claimed_root_regions >= _num_root_regions) {
tschatzl@52875 299 return NULL;
tschatzl@52875 300 }
tschatzl@52875 301
tschatzl@52875 302 size_t claimed_index = Atomic::add((size_t)1, &_claimed_root_regions) - 1;
tschatzl@52875 303 if (claimed_index < _num_root_regions) {
tschatzl@52875 304 return _root_regions[claimed_index];
tonyp@11584 305 }
mgerdin@38162 306 return NULL;
tonyp@11584 307 }
tonyp@11584 308
tschatzl@38270 309 uint G1CMRootRegions::num_root_regions() const {
tschatzl@52875 310 return (uint)_num_root_regions;
tschatzl@38270 311 }
tschatzl@38270 312
brutisso@36084 313 void G1CMRootRegions::notify_scan_done() {
brutisso@36084 314 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
brutisso@36084 315 _scan_in_progress = false;
brutisso@36084 316 RootRegionScan_lock->notify_all();
brutisso@36084 317 }
brutisso@36084 318
brutisso@36084 319 void G1CMRootRegions::cancel_scan() {
brutisso@36084 320 notify_scan_done();
brutisso@36084 321 }
brutisso@36084 322
ehelin@35943 323 void G1CMRootRegions::scan_finished() {
tonyp@11584 324 assert(scan_in_progress(), "pre-condition");
tonyp@11584 325
tonyp@11584 326 if (!_should_abort) {
tschatzl@52875 327 assert(_claimed_root_regions >= num_root_regions(),
tschatzl@52875 328 "we should have claimed all root regions, claimed " SIZE_FORMAT ", length = %u",
tschatzl@52875 329 _claimed_root_regions, num_root_regions());
tonyp@11584 330 }
tonyp@11584 331
brutisso@36084 332 notify_scan_done();
tonyp@11584 333 }
tonyp@11584 334
ehelin@35943 335 bool G1CMRootRegions::wait_until_scan_finished() {
tschatzl@49644 336 if (!scan_in_progress()) {
tschatzl@49644 337 return false;
tschatzl@49644 338 }
tonyp@11584 339
tonyp@11584 340 {
tonyp@11584 341 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
tonyp@11584 342 while (scan_in_progress()) {
tonyp@11584 343 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
tonyp@11584 344 }
tonyp@11584 345 }
tonyp@11584 346 return true;
tonyp@11584 347 }
tonyp@11584 348
tschatzl@47696 349 // Returns the maximum number of workers to be used in a concurrent
tschatzl@47696 350 // phase based on the number of GC workers being used in a STW
tschatzl@47696 351 // phase.
tschatzl@47696 352 static uint scale_concurrent_worker_threads(uint num_gc_workers) {
tschatzl@47696 353 return MAX2((num_gc_workers + 2) / 4, 1U);
jmasa@11174 354 }
jmasa@11174 355
tschatzl@47697 356 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
tschatzl@47697 357 G1RegionToSpaceMapper* prev_bitmap_storage,
tschatzl@47697 358 G1RegionToSpaceMapper* next_bitmap_storage) :
tschatzl@47697 359 // _cm_thread set inside the constructor
johnc@14740 360 _g1h(g1h),
tschatzl@47697 361 _completed_initialization(false),
tschatzl@47697 362
tschatzl@47678 363 _mark_bitmap_1(),
tschatzl@47678 364 _mark_bitmap_2(),
tschatzl@47678 365 _prev_mark_bitmap(&_mark_bitmap_1),
tschatzl@47678 366 _next_mark_bitmap(&_mark_bitmap_2),
ysr@1374 367
tschatzl@49644 368 _heap(_g1h->reserved_region()),
tschatzl@47697 369
tschatzl@52875 370 _root_regions(_g1h->max_regions()),
tschatzl@47697 371
tschatzl@39966 372 _global_mark_stack(),
tschatzl@47697 373
ysr@1374 374 // _finger set in set_non_marking_state
ysr@1374 375
kbarrett@53747 376 _worker_id_offset(G1DirtyCardQueueSet::num_par_ids() + G1ConcRefinementThreads),
tschatzl@47696 377 _max_num_tasks(ParallelGCThreads),
tschatzl@47697 378 // _num_active_tasks set in set_non_marking_state()
ysr@1374 379 // _tasks set inside the constructor
tschatzl@47697 380
tschatzl@47696 381 _task_queues(new G1CMTaskQueueSet((int) _max_num_tasks)),
zgu@52905 382 _terminator((int) _max_num_tasks, _task_queues),
ysr@1374 383
tschatzl@47697 384 _first_overflow_barrier_sync(),
tschatzl@47697 385 _second_overflow_barrier_sync(),
tschatzl@47697 386
ysr@1374 387 _has_overflown(false),
ysr@1374 388 _concurrent(false),
tonyp@2145 389 _has_aborted(false),
tonyp@2145 390 _restart_for_overflow(false),
sangheki@37137 391 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
sangheki@37137 392 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()),
ysr@1374 393
ysr@1374 394 // _verbose_level set below
ysr@1374 395
ysr@1374 396 _init_times(),
tschatzl@47678 397 _remark_times(),
tschatzl@47678 398 _remark_mark_times(),
tschatzl@47678 399 _remark_weak_ref_times(),
ysr@1374 400 _cleanup_times(),
tschatzl@49644 401 _total_cleanup_time(0.0),
johnc@11583 402
tschatzl@47697 403 _accum_task_vtime(NULL),
tschatzl@47697 404
tschatzl@47696 405 _concurrent_workers(NULL),
tschatzl@47697 406 _num_concurrent_workers(0),
tschatzl@49606 407 _max_concurrent_workers(0),
tschatzl@49606 408
tschatzl@49607 409 _region_mark_stats(NEW_C_HEAP_ARRAY(G1RegionMarkStats, _g1h->max_regions(), mtGC)),
tschatzl@49607 410 _top_at_rebuild_starts(NEW_C_HEAP_ARRAY(HeapWord*, _g1h->max_regions(), mtGC))
tschatzl@47697 411 {
tschatzl@47678 412 _mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage);
tschatzl@47678 413 _mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage);
ysr@1374 414
tschatzl@47697 415 // Create & start ConcurrentMark thread.
lkorinth@49659 416 _cm_thread = new G1ConcurrentMarkThread(this);
tschatzl@47678 417 if (_cm_thread->osthread() == NULL) {
tschatzl@47678 418 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
ehelin@22220 419 }
ysr@3262 420
tschatzl@47678 421 assert(CGC_lock != NULL, "CGC_lock must be initialized");
ysr@1374 422
tschatzl@47696 423 if (FLAG_IS_DEFAULT(ConcGCThreads) || ConcGCThreads == 0) {
tschatzl@47696 424 // Calculate the number of concurrent worker threads by scaling
tschatzl@47696 425 // the number of parallel GC threads.
tschatzl@47696 426 uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads);
tschatzl@47696 427 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
tschatzl@47696 428 }
tschatzl@47696 429
tschatzl@47696 430 assert(ConcGCThreads > 0, "ConcGCThreads have been set.");
jmasa@5035 431 if (ConcGCThreads > ParallelGCThreads) {
tschatzl@47695 432 log_warning(gc)("More ConcGCThreads (%u) than ParallelGCThreads (%u).",
brutisso@37073 433 ConcGCThreads, ParallelGCThreads);
johnc@14740 434 return;
ysr@1374 435 }
tschatzl@47695 436
tschatzl@49607 437 log_debug(gc)("ConcGCThreads: %u offset %u", ConcGCThreads, _worker_id_offset);
jprovino@38637 438 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
tschatzl@47695 439
tschatzl@47696 440 _num_concurrent_workers = ConcGCThreads;
tschatzl@47696 441 _max_concurrent_workers = _num_concurrent_workers;
tschatzl@47696 442
tschatzl@47696 443 _concurrent_workers = new WorkGang("G1 Conc", _max_concurrent_workers, false, true);
tschatzl@47696 444 _concurrent_workers->initialize_workers();
ysr@1374 445
johnc@14740 446 if (FLAG_IS_DEFAULT(MarkStackSize)) {
jwilhelm@29697 447 size_t mark_stack_size =
johnc@14740 448 MIN2(MarkStackSizeMax,
tschatzl@47696 449 MAX2(MarkStackSize, (size_t) (_max_concurrent_workers * TASKQUEUE_SIZE)));
johnc@14740 450 // Verify that the calculated value for MarkStackSize is in range.
johnc@14740 451 // It would be nice to use the private utility routine from Arguments.
johnc@14740 452 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
brutisso@37073 453 log_warning(gc)("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
brutisso@37073 454 "must be between 1 and " SIZE_FORMAT,
brutisso@37073 455 mark_stack_size, MarkStackSizeMax);
johnc@14740 456 return;
johnc@14740 457 }
jwilhelm@29697 458 FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size);
johnc@14740 459 } else {
johnc@14740 460 // Verify MarkStackSize is in range.
johnc@14740 461 if (FLAG_IS_CMDLINE(MarkStackSize)) {
johnc@14740 462 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
johnc@14740 463 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
brutisso@37073 464 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): "
brutisso@37073 465 "must be between 1 and " SIZE_FORMAT,
brutisso@37073 466 MarkStackSize, MarkStackSizeMax);
johnc@14740 467 return;
johnc@14740 468 }
johnc@14740 469 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
johnc@14740 470 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
brutisso@37073 471 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
brutisso@37073 472 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
brutisso@37073 473 MarkStackSize, MarkStackSizeMax);
johnc@14740 474 return;
johnc@14740 475 }
johnc@14740 476 }
johnc@14740 477 }
johnc@14740 478 }
johnc@14740 479
tschatzl@41176 480 if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
tschatzl@39966 481 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
johnc@14740 482 }
johnc@14740 483
tschatzl@47696 484 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC);
tschatzl@47696 485 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_num_tasks, mtGC);
johnc@14740 486
johnc@14740 487 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
tschatzl@47696 488 _num_active_tasks = _max_num_tasks;
tschatzl@47696 489
tschatzl@47696 490 for (uint i = 0; i < _max_num_tasks; ++i) {
ehelin@35943 491 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
johnc@14740 492 task_queue->initialize();
johnc@14740 493 _task_queues->register_queue(i, task_queue);
johnc@14740 494
tschatzl@49606 495 _tasks[i] = new G1CMTask(i, this, task_queue, _region_mark_stats, _g1h->max_regions());
johnc@14740 496
johnc@14740 497 _accum_task_vtime[i] = 0.0;
johnc@14740 498 }
johnc@14740 499
tschatzl@49644 500 reset_at_marking_complete();
johnc@14740 501 _completed_initialization = true;
ysr@1374 502 }
ysr@1374 503
ehelin@35943 504 void G1ConcurrentMark::reset() {
tschatzl@49664 505 _has_aborted = false;
tschatzl@49664 506
tschatzl@49644 507 reset_marking_for_restart();
ysr@1374 508
tschatzl@49606 509 // Reset all tasks, since different phases will use different number of active
tschatzl@49606 510 // threads. So, it's easiest to have all of them ready.
tschatzl@47696 511 for (uint i = 0; i < _max_num_tasks; ++i) {
tschatzl@47678 512 _tasks[i]->reset(_next_mark_bitmap);
johnc@6761 513 }
ysr@1374 514
tschatzl@49606 515 uint max_regions = _g1h->max_regions();
tschatzl@49606 516 for (uint i = 0; i < max_regions; i++) {
tschatzl@49607 517 _top_at_rebuild_starts[i] = NULL;
tschatzl@49606 518 _region_mark_stats[i].clear();
tschatzl@49606 519 }
ysr@1374 520 }
ysr@1374 521
tschatzl@49606 522 void G1ConcurrentMark::clear_statistics_in_region(uint region_idx) {
tschatzl@49606 523 for (uint j = 0; j < _max_num_tasks; ++j) {
tschatzl@49606 524 _tasks[j]->clear_mark_stats_cache(region_idx);
tschatzl@49606 525 }
tschatzl@49607 526 _top_at_rebuild_starts[region_idx] = NULL;
tschatzl@49606 527 _region_mark_stats[region_idx].clear();
tschatzl@49606 528 }
tschatzl@49606 529
tschatzl@49664 530 void G1ConcurrentMark::clear_statistics(HeapRegion* r) {
tschatzl@49606 531 uint const region_idx = r->hrm_index();
tschatzl@49606 532 if (r->is_humongous()) {
tschatzl@49606 533 assert(r->is_starts_humongous(), "Got humongous continues region here");
tschatzl@49606 534 uint const size_in_regions = (uint)_g1h->humongous_obj_size_in_regions(oop(r->humongous_start_region()->bottom())->size());
tschatzl@49606 535 for (uint j = region_idx; j < (region_idx + size_in_regions); j++) {
tschatzl@49606 536 clear_statistics_in_region(j);
tschatzl@49606 537 }
tschatzl@49606 538 } else {
tschatzl@49606 539 clear_statistics_in_region(region_idx);
tschatzl@49606 540 }
tschatzl@49604 541 }
johnc@15090 542
tschatzl@49727 543 static void clear_mark_if_set(G1CMBitMap* bitmap, HeapWord* addr) {
tschatzl@49727 544 if (bitmap->is_marked(addr)) {
tschatzl@49727 545 bitmap->clear(addr);
tschatzl@49727 546 }
tschatzl@49727 547 }
tschatzl@49727 548
tschatzl@49664 549 void G1ConcurrentMark::humongous_object_eagerly_reclaimed(HeapRegion* r) {
tschatzl@49664 550 assert_at_safepoint_on_vm_thread();
tschatzl@49664 551
tschatzl@49727 552 // Need to clear all mark bits of the humongous object.
tschatzl@49727 553 clear_mark_if_set(_prev_mark_bitmap, r->bottom());
tschatzl@49727 554 clear_mark_if_set(_next_mark_bitmap, r->bottom());
tschatzl@49664 555
tschatzl@49664 556 if (!_g1h->collector_state()->mark_or_rebuild_in_progress()) {
tschatzl@49664 557 return;
tschatzl@49664 558 }
tschatzl@49664 559
tschatzl@49664 560 // Clear any statistics about the region gathered so far.
tschatzl@49664 561 clear_statistics(r);
tschatzl@49664 562 }
tschatzl@49664 563
tschatzl@49644 564 void G1ConcurrentMark::reset_marking_for_restart() {
aharlap@46319 565 _global_mark_stack.set_empty();
aharlap@46447 566
aharlap@46447 567 // Expand the marking stack, if we have to and if we can.
aharlap@46447 568 if (has_overflown()) {
aharlap@46447 569 _global_mark_stack.expand();
tschatzl@49606 570
tschatzl@49606 571 uint max_regions = _g1h->max_regions();
tschatzl@49606 572 for (uint i = 0; i < max_regions; i++) {
tschatzl@49606 573 _region_mark_stats[i].clear_during_overflow();
tschatzl@49606 574 }
aharlap@46447 575 }
aharlap@46447 576
aharlap@46319 577 clear_has_overflown();
tschatzl@49644 578 _finger = _heap.start();
johnc@15090 579
tschatzl@47696 580 for (uint i = 0; i < _max_num_tasks; ++i) {
ehelin@35943 581 G1CMTaskQueue* queue = _task_queues->queue(i);
johnc@15090 582 queue->set_empty();
johnc@15090 583 }
johnc@15090 584 }
johnc@15090 585
ehelin@35943 586 void G1ConcurrentMark::set_concurrency(uint active_tasks) {
tschatzl@47696 587 assert(active_tasks <= _max_num_tasks, "we should not have more");
tschatzl@47696 588
tschatzl@47696 589 _num_active_tasks = active_tasks;
ysr@1374 590 // Need to update the three data structures below according to the
ysr@1374 591 // number of active threads for this phase.
zgu@53586 592 _terminator.terminator()->reset_for_reuse((int) active_tasks);
ysr@1374 593 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
ysr@1374 594 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
johnc@16389 595 }
johnc@16389 596
ehelin@35943 597 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
johnc@16389 598 set_concurrency(active_tasks);
ysr@1374 599
ysr@1374 600 _concurrent = concurrent;
ysr@1374 601
tschatzl@49666 602 if (!concurrent) {
tschatzl@49666 603 // At this point we should be in a STW phase, and completed marking.
tschatzl@49644 604 assert_at_safepoint_on_vm_thread();
pliden@24469 605 assert(out_of_regions(),
david@33105 606 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT,
tschatzl@49644 607 p2i(_finger), p2i(_heap.end()));
ysr@1374 608 }
ysr@1374 609 }
ysr@1374 610
tschatzl@49644 611 void G1ConcurrentMark::reset_at_marking_complete() {
ysr@1374 612 // We set the global marking state to some default values when we're
ysr@1374 613 // not doing marking.
tschatzl@49644 614 reset_marking_for_restart();
tschatzl@47696 615 _num_active_tasks = 0;
ysr@1374 616 }
ysr@1374 617
ehelin@35943 618 G1ConcurrentMark::~G1ConcurrentMark() {
tschatzl@49607 619 FREE_C_HEAP_ARRAY(HeapWord*, _top_at_rebuild_starts);
tschatzl@49606 620 FREE_C_HEAP_ARRAY(G1RegionMarkStats, _region_mark_stats);
ehelin@35943 621 // The G1ConcurrentMark instance is never freed.
stefank@11403 622 ShouldNotReachHere();
ysr@1374 623 }
ysr@1374 624
tschatzl@37122 625 class G1ClearBitMapTask : public AbstractGangTask {
tschatzl@38155 626 public:
tschatzl@38155 627 static size_t chunk_size() { return M; }
tschatzl@38155 628
tschatzl@38155 629 private:
tschatzl@37122 630 // Heap region closure used for clearing the given mark bitmap.
tschatzl@37122 631 class G1ClearBitmapHRClosure : public HeapRegionClosure {
tschatzl@37122 632 private:
tschatzl@37122 633 G1CMBitMap* _bitmap;
tschatzl@37122 634 G1ConcurrentMark* _cm;
tschatzl@37122 635 public:
tschatzl@51332 636 G1ClearBitmapHRClosure(G1CMBitMap* bitmap, G1ConcurrentMark* cm) : HeapRegionClosure(), _bitmap(bitmap), _cm(cm) {
tschatzl@37122 637 }
tschatzl@37122 638
jwilhelm@48969 639 virtual bool do_heap_region(HeapRegion* r) {
tschatzl@38155 640 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize;
tschatzl@37122 641
tschatzl@37122 642 HeapWord* cur = r->bottom();
tschatzl@37122 643 HeapWord* const end = r->end();
tschatzl@37122 644
tschatzl@37122 645 while (cur < end) {
tschatzl@37122 646 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end));
tschatzl@37122 647 _bitmap->clear_range(mr);
tschatzl@37122 648
tschatzl@37122 649 cur += chunk_size_in_words;
tschatzl@37122 650
tschatzl@37122 651 // Abort iteration if after yielding the marking has been aborted.
tschatzl@37122 652 if (_cm != NULL && _cm->do_yield_check() && _cm->has_aborted()) {
tschatzl@37122 653 return true;
tschatzl@37122 654 }
tschatzl@37122 655 // Repeat the asserts from before the start of the closure. We will do them
tschatzl@37122 656 // as asserts here to minimize their overhead on the product. However, we
tschatzl@37122 657 // will have them as guarantees at the beginning / end of the bitmap
tschatzl@37122 658 // clearing to get some checking in the product.
tschatzl@47678 659 assert(_cm == NULL || _cm->cm_thread()->during_cycle(), "invariant");
tschatzl@49643 660 assert(_cm == NULL || !G1CollectedHeap::heap()->collector_state()->mark_or_rebuild_in_progress(), "invariant");
tschatzl@37122 661 }
tschatzl@37122 662 assert(cur == end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index());
tschatzl@37122 663
tschatzl@37122 664 return false;
tschatzl@37122 665 }
tschatzl@37122 666 };
tschatzl@37122 667
tschatzl@37122 668 G1ClearBitmapHRClosure _cl;
tschatzl@37122 669 HeapRegionClaimer _hr_claimer;
tschatzl@37122 670 bool _suspendible; // If the task is suspendible, workers must join the STS.
tschatzl@37122 671
tschatzl@37122 672 public:
tschatzl@37122 673 G1ClearBitMapTask(G1CMBitMap* bitmap, G1ConcurrentMark* cm, uint n_workers, bool suspendible) :
tschatzl@38155 674 AbstractGangTask("G1 Clear Bitmap"),
tschatzl@37122 675 _cl(bitmap, suspendible ? cm : NULL),
tschatzl@37122 676 _hr_claimer(n_workers),
tschatzl@37122 677 _suspendible(suspendible)
tschatzl@37122 678 { }
tschatzl@37122 679
tschatzl@37122 680 void work(uint worker_id) {
tschatzl@37122 681 SuspendibleThreadSetJoiner sts_join(_suspendible);
sjohanss@47885 682 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
tschatzl@37122 683 }
tschatzl@37122 684
tschatzl@37122 685 bool is_complete() {
jwilhelm@48969 686 return _cl.is_complete();
tschatzl@37122 687 }
tschatzl@37122 688 };
tschatzl@37122 689
tschatzl@37122 690 void G1ConcurrentMark::clear_bitmap(G1CMBitMap* bitmap, WorkGang* workers, bool may_yield) {
tschatzl@37122 691 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
tschatzl@37122 692
tschatzl@38155 693 size_t const num_bytes_to_clear = (HeapRegion::GrainBytes * _g1h->num_regions()) / G1CMBitMap::heap_map_factor();
stefank@46619 694 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size();
tschatzl@38155 695
tschatzl@38155 696 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
tschatzl@38155 697
tschatzl@38155 698 G1ClearBitMapTask cl(bitmap, this, num_workers, may_yield);
tschatzl@38155 699
tschatzl@38155 700 log_debug(gc, ergo)("Running %s with %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks);
tschatzl@38155 701 workers->run_task(&cl, num_workers);
tschatzl@38155 702 guarantee(!may_yield || cl.is_complete(), "Must have completed iteration when not yielding.");
tschatzl@37122 703 }
tschatzl@37122 704
tschatzl@37122 705 void G1ConcurrentMark::cleanup_for_next_mark() {
tonyp@5243 706 // Make sure that the concurrent mark thread looks to still be in
tonyp@5243 707 // the current cycle.
tschatzl@47678 708 guarantee(cm_thread()->during_cycle(), "invariant");
tonyp@5243 709
tonyp@5243 710 // We are finishing up the current cycle by clearing the next
tonyp@5243 711 // marking bitmap and getting it ready for the next cycle. During
tonyp@5243 712 // this time no other cycle can start. So, let's make sure that this
tonyp@5243 713 // is the case.
tschatzl@49643 714 guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
tschatzl@37122 715
tschatzl@47696 716 clear_bitmap(_next_mark_bitmap, _concurrent_workers, true);
tschatzl@26160 717
tonyp@5243 718 // Repeat the asserts from above.
tschatzl@47678 719 guarantee(cm_thread()->during_cycle(), "invariant");
tschatzl@49643 720 guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
tschatzl@37122 721 }
tschatzl@37122 722
tschatzl@37122 723 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
tschatzl@49644 724 assert_at_safepoint_on_vm_thread();
tschatzl@47678 725 clear_bitmap(_prev_mark_bitmap, workers, false);
ysr@1374 726 }
ysr@1374 727
tschatzl@49644 728 class NoteStartOfMarkHRClosure : public HeapRegionClosure {
ysr@1374 729 public:
jwilhelm@48969 730 bool do_heap_region(HeapRegion* r) {
david@33786 731 r->note_start_of_marking();
ysr@1374 732 return false;
ysr@1374 733 }
ysr@1374 734 };
ysr@1374 735
tschatzl@49644 736 void G1ConcurrentMark::pre_initial_mark() {
jwilhelm@22551 737 // Initialize marking structures. This has to be done in a STW phase.
ysr@1374 738 reset();
tonyp@11455 739
tonyp@11455 740 // For each region note start of marking.
tonyp@11455 741 NoteStartOfMarkHRClosure startcl;
tschatzl@49644 742 _g1h->heap_region_iterate(&startcl);
tschatzl@52875 743
tschatzl@52875 744 _root_regions.reset();
ysr@1374 745 }
ysr@1374 746
ysr@1374 747
tschatzl@49644 748 void G1ConcurrentMark::post_initial_mark() {
johnc@10670 749 // Start Concurrent Marking weak-reference discovery.
tschatzl@49644 750 ReferenceProcessor* rp = _g1h->ref_processor_cm();
johnc@10670 751 // enable ("weak") refs discovery
kbarrett@28212 752 rp->enable_discovery();
ysr@1610 753 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
ysr@1374 754
pliden@49751 755 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
tonyp@5082 756 // This is the start of the marking cycle, we're expected all
tonyp@5082 757 // threads to have SATB queues with active set to false.
tonyp@5082 758 satb_mq_set.set_active_all_threads(true, /* new active value */
tonyp@5082 759 false /* expected_active */);
ysr@1374 760
tonyp@11584 761 _root_regions.prepare_for_scan();
tonyp@11584 762
ysr@1374 763 // update_g1_committed() will be called at the end of an evac pause
ysr@1374 764 // when marking is on. So, it's also called at the end of the
ysr@1374 765 // initial-mark pause to update the heap end, if the heap expands
ysr@1374 766 // during it. No need to call it here.
ysr@1374 767 }
ysr@1374 768
ysr@1374 769 /*
tonyp@9417 770 * Notice that in the next two methods, we actually leave the STS
tonyp@9417 771 * during the barrier sync and join it immediately afterwards. If we
tonyp@9417 772 * do not do this, the following deadlock can occur: one thread could
tonyp@9417 773 * be in the barrier sync code, waiting for the other thread to also
tonyp@9417 774 * sync up, whereas another one could be trying to yield, while also
tonyp@9417 775 * waiting for the other threads to sync up too.
tonyp@9417 776 *
tonyp@9417 777 * Note, however, that this code is also used during remark and in
tonyp@9417 778 * this case we should not attempt to leave / enter the STS, otherwise
jwilhelm@22551 779 * we'll either hit an assert (debug / fastdebug) or deadlock
tonyp@9417 780 * (product). So we should only leave / enter the STS if we are
tonyp@9417 781 * operating concurrently.
tonyp@9417 782 *
tonyp@9417 783 * Because the thread that does the sync barrier has left the STS, it
tonyp@9417 784 * is possible to be suspended for a Full GC or an evacuation pause
tonyp@9417 785 * could occur. This is actually safe, since the entering the sync
tonyp@9417 786 * barrier is one of the last things do_marking_step() does, and it
tonyp@9417 787 * doesn't manipulate any data structures afterwards.
tonyp@9417 788 */
ysr@1374 789
ehelin@35943 790 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
pliden@30613 791 bool barrier_aborted;
pliden@30613 792 {
pliden@30613 793 SuspendibleThreadSetLeaver sts_leave(concurrent());
pliden@30613 794 barrier_aborted = !_first_overflow_barrier_sync.enter();
tonyp@9417 795 }
pliden@24468 796
ysr@1374 797 // at this point everyone should have synced up and not be doing any
ysr@1374 798 // more work
ysr@1374 799
pliden@24468 800 if (barrier_aborted) {
pliden@24468 801 // If the barrier aborted we ignore the overflow condition and
pliden@24468 802 // just abort the whole marking phase as quickly as possible.
pliden@24468 803 return;
tonyp@9999 804 }
ysr@1374 805 }
ysr@1374 806
ehelin@35943 807 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
brutisso@33740 808 SuspendibleThreadSetLeaver sts_leave(concurrent());
brutisso@33740 809 _second_overflow_barrier_sync.enter();
pliden@24468 810
johnc@16389 811 // at this point everything should be re-initialized and ready to go
ysr@1374 812 }
ysr@1374 813
tschatzl@49644 814 class G1CMConcurrentMarkingTask : public AbstractGangTask {
ehelin@35943 815 G1ConcurrentMark* _cm;
lkorinth@49659 816
ysr@1374 817 public:
jmasa@11396 818 void work(uint worker_id) {
tschatzl@47695 819 assert(Thread::current()->is_ConcurrentGC_thread(), "Not a concurrent GC thread");
johnc@7399 820 ResourceMark rm;
ysr@1374 821
ysr@1374 822 double start_vtime = os::elapsedVTime();
ysr@1374 823
pliden@30613 824 {
pliden@30613 825 SuspendibleThreadSetJoiner sts_join;
pliden@30613 826
pliden@30613 827 assert(worker_id < _cm->active_tasks(), "invariant");
tschatzl@47695 828
tschatzl@47678 829 G1CMTask* task = _cm->task(worker_id);
tschatzl@47678 830 task->record_start_time();
pliden@30613 831 if (!_cm->has_aborted()) {
pliden@30613 832 do {
tschatzl@47695 833 task->do_marking_step(G1ConcMarkStepDurationMillis,
tschatzl@47678 834 true /* do_termination */,
tschatzl@47678 835 false /* is_serial*/);
pliden@30613 836
tschatzl@37414 837 _cm->do_yield_check();
tschatzl@47678 838 } while (!_cm->has_aborted() && task->has_aborted());
pliden@30613 839 }
tschatzl@47678 840 task->record_end_time();
tschatzl@47678 841 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant");
ysr@1374 842 }
ysr@1374 843
ysr@1374 844 double end_vtime = os::elapsedVTime();
jmasa@11396 845 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
ysr@1374 846 }
ysr@1374 847
tschatzl@49644 848 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm) :
tschatzl@49644 849 AbstractGangTask("Concurrent Mark"), _cm(cm) { }
ysr@1374 850
ehelin@35943 851 ~G1CMConcurrentMarkingTask() { }
ysr@1374 852 };
ysr@1374 853
tschatzl@47696 854 uint G1ConcurrentMark::calc_active_marking_workers() {
tschatzl@47696 855 uint result = 0;
mlarsson@27251 856 if (!UseDynamicNumberOfGCThreads ||
mlarsson@27251 857 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
mlarsson@27251 858 !ForceDynamicNumberOfGCThreads)) {
tschatzl@47696 859 result = _max_concurrent_workers;
mlarsson@27251 860 } else {
tschatzl@47696 861 result =
manc@52904 862 WorkerPolicy::calc_default_active_workers(_max_concurrent_workers,
manc@52904 863 1, /* Minimum workers */
manc@52904 864 _num_concurrent_workers,
manc@52904 865 Threads::number_of_non_daemon_threads());
tschatzl@47696 866 // Don't scale the result down by scale_concurrent_workers() because
tschatzl@47696 867 // that scaling has already gone into "_max_concurrent_workers".
jmasa@11174 868 }
tschatzl@47696 869 assert(result > 0 && result <= _max_concurrent_workers,
tschatzl@47696 870 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u",
tschatzl@47696 871 _max_concurrent_workers, result);
tschatzl@47696 872 return result;
jmasa@11174 873 }
jmasa@11174 874
tschatzl@49606 875 void G1ConcurrentMark::scan_root_region(HeapRegion* hr, uint worker_id) {
tschatzl@52875 876 assert(hr->is_old() || (hr->is_survivor() && hr->next_top_at_mark_start() == hr->bottom()),
tschatzl@52875 877 "Root regions must be old or survivor but region %u is %s", hr->hrm_index(), hr->get_type_str());
tschatzl@49606 878 G1RootRegionScanClosure cl(_g1h, this, worker_id);
tonyp@11584 879
tonyp@11584 880 const uintx interval = PrefetchScanIntervalInBytes;
tschatzl@52875 881 HeapWord* curr = hr->next_top_at_mark_start();
tonyp@11584 882 const HeapWord* end = hr->top();
tonyp@11584 883 while (curr < end) {
tonyp@11584 884 Prefetch::read(curr, interval);
tonyp@11584 885 oop obj = oop(curr);
sjohanss@32606 886 int size = obj->oop_iterate_size(&cl);
tonyp@11584 887 assert(size == obj->size(), "sanity");
tonyp@11584 888 curr += size;
tonyp@11584 889 }
tonyp@11584 890 }
tonyp@11584 891
ehelin@35943 892 class G1CMRootRegionScanTask : public AbstractGangTask {
ehelin@35943 893 G1ConcurrentMark* _cm;
tonyp@11584 894 public:
ehelin@35943 895 G1CMRootRegionScanTask(G1ConcurrentMark* cm) :
tschatzl@38270 896 AbstractGangTask("G1 Root Region Scan"), _cm(cm) { }
tonyp@11584 897
tonyp@11584 898 void work(uint worker_id) {
tonyp@11584 899 assert(Thread::current()->is_ConcurrentGC_thread(),
tonyp@11584 900 "this should only be done by a conc GC thread");
tonyp@11584 901
ehelin@35943 902 G1CMRootRegions* root_regions = _cm->root_regions();
tonyp@11584 903 HeapRegion* hr = root_regions->claim_next();
tonyp@11584 904 while (hr != NULL) {
tschatzl@49606 905 _cm->scan_root_region(hr, worker_id);
tonyp@11584 906 hr = root_regions->claim_next();
tonyp@11584 907 }
tonyp@11584 908 }
tonyp@11584 909 };
tonyp@11584 910
tschatzl@37123 911 void G1ConcurrentMark::scan_root_regions() {
tonyp@11584 912 // scan_in_progress() will have been set to true only if there was
tonyp@11584 913 // at least one root region to scan. So, if it's false, we
tonyp@11584 914 // should not attempt to do any further work.
tonyp@11584 915 if (root_regions()->scan_in_progress()) {
brutisso@36084 916 assert(!has_aborted(), "Aborting before root region scanning is finished not supported.");
brutisso@31397 917
tschatzl@47696 918 _num_concurrent_workers = MIN2(calc_active_marking_workers(),
tschatzl@47696 919 // We distribute work on a per-region basis, so starting
tschatzl@47696 920 // more threads than that is useless.
tschatzl@47696 921 root_regions()->num_root_regions());
tschatzl@47696 922 assert(_num_concurrent_workers <= _max_concurrent_workers,
tonyp@11584 923 "Maximum number of marking threads exceeded");
tonyp@11584 924
ehelin@35943 925 G1CMRootRegionScanTask task(this);
tschatzl@38270 926 log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
tschatzl@47696 927 task.name(), _num_concurrent_workers, root_regions()->num_root_regions());
tschatzl@47696 928 _concurrent_workers->run_task(&task, _num_concurrent_workers);
tonyp@11584 929
tonyp@11584 930 // It's possible that has_aborted() is true here without actually
tonyp@11584 931 // aborting the survivor scan earlier. This is OK as it's
tonyp@11584 932 // mainly used for sanity checking.
tonyp@11584 933 root_regions()->scan_finished();
tonyp@11584 934 }
tonyp@11584 935 }
tonyp@11584 936
sangheki@37137 937 void G1ConcurrentMark::concurrent_cycle_start() {
sangheki@37137 938 _gc_timer_cm->register_gc_start();
sangheki@37137 939
sangheki@37137 940 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start());
sangheki@37137 941
sangheki@37137 942 _g1h->trace_heap_before_gc(_gc_tracer_cm);
sangheki@35204 943 }
sangheki@35204 944
sangheki@37137 945 void G1ConcurrentMark::concurrent_cycle_end() {
tschatzl@49664 946 _g1h->collector_state()->set_clearing_next_bitmap(false);
tschatzl@49664 947
sangheki@37137 948 _g1h->trace_heap_after_gc(_gc_tracer_cm);
sangheki@37137 949
sangheki@37137 950 if (has_aborted()) {
tschatzl@49632 951 log_info(gc, marking)("Concurrent Mark Abort");
sangheki@37137 952 _gc_tracer_cm->report_concurrent_mode_failure();
sangheki@36588 953 }
sangheki@36588 954
sangheki@37137 955 _gc_timer_cm->register_gc_end();
sangheki@37137 956
sangheki@37137 957 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
sangheki@35204 958 }
sangheki@35204 959
tschatzl@37123 960 void G1ConcurrentMark::mark_from_roots() {
ysr@1374 961 _restart_for_overflow = false;
jmasa@11174 962
tschatzl@47696 963 _num_concurrent_workers = calc_active_marking_workers();
tschatzl@47696 964
tschatzl@47696 965 uint active_workers = MAX2(1U, _num_concurrent_workers);
johnc@11250 966
jmasa@40096 967 // Setting active workers is not guaranteed since fewer
jmasa@40096 968 // worker threads may currently exist and more may not be
jmasa@40096 969 // available.
tschatzl@47696 970 active_workers = _concurrent_workers->update_active_workers(active_workers);
tschatzl@47696 971 log_info(gc, task)("Using %u workers of %u for marking", active_workers, _concurrent_workers->total_workers());
tschatzl@40922 972
johnc@16389 973 // Parallel task terminator is set in "set_concurrency_and_phase()"
johnc@16389 974 set_concurrency_and_phase(active_workers, true /* concurrent */);
ysr@1374 975
tschatzl@49644 976 G1CMConcurrentMarkingTask marking_task(this);
tschatzl@47696 977 _concurrent_workers->run_task(&marking_task);
ysr@1374 978 print_stats();
ysr@1374 979 }
ysr@1374 980
tschatzl@49664 981 void G1ConcurrentMark::verify_during_pause(G1HeapVerifier::G1VerifyType type, VerifyOption vo, const char* caller) {
tschatzl@49664 982 G1HeapVerifier* verifier = _g1h->verifier();
tschatzl@49664 983
tschatzl@49664 984 verifier->verify_region_sets_optional();
tschatzl@49664 985
tschatzl@49664 986 if (VerifyDuringGC) {
tschatzl@49807 987 GCTraceTime(Debug, gc, phases) debug(caller, _gc_timer_cm);
tschatzl@49664 988
tschatzl@49664 989 size_t const BufLen = 512;
tschatzl@49664 990 char buffer[BufLen];
tschatzl@49664 991
tschatzl@49664 992 jio_snprintf(buffer, BufLen, "During GC (%s)", caller);
tschatzl@49664 993 verifier->verify(type, vo, buffer);
tschatzl@49664 994 }
tschatzl@49664 995
tschatzl@49664 996 verifier->check_bitmaps(caller);
tschatzl@49664 997 }
tschatzl@49664 998
tschatzl@49811 999 class G1UpdateRemSetTrackingBeforeRebuildTask : public AbstractGangTask {
tschatzl@49607 1000 G1CollectedHeap* _g1h;
tschatzl@49607 1001 G1ConcurrentMark* _cm;
tschatzl@49811 1002 HeapRegionClaimer _hrclaimer;
tschatzl@49811 1003 uint volatile _total_selected_for_rebuild;
tschatzl@49607 1004
tschatzl@49803 1005 G1PrintRegionLivenessInfoClosure _cl;
tschatzl@49803 1006
tschatzl@49811 1007 class G1UpdateRemSetTrackingBeforeRebuild : public HeapRegionClosure {
tschatzl@49811 1008 G1CollectedHeap* _g1h;
tschatzl@49811 1009 G1ConcurrentMark* _cm;
tschatzl@49811 1010
tschatzl@49811 1011 G1PrintRegionLivenessInfoClosure* _cl;
tschatzl@49811 1012
tschatzl@49811 1013 uint _num_regions_selected_for_rebuild; // The number of regions actually selected for rebuild.
tschatzl@49811 1014
tschatzl@51007 1015 void update_remset_before_rebuild(HeapRegion* hr) {
tschatzl@53985 1016 G1RemSetTrackingPolicy* tracking_policy = _g1h->policy()->remset_tracker();
tschatzl@49811 1017
tschatzl@51007 1018 bool selected_for_rebuild;
tschatzl@51007 1019 if (hr->is_humongous()) {
tschatzl@51007 1020 bool const is_live = _cm->liveness(hr->humongous_start_region()->hrm_index()) > 0;
tschatzl@51007 1021 selected_for_rebuild = tracking_policy->update_humongous_before_rebuild(hr, is_live);
tschatzl@51007 1022 } else {
tschatzl@51007 1023 size_t const live_bytes = _cm->liveness(hr->hrm_index());
tschatzl@51007 1024 selected_for_rebuild = tracking_policy->update_before_rebuild(hr, live_bytes);
tschatzl@51007 1025 }
tschatzl@49811 1026 if (selected_for_rebuild) {
tschatzl@49811 1027 _num_regions_selected_for_rebuild++;
tschatzl@49811 1028 }
tschatzl@49811 1029 _cm->update_top_at_rebuild_start(hr);
tschatzl@49607 1030 }
tschatzl@49811 1031
tschatzl@49811 1032 // Distribute the given words across the humongous object starting with hr and
tschatzl@49811 1033 // note end of marking.
tschatzl@49811 1034 void distribute_marked_bytes(HeapRegion* hr, size_t marked_words) {
tschatzl@49811 1035 uint const region_idx = hr->hrm_index();
tschatzl@49811 1036 size_t const obj_size_in_words = (size_t)oop(hr->bottom())->size();
tschatzl@49811 1037 uint const num_regions_in_humongous = (uint)G1CollectedHeap::humongous_obj_size_in_regions(obj_size_in_words);
tschatzl@49811 1038
tschatzl@49811 1039 // "Distributing" zero words means that we only note end of marking for these
tschatzl@49811 1040 // regions.
tschatzl@49811 1041 assert(marked_words == 0 || obj_size_in_words == marked_words,
tschatzl@49811 1042 "Marked words should either be 0 or the same as humongous object (" SIZE_FORMAT ") but is " SIZE_FORMAT,
tschatzl@49811 1043 obj_size_in_words, marked_words);
tschatzl@49811 1044
tschatzl@49811 1045 for (uint i = region_idx; i < (region_idx + num_regions_in_humongous); i++) {
tschatzl@49811 1046 HeapRegion* const r = _g1h->region_at(i);
tschatzl@49811 1047 size_t const words_to_add = MIN2(HeapRegion::GrainWords, marked_words);
tschatzl@49811 1048
tschatzl@49811 1049 log_trace(gc, marking)("Adding " SIZE_FORMAT " words to humongous region %u (%s)",
tschatzl@49811 1050 words_to_add, i, r->get_type_str());
tschatzl@49811 1051 add_marked_bytes_and_note_end(r, words_to_add * HeapWordSize);
tschatzl@49811 1052 marked_words -= words_to_add;
tschatzl@49811 1053 }
tschatzl@49811 1054 assert(marked_words == 0,
tschatzl@49811 1055 SIZE_FORMAT " words left after distributing space across %u regions",
tschatzl@49811 1056 marked_words, num_regions_in_humongous);
tschatzl@49811 1057 }
tschatzl@49811 1058
tschatzl@49811 1059 void update_marked_bytes(HeapRegion* hr) {
tschatzl@49811 1060 uint const region_idx = hr->hrm_index();
tschatzl@49811 1061 size_t const marked_words = _cm->liveness(region_idx);
tschatzl@49811 1062 // The marking attributes the object's size completely to the humongous starts
tschatzl@49811 1063 // region. We need to distribute this value across the entire set of regions a
tschatzl@49811 1064 // humongous object spans.
tschatzl@49811 1065 if (hr->is_humongous()) {
tschatzl@49811 1066 assert(hr->is_starts_humongous() || marked_words == 0,
tschatzl@49811 1067 "Should not have marked words " SIZE_FORMAT " in non-starts humongous region %u (%s)",
tschatzl@49811 1068 marked_words, region_idx, hr->get_type_str());
tschatzl@49811 1069 if (hr->is_starts_humongous()) {
tschatzl@49811 1070 distribute_marked_bytes(hr, marked_words);
tschatzl@49811 1071 }
tschatzl@49811 1072 } else {
tschatzl@49811 1073 log_trace(gc, marking)("Adding " SIZE_FORMAT " words to region %u (%s)", marked_words, region_idx, hr->get_type_str());
tschatzl@49811 1074 add_marked_bytes_and_note_end(hr, marked_words * HeapWordSize);
tschatzl@49811 1075 }
tschatzl@49811 1076 }
tschatzl@49811 1077
tschatzl@49811 1078 void add_marked_bytes_and_note_end(HeapRegion* hr, size_t marked_bytes) {
tschatzl@49811 1079 hr->add_to_marked_bytes(marked_bytes);
tschatzl@49811 1080 _cl->do_heap_region(hr);
tschatzl@49811 1081 hr->note_end_of_marking();
tschatzl@49811 1082 }
tschatzl@49811 1083
tschatzl@49811 1084 public:
tschatzl@49811 1085 G1UpdateRemSetTrackingBeforeRebuild(G1CollectedHeap* g1h, G1ConcurrentMark* cm, G1PrintRegionLivenessInfoClosure* cl) :
tschatzl@51332 1086 _g1h(g1h), _cm(cm), _cl(cl), _num_regions_selected_for_rebuild(0) { }
tschatzl@49811 1087
tschatzl@49811 1088 virtual bool do_heap_region(HeapRegion* r) {
tschatzl@49811 1089 update_remset_before_rebuild(r);
tschatzl@49811 1090 update_marked_bytes(r);
tschatzl@49811 1091
tschatzl@49811 1092 return false;
tschatzl@49811 1093 }
tschatzl@49811 1094
tschatzl@49811 1095 uint num_selected_for_rebuild() const { return _num_regions_selected_for_rebuild; }
tschatzl@49811 1096 };
tschatzl@49811 1097
tschatzl@49811 1098 public:
tschatzl@49811 1099 G1UpdateRemSetTrackingBeforeRebuildTask(G1CollectedHeap* g1h, G1ConcurrentMark* cm, uint num_workers) :
tschatzl@49811 1100 AbstractGangTask("G1 Update RemSet Tracking Before Rebuild"),
tschatzl@49811 1101 _g1h(g1h), _cm(cm), _hrclaimer(num_workers), _total_selected_for_rebuild(0), _cl("Post-Marking") { }
tschatzl@49811 1102
tschatzl@49811 1103 virtual void work(uint worker_id) {
tschatzl@49811 1104 G1UpdateRemSetTrackingBeforeRebuild update_cl(_g1h, _cm, &_cl);
tschatzl@49811 1105 _g1h->heap_region_par_iterate_from_worker_offset(&update_cl, &_hrclaimer, worker_id);
tschatzl@49811 1106 Atomic::add(update_cl.num_selected_for_rebuild(), &_total_selected_for_rebuild);
tschatzl@49607 1107 }
tschatzl@49607 1108
tschatzl@49811 1109 uint total_selected_for_rebuild() const { return _total_selected_for_rebuild; }
tschatzl@49811 1110
tschatzl@49811 1111 // Number of regions for which roughly one thread should be spawned for this work.
tschatzl@49811 1112 static const uint RegionsPerThread = 384;
tschatzl@49607 1113 };
tschatzl@49607 1114
tschatzl@49607 1115 class G1UpdateRemSetTrackingAfterRebuild : public HeapRegionClosure {
tschatzl@49607 1116 G1CollectedHeap* _g1h;
tschatzl@49607 1117 public:
tschatzl@49607 1118 G1UpdateRemSetTrackingAfterRebuild(G1CollectedHeap* g1h) : _g1h(g1h) { }
tschatzl@49607 1119
tschatzl@49607 1120 virtual bool do_heap_region(HeapRegion* r) {
tschatzl@53985 1121 _g1h->policy()->remset_tracker()->update_after_rebuild(r);
tschatzl@49607 1122 return false;
tschatzl@49607 1123 }
tschatzl@49607 1124 };
tschatzl@49607 1125
tschatzl@49644 1126 void G1ConcurrentMark::remark() {
tschatzl@49644 1127 assert_at_safepoint_on_vm_thread();
tschatzl@49644 1128
tschatzl@49644 1129 // If a full collection has happened, we should not continue. However we might
tschatzl@49644 1130 // have ended up here as the Remark VM operation has been scheduled already.
ysr@1374 1131 if (has_aborted()) {
ysr@1374 1132 return;
ysr@1374 1133 }
ysr@1374 1134
tschatzl@53985 1135 G1Policy* policy = _g1h->policy();
tschatzl@53985 1136 policy->record_concurrent_mark_remark_start();
ysr@1374 1137
ysr@1374 1138 double start = os::elapsedTime();
ysr@1374 1139
tschatzl@49664 1140 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "Remark before");
tschatzl@49664 1141
tschatzl@49664 1142 {
tschatzl@49807 1143 GCTraceTime(Debug, gc, phases) debug("Finalize Marking", _gc_timer_cm);
tschatzl@49664 1144 finalize_marking();
tschatzl@49664 1145 }
ysr@1374 1146
ysr@1374 1147 double mark_work_end = os::elapsedTime();
ysr@1374 1148
tschatzl@49664 1149 bool const mark_finished = !has_overflown();
tschatzl@49664 1150 if (mark_finished) {
tschatzl@49664 1151 weak_refs_work(false /* clear_all_soft_refs */);
tschatzl@49664 1152
pliden@49751 1153 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
ysr@1374 1154 // We're done with marking.
tschatzl@49644 1155 // This is the end of the marking cycle, we're expected all
tonyp@5082 1156 // threads to have SATB queues with active set to true.
tonyp@7920 1157 satb_mq_set.set_active_all_threads(false, /* new active value */
tonyp@7920 1158 true /* expected_active */);
tonyp@3000 1159
tschatzl@49606 1160 {
tschatzl@49807 1161 GCTraceTime(Debug, gc, phases) debug("Flush Task Caches", _gc_timer_cm);
tschatzl@49606 1162 flush_all_task_caches();
tschatzl@49606 1163 }
tschatzl@49606 1164
tschatzl@49803 1165 // Install newly created mark bitmap as "prev".
tschatzl@49803 1166 swap_mark_bitmaps();
tschatzl@49607 1167 {
tschatzl@49807 1168 GCTraceTime(Debug, gc, phases) debug("Update Remembered Set Tracking Before Rebuild", _gc_timer_cm);
tschatzl@49811 1169
tschatzl@49811 1170 uint const workers_by_capacity = (_g1h->num_regions() + G1UpdateRemSetTrackingBeforeRebuildTask::RegionsPerThread - 1) /
tschatzl@49811 1171 G1UpdateRemSetTrackingBeforeRebuildTask::RegionsPerThread;
tschatzl@49811 1172 uint const num_workers = MIN2(_g1h->workers()->active_workers(), workers_by_capacity);
tschatzl@49811 1173
tschatzl@49811 1174 G1UpdateRemSetTrackingBeforeRebuildTask cl(_g1h, this, num_workers);
tschatzl@49811 1175 log_debug(gc,ergo)("Running %s using %u workers for %u regions in heap", cl.name(), num_workers, _g1h->num_regions());
tschatzl@49811 1176 _g1h->workers()->run_task(&cl, num_workers);
tschatzl@49811 1177
tschatzl@49607 1178 log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u",
tschatzl@49811 1179 _g1h->num_regions(), cl.total_selected_for_rebuild());
tschatzl@49607 1180 }
tschatzl@49804 1181 {
tschatzl@49807 1182 GCTraceTime(Debug, gc, phases) debug("Reclaim Empty Regions", _gc_timer_cm);
tschatzl@49804 1183 reclaim_empty_regions();
tschatzl@49804 1184 }
tschatzl@49804 1185
tschatzl@49804 1186 // Clean out dead classes
tschatzl@49804 1187 if (ClassUnloadingWithConcurrentMark) {
tschatzl@49807 1188 GCTraceTime(Debug, gc, phases) debug("Purge Metaspace", _gc_timer_cm);
tschatzl@49804 1189 ClassLoaderDataGraph::purge();
tschatzl@49804 1190 }
tschatzl@49804 1191
tschatzl@52346 1192 _g1h->resize_heap_if_necessary();
tschatzl@52346 1193
tschatzl@49804 1194 compute_new_sizes();
tschatzl@49607 1195
tschatzl@49803 1196 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "Remark after");
tschatzl@49664 1197
johnc@8073 1198 assert(!restart_for_overflow(), "sanity");
johnc@15090 1199 // Completely reset the marking state since marking completed
tschatzl@49644 1200 reset_at_marking_complete();
tschatzl@49664 1201 } else {
tschatzl@49664 1202 // We overflowed. Restart concurrent marking.
tschatzl@49664 1203 _restart_for_overflow = true;
tschatzl@49664 1204
tschatzl@49664 1205 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "Remark overflow");
tschatzl@49664 1206
tschatzl@49664 1207 // Clear the marking state because we will be restarting
tschatzl@49664 1208 // marking due to overflowing the global mark stack.
tschatzl@49664 1209 reset_marking_for_restart();
tschatzl@49664 1210 }
tschatzl@49664 1211
tschatzl@49664 1212 {
tschatzl@49807 1213 GCTraceTime(Debug, gc, phases) debug("Report Object Count", _gc_timer_cm);
tschatzl@49803 1214 report_object_count(mark_finished);
johnc@8073 1215 }
johnc@8073 1216
ysr@1374 1217 // Statistics
ysr@1374 1218 double now = os::elapsedTime();
ysr@1374 1219 _remark_mark_times.add((mark_work_end - start) * 1000.0);
ysr@1374 1220 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
ysr@1374 1221 _remark_times.add((now - start) * 1000.0);
ysr@1374 1222
tschatzl@53985 1223 policy->record_concurrent_mark_remark_end();
ysr@1374 1224 }
ysr@1374 1225
tschatzl@49804 1226 class G1ReclaimEmptyRegionsTask : public AbstractGangTask {
tschatzl@49632 1227 // Per-region work during the Cleanup pause.
tschatzl@49804 1228 class G1ReclaimEmptyRegionsClosure : public HeapRegionClosure {
tschatzl@49644 1229 G1CollectedHeap* _g1h;
tschatzl@49632 1230 size_t _freed_bytes;
tschatzl@49632 1231 FreeRegionList* _local_cleanup_list;
tschatzl@49632 1232 uint _old_regions_removed;
tschatzl@49632 1233 uint _humongous_regions_removed;
tschatzl@49632 1234
tschatzl@49632 1235 public:
tschatzl@49806 1236 G1ReclaimEmptyRegionsClosure(G1CollectedHeap* g1h,
tschatzl@52716 1237 FreeRegionList* local_cleanup_list) :
tschatzl@49806 1238 _g1h(g1h),
tschatzl@49632 1239 _freed_bytes(0),
tschatzl@49632 1240 _local_cleanup_list(local_cleanup_list),
tschatzl@49632 1241 _old_regions_removed(0),
tschatzl@52716 1242 _humongous_regions_removed(0) { }
tschatzl@49632 1243
tschatzl@49632 1244 size_t freed_bytes() { return _freed_bytes; }
tschatzl@49632 1245 const uint old_regions_removed() { return _old_regions_removed; }
tschatzl@49632 1246 const uint humongous_regions_removed() { return _humongous_regions_removed; }
tschatzl@49632 1247
tschatzl@49632 1248 bool do_heap_region(HeapRegion *hr) {
tschatzl@49632 1249 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) {
tschatzl@49632 1250 _freed_bytes += hr->used();
tschatzl@49632 1251 hr->set_containing_set(NULL);
tschatzl@49632 1252 if (hr->is_humongous()) {
tschatzl@49632 1253 _humongous_regions_removed++;
tschatzl@49644 1254 _g1h->free_humongous_region(hr, _local_cleanup_list);
tschatzl@49632 1255 } else {
tschatzl@49632 1256 _old_regions_removed++;
tschatzl@49644 1257 _g1h->free_region(hr, _local_cleanup_list, false /* skip_remset */, false /* skip_hcc */, true /* locked */);
tschatzl@49632 1258 }
tschatzl@49632 1259 hr->clear_cardtable();
tschatzl@49664 1260 _g1h->concurrent_mark()->clear_statistics_in_region(hr->hrm_index());
tschatzl@49664 1261 log_trace(gc)("Reclaimed empty region %u (%s) bot " PTR_FORMAT, hr->hrm_index(), hr->get_short_type_str(), p2i(hr->bottom()));
brutisso@23450 1262 }
tschatzl@49632 1263
tschatzl@49632 1264 return false;
brutisso@23450 1265 }
tschatzl@49632 1266 };
tschatzl@49632 1267
ysr@1374 1268 G1CollectedHeap* _g1h;
tonyp@7923 1269 FreeRegionList* _cleanup_list;
mlarsson@27009 1270 HeapRegionClaimer _hrclaimer;
tonyp@7923 1271
ysr@1374 1272 public:
tschatzl@49804 1273 G1ReclaimEmptyRegionsTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
tschatzl@49632 1274 AbstractGangTask("G1 Cleanup"),
tschatzl@49632 1275 _g1h(g1h),
tschatzl@49632 1276 _cleanup_list(cleanup_list),
tschatzl@49632 1277 _hrclaimer(n_workers) {
mlarsson@27009 1278 }
ysr@1374 1279
jmasa@11396 1280 void work(uint worker_id) {
tonyp@8072 1281 FreeRegionList local_cleanup_list("Local Cleanup List");
tschatzl@52716 1282 G1ReclaimEmptyRegionsClosure cl(_g1h, &local_cleanup_list);
tschatzl@49632 1283 _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hrclaimer, worker_id);
tschatzl@49632 1284 assert(cl.is_complete(), "Shouldn't have aborted!");
tschatzl@49632 1285
tschatzl@49632 1286 // Now update the old/humongous region sets
tschatzl@49632 1287 _g1h->remove_from_old_sets(cl.old_regions_removed(), cl.humongous_regions_removed());
ysr@1374 1288 {
ysr@1374 1289 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
tschatzl@49632 1290 _g1h->decrement_summary_bytes(cl.freed_bytes());
tonyp@10001 1291
jwilhelm@23471 1292 _cleanup_list->add_ordered(&local_cleanup_list);
tonyp@8072 1293 assert(local_cleanup_list.is_empty(), "post-condition");
ysr@1374 1294 }
ysr@1374 1295 }
ysr@1374 1296 };
ysr@1374 1297
tschatzl@49632 1298 void G1ConcurrentMark::reclaim_empty_regions() {
tschatzl@49632 1299 WorkGang* workers = _g1h->workers();
tschatzl@49632 1300 FreeRegionList empty_regions_list("Empty Regions After Mark List");
tschatzl@49632 1301
tschatzl@49804 1302 G1ReclaimEmptyRegionsTask cl(_g1h, &empty_regions_list, workers->active_workers());
tschatzl@49632 1303 workers->run_task(&cl);
tschatzl@49632 1304
tschatzl@49632 1305 if (!empty_regions_list.is_empty()) {
tschatzl@49664 1306 log_debug(gc)("Reclaimed %u empty regions", empty_regions_list.length());
tschatzl@49632 1307 // Now print the empty regions list.
tschatzl@49632 1308 G1HRPrinter* hrp = _g1h->hr_printer();
tschatzl@49632 1309 if (hrp->is_active()) {
tschatzl@49632 1310 FreeRegionListIterator iter(&empty_regions_list);
tschatzl@49632 1311 while (iter.more_available()) {
tschatzl@49632 1312 HeapRegion* hr = iter.get_next();
tschatzl@49632 1313 hrp->cleanup(hr);
tschatzl@49632 1314 }
tschatzl@49632 1315 }
tschatzl@49632 1316 // And actually make them available.
tschatzl@49632 1317 _g1h->prepend_to_freelist(&empty_regions_list);
tschatzl@49632 1318 }
tschatzl@49632 1319 }
tschatzl@49632 1320
tschatzl@49804 1321 void G1ConcurrentMark::compute_new_sizes() {
tschatzl@49804 1322 MetaspaceGC::compute_new_size();
tschatzl@49804 1323
tschatzl@49804 1324 // Cleanup will have freed any regions completely full of garbage.
tschatzl@49804 1325 // Update the soft reference policy with the new heap occupancy.
tschatzl@49804 1326 Universe::update_heap_info_at_gc();
tschatzl@49804 1327
tschatzl@49804 1328 // We reclaimed old regions so we should calculate the sizes to make
tschatzl@49804 1329 // sure we update the old gen/space data.
tschatzl@49804 1330 _g1h->g1mm()->update_sizes();
tschatzl@49804 1331 }
tschatzl@49804 1332
ehelin@35943 1333 void G1ConcurrentMark::cleanup() {
tschatzl@49644 1334 assert_at_safepoint_on_vm_thread();
ysr@1374 1335
ysr@1374 1336 // If a full collection has happened, we shouldn't do this.
ysr@1374 1337 if (has_aborted()) {
ysr@1374 1338 return;
ysr@1374 1339 }
ysr@1374 1340
tschatzl@53985 1341 G1Policy* policy = _g1h->policy();
tschatzl@53985 1342 policy->record_concurrent_mark_cleanup_start();
ysr@1374 1343
ysr@1374 1344 double start = os::elapsedTime();
ysr@1374 1345
tschatzl@49803 1346 verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "Cleanup before");
tschatzl@49664 1347
tschatzl@37413 1348 {
tschatzl@49807 1349 GCTraceTime(Debug, gc, phases) debug("Update Remembered Set Tracking After Rebuild", _gc_timer_cm);
tschatzl@49607 1350 G1UpdateRemSetTrackingAfterRebuild cl(_g1h);
tschatzl@49644 1351 _g1h->heap_region_iterate(&cl);
johnc@11583 1352 }
johnc@11583 1353
brutisso@35061 1354 if (log_is_enabled(Trace, gc, liveness)) {
tschatzl@49607 1355 G1PrintRegionLivenessInfoClosure cl("Post-Cleanup");
tonyp@8930 1356 _g1h->heap_region_iterate(&cl);
tonyp@8930 1357 }
tonyp@8930 1358
tschatzl@49664 1359 verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "Cleanup after");
tschatzl@49664 1360
tschatzl@49664 1361 // We need to make this be a "collection" so any collection pause that
tschatzl@49664 1362 // races with it goes around and waits for Cleanup to finish.
tschatzl@49664 1363 _g1h->increment_total_collections();
tschatzl@49664 1364
tschatzl@49664 1365 // Local statistics
tschatzl@49664 1366 double recent_cleanup_time = (os::elapsedTime() - start);
tschatzl@49664 1367 _total_cleanup_time += recent_cleanup_time;
tschatzl@49664 1368 _cleanup_times.add(recent_cleanup_time);
tschatzl@49664 1369
tschatzl@49607 1370 {
tschatzl@49807 1371 GCTraceTime(Debug, gc, phases) debug("Finalize Concurrent Mark Cleanup", _gc_timer_cm);
tschatzl@53985 1372 policy->record_concurrent_mark_cleanup_end();
ysr@1374 1373 }
ysr@1374 1374 }
ysr@1374 1375
johnc@15495 1376 // 'Keep Alive' oop closure used by both serial parallel reference processing.
ehelin@35943 1377 // Uses the G1CMTask associated with a worker thread (for serial reference
ehelin@35943 1378 // processing the G1CMTask for worker 0 is used) to preserve (mark) and
johnc@15495 1379 // trace referent objects.
johnc@15495 1380 //
ehelin@35943 1381 // Using the G1CMTask and embedded local queues avoids having the worker
johnc@15495 1382 // threads operating on the global mark stack. This reduces the risk
johnc@15495 1383 // of overflowing the stack - which we would rather avoid at this late
johnc@15495 1384 // state. Also using the tasks' local queues removes the potential
johnc@15495 1385 // of the workers interfering with each other that could occur if
johnc@15495 1386 // operating on the global stack.
johnc@15495 1387
tschatzl@49644 1388 class G1CMKeepAliveAndDrainClosure : public OopClosure {
ehelin@35943 1389 G1ConcurrentMark* _cm;
ehelin@35943 1390 G1CMTask* _task;
tschatzl@49819 1391 uint _ref_counter_limit;
tschatzl@49819 1392 uint _ref_counter;
ehelin@35943 1393 bool _is_serial;
tschatzl@49644 1394 public:
ehelin@35943 1395 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
tschatzl@51332 1396 _cm(cm), _task(task), _ref_counter_limit(G1RefProcDrainInterval),
tschatzl@51332 1397 _ref_counter(_ref_counter_limit), _is_serial(is_serial) {
johnc@16388 1398 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
johnc@8073 1399 }
johnc@8073 1400
johnc@8073 1401 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
johnc@8073 1402 virtual void do_oop( oop* p) { do_oop_work(p); }
johnc@8073 1403
johnc@8073 1404 template <class T> void do_oop_work(T* p) {
tschatzl@49809 1405 if (_cm->has_overflown()) {
tschatzl@49809 1406 return;
tschatzl@49809 1407 }
tschatzl@49809 1408 if (!_task->deal_with_reference(p)) {
tschatzl@49809 1409 // We did not add anything to the mark bitmap (or mark stack), so there is
tschatzl@49809 1410 // no point trying to drain it.
tschatzl@49809 1411 return;
tschatzl@49809 1412 }
tschatzl@49809 1413 _ref_counter--;
tschatzl@49809 1414
tschatzl@49809 1415 if (_ref_counter == 0) {
tschatzl@49809 1416 // We have dealt with _ref_counter_limit references, pushing them
tschatzl@49809 1417 // and objects reachable from them on to the local stack (and
tschatzl@49809 1418 // possibly the global stack). Call G1CMTask::do_marking_step() to
tschatzl@49809 1419 // process these entries.
tschatzl@49809 1420 //
tschatzl@49809 1421 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if
tschatzl@49809 1422 // there's nothing more to do (i.e. we're done with the entries that
tschatzl@49809 1423 // were pushed as a result of the G1CMTask::deal_with_reference() calls
tschatzl@49809 1424 // above) or we overflow.
tschatzl@49809 1425 //
tschatzl@49809 1426 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
tschatzl@49809 1427 // flag while there may still be some work to do. (See the comment at
tschatzl@49809 1428 // the beginning of G1CMTask::do_marking_step() for those conditions -
tschatzl@49809 1429 // one of which is reaching the specified time target.) It is only
tschatzl@49809 1430 // when G1CMTask::do_marking_step() returns without setting the
tschatzl@49809 1431 // has_aborted() flag that the marking step has completed.
tschatzl@49809 1432 do {
tschatzl@49809 1433 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
tschatzl@49809 1434 _task->do_marking_step(mark_step_duration_ms,
tschatzl@49809 1435 false /* do_termination */,
tschatzl@49809 1436 _is_serial);
tschatzl@49809 1437 } while (_task->has_aborted() && !_cm->has_overflown());
tschatzl@49809 1438 _ref_counter = _ref_counter_limit;
johnc@8073 1439 }
johnc@8073 1440 }
johnc@8073 1441 };
johnc@8073 1442
johnc@15495 1443 // 'Drain' oop closure used by both serial and parallel reference processing.
ehelin@35943 1444 // Uses the G1CMTask associated with a given worker thread (for serial
ehelin@35943 1445 // reference processing the G1CMtask for worker 0 is used). Calls the
johnc@15495 1446 // do_marking_step routine, with an unbelievably large timeout value,
johnc@15495 1447 // to drain the marking data structures of the remaining entries
johnc@15495 1448 // added by the 'keep alive' oop closure above.
johnc@15495 1449
tschatzl@49644 1450 class G1CMDrainMarkingStackClosure : public VoidClosure {
ehelin@35943 1451 G1ConcurrentMark* _cm;
ehelin@35943 1452 G1CMTask* _task;
ehelin@35943 1453 bool _is_serial;
johnc@8073 1454 public:
ehelin@35943 1455 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
johnc@16388 1456 _cm(cm), _task(task), _is_serial(is_serial) {
johnc@16388 1457 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
johnc@15495 1458 }
johnc@8073 1459
johnc@8073 1460 void do_void() {
johnc@8073 1461 do {
ehelin@35943 1462 // We call G1CMTask::do_marking_step() to completely drain the local
johnc@15495 1463 // and global marking stacks of entries pushed by the 'keep alive'
johnc@15495 1464 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above).
johnc@15495 1465 //
ehelin@35943 1466 // G1CMTask::do_marking_step() is called in a loop, which we'll exit
jwilhelm@22551 1467 // if there's nothing more to do (i.e. we've completely drained the
johnc@15495 1468 // entries that were pushed as a a result of applying the 'keep alive'
johnc@15495 1469 // closure to the entries on the discovered ref lists) or we overflow
johnc@15495 1470 // the global marking stack.
johnc@15495 1471 //
ehelin@35943 1472 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
johnc@15495 1473 // flag while there may still be some work to do. (See the comment at
ehelin@35943 1474 // the beginning of G1CMTask::do_marking_step() for those conditions -
johnc@15495 1475 // one of which is reaching the specified time target.) It is only
ehelin@35943 1476 // when G1CMTask::do_marking_step() returns without setting the
johnc@15495 1477 // has_aborted() flag that the marking step has completed.
johnc@8073 1478
johnc@8073 1479 _task->do_marking_step(1000000000.0 /* something very large */,
johnc@16388 1480 true /* do_termination */,
johnc@16388 1481 _is_serial);
johnc@8073 1482 } while (_task->has_aborted() && !_cm->has_overflown());
johnc@8073 1483 }
johnc@8073 1484 };
johnc@8073 1485
johnc@10670 1486 // Implementation of AbstractRefProcTaskExecutor for parallel
johnc@10670 1487 // reference processing at the end of G1 concurrent marking
johnc@10670 1488
tschatzl@49644 1489 class G1CMRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
johnc@8073 1490 private:
ehelin@35943 1491 G1CollectedHeap* _g1h;
ehelin@35943 1492 G1ConcurrentMark* _cm;
ehelin@35943 1493 WorkGang* _workers;
ehelin@35943 1494 uint _active_workers;
johnc@8073 1495
johnc@8073 1496 public:
johnc@10670 1497 G1CMRefProcTaskExecutor(G1CollectedHeap* g1h,
ehelin@35943 1498 G1ConcurrentMark* cm,
eistepan@30585 1499 WorkGang* workers,
eistepan@30585 1500 uint n_workers) :
johnc@11172 1501 _g1h(g1h), _cm(cm),
johnc@11172 1502 _workers(workers), _active_workers(n_workers) { }
johnc@8073 1503
tschatzl@50606 1504 virtual void execute(ProcessTask& task, uint ergo_workers);
johnc@8073 1505 };
johnc@8073 1506
tschatzl@49644 1507 class G1CMRefProcTaskProxy : public AbstractGangTask {
johnc@8073 1508 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
ehelin@35943 1509 ProcessTask& _proc_task;
ehelin@35943 1510 G1CollectedHeap* _g1h;
ehelin@35943 1511 G1ConcurrentMark* _cm;
johnc@8073 1512
johnc@8073 1513 public:
johnc@10670 1514 G1CMRefProcTaskProxy(ProcessTask& proc_task,
ehelin@35943 1515 G1CollectedHeap* g1h,
ehelin@35943 1516 G1ConcurrentMark* cm) :
johnc@8073 1517 AbstractGangTask("Process reference objects in parallel"),
johnc@15495 1518 _proc_task(proc_task), _g1h(g1h), _cm(cm) {
johnc@16388 1519 ReferenceProcessor* rp = _g1h->ref_processor_cm();
johnc@16388 1520 assert(rp->processing_is_mt(), "shouldn't be here otherwise");
johnc@16388 1521 }
johnc@8073 1522
jmasa@11396 1523 virtual void work(uint worker_id) {
mdoerr@25893 1524 ResourceMark rm;
mdoerr@25893 1525 HandleMark hm;
ehelin@35943 1526 G1CMTask* task = _cm->task(worker_id);
johnc@8073 1527 G1CMIsAliveClosure g1_is_alive(_g1h);
johnc@16388 1528 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
johnc@16388 1529 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
johnc@8073 1530
jmasa@11396 1531 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
johnc@8073 1532 }
johnc@8073 1533 };
johnc@8073 1534
tschatzl@50606 1535 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task, uint ergo_workers) {
johnc@8073 1536 assert(_workers != NULL, "Need parallel worker threads.");
johnc@15495 1537 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
tschatzl@50606 1538 assert(_workers->active_workers() >= ergo_workers,
tschatzl@50606 1539 "Ergonomically chosen workers(%u) should be less than or equal to active workers(%u)",
tschatzl@50606 1540 ergo_workers, _workers->active_workers());
johnc@8073 1541
johnc@11172 1542 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
johnc@8073 1543
johnc@16389 1544 // We need to reset the concurrency level before each
johnc@16389 1545 // proxy task execution, so that the termination protocol
ehelin@35943 1546 // and overflow handling in G1CMTask::do_marking_step() knows
johnc@16389 1547 // how many workers to wait for.
tschatzl@50606 1548 _cm->set_concurrency(ergo_workers);
tschatzl@50606 1549 _workers->run_task(&proc_task_proxy, ergo_workers);
johnc@8073 1550 }
johnc@8073 1551
tschatzl@47678 1552 void G1ConcurrentMark::weak_refs_work(bool clear_all_soft_refs) {
ysr@1374 1553 ResourceMark rm;
ysr@1374 1554 HandleMark hm;
johnc@10666 1555
johnc@10666 1556 // Is alive closure.
tschatzl@49644 1557 G1CMIsAliveClosure g1_is_alive(_g1h);
johnc@10666 1558
gziemski@51405 1559 // Inner scope to exclude the cleaning of the string table
gziemski@51405 1560 // from the displayed time.
johnc@10666 1561 {
tschatzl@49807 1562 GCTraceTime(Debug, gc, phases) debug("Reference Processing", _gc_timer_cm);
johnc@10666 1563
tschatzl@49644 1564 ReferenceProcessor* rp = _g1h->ref_processor_cm();
johnc@10666 1565
johnc@10666 1566 // See the comment in G1CollectedHeap::ref_processing_init()
johnc@10666 1567 // about how reference processing currently works in G1.
johnc@10666 1568
johnc@15495 1569 // Set the soft reference policy
johnc@10666 1570 rp->setup_policy(clear_all_soft_refs);
tschatzl@39966 1571 assert(_global_mark_stack.is_empty(), "mark stack should be empty");
johnc@10666 1572
stefank@47676 1573 // Instances of the 'Keep Alive' and 'Complete GC' closures used
stefank@47676 1574 // in serial reference processing. Note these closures are also
stefank@47676 1575 // used for serially processing (by the the current thread) the
stefank@47676 1576 // JNI references during parallel reference processing.
stefank@47676 1577 //
stefank@47676 1578 // These closures do not need to synchronize with the worker
stefank@47676 1579 // threads involved in parallel reference processing as these
stefank@47676 1580 // instances are executed serially by the current thread (e.g.
stefank@47676 1581 // reference processing is not multi-threaded and is thus
stefank@47676 1582 // performed by the current thread instead of a gang worker).
stefank@47676 1583 //
stefank@47676 1584 // The gang tasks involved in parallel reference processing create
stefank@47676 1585 // their own instances of these closures, which do their own
stefank@47676 1586 // synchronization among themselves.
stefank@47676 1587 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
stefank@47676 1588 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
stefank@47676 1589
johnc@16388 1590 // We need at least one active thread. If reference processing
johnc@16388 1591 // is not multi-threaded we use the current (VMThread) thread,
johnc@16388 1592 // otherwise we use the work gang from the G1CollectedHeap and
johnc@16388 1593 // we utilize all the worker threads we can.
mlarsson@27251 1594 bool processing_is_mt = rp->processing_is_mt();
tschatzl@49644 1595 uint active_workers = (processing_is_mt ? _g1h->workers()->active_workers() : 1U);
tschatzl@47696 1596 active_workers = MAX2(MIN2(active_workers, _max_num_tasks), 1U);
johnc@10666 1597
johnc@16388 1598 // Parallel processing task executor.
tschatzl@49644 1599 G1CMRefProcTaskExecutor par_task_executor(_g1h, this,
tschatzl@49644 1600 _g1h->workers(), active_workers);
johnc@16388 1601 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
johnc@15495 1602
johnc@16389 1603 // Set the concurrency level. The phase was already set prior to
johnc@16389 1604 // executing the remark task.
johnc@16389 1605 set_concurrency(active_workers);
johnc@16389 1606
johnc@15495 1607 // Set the degree of MT processing here. If the discovery was done MT,
johnc@15495 1608 // the number of threads involved during discovery could differ from
johnc@15495 1609 // the number of active workers. This is OK as long as the discovered
johnc@15495 1610 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
johnc@15495 1611 rp->set_active_mt_degree(active_workers);
johnc@15495 1612
sangheki@50396 1613 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->max_num_queues());
sangheki@46795 1614
johnc@15495 1615 // Process the weak references.
sla@18025 1616 const ReferenceProcessorStats& stats =
sla@18025 1617 rp->process_discovered_references(&g1_is_alive,
sla@18025 1618 &g1_keep_alive,
sla@18025 1619 &g1_drain_mark_stack,
sla@18025 1620 executor,
sangheki@46795 1621 &pt);
sangheki@37137 1622 _gc_tracer_cm->report_gc_reference_stats(stats);
sangheki@46795 1623 pt.print_all_references();
johnc@15495 1624
johnc@15495 1625 // The do_oop work routines of the keep_alive and drain_marking_stack
johnc@15495 1626 // oop closures will set the has_overflown flag if we overflow the
johnc@15495 1627 // global marking stack.
johnc@10666 1628
aharlap@46319 1629 assert(has_overflown() || _global_mark_stack.is_empty(),
tschatzl@49644 1630 "Mark stack should be empty (unless it has overflown)");
johnc@10666 1631
tschatzl@49967 1632 assert(rp->num_queues() == active_workers, "why not");
johnc@15495 1633
johnc@10666 1634 rp->verify_no_references_recorded();
johnc@10670 1635 assert(!rp->discovery_enabled(), "Post condition");
johnc@8073 1636 }
johnc@8073 1637
kbarrett@49861 1638 if (has_overflown()) {
tschatzl@50442 1639 // We can not trust g1_is_alive and the contents of the heap if the marking stack
tschatzl@50442 1640 // overflowed while processing references. Exit the VM.
tschatzl@50442 1641 fatal("Overflow during reference processing, can not continue. Please "
tschatzl@50442 1642 "increase MarkStackSizeMax (current value: " SIZE_FORMAT ") and "
tschatzl@50442 1643 "restart.", MarkStackSizeMax);
kbarrett@49861 1644 return;
kbarrett@49861 1645 }
kbarrett@49861 1646
kbarrett@49861 1647 assert(_global_mark_stack.is_empty(), "Marking should have completed");
stefank@47676 1648
stefank@47648 1649 {
stefank@47648 1650 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm);
kbarrett@51546 1651 WeakProcessor::weak_oops_do(_g1h->workers(), &g1_is_alive, &do_nothing_cl, 1);
stefank@47648 1652 }
stefank@47648 1653
gziemski@51405 1654 // Unload Klasses, String, Code Cache, etc.
stefank@37146 1655 if (ClassUnloadingWithConcurrentMark) {
sjohanss@46348 1656 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
tschatzl@52498 1657 bool purged_classes = SystemDictionary::do_unloading(_gc_timer_cm);
tschatzl@49644 1658 _g1h->complete_cleaning(&g1_is_alive, purged_classes);
tschatzl@53536 1659 } else if (StringDedup::is_enabled()) {
tschatzl@53536 1660 GCTraceTime(Debug, gc, phases) debug("String Deduplication", _gc_timer_cm);
tschatzl@53536 1661 _g1h->string_dedup_cleaning(&g1_is_alive, NULL);
stefank@37146 1662 }
ysr@1374 1663 }
ysr@1374 1664
tschatzl@50097 1665 class G1PrecleanYieldClosure : public YieldClosure {
tschatzl@50097 1666 G1ConcurrentMark* _cm;
tschatzl@50097 1667
tschatzl@50097 1668 public:
tschatzl@50097 1669 G1PrecleanYieldClosure(G1ConcurrentMark* cm) : _cm(cm) { }
tschatzl@50097 1670
tschatzl@50097 1671 virtual bool should_return() {
tschatzl@50097 1672 return _cm->has_aborted();
tschatzl@50097 1673 }
tschatzl@50097 1674
tschatzl@50097 1675 virtual bool should_return_fine_grain() {
tschatzl@50097 1676 _cm->do_yield_check();
tschatzl@50097 1677 return _cm->has_aborted();
tschatzl@50097 1678 }
tschatzl@50097 1679 };
tschatzl@50097 1680
tschatzl@50097 1681 void G1ConcurrentMark::preclean() {
tschatzl@50097 1682 assert(G1UseReferencePrecleaning, "Precleaning must be enabled.");
tschatzl@50097 1683
tschatzl@50097 1684 SuspendibleThreadSetJoiner joiner;
tschatzl@50097 1685
tschatzl@50097 1686 G1CMKeepAliveAndDrainClosure keep_alive(this, task(0), true /* is_serial */);
tschatzl@50097 1687 G1CMDrainMarkingStackClosure drain_mark_stack(this, task(0), true /* is_serial */);
tschatzl@50097 1688
tschatzl@50097 1689 set_concurrency_and_phase(1, true);
tschatzl@50097 1690
tschatzl@50097 1691 G1PrecleanYieldClosure yield_cl(this);
tschatzl@50097 1692
tschatzl@50097 1693 ReferenceProcessor* rp = _g1h->ref_processor_cm();
tschatzl@50097 1694 // Precleaning is single threaded. Temporarily disable MT discovery.
tschatzl@50097 1695 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false);
tschatzl@50097 1696 rp->preclean_discovered_references(rp->is_alive_non_header(),
tschatzl@50097 1697 &keep_alive,
tschatzl@50097 1698 &drain_mark_stack,
tschatzl@50097 1699 &yield_cl,
tschatzl@50097 1700 _gc_timer_cm);
tschatzl@50097 1701 }
tschatzl@50097 1702
tschatzl@49803 1703 // When sampling object counts, we already swapped the mark bitmaps, so we need to use
tschatzl@49803 1704 // the prev bitmap determining liveness.
tschatzl@49803 1705 class G1ObjectCountIsAliveClosure: public BoolObjectClosure {
tschatzl@49806 1706 G1CollectedHeap* _g1h;
tschatzl@49806 1707 public:
tschatzl@49806 1708 G1ObjectCountIsAliveClosure(G1CollectedHeap* g1h) : _g1h(g1h) { }
tschatzl@49803 1709
tschatzl@49803 1710 bool do_object_b(oop obj) {
tschatzl@49803 1711 HeapWord* addr = (HeapWord*)obj;
tschatzl@49803 1712 return addr != NULL &&
tschatzl@49806 1713 (!_g1h->is_in_g1_reserved(addr) || !_g1h->is_obj_dead(obj));
tschatzl@49803 1714 }
tschatzl@49803 1715 };
tschatzl@49803 1716
tschatzl@49803 1717 void G1ConcurrentMark::report_object_count(bool mark_completed) {
tschatzl@49803 1718 // Depending on the completion of the marking liveness needs to be determined
tschatzl@49803 1719 // using either the next or prev bitmap.
tschatzl@49803 1720 if (mark_completed) {
tschatzl@49803 1721 G1ObjectCountIsAliveClosure is_alive(_g1h);
tschatzl@49803 1722 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
tschatzl@49803 1723 } else {
tschatzl@49803 1724 G1CMIsAliveClosure is_alive(_g1h);
tschatzl@49803 1725 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
tschatzl@49803 1726 }
tschatzl@49664 1727 }
tschatzl@49664 1728
tschatzl@49803 1729
tschatzl@47678 1730 void G1ConcurrentMark::swap_mark_bitmaps() {
tschatzl@47678 1731 G1CMBitMap* temp = _prev_mark_bitmap;
tschatzl@47678 1732 _prev_mark_bitmap = _next_mark_bitmap;
tschatzl@47678 1733 _next_mark_bitmap = temp;
tschatzl@49664 1734 _g1h->collector_state()->set_clearing_next_bitmap(true);
ysr@1374 1735 }
ysr@1374 1736
kbarrett@30577 1737 // Closure for marking entries in SATB buffers.
ehelin@35943 1738 class G1CMSATBBufferClosure : public SATBBufferClosure {
stefank@25492 1739 private:
ehelin@35943 1740 G1CMTask* _task;
kbarrett@30577 1741 G1CollectedHeap* _g1h;
kbarrett@30577 1742
ehelin@35943 1743 // This is very similar to G1CMTask::deal_with_reference, but with
kbarrett@30577 1744 // more relaxed requirements for the argument, so this must be more
kbarrett@30577 1745 // circumspect about treating the argument as an object.
kbarrett@30577 1746 void do_entry(void* entry) const {
kbarrett@30577 1747 _task->increment_refs_reached();
tschatzl@46752 1748 oop const obj = static_cast<oop>(entry);
tschatzl@46752 1749 _task->make_reference_grey(obj);
kbarrett@30577 1750 }
stefank@25492 1751
stefank@25492 1752 public:
ehelin@35943 1753 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
kbarrett@30577 1754 : _task(task), _g1h(g1h) { }
kbarrett@30577 1755
kbarrett@30577 1756 virtual void do_buffer(void** buffer, size_t size) {
kbarrett@30577 1757 for (size_t i = 0; i < size; ++i) {
kbarrett@30577 1758 do_entry(buffer[i]);
kbarrett@30577 1759 }
stefank@25492 1760 }
stefank@25492 1761 };
stefank@25492 1762
stefank@25492 1763 class G1RemarkThreadsClosure : public ThreadClosure {
ehelin@35943 1764 G1CMSATBBufferClosure _cm_satb_cl;
stefank@25492 1765 G1CMOopClosure _cm_cl;
stefank@25492 1766 MarkingCodeBlobClosure _code_cl;
stefank@25492 1767 int _thread_parity;
stefank@25492 1768
stefank@25492 1769 public:
ehelin@35943 1770 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) :
kbarrett@30577 1771 _cm_satb_cl(task, g1h),
tschatzl@49644 1772 _cm_cl(g1h, task),
kbarrett@30577 1773 _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
mgerdin@29802 1774 _thread_parity(Threads::thread_claim_parity()) {}
stefank@25492 1775
stefank@25492 1776 void do_thread(Thread* thread) {
kbarrett@54006 1777 if (thread->claim_oops_do(true, _thread_parity)) {
kbarrett@54006 1778 SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread);
kbarrett@54006 1779 queue.apply_closure_and_empty(&_cm_satb_cl);
kbarrett@54006 1780 if (thread->is_Java_thread()) {
stefank@25492 1781 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
stefank@25492 1782 // however the liveness of oops reachable from nmethods have very complex lifecycles:
stefank@25492 1783 // * Alive if on the stack of an executing method
stefank@25492 1784 // * Weakly reachable otherwise
stefank@25492 1785 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
stefank@25492 1786 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
kbarrett@54006 1787 JavaThread* jt = (JavaThread*)thread;
stefank@25492 1788 jt->nmethods_do(&_code_cl);
stefank@25492 1789 }
stefank@25492 1790 }
stefank@25492 1791 }
stefank@25492 1792 };
stefank@25492 1793
tschatzl@49644 1794 class G1CMRemarkTask : public AbstractGangTask {
ehelin@35943 1795 G1ConcurrentMark* _cm;
ysr@1374 1796 public:
jmasa@11396 1797 void work(uint worker_id) {
lkorinth@48091 1798 G1CMTask* task = _cm->task(worker_id);
lkorinth@48091 1799 task->record_start_time();
lkorinth@48091 1800 {
lkorinth@48091 1801 ResourceMark rm;
lkorinth@48091 1802 HandleMark hm;
lkorinth@48091 1803
lkorinth@48091 1804 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
lkorinth@48091 1805 Threads::threads_do(&threads_f);
ysr@1374 1806 }
lkorinth@48091 1807
lkorinth@48091 1808 do {
lkorinth@48091 1809 task->do_marking_step(1000000000.0 /* something very large */,
lkorinth@48091 1810 true /* do_termination */,
lkorinth@48091 1811 false /* is_serial */);
lkorinth@48091 1812 } while (task->has_aborted() && !_cm->has_overflown());
lkorinth@48091 1813 // If we overflow, then we do not want to restart. We instead
lkorinth@48091 1814 // want to abort remark and do concurrent marking again.
lkorinth@48091 1815 task->record_end_time();
ysr@1374 1816 }
ysr@1374 1817
ehelin@35943 1818 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) :
mlarsson@27251 1819 AbstractGangTask("Par Remark"), _cm(cm) {
johnc@11250 1820 _cm->terminator()->reset_for_reuse(active_workers);
jmasa@11174 1821 }
ysr@1374 1822 };
ysr@1374 1823
tschatzl@49644 1824 void G1ConcurrentMark::finalize_marking() {
ysr@1374 1825 ResourceMark rm;
ysr@1374 1826 HandleMark hm;
ysr@1374 1827
tschatzl@49644 1828 _g1h->ensure_parsability(false);
ysr@1374 1829
mlarsson@27251 1830 // this is remark, so we'll use up all active threads
tschatzl@49644 1831 uint active_workers = _g1h->workers()->active_workers();
mlarsson@27251 1832 set_concurrency_and_phase(active_workers, false /* concurrent */);
mlarsson@27251 1833 // Leave _parallel_marking_threads at it's
ehelin@35943 1834 // value originally calculated in the G1ConcurrentMark
mlarsson@27251 1835 // constructor and pass values of the active workers
mlarsson@27251 1836 // through the gang in the task.
mlarsson@27251 1837
stefank@30868 1838 {
stefank@30868 1839 StrongRootsScope srs(active_workers);
stefank@30868 1840
ehelin@35943 1841 G1CMRemarkTask remarkTask(this, active_workers);
stefank@30868 1842 // We will start all available threads, even if we decide that the
stefank@30868 1843 // active_workers will be fewer. The extra ones will just bail out
stefank@30868 1844 // immediately.
tschatzl@49644 1845 _g1h->workers()->run_task(&remarkTask);
stefank@30868 1846 }
mlarsson@27251 1847
pliden@49751 1848 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
johnc@16390 1849 guarantee(has_overflown() ||
johnc@16390 1850 satb_mq_set.completed_buffers_num() == 0,
jprovino@36371 1851 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
david@33105 1852 BOOL_TO_STR(has_overflown()),
david@33105 1853 satb_mq_set.completed_buffers_num());
ysr@1374 1854
ysr@1374 1855 print_stats();
ysr@1374 1856 }
ysr@1374 1857
tschatzl@49606 1858 void G1ConcurrentMark::flush_all_task_caches() {
tschatzl@49606 1859 size_t hits = 0;
tschatzl@49606 1860 size_t misses = 0;
tschatzl@49606 1861 for (uint i = 0; i < _max_num_tasks; i++) {
tschatzl@49606 1862 Pair<size_t, size_t> stats = _tasks[i]->flush_mark_stats_cache();
tschatzl@49606 1863 hits += stats.first;
tschatzl@49606 1864 misses += stats.second;
tschatzl@49606 1865 }
tschatzl@49606 1866 size_t sum = hits + misses;
tschatzl@49606 1867 log_debug(gc, stats)("Mark stats cache hits " SIZE_FORMAT " misses " SIZE_FORMAT " ratio %1.3lf",
tschatzl@49606 1868 hits, misses, percent_of(hits, sum));
tschatzl@49606 1869 }
tschatzl@49606 1870
tschatzl@47678 1871 void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) {
tschatzl@47678 1872 _prev_mark_bitmap->clear_range(mr);
tonyp@11455 1873 }
tonyp@11455 1874
ysr@1374 1875 HeapRegion*
ehelin@35943 1876 G1ConcurrentMark::claim_region(uint worker_id) {
ysr@1374 1877 // "checkpoint" the finger
ysr@1374 1878 HeapWord* finger = _finger;
ysr@1374 1879
tschatzl@49644 1880 while (finger < _heap.end()) {
tonyp@4026 1881 assert(_g1h->is_in_g1_reserved(finger), "invariant");
ysr@1374 1882
david@33786 1883 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
tschatzl@41081 1884 // Make sure that the reads below do not float before loading curr_region.
tschatzl@41081 1885 OrderAccess::loadload();
david@33786 1886 // Above heap_region_containing may return NULL as we always scan claim
tschatzl@26160 1887 // until the end of the heap. In this case, just jump to the next region.
tschatzl@26160 1888 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
tonyp@9994 1889
tonyp@9994 1890 // Is the gap between reading the finger and doing the CAS too long?
coleenp@47634 1891 HeapWord* res = Atomic::cmpxchg(end, &_finger, finger);
tschatzl@26160 1892 if (res == finger && curr_region != NULL) {
ysr@1374 1893 // we succeeded
tschatzl@26160 1894 HeapWord* bottom = curr_region->bottom();
tschatzl@26160 1895 HeapWord* limit = curr_region->next_top_at_mark_start();
tschatzl@26160 1896
ysr@1374 1897 // notice that _finger == end cannot be guaranteed here since,
ysr@1374 1898 // someone else might have moved the finger even further
tonyp@4026 1899 assert(_finger >= end, "the finger should have moved forward");
ysr@1374 1900
ysr@1374 1901 if (limit > bottom) {
ysr@1374 1902 return curr_region;
ysr@1374 1903 } else {
tonyp@4026 1904 assert(limit == bottom,
tonyp@4026 1905 "the region limit should be at bottom");
ysr@1374 1906 // we return NULL and the caller should try calling
ysr@1374 1907 // claim_region() again.
ysr@1374 1908 return NULL;
ysr@1374 1909 }
ysr@1374 1910 } else {
tonyp@4026 1911 assert(_finger > finger, "the finger should have moved forward");
ysr@1374 1912 // read it again
ysr@1374 1913 finger = _finger;
ysr@1374 1914 }
ysr@1374 1915 }
ysr@1374 1916
ysr@1374 1917 return NULL;
ysr@1374 1918 }
ysr@1374 1919
tonyp@11455 1920 #ifndef PRODUCT
coleenp@49392 1921 class VerifyNoCSetOops {
tonyp@11455 1922 G1CollectedHeap* _g1h;
kbarrett@31994 1923 const char* _phase;
tonyp@11455 1924 int _info;
tonyp@11455 1925
kbarrett@31994 1926 public:
kbarrett@31994 1927 VerifyNoCSetOops(const char* phase, int info = -1) :
kbarrett@31994 1928 _g1h(G1CollectedHeap::heap()),
kbarrett@31994 1929 _phase(phase),
kbarrett@31994 1930 _info(info)
kbarrett@31994 1931 { }
kbarrett@31994 1932
tschatzl@46328 1933 void operator()(G1TaskQueueEntry task_entry) const {
tschatzl@46328 1934 if (task_entry.is_array_slice()) {
tschatzl@46328 1935 guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice()));
tschatzl@46328 1936 return;
tschatzl@46328 1937 }
coleenp@46968 1938 guarantee(oopDesc::is_oop(task_entry.obj()),
david@33105 1939 "Non-oop " PTR_FORMAT ", phase: %s, info: %d",
tschatzl@46328 1940 p2i(task_entry.obj()), _phase, _info);
tschatzl@46328 1941 guarantee(!_g1h->is_in_cset(task_entry.obj()),
david@33105 1942 "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d",
tschatzl@46328 1943 p2i(task_entry.obj()), _phase, _info);
tonyp@11455 1944 }
tonyp@11455 1945 };
tonyp@11455 1946
ehelin@35943 1947 void G1ConcurrentMark::verify_no_cset_oops() {
tonyp@11455 1948 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
tschatzl@49644 1949 if (!_g1h->collector_state()->mark_or_rebuild_in_progress()) {
tonyp@11455 1950 return;
tonyp@11455 1951 }
tonyp@11455 1952
kbarrett@30279 1953 // Verify entries on the global mark stack
tschatzl@39966 1954 _global_mark_stack.iterate(VerifyNoCSetOops("Stack"));
kbarrett@30279 1955
kbarrett@30279 1956 // Verify entries on the task queues
tschatzl@47696 1957 for (uint i = 0; i < _max_num_tasks; ++i) {
ehelin@35943 1958 G1CMTaskQueue* queue = _task_queues->queue(i);
kbarrett@31994 1959 queue->iterate(VerifyNoCSetOops("Queue", i));
tonyp@11455 1960 }
tonyp@11455 1961
kbarrett@30279 1962 // Verify the global finger
kbarrett@30279 1963 HeapWord* global_finger = finger();
tschatzl@49644 1964 if (global_finger != NULL && global_finger < _heap.end()) {
kbarrett@30279 1965 // Since we always iterate over all regions, we might get a NULL HeapRegion
kbarrett@30279 1966 // here.
david@33786 1967 HeapRegion* global_hr = _g1h->heap_region_containing(global_finger);
kbarrett@30279 1968 guarantee(global_hr == NULL || global_finger == global_hr->bottom(),
david@33105 1969 "global finger: " PTR_FORMAT " region: " HR_FORMAT,
david@33105 1970 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
tonyp@11455 1971 }
tonyp@11455 1972
kbarrett@30279 1973 // Verify the task fingers
tschatzl@47696 1974 assert(_num_concurrent_workers <= _max_num_tasks, "sanity");
tschatzl@47696 1975 for (uint i = 0; i < _num_concurrent_workers; ++i) {
ehelin@35943 1976 G1CMTask* task = _tasks[i];
kbarrett@30279 1977 HeapWord* task_finger = task->finger();
tschatzl@49644 1978 if (task_finger != NULL && task_finger < _heap.end()) {
kbarrett@30279 1979 // See above note on the global finger verification.
david@33786 1980 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
kbarrett@30279 1981 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
kbarrett@30279 1982 !task_hr->in_collection_set(),
david@33105 1983 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
david@33105 1984 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
tonyp@11455 1985 }
tonyp@11455 1986 }
ysr@1374 1987 }
tonyp@11455 1988 #endif // PRODUCT
tschatzl@49607 1989
tschatzl@49607 1990 void G1ConcurrentMark::rebuild_rem_set_concurrently() {
tschatzl@53985 1991 _g1h->rem_set()->rebuild_rem_set(this, _concurrent_workers, _worker_id_offset);
johnc@11583 1992 }
johnc@11583 1993
ehelin@35943 1994 void G1ConcurrentMark::print_stats() {
brutisso@35061 1995 if (!log_is_enabled(Debug, gc, stats)) {
brutisso@35061 1996 return;
brutisso@35061 1997 }
brutisso@35061 1998 log_debug(gc, stats)("---------------------------------------------------------------------");
tschatzl@47696 1999 for (size_t i = 0; i < _num_active_tasks; ++i) {
brutisso@35061 2000 _tasks[i]->print_stats();
brutisso@35061 2001 log_debug(gc, stats)("---------------------------------------------------------------------");
ysr@1374 2002 }
ysr@1374 2003 }
ysr@1374 2004
tschatzl@49644 2005 void G1ConcurrentMark::concurrent_cycle_abort() {
tschatzl@47678 2006 if (!cm_thread()->during_cycle() || _has_aborted) {
brutisso@31397 2007 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
brutisso@31397 2008 return;
brutisso@31397 2009 }
brutisso@31397 2010
tschatzl@25728 2011 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
tschatzl@25728 2012 // concurrent bitmap clearing.
tschatzl@37414 2013 {
tschatzl@49807 2014 GCTraceTime(Debug, gc) debug("Clear Next Bitmap");
tschatzl@47678 2015 clear_bitmap(_next_mark_bitmap, _g1h->workers(), false);
tschatzl@37414 2016 }
brutisso@24244 2017 // Note we cannot clear the previous marking bitmap here
brutisso@24244 2018 // since VerifyDuringGC verifies the objects marked during
brutisso@24244 2019 // a full GC against the previous bitmap.
brutisso@24244 2020
ysr@1374 2021 // Empty mark stack
tschatzl@49644 2022 reset_marking_for_restart();
tschatzl@47696 2023 for (uint i = 0; i < _max_num_tasks; ++i) {
ysr@1374 2024 _tasks[i]->clear_region_fields();
johnc@6761 2025 }
pliden@24468 2026 _first_overflow_barrier_sync.abort();
pliden@24468 2027 _second_overflow_barrier_sync.abort();
pliden@24848 2028 _has_aborted = true;
ysr@1374 2029
pliden@49751 2030 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
ysr@1374 2031 satb_mq_set.abandon_partial_marking();
tonyp@5082 2032 // This can be called either during or outside marking, we'll read
tonyp@5082 2033 // the expected_active value from the SATB queue set.
tonyp@5082 2034 satb_mq_set.set_active_all_threads(
tonyp@5082 2035 false, /* new active value */
tonyp@5082 2036 satb_mq_set.is_active() /* expected_active */);
ysr@1374 2037 }
ysr@1374 2038
ysr@1374 2039 static void print_ms_time_info(const char* prefix, const char* name,
ysr@1374 2040 NumberSeq& ns) {
brutisso@35061 2041 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
ysr@1374 2042 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
ysr@1374 2043 if (ns.num() > 0) {
brutisso@35061 2044 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
ysr@1374 2045 prefix, ns.sd(), ns.maximum());
ysr@1374 2046 }
ysr@1374 2047 }
ysr@1374 2048
ehelin@35943 2049 void G1ConcurrentMark::print_summary_info() {
stefank@37242 2050 Log(gc, marking) log;
brutisso@35061 2051 if (!log.is_trace()) {
brutisso@35061 2052 return;
brutisso@35061 2053 }
brutisso@35061 2054
brutisso@35061 2055 log.trace(" Concurrent marking:");
ysr@1374 2056 print_ms_time_info(" ", "init marks", _init_times);
ysr@1374 2057 print_ms_time_info(" ", "remarks", _remark_times);
ysr@1374 2058 {
ysr@1374 2059 print_ms_time_info(" ", "final marks", _remark_mark_times);
ysr@1374 2060 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
ysr@1374 2061
ysr@1374 2062 }
ysr@1374 2063 print_ms_time_info(" ", "cleanups", _cleanup_times);
tschatzl@37413 2064 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
tschatzl@49644 2065 _total_cleanup_time, (_cleanup_times.num() > 0 ? _total_cleanup_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
brutisso@35061 2066 log.trace(" Total stop_world time = %8.2f s.",
brutisso@35061 2067 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
brutisso@35061 2068 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
tschatzl@47678 2069 cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
ysr@1374 2070 }
ysr@1374 2071
ehelin@35943 2072 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
tschatzl@47696 2073 _concurrent_workers->print_worker_threads_on(st);
tonyp@4022 2074 }
tonyp@4022 2075
drwhite@37474 2076 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
tschatzl@47696 2077 _concurrent_workers->threads_do(tc);
drwhite@37474 2078 }
drwhite@37474 2079
ehelin@35943 2080 void G1ConcurrentMark::print_on_error(outputStream* st) const {
stefank@16685 2081 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
tschatzl@47678 2082 p2i(_prev_mark_bitmap), p2i(_next_mark_bitmap));
tschatzl@47678 2083 _prev_mark_bitmap->print_on_error(st, " Prev Bits: ");
tschatzl@47678 2084 _next_mark_bitmap->print_on_error(st, " Next Bits: ");
stefank@16685 2085 }
stefank@16685 2086
kbarrett@33226 2087 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
kbarrett@36373 2088 ReferenceProcessor* result = g1h->ref_processor_cm();
kbarrett@36373 2089 assert(result != NULL, "CM reference processor should not be NULL");
kbarrett@33226 2090 return result;
kbarrett@33226 2091 }
kbarrett@33226 2092
tonyp@9994 2093 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
ehelin@35943 2094 G1CMTask* task)
stefank@50752 2095 : MetadataVisitingOopIterateClosure(get_cm_oop_closure_ref_processor(g1h)),
tschatzl@49644 2096 _g1h(g1h), _task(task)
kbarrett@33226 2097 { }
ysr@1374 2098
ehelin@35943 2099 void G1CMTask::setup_for_region(HeapRegion* hr) {
tonyp@4026 2100 assert(hr != NULL,
brutisso@24106 2101 "claim_region() should have filtered out NULL regions");
ysr@1374 2102 _curr_region = hr;
ysr@1374 2103 _finger = hr->bottom();
ysr@1374 2104 update_region_limit();
ysr@1374 2105 }
ysr@1374 2106
ehelin@35943 2107 void G1CMTask::update_region_limit() {
ysr@1374 2108 HeapRegion* hr = _curr_region;
ysr@1374 2109 HeapWord* bottom = hr->bottom();
ysr@1374 2110 HeapWord* limit = hr->next_top_at_mark_start();
ysr@1374 2111
ysr@1374 2112 if (limit == bottom) {
ysr@1374 2113 // The region was collected underneath our feet.
ysr@1374 2114 // We set the finger to bottom to ensure that the bitmap
ysr@1374 2115 // iteration that will follow this will not do anything.
ysr@1374 2116 // (this is not a condition that holds when we set the region up,
ysr@1374 2117 // as the region is not supposed to be empty in the first place)
ysr@1374 2118 _finger = bottom;
ysr@1374 2119 } else if (limit >= _region_limit) {
tonyp@4026 2120 assert(limit >= _finger, "peace of mind");
ysr@1374 2121 } else {
tonyp@4026 2122 assert(limit < _region_limit, "only way to get here");
ysr@1374 2123 // This can happen under some pretty unusual circumstances. An
ysr@1374 2124 // evacuation pause empties the region underneath our feet (NTAMS
ysr@1374 2125 // at bottom). We then do some allocation in the region (NTAMS
ysr@1374 2126 // stays at bottom), followed by the region being used as a GC
ysr@1374 2127 // alloc region (NTAMS will move to top() and the objects
ysr@1374 2128 // originally below it will be grayed). All objects now marked in
ysr@1374 2129 // the region are explicitly grayed, if below the global finger,
ysr@1374 2130 // and we do not need in fact to scan anything else. So, we simply
ysr@1374 2131 // set _finger to be limit to ensure that the bitmap iteration
ysr@1374 2132 // doesn't do anything.
ysr@1374 2133 _finger = limit;
ysr@1374 2134 }
ysr@1374 2135
ysr@1374 2136 _region_limit = limit;
ysr@1374 2137 }
ysr@1374 2138
ehelin@35943 2139 void G1CMTask::giveup_current_region() {
tonyp@4026 2140 assert(_curr_region != NULL, "invariant");
ysr@1374 2141 clear_region_fields();
ysr@1374 2142 }
ysr@1374 2143
ehelin@35943 2144 void G1CMTask::clear_region_fields() {
ysr@1374 2145 // Values for these three fields that indicate that we're not
ysr@1374 2146 // holding on to a region.
ysr@1374 2147 _curr_region = NULL;
ysr@1374 2148 _finger = NULL;
ysr@1374 2149 _region_limit = NULL;
ysr@1374 2150 }
ysr@1374 2151
ehelin@35943 2152 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
tonyp@9994 2153 if (cm_oop_closure == NULL) {
tonyp@9994 2154 assert(_cm_oop_closure != NULL, "invariant");
tonyp@9994 2155 } else {
tonyp@9994 2156 assert(_cm_oop_closure == NULL, "invariant");
tonyp@9994 2157 }
tonyp@9994 2158 _cm_oop_closure = cm_oop_closure;
tonyp@9994 2159 }
tonyp@9994 2160
tschatzl@47678 2161 void G1CMTask::reset(G1CMBitMap* next_mark_bitmap) {
tschatzl@47678 2162 guarantee(next_mark_bitmap != NULL, "invariant");
tschatzl@47678 2163 _next_mark_bitmap = next_mark_bitmap;
ysr@1374 2164 clear_region_fields();
ysr@1374 2165
ysr@1374 2166 _calls = 0;
ysr@1374 2167 _elapsed_time_ms = 0.0;
ysr@1374 2168 _termination_time_ms = 0.0;
ysr@1374 2169 _termination_start_time_ms = 0.0;
tschatzl@49606 2170
tschatzl@49606 2171 _mark_stats_cache.reset();
ysr@1374 2172 }
ysr@1374 2173
ehelin@35943 2174 bool G1CMTask::should_exit_termination() {
zgu@53348 2175 if (!regular_clock_call()) {
zgu@53348 2176 return true;
zgu@53348 2177 }
zgu@53348 2178
ysr@1374 2179 // This is called when we are in the termination protocol. We should
ysr@1374 2180 // quit if, for some reason, this task wants to abort or the global
ysr@1374 2181 // stack is not empty (this means that we can get work from it).
ysr@1374 2182 return !_cm->mark_stack_empty() || has_aborted();
ysr@1374 2183 }
ysr@1374 2184
ehelin@35943 2185 void G1CMTask::reached_limit() {
tonyp@4026 2186 assert(_words_scanned >= _words_scanned_limit ||
tonyp@4026 2187 _refs_reached >= _refs_reached_limit ,
tonyp@4026 2188 "shouldn't have been called otherwise");
zgu@53348 2189 abort_marking_if_regular_check_fail();
ysr@1374 2190 }
ysr@1374 2191
zgu@53348 2192 bool G1CMTask::regular_clock_call() {
tschatzl@49644 2193 if (has_aborted()) {
zgu@53348 2194 return false;
tschatzl@49644 2195 }
ysr@1374 2196
ysr@1374 2197 // First, we need to recalculate the words scanned and refs reached
ysr@1374 2198 // limits for the next clock call.
ysr@1374 2199 recalculate_limits();
ysr@1374 2200
ysr@1374 2201 // During the regular clock call we do the following
ysr@1374 2202
ysr@1374 2203 // (1) If an overflow has been flagged, then we abort.
ysr@1374 2204 if (_cm->has_overflown()) {
zgu@53348 2205 return false;
ysr@1374 2206 }
ysr@1374 2207
ysr@1374 2208 // If we are not concurrent (i.e. we're doing remark) we don't need
ysr@1374 2209 // to check anything else. The other steps are only needed during
ysr@1374 2210 // the concurrent marking phase.
tschatzl@49665 2211 if (!_cm->concurrent()) {
zgu@53348 2212 return true;
tschatzl@47678 2213 }
ysr@1374 2214
ysr@1374 2215 // (2) If marking has been aborted for Full GC, then we also abort.
ysr@1374 2216 if (_cm->has_aborted()) {
zgu@53348 2217 return false;
ysr@1374 2218 }
ysr@1374 2219
ysr@1374 2220 double curr_time_ms = os::elapsedVTime() * 1000.0;
ysr@1374 2221
ysr@1374 2222 // (4) We check whether we should yield. If we have to, then we abort.
pliden@24094 2223 if (SuspendibleThreadSet::should_yield()) {
ysr@1374 2224 // We should yield. To do this we abort the task. The caller is
ysr@1374 2225 // responsible for yielding.
zgu@53348 2226 return false;
ysr@1374 2227 }
ysr@1374 2228
ysr@1374 2229 // (5) We check whether we've reached our time quota. If we have,
ysr@1374 2230 // then we abort.
ysr@1374 2231 double elapsed_time_ms = curr_time_ms - _start_time_ms;
ysr@1374 2232 if (elapsed_time_ms > _time_target_ms) {
johnc@8073 2233 _has_timed_out = true;
zgu@53348 2234 return false;
ysr@1374 2235 }
ysr@1374 2236
ysr@1374 2237 // (6) Finally, we check whether there are enough completed STAB
ysr@1374 2238 // buffers available for processing. If there are, we abort.
pliden@49751 2239 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
ysr@1374 2240 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
ysr@1374 2241 // we do need to process SATB buffers, we'll abort and restart
ysr@1374 2242 // the marking task to do so
zgu@53348 2243 return false;
ysr@1374 2244 }
zgu@53348 2245 return true;
ysr@1374 2246 }
ysr@1374 2247
ehelin@35943 2248 void G1CMTask::recalculate_limits() {
ysr@1374 2249 _real_words_scanned_limit = _words_scanned + words_scanned_period;
ysr@1374 2250 _words_scanned_limit = _real_words_scanned_limit;
ysr@1374 2251
ysr@1374 2252 _real_refs_reached_limit = _refs_reached + refs_reached_period;
ysr@1374 2253 _refs_reached_limit = _real_refs_reached_limit;
ysr@1374 2254 }
ysr@1374 2255
ehelin@35943 2256 void G1CMTask::decrease_limits() {
ysr@1374 2257 // This is called when we believe that we're going to do an infrequent
ysr@1374 2258 // operation which will increase the per byte scanned cost (i.e. move
ysr@1374 2259 // entries to/from the global stack). It basically tries to decrease the
ysr@1374 2260 // scanning limit so that the clock is called earlier.
ysr@1374 2261
tschatzl@47678 2262 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4;
tschatzl@47678 2263 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4;
ysr@1374 2264 }
ysr@1374 2265
ehelin@35943 2266 void G1CMTask::move_entries_to_global_stack() {
tschatzl@41176 2267 // Local array where we'll store the entries that will be popped
tschatzl@41176 2268 // from the local queue.
tschatzl@46328 2269 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
tschatzl@41176 2270
tschatzl@41176 2271 size_t n = 0;
tschatzl@46328 2272 G1TaskQueueEntry task_entry;
tschatzl@46328 2273 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) {
tschatzl@46328 2274 buffer[n] = task_entry;
ysr@1374 2275 ++n;
ysr@1374 2276 }
tschatzl@46328 2277 if (n < G1CMMarkStack::EntriesPerChunk) {
tschatzl@46328 2278 buffer[n] = G1TaskQueueEntry();
tschatzl@41176 2279 }
ysr@1374 2280
ysr@1374 2281 if (n > 0) {
tschatzl@41176 2282 if (!_cm->mark_stack_push(buffer)) {
ysr@1374 2283 set_has_aborted();
ysr@1374 2284 }
ysr@1374 2285 }
ysr@1374 2286
tschatzl@41176 2287 // This operation was quite expensive, so decrease the limits.
ysr@1374 2288 decrease_limits();
ysr@1374 2289 }
ysr@1374 2290
tschatzl@41176 2291 bool G1CMTask::get_entries_from_global_stack() {
tschatzl@41176 2292 // Local array where we'll store the entries that will be popped
ysr@1374 2293 // from the global stack.
tschatzl@46328 2294 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
tschatzl@41176 2295
tschatzl@41176 2296 if (!_cm->mark_stack_pop(buffer)) {
tschatzl@41176 2297 return false;
tschatzl@41176 2298 }
tschatzl@41176 2299
tschatzl@41176 2300 // We did actually pop at least one entry.
tschatzl@46328 2301 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) {
tschatzl@46328 2302 G1TaskQueueEntry task_entry = buffer[i];
tschatzl@46328 2303 if (task_entry.is_null()) {
tschatzl@41176 2304 break;
ysr@1374 2305 }
coleenp@46968 2306 assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj()));
tschatzl@46328 2307 bool success = _task_queue->push(task_entry);
tschatzl@41176 2308 // We only call this when the local queue is empty or under a
tschatzl@41176 2309 // given target limit. So, we do not expect this push to fail.
tschatzl@41176 2310 assert(success, "invariant");
ysr@1374 2311 }
ysr@1374 2312
tschatzl@41176 2313 // This operation was quite expensive, so decrease the limits
ysr@1374 2314 decrease_limits();
tschatzl@41176 2315 return true;
ysr@1374 2316 }
ysr@1374 2317
ehelin@35943 2318 void G1CMTask::drain_local_queue(bool partially) {
tschatzl@42597 2319 if (has_aborted()) {
tschatzl@42597 2320 return;
tschatzl@42597 2321 }
ysr@1374 2322
ysr@1374 2323 // Decide what the target size is, depending whether we're going to
ysr@1374 2324 // drain it partially (so that other tasks can steal if they run out
ysr@1374 2325 // of things to do) or totally (at the very end).
ysr@1374 2326 size_t target_size;
tonyp@9999 2327 if (partially) {
chrisphi@50676 2328 target_size = MIN2((size_t)_task_queue->max_elems()/3, (size_t)GCDrainStackTargetSize);
tonyp@9999 2329 } else {
ysr@1374 2330 target_size = 0;
tonyp@9999 2331 }
ysr@1374 2332
ysr@1374 2333 if (_task_queue->size() > target_size) {
tschatzl@46328 2334 G1TaskQueueEntry entry;
tschatzl@46328 2335 bool ret = _task_queue->pop_local(entry);
ysr@1374 2336 while (ret) {
tschatzl@46328 2337 scan_task_entry(entry);
tonyp@9999 2338 if (_task_queue->size() <= target_size || has_aborted()) {
ysr@1374 2339 ret = false;
tonyp@9999 2340 } else {
tschatzl@46328 2341 ret = _task_queue->pop_local(entry);
tonyp@9999 2342 }
ysr@1374 2343 }
ysr@1374 2344 }
ysr@1374 2345 }
ysr@1374 2346
ehelin@35943 2347 void G1CMTask::drain_global_stack(bool partially) {
tschatzl@49644 2348 if (has_aborted()) {
tschatzl@49644 2349 return;
tschatzl@49644 2350 }
ysr@1374 2351
ysr@1374 2352 // We have a policy to drain the local queue before we attempt to
ysr@1374 2353 // drain the global stack.
tonyp@4026 2354 assert(partially || _task_queue->size() == 0, "invariant");
ysr@1374 2355
ysr@1374 2356 // Decide what the target size is, depending whether we're going to
ysr@1374 2357 // drain it partially (so that other tasks can steal if they run out
tschatzl@41176 2358 // of things to do) or totally (at the very end).
tschatzl@41176 2359 // Notice that when draining the global mark stack partially, due to the racyness
tschatzl@41176 2360 // of the mark stack size update we might in fact drop below the target. But,
tschatzl@41176 2361 // this is not a problem.
tschatzl@41176 2362 // In case of total draining, we simply process until the global mark stack is
tschatzl@41176 2363 // totally empty, disregarding the size counter.
tonyp@9999 2364 if (partially) {
tschatzl@41176 2365 size_t const target_size = _cm->partial_mark_stack_size_target();
tschatzl@41176 2366 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
tschatzl@41176 2367 if (get_entries_from_global_stack()) {
tschatzl@41176 2368 drain_local_queue(partially);
tschatzl@41176 2369 }
tschatzl@41176 2370 }
tonyp@9999 2371 } else {
tschatzl@41176 2372 while (!has_aborted() && get_entries_from_global_stack()) {
ysr@1374 2373 drain_local_queue(partially);
ysr@1374 2374 }
ysr@1374 2375 }
ysr@1374 2376 }
ysr@1374 2377
ysr@1374 2378 // SATB Queue has several assumptions on whether to call the par or
ysr@1374 2379 // non-par versions of the methods. this is why some of the code is
ysr@1374 2380 // replicated. We should really get rid of the single-threaded version
ysr@1374 2381 // of the code to simplify things.
ehelin@35943 2382 void G1CMTask::drain_satb_buffers() {
tschatzl@49644 2383 if (has_aborted()) {
tschatzl@49644 2384 return;
tschatzl@49644 2385 }
ysr@1374 2386
ysr@1374 2387 // We set this so that the regular clock knows that we're in the
ysr@1374 2388 // middle of draining buffers and doesn't set the abort flag when it
ysr@1374 2389 // notices that SATB buffers are available for draining. It'd be
ysr@1374 2390 // very counter productive if it did that. :-)
ysr@1374 2391 _draining_satb_buffers = true;
ysr@1374 2392
ehelin@35943 2393 G1CMSATBBufferClosure satb_cl(this, _g1h);
pliden@49751 2394 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
ysr@1374 2395
ysr@1374 2396 // This keeps claiming and applying the closure to completed buffers
ysr@1374 2397 // until we run out of buffers or we need to abort.
mlarsson@27251 2398 while (!has_aborted() &&
kbarrett@30577 2399 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
zgu@53348 2400 abort_marking_if_regular_check_fail();
ysr@1374 2401 }
ysr@1374 2402
ysr@1374 2403 _draining_satb_buffers = false;
ysr@1374 2404
tonyp@4026 2405 assert(has_aborted() ||
tschatzl@49665 2406 _cm->concurrent() ||
tonyp@4026 2407 satb_mq_set.completed_buffers_num() == 0, "invariant");
ysr@1374 2408
ysr@1374 2409 // again, this was a potentially expensive operation, decrease the
ysr@1374 2410 // limits to get the regular clock call early
ysr@1374 2411 decrease_limits();
ysr@1374 2412 }
ysr@1374 2413
tschatzl@49606 2414 void G1CMTask::clear_mark_stats_cache(uint region_idx) {
tschatzl@49606 2415 _mark_stats_cache.reset(region_idx);
tschatzl@49606 2416 }
tschatzl@49606 2417
tschatzl@49606 2418 Pair<size_t, size_t> G1CMTask::flush_mark_stats_cache() {
tschatzl@49606 2419 return _mark_stats_cache.evict_all();
tschatzl@49606 2420 }
tschatzl@49606 2421
ehelin@35943 2422 void G1CMTask::print_stats() {
tschatzl@49606 2423 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u", _worker_id, _calls);
brutisso@35061 2424 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
brutisso@35061 2425 _elapsed_time_ms, _termination_time_ms);
tschatzl@49606 2426 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms max = %1.2lfms, total = %1.2lfms",
tschatzl@49606 2427 _step_times_ms.num(),
tschatzl@49606 2428 _step_times_ms.avg(),
tschatzl@49606 2429 _step_times_ms.sd(),
tschatzl@49606 2430 _step_times_ms.maximum(),
tschatzl@49606 2431 _step_times_ms.sum());
tschatzl@49606 2432 size_t const hits = _mark_stats_cache.hits();
tschatzl@49606 2433 size_t const misses = _mark_stats_cache.misses();
tschatzl@49606 2434 log_debug(gc, stats)(" Mark Stats Cache: hits " SIZE_FORMAT " misses " SIZE_FORMAT " ratio %.3f",
tschatzl@49606 2435 hits, misses, percent_of(hits, hits + misses));
ysr@1374 2436 }
ysr@1374 2437
zgu@51292 2438 bool G1ConcurrentMark::try_stealing(uint worker_id, G1TaskQueueEntry& task_entry) {
zgu@51292 2439 return _task_queues->steal(worker_id, task_entry);
stefank@30566 2440 }
stefank@30566 2441
ysr@1374 2442 /*****************************************************************************
ysr@1374 2443
johnc@16388 2444 The do_marking_step(time_target_ms, ...) method is the building
johnc@16388 2445 block of the parallel marking framework. It can be called in parallel
ysr@1374 2446 with other invocations of do_marking_step() on different tasks
ysr@1374 2447 (but only one per task, obviously) and concurrently with the
ysr@1374 2448 mutator threads, or during remark, hence it eliminates the need
ysr@1374 2449 for two versions of the code. When called during remark, it will
ysr@1374 2450 pick up from where the task left off during the concurrent marking
ysr@1374 2451 phase. Interestingly, tasks are also claimable during evacuation
ysr@1374 2452 pauses too, since do_marking_step() ensures that it aborts before
ysr@1374 2453 it needs to yield.
ysr@1374 2454
johnc@16388 2455 The data structures that it uses to do marking work are the
ysr@1374 2456 following:
ysr@1374 2457
ysr@1374 2458 (1) Marking Bitmap. If there are gray objects that appear only
ysr@1374 2459 on the bitmap (this happens either when dealing with an overflow
ysr@1374 2460 or when the initial marking phase has simply marked the roots
ysr@1374 2461 and didn't push them on the stack), then tasks claim heap
ysr@1374 2462 regions whose bitmap they then scan to find gray objects. A
ysr@1374 2463 global finger indicates where the end of the last claimed region
ysr@1374 2464 is. A local finger indicates how far into the region a task has
ysr@1374 2465 scanned. The two fingers are used to determine how to gray an
ysr@1374 2466 object (i.e. whether simply marking it is OK, as it will be
ysr@1374 2467 visited by a task in the future, or whether it needs to be also
ysr@1374 2468 pushed on a stack).
ysr@1374 2469
ysr@1374 2470 (2) Local Queue. The local queue of the task which is accessed
ysr@1374 2471 reasonably efficiently by the task. Other tasks can steal from
ysr@1374 2472 it when they run out of work. Throughout the marking phase, a
ysr@1374 2473 task attempts to keep its local queue short but not totally
ysr@1374 2474 empty, so that entries are available for stealing by other
ysr@1374 2475 tasks. Only when there is no more work, a task will totally
ysr@1374 2476 drain its local queue.
ysr@1374 2477
ysr@1374 2478 (3) Global Mark Stack. This handles local queue overflow. During
ysr@1374 2479 marking only sets of entries are moved between it and the local
ysr@1374 2480 queues, as access to it requires a mutex and more fine-grain
ysr@1374 2481 interaction with it which might cause contention. If it
ysr@1374 2482 overflows, then the marking phase should restart and iterate
ysr@1374 2483 over the bitmap to identify gray objects. Throughout the marking
ysr@1374 2484 phase, tasks attempt to keep the global mark stack at a small
ysr@1374 2485 length but not totally empty, so that entries are available for
ysr@1374 2486 popping by other tasks. Only when there is no more work, tasks
ysr@1374 2487 will totally drain the global mark stack.
ysr@1374 2488
tonyp@12272 2489 (4) SATB Buffer Queue. This is where completed SATB buffers are
ysr@1374 2490 made available. Buffers are regularly removed from this queue
ysr@1374 2491 and scanned for roots, so that the queue doesn't get too
ysr@1374 2492 long. During remark, all completed buffers are processed, as
ysr@1374 2493 well as the filled in parts of any uncompleted buffers.
ysr@1374 2494
ysr@1374 2495 The do_marking_step() method tries to abort when the time target
ysr@1374 2496 has been reached. There are a few other cases when the
ysr@1374 2497 do_marking_step() method also aborts:
ysr@1374 2498
ysr@1374 2499 (1) When the marking phase has been aborted (after a Full GC).
ysr@1374 2500
tonyp@12272 2501 (2) When a global overflow (on the global stack) has been
tonyp@12272 2502 triggered. Before the task aborts, it will actually sync up with
tonyp@12272 2503 the other tasks to ensure that all the marking data structures
johnc@16389 2504 (local queues, stacks, fingers etc.) are re-initialized so that
tonyp@12272 2505 when do_marking_step() completes, the marking phase can
tonyp@12272 2506 immediately restart.
ysr@1374 2507
ysr@1374 2508 (3) When enough completed SATB buffers are available. The
ysr@1374 2509 do_marking_step() method only tries to drain SATB buffers right
ysr@1374 2510 at the beginning. So, if enough buffers are available, the
ysr@1374 2511 marking step aborts and the SATB buffers are processed at
ysr@1374 2512 the beginning of the next invocation.
ysr@1374 2513
ysr@1374 2514 (4) To yield. when we have to yield then we abort and yield
ysr@1374 2515 right at the end of do_marking_step(). This saves us from a lot
ysr@1374 2516 of hassle as, by yielding we might allow a Full GC. If this
ysr@1374 2517 happens then objects will be compacted underneath our feet, the
ysr@1374 2518 heap might shrink, etc. We save checking for this by just
ysr@1374 2519 aborting and doing the yield right at the end.
ysr@1374 2520
ysr@1374 2521 From the above it follows that the do_marking_step() method should
ysr@1374 2522 be called in a loop (or, otherwise, regularly) until it completes.
ysr@1374 2523
ysr@1374 2524 If a marking step completes without its has_aborted() flag being
ysr@1374 2525 true, it means it has completed the current marking phase (and
ysr@1374 2526 also all other marking tasks have done so and have all synced up).
ysr@1374 2527
ysr@1374 2528 A method called regular_clock_call() is invoked "regularly" (in
ysr@1374 2529 sub ms intervals) throughout marking. It is this clock method that
ysr@1374 2530 checks all the abort conditions which were mentioned above and
ysr@1374 2531 decides when the task should abort. A work-based scheme is used to
ysr@1374 2532 trigger this clock method: when the number of object words the
ysr@1374 2533 marking phase has scanned or the number of references the marking
ysr@1374 2534 phase has visited reach a given limit. Additional invocations to
ysr@1374 2535 the method clock have been planted in a few other strategic places
ysr@1374 2536 too. The initial reason for the clock method was to avoid calling
ysr@1374 2537 vtime too regularly, as it is quite expensive. So, once it was in
ysr@1374 2538 place, it was natural to piggy-back all the other conditions on it
ysr@1374 2539 too and not constantly check them throughout the code.
ysr@1374 2540
johnc@16388 2541 If do_termination is true then do_marking_step will enter its
johnc@16388 2542 termination protocol.
johnc@16388 2543
johnc@16388 2544 The value of is_serial must be true when do_marking_step is being
johnc@16388 2545 called serially (i.e. by the VMThread) and do_marking_step should
johnc@16388 2546 skip any synchronization in the termination and overflow code.
johnc@16388 2547 Examples include the serial remark code and the serial reference
johnc@16388 2548 processing closures.
johnc@16388 2549
johnc@16388 2550 The value of is_serial must be false when do_marking_step is
johnc@16388 2551 being called by any of the worker threads in a work gang.
johnc@16388 2552 Examples include the concurrent marking code (CMMarkingTask),
johnc@16388 2553 the MT remark code, and the MT reference processing closures.
johnc@16388 2554
ysr@1374 2555 *****************************************************************************/
ysr@1374 2556
ehelin@35943 2557 void G1CMTask::do_marking_step(double time_target_ms,
ehelin@35943 2558 bool do_termination,
ehelin@35943 2559 bool is_serial) {
tonyp@4026 2560 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
ysr@1374 2561
ysr@1374 2562 _start_time_ms = os::elapsedVTime() * 1000.0;
ysr@1374 2563
johnc@16388 2564 // If do_stealing is true then do_marking_step will attempt to
ehelin@35943 2565 // steal work from the other G1CMTasks. It only makes sense to
johnc@16388 2566 // enable stealing when the termination protocol is enabled
johnc@16388 2567 // and do_marking_step() is not being called serially.
johnc@16388 2568 bool do_stealing = do_termination && !is_serial;
johnc@16388 2569
tschatzl@53985 2570 double diff_prediction_ms = _g1h->policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
ysr@1374 2571 _time_target_ms = time_target_ms - diff_prediction_ms;
ysr@1374 2572
ysr@1374 2573 // set up the variables that are used in the work-based scheme to
ysr@1374 2574 // call the regular clock method
ysr@1374 2575 _words_scanned = 0;
ysr@1374 2576 _refs_reached = 0;
ysr@1374 2577 recalculate_limits();
ysr@1374 2578
ysr@1374 2579 // clear all flags
ysr@1374 2580 clear_has_aborted();
johnc@8073 2581 _has_timed_out = false;
ysr@1374 2582 _draining_satb_buffers = false;
ysr@1374 2583
ysr@1374 2584 ++_calls;
ysr@1374 2585
ysr@1374 2586 // Set up the bitmap and oop closures. Anything that uses them is
ysr@1374 2587 // eventually called from this method, so it is OK to allocate these
ysr@1374 2588 // statically.
tschatzl@46750 2589 G1CMBitMapClosure bitmap_closure(this, _cm);
tschatzl@49644 2590 G1CMOopClosure cm_oop_closure(_g1h, this);
tonyp@9994 2591 set_cm_oop_closure(&cm_oop_closure);
ysr@1374 2592
ysr@1374 2593 if (_cm->has_overflown()) {
tonyp@12272 2594 // This can happen if the mark stack overflows during a GC pause
tonyp@12272 2595 // and this task, after a yield point, restarts. We have to abort
tonyp@12272 2596 // as we need to get into the overflow protocol which happens
tonyp@12272 2597 // right at the end of this task.
ysr@1374 2598 set_has_aborted();
ysr@1374 2599 }
ysr@1374 2600
ysr@1374 2601 // First drain any available SATB buffers. After this, we will not
ysr@1374 2602 // look at SATB buffers before the next invocation of this method.
ysr@1374 2603 // If enough completed SATB buffers are queued up, the regular clock
ysr@1374 2604 // will abort this task so that it restarts.
ysr@1374 2605 drain_satb_buffers();
ysr@1374 2606 // ...then partially drain the local queue and the global stack
ysr@1374 2607 drain_local_queue(true);
ysr@1374 2608 drain_global_stack(true);
ysr@1374 2609
ysr@1374 2610 do {
ysr@1374 2611 if (!has_aborted() && _curr_region != NULL) {
ysr@1374 2612 // This means that we're already holding on to a region.
tonyp@4026 2613 assert(_finger != NULL, "if region is not NULL, then the finger "
tonyp@4026 2614 "should not be NULL either");
ysr@1374 2615
ysr@1374 2616 // We might have restarted this task after an evacuation pause
ysr@1374 2617 // which might have evacuated the region we're holding on to
ysr@1374 2618 // underneath our feet. Let's read its limit again to make sure
ysr@1374 2619 // that we do not iterate over a region of the heap that
ysr@1374 2620 // contains garbage (update_region_limit() will also move
ysr@1374 2621 // _finger to the start of the region if it is found empty).
ysr@1374 2622 update_region_limit();
ysr@1374 2623 // We will start from _finger not from the start of the region,
ysr@1374 2624 // as we might be restarting this task after aborting half-way
ysr@1374 2625 // through scanning this region. In this case, _finger points to
ysr@1374 2626 // the address where we last found a marked object. If this is a
ysr@1374 2627 // fresh region, _finger points to start().
ysr@1374 2628 MemRegion mr = MemRegion(_finger, _region_limit);
ysr@1374 2629
tonyp@26846 2630 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
johnc@15609 2631 "humongous regions should go around loop once only");
johnc@15609 2632
johnc@15609 2633 // Some special cases:
johnc@15609 2634 // If the memory region is empty, we can just give up the region.
johnc@15609 2635 // If the current region is humongous then we only need to check
johnc@15609 2636 // the bitmap for the bit associated with the start of the object,
johnc@15609 2637 // scan the object if it's live, and give up the region.
johnc@15609 2638 // Otherwise, let's iterate over the bitmap of the part of the region
johnc@15609 2639 // that is left.
johnc@15604 2640 // If the iteration is successful, give up the region.
johnc@15609 2641 if (mr.is_empty()) {
johnc@15609 2642 giveup_current_region();
zgu@53348 2643 abort_marking_if_regular_check_fail();
tonyp@26846 2644 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
tschatzl@47678 2645 if (_next_mark_bitmap->is_marked(mr.start())) {
johnc@15609 2646 // The object is marked - apply the closure
tschatzl@46750 2647 bitmap_closure.do_addr(mr.start());
johnc@15609 2648 }
johnc@15609 2649 // Even if this task aborted while scanning the humongous object
johnc@15609 2650 // we can (and should) give up the current region.
johnc@15609 2651 giveup_current_region();
zgu@53348 2652 abort_marking_if_regular_check_fail();
tschatzl@47678 2653 } else if (_next_mark_bitmap->iterate(&bitmap_closure, mr)) {
ysr@1374 2654 giveup_current_region();
zgu@53348 2655 abort_marking_if_regular_check_fail();
ysr@1374 2656 } else {
tonyp@4026 2657 assert(has_aborted(), "currently the only way to do so");
ysr@1374 2658 // The only way to abort the bitmap iteration is to return
ysr@1374 2659 // false from the do_bit() method. However, inside the
ysr@1374 2660 // do_bit() method we move the _finger to point to the
ysr@1374 2661 // object currently being looked at. So, if we bail out, we
ysr@1374 2662 // have definitely set _finger to something non-null.
tonyp@4026 2663 assert(_finger != NULL, "invariant");
ysr@1374 2664
ysr@1374 2665 // Region iteration was actually aborted. So now _finger
ysr@1374 2666 // points to the address of the object we last scanned. If we
ysr@1374 2667 // leave it there, when we restart this task, we will rescan
ysr@1374 2668 // the object. It is easy to avoid this. We move the finger by
tschatzl@46750 2669 // enough to point to the next possible object header.
apetrusenko@5079 2670 assert(_finger < _region_limit, "invariant");
tschatzl@46750 2671 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
apetrusenko@5079 2672 // Check if bitmap iteration was aborted while scanning the last object
apetrusenko@5079 2673 if (new_finger >= _region_limit) {
tonyp@12272 2674 giveup_current_region();
apetrusenko@5079 2675 } else {
tonyp@12272 2676 move_finger_to(new_finger);
apetrusenko@5079 2677 }
ysr@1374 2678 }
ysr@1374 2679 }
ysr@1374 2680 // At this point we have either completed iterating over the
ysr@1374 2681 // region we were holding on to, or we have aborted.
ysr@1374 2682
ysr@1374 2683 // We then partially drain the local queue and the global stack.
ysr@1374 2684 // (Do we really need this?)
ysr@1374 2685 drain_local_queue(true);
ysr@1374 2686 drain_global_stack(true);
ysr@1374 2687
ysr@1374 2688 // Read the note on the claim_region() method on why it might
ysr@1374 2689 // return NULL with potentially more regions available for
ysr@1374 2690 // claiming and why we have to check out_of_regions() to determine
ysr@1374 2691 // whether we're done or not.
ysr@1374 2692 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
ysr@1374 2693 // We are going to try to claim a new region. We should have
ysr@1374 2694 // given up on the previous one.
tonyp@4026 2695 // Separated the asserts so that we know which one fires.
tonyp@4026 2696 assert(_curr_region == NULL, "invariant");
tonyp@4026 2697 assert(_finger == NULL, "invariant");
tonyp@4026 2698 assert(_region_limit == NULL, "invariant");
johnc@14073 2699 HeapRegion* claimed_region = _cm->claim_region(_worker_id);
ysr@1374 2700 if (claimed_region != NULL) {
ysr@1374 2701 // Yes, we managed to claim one
ysr@1374 2702 setup_for_region(claimed_region);
tonyp@4026 2703 assert(_curr_region == claimed_region, "invariant");
ysr@1374 2704 }
ysr@1374 2705 // It is important to call the regular clock here. It might take
ysr@1374 2706 // a while to claim a region if, for example, we hit a large
ysr@1374 2707 // block of empty regions. So we need to call the regular clock
ysr@1374 2708 // method once round the loop to make sure it's called
ysr@1374 2709 // frequently enough.
zgu@53348 2710 abort_marking_if_regular_check_fail();
ysr@1374 2711 }
ysr@1374 2712
ysr@1374 2713 if (!has_aborted() && _curr_region == NULL) {
tonyp@4026 2714 assert(_cm->out_of_regions(),
tonyp@4026 2715 "at this point we should be out of regions");
ysr@1374 2716 }
ysr@1374 2717 } while ( _curr_region != NULL && !has_aborted());
ysr@1374 2718
ysr@1374 2719 if (!has_aborted()) {
ysr@1374 2720 // We cannot check whether the global stack is empty, since other
tonyp@12272 2721 // tasks might be pushing objects to it concurrently.
tonyp@4026 2722 assert(_cm->out_of_regions(),
tonyp@4026 2723 "at this point we should be out of regions");
ysr@1374 2724 // Try to reduce the number of available SATB buffers so that
ysr@1374 2725 // remark has less work to do.
ysr@1374 2726 drain_satb_buffers();
ysr@1374 2727 }
ysr@1374 2728
ysr@1374 2729 // Since we've done everything else, we can now totally drain the
ysr@1374 2730 // local queue and global stack.
ysr@1374 2731 drain_local_queue(false);
ysr@1374 2732 drain_global_stack(false);
ysr@1374 2733
ysr@1374 2734 // Attempt at work stealing from other task's queues.
johnc@8073 2735 if (do_stealing && !has_aborted()) {
ysr@1374 2736 // We have not aborted. This means that we have finished all that
ysr@1374 2737 // we could. Let's try to do some stealing...
ysr@1374 2738
ysr@1374 2739 // We cannot check whether the global stack is empty, since other
tonyp@12272 2740 // tasks might be pushing objects to it concurrently.
tonyp@4026 2741 assert(_cm->out_of_regions() && _task_queue->size() == 0,
tonyp@4026 2742 "only way to reach here");
ysr@1374 2743 while (!has_aborted()) {
tschatzl@46328 2744 G1TaskQueueEntry entry;
zgu@51292 2745 if (_cm->try_stealing(_worker_id, entry)) {
tschatzl@46328 2746 scan_task_entry(entry);
ysr@1374 2747
ysr@1374 2748 // And since we're towards the end, let's totally drain the
ysr@1374 2749 // local queue and global stack.
ysr@1374 2750 drain_local_queue(false);
ysr@1374 2751 drain_global_stack(false);
ysr@1374 2752 } else {
ysr@1374 2753 break;
ysr@1374 2754 }
ysr@1374 2755 }
ysr@1374 2756 }
ysr@1374 2757
ysr@1374 2758 // We still haven't aborted. Now, let's try to get into the
ysr@1374 2759 // termination protocol.
johnc@8073 2760 if (do_termination && !has_aborted()) {
ysr@1374 2761 // We cannot check whether the global stack is empty, since other
tonyp@12272 2762 // tasks might be concurrently pushing objects on it.
tonyp@4026 2763 // Separated the asserts so that we know which one fires.
tonyp@4026 2764 assert(_cm->out_of_regions(), "only way to reach here");
tonyp@4026 2765 assert(_task_queue->size() == 0, "only way to reach here");
ysr@1374 2766 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
johnc@16388 2767
ehelin@35943 2768 // The G1CMTask class also extends the TerminatorTerminator class,
ysr@1374 2769 // hence its should_exit_termination() method will also decide
ysr@1374 2770 // whether to exit the termination protocol or not.
johnc@16388 2771 bool finished = (is_serial ||
johnc@16388 2772 _cm->terminator()->offer_termination(this));
ysr@1374 2773 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
ysr@1374 2774 _termination_time_ms +=
ysr@1374 2775 termination_end_time_ms - _termination_start_time_ms;
ysr@1374 2776
ysr@1374 2777 if (finished) {
ysr@1374 2778 // We're all done.
ysr@1374 2779
ysr@1374 2780 // We can now guarantee that the global stack is empty, since
tonyp@4026 2781 // all other tasks have finished. We separated the guarantees so
tonyp@4026 2782 // that, if a condition is false, we can immediately find out
tonyp@4026 2783 // which one.
tonyp@4026 2784 guarantee(_cm->out_of_regions(), "only way to reach here");
tonyp@4026 2785 guarantee(_cm->mark_stack_empty(), "only way to reach here");
tonyp@4026 2786 guarantee(_task_queue->size() == 0, "only way to reach here");
tonyp@4026 2787 guarantee(!_cm->has_overflown(), "only way to reach here");
zgu@53348 2788 guarantee(!has_aborted(), "should never happen if termination has completed");
ysr@1374 2789 } else {
ysr@1374 2790 // Apparently there's more work to do. Let's abort this task. It
ysr@1374 2791 // will restart it and we can hopefully find more things to do.
ysr@1374 2792 set_has_aborted();
ysr@1374 2793 }
ysr@1374 2794 }
ysr@1374 2795
ysr@1374 2796 // Mainly for debugging purposes to make sure that a pointer to the
ysr@1374 2797 // closure which was statically allocated in this frame doesn't
ysr@1374 2798 // escape it by accident.
tonyp@9994 2799 set_cm_oop_closure(NULL);
ysr@1374 2800 double end_time_ms = os::elapsedVTime() * 1000.0;
ysr@1374 2801 double elapsed_time_ms = end_time_ms - _start_time_ms;
ysr@1374 2802 // Update the step history.
ysr@1374 2803 _step_times_ms.add(elapsed_time_ms);
ysr@1374 2804
ysr@1374 2805 if (has_aborted()) {
ysr@1374 2806 // The task was aborted for some reason.
johnc@8073 2807 if (_has_timed_out) {
ysr@1374 2808 double diff_ms = elapsed_time_ms - _time_target_ms;
ysr@1374 2809 // Keep statistics of how well we did with respect to hitting
ysr@1374 2810 // our target only if we actually timed out (if we aborted for
ysr@1374 2811 // other reasons, then the results might get skewed).
ysr@1374 2812 _marking_step_diffs_ms.add(diff_ms);
ysr@1374 2813 }
ysr@1374 2814
ysr@1374 2815 if (_cm->has_overflown()) {
ysr@1374 2816 // This is the interesting one. We aborted because a global
ysr@1374 2817 // overflow was raised. This means we have to restart the
ysr@1374 2818 // marking phase and start iterating over regions. However, in
ysr@1374 2819 // order to do this we have to make sure that all tasks stop
jwilhelm@22551 2820 // what they are doing and re-initialize in a safe manner. We
ysr@1374 2821 // will achieve this with the use of two barrier sync points.
ysr@1374 2822
johnc@16388 2823 if (!is_serial) {
johnc@16388 2824 // We only need to enter the sync barrier if being called
johnc@16388 2825 // from a parallel context
johnc@16388 2826 _cm->enter_first_sync_barrier(_worker_id);
johnc@16388 2827
johnc@16388 2828 // When we exit this sync barrier we know that all tasks have
johnc@16388 2829 // stopped doing marking work. So, it's now safe to
tschatzl@49606 2830 // re-initialize our data structures.
johnc@16388 2831 }
ysr@1374 2832
ysr@1374 2833 clear_region_fields();
tschatzl@49606 2834 flush_mark_stats_cache();
ysr@1374 2835
johnc@16388 2836 if (!is_serial) {
tschatzl@49606 2837 // If we're executing the concurrent phase of marking, reset the marking
tschatzl@49606 2838 // state; otherwise the marking state is reset after reference processing,
tschatzl@49606 2839 // during the remark pause.
tschatzl@49606 2840 // If we reset here as a result of an overflow during the remark we will
tschatzl@49606 2841 // see assertion failures from any subsequent set_concurrency_and_phase()
tschatzl@49606 2842 // calls.
tschatzl@49606 2843 if (_cm->concurrent() && _worker_id == 0) {
tschatzl@49606 2844 // Worker 0 is responsible for clearing the global data structures because
tschatzl@49606 2845 // of an overflow. During STW we should not clear the overflow flag (in
tschatzl@49606 2846 // G1ConcurrentMark::reset_marking_state()) since we rely on it being true when we exit
tschatzl@49606 2847 // method to abort the pause and restart concurrent marking.
tschatzl@49644 2848 _cm->reset_marking_for_restart();
tschatzl@49606 2849
tschatzl@49606 2850 log_info(gc, marking)("Concurrent Mark reset for overflow");
tschatzl@49606 2851 }
tschatzl@49606 2852
johnc@16388 2853 // ...and enter the second barrier.
johnc@16388 2854 _cm->enter_second_sync_barrier(_worker_id);
johnc@16388 2855 }
johnc@16389 2856 // At this point, if we're during the concurrent phase of
johnc@16389 2857 // marking, everything has been re-initialized and we're
ysr@1374 2858 // ready to restart.
ysr@1374 2859 }
ysr@1374 2860 }
ysr@1374 2861 }
ysr@1374 2862
tschatzl@49606 2863 G1CMTask::G1CMTask(uint worker_id,
tschatzl@49606 2864 G1ConcurrentMark* cm,
tschatzl@49606 2865 G1CMTaskQueue* task_queue,
tschatzl@49606 2866 G1RegionMarkStats* mark_stats,
tschatzl@49606 2867 uint max_regions) :
tschatzl@47697 2868 _objArray_processor(this),
tschatzl@47697 2869 _worker_id(worker_id),
tschatzl@47697 2870 _g1h(G1CollectedHeap::heap()),
tschatzl@47697 2871 _cm(cm),
tschatzl@47697 2872 _next_mark_bitmap(NULL),
tschatzl@47697 2873 _task_queue(task_queue),
tschatzl@49606 2874 _mark_stats_cache(mark_stats, max_regions, RegionMarkStatsCacheSize),
tschatzl@47697 2875 _calls(0),
tschatzl@47697 2876 _time_target_ms(0.0),
tschatzl@47697 2877 _start_time_ms(0.0),
tschatzl@47697 2878 _cm_oop_closure(NULL),
tschatzl@47697 2879 _curr_region(NULL),
tschatzl@47697 2880 _finger(NULL),
tschatzl@47697 2881 _region_limit(NULL),
tschatzl@47697 2882 _words_scanned(0),
tschatzl@47697 2883 _words_scanned_limit(0),
tschatzl@47697 2884 _real_words_scanned_limit(0),
tschatzl@47697 2885 _refs_reached(0),
tschatzl@47697 2886 _refs_reached_limit(0),
tschatzl@47697 2887 _real_refs_reached_limit(0),
tschatzl@47697 2888 _has_aborted(false),
tschatzl@47697 2889 _has_timed_out(false),
tschatzl@47697 2890 _draining_satb_buffers(false),
tschatzl@47697 2891 _step_times_ms(),
tschatzl@47697 2892 _elapsed_time_ms(0.0),
tschatzl@47697 2893 _termination_time_ms(0.0),
tschatzl@47697 2894 _termination_start_time_ms(0.0),
tschatzl@47697 2895 _marking_step_diffs_ms()
tschatzl@47697 2896 {
tonyp@4026 2897 guarantee(task_queue != NULL, "invariant");
ysr@1374 2898
ysr@1374 2899 _marking_step_diffs_ms.add(0.5);
ysr@1374 2900 }
tonyp@8930 2901
tonyp@8930 2902 // These are formatting macros that are used below to ensure
tonyp@8930 2903 // consistent formatting. The *_H_* versions are used to format the
tonyp@8930 2904 // header for a particular value and they should be kept consistent
tonyp@8930 2905 // with the corresponding macro. Also note that most of the macros add
tonyp@8930 2906 // the necessary white space (as a prefix) which makes them a bit
tonyp@8930 2907 // easier to compose.
tonyp@8930 2908
tonyp@8930 2909 // All the output lines are prefixed with this string to be able to
tonyp@8930 2910 // identify them easily in a large log file.
tonyp@8930 2911 #define G1PPRL_LINE_PREFIX "###"
tonyp@8930 2912
bpittore@31592 2913 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
tonyp@8930 2914 #ifdef _LP64
tonyp@8930 2915 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
tonyp@8930 2916 #else // _LP64
tonyp@8930 2917 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
tonyp@8930 2918 #endif // _LP64
tonyp@8930 2919
tonyp@8930 2920 // For per-region info
tonyp@8930 2921 #define G1PPRL_TYPE_FORMAT " %-4s"
tonyp@8930 2922 #define G1PPRL_TYPE_H_FORMAT " %4s"
tschatzl@49672 2923 #define G1PPRL_STATE_FORMAT " %-5s"
tschatzl@49672 2924 #define G1PPRL_STATE_H_FORMAT " %5s"
bpittore@31592 2925 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
tonyp@8930 2926 #define G1PPRL_BYTE_H_FORMAT " %9s"
tonyp@8930 2927 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
tonyp@8930 2928 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
tonyp@8930 2929
tonyp@8930 2930 // For summary info
bpittore@31592 2931 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
bpittore@31592 2932 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
bpittore@31592 2933 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
bpittore@31592 2934 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
tonyp@8930 2935
tschatzl@47697 2936 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) :
tschatzl@47697 2937 _total_used_bytes(0), _total_capacity_bytes(0),
tschatzl@47697 2938 _total_prev_live_bytes(0), _total_next_live_bytes(0),
tschatzl@47697 2939 _total_remset_bytes(0), _total_strong_code_roots_bytes(0)
tschatzl@47697 2940 {
tschatzl@49811 2941 if (!log_is_enabled(Trace, gc, liveness)) {
tschatzl@49811 2942 return;
tschatzl@49811 2943 }
tschatzl@49811 2944
tonyp@8930 2945 G1CollectedHeap* g1h = G1CollectedHeap::heap();
tonyp@8930 2946 MemRegion g1_reserved = g1h->g1_reserved();
tonyp@8930 2947 double now = os::elapsedTime();
tonyp@8930 2948
tonyp@8930 2949 // Print the header of the output.
brutisso@35061 2950 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
brutisso@35061 2951 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
brutisso@35061 2952 G1PPRL_SUM_ADDR_FORMAT("reserved")
brutisso@35061 2953 G1PPRL_SUM_BYTE_FORMAT("region-size"),
brutisso@35061 2954 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
brutisso@35061 2955 HeapRegion::GrainBytes);
brutisso@35061 2956 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
brutisso@35061 2957 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
brutisso@35061 2958 G1PPRL_TYPE_H_FORMAT
brutisso@35061 2959 G1PPRL_ADDR_BASE_H_FORMAT
brutisso@35061 2960 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2961 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2962 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2963 G1PPRL_DOUBLE_H_FORMAT
brutisso@35061 2964 G1PPRL_BYTE_H_FORMAT
tschatzl@49672 2965 G1PPRL_STATE_H_FORMAT
brutisso@35061 2966 G1PPRL_BYTE_H_FORMAT,
brutisso@35061 2967 "type", "address-range",
brutisso@35061 2968 "used", "prev-live", "next-live", "gc-eff",
tschatzl@49672 2969 "remset", "state", "code-roots");
brutisso@35061 2970 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
brutisso@35061 2971 G1PPRL_TYPE_H_FORMAT
brutisso@35061 2972 G1PPRL_ADDR_BASE_H_FORMAT
brutisso@35061 2973 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2974 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2975 G1PPRL_BYTE_H_FORMAT
brutisso@35061 2976 G1PPRL_DOUBLE_H_FORMAT
brutisso@35061 2977 G1PPRL_BYTE_H_FORMAT
tschatzl@49672 2978 G1PPRL_STATE_H_FORMAT
brutisso@35061 2979 G1PPRL_BYTE_H_FORMAT,
brutisso@35061 2980 "", "",
brutisso@35061 2981 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
tschatzl@49672 2982 "(bytes)", "", "(bytes)");
tonyp@8930 2983 }
tonyp@8930 2984
jwilhelm@48969 2985 bool G1PrintRegionLivenessInfoClosure::do_heap_region(HeapRegion* r) {
tschatzl@49811 2986 if (!log_is_enabled(Trace, gc, liveness)) {
tschatzl@49811 2987 return false;
tschatzl@49811 2988 }
tschatzl@49811 2989
tonyp@26696 2990 const char* type = r->get_type_str();
tonyp@8930 2991 HeapWord* bottom = r->bottom();
tonyp@8930 2992 HeapWord* end = r->end();
tonyp@8930 2993 size_t capacity_bytes = r->capacity();
tonyp@8930 2994 size_t used_bytes = r->used();
tonyp@8930 2995 size_t prev_live_bytes = r->live_bytes();
tonyp@8930 2996 size_t next_live_bytes = r->next_live_bytes();
tonyp@8930 2997 double gc_eff = r->gc_efficiency();
tschatzl@17395 2998 size_t remset_bytes = r->rem_set()->mem_size();
johnc@19339 2999 size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size();
tschatzl@49672 3000 const char* remset_type = r->rem_set()->get_short_state_str();
johnc@19339 3001
tonyp@8930 3002 _total_used_bytes += used_bytes;
tonyp@8930 3003 _total_capacity_bytes += capacity_bytes;
tonyp@8930 3004 _total_prev_live_bytes += prev_live_bytes;
tonyp@8930 3005 _total_next_live_bytes += next_live_bytes;
tschatzl@17395 3006 _total_remset_bytes += remset_bytes;
johnc@19339 3007 _total_strong_code_roots_bytes += strong_code_roots_bytes;
tonyp@8930 3008
tonyp@8930 3009 // Print a line for this particular region.
brutisso@35061 3010 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
brutisso@35061 3011 G1PPRL_TYPE_FORMAT
brutisso@35061 3012 G1PPRL_ADDR_BASE_FORMAT
brutisso@35061 3013 G1PPRL_BYTE_FORMAT
brutisso@35061 3014 G1PPRL_BYTE_FORMAT
brutisso@35061 3015 G1PPRL_BYTE_FORMAT
brutisso@35061 3016 G1PPRL_DOUBLE_FORMAT
brutisso@35061 3017 G1PPRL_BYTE_FORMAT
tschatzl@49672 3018 G1PPRL_STATE_FORMAT
brutisso@35061 3019 G1PPRL_BYTE_FORMAT,
brutisso@35061 3020 type, p2i(bottom), p2i(end),
brutisso@35061 3021 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
tschatzl@49672 3022 remset_bytes, remset_type, strong_code_roots_bytes);
tonyp@8930 3023
tonyp@8930 3024 return false;
tonyp@8930 3025 }
tonyp@8930 3026
tonyp@8930 3027 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
tschatzl@49811 3028 if (!log_is_enabled(Trace, gc, liveness)) {
tschatzl@49811 3029 return;
tschatzl@49811 3030 }
tschatzl@49811 3031
tschatzl@17395 3032 // add sta