annotate src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 2187:234761c55641

6608385: G1: need to support parallel reference processing Summary: Implement support for ParallelRefProcEnabled in the reference processing that takes place at the end of G1 concurrent marking. Reviewed-by: tonyp, ysr
author johnc
date Tue, 25 Jan 2011 10:56:22 -0800
parents 0fa27f37d4d4
children 81668b1f4877
rev   line source
ysr@345 1 /*
tonyp@2162 2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
ysr@345 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
ysr@345 4 *
ysr@345 5 * This code is free software; you can redistribute it and/or modify it
ysr@345 6 * under the terms of the GNU General Public License version 2 only, as
ysr@345 7 * published by the Free Software Foundation.
ysr@345 8 *
ysr@345 9 * This code is distributed in the hope that it will be useful, but WITHOUT
ysr@345 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
ysr@345 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
ysr@345 12 * version 2 for more details (a copy is included in the LICENSE file that
ysr@345 13 * accompanied this code).
ysr@345 14 *
ysr@345 15 * You should have received a copy of the GNU General Public License version
ysr@345 16 * 2 along with this work; if not, write to the Free Software Foundation,
ysr@345 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
ysr@345 18 *
trims@1563 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@1563 20 * or visit www.oracle.com if you need additional information or have any
trims@1563 21 * questions.
ysr@345 22 *
ysr@345 23 */
ysr@345 24
stefank@1992 25 #include "precompiled.hpp"
stefank@1992 26 #include "classfile/symbolTable.hpp"
stefank@1992 27 #include "gc_implementation/g1/concurrentMark.hpp"
stefank@1992 28 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"
stefank@1992 29 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
stefank@1992 30 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
stefank@1992 31 #include "gc_implementation/g1/g1RemSet.hpp"
stefank@1992 32 #include "gc_implementation/g1/heapRegionRemSet.hpp"
stefank@1992 33 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
kamg@2138 34 #include "gc_implementation/shared/vmGCOperations.hpp"
stefank@1992 35 #include "memory/genOopClosures.inline.hpp"
stefank@1992 36 #include "memory/referencePolicy.hpp"
stefank@1992 37 #include "memory/resourceArea.hpp"
stefank@1992 38 #include "oops/oop.inline.hpp"
stefank@1992 39 #include "runtime/handles.inline.hpp"
stefank@1992 40 #include "runtime/java.hpp"
ysr@345 41
ysr@345 42 //
ysr@345 43 // CMS Bit Map Wrapper
ysr@345 44
ysr@345 45 CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter):
ysr@345 46 _bm((uintptr_t*)NULL,0),
ysr@345 47 _shifter(shifter) {
ysr@345 48 _bmStartWord = (HeapWord*)(rs.base());
ysr@345 49 _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes
ysr@345 50 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
ysr@345 51 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
ysr@345 52
ysr@345 53 guarantee(brs.is_reserved(), "couldn't allocate CMS bit map");
ysr@345 54 // For now we'll just commit all of the bit map up fromt.
ysr@345 55 // Later on we'll try to be more parsimonious with swap.
ysr@345 56 guarantee(_virtual_space.initialize(brs, brs.size()),
ysr@345 57 "couldn't reseve backing store for CMS bit map");
ysr@345 58 assert(_virtual_space.committed_size() == brs.size(),
ysr@345 59 "didn't reserve backing store for all of CMS bit map?");
ysr@345 60 _bm.set_map((uintptr_t*)_virtual_space.low());
ysr@345 61 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
ysr@345 62 _bmWordSize, "inconsistency in bit map sizing");
ysr@345 63 _bm.set_size(_bmWordSize >> _shifter);
ysr@345 64 }
ysr@345 65
ysr@345 66 HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
ysr@345 67 HeapWord* limit) const {
ysr@345 68 // First we must round addr *up* to a possible object boundary.
ysr@345 69 addr = (HeapWord*)align_size_up((intptr_t)addr,
ysr@345 70 HeapWordSize << _shifter);
ysr@345 71 size_t addrOffset = heapWordToOffset(addr);
ysr@345 72 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
ysr@345 73 size_t limitOffset = heapWordToOffset(limit);
ysr@345 74 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
ysr@345 75 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
ysr@345 76 assert(nextAddr >= addr, "get_next_one postcondition");
ysr@345 77 assert(nextAddr == limit || isMarked(nextAddr),
ysr@345 78 "get_next_one postcondition");
ysr@345 79 return nextAddr;
ysr@345 80 }
ysr@345 81
ysr@345 82 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
ysr@345 83 HeapWord* limit) const {
ysr@345 84 size_t addrOffset = heapWordToOffset(addr);
ysr@345 85 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
ysr@345 86 size_t limitOffset = heapWordToOffset(limit);
ysr@345 87 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
ysr@345 88 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
ysr@345 89 assert(nextAddr >= addr, "get_next_one postcondition");
ysr@345 90 assert(nextAddr == limit || !isMarked(nextAddr),
ysr@345 91 "get_next_one postcondition");
ysr@345 92 return nextAddr;
ysr@345 93 }
ysr@345 94
ysr@345 95 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
ysr@345 96 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
ysr@345 97 return (int) (diff >> _shifter);
ysr@345 98 }
ysr@345 99
ysr@345 100 bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
ysr@345 101 HeapWord* left = MAX2(_bmStartWord, mr.start());
ysr@345 102 HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
ysr@345 103 if (right > left) {
ysr@345 104 // Right-open interval [leftOffset, rightOffset).
ysr@345 105 return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
ysr@345 106 } else {
ysr@345 107 return true;
ysr@345 108 }
ysr@345 109 }
ysr@345 110
ysr@345 111 void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap,
ysr@345 112 size_t from_start_index,
ysr@345 113 HeapWord* to_start_word,
ysr@345 114 size_t word_num) {
ysr@345 115 _bm.mostly_disjoint_range_union(from_bitmap,
ysr@345 116 from_start_index,
ysr@345 117 heapWordToOffset(to_start_word),
ysr@345 118 word_num);
ysr@345 119 }
ysr@345 120
ysr@345 121 #ifndef PRODUCT
ysr@345 122 bool CMBitMapRO::covers(ReservedSpace rs) const {
ysr@345 123 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
kvn@683 124 assert(((size_t)_bm.size() * (size_t)(1 << _shifter)) == _bmWordSize,
ysr@345 125 "size inconsistency");
ysr@345 126 return _bmStartWord == (HeapWord*)(rs.base()) &&
ysr@345 127 _bmWordSize == rs.size()>>LogHeapWordSize;
ysr@345 128 }
ysr@345 129 #endif
ysr@345 130
ysr@345 131 void CMBitMap::clearAll() {
ysr@345 132 _bm.clear();
ysr@345 133 return;
ysr@345 134 }
ysr@345 135
ysr@345 136 void CMBitMap::markRange(MemRegion mr) {
ysr@345 137 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
ysr@345 138 assert(!mr.is_empty(), "unexpected empty region");
ysr@345 139 assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
ysr@345 140 ((HeapWord *) mr.end())),
ysr@345 141 "markRange memory region end is not card aligned");
ysr@345 142 // convert address range into offset range
ysr@345 143 _bm.at_put_range(heapWordToOffset(mr.start()),
ysr@345 144 heapWordToOffset(mr.end()), true);
ysr@345 145 }
ysr@345 146
ysr@345 147 void CMBitMap::clearRange(MemRegion mr) {
ysr@345 148 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
ysr@345 149 assert(!mr.is_empty(), "unexpected empty region");
ysr@345 150 // convert address range into offset range
ysr@345 151 _bm.at_put_range(heapWordToOffset(mr.start()),
ysr@345 152 heapWordToOffset(mr.end()), false);
ysr@345 153 }
ysr@345 154
ysr@345 155 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
ysr@345 156 HeapWord* end_addr) {
ysr@345 157 HeapWord* start = getNextMarkedWordAddress(addr);
ysr@345 158 start = MIN2(start, end_addr);
ysr@345 159 HeapWord* end = getNextUnmarkedWordAddress(start);
ysr@345 160 end = MIN2(end, end_addr);
ysr@345 161 assert(start <= end, "Consistency check");
ysr@345 162 MemRegion mr(start, end);
ysr@345 163 if (!mr.is_empty()) {
ysr@345 164 clearRange(mr);
ysr@345 165 }
ysr@345 166 return mr;
ysr@345 167 }
ysr@345 168
ysr@345 169 CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
ysr@345 170 _base(NULL), _cm(cm)
ysr@345 171 #ifdef ASSERT
ysr@345 172 , _drain_in_progress(false)
ysr@345 173 , _drain_in_progress_yields(false)
ysr@345 174 #endif
ysr@345 175 {}
ysr@345 176
ysr@345 177 void CMMarkStack::allocate(size_t size) {
ysr@345 178 _base = NEW_C_HEAP_ARRAY(oop, size);
ysr@345 179 if (_base == NULL)
ysr@345 180 vm_exit_during_initialization("Failed to allocate "
ysr@345 181 "CM region mark stack");
ysr@345 182 _index = 0;
ysr@345 183 // QQQQ cast ...
ysr@345 184 _capacity = (jint) size;
ysr@345 185 _oops_do_bound = -1;
ysr@345 186 NOT_PRODUCT(_max_depth = 0);
ysr@345 187 }
ysr@345 188
ysr@345 189 CMMarkStack::~CMMarkStack() {
ysr@345 190 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
ysr@345 191 }
ysr@345 192
ysr@345 193 void CMMarkStack::par_push(oop ptr) {
ysr@345 194 while (true) {
ysr@345 195 if (isFull()) {
ysr@345 196 _overflow = true;
ysr@345 197 return;
ysr@345 198 }
ysr@345 199 // Otherwise...
ysr@345 200 jint index = _index;
ysr@345 201 jint next_index = index+1;
ysr@345 202 jint res = Atomic::cmpxchg(next_index, &_index, index);
ysr@345 203 if (res == index) {
ysr@345 204 _base[index] = ptr;
ysr@345 205 // Note that we don't maintain this atomically. We could, but it
ysr@345 206 // doesn't seem necessary.
ysr@345 207 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
ysr@345 208 return;
ysr@345 209 }
ysr@345 210 // Otherwise, we need to try again.
ysr@345 211 }
ysr@345 212 }
ysr@345 213
ysr@345 214 void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) {
ysr@345 215 while (true) {
ysr@345 216 if (isFull()) {
ysr@345 217 _overflow = true;
ysr@345 218 return;
ysr@345 219 }
ysr@345 220 // Otherwise...
ysr@345 221 jint index = _index;
ysr@345 222 jint next_index = index + n;
ysr@345 223 if (next_index > _capacity) {
ysr@345 224 _overflow = true;
ysr@345 225 return;
ysr@345 226 }
ysr@345 227 jint res = Atomic::cmpxchg(next_index, &_index, index);
ysr@345 228 if (res == index) {
ysr@345 229 for (int i = 0; i < n; i++) {
ysr@345 230 int ind = index + i;
ysr@345 231 assert(ind < _capacity, "By overflow test above.");
ysr@345 232 _base[ind] = ptr_arr[i];
ysr@345 233 }
ysr@345 234 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
ysr@345 235 return;
ysr@345 236 }
ysr@345 237 // Otherwise, we need to try again.
ysr@345 238 }
ysr@345 239 }
ysr@345 240
ysr@345 241
ysr@345 242 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
ysr@345 243 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
ysr@345 244 jint start = _index;
ysr@345 245 jint next_index = start + n;
ysr@345 246 if (next_index > _capacity) {
ysr@345 247 _overflow = true;
ysr@345 248 return;
ysr@345 249 }
ysr@345 250 // Otherwise.
ysr@345 251 _index = next_index;
ysr@345 252 for (int i = 0; i < n; i++) {
ysr@345 253 int ind = start + i;
tonyp@1082 254 assert(ind < _capacity, "By overflow test above.");
ysr@345 255 _base[ind] = ptr_arr[i];
ysr@345 256 }
ysr@345 257 }
ysr@345 258
ysr@345 259
ysr@345 260 bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) {
ysr@345 261 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
ysr@345 262 jint index = _index;
ysr@345 263 if (index == 0) {
ysr@345 264 *n = 0;
ysr@345 265 return false;
ysr@345 266 } else {
ysr@345 267 int k = MIN2(max, index);
ysr@345 268 jint new_ind = index - k;
ysr@345 269 for (int j = 0; j < k; j++) {
ysr@345 270 ptr_arr[j] = _base[new_ind + j];
ysr@345 271 }
ysr@345 272 _index = new_ind;
ysr@345 273 *n = k;
ysr@345 274 return true;
ysr@345 275 }
ysr@345 276 }
ysr@345 277
ysr@345 278
ysr@345 279 CMRegionStack::CMRegionStack() : _base(NULL) {}
ysr@345 280
ysr@345 281 void CMRegionStack::allocate(size_t size) {
ysr@345 282 _base = NEW_C_HEAP_ARRAY(MemRegion, size);
ysr@345 283 if (_base == NULL)
ysr@345 284 vm_exit_during_initialization("Failed to allocate "
ysr@345 285 "CM region mark stack");
ysr@345 286 _index = 0;
ysr@345 287 // QQQQ cast ...
ysr@345 288 _capacity = (jint) size;
ysr@345 289 }
ysr@345 290
ysr@345 291 CMRegionStack::~CMRegionStack() {
ysr@345 292 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
ysr@345 293 }
ysr@345 294
johnc@1857 295 void CMRegionStack::push_lock_free(MemRegion mr) {
ysr@345 296 assert(mr.word_size() > 0, "Precondition");
ysr@345 297 while (true) {
johnc@1857 298 jint index = _index;
johnc@1857 299
johnc@1857 300 if (index >= _capacity) {
ysr@345 301 _overflow = true;
ysr@345 302 return;
ysr@345 303 }
ysr@345 304 // Otherwise...
ysr@345 305 jint next_index = index+1;
ysr@345 306 jint res = Atomic::cmpxchg(next_index, &_index, index);
ysr@345 307 if (res == index) {
ysr@345 308 _base[index] = mr;
ysr@345 309 return;
ysr@345 310 }
ysr@345 311 // Otherwise, we need to try again.
ysr@345 312 }
ysr@345 313 }
ysr@345 314
johnc@1857 315 // Lock-free pop of the region stack. Called during the concurrent
johnc@1857 316 // marking / remark phases. Should only be called in tandem with
johnc@1857 317 // other lock-free pops.
johnc@1857 318 MemRegion CMRegionStack::pop_lock_free() {
ysr@345 319 while (true) {
ysr@345 320 jint index = _index;
ysr@345 321
ysr@345 322 if (index == 0) {
ysr@345 323 return MemRegion();
ysr@345 324 }
johnc@1857 325 // Otherwise...
ysr@345 326 jint next_index = index-1;
ysr@345 327 jint res = Atomic::cmpxchg(next_index, &_index, index);
ysr@345 328 if (res == index) {
ysr@345 329 MemRegion mr = _base[next_index];
ysr@345 330 if (mr.start() != NULL) {
tonyp@1082 331 assert(mr.end() != NULL, "invariant");
tonyp@1082 332 assert(mr.word_size() > 0, "invariant");
ysr@345 333 return mr;
ysr@345 334 } else {
ysr@345 335 // that entry was invalidated... let's skip it
tonyp@1082 336 assert(mr.end() == NULL, "invariant");
ysr@345 337 }
ysr@345 338 }
ysr@345 339 // Otherwise, we need to try again.
ysr@345 340 }
ysr@345 341 }
johnc@1857 342
johnc@1857 343 #if 0
johnc@1857 344 // The routines that manipulate the region stack with a lock are
johnc@1857 345 // not currently used. They should be retained, however, as a
johnc@1857 346 // diagnostic aid.
tonyp@1447 347
tonyp@1447 348 void CMRegionStack::push_with_lock(MemRegion mr) {
tonyp@1447 349 assert(mr.word_size() > 0, "Precondition");
tonyp@1447 350 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
tonyp@1447 351
tonyp@1447 352 if (isFull()) {
tonyp@1447 353 _overflow = true;
tonyp@1447 354 return;
tonyp@1447 355 }
tonyp@1447 356
tonyp@1447 357 _base[_index] = mr;
tonyp@1447 358 _index += 1;
tonyp@1447 359 }
tonyp@1447 360
tonyp@1447 361 MemRegion CMRegionStack::pop_with_lock() {
tonyp@1447 362 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
tonyp@1447 363
tonyp@1447 364 while (true) {
tonyp@1447 365 if (_index == 0) {
tonyp@1447 366 return MemRegion();
tonyp@1447 367 }
tonyp@1447 368 _index -= 1;
tonyp@1447 369
tonyp@1447 370 MemRegion mr = _base[_index];
tonyp@1447 371 if (mr.start() != NULL) {
tonyp@1447 372 assert(mr.end() != NULL, "invariant");
tonyp@1447 373 assert(mr.word_size() > 0, "invariant");
tonyp@1447 374 return mr;
tonyp@1447 375 } else {
tonyp@1447 376 // that entry was invalidated... let's skip it
tonyp@1447 377 assert(mr.end() == NULL, "invariant");
tonyp@1447 378 }
tonyp@1447 379 }
tonyp@1447 380 }
johnc@1857 381 #endif
ysr@345 382
ysr@345 383 bool CMRegionStack::invalidate_entries_into_cset() {
ysr@345 384 bool result = false;
ysr@345 385 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 386 for (int i = 0; i < _oops_do_bound; ++i) {
ysr@345 387 MemRegion mr = _base[i];
ysr@345 388 if (mr.start() != NULL) {
tonyp@1082 389 assert(mr.end() != NULL, "invariant");
tonyp@1082 390 assert(mr.word_size() > 0, "invariant");
ysr@345 391 HeapRegion* hr = g1h->heap_region_containing(mr.start());
tonyp@1082 392 assert(hr != NULL, "invariant");
ysr@345 393 if (hr->in_collection_set()) {
ysr@345 394 // The region points into the collection set
ysr@345 395 _base[i] = MemRegion();
ysr@345 396 result = true;
ysr@345 397 }
ysr@345 398 } else {
ysr@345 399 // that entry was invalidated... let's skip it
tonyp@1082 400 assert(mr.end() == NULL, "invariant");
ysr@345 401 }
ysr@345 402 }
ysr@345 403 return result;
ysr@345 404 }
ysr@345 405
ysr@345 406 template<class OopClosureClass>
ysr@345 407 bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) {
ysr@345 408 assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after
ysr@345 409 || SafepointSynchronize::is_at_safepoint(),
ysr@345 410 "Drain recursion must be yield-safe.");
ysr@345 411 bool res = true;
ysr@345 412 debug_only(_drain_in_progress = true);
ysr@345 413 debug_only(_drain_in_progress_yields = yield_after);
ysr@345 414 while (!isEmpty()) {
ysr@345 415 oop newOop = pop();
ysr@345 416 assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop");
ysr@345 417 assert(newOop->is_oop(), "Expected an oop");
ysr@345 418 assert(bm == NULL || bm->isMarked((HeapWord*)newOop),
ysr@345 419 "only grey objects on this stack");
ysr@345 420 // iterate over the oops in this oop, marking and pushing
ysr@345 421 // the ones in CMS generation.
ysr@345 422 newOop->oop_iterate(cl);
ysr@345 423 if (yield_after && _cm->do_yield_check()) {
ysr@345 424 res = false; break;
ysr@345 425 }
ysr@345 426 }
ysr@345 427 debug_only(_drain_in_progress = false);
ysr@345 428 return res;
ysr@345 429 }
ysr@345 430
ysr@345 431 void CMMarkStack::oops_do(OopClosure* f) {
ysr@345 432 if (_index == 0) return;
ysr@345 433 assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
ysr@345 434 "Bound must be set.");
ysr@345 435 for (int i = 0; i < _oops_do_bound; i++) {
ysr@345 436 f->do_oop(&_base[i]);
ysr@345 437 }
ysr@345 438 _oops_do_bound = -1;
ysr@345 439 }
ysr@345 440
ysr@345 441 bool ConcurrentMark::not_yet_marked(oop obj) const {
ysr@345 442 return (_g1h->is_obj_ill(obj)
ysr@345 443 || (_g1h->is_in_permanent(obj)
ysr@345 444 && !nextMarkBitMap()->isMarked((HeapWord*)obj)));
ysr@345 445 }
ysr@345 446
ysr@345 447 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
ysr@345 448 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
ysr@345 449 #endif // _MSC_VER
ysr@345 450
ysr@345 451 ConcurrentMark::ConcurrentMark(ReservedSpace rs,
ysr@345 452 int max_regions) :
ysr@345 453 _markBitMap1(rs, MinObjAlignment - 1),
ysr@345 454 _markBitMap2(rs, MinObjAlignment - 1),
ysr@345 455
ysr@345 456 _parallel_marking_threads(0),
ysr@345 457 _sleep_factor(0.0),
ysr@345 458 _marking_task_overhead(1.0),
ysr@345 459 _cleanup_sleep_factor(0.0),
ysr@345 460 _cleanup_task_overhead(1.0),
tonyp@2165 461 _cleanup_list("Cleanup List"),
ysr@345 462 _region_bm(max_regions, false /* in_resource_area*/),
ysr@345 463 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
ysr@345 464 CardTableModRefBS::card_shift,
ysr@345 465 false /* in_resource_area*/),
ysr@345 466 _prevMarkBitMap(&_markBitMap1),
ysr@345 467 _nextMarkBitMap(&_markBitMap2),
ysr@345 468 _at_least_one_mark_complete(false),
ysr@345 469
ysr@345 470 _markStack(this),
ysr@345 471 _regionStack(),
ysr@345 472 // _finger set in set_non_marking_state
ysr@345 473
ysr@345 474 _max_task_num(MAX2(ParallelGCThreads, (size_t)1)),
ysr@345 475 // _active_tasks set in set_non_marking_state
ysr@345 476 // _tasks set inside the constructor
ysr@345 477 _task_queues(new CMTaskQueueSet((int) _max_task_num)),
ysr@345 478 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)),
ysr@345 479
ysr@345 480 _has_overflown(false),
ysr@345 481 _concurrent(false),
tonyp@657 482 _has_aborted(false),
tonyp@657 483 _restart_for_overflow(false),
tonyp@657 484 _concurrent_marking_in_progress(false),
tonyp@657 485 _should_gray_objects(false),
ysr@345 486
ysr@345 487 // _verbose_level set below
ysr@345 488
ysr@345 489 _init_times(),
ysr@345 490 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
ysr@345 491 _cleanup_times(),
ysr@345 492 _total_counting_time(0.0),
ysr@345 493 _total_rs_scrub_time(0.0),
ysr@345 494
tonyp@990 495 _parallel_workers(NULL)
ysr@345 496 {
ysr@345 497 CMVerboseLevel verbose_level =
ysr@345 498 (CMVerboseLevel) G1MarkingVerboseLevel;
ysr@345 499 if (verbose_level < no_verbose)
ysr@345 500 verbose_level = no_verbose;
ysr@345 501 if (verbose_level > high_verbose)
ysr@345 502 verbose_level = high_verbose;
ysr@345 503 _verbose_level = verbose_level;
ysr@345 504
ysr@345 505 if (verbose_low())
ysr@345 506 gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", "
ysr@345 507 "heap end = "PTR_FORMAT, _heap_start, _heap_end);
ysr@345 508
jmasa@1371 509 _markStack.allocate(MarkStackSize);
johnc@791 510 _regionStack.allocate(G1MarkRegionStackSize);
ysr@345 511
ysr@345 512 // Create & start a ConcurrentMark thread.
ysr@896 513 _cmThread = new ConcurrentMarkThread(this);
ysr@896 514 assert(cmThread() != NULL, "CM Thread should have been created");
ysr@896 515 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
ysr@896 516
ysr@345 517 _g1h = G1CollectedHeap::heap();
ysr@345 518 assert(CGC_lock != NULL, "Where's the CGC_lock?");
ysr@345 519 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
ysr@345 520 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
ysr@345 521
ysr@345 522 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
tonyp@1369 523 satb_qs.set_buffer_size(G1SATBBufferSize);
ysr@345 524
ysr@345 525 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
ysr@345 526 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
ysr@345 527
ysr@345 528 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
ysr@345 529 _active_tasks = _max_task_num;
ysr@345 530 for (int i = 0; i < (int) _max_task_num; ++i) {
ysr@345 531 CMTaskQueue* task_queue = new CMTaskQueue();
ysr@345 532 task_queue->initialize();
ysr@345 533 _task_queues->register_queue(i, task_queue);
ysr@345 534
ysr@345 535 _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
ysr@345 536 _accum_task_vtime[i] = 0.0;
ysr@345 537 }
ysr@345 538
jmasa@1371 539 if (ConcGCThreads > ParallelGCThreads) {
jmasa@1371 540 vm_exit_during_initialization("Can't have more ConcGCThreads "
ysr@345 541 "than ParallelGCThreads.");
ysr@345 542 }
ysr@345 543 if (ParallelGCThreads == 0) {
ysr@345 544 // if we are not running with any parallel GC threads we will not
ysr@345 545 // spawn any marking threads either
ysr@345 546 _parallel_marking_threads = 0;
ysr@345 547 _sleep_factor = 0.0;
ysr@345 548 _marking_task_overhead = 1.0;
ysr@345 549 } else {
jmasa@1371 550 if (ConcGCThreads > 0) {
jmasa@1371 551 // notice that ConcGCThreads overwrites G1MarkingOverheadPercent
ysr@345 552 // if both are set
ysr@345 553
jmasa@1371 554 _parallel_marking_threads = ConcGCThreads;
ysr@345 555 _sleep_factor = 0.0;
ysr@345 556 _marking_task_overhead = 1.0;
johnc@791 557 } else if (G1MarkingOverheadPercent > 0) {
ysr@345 558 // we will calculate the number of parallel marking threads
ysr@345 559 // based on a target overhead with respect to the soft real-time
ysr@345 560 // goal
ysr@345 561
johnc@791 562 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
ysr@345 563 double overall_cm_overhead =
johnc@791 564 (double) MaxGCPauseMillis * marking_overhead /
johnc@791 565 (double) GCPauseIntervalMillis;
ysr@345 566 double cpu_ratio = 1.0 / (double) os::processor_count();
ysr@345 567 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
ysr@345 568 double marking_task_overhead =
ysr@345 569 overall_cm_overhead / marking_thread_num *
ysr@345 570 (double) os::processor_count();
ysr@345 571 double sleep_factor =
ysr@345 572 (1.0 - marking_task_overhead) / marking_task_overhead;
ysr@345 573
ysr@345 574 _parallel_marking_threads = (size_t) marking_thread_num;
ysr@345 575 _sleep_factor = sleep_factor;
ysr@345 576 _marking_task_overhead = marking_task_overhead;
ysr@345 577 } else {
ysr@345 578 _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1);
ysr@345 579 _sleep_factor = 0.0;
ysr@345 580 _marking_task_overhead = 1.0;
ysr@345 581 }
ysr@345 582
ysr@345 583 if (parallel_marking_threads() > 1)
ysr@345 584 _cleanup_task_overhead = 1.0;
ysr@345 585 else
ysr@345 586 _cleanup_task_overhead = marking_task_overhead();
ysr@345 587 _cleanup_sleep_factor =
ysr@345 588 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
ysr@345 589
ysr@345 590 #if 0
ysr@345 591 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
ysr@345 592 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
ysr@345 593 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
ysr@345 594 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
ysr@345 595 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
ysr@345 596 #endif
ysr@345 597
tonyp@1082 598 guarantee(parallel_marking_threads() > 0, "peace of mind");
jmasa@1855 599 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads",
jmasa@1855 600 (int) _parallel_marking_threads, false, true);
jmasa@1855 601 if (_parallel_workers == NULL) {
ysr@345 602 vm_exit_during_initialization("Failed necessary allocation.");
jmasa@1855 603 } else {
jmasa@1855 604 _parallel_workers->initialize_workers();
jmasa@1855 605 }
ysr@345 606 }
ysr@345 607
ysr@345 608 // so that the call below can read a sensible value
ysr@345 609 _heap_start = (HeapWord*) rs.base();
ysr@345 610 set_non_marking_state();
ysr@345 611 }
ysr@345 612
ysr@345 613 void ConcurrentMark::update_g1_committed(bool force) {
ysr@345 614 // If concurrent marking is not in progress, then we do not need to
ysr@345 615 // update _heap_end. This has a subtle and important
ysr@345 616 // side-effect. Imagine that two evacuation pauses happen between
ysr@345 617 // marking completion and remark. The first one can grow the
ysr@345 618 // heap (hence now the finger is below the heap end). Then, the
ysr@345 619 // second one could unnecessarily push regions on the region
ysr@345 620 // stack. This causes the invariant that the region stack is empty
ysr@345 621 // at the beginning of remark to be false. By ensuring that we do
ysr@345 622 // not observe heap expansions after marking is complete, then we do
ysr@345 623 // not have this problem.
ysr@345 624 if (!concurrent_marking_in_progress() && !force)
ysr@345 625 return;
ysr@345 626
ysr@345 627 MemRegion committed = _g1h->g1_committed();
tonyp@1082 628 assert(committed.start() == _heap_start, "start shouldn't change");
ysr@345 629 HeapWord* new_end = committed.end();
ysr@345 630 if (new_end > _heap_end) {
ysr@345 631 // The heap has been expanded.
ysr@345 632
ysr@345 633 _heap_end = new_end;
ysr@345 634 }
ysr@345 635 // Notice that the heap can also shrink. However, this only happens
ysr@345 636 // during a Full GC (at least currently) and the entire marking
ysr@345 637 // phase will bail out and the task will not be restarted. So, let's
ysr@345 638 // do nothing.
ysr@345 639 }
ysr@345 640
ysr@345 641 void ConcurrentMark::reset() {
ysr@345 642 // Starting values for these two. This should be called in a STW
ysr@345 643 // phase. CM will be notified of any future g1_committed expansions
ysr@345 644 // will be at the end of evacuation pauses, when tasks are
ysr@345 645 // inactive.
ysr@345 646 MemRegion committed = _g1h->g1_committed();
ysr@345 647 _heap_start = committed.start();
ysr@345 648 _heap_end = committed.end();
ysr@345 649
tonyp@1082 650 // Separated the asserts so that we know which one fires.
tonyp@1082 651 assert(_heap_start != NULL, "heap bounds should look ok");
tonyp@1082 652 assert(_heap_end != NULL, "heap bounds should look ok");
tonyp@1082 653 assert(_heap_start < _heap_end, "heap bounds should look ok");
ysr@345 654
ysr@345 655 // reset all the marking data structures and any necessary flags
ysr@345 656 clear_marking_state();
ysr@345 657
ysr@345 658 if (verbose_low())
ysr@345 659 gclog_or_tty->print_cr("[global] resetting");
ysr@345 660
ysr@345 661 // We do reset all of them, since different phases will use
ysr@345 662 // different number of active threads. So, it's easiest to have all
ysr@345 663 // of them ready.
johnc@1857 664 for (int i = 0; i < (int) _max_task_num; ++i) {
ysr@345 665 _tasks[i]->reset(_nextMarkBitMap);
johnc@1857 666 }
ysr@345 667
ysr@345 668 // we need this to make sure that the flag is on during the evac
ysr@345 669 // pause with initial mark piggy-backed
ysr@345 670 set_concurrent_marking_in_progress();
ysr@345 671 }
ysr@345 672
ysr@345 673 void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) {
tonyp@1082 674 assert(active_tasks <= _max_task_num, "we should not have more");
ysr@345 675
ysr@345 676 _active_tasks = active_tasks;
ysr@345 677 // Need to update the three data structures below according to the
ysr@345 678 // number of active threads for this phase.
ysr@345 679 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
ysr@345 680 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
ysr@345 681 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
ysr@345 682
ysr@345 683 _concurrent = concurrent;
ysr@345 684 // We propagate this to all tasks, not just the active ones.
ysr@345 685 for (int i = 0; i < (int) _max_task_num; ++i)
ysr@345 686 _tasks[i]->set_concurrent(concurrent);
ysr@345 687
ysr@345 688 if (concurrent) {
ysr@345 689 set_concurrent_marking_in_progress();
ysr@345 690 } else {
ysr@345 691 // We currently assume that the concurrent flag has been set to
ysr@345 692 // false before we start remark. At this point we should also be
ysr@345 693 // in a STW phase.
tonyp@1082 694 assert(!concurrent_marking_in_progress(), "invariant");
tonyp@1082 695 assert(_finger == _heap_end, "only way to get here");
ysr@345 696 update_g1_committed(true);
ysr@345 697 }
ysr@345 698 }
ysr@345 699
ysr@345 700 void ConcurrentMark::set_non_marking_state() {
ysr@345 701 // We set the global marking state to some default values when we're
ysr@345 702 // not doing marking.
ysr@345 703 clear_marking_state();
ysr@345 704 _active_tasks = 0;
ysr@345 705 clear_concurrent_marking_in_progress();
ysr@345 706 }
ysr@345 707
ysr@345 708 ConcurrentMark::~ConcurrentMark() {
ysr@345 709 for (int i = 0; i < (int) _max_task_num; ++i) {
ysr@345 710 delete _task_queues->queue(i);
ysr@345 711 delete _tasks[i];
ysr@345 712 }
ysr@345 713 delete _task_queues;
ysr@345 714 FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
ysr@345 715 }
ysr@345 716
ysr@345 717 // This closure is used to mark refs into the g1 generation
ysr@345 718 // from external roots in the CMS bit map.
ysr@345 719 // Called at the first checkpoint.
ysr@345 720 //
ysr@345 721
ysr@345 722 void ConcurrentMark::clearNextBitmap() {
tonyp@1448 723 G1CollectedHeap* g1h = G1CollectedHeap::heap();
tonyp@1448 724 G1CollectorPolicy* g1p = g1h->g1_policy();
tonyp@1448 725
tonyp@1448 726 // Make sure that the concurrent mark thread looks to still be in
tonyp@1448 727 // the current cycle.
tonyp@1448 728 guarantee(cmThread()->during_cycle(), "invariant");
tonyp@1448 729
tonyp@1448 730 // We are finishing up the current cycle by clearing the next
tonyp@1448 731 // marking bitmap and getting it ready for the next cycle. During
tonyp@1448 732 // this time no other cycle can start. So, let's make sure that this
tonyp@1448 733 // is the case.
tonyp@1448 734 guarantee(!g1h->mark_in_progress(), "invariant");
tonyp@1448 735
tonyp@1448 736 // clear the mark bitmap (no grey objects to start with).
tonyp@1448 737 // We need to do this in chunks and offer to yield in between
tonyp@1448 738 // each chunk.
tonyp@1448 739 HeapWord* start = _nextMarkBitMap->startWord();
tonyp@1448 740 HeapWord* end = _nextMarkBitMap->endWord();
tonyp@1448 741 HeapWord* cur = start;
tonyp@1448 742 size_t chunkSize = M;
tonyp@1448 743 while (cur < end) {
tonyp@1448 744 HeapWord* next = cur + chunkSize;
tonyp@1448 745 if (next > end)
tonyp@1448 746 next = end;
tonyp@1448 747 MemRegion mr(cur,next);
tonyp@1448 748 _nextMarkBitMap->clearRange(mr);
tonyp@1448 749 cur = next;
tonyp@1448 750 do_yield_check();
tonyp@1448 751
tonyp@1448 752 // Repeat the asserts from above. We'll do them as asserts here to
tonyp@1448 753 // minimize their overhead on the product. However, we'll have
tonyp@1448 754 // them as guarantees at the beginning / end of the bitmap
tonyp@1448 755 // clearing to get some checking in the product.
tonyp@1448 756 assert(cmThread()->during_cycle(), "invariant");
tonyp@1448 757 assert(!g1h->mark_in_progress(), "invariant");
tonyp@1448 758 }
tonyp@1448 759
tonyp@1448 760 // Repeat the asserts from above.
tonyp@1448 761 guarantee(cmThread()->during_cycle(), "invariant");
tonyp@1448 762 guarantee(!g1h->mark_in_progress(), "invariant");
ysr@345 763 }
ysr@345 764
ysr@345 765 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
ysr@345 766 public:
ysr@345 767 bool doHeapRegion(HeapRegion* r) {
ysr@345 768 if (!r->continuesHumongous()) {
ysr@345 769 r->note_start_of_marking(true);
ysr@345 770 }
ysr@345 771 return false;
ysr@345 772 }
ysr@345 773 };
ysr@345 774
ysr@345 775 void ConcurrentMark::checkpointRootsInitialPre() {
ysr@345 776 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 777 G1CollectorPolicy* g1p = g1h->g1_policy();
ysr@345 778
ysr@345 779 _has_aborted = false;
ysr@345 780
jcoomes@1558 781 #ifndef PRODUCT
tonyp@1106 782 if (G1PrintReachableAtInitialMark) {
tonyp@1477 783 print_reachable("at-cycle-start",
tonyp@1477 784 true /* use_prev_marking */, true /* all */);
tonyp@1106 785 }
jcoomes@1558 786 #endif
ysr@345 787
ysr@345 788 // Initialise marking structures. This has to be done in a STW phase.
ysr@345 789 reset();
ysr@345 790 }
ysr@345 791
ysr@345 792 class CMMarkRootsClosure: public OopsInGenClosure {
ysr@345 793 private:
ysr@345 794 ConcurrentMark* _cm;
ysr@345 795 G1CollectedHeap* _g1h;
ysr@345 796 bool _do_barrier;
ysr@345 797
ysr@345 798 public:
ysr@345 799 CMMarkRootsClosure(ConcurrentMark* cm,
ysr@345 800 G1CollectedHeap* g1h,
ysr@345 801 bool do_barrier) : _cm(cm), _g1h(g1h),
ysr@345 802 _do_barrier(do_barrier) { }
ysr@345 803
ysr@896 804 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
ysr@896 805 virtual void do_oop( oop* p) { do_oop_work(p); }
ysr@896 806
ysr@896 807 template <class T> void do_oop_work(T* p) {
ysr@896 808 T heap_oop = oopDesc::load_heap_oop(p);
ysr@896 809 if (!oopDesc::is_null(heap_oop)) {
ysr@896 810 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
ysr@896 811 assert(obj->is_oop() || obj->mark() == NULL,
ysr@345 812 "expected an oop, possibly with mark word displaced");
ysr@896 813 HeapWord* addr = (HeapWord*)obj;
ysr@345 814 if (_g1h->is_in_g1_reserved(addr)) {
ysr@896 815 _cm->grayRoot(obj);
ysr@345 816 }
ysr@345 817 }
ysr@345 818 if (_do_barrier) {
ysr@345 819 assert(!_g1h->is_in_g1_reserved(p),
ysr@345 820 "Should be called on external roots");
ysr@345 821 do_barrier(p);
ysr@345 822 }
ysr@345 823 }
ysr@345 824 };
ysr@345 825
ysr@345 826 void ConcurrentMark::checkpointRootsInitialPost() {
ysr@345 827 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 828
ysr@345 829 // For each region note start of marking.
ysr@345 830 NoteStartOfMarkHRClosure startcl;
ysr@345 831 g1h->heap_region_iterate(&startcl);
ysr@345 832
ysr@345 833 // Start weak-reference discovery.
ysr@345 834 ReferenceProcessor* rp = g1h->ref_processor();
ysr@345 835 rp->verify_no_references_recorded();
ysr@345 836 rp->enable_discovery(); // enable ("weak") refs discovery
ysr@464 837 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
ysr@345 838
ysr@345 839 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
tonyp@1404 840 // This is the start of the marking cycle, we're expected all
tonyp@1404 841 // threads to have SATB queues with active set to false.
tonyp@1404 842 satb_mq_set.set_active_all_threads(true, /* new active value */
tonyp@1404 843 false /* expected_active */);
ysr@345 844
ysr@345 845 // update_g1_committed() will be called at the end of an evac pause
ysr@345 846 // when marking is on. So, it's also called at the end of the
ysr@345 847 // initial-mark pause to update the heap end, if the heap expands
ysr@345 848 // during it. No need to call it here.
ysr@345 849 }
ysr@345 850
ysr@345 851 // Checkpoint the roots into this generation from outside
ysr@345 852 // this generation. [Note this initial checkpoint need only
ysr@345 853 // be approximate -- we'll do a catch up phase subsequently.]
ysr@345 854 void ConcurrentMark::checkpointRootsInitial() {
ysr@345 855 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
ysr@345 856 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 857
ysr@345 858 double start = os::elapsedTime();
ysr@345 859
ysr@345 860 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
ysr@345 861 g1p->record_concurrent_mark_init_start();
ysr@345 862 checkpointRootsInitialPre();
ysr@345 863
ysr@345 864 // YSR: when concurrent precleaning is in place, we'll
ysr@345 865 // need to clear the cached card table here
ysr@345 866
ysr@345 867 ResourceMark rm;
ysr@345 868 HandleMark hm;
ysr@345 869
ysr@345 870 g1h->ensure_parsability(false);
ysr@345 871 g1h->perm_gen()->save_marks();
ysr@345 872
ysr@345 873 CMMarkRootsClosure notOlder(this, g1h, false);
ysr@345 874 CMMarkRootsClosure older(this, g1h, true);
ysr@345 875
ysr@345 876 g1h->set_marking_started();
ysr@345 877 g1h->rem_set()->prepare_for_younger_refs_iterate(false);
ysr@345 878
jrose@1047 879 g1h->process_strong_roots(true, // activate StrongRootsScope
jrose@1047 880 false, // fake perm gen collection
ysr@345 881 SharedHeap::SO_AllClasses,
ysr@345 882 &notOlder, // Regular roots
jrose@1047 883 NULL, // do not visit active blobs
ysr@345 884 &older // Perm Gen Roots
ysr@345 885 );
ysr@345 886 checkpointRootsInitialPost();
ysr@345 887
ysr@345 888 // Statistics.
ysr@345 889 double end = os::elapsedTime();
ysr@345 890 _init_times.add((end - start) * 1000.0);
ysr@345 891
ysr@345 892 g1p->record_concurrent_mark_init_end();
ysr@345 893 }
ysr@345 894
ysr@345 895 /*
ysr@345 896 Notice that in the next two methods, we actually leave the STS
ysr@345 897 during the barrier sync and join it immediately afterwards. If we
ysr@345 898 do not do this, this then the following deadlock can occur: one
ysr@345 899 thread could be in the barrier sync code, waiting for the other
ysr@345 900 thread to also sync up, whereas another one could be trying to
ysr@345 901 yield, while also waiting for the other threads to sync up too.
ysr@345 902
ysr@345 903 Because the thread that does the sync barrier has left the STS, it
ysr@345 904 is possible to be suspended for a Full GC or an evacuation pause
ysr@345 905 could occur. This is actually safe, since the entering the sync
ysr@345 906 barrier is one of the last things do_marking_step() does, and it
ysr@345 907 doesn't manipulate any data structures afterwards.
ysr@345 908 */
ysr@345 909
ysr@345 910 void ConcurrentMark::enter_first_sync_barrier(int task_num) {
ysr@345 911 if (verbose_low())
ysr@345 912 gclog_or_tty->print_cr("[%d] entering first barrier", task_num);
ysr@345 913
ysr@345 914 ConcurrentGCThread::stsLeave();
ysr@345 915 _first_overflow_barrier_sync.enter();
ysr@345 916 ConcurrentGCThread::stsJoin();
ysr@345 917 // at this point everyone should have synced up and not be doing any
ysr@345 918 // more work
ysr@345 919
ysr@345 920 if (verbose_low())
ysr@345 921 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num);
ysr@345 922
ysr@345 923 // let task 0 do this
ysr@345 924 if (task_num == 0) {
ysr@345 925 // task 0 is responsible for clearing the global data structures
ysr@345 926 clear_marking_state();
ysr@345 927
ysr@345 928 if (PrintGC) {
ysr@345 929 gclog_or_tty->date_stamp(PrintGCDateStamps);
ysr@345 930 gclog_or_tty->stamp(PrintGCTimeStamps);
ysr@345 931 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
ysr@345 932 }
ysr@345 933 }
ysr@345 934
ysr@345 935 // after this, each task should reset its own data structures then
ysr@345 936 // then go into the second barrier
ysr@345 937 }
ysr@345 938
ysr@345 939 void ConcurrentMark::enter_second_sync_barrier(int task_num) {
ysr@345 940 if (verbose_low())
ysr@345 941 gclog_or_tty->print_cr("[%d] entering second barrier", task_num);
ysr@345 942
ysr@345 943 ConcurrentGCThread::stsLeave();
ysr@345 944 _second_overflow_barrier_sync.enter();
ysr@345 945 ConcurrentGCThread::stsJoin();
ysr@345 946 // at this point everything should be re-initialised and ready to go
ysr@345 947
ysr@345 948 if (verbose_low())
ysr@345 949 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num);
ysr@345 950 }
ysr@345 951
ysr@345 952 void ConcurrentMark::grayRoot(oop p) {
ysr@345 953 HeapWord* addr = (HeapWord*) p;
ysr@345 954 // We can't really check against _heap_start and _heap_end, since it
ysr@345 955 // is possible during an evacuation pause with piggy-backed
ysr@345 956 // initial-mark that the committed space is expanded during the
ysr@345 957 // pause without CM observing this change. So the assertions below
ysr@345 958 // is a bit conservative; but better than nothing.
tonyp@1082 959 assert(_g1h->g1_committed().contains(addr),
tonyp@1082 960 "address should be within the heap bounds");
ysr@345 961
ysr@345 962 if (!_nextMarkBitMap->isMarked(addr))
ysr@345 963 _nextMarkBitMap->parMark(addr);
ysr@345 964 }
ysr@345 965
ysr@345 966 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
ysr@345 967 // The objects on the region have already been marked "in bulk" by
ysr@345 968 // the caller. We only need to decide whether to push the region on
ysr@345 969 // the region stack or not.
ysr@345 970
ysr@345 971 if (!concurrent_marking_in_progress() || !_should_gray_objects)
ysr@345 972 // We're done with marking and waiting for remark. We do not need to
ysr@345 973 // push anything else on the region stack.
ysr@345 974 return;
ysr@345 975
ysr@345 976 HeapWord* finger = _finger;
ysr@345 977
ysr@345 978 if (verbose_low())
ysr@345 979 gclog_or_tty->print_cr("[global] attempting to push "
ysr@345 980 "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at "
ysr@345 981 PTR_FORMAT, mr.start(), mr.end(), finger);
ysr@345 982
ysr@345 983 if (mr.start() < finger) {
ysr@345 984 // The finger is always heap region aligned and it is not possible
ysr@345 985 // for mr to span heap regions.
tonyp@1082 986 assert(mr.end() <= finger, "invariant");
tonyp@1082 987
tonyp@1082 988 // Separated the asserts so that we know which one fires.
tonyp@1082 989 assert(mr.start() <= mr.end(),
tonyp@1082 990 "region boundaries should fall within the committed space");
tonyp@1082 991 assert(_heap_start <= mr.start(),
tonyp@1082 992 "region boundaries should fall within the committed space");
tonyp@1082 993 assert(mr.end() <= _heap_end,
tonyp@1082 994 "region boundaries should fall within the committed space");
ysr@345 995 if (verbose_low())
ysr@345 996 gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") "
ysr@345 997 "below the finger, pushing it",
ysr@345 998 mr.start(), mr.end());
ysr@345 999
johnc@1857 1000 if (!region_stack_push_lock_free(mr)) {
ysr@345 1001 if (verbose_low())
ysr@345 1002 gclog_or_tty->print_cr("[global] region stack has overflown.");
ysr@345 1003 }
ysr@345 1004 }
ysr@345 1005 }
ysr@345 1006
ysr@345 1007 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
ysr@345 1008 // The object is not marked by the caller. We need to at least mark
ysr@345 1009 // it and maybe push in on the stack.
ysr@345 1010
ysr@345 1011 HeapWord* addr = (HeapWord*)p;
ysr@345 1012 if (!_nextMarkBitMap->isMarked(addr)) {
ysr@345 1013 // We definitely need to mark it, irrespective whether we bail out
ysr@345 1014 // because we're done with marking.
ysr@345 1015 if (_nextMarkBitMap->parMark(addr)) {
ysr@345 1016 if (!concurrent_marking_in_progress() || !_should_gray_objects)
ysr@345 1017 // If we're done with concurrent marking and we're waiting for
ysr@345 1018 // remark, then we're not pushing anything on the stack.
ysr@345 1019 return;
ysr@345 1020
ysr@345 1021 // No OrderAccess:store_load() is needed. It is implicit in the
ysr@345 1022 // CAS done in parMark(addr) above
ysr@345 1023 HeapWord* finger = _finger;
ysr@345 1024
ysr@345 1025 if (addr < finger) {
ysr@345 1026 if (!mark_stack_push(oop(addr))) {
ysr@345 1027 if (verbose_low())
ysr@345 1028 gclog_or_tty->print_cr("[global] global stack overflow "
ysr@345 1029 "during parMark");
ysr@345 1030 }
ysr@345 1031 }
ysr@345 1032 }
ysr@345 1033 }
ysr@345 1034 }
ysr@345 1035
ysr@345 1036 class CMConcurrentMarkingTask: public AbstractGangTask {
ysr@345 1037 private:
ysr@345 1038 ConcurrentMark* _cm;
ysr@345 1039 ConcurrentMarkThread* _cmt;
ysr@345 1040
ysr@345 1041 public:
ysr@345 1042 void work(int worker_i) {
tonyp@1082 1043 assert(Thread::current()->is_ConcurrentGC_thread(),
tonyp@1082 1044 "this should only be done by a conc GC thread");
johnc@1994 1045 ResourceMark rm;
ysr@345 1046
ysr@345 1047 double start_vtime = os::elapsedVTime();
ysr@345 1048
ysr@345 1049 ConcurrentGCThread::stsJoin();
ysr@345 1050
tonyp@1082 1051 assert((size_t) worker_i < _cm->active_tasks(), "invariant");
ysr@345 1052 CMTask* the_task = _cm->task(worker_i);
ysr@345 1053 the_task->record_start_time();
ysr@345 1054 if (!_cm->has_aborted()) {
ysr@345 1055 do {
ysr@345 1056 double start_vtime_sec = os::elapsedVTime();
ysr@345 1057 double start_time_sec = os::elapsedTime();
johnc@2187 1058 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
johnc@2187 1059
johnc@2187 1060 the_task->do_marking_step(mark_step_duration_ms,
johnc@2187 1061 true /* do_stealing */,
johnc@2187 1062 true /* do_termination */);
johnc@2187 1063
ysr@345 1064 double end_time_sec = os::elapsedTime();
ysr@345 1065 double end_vtime_sec = os::elapsedVTime();
ysr@345 1066 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
ysr@345 1067 double elapsed_time_sec = end_time_sec - start_time_sec;
ysr@345 1068 _cm->clear_has_overflown();
ysr@345 1069
ysr@345 1070 bool ret = _cm->do_yield_check(worker_i);
ysr@345 1071
ysr@345 1072 jlong sleep_time_ms;
ysr@345 1073 if (!_cm->has_aborted() && the_task->has_aborted()) {
ysr@345 1074 sleep_time_ms =
ysr@345 1075 (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0);
ysr@345 1076 ConcurrentGCThread::stsLeave();
ysr@345 1077 os::sleep(Thread::current(), sleep_time_ms, false);
ysr@345 1078 ConcurrentGCThread::stsJoin();
ysr@345 1079 }
ysr@345 1080 double end_time2_sec = os::elapsedTime();
ysr@345 1081 double elapsed_time2_sec = end_time2_sec - start_time_sec;
ysr@345 1082
ysr@345 1083 #if 0
ysr@345 1084 gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, "
ysr@345 1085 "overhead %1.4lf",
ysr@345 1086 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
ysr@345 1087 the_task->conc_overhead(os::elapsedTime()) * 8.0);
ysr@345 1088 gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms",
ysr@345 1089 elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0);
ysr@345 1090 #endif
ysr@345 1091 } while (!_cm->has_aborted() && the_task->has_aborted());
ysr@345 1092 }
ysr@345 1093 the_task->record_end_time();
tonyp@1082 1094 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant");
ysr@345 1095
ysr@345 1096 ConcurrentGCThread::stsLeave();
ysr@345 1097
ysr@345 1098 double end_vtime = os::elapsedVTime();
ysr@345 1099 _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime);
ysr@345 1100 }
ysr@345 1101
ysr@345 1102 CMConcurrentMarkingTask(ConcurrentMark* cm,
ysr@345 1103 ConcurrentMarkThread* cmt) :
ysr@345 1104 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
ysr@345 1105
ysr@345 1106 ~CMConcurrentMarkingTask() { }
ysr@345 1107 };
ysr@345 1108
ysr@345 1109 void ConcurrentMark::markFromRoots() {
ysr@345 1110 // we might be tempted to assert that:
ysr@345 1111 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
ysr@345 1112 // "inconsistent argument?");
ysr@345 1113 // However that wouldn't be right, because it's possible that
ysr@345 1114 // a safepoint is indeed in progress as a younger generation
ysr@345 1115 // stop-the-world GC happens even as we mark in this generation.
ysr@345 1116
ysr@345 1117 _restart_for_overflow = false;
ysr@345 1118
johnc@2187 1119 size_t active_workers = MAX2((size_t) 1, parallel_marking_threads());
johnc@2187 1120 set_phase(active_workers, true /* concurrent */);
ysr@345 1121
ysr@345 1122 CMConcurrentMarkingTask markingTask(this, cmThread());
ysr@345 1123 if (parallel_marking_threads() > 0)
ysr@345 1124 _parallel_workers->run_task(&markingTask);
ysr@345 1125 else
ysr@345 1126 markingTask.work(0);
ysr@345 1127 print_stats();
ysr@345 1128 }
ysr@345 1129
ysr@345 1130 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
ysr@345 1131 // world is stopped at this checkpoint
ysr@345 1132 assert(SafepointSynchronize::is_at_safepoint(),
ysr@345 1133 "world should be stopped");
ysr@345 1134 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 1135
ysr@345 1136 // If a full collection has happened, we shouldn't do this.
ysr@345 1137 if (has_aborted()) {
ysr@345 1138 g1h->set_marking_complete(); // So bitmap clearing isn't confused
ysr@345 1139 return;
ysr@345 1140 }
ysr@345 1141
kamg@2138 1142 SvcGCMarker sgcm(SvcGCMarker::OTHER);
kamg@2138 1143
ysr@896 1144 if (VerifyDuringGC) {
ysr@896 1145 HandleMark hm; // handle scope
ysr@896 1146 gclog_or_tty->print(" VerifyDuringGC:(before)");
ysr@896 1147 Universe::heap()->prepare_for_verify();
ysr@896 1148 Universe::verify(true, false, true);
ysr@896 1149 }
ysr@896 1150
ysr@345 1151 G1CollectorPolicy* g1p = g1h->g1_policy();
ysr@345 1152 g1p->record_concurrent_mark_remark_start();
ysr@345 1153
ysr@345 1154 double start = os::elapsedTime();
ysr@345 1155
ysr@345 1156 checkpointRootsFinalWork();
ysr@345 1157
ysr@345 1158 double mark_work_end = os::elapsedTime();
ysr@345 1159
ysr@345 1160 weakRefsWork(clear_all_soft_refs);
ysr@345 1161
ysr@345 1162 if (has_overflown()) {
ysr@345 1163 // Oops. We overflowed. Restart concurrent marking.
ysr@345 1164 _restart_for_overflow = true;
ysr@345 1165 // Clear the flag. We do not need it any more.
ysr@345 1166 clear_has_overflown();
ysr@345 1167 if (G1TraceMarkStackOverflow)
ysr@345 1168 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
ysr@345 1169 } else {
tonyp@2162 1170 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
ysr@345 1171 // We're done with marking.
tonyp@1404 1172 // This is the end of the marking cycle, we're expected all
tonyp@1404 1173 // threads to have SATB queues with active set to true.
tonyp@2162 1174 satb_mq_set.set_active_all_threads(false, /* new active value */
tonyp@2162 1175 true /* expected_active */);
tonyp@860 1176
tonyp@860 1177 if (VerifyDuringGC) {
ysr@896 1178 HandleMark hm; // handle scope
ysr@896 1179 gclog_or_tty->print(" VerifyDuringGC:(after)");
ysr@896 1180 Universe::heap()->prepare_for_verify();
ysr@896 1181 Universe::heap()->verify(/* allow_dirty */ true,
ysr@896 1182 /* silent */ false,
ysr@896 1183 /* use_prev_marking */ false);
tonyp@860 1184 }
johnc@2187 1185 assert(!restart_for_overflow(), "sanity");
johnc@2187 1186 }
johnc@2187 1187
johnc@2187 1188 // Reset the marking state if marking completed
johnc@2187 1189 if (!restart_for_overflow()) {
johnc@2187 1190 set_non_marking_state();
ysr@345 1191 }
ysr@345 1192
ysr@345 1193 #if VERIFY_OBJS_PROCESSED
ysr@345 1194 _scan_obj_cl.objs_processed = 0;
ysr@345 1195 ThreadLocalObjQueue::objs_enqueued = 0;
ysr@345 1196 #endif
ysr@345 1197
ysr@345 1198 // Statistics
ysr@345 1199 double now = os::elapsedTime();
ysr@345 1200 _remark_mark_times.add((mark_work_end - start) * 1000.0);
ysr@345 1201 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
ysr@345 1202 _remark_times.add((now - start) * 1000.0);
ysr@345 1203
ysr@345 1204 g1p->record_concurrent_mark_remark_end();
ysr@345 1205 }
ysr@345 1206
ysr@345 1207
ysr@345 1208 #define CARD_BM_TEST_MODE 0
ysr@345 1209
ysr@345 1210 class CalcLiveObjectsClosure: public HeapRegionClosure {
ysr@345 1211
ysr@345 1212 CMBitMapRO* _bm;
ysr@345 1213 ConcurrentMark* _cm;
ysr@345 1214 bool _changed;
ysr@345 1215 bool _yield;
ysr@345 1216 size_t _words_done;
ysr@345 1217 size_t _tot_live;
ysr@345 1218 size_t _tot_used;
ysr@345 1219 size_t _regions_done;
ysr@345 1220 double _start_vtime_sec;
ysr@345 1221
ysr@345 1222 BitMap* _region_bm;
ysr@345 1223 BitMap* _card_bm;
ysr@345 1224 intptr_t _bottom_card_num;
ysr@345 1225 bool _final;
ysr@345 1226
ysr@345 1227 void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
ysr@345 1228 for (intptr_t i = start_card_num; i <= last_card_num; i++) {
ysr@345 1229 #if CARD_BM_TEST_MODE
tonyp@1082 1230 guarantee(_card_bm->at(i - _bottom_card_num), "Should already be set.");
ysr@345 1231 #else
ysr@345 1232 _card_bm->par_at_put(i - _bottom_card_num, 1);
ysr@345 1233 #endif
ysr@345 1234 }
ysr@345 1235 }
ysr@345 1236
ysr@345 1237 public:
ysr@345 1238 CalcLiveObjectsClosure(bool final,
ysr@345 1239 CMBitMapRO *bm, ConcurrentMark *cm,
tonyp@990 1240 BitMap* region_bm, BitMap* card_bm) :
ysr@345 1241 _bm(bm), _cm(cm), _changed(false), _yield(true),
ysr@345 1242 _words_done(0), _tot_live(0), _tot_used(0),
tonyp@990 1243 _region_bm(region_bm), _card_bm(card_bm),_final(final),
ysr@345 1244 _regions_done(0), _start_vtime_sec(0.0)
ysr@345 1245 {
ysr@345 1246 _bottom_card_num =
ysr@345 1247 intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
ysr@345 1248 CardTableModRefBS::card_shift);
ysr@345 1249 }
ysr@345 1250
tonyp@878 1251 // It takes a region that's not empty (i.e., it has at least one
tonyp@878 1252 // live object in it and sets its corresponding bit on the region
tonyp@878 1253 // bitmap to 1. If the region is "starts humongous" it will also set
tonyp@878 1254 // to 1 the bits on the region bitmap that correspond to its
tonyp@878 1255 // associated "continues humongous" regions.
tonyp@878 1256 void set_bit_for_region(HeapRegion* hr) {
tonyp@878 1257 assert(!hr->continuesHumongous(), "should have filtered those out");
tonyp@878 1258
tonyp@878 1259 size_t index = hr->hrs_index();
tonyp@878 1260 if (!hr->startsHumongous()) {
tonyp@878 1261 // Normal (non-humongous) case: just set the bit.
tonyp@878 1262 _region_bm->par_at_put((BitMap::idx_t) index, true);
tonyp@878 1263 } else {
tonyp@878 1264 // Starts humongous case: calculate how many regions are part of
tonyp@878 1265 // this humongous region and then set the bit range. It might
tonyp@878 1266 // have been a bit more efficient to look at the object that
tonyp@878 1267 // spans these humongous regions to calculate their number from
tonyp@878 1268 // the object's size. However, it's a good idea to calculate
tonyp@878 1269 // this based on the metadata itself, and not the region
tonyp@878 1270 // contents, so that this code is not aware of what goes into
tonyp@878 1271 // the humongous regions (in case this changes in the future).
tonyp@878 1272 G1CollectedHeap* g1h = G1CollectedHeap::heap();
tonyp@878 1273 size_t end_index = index + 1;
tonyp@880 1274 while (end_index < g1h->n_regions()) {
tonyp@880 1275 HeapRegion* chr = g1h->region_at(end_index);
tonyp@878 1276 if (!chr->continuesHumongous()) {
tonyp@878 1277 break;
tonyp@878 1278 }
tonyp@878 1279 end_index += 1;
tonyp@878 1280 }
tonyp@878 1281 _region_bm->par_at_put_range((BitMap::idx_t) index,
tonyp@878 1282 (BitMap::idx_t) end_index, true);
tonyp@878 1283 }
tonyp@878 1284 }
tonyp@878 1285
ysr@345 1286 bool doHeapRegion(HeapRegion* hr) {
ysr@345 1287 if (!_final && _regions_done == 0)
ysr@345 1288 _start_vtime_sec = os::elapsedVTime();
ysr@345 1289
iveresov@677 1290 if (hr->continuesHumongous()) {
tonyp@878 1291 // We will ignore these here and process them when their
tonyp@878 1292 // associated "starts humongous" region is processed (see
tonyp@878 1293 // set_bit_for_heap_region()). Note that we cannot rely on their
tonyp@878 1294 // associated "starts humongous" region to have their bit set to
tonyp@878 1295 // 1 since, due to the region chunking in the parallel region
tonyp@878 1296 // iteration, a "continues humongous" region might be visited
tonyp@878 1297 // before its associated "starts humongous".
iveresov@677 1298 return false;
iveresov@677 1299 }
ysr@345 1300
ysr@345 1301 HeapWord* nextTop = hr->next_top_at_mark_start();
ysr@345 1302 HeapWord* start = hr->top_at_conc_mark_count();
ysr@345 1303 assert(hr->bottom() <= start && start <= hr->end() &&
ysr@345 1304 hr->bottom() <= nextTop && nextTop <= hr->end() &&
ysr@345 1305 start <= nextTop,
ysr@345 1306 "Preconditions.");
ysr@345 1307 // Otherwise, record the number of word's we'll examine.
ysr@345 1308 size_t words_done = (nextTop - start);
ysr@345 1309 // Find the first marked object at or after "start".
ysr@345 1310 start = _bm->getNextMarkedWordAddress(start, nextTop);
ysr@345 1311 size_t marked_bytes = 0;
ysr@345 1312
ysr@345 1313 // Below, the term "card num" means the result of shifting an address
ysr@345 1314 // by the card shift -- address 0 corresponds to card number 0. One
ysr@345 1315 // must subtract the card num of the bottom of the heap to obtain a
ysr@345 1316 // card table index.
ysr@345 1317 // The first card num of the sequence of live cards currently being
ysr@345 1318 // constructed. -1 ==> no sequence.
ysr@345 1319 intptr_t start_card_num = -1;
ysr@345 1320 // The last card num of the sequence of live cards currently being
ysr@345 1321 // constructed. -1 ==> no sequence.
ysr@345 1322 intptr_t last_card_num = -1;
ysr@345 1323
ysr@345 1324 while (start < nextTop) {
ysr@345 1325 if (_yield && _cm->do_yield_check()) {
ysr@345 1326 // We yielded. It might be for a full collection, in which case
ysr@345 1327 // all bets are off; terminate the traversal.
ysr@345 1328 if (_cm->has_aborted()) {
ysr@345 1329 _changed = false;
ysr@345 1330 return true;
ysr@345 1331 } else {
ysr@345 1332 // Otherwise, it might be a collection pause, and the region
ysr@345 1333 // we're looking at might be in the collection set. We'll
ysr@345 1334 // abandon this region.
ysr@345 1335 return false;
ysr@345 1336 }
ysr@345 1337 }
ysr@345 1338 oop obj = oop(start);
ysr@345 1339 int obj_sz = obj->size();
ysr@345 1340 // The card num of the start of the current object.
ysr@345 1341 intptr_t obj_card_num =
ysr@345 1342 intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
ysr@345 1343
ysr@345 1344 HeapWord* obj_last = start + obj_sz - 1;
ysr@345 1345 intptr_t obj_last_card_num =
ysr@345 1346 intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
ysr@345 1347
ysr@345 1348 if (obj_card_num != last_card_num) {
ysr@345 1349 if (start_card_num == -1) {
ysr@345 1350 assert(last_card_num == -1, "Both or neither.");
ysr@345 1351 start_card_num = obj_card_num;
ysr@345 1352 } else {
ysr@345 1353 assert(last_card_num != -1, "Both or neither.");
ysr@345 1354 assert(obj_card_num >= last_card_num, "Inv");
ysr@345 1355 if ((obj_card_num - last_card_num) > 1) {
ysr@345 1356 // Mark the last run, and start a new one.
ysr@345 1357 mark_card_num_range(start_card_num, last_card_num);
ysr@345 1358 start_card_num = obj_card_num;
ysr@345 1359 }
ysr@345 1360 }
ysr@345 1361 #if CARD_BM_TEST_MODE
ysr@345 1362 /*
ysr@345 1363 gclog_or_tty->print_cr("Setting bits from %d/%d.",
ysr@345 1364 obj_card_num - _bottom_card_num,
ysr@345 1365 obj_last_card_num - _bottom_card_num);
ysr@345 1366 */
ysr@345 1367 for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
ysr@345 1368 _card_bm->par_at_put(j - _bottom_card_num, 1);
ysr@345 1369 }
ysr@345 1370 #endif
ysr@345 1371 }
ysr@345 1372 // In any case, we set the last card num.
ysr@345 1373 last_card_num = obj_last_card_num;
ysr@345 1374
apetrusenko@1089 1375 marked_bytes += (size_t)obj_sz * HeapWordSize;
ysr@345 1376 // Find the next marked object after this one.
ysr@345 1377 start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
ysr@345 1378 _changed = true;
ysr@345 1379 }
ysr@345 1380 // Handle the last range, if any.
ysr@345 1381 if (start_card_num != -1)
ysr@345 1382 mark_card_num_range(start_card_num, last_card_num);
ysr@345 1383 if (_final) {
ysr@345 1384 // Mark the allocated-since-marking portion...
ysr@345 1385 HeapWord* tp = hr->top();
ysr@345 1386 if (nextTop < tp) {
ysr@345 1387 start_card_num =
ysr@345 1388 intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
ysr@345 1389 last_card_num =
ysr@345 1390 intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
ysr@345 1391 mark_card_num_range(start_card_num, last_card_num);
ysr@345 1392 // This definitely means the region has live objects.
tonyp@878 1393 set_bit_for_region(hr);
ysr@345 1394 }
ysr@345 1395 }
ysr@345 1396
ysr@345 1397 hr->add_to_marked_bytes(marked_bytes);
ysr@345 1398 // Update the live region bitmap.
ysr@345 1399 if (marked_bytes > 0) {
tonyp@878 1400 set_bit_for_region(hr);
ysr@345 1401 }
ysr@345 1402 hr->set_top_at_conc_mark_count(nextTop);
ysr@345 1403 _tot_live += hr->next_live_bytes();
ysr@345 1404 _tot_used += hr->used();
ysr@345 1405 _words_done = words_done;
ysr@345 1406
ysr@345 1407 if (!_final) {
ysr@345 1408 ++_regions_done;
ysr@345 1409 if (_regions_done % 10 == 0) {
ysr@345 1410 double end_vtime_sec = os::elapsedVTime();
ysr@345 1411 double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
ysr@345 1412 if (elapsed_vtime_sec > (10.0 / 1000.0)) {
ysr@345 1413 jlong sleep_time_ms =
ysr@345 1414 (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
ysr@345 1415 os::sleep(Thread::current(), sleep_time_ms, false);
ysr@345 1416 _start_vtime_sec = end_vtime_sec;
ysr@345 1417 }
ysr@345 1418 }
ysr@345 1419 }
ysr@345 1420
ysr@345 1421 return false;
ysr@345 1422 }
ysr@345 1423
ysr@345 1424 bool changed() { return _changed; }
ysr@345 1425 void reset() { _changed = false; _words_done = 0; }
ysr@345 1426 void no_yield() { _yield = false; }
ysr@345 1427 size_t words_done() { return _words_done; }
ysr@345 1428 size_t tot_live() { return _tot_live; }
ysr@345 1429 size_t tot_used() { return _tot_used; }
ysr@345 1430 };
ysr@345 1431
ysr@345 1432
ysr@345 1433 void ConcurrentMark::calcDesiredRegions() {
ysr@345 1434 _region_bm.clear();
ysr@345 1435 _card_bm.clear();
ysr@345 1436 CalcLiveObjectsClosure calccl(false /*final*/,
ysr@345 1437 nextMarkBitMap(), this,
tonyp@990 1438 &_region_bm, &_card_bm);
ysr@345 1439 G1CollectedHeap *g1h = G1CollectedHeap::heap();
ysr@345 1440 g1h->heap_region_iterate(&calccl);
ysr@345 1441
ysr@345 1442 do {
ysr@345 1443 calccl.reset();
ysr@345 1444 g1h->heap_region_iterate(&calccl);
ysr@345 1445 } while (calccl.changed());
ysr@345 1446 }
ysr@345 1447
ysr@345 1448 class G1ParFinalCountTask: public AbstractGangTask {
ysr@345 1449 protected:
ysr@345 1450 G1CollectedHeap* _g1h;
ysr@345 1451 CMBitMap* _bm;
ysr@345 1452 size_t _n_workers;
ysr@345 1453 size_t *_live_bytes;
ysr@345 1454 size_t *_used_bytes;
ysr@345 1455 BitMap* _region_bm;
ysr@345 1456 BitMap* _card_bm;
ysr@345 1457 public:
ysr@345 1458 G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
ysr@345 1459 BitMap* region_bm, BitMap* card_bm) :
ysr@345 1460 AbstractGangTask("G1 final counting"), _g1h(g1h),
ysr@345 1461 _bm(bm), _region_bm(region_bm), _card_bm(card_bm)
ysr@345 1462 {
ysr@345 1463 if (ParallelGCThreads > 0)
ysr@345 1464 _n_workers = _g1h->workers()->total_workers();
ysr@345 1465 else
ysr@345 1466 _n_workers = 1;
ysr@345 1467 _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
ysr@345 1468 _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
ysr@345 1469 }
ysr@345 1470
ysr@345 1471 ~G1ParFinalCountTask() {
ysr@345 1472 FREE_C_HEAP_ARRAY(size_t, _live_bytes);
ysr@345 1473 FREE_C_HEAP_ARRAY(size_t, _used_bytes);
ysr@345 1474 }
ysr@345 1475
ysr@345 1476 void work(int i) {
ysr@345 1477 CalcLiveObjectsClosure calccl(true /*final*/,
ysr@345 1478 _bm, _g1h->concurrent_mark(),
tonyp@990 1479 _region_bm, _card_bm);
ysr@345 1480 calccl.no_yield();
jmasa@1855 1481 if (G1CollectedHeap::use_parallel_gc_threads()) {
tonyp@358 1482 _g1h->heap_region_par_iterate_chunked(&calccl, i,
tonyp@358 1483 HeapRegion::FinalCountClaimValue);
ysr@345 1484 } else {
ysr@345 1485 _g1h->heap_region_iterate(&calccl);
ysr@345 1486 }
ysr@345 1487 assert(calccl.complete(), "Shouldn't have yielded!");
ysr@345 1488
tonyp@1082 1489 assert((size_t) i < _n_workers, "invariant");
ysr@345 1490 _live_bytes[i] = calccl.tot_live();
ysr@345 1491 _used_bytes[i] = calccl.tot_used();
ysr@345 1492 }
ysr@345 1493 size_t live_bytes() {
ysr@345 1494 size_t live_bytes = 0;
ysr@345 1495 for (size_t i = 0; i < _n_workers; ++i)
ysr@345 1496 live_bytes += _live_bytes[i];
ysr@345 1497 return live_bytes;
ysr@345 1498 }
ysr@345 1499 size_t used_bytes() {
ysr@345 1500 size_t used_bytes = 0;
ysr@345 1501 for (size_t i = 0; i < _n_workers; ++i)
ysr@345 1502 used_bytes += _used_bytes[i];
ysr@345 1503 return used_bytes;
ysr@345 1504 }
ysr@345 1505 };
ysr@345 1506
ysr@345 1507 class G1ParNoteEndTask;
ysr@345 1508
ysr@345 1509 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
ysr@345 1510 G1CollectedHeap* _g1;
ysr@345 1511 int _worker_num;
ysr@345 1512 size_t _max_live_bytes;
ysr@345 1513 size_t _regions_claimed;
ysr@345 1514 size_t _freed_bytes;
tonyp@2165 1515 FreeRegionList _local_cleanup_list;
tonyp@2165 1516 HumongousRegionSet _humongous_proxy_set;
ysr@345 1517 double _claimed_region_time;
ysr@345 1518 double _max_region_time;
ysr@345 1519
ysr@345 1520 public:
ysr@345 1521 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
ysr@345 1522 int worker_num);
ysr@345 1523 size_t freed_bytes() { return _freed_bytes; }
tonyp@2165 1524 FreeRegionList* local_cleanup_list() {
tonyp@2165 1525 return &_local_cleanup_list;
tonyp@2165 1526 }
tonyp@2165 1527 HumongousRegionSet* humongous_proxy_set() {
tonyp@2165 1528 return &_humongous_proxy_set;
ysr@345 1529 }
ysr@345 1530
ysr@345 1531 bool doHeapRegion(HeapRegion *r);
ysr@345 1532
ysr@345 1533 size_t max_live_bytes() { return _max_live_bytes; }
ysr@345 1534 size_t regions_claimed() { return _regions_claimed; }
ysr@345 1535 double claimed_region_time_sec() { return _claimed_region_time; }
ysr@345 1536 double max_region_time_sec() { return _max_region_time; }
ysr@345 1537 };
ysr@345 1538
ysr@345 1539 class G1ParNoteEndTask: public AbstractGangTask {
ysr@345 1540 friend class G1NoteEndOfConcMarkClosure;
tonyp@2165 1541
ysr@345 1542 protected:
ysr@345 1543 G1CollectedHeap* _g1h;
ysr@345 1544 size_t _max_live_bytes;
ysr@345 1545 size_t _freed_bytes;
tonyp@2165 1546 FreeRegionList* _cleanup_list;
tonyp@2165 1547
ysr@345 1548 public:
ysr@345 1549 G1ParNoteEndTask(G1CollectedHeap* g1h,
tonyp@2165 1550 FreeRegionList* cleanup_list) :
ysr@345 1551 AbstractGangTask("G1 note end"), _g1h(g1h),
tonyp@2165 1552 _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { }
ysr@345 1553
ysr@345 1554 void work(int i) {
ysr@345 1555 double start = os::elapsedTime();
tonyp@2165 1556 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i);
jmasa@1855 1557 if (G1CollectedHeap::use_parallel_gc_threads()) {
tonyp@358 1558 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
tonyp@358 1559 HeapRegion::NoteEndClaimValue);
ysr@345 1560 } else {
ysr@345 1561 _g1h->heap_region_iterate(&g1_note_end);
ysr@345 1562 }
ysr@345 1563 assert(g1_note_end.complete(), "Shouldn't have yielded!");
ysr@345 1564
tonyp@2165 1565 // Now update the lists
tonyp@2165 1566 _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(),
tonyp@2165 1567 NULL /* free_list */,
tonyp@2165 1568 g1_note_end.humongous_proxy_set(),
tonyp@2165 1569 true /* par */);
ysr@345 1570 {
ysr@345 1571 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
ysr@345 1572 _max_live_bytes += g1_note_end.max_live_bytes();
ysr@345 1573 _freed_bytes += g1_note_end.freed_bytes();
tonyp@2165 1574
tonyp@2165 1575 _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list());
tonyp@2165 1576 assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition");
ysr@345 1577 }
ysr@345 1578 double end = os::elapsedTime();
ysr@345 1579 if (G1PrintParCleanupStats) {
ysr@345 1580 gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
ysr@345 1581 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
ysr@345 1582 i, start, end, (end-start)*1000.0,
ysr@345 1583 g1_note_end.regions_claimed(),
ysr@345 1584 g1_note_end.claimed_region_time_sec()*1000.0,
ysr@345 1585 g1_note_end.max_region_time_sec()*1000.0);
ysr@345 1586 }
ysr@345 1587 }
ysr@345 1588 size_t max_live_bytes() { return _max_live_bytes; }
ysr@345 1589 size_t freed_bytes() { return _freed_bytes; }
ysr@345 1590 };
ysr@345 1591
ysr@345 1592 class G1ParScrubRemSetTask: public AbstractGangTask {
ysr@345 1593 protected:
ysr@345 1594 G1RemSet* _g1rs;
ysr@345 1595 BitMap* _region_bm;
ysr@345 1596 BitMap* _card_bm;
ysr@345 1597 public:
ysr@345 1598 G1ParScrubRemSetTask(G1CollectedHeap* g1h,
ysr@345 1599 BitMap* region_bm, BitMap* card_bm) :
ysr@345 1600 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()),
ysr@345 1601 _region_bm(region_bm), _card_bm(card_bm)
ysr@345 1602 {}
ysr@345 1603
ysr@345 1604 void work(int i) {
jmasa@1855 1605 if (G1CollectedHeap::use_parallel_gc_threads()) {
tonyp@358 1606 _g1rs->scrub_par(_region_bm, _card_bm, i,
tonyp@358 1607 HeapRegion::ScrubRemSetClaimValue);
ysr@345 1608 } else {
ysr@345 1609 _g1rs->scrub(_region_bm, _card_bm);
ysr@345 1610 }
ysr@345 1611 }
ysr@345 1612
ysr@345 1613 };
ysr@345 1614
ysr@345 1615 G1NoteEndOfConcMarkClosure::
ysr@345 1616 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
ysr@345 1617 int worker_num)
ysr@345 1618 : _g1(g1), _worker_num(worker_num),
ysr@345 1619 _max_live_bytes(0), _regions_claimed(0),
tonyp@2165 1620 _freed_bytes(0),
ysr@345 1621 _claimed_region_time(0.0), _max_region_time(0.0),
tonyp@2165 1622 _local_cleanup_list("Local Cleanup List"),
tonyp@2165 1623 _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { }
tonyp@2165 1624
tonyp@2165 1625 bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) {
ysr@345 1626 // We use a claim value of zero here because all regions
ysr@345 1627 // were claimed with value 1 in the FinalCount task.
tonyp@2165 1628 hr->reset_gc_time_stamp();
tonyp@2165 1629 if (!hr->continuesHumongous()) {
ysr@345 1630 double start = os::elapsedTime();
ysr@345 1631 _regions_claimed++;
tonyp@2165 1632 hr->note_end_of_marking();
tonyp@2165 1633 _max_live_bytes += hr->max_live_bytes();
tonyp@2165 1634 _g1->free_region_if_totally_empty(hr,
tonyp@2165 1635 &_freed_bytes,
tonyp@2165 1636 &_local_cleanup_list,
tonyp@2165 1637 &_humongous_proxy_set,
tonyp@2165 1638 true /* par */);
ysr@345 1639 double region_time = (os::elapsedTime() - start);
ysr@345 1640 _claimed_region_time += region_time;
ysr@345 1641 if (region_time > _max_region_time) _max_region_time = region_time;
ysr@345 1642 }
ysr@345 1643 return false;
ysr@345 1644 }
ysr@345 1645
ysr@345 1646 void ConcurrentMark::cleanup() {
ysr@345 1647 // world is stopped at this checkpoint
ysr@345 1648 assert(SafepointSynchronize::is_at_safepoint(),
ysr@345 1649 "world should be stopped");
ysr@345 1650 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 1651
ysr@345 1652 // If a full collection has happened, we shouldn't do this.
ysr@345 1653 if (has_aborted()) {
ysr@345 1654 g1h->set_marking_complete(); // So bitmap clearing isn't confused
ysr@345 1655 return;
ysr@345 1656 }
ysr@345 1657
tonyp@2165 1658 g1h->verify_region_sets_optional();
tonyp@2165 1659
ysr@896 1660 if (VerifyDuringGC) {
ysr@896 1661 HandleMark hm; // handle scope
ysr@896 1662 gclog_or_tty->print(" VerifyDuringGC:(before)");
ysr@896 1663 Universe::heap()->prepare_for_verify();
ysr@896 1664 Universe::verify(/* allow dirty */ true,
ysr@896 1665 /* silent */ false,
ysr@896 1666 /* prev marking */ true);
ysr@896 1667 }
ysr@896 1668
ysr@345 1669 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
ysr@345 1670 g1p->record_concurrent_mark_cleanup_start();
ysr@345 1671
ysr@345 1672 double start = os::elapsedTime();
ysr@345 1673
ysr@345 1674 // Do counting once more with the world stopped for good measure.
ysr@345 1675 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
ysr@345 1676 &_region_bm, &_card_bm);
jmasa@1855 1677 if (G1CollectedHeap::use_parallel_gc_threads()) {
tonyp@358 1678 assert(g1h->check_heap_region_claim_values(
tonyp@358 1679 HeapRegion::InitialClaimValue),
tonyp@358 1680 "sanity check");
tonyp@358 1681
ysr@345 1682 int n_workers = g1h->workers()->total_workers();
ysr@345 1683 g1h->set_par_threads(n_workers);
ysr@345 1684 g1h->workers()->run_task(&g1_par_count_task);
ysr@345 1685 g1h->set_par_threads(0);
tonyp@358 1686
tonyp@358 1687 assert(g1h->check_heap_region_claim_values(
tonyp@358 1688 HeapRegion::FinalCountClaimValue),
tonyp@358 1689 "sanity check");
ysr@345 1690 } else {
ysr@345 1691 g1_par_count_task.work(0);
ysr@345 1692 }
ysr@345 1693
ysr@345 1694 size_t known_garbage_bytes =
ysr@345 1695 g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
ysr@345 1696 #if 0
ysr@345 1697 gclog_or_tty->print_cr("used %1.2lf, live %1.2lf, garbage %1.2lf",
ysr@345 1698 (double) g1_par_count_task.used_bytes() / (double) (1024 * 1024),
ysr@345 1699 (double) g1_par_count_task.live_bytes() / (double) (1024 * 1024),
ysr@345 1700 (double) known_garbage_bytes / (double) (1024 * 1024));
ysr@345 1701 #endif // 0
ysr@345 1702 g1p->set_known_garbage_bytes(known_garbage_bytes);
ysr@345 1703
ysr@345 1704 size_t start_used_bytes = g1h->used();
ysr@345 1705 _at_least_one_mark_complete = true;
ysr@345 1706 g1h->set_marking_complete();
ysr@345 1707
ysr@345 1708 double count_end = os::elapsedTime();
ysr@345 1709 double this_final_counting_time = (count_end - start);
ysr@345 1710 if (G1PrintParCleanupStats) {
ysr@345 1711 gclog_or_tty->print_cr("Cleanup:");
ysr@345 1712 gclog_or_tty->print_cr(" Finalize counting: %8.3f ms",
ysr@345 1713 this_final_counting_time*1000.0);
ysr@345 1714 }
ysr@345 1715 _total_counting_time += this_final_counting_time;
ysr@345 1716
ysr@345 1717 // Install newly created mark bitMap as "prev".
ysr@345 1718 swapMarkBitMaps();
ysr@345 1719
ysr@345 1720 g1h->reset_gc_time_stamp();
ysr@345 1721
ysr@345 1722 // Note end of marking in all heap regions.
ysr@345 1723 double note_end_start = os::elapsedTime();
tonyp@2165 1724 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list);
jmasa@1855 1725 if (G1CollectedHeap::use_parallel_gc_threads()) {
ysr@345 1726 int n_workers = g1h->workers()->total_workers();
ysr@345 1727 g1h->set_par_threads(n_workers);
ysr@345 1728 g1h->workers()->run_task(&g1_par_note_end_task);
ysr@345 1729 g1h->set_par_threads(0);
tonyp@358 1730
tonyp@358 1731 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
tonyp@358 1732 "sanity check");
ysr@345 1733 } else {
ysr@345 1734 g1_par_note_end_task.work(0);
ysr@345 1735 }
tonyp@2165 1736
tonyp@2165 1737 if (!cleanup_list_is_empty()) {
tonyp@2165 1738 // The cleanup list is not empty, so we'll have to process it
tonyp@2165 1739 // concurrently. Notify anyone else that might be wanting free
tonyp@2165 1740 // regions that there will be more free regions coming soon.
tonyp@2165 1741 g1h->set_free_regions_coming();
tonyp@2165 1742 }
ysr@345 1743 double note_end_end = os::elapsedTime();
ysr@345 1744 if (G1PrintParCleanupStats) {
ysr@345 1745 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
ysr@345 1746 (note_end_end - note_end_start)*1000.0);
ysr@345 1747 }
ysr@345 1748
tonyp@358 1749
ysr@345 1750 // call below, since it affects the metric by which we sort the heap
ysr@345 1751 // regions.
ysr@345 1752 if (G1ScrubRemSets) {
ysr@345 1753 double rs_scrub_start = os::elapsedTime();
ysr@345 1754 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm);
jmasa@1855 1755 if (G1CollectedHeap::use_parallel_gc_threads()) {
ysr@345 1756 int n_workers = g1h->workers()->total_workers();
ysr@345 1757 g1h->set_par_threads(n_workers);
ysr@345 1758 g1h->workers()->run_task(&g1_par_scrub_rs_task);
ysr@345 1759 g1h->set_par_threads(0);
tonyp@358 1760
tonyp@358 1761 assert(g1h->check_heap_region_claim_values(
tonyp@358 1762 HeapRegion::ScrubRemSetClaimValue),
tonyp@358 1763 "sanity check");
ysr@345 1764 } else {
ysr@345 1765 g1_par_scrub_rs_task.work(0);
ysr@345 1766 }
ysr@345 1767
ysr@345 1768 double rs_scrub_end = os::elapsedTime();
ysr@345 1769 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
ysr@345 1770 _total_rs_scrub_time += this_rs_scrub_time;
ysr@345 1771 }
ysr@345 1772
ysr@345 1773 // this will also free any regions totally full of garbage objects,
ysr@345 1774 // and sort the regions.
ysr@345 1775 g1h->g1_policy()->record_concurrent_mark_cleanup_end(
ysr@345 1776 g1_par_note_end_task.freed_bytes(),
ysr@345 1777 g1_par_note_end_task.max_live_bytes());
ysr@345 1778
ysr@345 1779 // Statistics.
ysr@345 1780 double end = os::elapsedTime();
ysr@345 1781 _cleanup_times.add((end - start) * 1000.0);
ysr@345 1782
ysr@345 1783 // G1CollectedHeap::heap()->print();
ysr@345 1784 // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
ysr@345 1785 // G1CollectedHeap::heap()->get_gc_time_stamp());
ysr@345 1786
ysr@345 1787 if (PrintGC || PrintGCDetails) {
ysr@345 1788 g1h->print_size_transition(gclog_or_tty,
ysr@345 1789 start_used_bytes,
ysr@345 1790 g1h->used(),
ysr@345 1791 g1h->capacity());
ysr@345 1792 }
ysr@345 1793
ysr@345 1794 size_t cleaned_up_bytes = start_used_bytes - g1h->used();
ysr@345 1795 g1p->decrease_known_garbage_bytes(cleaned_up_bytes);
ysr@345 1796
ysr@345 1797 // We need to make this be a "collection" so any collection pause that
ysr@345 1798 // races with it goes around and waits for completeCleanup to finish.
ysr@345 1799 g1h->increment_total_collections();
ysr@345 1800
johnc@791 1801 if (VerifyDuringGC) {
ysr@896 1802 HandleMark hm; // handle scope
ysr@896 1803 gclog_or_tty->print(" VerifyDuringGC:(after)");
ysr@896 1804 Universe::heap()->prepare_for_verify();
ysr@896 1805 Universe::verify(/* allow dirty */ true,
ysr@896 1806 /* silent */ false,
ysr@896 1807 /* prev marking */ true);
ysr@345 1808 }
tonyp@2165 1809
tonyp@2165 1810 g1h->verify_region_sets_optional();
ysr@345 1811 }
ysr@345 1812
ysr@345 1813 void ConcurrentMark::completeCleanup() {
ysr@345 1814 if (has_aborted()) return;
ysr@345 1815
tonyp@2165 1816 G1CollectedHeap* g1h = G1CollectedHeap::heap();
tonyp@2165 1817
tonyp@2165 1818 _cleanup_list.verify_optional();
tonyp@2165 1819 FreeRegionList local_free_list("Local Cleanup List");
tonyp@2165 1820
tonyp@2165 1821 if (G1ConcRegionFreeingVerbose) {
tonyp@2165 1822 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
tonyp@2165 1823 "cleanup list has "SIZE_FORMAT" entries",
tonyp@2165 1824 _cleanup_list.length());
tonyp@2165 1825 }
tonyp@2165 1826
tonyp@2165 1827 // Noone else should be accessing the _cleanup_list at this point,
tonyp@2165 1828 // so it's not necessary to take any locks
tonyp@2165 1829 while (!_cleanup_list.is_empty()) {
tonyp@2165 1830 HeapRegion* hr = _cleanup_list.remove_head();
tonyp@2165 1831 assert(hr != NULL, "the list was not empty");
tonyp@2165 1832 hr->rem_set()->clear();
tonyp@2165 1833 local_free_list.add_as_tail(hr);
tonyp@2165 1834
tonyp@2165 1835 // Instead of adding one region at a time to the secondary_free_list,
tonyp@2165 1836 // we accumulate them in the local list and move them a few at a
tonyp@2165 1837 // time. This also cuts down on the number of notify_all() calls
tonyp@2165 1838 // we do during this process. We'll also append the local list when
tonyp@2165 1839 // _cleanup_list is empty (which means we just removed the last
tonyp@2165 1840 // region from the _cleanup_list).
tonyp@2165 1841 if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
tonyp@2165 1842 _cleanup_list.is_empty()) {
tonyp@2165 1843 if (G1ConcRegionFreeingVerbose) {
tonyp@2165 1844 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
tonyp@2165 1845 "appending "SIZE_FORMAT" entries to the "
tonyp@2165 1846 "secondary_free_list, clean list still has "
tonyp@2165 1847 SIZE_FORMAT" entries",
tonyp@2165 1848 local_free_list.length(),
tonyp@2165 1849 _cleanup_list.length());
ysr@345 1850 }
tonyp@2165 1851
tonyp@2165 1852 {
tonyp@2165 1853 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
tonyp@2165 1854 g1h->secondary_free_list_add_as_tail(&local_free_list);
tonyp@2165 1855 SecondaryFreeList_lock->notify_all();
tonyp@2165 1856 }
tonyp@2165 1857
tonyp@2165 1858 if (G1StressConcRegionFreeing) {
tonyp@2165 1859 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
tonyp@2165 1860 os::sleep(Thread::current(), (jlong) 1, false);
tonyp@2165 1861 }
tonyp@2165 1862 }
ysr@345 1863 }
ysr@345 1864 }
tonyp@2165 1865 assert(local_free_list.is_empty(), "post-condition");
ysr@345 1866 }
ysr@345 1867
johnc@2187 1868 // Support closures for reference procssing in G1
johnc@2187 1869
johnc@2057 1870 bool G1CMIsAliveClosure::do_object_b(oop obj) {
johnc@2057 1871 HeapWord* addr = (HeapWord*)obj;
johnc@2057 1872 return addr != NULL &&
johnc@2057 1873 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
johnc@2057 1874 }
ysr@345 1875
ysr@345 1876 class G1CMKeepAliveClosure: public OopClosure {
ysr@345 1877 G1CollectedHeap* _g1;
ysr@345 1878 ConcurrentMark* _cm;
ysr@345 1879 CMBitMap* _bitMap;
ysr@345 1880 public:
ysr@345 1881 G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
ysr@345 1882 CMBitMap* bitMap) :
ysr@345 1883 _g1(g1), _cm(cm),
ysr@345 1884 _bitMap(bitMap) {}
ysr@345 1885
ysr@896 1886 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
ysr@896 1887 virtual void do_oop( oop* p) { do_oop_work(p); }
ysr@896 1888
ysr@896 1889 template <class T> void do_oop_work(T* p) {
johnc@2187 1890 oop obj = oopDesc::load_decode_heap_oop(p);
johnc@2187 1891 HeapWord* addr = (HeapWord*)obj;
johnc@2187 1892
johnc@2187 1893 if (_cm->verbose_high())
johnc@2187 1894 gclog_or_tty->print_cr("\t[0] we're looking at location "
johnc@2187 1895 "*"PTR_FORMAT" = "PTR_FORMAT,
johnc@2187 1896 p, (void*) obj);
johnc@2187 1897
johnc@2187 1898 if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) {
ysr@345 1899 _bitMap->mark(addr);
johnc@2187 1900 _cm->mark_stack_push(obj);
ysr@345 1901 }
ysr@345 1902 }
ysr@345 1903 };
ysr@345 1904
ysr@345 1905 class G1CMDrainMarkingStackClosure: public VoidClosure {
ysr@345 1906 CMMarkStack* _markStack;
ysr@345 1907 CMBitMap* _bitMap;
ysr@345 1908 G1CMKeepAliveClosure* _oopClosure;
ysr@345 1909 public:
ysr@345 1910 G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
ysr@345 1911 G1CMKeepAliveClosure* oopClosure) :
ysr@345 1912 _bitMap(bitMap),
ysr@345 1913 _markStack(markStack),
ysr@345 1914 _oopClosure(oopClosure)
ysr@345 1915 {}
ysr@345 1916
ysr@345 1917 void do_void() {
ysr@345 1918 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
ysr@345 1919 }
ysr@345 1920 };
ysr@345 1921
johnc@2187 1922 // 'Keep Alive' closure used by parallel reference processing.
johnc@2187 1923 // An instance of this closure is used in the parallel reference processing
johnc@2187 1924 // code rather than an instance of G1CMKeepAliveClosure. We could have used
johnc@2187 1925 // the G1CMKeepAliveClosure as it is MT-safe. Also reference objects are
johnc@2187 1926 // placed on to discovered ref lists once so we can mark and push with no
johnc@2187 1927 // need to check whether the object has already been marked. Using the
johnc@2187 1928 // G1CMKeepAliveClosure would mean, however, having all the worker threads
johnc@2187 1929 // operating on the global mark stack. This means that an individual
johnc@2187 1930 // worker would be doing lock-free pushes while it processes its own
johnc@2187 1931 // discovered ref list followed by drain call. If the discovered ref lists
johnc@2187 1932 // are unbalanced then this could cause interference with the other
johnc@2187 1933 // workers. Using a CMTask (and its embedded local data structures)
johnc@2187 1934 // avoids that potential interference.
johnc@2187 1935 class G1CMParKeepAliveAndDrainClosure: public OopClosure {
johnc@2187 1936 ConcurrentMark* _cm;
johnc@2187 1937 CMTask* _task;
johnc@2187 1938 CMBitMap* _bitMap;
johnc@2187 1939 int _ref_counter_limit;
johnc@2187 1940 int _ref_counter;
johnc@2187 1941 public:
johnc@2187 1942 G1CMParKeepAliveAndDrainClosure(ConcurrentMark* cm,
johnc@2187 1943 CMTask* task,
johnc@2187 1944 CMBitMap* bitMap) :
johnc@2187 1945 _cm(cm), _task(task), _bitMap(bitMap),
johnc@2187 1946 _ref_counter_limit(G1RefProcDrainInterval)
johnc@2187 1947 {
johnc@2187 1948 assert(_ref_counter_limit > 0, "sanity");
johnc@2187 1949 _ref_counter = _ref_counter_limit;
johnc@2187 1950 }
johnc@2187 1951
johnc@2187 1952 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
johnc@2187 1953 virtual void do_oop( oop* p) { do_oop_work(p); }
johnc@2187 1954
johnc@2187 1955 template <class T> void do_oop_work(T* p) {
johnc@2187 1956 if (!_cm->has_overflown()) {
johnc@2187 1957 oop obj = oopDesc::load_decode_heap_oop(p);
johnc@2187 1958 if (_cm->verbose_high())
johnc@2187 1959 gclog_or_tty->print_cr("\t[%d] we're looking at location "
johnc@2187 1960 "*"PTR_FORMAT" = "PTR_FORMAT,
johnc@2187 1961 _task->task_id(), p, (void*) obj);
johnc@2187 1962
johnc@2187 1963 _task->deal_with_reference(obj);
johnc@2187 1964 _ref_counter--;
johnc@2187 1965
johnc@2187 1966 if (_ref_counter == 0) {
johnc@2187 1967 // We have dealt with _ref_counter_limit references, pushing them and objects
johnc@2187 1968 // reachable from them on to the local stack (and possibly the global stack).
johnc@2187 1969 // Call do_marking_step() to process these entries. We call the routine in a
johnc@2187 1970 // loop, which we'll exit if there's nothing more to do (i.e. we're done
johnc@2187 1971 // with the entries that we've pushed as a result of the deal_with_reference
johnc@2187 1972 // calls above) or we overflow.
johnc@2187 1973 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
johnc@2187 1974 // while there may still be some work to do. (See the comment at the
johnc@2187 1975 // beginning of CMTask::do_marking_step() for those conditions - one of which
johnc@2187 1976 // is reaching the specified time target.) It is only when
johnc@2187 1977 // CMTask::do_marking_step() returns without setting the has_aborted() flag
johnc@2187 1978 // that the marking has completed.
johnc@2187 1979 do {
johnc@2187 1980 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
johnc@2187 1981 _task->do_marking_step(mark_step_duration_ms,
johnc@2187 1982 false /* do_stealing */,
johnc@2187 1983 false /* do_termination */);
johnc@2187 1984 } while (_task->has_aborted() && !_cm->has_overflown());
johnc@2187 1985 _ref_counter = _ref_counter_limit;
johnc@2187 1986 }
johnc@2187 1987 } else {
johnc@2187 1988 if (_cm->verbose_high())
johnc@2187 1989 gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id());
johnc@2187 1990 }
johnc@2187 1991 }
johnc@2187 1992 };
johnc@2187 1993
johnc@2187 1994 class G1CMParDrainMarkingStackClosure: public VoidClosure {
johnc@2187 1995 ConcurrentMark* _cm;
johnc@2187 1996 CMTask* _task;
johnc@2187 1997 public:
johnc@2187 1998 G1CMParDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task) :
johnc@2187 1999 _cm(cm), _task(task)
johnc@2187 2000 {}
johnc@2187 2001
johnc@2187 2002 void do_void() {
johnc@2187 2003 do {
johnc@2187 2004 if (_cm->verbose_high())
johnc@2187 2005 gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", _task->task_id());
johnc@2187 2006
johnc@2187 2007 // We call CMTask::do_marking_step() to completely drain the local and
johnc@2187 2008 // global marking stacks. The routine is called in a loop, which we'll
johnc@2187 2009 // exit if there's nothing more to do (i.e. we'completely drained the
johnc@2187 2010 // entries that were pushed as a result of applying the
johnc@2187 2011 // G1CMParKeepAliveAndDrainClosure to the entries on the discovered ref
johnc@2187 2012 // lists above) or we overflow the global marking stack.
johnc@2187 2013 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
johnc@2187 2014 // while there may still be some work to do. (See the comment at the
johnc@2187 2015 // beginning of CMTask::do_marking_step() for those conditions - one of which
johnc@2187 2016 // is reaching the specified time target.) It is only when
johnc@2187 2017 // CMTask::do_marking_step() returns without setting the has_aborted() flag
johnc@2187 2018 // that the marking has completed.
johnc@2187 2019
johnc@2187 2020 _task->do_marking_step(1000000000.0 /* something very large */,
johnc@2187 2021 true /* do_stealing */,
johnc@2187 2022 true /* do_termination */);
johnc@2187 2023 } while (_task->has_aborted() && !_cm->has_overflown());
johnc@2187 2024 }
johnc@2187 2025 };
johnc@2187 2026
johnc@2187 2027 // Implementation of AbstractRefProcTaskExecutor for G1
johnc@2187 2028 class G1RefProcTaskExecutor: public AbstractRefProcTaskExecutor {
johnc@2187 2029 private:
johnc@2187 2030 G1CollectedHeap* _g1h;
johnc@2187 2031 ConcurrentMark* _cm;
johnc@2187 2032 CMBitMap* _bitmap;
johnc@2187 2033 WorkGang* _workers;
johnc@2187 2034 int _active_workers;
johnc@2187 2035
johnc@2187 2036 public:
johnc@2187 2037 G1RefProcTaskExecutor(G1CollectedHeap* g1h,
johnc@2187 2038 ConcurrentMark* cm,
johnc@2187 2039 CMBitMap* bitmap,
johnc@2187 2040 WorkGang* workers,
johnc@2187 2041 int n_workers) :
johnc@2187 2042 _g1h(g1h), _cm(cm), _bitmap(bitmap),
johnc@2187 2043 _workers(workers), _active_workers(n_workers)
johnc@2187 2044 { }
johnc@2187 2045
johnc@2187 2046 // Executes the given task using concurrent marking worker threads.
johnc@2187 2047 virtual void execute(ProcessTask& task);
johnc@2187 2048 virtual void execute(EnqueueTask& task);
johnc@2187 2049 };
johnc@2187 2050
johnc@2187 2051 class G1RefProcTaskProxy: public AbstractGangTask {
johnc@2187 2052 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
johnc@2187 2053 ProcessTask& _proc_task;
johnc@2187 2054 G1CollectedHeap* _g1h;
johnc@2187 2055 ConcurrentMark* _cm;
johnc@2187 2056 CMBitMap* _bitmap;
johnc@2187 2057
johnc@2187 2058 public:
johnc@2187 2059 G1RefProcTaskProxy(ProcessTask& proc_task,
johnc@2187 2060 G1CollectedHeap* g1h,
johnc@2187 2061 ConcurrentMark* cm,
johnc@2187 2062 CMBitMap* bitmap) :
johnc@2187 2063 AbstractGangTask("Process reference objects in parallel"),
johnc@2187 2064 _proc_task(proc_task), _g1h(g1h), _cm(cm), _bitmap(bitmap)
johnc@2187 2065 {}
johnc@2187 2066
johnc@2187 2067 virtual void work(int i) {
johnc@2187 2068 CMTask* marking_task = _cm->task(i);
johnc@2187 2069 G1CMIsAliveClosure g1_is_alive(_g1h);
johnc@2187 2070 G1CMParKeepAliveAndDrainClosure g1_par_keep_alive(_cm, marking_task, _bitmap);
johnc@2187 2071 G1CMParDrainMarkingStackClosure g1_par_drain(_cm, marking_task);
johnc@2187 2072
johnc@2187 2073 _proc_task.work(i, g1_is_alive, g1_par_keep_alive, g1_par_drain);
johnc@2187 2074 }
johnc@2187 2075 };
johnc@2187 2076
johnc@2187 2077 void G1RefProcTaskExecutor::execute(ProcessTask& proc_task) {
johnc@2187 2078 assert(_workers != NULL, "Need parallel worker threads.");
johnc@2187 2079
johnc@2187 2080 G1RefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap);
johnc@2187 2081
johnc@2187 2082 // We need to reset the phase for each task execution so that
johnc@2187 2083 // the termination protocol of CMTask::do_marking_step works.
johnc@2187 2084 _cm->set_phase(_active_workers, false /* concurrent */);
johnc@2187 2085 _g1h->set_par_threads(_active_workers);
johnc@2187 2086 _workers->run_task(&proc_task_proxy);
johnc@2187 2087 _g1h->set_par_threads(0);
johnc@2187 2088 }
johnc@2187 2089
johnc@2187 2090 class G1RefEnqueueTaskProxy: public AbstractGangTask {
johnc@2187 2091 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
johnc@2187 2092 EnqueueTask& _enq_task;
johnc@2187 2093
johnc@2187 2094 public:
johnc@2187 2095 G1RefEnqueueTaskProxy(EnqueueTask& enq_task) :
johnc@2187 2096 AbstractGangTask("Enqueue reference objects in parallel"),
johnc@2187 2097 _enq_task(enq_task)
johnc@2187 2098 { }
johnc@2187 2099
johnc@2187 2100 virtual void work(int i) {
johnc@2187 2101 _enq_task.work(i);
johnc@2187 2102 }
johnc@2187 2103 };
johnc@2187 2104
johnc@2187 2105 void G1RefProcTaskExecutor::execute(EnqueueTask& enq_task) {
johnc@2187 2106 assert(_workers != NULL, "Need parallel worker threads.");
johnc@2187 2107
johnc@2187 2108 G1RefEnqueueTaskProxy enq_task_proxy(enq_task);
johnc@2187 2109
johnc@2187 2110 _g1h->set_par_threads(_active_workers);
johnc@2187 2111 _workers->run_task(&enq_task_proxy);
johnc@2187 2112 _g1h->set_par_threads(0);
johnc@2187 2113 }
johnc@2187 2114
ysr@345 2115 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
ysr@345 2116 ResourceMark rm;
ysr@345 2117 HandleMark hm;
ysr@460 2118 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@460 2119 ReferenceProcessor* rp = g1h->ref_processor();
ysr@345 2120
johnc@1994 2121 // See the comment in G1CollectedHeap::ref_processing_init()
johnc@1994 2122 // about how reference processing currently works in G1.
johnc@1994 2123
ysr@345 2124 // Process weak references.
ysr@464 2125 rp->setup_policy(clear_all_soft_refs);
ysr@345 2126 assert(_markStack.isEmpty(), "mark stack should be empty");
ysr@345 2127
johnc@2057 2128 G1CMIsAliveClosure g1_is_alive(g1h);
johnc@2057 2129 G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap());
ysr@345 2130 G1CMDrainMarkingStackClosure
johnc@2057 2131 g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive);
ysr@345 2132
johnc@2187 2133 // We use the work gang from the G1CollectedHeap and we utilize all
johnc@2187 2134 // the worker threads.
johnc@2187 2135 int active_workers = MAX2(MIN2(g1h->workers()->total_workers(), (int)_max_task_num), 1);
johnc@2187 2136
johnc@2187 2137 G1RefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(),
johnc@2187 2138 g1h->workers(), active_workers);
johnc@2187 2139
johnc@2187 2140 if (rp->processing_is_mt()) {
johnc@2187 2141 // Set the degree of MT here. If the discovery is done MT, there
johnc@2187 2142 // may have been a different number of threads doing the discovery
johnc@2187 2143 // and a different number of discovered lists may have Ref objects.
johnc@2187 2144 // That is OK as long as the Reference lists are balanced (see
johnc@2187 2145 // balance_all_queues() and balance_queues()).
johnc@2187 2146 rp->set_mt_degree(active_workers);
johnc@2187 2147
johnc@2187 2148 rp->process_discovered_references(&g1_is_alive,
johnc@2187 2149 &g1_keep_alive,
johnc@2187 2150 &g1_drain_mark_stack,
johnc@2187 2151 &par_task_executor);
johnc@2187 2152
johnc@2187 2153 // The work routines of the parallel keep_alive and drain_marking_stack
johnc@2187 2154 // will set the has_overflown flag if we overflow the global marking
johnc@2187 2155 // stack.
johnc@2187 2156 } else {
johnc@2187 2157 rp->process_discovered_references(&g1_is_alive,
johnc@2187 2158 &g1_keep_alive,
johnc@2187 2159 &g1_drain_mark_stack,
johnc@2187 2160 NULL);
johnc@2187 2161
johnc@2187 2162 }
johnc@2187 2163
ysr@345 2164 assert(_markStack.overflow() || _markStack.isEmpty(),
johnc@2187 2165 "mark stack should be empty (unless it overflowed)");
ysr@345 2166 if (_markStack.overflow()) {
johnc@2187 2167 // Should have been done already when we tried to push an
johnc@2187 2168 // entry on to the global mark stack. But let's do it again.
ysr@345 2169 set_has_overflown();
ysr@345 2170 }
ysr@345 2171
johnc@2187 2172 if (rp->processing_is_mt()) {
johnc@2187 2173 assert(rp->num_q() == active_workers, "why not");
johnc@2187 2174 rp->enqueue_discovered_references(&par_task_executor);
johnc@2187 2175 } else {
johnc@2187 2176 rp->enqueue_discovered_references();
johnc@2187 2177 }
johnc@2187 2178
ysr@345 2179 rp->verify_no_references_recorded();
ysr@345 2180 assert(!rp->discovery_enabled(), "should have been disabled");
ysr@345 2181
ysr@345 2182 // Now clean up stale oops in SymbolTable and StringTable
johnc@2057 2183 SymbolTable::unlink(&g1_is_alive);
johnc@2057 2184 StringTable::unlink(&g1_is_alive);
ysr@345 2185 }
ysr@345 2186
ysr@345 2187 void ConcurrentMark::swapMarkBitMaps() {
ysr@345 2188 CMBitMapRO* temp = _prevMarkBitMap;
ysr@345 2189 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
ysr@345 2190 _nextMarkBitMap = (CMBitMap*) temp;
ysr@345 2191 }
ysr@345 2192
ysr@345 2193 class CMRemarkTask: public AbstractGangTask {
ysr@345 2194 private:
ysr@345 2195 ConcurrentMark *_cm;
ysr@345 2196
ysr@345 2197 public:
ysr@345 2198 void work(int worker_i) {
ysr@345 2199 // Since all available tasks are actually started, we should
ysr@345 2200 // only proceed if we're supposed to be actived.
ysr@345 2201 if ((size_t)worker_i < _cm->active_tasks()) {
ysr@345 2202 CMTask* task = _cm->task(worker_i);
ysr@345 2203 task->record_start_time();
ysr@345 2204 do {
johnc@2187 2205 task->do_marking_step(1000000000.0 /* something very large */,
johnc@2187 2206 true /* do_stealing */,
johnc@2187 2207 true /* do_termination */);
ysr@345 2208 } while (task->has_aborted() && !_cm->has_overflown());
ysr@345 2209 // If we overflow, then we do not want to restart. We instead
ysr@345 2210 // want to abort remark and do concurrent marking again.
ysr@345 2211 task->record_end_time();
ysr@345 2212 }
ysr@345 2213 }
ysr@345 2214
ysr@345 2215 CMRemarkTask(ConcurrentMark* cm) :
ysr@345 2216 AbstractGangTask("Par Remark"), _cm(cm) { }
ysr@345 2217 };
ysr@345 2218
ysr@345 2219 void ConcurrentMark::checkpointRootsFinalWork() {
ysr@345 2220 ResourceMark rm;
ysr@345 2221 HandleMark hm;
ysr@345 2222 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 2223
ysr@345 2224 g1h->ensure_parsability(false);
ysr@345 2225
jmasa@1855 2226 if (G1CollectedHeap::use_parallel_gc_threads()) {
jrose@1047 2227 G1CollectedHeap::StrongRootsScope srs(g1h);
ysr@345 2228 // this is remark, so we'll use up all available threads
ysr@345 2229 int active_workers = ParallelGCThreads;
johnc@2187 2230 set_phase(active_workers, false /* concurrent */);
ysr@345 2231
ysr@345 2232 CMRemarkTask remarkTask(this);
ysr@345 2233 // We will start all available threads, even if we decide that the
ysr@345 2234 // active_workers will be fewer. The extra ones will just bail out
ysr@345 2235 // immediately.
ysr@345 2236 int n_workers = g1h->workers()->total_workers();
ysr@345 2237 g1h->set_par_threads(n_workers);
ysr@345 2238 g1h->workers()->run_task(&remarkTask);
ysr@345 2239 g1h->set_par_threads(0);
ysr@345 2240 } else {
jrose@1047 2241 G1CollectedHeap::StrongRootsScope srs(g1h);
ysr@345 2242 // this is remark, so we'll use up all available threads
ysr@345 2243 int active_workers = 1;
johnc@2187 2244 set_phase(active_workers, false /* concurrent */);
ysr@345 2245
ysr@345 2246 CMRemarkTask remarkTask(this);
ysr@345 2247 // We will start all available threads, even if we decide that the
ysr@345 2248 // active_workers will be fewer. The extra ones will just bail out
ysr@345 2249 // immediately.
ysr@345 2250 remarkTask.work(0);
ysr@345 2251 }
tonyp@1082 2252 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
tonyp@1082 2253 guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
ysr@345 2254
ysr@345 2255 print_stats();
ysr@345 2256
ysr@345 2257 #if VERIFY_OBJS_PROCESSED
ysr@345 2258 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
ysr@345 2259 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
ysr@345 2260 _scan_obj_cl.objs_processed,
ysr@345 2261 ThreadLocalObjQueue::objs_enqueued);
ysr@345 2262 guarantee(_scan_obj_cl.objs_processed ==
ysr@345 2263 ThreadLocalObjQueue::objs_enqueued,
ysr@345 2264 "Different number of objs processed and enqueued.");
ysr@345 2265 }
ysr@345 2266 #endif
ysr@345 2267 }
ysr@345 2268
tonyp@1106 2269 #ifndef PRODUCT
tonyp@1106 2270
tonyp@1477 2271 class PrintReachableOopClosure: public OopClosure {
ysr@345 2272 private:
ysr@345 2273 G1CollectedHeap* _g1h;
ysr@345 2274 CMBitMapRO* _bitmap;
ysr@345 2275 outputStream* _out;
tonyp@1106 2276 bool _use_prev_marking;
tonyp@1477 2277 bool _all;
ysr@345 2278
ysr@345 2279 public:
tonyp@1477 2280 PrintReachableOopClosure(CMBitMapRO* bitmap,
tonyp@1477 2281 outputStream* out,
tonyp@1477 2282 bool use_prev_marking,
tonyp@1477 2283 bool all) :
tonyp@1106 2284 _g1h(G1CollectedHeap::heap()),
tonyp@1477 2285 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { }
ysr@345 2286
ysr@896 2287 void do_oop(narrowOop* p) { do_oop_work(p); }
ysr@896 2288 void do_oop( oop* p) { do_oop_work(p); }
ysr@896 2289
ysr@896 2290 template <class T> void do_oop_work(T* p) {
ysr@896 2291 oop obj = oopDesc::load_decode_heap_oop(p);
ysr@345 2292 const char* str = NULL;
ysr@345 2293 const char* str2 = "";
ysr@345 2294
tonyp@1477 2295 if (obj == NULL) {
tonyp@1477 2296 str = "";
tonyp@1477 2297 } else if (!_g1h->is_in_g1_reserved(obj)) {
tonyp@1477 2298 str = " O";
tonyp@1477 2299 } else {
ysr@345 2300 HeapRegion* hr = _g1h->heap_region_containing(obj);
tonyp@1082 2301 guarantee(hr != NULL, "invariant");
tonyp@1106 2302 bool over_tams = false;
tonyp@1106 2303 if (_use_prev_marking) {
tonyp@1106 2304 over_tams = hr->obj_allocated_since_prev_marking(obj);
tonyp@1106 2305 } else {
tonyp@1106 2306 over_tams = hr->obj_allocated_since_next_marking(obj);
tonyp@1106 2307 }
tonyp@1477 2308 bool marked = _bitmap->isMarked((HeapWord*) obj);
tonyp@1106 2309
tonyp@1106 2310 if (over_tams) {
tonyp@1477 2311 str = " >";
tonyp@1477 2312 if (marked) {
ysr@345 2313 str2 = " AND MARKED";
tonyp@1106 2314 }
tonyp@1477 2315 } else if (marked) {
tonyp@1477 2316 str = " M";
tonyp@1106 2317 } else {
tonyp@1477 2318 str = " NOT";
tonyp@1106 2319 }
ysr@345 2320 }
ysr@345 2321
tonyp@1477 2322 _out->print_cr(" "PTR_FORMAT": "PTR_FORMAT"%s%s",
ysr@345 2323 p, (void*) obj, str, str2);
ysr@345 2324 }
ysr@345 2325 };
ysr@345 2326
tonyp@1477 2327 class PrintReachableObjectClosure : public ObjectClosure {
ysr@345 2328 private:
tonyp@1106 2329 CMBitMapRO* _bitmap;
ysr@345 2330 outputStream* _out;
tonyp@1106 2331 bool _use_prev_marking;
tonyp@1477 2332 bool _all;
tonyp@1477 2333 HeapRegion* _hr;
ysr@345 2334
ysr@345 2335 public:
tonyp@1477 2336 PrintReachableObjectClosure(CMBitMapRO* bitmap,
tonyp@1477 2337 outputStream* out,
tonyp@1477 2338 bool use_prev_marking,
tonyp@1477 2339 bool all,
tonyp@1477 2340 HeapRegion* hr) :
tonyp@1477 2341 _bitmap(bitmap), _out(out),
tonyp@1477 2342 _use_prev_marking(use_prev_marking), _all(all), _hr(hr) { }
tonyp@1477 2343
tonyp@1477 2344 void do_object(oop o) {
tonyp@1477 2345 bool over_tams;
tonyp@1477 2346 if (_use_prev_marking) {
tonyp@1477 2347 over_tams = _hr->obj_allocated_since_prev_marking(o);
tonyp@1477 2348 } else {
tonyp@1477 2349 over_tams = _hr->obj_allocated_since_next_marking(o);
tonyp@1477 2350 }
tonyp@1477 2351 bool marked = _bitmap->isMarked((HeapWord*) o);
tonyp@1477 2352 bool print_it = _all || over_tams || marked;
tonyp@1477 2353
tonyp@1477 2354 if (print_it) {
tonyp@1477 2355 _out->print_cr(" "PTR_FORMAT"%s",
tonyp@1477 2356 o, (over_tams) ? " >" : (marked) ? " M" : "");
tonyp@1477 2357 PrintReachableOopClosure oopCl(_bitmap, _out, _use_prev_marking, _all);
tonyp@1477 2358 o->oop_iterate(&oopCl);
tonyp@1477 2359 }
ysr@345 2360 }
ysr@345 2361 };
ysr@345 2362
tonyp@1477 2363 class PrintReachableRegionClosure : public HeapRegionClosure {
ysr@345 2364 private:
tonyp@1106 2365 CMBitMapRO* _bitmap;
ysr@345 2366 outputStream* _out;
tonyp@1106 2367 bool _use_prev_marking;
tonyp@1477 2368 bool _all;
ysr@345 2369
ysr@345 2370 public:
ysr@345 2371 bool doHeapRegion(HeapRegion* hr) {
ysr@345 2372 HeapWord* b = hr->bottom();
ysr@345 2373 HeapWord* e = hr->end();
ysr@345 2374 HeapWord* t = hr->top();
tonyp@1106 2375 HeapWord* p = NULL;
tonyp@1106 2376 if (_use_prev_marking) {
tonyp@1106 2377 p = hr->prev_top_at_mark_start();
tonyp@1106 2378 } else {
tonyp@1106 2379 p = hr->next_top_at_mark_start();
tonyp@1106 2380 }
ysr@345 2381 _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
tonyp@1106 2382 "TAMS: "PTR_FORMAT, b, e, t, p);
tonyp@1477 2383 _out->cr();
tonyp@1477 2384
tonyp@1477 2385 HeapWord* from = b;
tonyp@1477 2386 HeapWord* to = t;
tonyp@1477 2387
tonyp@1477 2388 if (to > from) {
tonyp@1477 2389 _out->print_cr("Objects in ["PTR_FORMAT", "PTR_FORMAT"]", from, to);
tonyp@1477 2390 _out->cr();
tonyp@1477 2391 PrintReachableObjectClosure ocl(_bitmap, _out,
tonyp@1477 2392 _use_prev_marking, _all, hr);
tonyp@1477 2393 hr->object_iterate_mem_careful(MemRegion(from, to), &ocl);
tonyp@1477 2394 _out->cr();
tonyp@1477 2395 }
ysr@345 2396
ysr@345 2397 return false;
ysr@345 2398 }
ysr@345 2399
tonyp@1477 2400 PrintReachableRegionClosure(CMBitMapRO* bitmap,
tonyp@1477 2401 outputStream* out,
tonyp@1477 2402 bool use_prev_marking,
tonyp@1477 2403 bool all) :
tonyp@1477 2404 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { }
ysr@345 2405 };
ysr@345 2406
tonyp@1477 2407 void ConcurrentMark::print_reachable(const char* str,
tonyp@1477 2408 bool use_prev_marking,
tonyp@1477 2409 bool all) {
tonyp@1477 2410 gclog_or_tty->cr();
tonyp@1477 2411 gclog_or_tty->print_cr("== Doing heap dump... ");
tonyp@1106 2412
tonyp@1106 2413 if (G1PrintReachableBaseFile == NULL) {
tonyp@1106 2414 gclog_or_tty->print_cr(" #### error: no base file defined");
tonyp@1106 2415 return;
tonyp@1106 2416 }
tonyp@1106 2417
tonyp@1106 2418 if (strlen(G1PrintReachableBaseFile) + 1 + strlen(str) >
tonyp@1106 2419 (JVM_MAXPATHLEN - 1)) {
tonyp@1106 2420 gclog_or_tty->print_cr(" #### error: file name too long");
tonyp@1106 2421 return;
tonyp@1106 2422 }
tonyp@1106 2423
tonyp@1106 2424 char file_name[JVM_MAXPATHLEN];
tonyp@1106 2425 sprintf(file_name, "%s.%s", G1PrintReachableBaseFile, str);
tonyp@1106 2426 gclog_or_tty->print_cr(" dumping to file %s", file_name);
tonyp@1106 2427
tonyp@1106 2428 fileStream fout(file_name);
tonyp@1106 2429 if (!fout.is_open()) {
tonyp@1106 2430 gclog_or_tty->print_cr(" #### error: could not open file");
tonyp@1106 2431 return;
tonyp@1106 2432 }
tonyp@1106 2433
tonyp@1106 2434 outputStream* out = &fout;
tonyp@1106 2435
tonyp@1106 2436 CMBitMapRO* bitmap = NULL;
tonyp@1106 2437 if (use_prev_marking) {
tonyp@1106 2438 bitmap = _prevMarkBitMap;
tonyp@1106 2439 } else {
tonyp@1106 2440 bitmap = _nextMarkBitMap;
tonyp@1106 2441 }
tonyp@1106 2442
tonyp@1106 2443 out->print_cr("-- USING %s", (use_prev_marking) ? "PTAMS" : "NTAMS");
tonyp@1106 2444 out->cr();
tonyp@1106 2445
tonyp@1477 2446 out->print_cr("--- ITERATING OVER REGIONS");
tonyp@1106 2447 out->cr();
tonyp@1477 2448 PrintReachableRegionClosure rcl(bitmap, out, use_prev_marking, all);
ysr@345 2449 _g1h->heap_region_iterate(&rcl);
tonyp@1106 2450 out->cr();
tonyp@1106 2451
tonyp@1106 2452 gclog_or_tty->print_cr(" done");
tonyp@1477 2453 gclog_or_tty->flush();
ysr@345 2454 }
ysr@345 2455
tonyp@1106 2456 #endif // PRODUCT
tonyp@1106 2457
ysr@345 2458 // This note is for drainAllSATBBuffers and the code in between.
ysr@345 2459 // In the future we could reuse a task to do this work during an
ysr@345 2460 // evacuation pause (since now tasks are not active and can be claimed
ysr@345 2461 // during an evacuation pause). This was a late change to the code and
ysr@345 2462 // is currently not being taken advantage of.
ysr@345 2463
ysr@345 2464 class CMGlobalObjectClosure : public ObjectClosure {
ysr@345 2465 private:
ysr@345 2466 ConcurrentMark* _cm;
ysr@345 2467
ysr@345 2468 public:
ysr@345 2469 void do_object(oop obj) {
ysr@345 2470 _cm->deal_with_reference(obj);
ysr@345 2471 }
ysr@345 2472
ysr@345 2473 CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
ysr@345 2474 };
ysr@345 2475
ysr@345 2476 void ConcurrentMark::deal_with_reference(oop obj) {
ysr@345 2477 if (verbose_high())
ysr@345 2478 gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
ysr@345 2479 (void*) obj);
ysr@345 2480
ysr@345 2481
ysr@345 2482 HeapWord* objAddr = (HeapWord*) obj;
ysr@896 2483 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
ysr@345 2484 if (_g1h->is_in_g1_reserved(objAddr)) {
tonyp@1082 2485 assert(obj != NULL, "is_in_g1_reserved should ensure this");
ysr@345 2486 HeapRegion* hr = _g1h->heap_region_containing(obj);
ysr@345 2487 if (_g1h->is_obj_ill(obj, hr)) {
ysr@345 2488 if (verbose_high())
ysr@345 2489 gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered "
ysr@345 2490 "marked", (void*) obj);
ysr@345 2491
ysr@345 2492 // we need to mark it first
ysr@345 2493 if (_nextMarkBitMap->parMark(objAddr)) {
ysr@345 2494 // No OrderAccess:store_load() is needed. It is implicit in the
ysr@345 2495 // CAS done in parMark(objAddr) above
ysr@345 2496 HeapWord* finger = _finger;
ysr@345 2497 if (objAddr < finger) {
ysr@345 2498 if (verbose_high())
ysr@345 2499 gclog_or_tty->print_cr("[global] below the global finger "
ysr@345 2500 "("PTR_FORMAT"), pushing it", finger);
ysr@345 2501 if (!mark_stack_push(obj)) {
ysr@345 2502 if (verbose_low())
ysr@345 2503 gclog_or_tty->print_cr("[global] global stack overflow during "
ysr@345 2504 "deal_with_reference");
ysr@345 2505 }
ysr@345 2506 }
ysr@345 2507 }
ysr@345 2508 }
ysr@345 2509 }
ysr@345 2510 }
ysr@345 2511
ysr@345 2512 void ConcurrentMark::drainAllSATBBuffers() {
ysr@345 2513 CMGlobalObjectClosure oc(this);
ysr@345 2514 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
ysr@345 2515 satb_mq_set.set_closure(&oc);
ysr@345 2516
ysr@345 2517 while (satb_mq_set.apply_closure_to_completed_buffer()) {
ysr@345 2518 if (verbose_medium())
ysr@345 2519 gclog_or_tty->print_cr("[global] processed an SATB buffer");
ysr@345 2520 }
ysr@345 2521
ysr@345 2522 // no need to check whether we should do this, as this is only
ysr@345 2523 // called during an evacuation pause
ysr@345 2524 satb_mq_set.iterate_closure_all_threads();
ysr@345 2525
ysr@345 2526 satb_mq_set.set_closure(NULL);
tonyp@1082 2527 assert(satb_mq_set.completed_buffers_num() == 0, "invariant");
ysr@345 2528 }
ysr@345 2529
ysr@345 2530 void ConcurrentMark::markPrev(oop p) {
ysr@345 2531 // Note we are overriding the read-only view of the prev map here, via
ysr@345 2532 // the cast.
ysr@345 2533 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
ysr@345 2534 }
ysr@345 2535
ysr@345 2536 void ConcurrentMark::clear(oop p) {
ysr@345 2537 assert(p != NULL && p->is_oop(), "expected an oop");
ysr@345 2538 HeapWord* addr = (HeapWord*)p;
ysr@345 2539 assert(addr >= _nextMarkBitMap->startWord() ||
ysr@345 2540 addr < _nextMarkBitMap->endWord(), "in a region");
ysr@345 2541
ysr@345 2542 _nextMarkBitMap->clear(addr);
ysr@345 2543 }
ysr@345 2544
ysr@345 2545 void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
ysr@345 2546 // Note we are overriding the read-only view of the prev map here, via
ysr@345 2547 // the cast.
ysr@345 2548 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
ysr@345 2549 _nextMarkBitMap->clearRange(mr);
ysr@345 2550 }
ysr@345 2551
ysr@345 2552 HeapRegion*
ysr@345 2553 ConcurrentMark::claim_region(int task_num) {
ysr@345 2554 // "checkpoint" the finger
ysr@345 2555 HeapWord* finger = _finger;
ysr@345 2556
ysr@345 2557 // _heap_end will not change underneath our feet; it only changes at
ysr@345 2558 // yield points.
ysr@345 2559 while (finger < _heap_end) {
tonyp@1082 2560 assert(_g1h->is_in_g1_reserved(finger), "invariant");
ysr@345 2561
ysr@345 2562 // is the gap between reading the finger and doing the CAS too long?
ysr@345 2563
ysr@345 2564 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
ysr@345 2565 HeapWord* bottom = curr_region->bottom();
ysr@345 2566 HeapWord* end = curr_region->end();
ysr@345 2567 HeapWord* limit = curr_region->next_top_at_mark_start();
ysr@345 2568
ysr@345 2569 if (verbose_low())
ysr@345 2570 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" "
ysr@345 2571 "["PTR_FORMAT", "PTR_FORMAT"), "
ysr@345 2572 "limit = "PTR_FORMAT,
ysr@345 2573 task_num, curr_region, bottom, end, limit);
ysr@345 2574
ysr@345 2575 HeapWord* res =
ysr@345 2576 (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
ysr@345 2577 if (res == finger) {
ysr@345 2578 // we succeeded
ysr@345 2579
ysr@345 2580 // notice that _finger == end cannot be guaranteed here since,
ysr@345 2581 // someone else might have moved the finger even further
tonyp@1082 2582 assert(_finger >= end, "the finger should have moved forward");
ysr@345 2583
ysr@345 2584 if (verbose_low())
ysr@345 2585 gclog_or_tty->print_cr("[%d] we were successful with region = "
ysr@345 2586 PTR_FORMAT, task_num, curr_region);
ysr@345 2587
ysr@345 2588 if (limit > bottom) {
ysr@345 2589 if (verbose_low())
ysr@345 2590 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, "
ysr@345 2591 "returning it ", task_num, curr_region);
ysr@345 2592 return curr_region;
ysr@345 2593 } else {
tonyp@1082 2594 assert(limit == bottom,
tonyp@1082 2595 "the region limit should be at bottom");
ysr@345 2596 if (verbose_low())
ysr@345 2597 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, "
ysr@345 2598 "returning NULL", task_num, curr_region);
ysr@345 2599 // we return NULL and the caller should try calling
ysr@345 2600 // claim_region() again.
ysr@345 2601 return NULL;
ysr@345 2602 }
ysr@345 2603 } else {
tonyp@1082 2604 assert(_finger > finger, "the finger should have moved forward");
ysr@345 2605 if (verbose_low())
ysr@345 2606 gclog_or_tty->print_cr("[%d] somebody else moved the finger, "
ysr@345 2607 "global finger = "PTR_FORMAT", "
ysr@345 2608 "our finger = "PTR_FORMAT,
ysr@345 2609 task_num, _finger, finger);
ysr@345 2610
ysr@345 2611 // read it again
ysr@345 2612 finger = _finger;
ysr@345 2613 }
ysr@345 2614 }
ysr@345 2615
ysr@345 2616 return NULL;
ysr@345 2617 }
ysr@345 2618
johnc@1857 2619 bool ConcurrentMark::invalidate_aborted_regions_in_cset() {
johnc@1857 2620 bool result = false;
johnc@1857 2621 for (int i = 0; i < (int)_max_task_num; ++i) {
johnc@1857 2622 CMTask* the_task = _tasks[i];
johnc@1857 2623 MemRegion mr = the_task->aborted_region();
johnc@1857 2624 if (mr.start() != NULL) {
johnc@1857 2625 assert(mr.end() != NULL, "invariant");
johnc@1857 2626 assert(mr.word_size() > 0, "invariant");
johnc@1857 2627 HeapRegion* hr = _g1h->heap_region_containing(mr.start());
johnc@1857 2628 assert(hr != NULL, "invariant");
johnc@1857 2629 if (hr->in_collection_set()) {
johnc@1857 2630 // The region points into the collection set
johnc@1857 2631 the_task->set_aborted_region(MemRegion());
johnc@1857 2632 result = true;
johnc@1857 2633 }
johnc@1857 2634 }
johnc@1857 2635 }
johnc@1857 2636 return result;
johnc@1857 2637 }
johnc@1857 2638
johnc@1857 2639 bool ConcurrentMark::has_aborted_regions() {
johnc@1857 2640 for (int i = 0; i < (int)_max_task_num; ++i) {
johnc@1857 2641 CMTask* the_task = _tasks[i];
johnc@1857 2642 MemRegion mr = the_task->aborted_region();
johnc@1857 2643 if (mr.start() != NULL) {
johnc@1857 2644 assert(mr.end() != NULL, "invariant");
johnc@1857 2645 assert(mr.word_size() > 0, "invariant");
johnc@1857 2646 return true;
johnc@1857 2647 }
johnc@1857 2648 }
johnc@1857 2649 return false;
johnc@1857 2650 }
johnc@1857 2651
ysr@345 2652 void ConcurrentMark::oops_do(OopClosure* cl) {
ysr@345 2653 if (_markStack.size() > 0 && verbose_low())
ysr@345 2654 gclog_or_tty->print_cr("[global] scanning the global marking stack, "
ysr@345 2655 "size = %d", _markStack.size());
ysr@345 2656 // we first iterate over the contents of the mark stack...
ysr@345 2657 _markStack.oops_do(cl);
ysr@345 2658
ysr@345 2659 for (int i = 0; i < (int)_max_task_num; ++i) {
ysr@345 2660 OopTaskQueue* queue = _task_queues->queue((int)i);
ysr@345 2661
ysr@345 2662 if (queue->size() > 0 && verbose_low())
ysr@345 2663 gclog_or_tty->print_cr("[global] scanning task queue of task %d, "
ysr@345 2664 "size = %d", i, queue->size());
ysr@345 2665
ysr@345 2666 // ...then over the contents of the all the task queues.
ysr@345 2667 queue->oops_do(cl);
ysr@345 2668 }
ysr@345 2669
johnc@1857 2670 // Invalidate any entries, that are in the region stack, that
ysr@345 2671 // point into the collection set
ysr@345 2672 if (_regionStack.invalidate_entries_into_cset()) {
ysr@345 2673 // otherwise, any gray objects copied during the evacuation pause
ysr@345 2674 // might not be visited.
tonyp@1082 2675 assert(_should_gray_objects, "invariant");
ysr@345 2676 }
johnc@1857 2677
johnc@1857 2678 // Invalidate any aborted regions, recorded in the individual CM
johnc@1857 2679 // tasks, that point into the collection set.
johnc@1857 2680 if (invalidate_aborted_regions_in_cset()) {
johnc@1857 2681 // otherwise, any gray objects copied during the evacuation pause
johnc@1857 2682 // might not be visited.
johnc@1857 2683 assert(_should_gray_objects, "invariant");
johnc@1857 2684 }
johnc@1857 2685
ysr@345 2686 }
ysr@345 2687
ysr@345 2688 void ConcurrentMark::clear_marking_state() {
ysr@345 2689 _markStack.setEmpty();
ysr@345 2690 _markStack.clear_overflow();
ysr@345 2691 _regionStack.setEmpty();
ysr@345 2692 _regionStack.clear_overflow();
ysr@345 2693 clear_has_overflown();
ysr@345 2694 _finger = _heap_start;
ysr@345 2695
ysr@345 2696 for (int i = 0; i < (int)_max_task_num; ++i) {
ysr@345 2697 OopTaskQueue* queue = _task_queues->queue(i);
ysr@345 2698 queue->set_empty();
johnc@1910 2699 // Clear any partial regions from the CMTasks
johnc@1910 2700 _tasks[i]->clear_aborted_region();
ysr@345 2701 }
ysr@345 2702 }
ysr@345 2703
ysr@345 2704 void ConcurrentMark::print_stats() {
ysr@345 2705 if (verbose_stats()) {
ysr@345 2706 gclog_or_tty->print_cr("---------------------------------------------------------------------");
ysr@345 2707 for (size_t i = 0; i < _active_tasks; ++i) {
ysr@345 2708 _tasks[i]->print_stats();
ysr@345 2709 gclog_or_tty->print_cr("---------------------------------------------------------------------");
ysr@345 2710 }
ysr@345 2711 }
ysr@345 2712 }
ysr@345 2713
ysr@345 2714 class CSMarkOopClosure: public OopClosure {
ysr@345 2715 friend class CSMarkBitMapClosure;
ysr@345 2716
ysr@345 2717 G1CollectedHeap* _g1h;
ysr@345 2718 CMBitMap* _bm;
ysr@345 2719 ConcurrentMark* _cm;
ysr@345 2720 oop* _ms;
ysr@345 2721 jint* _array_ind_stack;
ysr@345 2722 int _ms_size;
ysr@345 2723 int _ms_ind;
ysr@345 2724 int _array_increment;
ysr@345 2725
ysr@345 2726 bool push(oop obj, int arr_ind = 0) {
ysr@345 2727 if (_ms_ind == _ms_size) {
ysr@345 2728 gclog_or_tty->print_cr("Mark stack is full.");
ysr@345 2729 return false;
ysr@345 2730 }
ysr@345 2731 _ms[_ms_ind] = obj;
ysr@345 2732 if (obj->is_objArray()) _array_ind_stack[_ms_ind] = arr_ind;
ysr@345 2733 _ms_ind++;
ysr@345 2734 return true;
ysr@345 2735 }
ysr@345 2736
ysr@345 2737 oop pop() {
ysr@345 2738 if (_ms_ind == 0) return NULL;
ysr@345 2739 else {
ysr@345 2740 _ms_ind--;
ysr@345 2741 return _ms[_ms_ind];
ysr@345 2742 }
ysr@345 2743 }
ysr@345 2744
ysr@896 2745 template <class T> bool drain() {
ysr@345 2746 while (_ms_ind > 0) {
ysr@345 2747 oop obj = pop();
ysr@345 2748 assert(obj != NULL, "Since index was non-zero.");
ysr@345 2749 if (obj->is_objArray()) {
ysr@345 2750 jint arr_ind = _array_ind_stack[_ms_ind];
ysr@345 2751 objArrayOop aobj = objArrayOop(obj);
ysr@345 2752 jint len = aobj->length();
ysr@345 2753 jint next_arr_ind = arr_ind + _array_increment;
ysr@345 2754 if (next_arr_ind < len) {
ysr@345 2755 push(obj, next_arr_ind);
ysr@345 2756 }
ysr@345 2757 // Now process this portion of this one.
ysr@345 2758 int lim = MIN2(next_arr_ind, len);
ysr@345 2759 for (int j = arr_ind; j < lim; j++) {
apetrusenko@963 2760 do_oop(aobj->objArrayOopDesc::obj_at_addr<T>(j));
ysr@345 2761 }
ysr@345 2762
ysr@345 2763 } else {
ysr@345 2764 obj->oop_iterate(this);
ysr@345 2765 }
ysr@345 2766 if (abort()) return false;
ysr@345 2767 }
ysr@345 2768 return true;
ysr@345 2769 }
ysr@345 2770
ysr@345 2771 public:
ysr@345 2772 CSMarkOopClosure(ConcurrentMark* cm, int ms_size) :
ysr@345 2773 _g1h(G1CollectedHeap::heap()),
ysr@345 2774 _cm(cm),
ysr@345 2775 _bm(cm->nextMarkBitMap()),
ysr@345 2776 _ms_size(ms_size), _ms_ind(0),
ysr@345 2777 _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
ysr@345 2778 _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
ysr@345 2779 _array_increment(MAX2(ms_size/8, 16))
ysr@345 2780 {}
ysr@345 2781
ysr@345 2782 ~CSMarkOopClosure() {
ysr@345 2783 FREE_C_HEAP_ARRAY(oop, _ms);
ysr@345 2784 FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
ysr@345 2785 }
ysr@345 2786
ysr@896 2787 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
ysr@896 2788 virtual void do_oop( oop* p) { do_oop_work(p); }
ysr@896 2789
ysr@896 2790 template <class T> void do_oop_work(T* p) {
ysr@896 2791 T heap_oop = oopDesc::load_heap_oop(p);
ysr@896 2792 if (oopDesc::is_null(heap_oop)) return;
ysr@896 2793 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
ysr@345 2794 if (obj->is_forwarded()) {
ysr@345 2795 // If the object has already been forwarded, we have to make sure
ysr@345 2796 // that it's marked. So follow the forwarding pointer. Note that
ysr@345 2797 // this does the right thing for self-forwarding pointers in the
ysr@345 2798 // evacuation failure case.
ysr@345 2799 obj = obj->forwardee();
ysr@345 2800 }
ysr@345 2801 HeapRegion* hr = _g1h->heap_region_containing(obj);
ysr@345 2802 if (hr != NULL) {
ysr@345 2803 if (hr->in_collection_set()) {
ysr@345 2804 if (_g1h->is_obj_ill(obj)) {
ysr@345 2805 _bm->mark((HeapWord*)obj);
ysr@345 2806 if (!push(obj)) {
ysr@345 2807 gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed.");
ysr@345 2808 set_abort();
ysr@345 2809 }
ysr@345 2810 }
ysr@345 2811 } else {
ysr@345 2812 // Outside the collection set; we need to gray it
ysr@345 2813 _cm->deal_with_reference(obj);
ysr@345 2814 }
ysr@345 2815 }
ysr@345 2816 }
ysr@345 2817 };
ysr@345 2818
ysr@345 2819 class CSMarkBitMapClosure: public BitMapClosure {
ysr@345 2820 G1CollectedHeap* _g1h;
ysr@345 2821 CMBitMap* _bitMap;
ysr@345 2822 ConcurrentMark* _cm;
ysr@345 2823 CSMarkOopClosure _oop_cl;
ysr@345 2824 public:
ysr@345 2825 CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) :
ysr@345 2826 _g1h(G1CollectedHeap::heap()),
ysr@345 2827 _bitMap(cm->nextMarkBitMap()),
ysr@345 2828 _oop_cl(cm, ms_size)
ysr@345 2829 {}
ysr@345 2830
ysr@345 2831 ~CSMarkBitMapClosure() {}
ysr@345 2832
ysr@345 2833 bool do_bit(size_t offset) {
ysr@345 2834 // convert offset into a HeapWord*
ysr@345 2835 HeapWord* addr = _bitMap->offsetToHeapWord(offset);
ysr@345 2836 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
ysr@345 2837 "address out of range");
ysr@345 2838 assert(_bitMap->isMarked(addr), "tautology");
ysr@345 2839 oop obj = oop(addr);
ysr@345 2840 if (!obj->is_forwarded()) {
ysr@345 2841 if (!_oop_cl.push(obj)) return false;
ysr@896 2842 if (UseCompressedOops) {
ysr@896 2843 if (!_oop_cl.drain<narrowOop>()) return false;
ysr@896 2844 } else {
ysr@896 2845 if (!_oop_cl.drain<oop>()) return false;
ysr@896 2846 }
ysr@345 2847 }
ysr@345 2848 // Otherwise...
ysr@345 2849 return true;
ysr@345 2850 }
ysr@345 2851 };
ysr@345 2852
ysr@345 2853
ysr@345 2854 class CompleteMarkingInCSHRClosure: public HeapRegionClosure {
ysr@345 2855 CMBitMap* _bm;
ysr@345 2856 CSMarkBitMapClosure _bit_cl;
ysr@345 2857 enum SomePrivateConstants {
ysr@345 2858 MSSize = 1000
ysr@345 2859 };
ysr@345 2860 bool _completed;
ysr@345 2861 public:
ysr@345 2862 CompleteMarkingInCSHRClosure(ConcurrentMark* cm) :
ysr@345 2863 _bm(cm->nextMarkBitMap()),
ysr@345 2864 _bit_cl(cm, MSSize),
ysr@345 2865 _completed(true)
ysr@345 2866 {}
ysr@345 2867
ysr@345 2868 ~CompleteMarkingInCSHRClosure() {}
ysr@345 2869
ysr@345 2870 bool doHeapRegion(HeapRegion* r) {
ysr@345 2871 if (!r->evacuation_failed()) {
ysr@345 2872 MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start());
ysr@345 2873 if (!mr.is_empty()) {
ysr@345 2874 if (!_bm->iterate(&_bit_cl, mr)) {
ysr@345 2875 _completed = false;
ysr@345 2876 return true;
ysr@345 2877 }
ysr@345 2878 }
ysr@345 2879 }
ysr@345 2880 return false;
ysr@345 2881 }
ysr@345 2882
ysr@345 2883 bool completed() { return _completed; }
ysr@345 2884 };
ysr@345 2885
ysr@345 2886 class ClearMarksInHRClosure: public HeapRegionClosure {
ysr@345 2887 CMBitMap* _bm;
ysr@345 2888 public:
ysr@345 2889 ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { }
ysr@345 2890
ysr@345 2891 bool doHeapRegion(HeapRegion* r) {
ysr@345 2892 if (!r->used_region().is_empty() && !r->evacuation_failed()) {
ysr@345 2893 MemRegion usedMR = r->used_region();
ysr@345 2894 _bm->clearRange(r->used_region());
ysr@345 2895 }
ysr@345 2896 return false;
ysr@345 2897 }
ysr@345 2898 };
ysr@345 2899
ysr@345 2900 void ConcurrentMark::complete_marking_in_collection_set() {
ysr@345 2901 G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@345 2902
ysr@345 2903 if (!g1h->mark_in_progress()) {
ysr@345 2904 g1h->g1_policy()->record_mark_closure_time(0.0);
ysr@345 2905 return;
ysr@345 2906 }
ysr@345 2907
ysr@345 2908 int i = 1;
ysr@345 2909 double start = os::elapsedTime();
ysr@345 2910 while (true) {
ysr@345 2911 i++;
ysr@345 2912 CompleteMarkingInCSHRClosure cmplt(this);
ysr@345 2913 g1h->collection_set_iterate(&cmplt);
ysr@345 2914 if (cmplt.completed()) break;
ysr@345 2915 }
ysr@345 2916 double end_time = os::elapsedTime();
ysr@345 2917 double elapsed_time_ms = (end_time - start) * 1000.0;
ysr@345 2918 g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms);
ysr@345 2919
ysr@345 2920 ClearMarksInHRClosure clr(nextMarkBitMap());
ysr@345 2921 g1h->collection_set_iterate(&clr);
ysr@345 2922 }
ysr@345 2923
ysr@345 2924 // The next two methods deal with the following optimisation. Some
ysr@345 2925 // objects are gray by being marked and located above the finger. If
ysr@345 2926 // they are copied, during an evacuation pause, below the finger then
ysr@345 2927 // the need to be pushed on the stack. The observation is that, if
ysr@345 2928 // there are no regions in the collection set located above the
ysr@345 2929 // finger, then the above cannot happen, hence we do not need to
ysr@345 2930 // explicitly gray any objects when copying them to below the
ysr@345 2931 // finger. The global stack will be scanned to ensure that, if it
ysr@345 2932 // points to objects being copied, it will update their
ysr@345 2933 // location. There is a tricky situation with the gray objects in
ysr@345 2934 // region stack that are being coped, however. See the comment in
ysr@345 2935 // newCSet().
ysr@345 2936
ysr@345 2937 void ConcurrentMark::newCSet() {
ysr@345 2938 if (!concurrent_marking_in_progress())
ysr@345 2939 // nothing to do if marking is not in progress
ysr@345 2940 return;
ysr@345 2941
ysr@345 2942 // find what the lowest finger is among the global and local fingers
ysr@345 2943 _min_finger = _finger;
ysr@345 2944 for (int i = 0; i < (int)_max_task_num; ++i) {
ysr@345 2945 CMTask* task = _tasks[i];
ysr@345 2946 HeapWord* task_finger = task->finger();
ysr@345 2947 if (task_finger != NULL && task_finger < _min_finger)
ysr@345 2948 _min_finger = task_finger;
ysr@345 2949 }
ysr@345 2950
ysr@345 2951 _should_gray_objects = false;
ysr@345 2952
ysr@345 2953 // This fixes a very subtle and fustrating bug. It might be the case
ysr@345 2954 // that, during en evacuation pause, heap regions that contain
ysr@345 2955 // objects that are gray (by being in regions contained in the
ysr@345 2956 // region stack) are included in the collection set. Since such gray
ysr@345 2957 // objects will be moved, and because it's not easy to redirect
ysr@345 2958 // region stack entries to point to a new location (because objects
ysr@345 2959 // in one region might be scattered to multiple regions after they
ysr@345 2960 // are copied), one option is to ensure that all marked objects
ysr@345 2961 // copied during a pause are pushed on the stack. Notice, however,
ysr@345 2962 // that this problem can only happen when the region stack is not
ysr@345 2963 // empty during an evacuation pause. So, we make the fix a bit less
ysr@345 2964 // conservative and ensure that regions are pushed on the stack,
ysr@345 2965 // irrespective whether all collection set regions are below the
ysr@345 2966 // finger, if the region stack is not empty. This is expected to be
ysr@345 2967 // a rare case, so I don't think it's necessary to be smarted about it.
johnc@1857 2968 if (!region_stack_empty() || has_aborted_regions())
ysr@345 2969 _should_gray_objects = true;
ysr@345 2970 }
ysr@345 2971
ysr@345 2972 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
ysr@345 2973 if (!concurrent_marking_in_progress())
ysr@345 2974 return;
ysr@345 2975
ysr@345 2976 HeapWord* region_end = hr->end();
ysr@345 2977 if (region_end > _min_finger)
ysr@345 2978 _should_gray_objects = true;
ysr@345 2979 }
ysr@345 2980
ysr@345 2981 // abandon current marking iteration due to a Full GC
ysr@345 2982 void ConcurrentMark::abort() {
ysr@345 2983 // Clear all marks to force marking thread to do nothing
ysr@345 2984 _nextMarkBitMap->clearAll();
ysr@345 2985 // Empty mark stack
ysr@345 2986 clear_marking_state();
johnc@1857 2987 for (int i = 0; i < (int)_max_task_num; ++i) {
ysr@345 2988 _tasks[i]->clear_region_fields();
johnc@1857 2989 }
ysr@345 2990 _has_aborted = true;
ysr@345 2991
ysr@345 2992 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
ysr@345 2993 satb_mq_set.abandon_partial_marking();
tonyp@1404 2994 // This can be called either during or outside marking, we'll read
tonyp@1404 2995 // the expected_active value from the SATB queue set.
tonyp@1404 2996 satb_mq_set.set_active_all_threads(
tonyp@1404 2997 false, /* new active value */
tonyp@1404 2998 satb_mq_set.is_active() /* expected_active */);
ysr@345 2999 }
ysr@345