annotate src/hotspot/share/gc/z/zMarkStackAllocator.cpp @ 53621:ec92cbf2152b

8212748: ZGC: Add reentrant locking functionality Reviewed-by: eosterlund, kbarrett
author pliden
date Fri, 09 Nov 2018 14:08:01 +0100
parents f3cf91d5373f
children
rev   line source
pliden@52107 1 /*
pliden@52107 2 * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
pliden@52107 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
pliden@52107 4 *
pliden@52107 5 * This code is free software; you can redistribute it and/or modify it
pliden@52107 6 * under the terms of the GNU General Public License version 2 only, as
pliden@52107 7 * published by the Free Software Foundation.
pliden@52107 8 *
pliden@52107 9 * This code is distributed in the hope that it will be useful, but WITHOUT
pliden@52107 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
pliden@52107 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
pliden@52107 12 * version 2 for more details (a copy is included in the LICENSE file that
pliden@52107 13 * accompanied this code).
pliden@52107 14 *
pliden@52107 15 * You should have received a copy of the GNU General Public License version
pliden@52107 16 * 2 along with this work; if not, write to the Free Software Foundation,
pliden@52107 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
pliden@52107 18 *
pliden@52107 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
pliden@52107 20 * or visit www.oracle.com if you need additional information or have any
pliden@52107 21 * questions.
pliden@52107 22 */
pliden@52107 23
pliden@52107 24 #include "precompiled.hpp"
pliden@52107 25 #include "gc/z/zLock.inline.hpp"
pliden@52107 26 #include "gc/z/zMarkStack.inline.hpp"
pliden@52107 27 #include "gc/z/zMarkStackAllocator.hpp"
pliden@52107 28 #include "logging/log.hpp"
pliden@52107 29 #include "runtime/atomic.hpp"
pliden@52107 30 #include "runtime/os.hpp"
pliden@52107 31 #include "utilities/debug.hpp"
pliden@52107 32
pliden@52107 33 uintptr_t ZMarkStackSpaceStart;
pliden@52107 34
pliden@52107 35 ZMarkStackSpace::ZMarkStackSpace() :
pliden@52107 36 _expand_lock(),
pliden@52107 37 _start(0),
pliden@52107 38 _top(0),
pliden@52107 39 _end(0) {
pliden@52107 40 assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
pliden@52107 41
pliden@52107 42 // Reserve address space
pliden@52107 43 const size_t size = ZMarkStackSpaceLimit;
pliden@52107 44 const size_t alignment = (size_t)os::vm_allocation_granularity();
pliden@52107 45 const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
pliden@52107 46 if (addr == 0) {
pliden@52107 47 log_error(gc, marking)("Failed to reserve address space for mark stacks");
pliden@52107 48 return;
pliden@52107 49 }
pliden@52107 50
pliden@52107 51 // Successfully initialized
pliden@52107 52 _start = _top = _end = addr;
pliden@52107 53
pliden@52107 54 // Register mark stack space start
pliden@52107 55 ZMarkStackSpaceStart = _start;
pliden@52107 56 }
pliden@52107 57
pliden@52107 58 bool ZMarkStackSpace::is_initialized() const {
pliden@52107 59 return _start != 0;
pliden@52107 60 }
pliden@52107 61
pliden@52107 62 uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
pliden@52107 63 uintptr_t top = Atomic::load(&_top);
pliden@52107 64
pliden@52107 65 for (;;) {
pliden@52107 66 const uintptr_t end = Atomic::load(&_end);
pliden@52107 67 const uintptr_t new_top = top + size;
pliden@52107 68 if (new_top > end) {
pliden@52107 69 // Not enough space left
pliden@52107 70 return 0;
pliden@52107 71 }
pliden@52107 72
pliden@52107 73 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top);
pliden@52107 74 if (prev_top == top) {
pliden@52107 75 // Success
pliden@52107 76 return top;
pliden@52107 77 }
pliden@52107 78
pliden@52107 79 // Retry
pliden@52107 80 top = prev_top;
pliden@52107 81 }
pliden@52107 82 }
pliden@52107 83
pliden@52107 84 uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
pliden@53621 85 ZLocker<ZLock> locker(&_expand_lock);
pliden@52107 86
pliden@52107 87 // Retry allocation before expanding
pliden@52107 88 uintptr_t addr = alloc_space(size);
pliden@52107 89 if (addr != 0) {
pliden@52107 90 return addr;
pliden@52107 91 }
pliden@52107 92
pliden@52107 93 // Check expansion limit
pliden@52107 94 const size_t expand_size = ZMarkStackSpaceExpandSize;
pliden@52107 95 const size_t old_size = _end - _start;
pliden@52107 96 const size_t new_size = old_size + expand_size;
pliden@52107 97 if (new_size > ZMarkStackSpaceLimit) {
pliden@52107 98 // Expansion limit reached. This is a fatal error since we
pliden@52107 99 // currently can't recover from running out of mark stack space.
pliden@52107 100 fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
pliden@52107 101 "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
pliden@52107 102 ZMarkStackSpaceLimit / M);
pliden@52107 103 }
pliden@52107 104
pliden@52107 105 log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
pliden@52107 106 old_size / M, new_size / M);
pliden@52107 107
pliden@52107 108 // Expand
pliden@52107 109 os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
pliden@52107 110
pliden@52107 111 // Increment top before end to make sure another
pliden@52107 112 // thread can't steal out newly expanded space.
pliden@52107 113 addr = Atomic::add(size, &_top) - size;
pliden@52107 114 Atomic::add(expand_size, &_end);
pliden@52107 115
pliden@52107 116 return addr;
pliden@52107 117 }
pliden@52107 118
pliden@52107 119 uintptr_t ZMarkStackSpace::alloc(size_t size) {
pliden@52107 120 const uintptr_t addr = alloc_space(size);
pliden@52107 121 if (addr != 0) {
pliden@52107 122 return addr;
pliden@52107 123 }
pliden@52107 124
pliden@52107 125 return expand_and_alloc_space(size);
pliden@52107 126 }
pliden@52107 127
pliden@52107 128 ZMarkStackAllocator::ZMarkStackAllocator() :
pliden@52107 129 _freelist(),
pliden@52107 130 _space() {
pliden@52107 131 guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch");
pliden@52107 132 guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch");
pliden@52107 133
pliden@52107 134 // Prime free list to avoid an immediate space
pliden@52107 135 // expansion when marking starts.
pliden@52107 136 if (_space.is_initialized()) {
pliden@52107 137 prime_freelist();
pliden@52107 138 }
pliden@52107 139 }
pliden@52107 140
pliden@52107 141 bool ZMarkStackAllocator::is_initialized() const {
pliden@52107 142 return _space.is_initialized();
pliden@52107 143 }
pliden@52107 144
pliden@52107 145 void ZMarkStackAllocator::prime_freelist() {
pliden@52107 146 for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {
pliden@52107 147 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
pliden@52107 148 ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);
pliden@52107 149 free_magazine(magazine);
pliden@52107 150 }
pliden@52107 151 }
pliden@52107 152
pliden@52107 153 ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
pliden@52107 154 assert(is_aligned(size, ZMarkStackSize), "Invalid size");
pliden@52107 155
pliden@52107 156 // Use first stack as magazine
pliden@52107 157 ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine();
pliden@52107 158 for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) {
pliden@52107 159 ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack();
pliden@52107 160 const bool success = magazine->push(stack);
pliden@52107 161 assert(success, "Magazine should never get full");
pliden@52107 162 }
pliden@52107 163
pliden@52107 164 return magazine;
pliden@52107 165 }
pliden@52107 166
pliden@52107 167 ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
pliden@52107 168 // Try allocating from the free list first
pliden@52107 169 ZMarkStackMagazine* const magazine = _freelist.pop_atomic();
pliden@52107 170 if (magazine != NULL) {
pliden@52107 171 return magazine;
pliden@52107 172 }
pliden@52107 173
pliden@52107 174 // Allocate new magazine
pliden@52107 175 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
pliden@52107 176 if (addr == 0) {
pliden@52107 177 return NULL;
pliden@52107 178 }
pliden@52107 179
pliden@52107 180 return create_magazine_from_space(addr, ZMarkStackMagazineSize);
pliden@52107 181 }
pliden@52107 182
pliden@52107 183 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
pliden@52107 184 _freelist.push_atomic(magazine);
pliden@52107 185 }