changeset 13447:ffaec77df3e5

8184347: Move G1CMBitMap and support classes into their own files Reviewed-by: rkennke, shade, mgerdin
author tschatzl
date Fri, 04 Aug 2017 14:24:11 +0200
parents 7653b75bfe57
children d7719b146b88
files src/share/vm/gc/g1/g1ConcurrentMark.cpp src/share/vm/gc/g1/g1ConcurrentMark.hpp src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp src/share/vm/gc/g1/g1ConcurrentMarkBitMap.cpp src/share/vm/gc/g1/g1ConcurrentMarkBitMap.hpp src/share/vm/gc/g1/g1ConcurrentMarkBitMap.inline.hpp
diffstat 6 files changed, 298 insertions(+), 197 deletions(-) [+]
line wrap: on
line diff
--- a/src/share/vm/gc/g1/g1ConcurrentMark.cpp	Fri Aug 04 14:15:42 2017 +0200
+++ b/src/share/vm/gc/g1/g1ConcurrentMark.cpp	Fri Aug 04 14:24:11 2017 +0200
@@ -60,43 +60,21 @@
 #include "utilities/align.hpp"
 #include "utilities/growableArray.hpp"
 
-void G1CMBitMap::print_on_error(outputStream* st, const char* prefix) const {
-  _bm.print_on_error(st, prefix);
-}
-
-size_t G1CMBitMap::compute_size(size_t heap_size) {
-  return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
-}
-
-size_t G1CMBitMap::mark_distance() {
-  return MinObjAlignmentInBytes * BitsPerByte;
-}
-
-void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
-  _covered = heap;
-
-  _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _covered.word_size() >> _shifter);
-
-  storage->set_mapping_changed_listener(&_listener);
-}
-
-void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
-  if (zero_filled) {
-    return;
-  }
-  // We need to clear the bitmap on commit, removing any existing information.
-  MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
-  _bm->clear_range(mr);
-}
-
-void G1CMBitMap::clear_range(MemRegion mr) {
-  MemRegion intersection = mr.intersection(_covered);
-  assert(!intersection.is_empty(),
-         "Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap",
-         p2i(mr.start()), p2i(mr.end()));
-  // convert address range into offset range
-  _bm.at_put_range(addr_to_offset(intersection.start()),
-                   addr_to_offset(intersection.end()), false);
+bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
+  assert(addr < _cm->finger(), "invariant");
+  assert(addr >= _task->finger(), "invariant");
+
+  // We move that task's local finger along.
+  _task->move_finger_to(addr);
+
+  _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
+  // we only partially drain the local queue and global stack
+  _task->drain_local_queue(true);
+  _task->drain_global_stack(true);
+
+  // if the has_aborted flag has been raised, we need to bail out of
+  // the iteration
+  return !_task->has_aborted();
 }
 
 G1CMMarkStack::G1CMMarkStack() :
@@ -2122,23 +2100,6 @@
   _nextMarkBitMap->print_on_error(st, " Next Bits: ");
 }
 
-bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
-  assert(addr < _cm->finger(), "invariant");
-  assert(addr >= _task->finger(), "invariant");
-
-  // We move that task's local finger along.
-  _task->move_finger_to(addr);
-
-  _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
-  // we only partially drain the local queue and global stack
-  _task->drain_local_queue(true);
-  _task->drain_global_stack(true);
-
-  // if the has_aborted flag has been raised, we need to bail out of
-  // the iteration
-  return !_task->has_aborted();
-}
-
 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
   ReferenceProcessor* result = g1h->ref_processor_cm();
   assert(result != NULL, "CM reference processor should not be NULL");
--- a/src/share/vm/gc/g1/g1ConcurrentMark.hpp	Fri Aug 04 14:15:42 2017 +0200
+++ b/src/share/vm/gc/g1/g1ConcurrentMark.hpp	Fri Aug 04 14:24:11 2017 +0200
@@ -26,13 +26,13 @@
 #define SHARE_VM_GC_G1_G1CONCURRENTMARK_HPP
 
 #include "classfile/javaClasses.hpp"
+#include "gc/g1/g1ConcurrentMarkBitMap.hpp"
 #include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp"
 #include "gc/g1/g1RegionToSpaceMapper.hpp"
 #include "gc/g1/heapRegionSet.hpp"
 #include "gc/shared/taskqueue.hpp"
 
 class G1CollectedHeap;
-class G1CMBitMap;
 class G1CMTask;
 class G1ConcurrentMark;
 class ConcurrentGCTimer;
@@ -110,95 +110,6 @@
   bool do_object_b(oop obj);
 };
 
-// Closure for iteration over bitmaps
-class G1CMBitMapClosure VALUE_OBJ_CLASS_SPEC {
-private:
-  G1ConcurrentMark* const _cm;
-  G1CMTask* const _task;
-public:
-  G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm) : _task(task), _cm(cm) { }
-
-  bool do_addr(HeapWord* const addr);
-};
-
-class G1CMBitMapMappingChangedListener : public G1MappingChangedListener {
- private:
-  G1CMBitMap* _bm;
- public:
-  G1CMBitMapMappingChangedListener() : _bm(NULL) {}
-
-  void set_bitmap(G1CMBitMap* bm) { _bm = bm; }
-
-  virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled);
-};
-
-// A generic mark bitmap for concurrent marking.  This is essentially a wrapper
-// around the BitMap class that is based on HeapWords, with one bit per (1 << _shifter) HeapWords.
-class G1CMBitMap VALUE_OBJ_CLASS_SPEC {
-private:
-  MemRegion _covered;    // The heap area covered by this bitmap.
-
-  const int _shifter;    // Shift amount from heap index to bit index in the bitmap.
-
-  BitMapView _bm;        // The actual bitmap.
-
-  G1CMBitMapMappingChangedListener _listener;
-
-  inline void check_mark(HeapWord* addr) NOT_DEBUG_RETURN;
-
-  // Convert from bit offset to address.
-  HeapWord* offset_to_addr(size_t offset) const {
-    return _covered.start() + (offset << _shifter);
-  }
-  // Convert from address to bit offset.
-  size_t addr_to_offset(const HeapWord* addr) const {
-    return pointer_delta(addr, _covered.start()) >> _shifter;
-  }
-public:
-  static size_t compute_size(size_t heap_size);
-  // Returns the amount of bytes on the heap between two marks in the bitmap.
-  static size_t mark_distance();
-  // Returns how many bytes (or bits) of the heap a single byte (or bit) of the
-  // mark bitmap corresponds to. This is the same as the mark distance above.
-  static size_t heap_map_factor() {
-    return mark_distance();
-  }
-
-  G1CMBitMap() : _covered(), _bm(), _shifter(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); }
-
-  // Initializes the underlying BitMap to cover the given area.
-  void initialize(MemRegion heap, G1RegionToSpaceMapper* storage);
-
-  // read marks
-  bool is_marked(HeapWord* addr) const {
-    assert(_covered.contains(addr),
-           "Address " PTR_FORMAT " is outside underlying space from " PTR_FORMAT " to " PTR_FORMAT,
-           p2i(addr), p2i(_covered.start()), p2i(_covered.end()));
-    return _bm.at(addr_to_offset(addr));
-  }
-
-  // Apply the closure to the addresses that correspond to marked bits in the bitmap.
-  inline bool iterate(G1CMBitMapClosure* cl, MemRegion mr);
-
-  // Return the address corresponding to the next marked bit at or after
-  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
-  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
-  inline HeapWord* get_next_marked_addr(const HeapWord* addr,
-                                        const HeapWord* limit) const;
-
-  // The argument addr should be the start address of a valid object
-  inline HeapWord* addr_after_obj(HeapWord* addr);
-
-  void print_on_error(outputStream* st, const char* prefix) const;
-
-  // Write marks.
-  inline void mark(HeapWord* addr);
-  inline void clear(HeapWord* addr);
-  inline bool par_mark(HeapWord* addr);
-
-  void clear_range(MemRegion mr);
-};
-
 // Represents the overflow mark stack used by concurrent marking.
 //
 // Stores oops in a huge buffer in virtual memory that is always fully committed.
--- a/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp	Fri Aug 04 14:15:42 2017 +0200
+++ b/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp	Fri Aug 04 14:24:11 2017 +0200
@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentMark.hpp"
+#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
 #include "gc/g1/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
@@ -36,59 +37,6 @@
   return _nextMarkBitMap->par_mark((HeapWord*)obj);
 }
 
-inline bool G1CMBitMap::iterate(G1CMBitMapClosure* cl, MemRegion mr) {
-  assert(!mr.is_empty(), "Does not support empty memregion to iterate over");
-  assert(_covered.contains(mr),
-         "Given MemRegion from " PTR_FORMAT " to " PTR_FORMAT " not contained in heap area",
-         p2i(mr.start()), p2i(mr.end()));
-
-  BitMap::idx_t const end_offset = addr_to_offset(mr.end());
-  BitMap::idx_t offset = _bm.get_next_one_offset(addr_to_offset(mr.start()), end_offset);
-
-  while (offset < end_offset) {
-    HeapWord* const addr = offset_to_addr(offset);
-    if (!cl->do_addr(addr)) {
-      return false;
-    }
-    size_t const obj_size = (size_t)((oop)addr)->size();
-    offset = _bm.get_next_one_offset(offset + (obj_size >> _shifter), end_offset);
-  }
-  return true;
-}
-
-inline HeapWord* G1CMBitMap::get_next_marked_addr(const HeapWord* addr,
-                                                  const HeapWord* limit) const {
-  assert(limit != NULL, "limit must not be NULL");
-  // Round addr up to a possible object boundary to be safe.
-  size_t const addr_offset = addr_to_offset(align_up(addr, HeapWordSize << _shifter));
-  size_t const limit_offset = addr_to_offset(limit);
-  size_t const nextOffset = _bm.get_next_one_offset(addr_offset, limit_offset);
-  return offset_to_addr(nextOffset);
-}
-
-#ifdef ASSERT
-inline void G1CMBitMap::check_mark(HeapWord* addr) {
-  assert(G1CollectedHeap::heap()->is_in_exact(addr),
-         "Trying to access bitmap " PTR_FORMAT " for address " PTR_FORMAT " not in the heap.",
-         p2i(this), p2i(addr));
-}
-#endif
-
-inline void G1CMBitMap::mark(HeapWord* addr) {
-  check_mark(addr);
-  _bm.set_bit(addr_to_offset(addr));
-}
-
-inline void G1CMBitMap::clear(HeapWord* addr) {
-  check_mark(addr);
-  _bm.clear_bit(addr_to_offset(addr));
-}
-
-inline bool G1CMBitMap::par_mark(HeapWord* addr) {
-  check_mark(addr);
-  return _bm.par_set_bit(addr_to_offset(addr));
-}
-
 #ifndef PRODUCT
 template<typename Fn>
 inline void G1CMMarkStack::iterate(Fn fn) const {
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.cpp	Fri Aug 04 14:24:11 2017 +0200
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
+#include "memory/virtualspace.hpp"
+
+void G1CMBitMap::print_on_error(outputStream* st, const char* prefix) const {
+  _bm.print_on_error(st, prefix);
+}
+
+size_t G1CMBitMap::compute_size(size_t heap_size) {
+  return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
+}
+
+size_t G1CMBitMap::mark_distance() {
+  return MinObjAlignmentInBytes * BitsPerByte;
+}
+
+void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
+  _covered = heap;
+
+  _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _covered.word_size() >> _shifter);
+
+  storage->set_mapping_changed_listener(&_listener);
+}
+
+void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
+  if (zero_filled) {
+    return;
+  }
+  // We need to clear the bitmap on commit, removing any existing information.
+  MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
+  _bm->clear_range(mr);
+}
+
+void G1CMBitMap::clear_range(MemRegion mr) {
+  MemRegion intersection = mr.intersection(_covered);
+  assert(!intersection.is_empty(),
+         "Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap",
+         p2i(mr.start()), p2i(mr.end()));
+  // convert address range into offset range
+  _bm.at_put_range(addr_to_offset(intersection.start()),
+                   addr_to_offset(intersection.end()), false);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.hpp	Fri Aug 04 14:24:11 2017 +0200
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP
+#define SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP
+
+#include "gc/g1/g1RegionToSpaceMapper.hpp"
+#include "memory/allocation.hpp"
+#include "memory/memRegion.hpp"
+#include "utilities/bitMap.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+
+class G1CMBitMap;
+class G1CMTask;
+class G1ConcurrentMark;
+
+// Closure for iteration over bitmaps
+class G1CMBitMapClosure VALUE_OBJ_CLASS_SPEC {
+private:
+  G1ConcurrentMark* const _cm;
+  G1CMTask* const _task;
+public:
+  G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm) : _task(task), _cm(cm) { }
+
+  bool do_addr(HeapWord* const addr);
+};
+
+class G1CMBitMapMappingChangedListener : public G1MappingChangedListener {
+ private:
+  G1CMBitMap* _bm;
+ public:
+  G1CMBitMapMappingChangedListener() : _bm(NULL) {}
+
+  void set_bitmap(G1CMBitMap* bm) { _bm = bm; }
+
+  virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled);
+};
+
+// A generic mark bitmap for concurrent marking.  This is essentially a wrapper
+// around the BitMap class that is based on HeapWords, with one bit per (1 << _shifter) HeapWords.
+class G1CMBitMap VALUE_OBJ_CLASS_SPEC {
+private:
+  MemRegion _covered;    // The heap area covered by this bitmap.
+
+  const int _shifter;    // Shift amount from heap index to bit index in the bitmap.
+
+  BitMapView _bm;        // The actual bitmap.
+
+  G1CMBitMapMappingChangedListener _listener;
+
+  inline void check_mark(HeapWord* addr) NOT_DEBUG_RETURN;
+
+  // Convert from bit offset to address.
+  HeapWord* offset_to_addr(size_t offset) const {
+    return _covered.start() + (offset << _shifter);
+  }
+  // Convert from address to bit offset.
+  size_t addr_to_offset(const HeapWord* addr) const {
+    return pointer_delta(addr, _covered.start()) >> _shifter;
+  }
+public:
+  static size_t compute_size(size_t heap_size);
+  // Returns the amount of bytes on the heap between two marks in the bitmap.
+  static size_t mark_distance();
+  // Returns how many bytes (or bits) of the heap a single byte (or bit) of the
+  // mark bitmap corresponds to. This is the same as the mark distance above.
+  static size_t heap_map_factor() {
+    return mark_distance();
+  }
+
+  G1CMBitMap() : _covered(), _bm(), _shifter(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); }
+
+  // Initializes the underlying BitMap to cover the given area.
+  void initialize(MemRegion heap, G1RegionToSpaceMapper* storage);
+
+  // Read marks
+  bool is_marked(HeapWord* addr) const {
+    assert(_covered.contains(addr),
+           "Address " PTR_FORMAT " is outside underlying space from " PTR_FORMAT " to " PTR_FORMAT,
+           p2i(addr), p2i(_covered.start()), p2i(_covered.end()));
+    return _bm.at(addr_to_offset(addr));
+  }
+
+  // Apply the closure to the addresses that correspond to marked bits in the bitmap.
+  inline bool iterate(G1CMBitMapClosure* cl, MemRegion mr);
+
+  // Return the address corresponding to the next marked bit at or after
+  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
+  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
+  inline HeapWord* get_next_marked_addr(const HeapWord* addr,
+                                        const HeapWord* limit) const;
+
+  // The argument addr should be the start address of a valid object
+  inline HeapWord* addr_after_obj(HeapWord* addr);
+
+  void print_on_error(outputStream* st, const char* prefix) const;
+
+  // Write marks.
+  inline void mark(HeapWord* addr);
+  inline void clear(HeapWord* addr);
+  inline bool par_mark(HeapWord* addr);
+
+  void clear_range(MemRegion mr);
+};
+
+#endif // SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.inline.hpp	Fri Aug 04 14:24:11 2017 +0200
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP
+#define SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP
+
+#include "gc/g1/g1ConcurrentMarkBitMap.hpp"
+#include "memory/memRegion.hpp"
+#include "utilities/align.hpp"
+#include "utilities/bitMap.inline.hpp"
+
+inline bool G1CMBitMap::iterate(G1CMBitMapClosure* cl, MemRegion mr) {
+  assert(!mr.is_empty(), "Does not support empty memregion to iterate over");
+  assert(_covered.contains(mr),
+         "Given MemRegion from " PTR_FORMAT " to " PTR_FORMAT " not contained in heap area",
+         p2i(mr.start()), p2i(mr.end()));
+
+  BitMap::idx_t const end_offset = addr_to_offset(mr.end());
+  BitMap::idx_t offset = _bm.get_next_one_offset(addr_to_offset(mr.start()), end_offset);
+
+  while (offset < end_offset) {
+    HeapWord* const addr = offset_to_addr(offset);
+    if (!cl->do_addr(addr)) {
+      return false;
+    }
+    size_t const obj_size = (size_t)((oop)addr)->size();
+    offset = _bm.get_next_one_offset(offset + (obj_size >> _shifter), end_offset);
+  }
+  return true;
+}
+
+inline HeapWord* G1CMBitMap::get_next_marked_addr(const HeapWord* addr,
+                                                  const HeapWord* limit) const {
+  assert(limit != NULL, "limit must not be NULL");
+  // Round addr up to a possible object boundary to be safe.
+  size_t const addr_offset = addr_to_offset(align_up(addr, HeapWordSize << _shifter));
+  size_t const limit_offset = addr_to_offset(limit);
+  size_t const nextOffset = _bm.get_next_one_offset(addr_offset, limit_offset);
+  return offset_to_addr(nextOffset);
+}
+
+#ifdef ASSERT
+inline void G1CMBitMap::check_mark(HeapWord* addr) {
+  assert(G1CollectedHeap::heap()->is_in_exact(addr),
+         "Trying to access bitmap " PTR_FORMAT " for address " PTR_FORMAT " not in the heap.",
+         p2i(this), p2i(addr));
+}
+#endif
+
+inline void G1CMBitMap::mark(HeapWord* addr) {
+  check_mark(addr);
+  _bm.set_bit(addr_to_offset(addr));
+}
+
+inline void G1CMBitMap::clear(HeapWord* addr) {
+  check_mark(addr);
+  _bm.clear_bit(addr_to_offset(addr));
+}
+
+inline bool G1CMBitMap::par_mark(HeapWord* addr) {
+  check_mark(addr);
+  return _bm.par_set_bit(addr_to_offset(addr));
+}
+
+#endif // SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP