--- a/make/linux/makefiles/top.make Fri Sep 26 13:33:15 2008 -0400
+++ b/make/linux/makefiles/top.make Tue Sep 30 12:24:27 2008 -0400
@@ -64,6 +64,7 @@ Include_DBs/GC = $(VM)/includeD
$(VM)/gc_implementation/includeDB_gc_parallelScavenge \
$(VM)/gc_implementation/includeDB_gc_concurrentMarkSweep \
$(VM)/gc_implementation/includeDB_gc_parNew \
+ $(VM)/gc_implementation/includeDB_gc_g1 \
$(VM)/gc_implementation/includeDB_gc_serial \
$(VM)/gc_implementation/includeDB_gc_shared
--- a/make/solaris/makefiles/top.make Fri Sep 26 13:33:15 2008 -0400
+++ b/make/solaris/makefiles/top.make Tue Sep 30 12:24:27 2008 -0400
@@ -54,6 +54,7 @@ Include_DBs/GC = $(VM)/includeDB_gc
$(VM)/gc_implementation/includeDB_gc_parallelScavenge \
$(VM)/gc_implementation/includeDB_gc_concurrentMarkSweep \
$(VM)/gc_implementation/includeDB_gc_parNew \
+ $(VM)/gc_implementation/includeDB_gc_g1 \
$(VM)/gc_implementation/includeDB_gc_serial \
$(VM)/gc_implementation/includeDB_gc_shared
--- a/make/windows/makefiles/generated.make Fri Sep 26 13:33:15 2008 -0400
+++ b/make/windows/makefiles/generated.make Tue Sep 30 12:24:27 2008 -0400
@@ -50,7 +50,8 @@ IncludeDBs_gc= $(WorkSpace)/src/share/vm
$(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_parallelScavenge \
$(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_shared \
$(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_parNew \
- $(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep
+ $(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep \
+ $(WorkSpace)/src/share/vm/gc_implementation/includeDB_gc_g1
IncludeDBs_core=$(IncludeDBs_base) $(IncludeDBs_gc) \
$(WorkSpace)/src/share/vm/includeDB_features
--- a/make/windows/makefiles/makedeps.make Fri Sep 26 13:33:15 2008 -0400
+++ b/make/windows/makefiles/makedeps.make Tue Sep 30 12:24:27 2008 -0400
@@ -64,6 +64,7 @@ MakeDepsIncludesPRIVATE=\
-relativeInclude src\share\vm\gc_implementation\shared \
-relativeInclude src\share\vm\gc_implementation\parNew \
-relativeInclude src\share\vm\gc_implementation\concurrentMarkSweep \
+ -relativeInclude src\share\vm\gc_implementation\g1 \
-relativeInclude src\share\vm\gc_interface \
-relativeInclude src\share\vm\asm \
-relativeInclude src\share\vm\memory \
@@ -115,6 +116,7 @@ MakeDepsIDEOptions=\
-additionalFile includeDB_gc_parallel \
-additionalFile includeDB_gc_parallelScavenge \
-additionalFile includeDB_gc_concurrentMarkSweep \
+ -additionalFile includeDB_gc_g1 \
-additionalFile includeDB_gc_parNew \
-additionalFile includeDB_gc_shared \
-additionalFile includeDB_gc_serial \
--- a/make/windows/makefiles/vm.make Fri Sep 26 13:33:15 2008 -0400
+++ b/make/windows/makefiles/vm.make Tue Sep 30 12:24:27 2008 -0400
@@ -117,6 +117,7 @@ CPP_INCLUDE_DIRS=\
/I "$(WorkSpace)\src\share\vm\gc_implementation\shared"\
/I "$(WorkSpace)\src\share\vm\gc_implementation\parNew"\
/I "$(WorkSpace)\src\share\vm\gc_implementation\concurrentMarkSweep"\
+ /I "$(WorkSpace)\src\share\vm\gc_implementation\g1"\
/I "$(WorkSpace)\src\share\vm\gc_interface"\
/I "$(WorkSpace)\src\share\vm\asm" \
/I "$(WorkSpace)\src\share\vm\memory" \
@@ -146,6 +147,7 @@ VM_PATH=$(VM_PATH);$(WorkSpace)/src/shar
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/gc_implementation/shared
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/gc_implementation/parNew
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/gc_implementation/concurrentMarkSweep
+VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/gc_implementation/g1
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/gc_interface
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/asm
VM_PATH=$(VM_PATH);$(WorkSpace)/src/share/vm/memory
@@ -222,6 +224,9 @@ bytecodeInterpreterWithChecks.obj: ..\ge
{$(WorkSpace)\src\share\vm\gc_implementation\concurrentMarkSweep}.cpp.obj::
$(CPP) $(CPP_FLAGS) $(CPP_USE_PCH) /c $<
+{$(WorkSpace)\src\share\vm\gc_implementation\g1}.cpp.obj::
+ $(CPP) $(CPP_FLAGS) $(CPP_USE_PCH) /c $<
+
{$(WorkSpace)\src\share\vm\gc_interface}.cpp.obj::
$(CPP) $(CPP_FLAGS) $(CPP_USE_PCH) /c $<
--- a/src/cpu/sparc/vm/assembler_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/assembler_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -128,6 +128,20 @@ int Assembler::branch_destination(int in
int AbstractAssembler::code_fill_byte() {
return 0x00; // illegal instruction 0x00000000
+}
+
+Assembler::Condition Assembler::reg_cond_to_cc_cond(Assembler::RCondition in) {
+ switch (in) {
+ case rc_z: return equal;
+ case rc_lez: return lessEqual;
+ case rc_lz: return less;
+ case rc_nz: return notEqual;
+ case rc_gz: return greater;
+ case rc_gez: return greaterEqual;
+ default:
+ ShouldNotReachHere();
+ }
+ return equal;
}
// Generate a bunch 'o stuff (including v9's
@@ -1213,29 +1227,17 @@ void MacroAssembler::set_vm_result(Regis
}
-void MacroAssembler::store_check(Register tmp, Register obj) {
- // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
-
- /* $$$ This stuff needs to go into one of the BarrierSet generator
- functions. (The particular barrier sets will have to be friends of
- MacroAssembler, I guess.) */
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
- CardTableModRefBS* ct = (CardTableModRefBS*)bs;
- assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
+void MacroAssembler::card_table_write(jbyte* byte_map_base,
+ Register tmp, Register obj) {
#ifdef _LP64
srlx(obj, CardTableModRefBS::card_shift, obj);
#else
srl(obj, CardTableModRefBS::card_shift, obj);
#endif
assert( tmp != obj, "need separate temp reg");
- Address rs(tmp, (address)ct->byte_map_base);
+ Address rs(tmp, (address)byte_map_base);
load_address(rs);
stb(G0, rs.base(), obj);
-}
-
-void MacroAssembler::store_check(Register tmp, Register obj, Register offset) {
- store_check(tmp, obj);
}
// %%% Note: The following six instructions have been moved,
@@ -1663,11 +1665,21 @@ void MacroAssembler::_verify_oop(Registe
if (reg == G0) return; // always NULL, which is always an oop
- char buffer[16];
+ char buffer[64];
+#ifdef COMPILER1
+ if (CommentedAssembly) {
+ snprintf(buffer, sizeof(buffer), "verify_oop at %d", offset());
+ block_comment(buffer);
+ }
+#endif
+
+ int len = strlen(file) + strlen(msg) + 1 + 4;
sprintf(buffer, "%d", line);
- int len = strlen(file) + strlen(msg) + 1 + 4 + strlen(buffer);
+ len += strlen(buffer);
+ sprintf(buffer, " at offset %d ", offset());
+ len += strlen(buffer);
char * real_msg = new char[len];
- sprintf(real_msg, "%s (%s:%d)", msg, file, line);
+ sprintf(real_msg, "%s%s(%s:%d)", msg, buffer, file, line);
// Call indirectly to solve generation ordering problem
Address a(O7, (address)StubRoutines::verify_oop_subroutine_entry_address());
@@ -2057,6 +2069,27 @@ void MacroAssembler::br_notnull( Registe
tst(s1);
br ( notZero, a, p, L );
#endif
+}
+
+void MacroAssembler::br_on_reg_cond( RCondition rc, bool a, Predict p,
+ Register s1, address d,
+ relocInfo::relocType rt ) {
+ if (VM_Version::v9_instructions_work()) {
+ bpr(rc, a, p, s1, d, rt);
+ } else {
+ tst(s1);
+ br(reg_cond_to_cc_cond(rc), a, p, d, rt);
+ }
+}
+
+void MacroAssembler::br_on_reg_cond( RCondition rc, bool a, Predict p,
+ Register s1, Label& L ) {
+ if (VM_Version::v9_instructions_work()) {
+ bpr(rc, a, p, s1, L);
+ } else {
+ tst(s1);
+ br(reg_cond_to_cc_cond(rc), a, p, L);
+ }
}
@@ -3241,68 +3274,74 @@ void MacroAssembler::eden_allocate(
assert(0 <= con_size_in_bytes && Assembler::is_simm13(con_size_in_bytes), "illegal object size");
assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
- // get eden boundaries
- // note: we need both top & top_addr!
- const Register top_addr = t1;
- const Register end = t2;
-
- CollectedHeap* ch = Universe::heap();
- set((intx)ch->top_addr(), top_addr);
- intx delta = (intx)ch->end_addr() - (intx)ch->top_addr();
- ld_ptr(top_addr, delta, end);
- ld_ptr(top_addr, 0, obj);
-
- // try to allocate
- Label retry;
- bind(retry);
+ if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
+ // No allocation in the shared eden.
+ br(Assembler::always, false, Assembler::pt, slow_case);
+ delayed()->nop();
+ } else {
+ // get eden boundaries
+ // note: we need both top & top_addr!
+ const Register top_addr = t1;
+ const Register end = t2;
+
+ CollectedHeap* ch = Universe::heap();
+ set((intx)ch->top_addr(), top_addr);
+ intx delta = (intx)ch->end_addr() - (intx)ch->top_addr();
+ ld_ptr(top_addr, delta, end);
+ ld_ptr(top_addr, 0, obj);
+
+ // try to allocate
+ Label retry;
+ bind(retry);
#ifdef ASSERT
- // make sure eden top is properly aligned
- {
- Label L;
- btst(MinObjAlignmentInBytesMask, obj);
- br(Assembler::zero, false, Assembler::pt, L);
- delayed()->nop();
- stop("eden top is not properly aligned");
- bind(L);
- }
+ // make sure eden top is properly aligned
+ {
+ Label L;
+ btst(MinObjAlignmentInBytesMask, obj);
+ br(Assembler::zero, false, Assembler::pt, L);
+ delayed()->nop();
+ stop("eden top is not properly aligned");
+ bind(L);
+ }
#endif // ASSERT
- const Register free = end;
- sub(end, obj, free); // compute amount of free space
- if (var_size_in_bytes->is_valid()) {
- // size is unknown at compile time
- cmp(free, var_size_in_bytes);
- br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
- delayed()->add(obj, var_size_in_bytes, end);
- } else {
- // size is known at compile time
- cmp(free, con_size_in_bytes);
- br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
- delayed()->add(obj, con_size_in_bytes, end);
- }
- // Compare obj with the value at top_addr; if still equal, swap the value of
- // end with the value at top_addr. If not equal, read the value at top_addr
- // into end.
- casx_under_lock(top_addr, obj, end, (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
- // if someone beat us on the allocation, try again, otherwise continue
- cmp(obj, end);
- brx(Assembler::notEqual, false, Assembler::pn, retry);
- delayed()->mov(end, obj); // nop if successfull since obj == end
+ const Register free = end;
+ sub(end, obj, free); // compute amount of free space
+ if (var_size_in_bytes->is_valid()) {
+ // size is unknown at compile time
+ cmp(free, var_size_in_bytes);
+ br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
+ delayed()->add(obj, var_size_in_bytes, end);
+ } else {
+ // size is known at compile time
+ cmp(free, con_size_in_bytes);
+ br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
+ delayed()->add(obj, con_size_in_bytes, end);
+ }
+ // Compare obj with the value at top_addr; if still equal, swap the value of
+ // end with the value at top_addr. If not equal, read the value at top_addr
+ // into end.
+ casx_under_lock(top_addr, obj, end, (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
+ // if someone beat us on the allocation, try again, otherwise continue
+ cmp(obj, end);
+ brx(Assembler::notEqual, false, Assembler::pn, retry);
+ delayed()->mov(end, obj); // nop if successfull since obj == end
#ifdef ASSERT
- // make sure eden top is properly aligned
- {
- Label L;
- const Register top_addr = t1;
-
- set((intx)ch->top_addr(), top_addr);
- ld_ptr(top_addr, 0, top_addr);
- btst(MinObjAlignmentInBytesMask, top_addr);
- br(Assembler::zero, false, Assembler::pt, L);
- delayed()->nop();
- stop("eden top is not properly aligned");
- bind(L);
- }
+ // make sure eden top is properly aligned
+ {
+ Label L;
+ const Register top_addr = t1;
+
+ set((intx)ch->top_addr(), top_addr);
+ ld_ptr(top_addr, 0, top_addr);
+ btst(MinObjAlignmentInBytesMask, top_addr);
+ br(Assembler::zero, false, Assembler::pt, L);
+ delayed()->nop();
+ stop("eden top is not properly aligned");
+ bind(L);
+ }
#endif // ASSERT
+ }
}
@@ -3554,6 +3593,468 @@ void MacroAssembler::bang_stack_size(Reg
}
}
+///////////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+static uint num_stores = 0;
+static uint num_null_pre_stores = 0;
+
+static void count_null_pre_vals(void* pre_val) {
+ num_stores++;
+ if (pre_val == NULL) num_null_pre_stores++;
+ if ((num_stores % 1000000) == 0) {
+ tty->print_cr(UINT32_FORMAT " stores, " UINT32_FORMAT " (%5.2f%%) with null pre-vals.",
+ num_stores, num_null_pre_stores,
+ 100.0*(float)num_null_pre_stores/(float)num_stores);
+ }
+}
+
+static address satb_log_enqueue_with_frame = 0;
+static u_char* satb_log_enqueue_with_frame_end = 0;
+
+static address satb_log_enqueue_frameless = 0;
+static u_char* satb_log_enqueue_frameless_end = 0;
+
+static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions?
+
+// The calls to this don't work. We'd need to do a fair amount of work to
+// make it work.
+static void check_index(int ind) {
+ assert(0 <= ind && ind <= 64*K && ((ind % oopSize) == 0),
+ "Invariants.")
+}
+
+static void generate_satb_log_enqueue(bool with_frame) {
+ BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize);
+ CodeBuffer buf(bb->instructions_begin(), bb->instructions_size());
+ MacroAssembler masm(&buf);
+ address start = masm.pc();
+ Register pre_val;
+
+ Label refill, restart;
+ if (with_frame) {
+ masm.save_frame(0);
+ pre_val = I0; // Was O0 before the save.
+ } else {
+ pre_val = O0;
+ }
+ int satb_q_index_byte_offset =
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_index());
+ int satb_q_buf_byte_offset =
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_buf());
+ assert(in_bytes(PtrQueue::byte_width_of_index()) == sizeof(intptr_t) &&
+ in_bytes(PtrQueue::byte_width_of_buf()) == sizeof(intptr_t),
+ "check sizes in assembly below");
+
+ masm.bind(restart);
+ masm.ld_ptr(G2_thread, satb_q_index_byte_offset, L0);
+
+ masm.br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pn, L0, refill);
+ // If the branch is taken, no harm in executing this in the delay slot.
+ masm.delayed()->ld_ptr(G2_thread, satb_q_buf_byte_offset, L1);
+ masm.sub(L0, oopSize, L0);
+
+ masm.st_ptr(pre_val, L1, L0); // [_buf + index] := I0
+ if (!with_frame) {
+ // Use return-from-leaf
+ masm.retl();
+ masm.delayed()->st_ptr(L0, G2_thread, satb_q_index_byte_offset);
+ } else {
+ // Not delayed.
+ masm.st_ptr(L0, G2_thread, satb_q_index_byte_offset);
+ }
+ if (with_frame) {
+ masm.ret();
+ masm.delayed()->restore();
+ }
+ masm.bind(refill);
+
+ address handle_zero =
+ CAST_FROM_FN_PTR(address,
+ &SATBMarkQueueSet::handle_zero_index_for_thread);
+ // This should be rare enough that we can afford to save all the
+ // scratch registers that the calling context might be using.
+ masm.mov(G1_scratch, L0);
+ masm.mov(G3_scratch, L1);
+ masm.mov(G4, L2);
+ // We need the value of O0 above (for the write into the buffer), so we
+ // save and restore it.
+ masm.mov(O0, L3);
+ // Since the call will overwrite O7, we save and restore that, as well.
+ masm.mov(O7, L4);
+ masm.call_VM_leaf(L5, handle_zero, G2_thread);
+ masm.mov(L0, G1_scratch);
+ masm.mov(L1, G3_scratch);
+ masm.mov(L2, G4);
+ masm.mov(L3, O0);
+ masm.br(Assembler::always, /*annul*/false, Assembler::pt, restart);
+ masm.delayed()->mov(L4, O7);
+
+ if (with_frame) {
+ satb_log_enqueue_with_frame = start;
+ satb_log_enqueue_with_frame_end = masm.pc();
+ } else {
+ satb_log_enqueue_frameless = start;
+ satb_log_enqueue_frameless_end = masm.pc();
+ }
+}
+
+static inline void generate_satb_log_enqueue_if_necessary(bool with_frame) {
+ if (with_frame) {
+ if (satb_log_enqueue_with_frame == 0) {
+ generate_satb_log_enqueue(with_frame);
+ assert(satb_log_enqueue_with_frame != 0, "postcondition.");
+ if (G1SATBPrintStubs) {
+ tty->print_cr("Generated with-frame satb enqueue:");
+ Disassembler::decode((u_char*)satb_log_enqueue_with_frame,
+ satb_log_enqueue_with_frame_end,
+ tty);
+ }
+ }
+ } else {
+ if (satb_log_enqueue_frameless == 0) {
+ generate_satb_log_enqueue(with_frame);
+ assert(satb_log_enqueue_frameless != 0, "postcondition.");
+ if (G1SATBPrintStubs) {
+ tty->print_cr("Generated frameless satb enqueue:");
+ Disassembler::decode((u_char*)satb_log_enqueue_frameless,
+ satb_log_enqueue_frameless_end,
+ tty);
+ }
+ }
+ }
+}
+
+void MacroAssembler::g1_write_barrier_pre(Register obj, Register index, int offset, Register tmp, bool preserve_o_regs) {
+ assert(offset == 0 || index == noreg, "choose one");
+
+ if (G1DisablePreBarrier) return;
+ // satb_log_barrier(tmp, obj, offset, preserve_o_regs);
+ Label filtered;
+ // satb_log_barrier_work0(tmp, filtered);
+ if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
+ ld(G2,
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()),
+ tmp);
+ } else {
+ guarantee(in_bytes(PtrQueue::byte_width_of_active()) == 1,
+ "Assumption");
+ ldsb(G2,
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()),
+ tmp);
+ }
+ // Check on whether to annul.
+ br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
+ delayed() -> nop();
+
+ // satb_log_barrier_work1(tmp, offset);
+ if (index == noreg) {
+ if (Assembler::is_simm13(offset)) {
+ ld_ptr(obj, offset, tmp);
+ } else {
+ set(offset, tmp);
+ ld_ptr(obj, tmp, tmp);
+ }
+ } else {
+ ld_ptr(obj, index, tmp);
+ }
+
+ // satb_log_barrier_work2(obj, tmp, offset);
+
+ // satb_log_barrier_work3(tmp, filtered, preserve_o_regs);
+
+ const Register pre_val = tmp;
+
+ if (G1SATBBarrierPrintNullPreVals) {
+ save_frame(0);
+ mov(pre_val, O0);
+ // Save G-regs that target may use.
+ mov(G1, L1);
+ mov(G2, L2);
+ mov(G3, L3);
+ mov(G4, L4);
+ mov(G5, L5);
+ call(CAST_FROM_FN_PTR(address, &count_null_pre_vals));
+ delayed()->nop();
+ // Restore G-regs that target may have used.
+ mov(L1, G1);
+ mov(L2, G2);
+ mov(L3, G3);
+ mov(L4, G4);
+ mov(L5, G5);
+ restore(G0, G0, G0);
+ }
+
+ // Check on whether to annul.
+ br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, pre_val, filtered);
+ delayed() -> nop();
+
+ // OK, it's not filtered, so we'll need to call enqueue. In the normal
+ // case, pre_val will be a scratch G-reg, but there's some cases in which
+ // it's an O-reg. In the first case, do a normal call. In the latter,
+ // do a save here and call the frameless version.
+
+ guarantee(pre_val->is_global() || pre_val->is_out(),
+ "Or we need to think harder.");
+ if (pre_val->is_global() && !preserve_o_regs) {
+ generate_satb_log_enqueue_if_necessary(true); // with frame.
+ call(satb_log_enqueue_with_frame);
+ delayed()->mov(pre_val, O0);
+ } else {
+ generate_satb_log_enqueue_if_necessary(false); // with frameless.
+ save_frame(0);
+ call(satb_log_enqueue_frameless);
+ delayed()->mov(pre_val->after_save(), O0);
+ restore();
+ }
+
+ bind(filtered);
+}
+
+static jint num_ct_writes = 0;
+static jint num_ct_writes_filtered_in_hr = 0;
+static jint num_ct_writes_filtered_null = 0;
+static jint num_ct_writes_filtered_pop = 0;
+static G1CollectedHeap* g1 = NULL;
+
+static Thread* count_ct_writes(void* filter_val, void* new_val) {
+ Atomic::inc(&num_ct_writes);
+ if (filter_val == NULL) {
+ Atomic::inc(&num_ct_writes_filtered_in_hr);
+ } else if (new_val == NULL) {
+ Atomic::inc(&num_ct_writes_filtered_null);
+ } else {
+ if (g1 == NULL) {
+ g1 = G1CollectedHeap::heap();
+ }
+ if ((HeapWord*)new_val < g1->popular_object_boundary()) {
+ Atomic::inc(&num_ct_writes_filtered_pop);
+ }
+ }
+ if ((num_ct_writes % 1000000) == 0) {
+ jint num_ct_writes_filtered =
+ num_ct_writes_filtered_in_hr +
+ num_ct_writes_filtered_null +
+ num_ct_writes_filtered_pop;
+
+ tty->print_cr("%d potential CT writes: %5.2f%% filtered\n"
+ " (%5.2f%% intra-HR, %5.2f%% null, %5.2f%% popular).",
+ num_ct_writes,
+ 100.0*(float)num_ct_writes_filtered/(float)num_ct_writes,
+ 100.0*(float)num_ct_writes_filtered_in_hr/
+ (float)num_ct_writes,
+ 100.0*(float)num_ct_writes_filtered_null/
+ (float)num_ct_writes,
+ 100.0*(float)num_ct_writes_filtered_pop/
+ (float)num_ct_writes);
+ }
+ return Thread::current();
+}
+
+static address dirty_card_log_enqueue = 0;
+static u_char* dirty_card_log_enqueue_end = 0;
+
+// This gets to assume that o0 contains the object address.
+static void generate_dirty_card_log_enqueue(jbyte* byte_map_base) {
+ BufferBlob* bb = BufferBlob::create("dirty_card_enqueue", EnqueueCodeSize*2);
+ CodeBuffer buf(bb->instructions_begin(), bb->instructions_size());
+ MacroAssembler masm(&buf);
+ address start = masm.pc();
+
+ Label not_already_dirty, restart, refill;
+
+#ifdef _LP64
+ masm.srlx(O0, CardTableModRefBS::card_shift, O0);
+#else
+ masm.srl(O0, CardTableModRefBS::card_shift, O0);
+#endif
+ Address rs(O1, (address)byte_map_base);
+ masm.load_address(rs); // O1 := <card table base>
+ masm.ldub(O0, O1, O2); // O2 := [O0 + O1]
+
+ masm.br_on_reg_cond(Assembler::rc_nz, /*annul*/false, Assembler::pt,
+ O2, not_already_dirty);
+ // Get O1 + O2 into a reg by itself -- useful in the take-the-branch
+ // case, harmless if not.
+ masm.delayed()->add(O0, O1, O3);
+
+ // We didn't take the branch, so we're already dirty: return.
+ // Use return-from-leaf
+ masm.retl();
+ masm.delayed()->nop();
+
+ // Not dirty.
+ masm.bind(not_already_dirty);
+ // First, dirty it.
+ masm.stb(G0, O3, G0); // [cardPtr] := 0 (i.e., dirty).
+ int dirty_card_q_index_byte_offset =
+ in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_index());
+ int dirty_card_q_buf_byte_offset =
+ in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_buf());
+ masm.bind(restart);
+ masm.ld_ptr(G2_thread, dirty_card_q_index_byte_offset, L0);
+
+ masm.br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pn,
+ L0, refill);
+ // If the branch is taken, no harm in executing this in the delay slot.
+ masm.delayed()->ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, L1);
+ masm.sub(L0, oopSize, L0);
+
+ masm.st_ptr(O3, L1, L0); // [_buf + index] := I0
+ // Use return-from-leaf
+ masm.retl();
+ masm.delayed()->st_ptr(L0, G2_thread, dirty_card_q_index_byte_offset);
+
+ masm.bind(refill);
+ address handle_zero =
+ CAST_FROM_FN_PTR(address,
+ &DirtyCardQueueSet::handle_zero_index_for_thread);
+ // This should be rare enough that we can afford to save all the
+ // scratch registers that the calling context might be using.
+ masm.mov(G1_scratch, L3);
+ masm.mov(G3_scratch, L5);
+ // We need the value of O3 above (for the write into the buffer), so we
+ // save and restore it.
+ masm.mov(O3, L6);
+ // Since the call will overwrite O7, we save and restore that, as well.
+ masm.mov(O7, L4);
+
+ masm.call_VM_leaf(L7_thread_cache, handle_zero, G2_thread);
+ masm.mov(L3, G1_scratch);
+ masm.mov(L5, G3_scratch);
+ masm.mov(L6, O3);
+ masm.br(Assembler::always, /*annul*/false, Assembler::pt, restart);
+ masm.delayed()->mov(L4, O7);
+
+ dirty_card_log_enqueue = start;
+ dirty_card_log_enqueue_end = masm.pc();
+ // XXX Should have a guarantee here about not going off the end!
+ // Does it already do so? Do an experiment...
+}
+
+static inline void
+generate_dirty_card_log_enqueue_if_necessary(jbyte* byte_map_base) {
+ if (dirty_card_log_enqueue == 0) {
+ generate_dirty_card_log_enqueue(byte_map_base);
+ assert(dirty_card_log_enqueue != 0, "postcondition.");
+ if (G1SATBPrintStubs) {
+ tty->print_cr("Generated dirty_card enqueue:");
+ Disassembler::decode((u_char*)dirty_card_log_enqueue,
+ dirty_card_log_enqueue_end,
+ tty);
+ }
+ }
+}
+
+
+void MacroAssembler::g1_write_barrier_post(Register store_addr, Register new_val, Register tmp) {
+
+ Label filtered;
+ MacroAssembler* post_filter_masm = this;
+
+ if (new_val == G0) return;
+ if (G1DisablePostBarrier) return;
+
+ G1SATBCardTableModRefBS* bs = (G1SATBCardTableModRefBS*) Universe::heap()->barrier_set();
+ assert(bs->kind() == BarrierSet::G1SATBCT ||
+ bs->kind() == BarrierSet::G1SATBCTLogging, "wrong barrier");
+ if (G1RSBarrierRegionFilter) {
+ xor3(store_addr, new_val, tmp);
+#ifdef _LP64
+ srlx(tmp, HeapRegion::LogOfHRGrainBytes, tmp);
+#else
+ srl(tmp, HeapRegion::LogOfHRGrainBytes, tmp);
+#endif
+ if (G1PrintCTFilterStats) {
+ guarantee(tmp->is_global(), "Or stats won't work...");
+ // This is a sleazy hack: I'm temporarily hijacking G2, which I
+ // promise to restore.
+ mov(new_val, G2);
+ save_frame(0);
+ mov(tmp, O0);
+ mov(G2, O1);
+ // Save G-regs that target may use.
+ mov(G1, L1);
+ mov(G2, L2);
+ mov(G3, L3);
+ mov(G4, L4);
+ mov(G5, L5);
+ call(CAST_FROM_FN_PTR(address, &count_ct_writes));
+ delayed()->nop();
+ mov(O0, G2);
+ // Restore G-regs that target may have used.
+ mov(L1, G1);
+ mov(L3, G3);
+ mov(L4, G4);
+ mov(L5, G5);
+ restore(G0, G0, G0);
+ }
+ // XXX Should I predict this taken or not? Does it mattern?
+ br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
+ delayed()->nop();
+ }
+
+ // Now we decide how to generate the card table write. If we're
+ // enqueueing, we call out to a generated function. Otherwise, we do it
+ // inline here.
+
+ if (G1RSBarrierUseQueue) {
+ // If the "store_addr" register is an "in" or "local" register, move it to
+ // a scratch reg so we can pass it as an argument.
+ bool use_scr = !(store_addr->is_global() || store_addr->is_out());
+ // Pick a scratch register different from "tmp".
+ Register scr = (tmp == G1_scratch ? G3_scratch : G1_scratch);
+ // Make sure we use up the delay slot!
+ if (use_scr) {
+ post_filter_masm->mov(store_addr, scr);
+ } else {
+ post_filter_masm->nop();
+ }
+ generate_dirty_card_log_enqueue_if_necessary(bs->byte_map_base);
+ save_frame(0);
+ call(dirty_card_log_enqueue);
+ if (use_scr) {
+ delayed()->mov(scr, O0);
+ } else {
+ delayed()->mov(store_addr->after_save(), O0);
+ }
+ restore();
+
+ } else {
+
+#ifdef _LP64
+ post_filter_masm->srlx(store_addr, CardTableModRefBS::card_shift, store_addr);
+#else
+ post_filter_masm->srl(store_addr, CardTableModRefBS::card_shift, store_addr);
+#endif
+ assert( tmp != store_addr, "need separate temp reg");
+ Address rs(tmp, (address)bs->byte_map_base);
+ load_address(rs);
+ stb(G0, rs.base(), store_addr);
+ }
+
+ bind(filtered);
+
+}
+
+#endif // SERIALGC
+///////////////////////////////////////////////////////////////////////////////////
+
+void MacroAssembler::card_write_barrier_post(Register store_addr, Register new_val, Register tmp) {
+ // If we're writing constant NULL, we can skip the write barrier.
+ if (new_val == G0) return;
+ CardTableModRefBS* bs = (CardTableModRefBS*) Universe::heap()->barrier_set();
+ assert(bs->kind() == BarrierSet::CardTableModRef ||
+ bs->kind() == BarrierSet::CardTableExtension, "wrong barrier");
+ card_table_write(bs->byte_map_base, tmp, store_addr);
+}
+
void MacroAssembler::load_klass(Register src_oop, Register klass) {
// The number of bytes in this code is used by
// MachCallDynamicJavaNode::ret_addr_offset()
--- a/src/cpu/sparc/vm/assembler_sparc.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/assembler_sparc.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -1439,7 +1439,11 @@ public:
// pp 214
void save( Register s1, Register s2, Register d ) { emit_long( op(arith_op) | rd(d) | op3(save_op3) | rs1(s1) | rs2(s2) ); }
- void save( Register s1, int simm13a, Register d ) { emit_long( op(arith_op) | rd(d) | op3(save_op3) | rs1(s1) | immed(true) | simm(simm13a, 13) ); }
+ void save( Register s1, int simm13a, Register d ) {
+ // make sure frame is at least large enough for the register save area
+ assert(-simm13a >= 16 * wordSize, "frame too small");
+ emit_long( op(arith_op) | rd(d) | op3(save_op3) | rs1(s1) | immed(true) | simm(simm13a, 13) );
+ }
void restore( Register s1 = G0, Register s2 = G0, Register d = G0 ) { emit_long( op(arith_op) | rd(d) | op3(restore_op3) | rs1(s1) | rs2(s2) ); }
void restore( Register s1, int simm13a, Register d ) { emit_long( op(arith_op) | rd(d) | op3(restore_op3) | rs1(s1) | immed(true) | simm(simm13a, 13) ); }
@@ -1594,6 +1598,11 @@ public:
inline void wrasi( Register d) { v9_only(); emit_long( op(arith_op) | rs1(d) | op3(wrreg_op3) | u_field(3, 29, 25)); }
inline void wrfprs( Register d) { v9_only(); emit_long( op(arith_op) | rs1(d) | op3(wrreg_op3) | u_field(6, 29, 25)); }
+ // For a given register condition, return the appropriate condition code
+ // Condition (the one you would use to get the same effect after "tst" on
+ // the target register.)
+ Assembler::Condition reg_cond_to_cc_cond(RCondition in);
+
// Creation
Assembler(CodeBuffer* code) : AbstractAssembler(code) {
@@ -1630,6 +1639,8 @@ class RegistersForDebugging : public Sta
// restore global registers in case C code disturbed them
static void restore_registers(MacroAssembler* a, Register r);
+
+
};
@@ -1722,6 +1733,12 @@ class MacroAssembler: public Assembler {
void br_null ( Register s1, bool a, Predict p, Label& L );
void br_notnull( Register s1, bool a, Predict p, Label& L );
+ // These versions will do the most efficient thing on v8 and v9. Perhaps
+ // this is what the routine above was meant to do, but it didn't (and
+ // didn't cover both target address kinds.)
+ void br_on_reg_cond( RCondition c, bool a, Predict p, Register s1, address d, relocInfo::relocType rt = relocInfo::none );
+ void br_on_reg_cond( RCondition c, bool a, Predict p, Register s1, Label& L);
+
inline void bp( Condition c, bool a, CC cc, Predict p, address d, relocInfo::relocType rt = relocInfo::none );
inline void bp( Condition c, bool a, CC cc, Predict p, Label& L );
@@ -2056,9 +2073,23 @@ class MacroAssembler: public Assembler {
#endif // ASSERT
public:
- // Stores
- void store_check(Register tmp, Register obj); // store check for obj - register is destroyed afterwards
- void store_check(Register tmp, Register obj, Register offset); // store check for obj - register is destroyed afterwards
+
+ // Write to card table for - register is destroyed afterwards.
+ void card_table_write(jbyte* byte_map_base, Register tmp, Register obj);
+
+ void card_write_barrier_post(Register store_addr, Register new_val, Register tmp);
+
+#ifndef SERIALGC
+ // Array store and offset
+ void g1_write_barrier_pre(Register obj, Register index, int offset, Register tmp, bool preserve_o_regs);
+
+ void g1_write_barrier_post(Register store_addr, Register new_val, Register tmp);
+
+ // May do filtering, depending on the boolean arguments.
+ void g1_card_table_write(jbyte* byte_map_base,
+ Register tmp, Register obj, Register new_val,
+ bool region_filter, bool null_filter);
+#endif // SERIALGC
// pushes double TOS element of FPU stack on CPU stack; pops from FPU stack
void push_fTOS();
--- a/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -404,4 +404,55 @@ void ArrayCopyStub::emit_code(LIR_Assemb
}
+///////////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+void G1PreBarrierStub::emit_code(LIR_Assembler* ce) {
+ __ bind(_entry);
+
+ assert(pre_val()->is_register(), "Precondition.");
+
+ Register pre_val_reg = pre_val()->as_register();
+
+ ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false);
+ __ br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pt,
+ pre_val_reg, _continuation);
+ __ delayed()->nop();
+
+ __ call(Runtime1::entry_for(Runtime1::Runtime1::g1_pre_barrier_slow_id));
+ __ delayed()->mov(pre_val_reg, G4);
+ __ br(Assembler::always, false, Assembler::pt, _continuation);
+ __ delayed()->nop();
+
+}
+
+jbyte* G1PostBarrierStub::_byte_map_base = NULL;
+
+jbyte* G1PostBarrierStub::byte_map_base_slow() {
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ assert(bs->is_a(BarrierSet::G1SATBCTLogging),
+ "Must be if we're using this.");
+ return ((G1SATBCardTableModRefBS*)bs)->byte_map_base;
+}
+
+void G1PostBarrierStub::emit_code(LIR_Assembler* ce) {
+ __ bind(_entry);
+
+ assert(addr()->is_register(), "Precondition.");
+ assert(new_val()->is_register(), "Precondition.");
+ Register addr_reg = addr()->as_pointer_register();
+ Register new_val_reg = new_val()->as_register();
+ __ br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pt,
+ new_val_reg, _continuation);
+ __ delayed()->nop();
+
+ __ call(Runtime1::entry_for(Runtime1::Runtime1::g1_post_barrier_slow_id));
+ __ delayed()->mov(addr_reg, G4);
+ __ br(Assembler::always, false, Assembler::pt, _continuation);
+ __ delayed()->nop();
+}
+
+#endif // SERIALGC
+///////////////////////////////////////////////////////////////////////////////////
+
#undef __
--- a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -2093,7 +2093,11 @@ void LIR_Assembler::emit_arraycopy(LIR_O
// the known type isn't loaded since the code sanity checks
// in debug mode and the type isn't required when we know the exact type
// also check that the type is an array type.
- if (op->expected_type() == NULL) {
+ // We also, for now, always call the stub if the barrier set requires a
+ // write_ref_pre barrier (which the stub does, but none of the optimized
+ // cases currently does).
+ if (op->expected_type() == NULL ||
+ Universe::heap()->barrier_set()->has_write_ref_pre_barrier()) {
__ mov(src, O0);
__ mov(src_pos, O1);
__ mov(dst, O2);
--- a/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -365,6 +365,10 @@ void LIRGenerator::do_StoreIndexed(Store
__ store_check(value.result(), array.result(), tmp1, tmp2, tmp3, store_check_info);
}
+ if (obj_store) {
+ // Needs GC write barriers.
+ pre_barrier(LIR_OprFact::address(array_addr), false, NULL);
+ }
__ move(value.result(), array_addr, null_check_info);
if (obj_store) {
// Is this precise?
@@ -663,6 +667,10 @@ void LIRGenerator::do_CompareAndSwap(Int
__ add(obj.result(), offset.result(), addr);
+ if (type == objectType) { // Write-barrier needed for Object fields.
+ pre_barrier(obj.result(), false, NULL);
+ }
+
if (type == objectType)
__ cas_obj(addr, cmp.result(), val.result(), t1, t2);
else if (type == intType)
@@ -677,7 +685,11 @@ void LIRGenerator::do_CompareAndSwap(Int
LIR_Opr result = rlock_result(x);
__ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result);
if (type == objectType) { // Write-barrier needed for Object fields.
+#ifdef PRECISE_CARDMARK
+ post_barrier(addr, val.result());
+#else
post_barrier(obj.result(), val.result());
+#endif // PRECISE_CARDMARK
}
}
@@ -1154,6 +1166,10 @@ void LIRGenerator::put_Object_unsafe(LIR
addr = new LIR_Address(base_op, index_op, type);
}
+ if (is_obj) {
+ pre_barrier(LIR_OprFact::address(addr), false, NULL);
+ // _bs->c1_write_barrier_pre(this, LIR_OprFact::address(addr));
+ }
__ move(data, addr);
if (is_obj) {
// This address is precise
--- a/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -832,6 +832,163 @@ OopMapSet* Runtime1::generate_code_for(S
}
break;
+#ifndef SERIALGC
+ case g1_pre_barrier_slow_id:
+ { // G4: previous value of memory
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ if (bs->kind() != BarrierSet::G1SATBCTLogging) {
+ __ save_frame(0);
+ __ set((int)id, O1);
+ __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0);
+ __ should_not_reach_here();
+ break;
+ }
+
+ __ set_info("g1_pre_barrier_slow_id", dont_gc_arguments);
+
+ Register pre_val = G4;
+ Register tmp = G1_scratch;
+ Register tmp2 = G3_scratch;
+
+ Label refill, restart;
+ bool with_frame = false; // I don't know if we can do with-frame.
+ int satb_q_index_byte_offset =
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_index());
+ int satb_q_buf_byte_offset =
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_buf());
+ __ bind(restart);
+ __ ld_ptr(G2_thread, satb_q_index_byte_offset, tmp);
+
+ __ br_on_reg_cond(Assembler::rc_z, /*annul*/false,
+ Assembler::pn, tmp, refill);
+
+ // If the branch is taken, no harm in executing this in the delay slot.
+ __ delayed()->ld_ptr(G2_thread, satb_q_buf_byte_offset, tmp2);
+ __ sub(tmp, oopSize, tmp);
+
+ __ st_ptr(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card>
+ // Use return-from-leaf
+ __ retl();
+ __ delayed()->st_ptr(tmp, G2_thread, satb_q_index_byte_offset);
+
+ __ bind(refill);
+ __ save_frame(0);
+
+ __ mov(pre_val, L0);
+ __ mov(tmp, L1);
+ __ mov(tmp2, L2);
+
+ __ call_VM_leaf(L7_thread_cache,
+ CAST_FROM_FN_PTR(address,
+ SATBMarkQueueSet::handle_zero_index_for_thread),
+ G2_thread);
+
+ __ mov(L0, pre_val);
+ __ mov(L1, tmp);
+ __ mov(L2, tmp2);
+
+ __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
+ __ delayed()->restore();
+ }
+ break;
+
+ case g1_post_barrier_slow_id:
+ {
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ if (bs->kind() != BarrierSet::G1SATBCTLogging) {
+ __ save_frame(0);
+ __ set((int)id, O1);
+ __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0);
+ __ should_not_reach_here();
+ break;
+ }
+
+ __ set_info("g1_post_barrier_slow_id", dont_gc_arguments);
+
+ Register addr = G4;
+ Register cardtable = G5;
+ Register tmp = G1_scratch;
+ Register tmp2 = G3_scratch;
+ jbyte* byte_map_base = ((CardTableModRefBS*)bs)->byte_map_base;
+
+ Label not_already_dirty, restart, refill;
+
+#ifdef _LP64
+ __ srlx(addr, CardTableModRefBS::card_shift, addr);
+#else
+ __ srl(addr, CardTableModRefBS::card_shift, addr);
+#endif
+
+ Address rs(cardtable, (address)byte_map_base);
+ __ load_address(rs); // cardtable := <card table base>
+ __ ldub(addr, cardtable, tmp); // tmp := [addr + cardtable]
+
+ __ br_on_reg_cond(Assembler::rc_nz, /*annul*/false, Assembler::pt,
+ tmp, not_already_dirty);
+ // Get cardtable + tmp into a reg by itself -- useful in the take-the-branch
+ // case, harmless if not.
+ __ delayed()->add(addr, cardtable, tmp2);
+
+ // We didn't take the branch, so we're already dirty: return.
+ // Use return-from-leaf
+ __ retl();
+ __ delayed()->nop();
+
+ // Not dirty.
+ __ bind(not_already_dirty);
+ // First, dirty it.
+ __ stb(G0, tmp2, 0); // [cardPtr] := 0 (i.e., dirty).
+
+ Register tmp3 = cardtable;
+ Register tmp4 = tmp;
+
+ // these registers are now dead
+ addr = cardtable = tmp = noreg;
+
+ int dirty_card_q_index_byte_offset =
+ in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_index());
+ int dirty_card_q_buf_byte_offset =
+ in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_buf());
+ __ bind(restart);
+ __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, tmp3);
+
+ __ br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pn,
+ tmp3, refill);
+ // If the branch is taken, no harm in executing this in the delay slot.
+ __ delayed()->ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, tmp4);
+ __ sub(tmp3, oopSize, tmp3);
+
+ __ st_ptr(tmp2, tmp4, tmp3); // [_buf + index] := <address_of_card>
+ // Use return-from-leaf
+ __ retl();
+ __ delayed()->st_ptr(tmp3, G2_thread, dirty_card_q_index_byte_offset);
+
+ __ bind(refill);
+ __ save_frame(0);
+
+ __ mov(tmp2, L0);
+ __ mov(tmp3, L1);
+ __ mov(tmp4, L2);
+
+ __ call_VM_leaf(L7_thread_cache,
+ CAST_FROM_FN_PTR(address,
+ DirtyCardQueueSet::handle_zero_index_for_thread),
+ G2_thread);
+
+ __ mov(L0, tmp2);
+ __ mov(L1, tmp3);
+ __ mov(L2, tmp4);
+
+ __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
+ __ delayed()->restore();
+ }
+ break;
+#endif // !SERIALGC
+
default:
{ __ set_info("unimplemented entry", dont_gc_arguments);
__ save_frame(0);
--- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1110,30 +1110,31 @@ class StubGenerator: public StubCodeGene
// The input registers are overwritten.
//
void gen_write_ref_array_pre_barrier(Register addr, Register count) {
-#if 0 // G1 only
BarrierSet* bs = Universe::heap()->barrier_set();
if (bs->has_write_ref_pre_barrier()) {
assert(bs->has_write_ref_array_pre_opt(),
"Else unsupported barrier set.");
- assert(addr->is_global() && count->is_global(),
- "If not, then we have to fix this code to handle more "
- "general cases.");
- // Get some new fresh output registers.
__ save_frame(0);
// Save the necessary global regs... will be used after.
- __ mov(addr, L0);
- __ mov(count, L1);
-
- __ mov(addr, O0);
+ if (addr->is_global()) {
+ __ mov(addr, L0);
+ }
+ if (count->is_global()) {
+ __ mov(count, L1);
+ }
+ __ mov(addr->after_save(), O0);
// Get the count into O1
__ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre));
- __ delayed()->mov(count, O1);
- __ mov(L0, addr);
- __ mov(L1, count);
+ __ delayed()->mov(count->after_save(), O1);
+ if (addr->is_global()) {
+ __ mov(L0, addr);
+ }
+ if (count->is_global()) {
+ __ mov(L1, count);
+ }
__ restore();
}
-#endif // 0
}
//
// Generate post-write barrier for array.
@@ -1150,22 +1151,17 @@ class StubGenerator: public StubCodeGene
BarrierSet* bs = Universe::heap()->barrier_set();
switch (bs->kind()) {
-#if 0 // G1 - only
case BarrierSet::G1SATBCT:
case BarrierSet::G1SATBCTLogging:
{
- assert(addr->is_global() && count->is_global(),
- "If not, then we have to fix this code to handle more "
- "general cases.");
// Get some new fresh output registers.
__ save_frame(0);
- __ mov(addr, O0);
+ __ mov(addr->after_save(), O0);
__ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
- __ delayed()->mov(count, O1);
+ __ delayed()->mov(count->after_save(), O1);
__ restore();
}
break;
-#endif // 0 G1 - only
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
{
@@ -2412,8 +2408,7 @@ class StubGenerator: public StubCodeGene
StubCodeMark mark(this, "StubRoutines", name);
address start = __ pc();
- gen_write_ref_array_pre_barrier(G1, G5);
-
+ gen_write_ref_array_pre_barrier(O1, O2);
#ifdef ASSERT
// We sometimes save a frame (see partial_subtype_check below).
--- a/src/cpu/sparc/vm/templateTable_sparc.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/sparc/vm/templateTable_sparc.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -28,6 +28,79 @@
#ifndef CC_INTERP
#define __ _masm->
+// Misc helpers
+
+// Do an oop store like *(base + index + offset) = val
+// index can be noreg,
+static void do_oop_store(InterpreterMacroAssembler* _masm,
+ Register base,
+ Register index,
+ int offset,
+ Register val,
+ Register tmp,
+ BarrierSet::Name barrier,
+ bool precise) {
+ assert(tmp != val && tmp != base && tmp != index, "register collision");
+ assert(index == noreg || offset == 0, "only one offset");
+ switch (barrier) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ {
+ __ g1_write_barrier_pre( base, index, offset, tmp, /*preserve_o_regs*/true);
+ if (index == noreg ) {
+ assert(Assembler::is_simm13(offset), "fix this code");
+ __ store_heap_oop(val, base, offset);
+ } else {
+ __ store_heap_oop(val, base, index);
+ }
+
+ // No need for post barrier if storing NULL
+ if (val != G0) {
+ if (precise) {
+ if (index == noreg) {
+ __ add(base, offset, base);
+ } else {
+ __ add(base, index, base);
+ }
+ }
+ __ g1_write_barrier_post(base, val, tmp);
+ }
+ }
+ break;
+#endif // SERIALGC
+ case BarrierSet::CardTableModRef:
+ case BarrierSet::CardTableExtension:
+ {
+ if (index == noreg ) {
+ assert(Assembler::is_simm13(offset), "fix this code");
+ __ store_heap_oop(val, base, offset);
+ } else {
+ __ store_heap_oop(val, base, index);
+ }
+ // No need for post barrier if storing NULL
+ if (val != G0) {
+ if (precise) {
+ if (index == noreg) {
+ __ add(base, offset, base);
+ } else {
+ __ add(base, index, base);
+ }
+ }
+ __ card_write_barrier_post(base, val, tmp);
+ }
+ }
+ break;
+ case BarrierSet::ModRef:
+ case BarrierSet::Other:
+ ShouldNotReachHere();
+ break;
+ default :
+ ShouldNotReachHere();
+
+ }
+}
+
//----------------------------------------------------------------------------------------------------
// Platform-dependent initialization
@@ -758,6 +831,8 @@ void TemplateTable::aastore() {
// O4: array element klass
// O5: value klass
+ // Address element(O1, 0, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
+
// Generate a fast subtype check. Branch to store_ok if no
// failure. Throw if failure.
__ gen_subtype_check( O5, O4, G3_scratch, G4_scratch, G1_scratch, store_ok );
@@ -767,18 +842,14 @@ void TemplateTable::aastore() {
// Store is OK.
__ bind(store_ok);
- __ store_heap_oop(Otos_i, O1, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
- // Quote from rememberedSet.hpp: For objArrays, the precise card
- // corresponding to the pointer store is dirtied so we don't need to
- // scavenge the entire array.
- Address element(O1, 0, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
- __ add(element, O1); // address the element precisely
- __ store_check(G3_scratch, O1);
+ do_oop_store(_masm, O1, noreg, arrayOopDesc::base_offset_in_bytes(T_OBJECT), Otos_i, G3_scratch, _bs->kind(), true);
+
__ ba(false,done);
__ delayed()->inc(Lesp, 3* Interpreter::stackElementSize()); // adj sp (pops array, index and value)
__ bind(is_null);
- __ store_heap_oop(Otos_i, element);
+ do_oop_store(_masm, O1, noreg, arrayOopDesc::base_offset_in_bytes(T_OBJECT), G0, G4_scratch, _bs->kind(), true);
+
__ profile_null_seen(G3_scratch);
__ inc(Lesp, 3* Interpreter::stackElementSize()); // adj sp (pops array, index and value)
__ bind(done);
@@ -2449,8 +2520,9 @@ void TemplateTable::putfield_or_static(i
// atos
__ pop_ptr();
__ verify_oop(Otos_i);
- __ store_heap_oop(Otos_i, Rclass, Roffset);
- __ store_check(G1_scratch, Rclass, Roffset);
+
+ do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
+
__ ba(false, checkVolatile);
__ delayed()->tst(Lscratch);
@@ -2491,8 +2563,9 @@ void TemplateTable::putfield_or_static(i
__ pop_ptr();
pop_and_check_object(Rclass);
__ verify_oop(Otos_i);
- __ store_heap_oop(Otos_i, Rclass, Roffset);
- __ store_check(G1_scratch, Rclass, Roffset);
+
+ do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
+
patch_bytecode(Bytecodes::_fast_aputfield, G3_scratch, G4_scratch);
__ ba(false, checkVolatile);
__ delayed()->tst(Lscratch);
@@ -2646,8 +2719,7 @@ void TemplateTable::fast_storefield(TosS
__ stf(FloatRegisterImpl::D, Ftos_d, Rclass, Roffset);
break;
case Bytecodes::_fast_aputfield:
- __ store_heap_oop(Otos_i, Rclass, Roffset);
- __ store_check(G1_scratch, Rclass, Roffset);
+ do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
break;
default:
ShouldNotReachHere();
--- a/src/cpu/x86/vm/assembler_x86.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/assembler_x86.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -5935,26 +5935,30 @@ void MacroAssembler::eden_allocate(Regis
Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, var_size_in_bytes, t1);
- Register end = t1;
- Label retry;
- bind(retry);
- ExternalAddress heap_top((address) Universe::heap()->top_addr());
- movptr(obj, heap_top);
- if (var_size_in_bytes == noreg) {
- lea(end, Address(obj, con_size_in_bytes));
- } else {
- lea(end, Address(obj, var_size_in_bytes, Address::times_1));
- }
- // if end < obj then we wrapped around => object too long => slow case
- cmpptr(end, obj);
- jcc(Assembler::below, slow_case);
- cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
- jcc(Assembler::above, slow_case);
- // Compare obj with the top addr, and if still equal, store the new top addr in
- // end at the address of the top addr pointer. Sets ZF if was equal, and clears
- // it otherwise. Use lock prefix for atomicity on MPs.
- locked_cmpxchgptr(end, heap_top);
- jcc(Assembler::notEqual, retry);
+ if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
+ jmp(slow_case);
+ } else {
+ Register end = t1;
+ Label retry;
+ bind(retry);
+ ExternalAddress heap_top((address) Universe::heap()->top_addr());
+ movptr(obj, heap_top);
+ if (var_size_in_bytes == noreg) {
+ lea(end, Address(obj, con_size_in_bytes));
+ } else {
+ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
+ }
+ // if end < obj then we wrapped around => object too long => slow case
+ cmpptr(end, obj);
+ jcc(Assembler::below, slow_case);
+ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
+ jcc(Assembler::above, slow_case);
+ // Compare obj with the top addr, and if still equal, store the new top addr in
+ // end at the address of the top addr pointer. Sets ZF if was equal, and clears
+ // it otherwise. Use lock prefix for atomicity on MPs.
+ locked_cmpxchgptr(end, heap_top);
+ jcc(Assembler::notEqual, retry);
+ }
}
void MacroAssembler::enter() {
@@ -6490,6 +6494,179 @@ void MacroAssembler::sign_extend_short(R
sarl(reg, 16);
}
}
+
+//////////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+void MacroAssembler::g1_write_barrier_pre(Register obj,
+#ifndef _LP64
+ Register thread,
+#endif
+ Register tmp,
+ Register tmp2,
+ bool tosca_live) {
+ LP64_ONLY(Register thread = r15_thread;)
+ Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()));
+
+ Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_index()));
+ Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_buf()));
+
+
+ Label done;
+ Label runtime;
+
+ // if (!marking_in_progress) goto done;
+ if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
+ cmpl(in_progress, 0);
+ } else {
+ assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");
+ cmpb(in_progress, 0);
+ }
+ jcc(Assembler::equal, done);
+
+ // if (x.f == NULL) goto done;
+ cmpptr(Address(obj, 0), NULL_WORD);
+ jcc(Assembler::equal, done);
+
+ // Can we store original value in the thread's buffer?
+
+ LP64_ONLY(movslq(tmp, index);)
+ movptr(tmp2, Address(obj, 0));
+#ifdef _LP64
+ cmpq(tmp, 0);
+#else
+ cmpl(index, 0);
+#endif
+ jcc(Assembler::equal, runtime);
+#ifdef _LP64
+ subq(tmp, wordSize);
+ movl(index, tmp);
+ addq(tmp, buffer);
+#else
+ subl(index, wordSize);
+ movl(tmp, buffer);
+ addl(tmp, index);
+#endif
+ movptr(Address(tmp, 0), tmp2);
+ jmp(done);
+ bind(runtime);
+ // save the live input values
+ if(tosca_live) push(rax);
+ push(obj);
+#ifdef _LP64
+ movq(c_rarg0, Address(obj, 0));
+ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, r15_thread);
+#else
+ push(thread);
+ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, thread);
+ pop(thread);
+#endif
+ pop(obj);
+ if(tosca_live) pop(rax);
+ bind(done);
+
+}
+
+void MacroAssembler::g1_write_barrier_post(Register store_addr,
+ Register new_val,
+#ifndef _LP64
+ Register thread,
+#endif
+ Register tmp,
+ Register tmp2) {
+
+ LP64_ONLY(Register thread = r15_thread;)
+ Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_index()));
+ Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_buf()));
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ CardTableModRefBS* ct = (CardTableModRefBS*)bs;
+ Label done;
+ Label runtime;
+
+ // Does store cross heap regions?
+
+ movptr(tmp, store_addr);
+ xorptr(tmp, new_val);
+ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
+ jcc(Assembler::equal, done);
+
+ // crosses regions, storing NULL?
+
+ cmpptr(new_val, (int32_t) NULL_WORD);
+ jcc(Assembler::equal, done);
+
+ // storing region crossing non-NULL, is card already dirty?
+
+ ExternalAddress cardtable((address) ct->byte_map_base);
+ assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
+#ifdef _LP64
+ const Register card_addr = tmp;
+
+ movq(card_addr, store_addr);
+ shrq(card_addr, CardTableModRefBS::card_shift);
+
+ lea(tmp2, cardtable);
+
+ // get the address of the card
+ addq(card_addr, tmp2);
+#else
+ const Register card_index = tmp;
+
+ movl(card_index, store_addr);
+ shrl(card_index, CardTableModRefBS::card_shift);
+
+ Address index(noreg, card_index, Address::times_1);
+ const Register card_addr = tmp;
+ lea(card_addr, as_Address(ArrayAddress(cardtable, index)));
+#endif
+ cmpb(Address(card_addr, 0), 0);
+ jcc(Assembler::equal, done);
+
+ // storing a region crossing, non-NULL oop, card is clean.
+ // dirty card and log.
+
+ movb(Address(card_addr, 0), 0);
+
+ cmpl(queue_index, 0);
+ jcc(Assembler::equal, runtime);
+ subl(queue_index, wordSize);
+ movptr(tmp2, buffer);
+#ifdef _LP64
+ movslq(rscratch1, queue_index);
+ addq(tmp2, rscratch1);
+ movq(Address(tmp2, 0), card_addr);
+#else
+ addl(tmp2, queue_index);
+ movl(Address(tmp2, 0), card_index);
+#endif
+ jmp(done);
+
+ bind(runtime);
+ // save the live input values
+ push(store_addr);
+ push(new_val);
+#ifdef _LP64
+ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
+#else
+ push(thread);
+ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
+ pop(thread);
+#endif
+ pop(new_val);
+ pop(store_addr);
+
+ bind(done);
+
+}
+
+#endif // SERIALGC
+//////////////////////////////////////////////////////////////////////////////////
+
void MacroAssembler::store_check(Register obj) {
// Does a store check for the oop in register obj. The content of
--- a/src/cpu/x86/vm/assembler_x86.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/assembler_x86.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -227,9 +227,11 @@ class Address VALUE_OBJ_CLASS_SPEC {
#endif // ASSERT
// accessors
- bool uses(Register reg) const {
- return _base == reg || _index == reg;
- }
+ bool uses(Register reg) const { return _base == reg || _index == reg; }
+ Register base() const { return _base; }
+ Register index() const { return _index; }
+ ScaleFactor scale() const { return _scale; }
+ int disp() const { return _disp; }
// Convert the raw encoding form into the form expected by the constructor for
// Address. An index of 4 (rsp) corresponds to having no index, so convert
@@ -1310,7 +1312,8 @@ private:
// on arguments should also go in here.
class MacroAssembler: public Assembler {
- friend class LIR_Assembler;
+ friend class LIR_Assembler;
+ friend class Runtime1; // as_Address()
protected:
Address as_Address(AddressLiteral adr);
@@ -1452,6 +1455,7 @@ class MacroAssembler: public Assembler {
// Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
// The pointer will be loaded into the thread register.
void get_thread(Register thread);
+
// Support for VM calls
//
@@ -1527,6 +1531,22 @@ class MacroAssembler: public Assembler {
void store_check(Register obj); // store check for obj - register is destroyed afterwards
void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed)
+ void g1_write_barrier_pre(Register obj,
+#ifndef _LP64
+ Register thread,
+#endif
+ Register tmp,
+ Register tmp2,
+ bool tosca_live);
+ void g1_write_barrier_post(Register store_addr,
+ Register new_val,
+#ifndef _LP64
+ Register thread,
+#endif
+ Register tmp,
+ Register tmp2);
+
+
// split store_check(Register obj) to enhance instruction interleaving
void store_check_part_1(Register obj);
void store_check_part_2(Register obj);
--- a/src/cpu/x86/vm/c1_CodeStubs_x86.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/c1_CodeStubs_x86.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -456,5 +456,50 @@ void ArrayCopyStub::emit_code(LIR_Assemb
__ jmp(_continuation);
}
+/////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+void G1PreBarrierStub::emit_code(LIR_Assembler* ce) {
+
+ // At this point we know that marking is in progress
+
+ __ bind(_entry);
+ assert(pre_val()->is_register(), "Precondition.");
+
+ Register pre_val_reg = pre_val()->as_register();
+
+ ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false);
+
+ __ cmpptr(pre_val_reg, (int32_t) NULL_WORD);
+ __ jcc(Assembler::equal, _continuation);
+ ce->store_parameter(pre_val()->as_register(), 0);
+ __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_pre_barrier_slow_id)));
+ __ jmp(_continuation);
+
+}
+
+jbyte* G1PostBarrierStub::_byte_map_base = NULL;
+
+jbyte* G1PostBarrierStub::byte_map_base_slow() {
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ assert(bs->is_a(BarrierSet::G1SATBCTLogging),
+ "Must be if we're using this.");
+ return ((G1SATBCardTableModRefBS*)bs)->byte_map_base;
+}
+
+void G1PostBarrierStub::emit_code(LIR_Assembler* ce) {
+ __ bind(_entry);
+ assert(addr()->is_register(), "Precondition.");
+ assert(new_val()->is_register(), "Precondition.");
+ Register new_val_reg = new_val()->as_register();
+ __ cmpptr(new_val_reg, (int32_t) NULL_WORD);
+ __ jcc(Assembler::equal, _continuation);
+ ce->store_parameter(addr()->as_register(), 0);
+ __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id)));
+ __ jmp(_continuation);
+}
+
+#endif // SERIALGC
+/////////////////////////////////////////////////////////////////////////////
#undef __
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -302,6 +302,8 @@ void LIRGenerator::do_StoreIndexed(Store
}
if (obj_store) {
+ // Needs GC write barriers.
+ pre_barrier(LIR_OprFact::address(array_addr), false, NULL);
__ move(value.result(), array_addr, null_check_info);
// Seems to be a precise
post_barrier(LIR_OprFact::address(array_addr), value.result());
@@ -756,7 +758,10 @@ void LIRGenerator::do_CompareAndSwap(Int
__ move(obj.result(), addr);
__ add(addr, offset.result(), addr);
-
+ if (type == objectType) { // Write-barrier needed for Object fields.
+ // Do the pre-write barrier, if any.
+ pre_barrier(addr, false, NULL);
+ }
LIR_Opr ill = LIR_OprFact::illegalOpr; // for convenience
if (type == objectType)
@@ -1286,6 +1291,8 @@ void LIRGenerator::put_Object_unsafe(LIR
LIR_Address* addr = new LIR_Address(src, offset, type);
bool is_obj = (type == T_ARRAY || type == T_OBJECT);
if (is_obj) {
+ // Do the pre-write barrier, if any.
+ pre_barrier(LIR_OprFact::address(addr), false, NULL);
__ move(data, addr);
assert(src->is_register(), "must be register");
// Seems to be a precise address
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1583,6 +1583,166 @@ OopMapSet* Runtime1::generate_code_for(S
}
break;
+#ifndef SERIALGC
+ case g1_pre_barrier_slow_id:
+ {
+ StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
+ // arg0 : previous value of memory
+
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ if (bs->kind() != BarrierSet::G1SATBCTLogging) {
+ __ movptr(rax, (int)id);
+ __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
+ __ should_not_reach_here();
+ break;
+ }
+
+ __ push(rax);
+ __ push(rdx);
+
+ const Register pre_val = rax;
+ const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+ const Register tmp = rdx;
+
+ NOT_LP64(__ get_thread(thread);)
+
+ Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()));
+
+ Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_index()));
+ Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_buf()));
+
+
+ Label done;
+ Label runtime;
+
+ // Can we store original value in the thread's buffer?
+
+ LP64_ONLY(__ movslq(tmp, queue_index);)
+#ifdef _LP64
+ __ cmpq(tmp, 0);
+#else
+ __ cmpl(queue_index, 0);
+#endif
+ __ jcc(Assembler::equal, runtime);
+#ifdef _LP64
+ __ subq(tmp, wordSize);
+ __ movl(queue_index, tmp);
+ __ addq(tmp, buffer);
+#else
+ __ subl(queue_index, wordSize);
+ __ movl(tmp, buffer);
+ __ addl(tmp, queue_index);
+#endif
+
+ // prev_val (rax)
+ f.load_argument(0, pre_val);
+ __ movptr(Address(tmp, 0), pre_val);
+ __ jmp(done);
+
+ __ bind(runtime);
+ // load the pre-value
+ __ push(rcx);
+ f.load_argument(0, rcx);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), rcx, thread);
+ __ pop(rcx);
+
+ __ bind(done);
+ __ pop(rdx);
+ __ pop(rax);
+ }
+ break;
+
+ case g1_post_barrier_slow_id:
+ {
+ StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
+
+
+ // arg0: store_address
+ Address store_addr(rbp, 2*BytesPerWord);
+
+ BarrierSet* bs = Universe::heap()->barrier_set();
+ CardTableModRefBS* ct = (CardTableModRefBS*)bs;
+ Label done;
+ Label runtime;
+
+ // At this point we know new_value is non-NULL and the new_value crosses regsion.
+ // Must check to see if card is already dirty
+
+ const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+
+ Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_index()));
+ Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
+ PtrQueue::byte_offset_of_buf()));
+
+ __ push(rax);
+ __ push(rdx);
+
+ NOT_LP64(__ get_thread(thread);)
+ ExternalAddress cardtable((address)ct->byte_map_base);
+ assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
+
+ const Register card_addr = rdx;
+#ifdef _LP64
+ const Register tmp = rscratch1;
+ f.load_argument(0, card_addr);
+ __ shrq(card_addr, CardTableModRefBS::card_shift);
+ __ lea(tmp, cardtable);
+ // get the address of the card
+ __ addq(card_addr, tmp);
+#else
+ const Register card_index = rdx;
+ f.load_argument(0, card_index);
+ __ shrl(card_index, CardTableModRefBS::card_shift);
+
+ Address index(noreg, card_index, Address::times_1);
+ __ leal(card_addr, __ as_Address(ArrayAddress(cardtable, index)));
+#endif
+
+ __ cmpb(Address(card_addr, 0), 0);
+ __ jcc(Assembler::equal, done);
+
+ // storing region crossing non-NULL, card is clean.
+ // dirty card and log.
+
+ __ movb(Address(card_addr, 0), 0);
+
+ __ cmpl(queue_index, 0);
+ __ jcc(Assembler::equal, runtime);
+ __ subl(queue_index, wordSize);
+
+ const Register buffer_addr = rbx;
+ __ push(rbx);
+
+ __ movptr(buffer_addr, buffer);
+
+#ifdef _LP64
+ __ movslq(rscratch1, queue_index);
+ __ addptr(buffer_addr, rscratch1);
+#else
+ __ addptr(buffer_addr, queue_index);
+#endif
+ __ movptr(Address(buffer_addr, 0), card_addr);
+
+ __ pop(rbx);
+ __ jmp(done);
+
+ __ bind(runtime);
+ NOT_LP64(__ push(rcx);)
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
+ NOT_LP64(__ pop(rcx);)
+
+ __ bind(done);
+ __ pop(rdx);
+ __ pop(rax);
+
+ }
+ break;
+#endif // !SERIALGC
+
default:
{ StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
__ movptr(rax, (int)id);
--- a/src/cpu/x86/vm/interp_masm_x86_64.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/interp_masm_x86_64.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -44,8 +44,13 @@ void InterpreterMacroAssembler::call_VM_
// Note: No need to save/restore bcp & locals (r13 & r14) pointer
// since these are callee saved registers and no blocking/
// GC can happen in leaf calls.
+ // Further Note: DO NOT save/restore bcp/locals. If a caller has
+ // already saved them so that it can use esi/edi as temporaries
+ // then a save/restore here will DESTROY the copy the caller
+ // saved! There used to be a save_bcp() that only happened in
+ // the ASSERT path (no restore_bcp). Which caused bizarre failures
+ // when jvm built with ASSERTs.
#ifdef ASSERT
- save_bcp();
{
Label L;
cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
@@ -58,24 +63,9 @@ void InterpreterMacroAssembler::call_VM_
// super call
MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
// interpreter specific
-#ifdef ASSERT
- {
- Label L;
- cmpptr(r13, Address(rbp, frame::interpreter_frame_bcx_offset * wordSize));
- jcc(Assembler::equal, L);
- stop("InterpreterMacroAssembler::call_VM_leaf_base:"
- " r13 not callee saved?");
- bind(L);
- }
- {
- Label L;
- cmpptr(r14, Address(rbp, frame::interpreter_frame_locals_offset * wordSize));
- jcc(Assembler::equal, L);
- stop("InterpreterMacroAssembler::call_VM_leaf_base:"
- " r14 not callee saved?");
- bind(L);
- }
-#endif
+ // Used to ASSERT that r13/r14 were equal to frame's bcp/locals
+ // but since they may not have been saved (and we don't want to
+ // save thme here (see note above) the assert is invalid.
}
void InterpreterMacroAssembler::call_VM_base(Register oop_result,
--- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -712,7 +712,6 @@ class StubGenerator: public StubCodeGene
// end - element count
void gen_write_ref_array_pre_barrier(Register start, Register count) {
assert_different_registers(start, count);
-#if 0 // G1 only
BarrierSet* bs = Universe::heap()->barrier_set();
switch (bs->kind()) {
case BarrierSet::G1SATBCT:
@@ -721,8 +720,8 @@ class StubGenerator: public StubCodeGene
__ pusha(); // push registers
__ push(count);
__ push(start);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre));
- __ addl(esp, wordSize * 2);
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)));
+ __ addptr(rsp, 2*wordSize);
__ popa();
}
break;
@@ -734,7 +733,6 @@ class StubGenerator: public StubCodeGene
ShouldNotReachHere();
}
-#endif // 0 - G1 only
}
@@ -750,20 +748,18 @@ class StubGenerator: public StubCodeGene
BarrierSet* bs = Universe::heap()->barrier_set();
assert_different_registers(start, count);
switch (bs->kind()) {
-#if 0 // G1 only
case BarrierSet::G1SATBCT:
case BarrierSet::G1SATBCTLogging:
{
__ pusha(); // push registers
__ push(count);
__ push(start);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
- __ addl(esp, wordSize * 2);
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
+ __ addptr(rsp, 2*wordSize);
__ popa();
}
break;
-#endif // 0 G1 only
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
@@ -1378,9 +1374,9 @@ class StubGenerator: public StubCodeGene
Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
// Copy from low to high addresses, indexed from the end of each array.
+ gen_write_ref_array_pre_barrier(to, count);
__ lea(end_from, end_from_addr);
__ lea(end_to, end_to_addr);
- gen_write_ref_array_pre_barrier(to, count);
assert(length == count, ""); // else fix next line:
__ negptr(count); // negate and test the length
__ jccb(Assembler::notZero, L_load_element);
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1153,18 +1153,26 @@ class StubGenerator: public StubCodeGene
// Destroy no registers!
//
void gen_write_ref_array_pre_barrier(Register addr, Register count) {
-#if 0 // G1 - only
- assert_different_registers(addr, c_rarg1);
- assert_different_registers(count, c_rarg0);
BarrierSet* bs = Universe::heap()->barrier_set();
switch (bs->kind()) {
case BarrierSet::G1SATBCT:
case BarrierSet::G1SATBCTLogging:
{
__ pusha(); // push registers
- __ movptr(c_rarg0, addr);
- __ movptr(c_rarg1, count);
- __ call(RuntimeAddress(BarrierSet::static_write_ref_array_pre));
+ if (count == c_rarg0) {
+ if (addr == c_rarg1) {
+ // exactly backwards!!
+ __ xchgptr(c_rarg1, c_rarg0);
+ } else {
+ __ movptr(c_rarg1, count);
+ __ movptr(c_rarg0, addr);
+ }
+
+ } else {
+ __ movptr(c_rarg0, addr);
+ __ movptr(c_rarg1, count);
+ }
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)));
__ popa();
}
break;
@@ -1172,11 +1180,10 @@ class StubGenerator: public StubCodeGene
case BarrierSet::CardTableExtension:
case BarrierSet::ModRef:
break;
- default :
+ default:
ShouldNotReachHere();
}
-#endif // 0 G1 - only
}
//
@@ -1193,7 +1200,6 @@ class StubGenerator: public StubCodeGene
assert_different_registers(start, end, scratch);
BarrierSet* bs = Universe::heap()->barrier_set();
switch (bs->kind()) {
-#if 0 // G1 - only
case BarrierSet::G1SATBCT:
case BarrierSet::G1SATBCTLogging:
@@ -1206,11 +1212,10 @@ class StubGenerator: public StubCodeGene
__ shrptr(scratch, LogBytesPerWord);
__ mov(c_rarg0, start);
__ mov(c_rarg1, scratch);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
__ popa();
}
break;
-#endif // 0 G1 - only
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
{
@@ -1239,8 +1244,12 @@ class StubGenerator: public StubCodeGene
__ decrement(count);
__ jcc(Assembler::greaterEqual, L_loop);
}
- }
- }
+ break;
+ default:
+ ShouldNotReachHere();
+
+ }
+ }
// Copy big chunks forward
//
@@ -2282,7 +2291,7 @@ class StubGenerator: public StubCodeGene
// and report their number to the caller.
assert_different_registers(rax, r14_length, count, to, end_to, rcx);
__ lea(end_to, to_element_addr);
- gen_write_ref_array_post_barrier(to, end_to, rcx);
+ gen_write_ref_array_post_barrier(to, end_to, rscratch1);
__ movptr(rax, r14_length); // original oops
__ addptr(rax, count); // K = (original - remaining) oops
__ notptr(rax); // report (-1^K) to caller
@@ -2291,7 +2300,7 @@ class StubGenerator: public StubCodeGene
// Come here on success only.
__ BIND(L_do_card_marks);
__ addptr(end_to, -wordSize); // make an inclusive end pointer
- gen_write_ref_array_post_barrier(to, end_to, rcx);
+ gen_write_ref_array_post_barrier(to, end_to, rscratch1);
__ xorptr(rax, rax); // return 0 on success
// Common exit point (success or failure).
--- a/src/cpu/x86/vm/templateTable_x86_32.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/templateTable_x86_32.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -106,6 +106,78 @@ static Assembler::Condition j_not(Templa
//----------------------------------------------------------------------------------------------------
// Miscelaneous helper routines
+
+// Store an oop (or NULL) at the address described by obj.
+// If val == noreg this means store a NULL
+
+static void do_oop_store(InterpreterMacroAssembler* _masm,
+ Address obj,
+ Register val,
+ BarrierSet::Name barrier,
+ bool precise) {
+ assert(val == noreg || val == rax, "parameter is just for looks");
+ switch (barrier) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ {
+ // flatten object address if needed
+ // We do it regardless of precise because we need the registers
+ if (obj.index() == noreg && obj.disp() == 0) {
+ if (obj.base() != rdx) {
+ __ movl(rdx, obj.base());
+ }
+ } else {
+ __ leal(rdx, obj);
+ }
+ __ get_thread(rcx);
+ __ save_bcp();
+ __ g1_write_barrier_pre(rdx, rcx, rsi, rbx, val != noreg);
+
+ // Do the actual store
+ // noreg means NULL
+ if (val == noreg) {
+ __ movl(Address(rdx, 0), NULL_WORD);
+ // No post barrier for NULL
+ } else {
+ __ movl(Address(rdx, 0), val);
+ __ g1_write_barrier_post(rdx, rax, rcx, rbx, rsi);
+ }
+ __ restore_bcp();
+
+ }
+ break;
+#endif // SERIALGC
+ case BarrierSet::CardTableModRef:
+ case BarrierSet::CardTableExtension:
+ {
+ if (val == noreg) {
+ __ movl(obj, NULL_WORD);
+ } else {
+ __ movl(obj, val);
+ // flatten object address if needed
+ if (!precise || (obj.index() == noreg && obj.disp() == 0)) {
+ __ store_check(obj.base());
+ } else {
+ __ leal(rdx, obj);
+ __ store_check(rdx);
+ }
+ }
+ }
+ break;
+ case BarrierSet::ModRef:
+ case BarrierSet::Other:
+ if (val == noreg) {
+ __ movl(obj, NULL_WORD);
+ } else {
+ __ movl(obj, val);
+ }
+ break;
+ default :
+ ShouldNotReachHere();
+
+ }
+}
Address TemplateTable::at_bcp(int offset) {
assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
@@ -876,6 +948,8 @@ void TemplateTable::aastore() {
__ movptr(rax, at_tos()); // Value
__ movl(rcx, at_tos_p1()); // Index
__ movptr(rdx, at_tos_p2()); // Array
+
+ Address element_address(rdx, rcx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
index_check_without_pop(rdx, rcx); // kills rbx,
// do array store check - check for NULL value first
__ testptr(rax, rax);
@@ -887,7 +961,7 @@ void TemplateTable::aastore() {
__ movptr(rax, Address(rdx, oopDesc::klass_offset_in_bytes()));
__ movptr(rax, Address(rax, sizeof(oopDesc) + objArrayKlass::element_klass_offset_in_bytes()));
// Compress array+index*wordSize+12 into a single register. Frees ECX.
- __ lea(rdx, Address(rdx, rcx, Address::times_ptr, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
+ __ lea(rdx, element_address);
// Generate subtype check. Blows ECX. Resets EDI to locals.
// Superklass in EAX. Subklass in EBX.
@@ -899,15 +973,20 @@ void TemplateTable::aastore() {
// Come here on success
__ bind(ok_is_subtype);
- __ movptr(rax, at_rsp()); // Value
- __ movptr(Address(rdx, 0), rax);
- __ store_check(rdx);
- __ jmpb(done);
+
+ // Get the value to store
+ __ movptr(rax, at_rsp());
+ // and store it with appropriate barrier
+ do_oop_store(_masm, Address(rdx, 0), rax, _bs->kind(), true);
+
+ __ jmp(done);
// Have a NULL in EAX, EDX=array, ECX=index. Store NULL at ary[idx]
__ bind(is_null);
__ profile_null_seen(rbx);
- __ movptr(Address(rdx, rcx, Address::times_ptr, arrayOopDesc::base_offset_in_bytes(T_OBJECT)), rax);
+
+ // Store NULL, (noreg means NULL to do_oop_store)
+ do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
// Pop stack arguments
__ bind(done);
@@ -1515,7 +1594,7 @@ void TemplateTable::branch(bool is_jsr,
// compute return address as bci in rax,
__ lea(rax, at_bcp((is_wide ? 5 : 3) - in_bytes(constMethodOopDesc::codes_offset())));
__ subptr(rax, Address(rcx, methodOopDesc::const_offset()));
- // Adjust the bcp in ESI by the displacement in EDX
+ // Adjust the bcp in RSI by the displacement in EDX
__ addptr(rsi, rdx);
// Push return address
__ push_i(rax);
@@ -1526,7 +1605,7 @@ void TemplateTable::branch(bool is_jsr,
// Normal (non-jsr) branch handling
- // Adjust the bcp in ESI by the displacement in EDX
+ // Adjust the bcp in RSI by the displacement in EDX
__ addptr(rsi, rdx);
assert(UseLoopCounter || !UseOnStackReplacement, "on-stack-replacement requires loop counters");
@@ -2439,11 +2518,12 @@ void TemplateTable::putfield_or_static(i
__ pop(atos);
if (!is_static) pop_and_check_object(obj);
- __ movptr(lo, rax );
- __ store_check(obj, lo); // Need to mark card
+ do_oop_store(_masm, lo, rax, _bs->kind(), false);
+
if (!is_static) {
patch_bytecode(Bytecodes::_fast_aputfield, rcx, rbx);
}
+
__ jmp(Done);
__ bind(notObj);
@@ -2664,7 +2744,10 @@ void TemplateTable::fast_storefield(TosS
break;
case Bytecodes::_fast_fputfield: __ fstp_s(lo); break;
case Bytecodes::_fast_dputfield: __ fstp_d(lo); break;
- case Bytecodes::_fast_aputfield: __ movptr(lo, rax); __ store_check(rcx, lo); break;
+ case Bytecodes::_fast_aputfield: {
+ do_oop_store(_masm, lo, rax, _bs->kind(), false);
+ break;
+ }
default:
ShouldNotReachHere();
}
@@ -2672,7 +2755,8 @@ void TemplateTable::fast_storefield(TosS
Label done;
volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
Assembler::StoreStore));
- __ jmpb(done);
+ // Barriers are so large that short branch doesn't reach!
+ __ jmp(done);
// Same code as above, but don't need rdx to test for volatile.
__ bind(notVolatile);
@@ -2694,7 +2778,10 @@ void TemplateTable::fast_storefield(TosS
break;
case Bytecodes::_fast_fputfield: __ fstp_s(lo); break;
case Bytecodes::_fast_dputfield: __ fstp_d(lo); break;
- case Bytecodes::_fast_aputfield: __ movptr(lo, rax); __ store_check(rcx, lo); break;
+ case Bytecodes::_fast_aputfield: {
+ do_oop_store(_masm, lo, rax, _bs->kind(), false);
+ break;
+ }
default:
ShouldNotReachHere();
}
@@ -3054,8 +3141,6 @@ void TemplateTable::_new() {
Label initialize_object; // including clearing the fields
Label allocate_shared;
- ExternalAddress heap_top((address)Universe::heap()->top_addr());
-
__ get_cpool_and_tags(rcx, rax);
// get instanceKlass
__ movptr(rcx, Address(rcx, rdx, Address::times_ptr, sizeof(constantPoolOopDesc)));
@@ -3111,6 +3196,8 @@ void TemplateTable::_new() {
// rdx: instance size in bytes
if (allow_shared_alloc) {
__ bind(allocate_shared);
+
+ ExternalAddress heap_top((address)Universe::heap()->top_addr());
Label retry;
__ bind(retry);
--- a/src/cpu/x86/vm/templateTable_x86_64.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/cpu/x86/vm/templateTable_x86_64.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -115,6 +115,69 @@ static Assembler::Condition j_not(Templa
// Miscelaneous helper routines
+// Store an oop (or NULL) at the address described by obj.
+// If val == noreg this means store a NULL
+
+static void do_oop_store(InterpreterMacroAssembler* _masm,
+ Address obj,
+ Register val,
+ BarrierSet::Name barrier,
+ bool precise) {
+ assert(val == noreg || val == rax, "parameter is just for looks");
+ switch (barrier) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ {
+ // flatten object address if needed
+ if (obj.index() == noreg && obj.disp() == 0) {
+ if (obj.base() != rdx) {
+ __ movq(rdx, obj.base());
+ }
+ } else {
+ __ leaq(rdx, obj);
+ }
+ __ g1_write_barrier_pre(rdx, r8, rbx, val != noreg);
+ if (val == noreg) {
+ __ store_heap_oop(Address(rdx, 0), NULL_WORD);
+ } else {
+ __ store_heap_oop(Address(rdx, 0), val);
+ __ g1_write_barrier_post(rdx, val, r8, rbx);
+ }
+
+ }
+ break;
+#endif // SERIALGC
+ case BarrierSet::CardTableModRef:
+ case BarrierSet::CardTableExtension:
+ {
+ if (val == noreg) {
+ __ store_heap_oop(obj, NULL_WORD);
+ } else {
+ __ store_heap_oop(obj, val);
+ // flatten object address if needed
+ if (!precise || (obj.index() == noreg && obj.disp() == 0)) {
+ __ store_check(obj.base());
+ } else {
+ __ leaq(rdx, obj);
+ __ store_check(rdx);
+ }
+ }
+ }
+ break;
+ case BarrierSet::ModRef:
+ case BarrierSet::Other:
+ if (val == noreg) {
+ __ store_heap_oop(obj, NULL_WORD);
+ } else {
+ __ store_heap_oop(obj, val);
+ }
+ break;
+ default :
+ ShouldNotReachHere();
+
+ }
+}
Address TemplateTable::at_bcp(int offset) {
assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
@@ -560,8 +623,8 @@ void TemplateTable::aaload() {
// rdx: array
index_check(rdx, rax); // kills rbx
__ load_heap_oop(rax, Address(rdx, rax,
- UseCompressedOops ? Address::times_4 : Address::times_8,
- arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
+ UseCompressedOops ? Address::times_4 : Address::times_8,
+ arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
}
void TemplateTable::baload() {
@@ -866,6 +929,11 @@ void TemplateTable::aastore() {
__ movptr(rax, at_tos()); // value
__ movl(rcx, at_tos_p1()); // index
__ movptr(rdx, at_tos_p2()); // array
+
+ Address element_address(rdx, rcx,
+ UseCompressedOops? Address::times_4 : Address::times_8,
+ arrayOopDesc::base_offset_in_bytes(T_OBJECT));
+
index_check(rdx, rcx); // kills rbx
// do array store check - check for NULL value first
__ testptr(rax, rax);
@@ -879,9 +947,7 @@ void TemplateTable::aastore() {
sizeof(oopDesc) +
objArrayKlass::element_klass_offset_in_bytes()));
// Compress array + index*oopSize + 12 into a single register. Frees rcx.
- __ lea(rdx, Address(rdx, rcx,
- UseCompressedOops ? Address::times_4 : Address::times_8,
- arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
+ __ lea(rdx, element_address);
// Generate subtype check. Blows rcx, rdi
// Superklass in rax. Subklass in rbx.
@@ -893,18 +959,19 @@ void TemplateTable::aastore() {
// Come here on success
__ bind(ok_is_subtype);
- __ movptr(rax, at_tos()); // Value
- __ store_heap_oop(Address(rdx, 0), rax);
- __ store_check(rdx);
+
+ // Get the value we will store
+ __ movptr(rax, at_tos());
+ // Now store using the appropriate barrier
+ do_oop_store(_masm, Address(rdx, 0), rax, _bs->kind(), true);
__ jmp(done);
// Have a NULL in rax, rdx=array, ecx=index. Store NULL at ary[idx]
__ bind(is_null);
__ profile_null_seen(rbx);
- __ store_heap_oop(Address(rdx, rcx,
- UseCompressedOops ? Address::times_4 : Address::times_8,
- arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
- rax);
+
+ // Store a NULL
+ do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
// Pop stack arguments
__ bind(done);
@@ -2396,8 +2463,10 @@ void TemplateTable::putfield_or_static(i
// atos
__ pop(atos);
if (!is_static) pop_and_check_object(obj);
- __ store_heap_oop(field, rax);
- __ store_check(obj, field); // Need to mark card
+
+ // Store into the field
+ do_oop_store(_masm, field, rax, _bs->kind(), false);
+
if (!is_static) {
patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx);
}
@@ -2584,8 +2653,7 @@ void TemplateTable::fast_storefield(TosS
// access field
switch (bytecode()) {
case Bytecodes::_fast_aputfield:
- __ store_heap_oop(field, rax);
- __ store_check(rcx, field);
+ do_oop_store(_masm, field, rax, _bs->kind(), false);
break;
case Bytecodes::_fast_lputfield:
__ movq(field, rax);
@@ -3044,8 +3112,6 @@ void TemplateTable::_new() {
Label initialize_header;
Label initialize_object; // including clearing the fields
Label allocate_shared;
- ExternalAddress top((address)Universe::heap()->top_addr());
- ExternalAddress end((address)Universe::heap()->end_addr());
__ get_cpool_and_tags(rsi, rax);
// get instanceKlass
@@ -3105,6 +3171,9 @@ void TemplateTable::_new() {
// rdx: instance size in bytes
if (allow_shared_alloc) {
__ bind(allocate_shared);
+
+ ExternalAddress top((address)Universe::heap()->top_addr());
+ ExternalAddress end((address)Universe::heap()->end_addr());
const Register RtopAddr = rscratch1;
const Register RendAddr = rscratch2;
--- a/src/os/linux/vm/os_linux.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/os/linux/vm/os_linux.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1259,6 +1259,17 @@ jlong os::elapsed_counter() {
jlong os::elapsed_frequency() {
return (1000 * 1000);
+}
+
+// For now, we say that linux does not support vtime. I have no idea
+// whether it can actually be made to (DLD, 9/13/05).
+
+bool os::supports_vtime() { return false; }
+bool os::enable_vtime() { return false; }
+bool os::vtime_enabled() { return false; }
+double os::elapsedVTime() {
+ // better than nothing, but not much
+ return elapsedTime();
}
jlong os::javaTimeMillis() {
--- a/src/os/solaris/vm/os_solaris.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/os/solaris/vm/os_solaris.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1691,6 +1691,40 @@ bool os::getTimesSecs(double* process_re
}
}
+bool os::supports_vtime() { return true; }
+
+bool os::enable_vtime() {
+ int fd = open("/proc/self/ctl", O_WRONLY);
+ if (fd == -1)
+ return false;
+
+ long cmd[] = { PCSET, PR_MSACCT };
+ int res = write(fd, cmd, sizeof(long) * 2);
+ close(fd);
+ if (res != sizeof(long) * 2)
+ return false;
+
+ return true;
+}
+
+bool os::vtime_enabled() {
+ int fd = open("/proc/self/status", O_RDONLY);
+ if (fd == -1)
+ return false;
+
+ pstatus_t status;
+ int res = read(fd, (void*) &status, sizeof(pstatus_t));
+ close(fd);
+ if (res != sizeof(pstatus_t))
+ return false;
+
+ return status.pr_flags & PR_MSACCT;
+}
+
+double os::elapsedVTime() {
+ return (double)gethrvtime() / (double)hrtime_hz;
+}
+
// Used internally for comparisons only
// getTimeMillis guaranteed to not move backwards on Solaris
jlong getTimeMillis() {
@@ -2688,7 +2722,7 @@ size_t os::numa_get_leaf_groups(int *ids
return bottom;
}
-// Detect the topology change. Typically happens during CPU pluggin-unplugging.
+// Detect the topology change. Typically happens during CPU plugging-unplugging.
bool os::numa_topology_changed() {
int is_stale = Solaris::lgrp_cookie_stale(Solaris::lgrp_cookie());
if (is_stale != -1 && is_stale) {
--- a/src/os/windows/vm/os_windows.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/os/windows/vm/os_windows.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -735,6 +735,17 @@ FILETIME java_to_windows_time(jlong l) {
result.dwHighDateTime = high(a);
result.dwLowDateTime = low(a);
return result;
+}
+
+// For now, we say that Windows does not support vtime. I have no idea
+// whether it can actually be made to (DLD, 9/13/05).
+
+bool os::supports_vtime() { return false; }
+bool os::enable_vtime() { return false; }
+bool os::vtime_enabled() { return false; }
+double os::elapsedVTime() {
+ // better than nothing, but not much
+ return elapsedTime();
}
jlong os::javaTimeMillis() {
--- a/src/share/vm/adlc/formssel.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/adlc/formssel.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -3768,6 +3768,10 @@ int MatchRule::is_ideal_copy() const {
int MatchRule::is_ideal_copy() const {
if( _rChild ) {
const char *opType = _rChild->_opType;
+#if 1
+ if( strcmp(opType,"CastIP")==0 )
+ return 1;
+#else
if( strcmp(opType,"CastII")==0 )
return 1;
// Do not treat *CastPP this way, because it
@@ -3787,6 +3791,7 @@ int MatchRule::is_ideal_copy() const {
// return 1;
//if( strcmp(opType,"CastP2X")==0 )
// return 1;
+#endif
}
if( is_chain_rule(_AD.globalNames()) &&
_lChild && strncmp(_lChild->_opType,"stackSlot",9)==0 )
--- a/src/share/vm/c1/c1_CodeStubs.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_CodeStubs.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -482,3 +482,81 @@ class ArrayCopyStub: public CodeStub {
virtual void print_name(outputStream* out) const { out->print("ArrayCopyStub"); }
#endif // PRODUCT
};
+
+//////////////////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+// Code stubs for Garbage-First barriers.
+class G1PreBarrierStub: public CodeStub {
+ private:
+ LIR_Opr _addr;
+ LIR_Opr _pre_val;
+ LIR_PatchCode _patch_code;
+ CodeEmitInfo* _info;
+
+ public:
+ // pre_val (a temporary register) must be a register;
+ // addr (the address of the field to be read) must be a LIR_Address
+ G1PreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) :
+ _addr(addr), _pre_val(pre_val), _patch_code(patch_code), _info(info)
+ {
+ assert(_pre_val->is_register(), "should be temporary register");
+ assert(_addr->is_address(), "should be the address of the field");
+ }
+
+ LIR_Opr addr() const { return _addr; }
+ LIR_Opr pre_val() const { return _pre_val; }
+ LIR_PatchCode patch_code() const { return _patch_code; }
+ CodeEmitInfo* info() const { return _info; }
+
+ virtual void emit_code(LIR_Assembler* e);
+ virtual void visit(LIR_OpVisitState* visitor) {
+ // don't pass in the code emit info since it's processed in the fast
+ // path
+ if (_info != NULL)
+ visitor->do_slow_case(_info);
+ else
+ visitor->do_slow_case();
+ visitor->do_input(_addr);
+ visitor->do_temp(_pre_val);
+ }
+#ifndef PRODUCT
+ virtual void print_name(outputStream* out) const { out->print("G1PreBarrierStub"); }
+#endif // PRODUCT
+};
+
+class G1PostBarrierStub: public CodeStub {
+ private:
+ LIR_Opr _addr;
+ LIR_Opr _new_val;
+
+ static jbyte* _byte_map_base;
+ static jbyte* byte_map_base_slow();
+ static jbyte* byte_map_base() {
+ if (_byte_map_base == NULL) {
+ _byte_map_base = byte_map_base_slow();
+ }
+ return _byte_map_base;
+ }
+
+ public:
+ // addr (the address of the object head) and new_val must be registers.
+ G1PostBarrierStub(LIR_Opr addr, LIR_Opr new_val): _addr(addr), _new_val(new_val) { }
+
+ LIR_Opr addr() const { return _addr; }
+ LIR_Opr new_val() const { return _new_val; }
+
+ virtual void emit_code(LIR_Assembler* e);
+ virtual void visit(LIR_OpVisitState* visitor) {
+ // don't pass in the code emit info since it's processed in the fast path
+ visitor->do_slow_case();
+ visitor->do_input(_addr);
+ visitor->do_input(_new_val);
+ }
+#ifndef PRODUCT
+ virtual void print_name(outputStream* out) const { out->print("G1PostBarrierStub"); }
+#endif // PRODUCT
+};
+
+#endif // SERIALGC
+//////////////////////////////////////////////////////////////////////////////////////////
--- a/src/share/vm/c1/c1_LIRAssembler.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_LIRAssembler.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -74,6 +74,7 @@ LIR_Assembler::LIR_Assembler(Compilation
LIR_Assembler::LIR_Assembler(Compilation* c):
_compilation(c)
, _masm(c->masm())
+ , _bs(Universe::heap()->barrier_set())
, _frame_map(c->frame_map())
, _current_block(NULL)
, _pending_non_safepoint(NULL)
--- a/src/share/vm/c1/c1_LIRAssembler.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_LIRAssembler.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -24,11 +24,13 @@
class Compilation;
class ScopeValue;
+class BarrierSet;
class LIR_Assembler: public CompilationResourceObj {
private:
C1_MacroAssembler* _masm;
CodeStubList* _slow_case_stubs;
+ BarrierSet* _bs;
Compilation* _compilation;
FrameMap* _frame_map;
--- a/src/share/vm/c1/c1_LIRGenerator.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -285,16 +285,7 @@ jlong LIRItem::get_jlong_constant() cons
void LIRGenerator::init() {
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
- CardTableModRefBS* ct = (CardTableModRefBS*)bs;
- assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
-
-#ifdef _LP64
- _card_table_base = new LIR_Const((jlong)ct->byte_map_base);
-#else
- _card_table_base = new LIR_Const((jint)ct->byte_map_base);
-#endif
+ _bs = Universe::heap()->barrier_set();
}
@@ -1239,8 +1230,37 @@ LIR_Opr LIRGenerator::load_constant(LIR_
// Various barriers
+void LIRGenerator::pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info) {
+ // Do the pre-write barrier, if any.
+ switch (_bs->kind()) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ G1SATBCardTableModRef_pre_barrier(addr_opr, patch, info);
+ break;
+#endif // SERIALGC
+ case BarrierSet::CardTableModRef:
+ case BarrierSet::CardTableExtension:
+ // No pre barriers
+ break;
+ case BarrierSet::ModRef:
+ case BarrierSet::Other:
+ // No pre barriers
+ break;
+ default :
+ ShouldNotReachHere();
+
+ }
+}
+
void LIRGenerator::post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
- switch (Universe::heap()->barrier_set()->kind()) {
+ switch (_bs->kind()) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ G1SATBCardTableModRef_post_barrier(addr, new_val);
+ break;
+#endif // SERIALGC
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
CardTableModRef_post_barrier(addr, new_val);
@@ -1254,11 +1274,120 @@ void LIRGenerator::post_barrier(LIR_OprD
}
}
+////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+void LIRGenerator::G1SATBCardTableModRef_pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info) {
+ if (G1DisablePreBarrier) return;
+
+ // First we test whether marking is in progress.
+ BasicType flag_type;
+ if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
+ flag_type = T_INT;
+ } else {
+ guarantee(in_bytes(PtrQueue::byte_width_of_active()) == 1,
+ "Assumption");
+ flag_type = T_BYTE;
+ }
+ LIR_Opr thrd = getThreadPointer();
+ LIR_Address* mark_active_flag_addr =
+ new LIR_Address(thrd,
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()),
+ flag_type);
+ // Read the marking-in-progress flag.
+ LIR_Opr flag_val = new_register(T_INT);
+ __ load(mark_active_flag_addr, flag_val);
+
+ LabelObj* start_store = new LabelObj();
+
+ LIR_PatchCode pre_val_patch_code =
+ patch ? lir_patch_normal : lir_patch_none;
+
+ LIR_Opr pre_val = new_register(T_OBJECT);
+
+ __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+ if (!addr_opr->is_address()) {
+ assert(addr_opr->is_register(), "must be");
+ addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, 0, T_OBJECT));
+ }
+ CodeStub* slow = new G1PreBarrierStub(addr_opr, pre_val, pre_val_patch_code,
+ info);
+ __ branch(lir_cond_notEqual, T_INT, slow);
+ __ branch_destination(slow->continuation());
+}
+
+void LIRGenerator::G1SATBCardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
+ if (G1DisablePostBarrier) return;
+
+ // If the "new_val" is a constant NULL, no barrier is necessary.
+ if (new_val->is_constant() &&
+ new_val->as_constant_ptr()->as_jobject() == NULL) return;
+
+ if (!new_val->is_register()) {
+ LIR_Opr new_val_reg = new_pointer_register();
+ if (new_val->is_constant()) {
+ __ move(new_val, new_val_reg);
+ } else {
+ __ leal(new_val, new_val_reg);
+ }
+ new_val = new_val_reg;
+ }
+ assert(new_val->is_register(), "must be a register at this point");
+
+ if (addr->is_address()) {
+ LIR_Address* address = addr->as_address_ptr();
+ LIR_Opr ptr = new_pointer_register();
+ if (!address->index()->is_valid() && address->disp() == 0) {
+ __ move(address->base(), ptr);
+ } else {
+ assert(address->disp() != max_jint, "lea doesn't support patched addresses!");
+ __ leal(addr, ptr);
+ }
+ addr = ptr;
+ }
+ assert(addr->is_register(), "must be a register at this point");
+
+ LIR_Opr xor_res = new_pointer_register();
+ LIR_Opr xor_shift_res = new_pointer_register();
+
+ if (TwoOperandLIRForm ) {
+ __ move(addr, xor_res);
+ __ logical_xor(xor_res, new_val, xor_res);
+ __ move(xor_res, xor_shift_res);
+ __ unsigned_shift_right(xor_shift_res,
+ LIR_OprFact::intConst(HeapRegion::LogOfHRGrainBytes),
+ xor_shift_res,
+ LIR_OprDesc::illegalOpr());
+ } else {
+ __ logical_xor(addr, new_val, xor_res);
+ __ unsigned_shift_right(xor_res,
+ LIR_OprFact::intConst(HeapRegion::LogOfHRGrainBytes),
+ xor_shift_res,
+ LIR_OprDesc::illegalOpr());
+ }
+
+ if (!new_val->is_register()) {
+ LIR_Opr new_val_reg = new_pointer_register();
+ __ leal(new_val, new_val_reg);
+ new_val = new_val_reg;
+ }
+ assert(new_val->is_register(), "must be a register at this point");
+
+ __ cmp(lir_cond_notEqual, xor_shift_res, LIR_OprFact::intptrConst(NULL_WORD));
+
+ CodeStub* slow = new G1PostBarrierStub(addr, new_val);
+ __ branch(lir_cond_notEqual, T_INT, slow);
+ __ branch_destination(slow->continuation());
+}
+
+#endif // SERIALGC
+////////////////////////////////////////////////////////////////////////
+
void LIRGenerator::CardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(sizeof(*((CardTableModRefBS*)bs)->byte_map_base) == sizeof(jbyte), "adjust this code");
- LIR_Const* card_table_base = new LIR_Const(((CardTableModRefBS*)bs)->byte_map_base);
+ assert(sizeof(*((CardTableModRefBS*)_bs)->byte_map_base) == sizeof(jbyte), "adjust this code");
+ LIR_Const* card_table_base = new LIR_Const(((CardTableModRefBS*)_bs)->byte_map_base);
if (addr->is_address()) {
LIR_Address* address = addr->as_address_ptr();
LIR_Opr ptr = new_register(T_OBJECT);
@@ -1388,6 +1517,13 @@ void LIRGenerator::do_StoreField(StoreFi
__ membar_release();
}
+ if (is_oop) {
+ // Do the pre-write barrier, if any.
+ pre_barrier(LIR_OprFact::address(address),
+ needs_patching,
+ (info ? new CodeEmitInfo(info) : NULL));
+ }
+
if (is_volatile) {
assert(!needs_patching && x->is_loaded(),
"how do we know it's volatile if it's not loaded");
@@ -1398,7 +1534,12 @@ void LIRGenerator::do_StoreField(StoreFi
}
if (is_oop) {
+#ifdef PRECISE_CARDMARK
+ // Precise cardmarks don't work
+ post_barrier(LIR_OprFact::address(address), value.result());
+#else
post_barrier(object.result(), value.result());
+#endif // PRECISE_CARDMARK
}
if (is_volatile && os::is_MP()) {
--- a/src/share/vm/c1/c1_LIRGenerator.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_LIRGenerator.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -145,6 +145,7 @@ class PhiResolver: public CompilationRes
// only the classes below belong in the same file
class LIRGenerator: public InstructionVisitor, public BlockClosure {
+
private:
Compilation* _compilation;
ciMethod* _method; // method that we are compiling
@@ -154,6 +155,7 @@ class LIRGenerator: public InstructionVi
Values _instruction_for_operand;
BitMap2D _vreg_flags; // flags which can be set on a per-vreg basis
LIR_List* _lir;
+ BarrierSet* _bs;
LIRGenerator* gen() {
return this;
@@ -173,8 +175,6 @@ class LIRGenerator: public InstructionVi
GrowableArray<LIR_Const*> _constants;
LIR_OprList _reg_for_constants;
Values _unpinned_constants;
-
- LIR_Const* _card_table_base;
friend class PhiResolver;
@@ -195,8 +195,6 @@ class LIRGenerator: public InstructionVi
// get a constant into a register and get track of what register was used
LIR_Opr load_constant(Constant* x);
LIR_Opr load_constant(LIR_Const* constant);
-
- LIR_Const* card_table_base() const { return _card_table_base; }
void set_result(Value x, LIR_Opr opr) {
assert(opr->is_valid(), "must set to valid value");
@@ -253,12 +251,17 @@ class LIRGenerator: public InstructionVi
// generic interface
+ void pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info);
void post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
// specific implementations
+ // pre barriers
+
+ void G1SATBCardTableModRef_pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info);
// post barriers
+ void G1SATBCardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
void CardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
--- a/src/share/vm/c1/c1_Runtime1.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_Runtime1.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -168,6 +168,8 @@ void Runtime1::generate_blob_for(StubID
switch (id) {
// These stubs don't need to have an oopmap
case dtrace_object_alloc_id:
+ case g1_pre_barrier_slow_id:
+ case g1_post_barrier_slow_id:
case slow_subtype_check_id:
case fpu2long_stub_id:
case unwind_exception_id:
--- a/src/share/vm/c1/c1_Runtime1.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_Runtime1.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -56,6 +56,8 @@ class StubAssembler;
stub(access_field_patching) \
stub(load_klass_patching) \
stub(jvmti_exception_throw) \
+ stub(g1_pre_barrier_slow) \
+ stub(g1_post_barrier_slow) \
stub(fpu2long_stub) \
stub(counter_overflow) \
last_entry(number_of_ids)
--- a/src/share/vm/c1/c1_globals.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/c1/c1_globals.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -213,9 +213,6 @@
develop(bool, UseFastLocking, true, \
"Use fast inlined locking code") \
\
- product(bool, FastTLABRefill, true, \
- "Use fast TLAB refill code") \
- \
develop(bool, UseSlowPath, false, \
"For debugging: test slow cases by always using them") \
\
--- a/src/share/vm/compiler/methodLiveness.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/compiler/methodLiveness.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -76,8 +76,9 @@ class BitCounter: public BitMapClosure {
BitCounter() : _count(0) {}
// Callback when bit in map is set
- virtual void do_bit(size_t offset) {
+ virtual bool do_bit(size_t offset) {
_count++;
+ return true;
}
int count() {
@@ -467,7 +468,7 @@ MethodLivenessResult MethodLiveness::get
bci = 0;
}
- MethodLivenessResult answer(NULL,0);
+ MethodLivenessResult answer((uintptr_t*)NULL,0);
if (_block_count > 0) {
if (TimeLivenessAnalysis) _time_total.start();
--- a/src/share/vm/compiler/methodLiveness.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/compiler/methodLiveness.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -29,7 +29,7 @@ class MethodLivenessResult : public BitM
bool _is_valid;
public:
- MethodLivenessResult(uintptr_t* map, idx_t size_in_bits)
+ MethodLivenessResult(BitMap::bm_word_t* map, idx_t size_in_bits)
: BitMap(map, size_in_bits)
, _is_valid(false)
{}
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -790,7 +790,7 @@ CompactibleFreeListSpace::object_iterate
}
-HeapWord* CompactibleFreeListSpace::block_start(const void* p) const {
+HeapWord* CompactibleFreeListSpace::block_start_const(const void* p) const {
NOT_PRODUCT(verify_objects_initialized());
return _bt.block_start(p);
}
@@ -2286,9 +2286,9 @@ void CompactibleFreeListSpace::verifyInd
}
void CompactibleFreeListSpace::verifyIndexedFreeList(size_t size) const {
- guarantee(size % 2 == 0, "Odd slots should be empty");
- for (FreeChunk* fc = _indexedFreeList[size].head(); fc != NULL;
- fc = fc->next()) {
+ FreeChunk* fc = _indexedFreeList[size].head();
+ guarantee((size % 2 == 0) || fc == NULL, "Odd slots should be empty");
+ for (; fc != NULL; fc = fc->next()) {
guarantee(fc->size() == size, "Size inconsistency");
guarantee(fc->isFree(), "!free?");
guarantee(fc->next() == NULL || fc->next()->prev() == fc, "Broken list");
@@ -2790,10 +2790,11 @@ initialize_sequential_subtasks_for_resca
assert(n_threads > 0, "Unexpected n_threads argument");
const size_t task_size = rescan_task_size();
size_t n_tasks = (used_region().word_size() + task_size - 1)/task_size;
- assert((used_region().start() + (n_tasks - 1)*task_size <
- used_region().end()) &&
- (used_region().start() + n_tasks*task_size >=
- used_region().end()), "n_task calculation incorrect");
+ assert((n_tasks == 0) == used_region().is_empty(), "n_tasks incorrect");
+ assert(n_tasks == 0 ||
+ ((used_region().start() + (n_tasks - 1)*task_size < used_region().end()) &&
+ (used_region().start() + n_tasks*task_size >= used_region().end())),
+ "n_tasks calculation incorrect");
SequentialSubTasksDone* pst = conc_par_seq_tasks();
assert(!pst->valid(), "Clobbering existing data?");
pst->set_par_threads(n_threads);
@@ -2833,7 +2834,7 @@ initialize_sequential_subtasks_for_marki
assert(n_tasks == 0 ||
((span.start() + (n_tasks - 1)*task_size < span.end()) &&
(span.start() + n_tasks*task_size >= span.end())),
- "n_task calculation incorrect");
+ "n_tasks calculation incorrect");
SequentialSubTasksDone* pst = conc_par_seq_tasks();
assert(!pst->valid(), "Clobbering existing data?");
pst->set_par_threads(n_threads);
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -502,7 +502,7 @@ class CompactibleFreeListSpace: public C
void blk_iterate(BlkClosure* cl);
void blk_iterate_careful(BlkClosureCareful* cl);
- HeapWord* block_start(const void* p) const;
+ HeapWord* block_start_const(const void* p) const;
HeapWord* block_start_careful(const void* p) const;
size_t block_size(const HeapWord* p) const;
size_t block_size_no_stall(HeapWord* p, const CMSCollector* c) const;
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -2761,13 +2761,14 @@ class VerifyMarkedClosure: public BitMap
public:
VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
- void do_bit(size_t offset) {
+ bool do_bit(size_t offset) {
HeapWord* addr = _marks->offsetToHeapWord(offset);
if (!_marks->isMarked(addr)) {
oop(addr)->print();
gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
_failed = true;
}
+ return true;
}
bool failed() { return _failed; }
@@ -3650,6 +3651,7 @@ class CMSConcMarkingTask: public Yieldin
CompactibleFreeListSpace* _cms_space;
CompactibleFreeListSpace* _perm_space;
HeapWord* _global_finger;
+ HeapWord* _restart_addr;
// Exposed here for yielding support
Mutex* const _bit_map_lock;
@@ -3680,7 +3682,7 @@ class CMSConcMarkingTask: public Yieldin
_term.set_task(this);
assert(_cms_space->bottom() < _perm_space->bottom(),
"Finger incorrectly initialized below");
- _global_finger = _cms_space->bottom();
+ _restart_addr = _global_finger = _cms_space->bottom();
}
@@ -3698,6 +3700,10 @@ class CMSConcMarkingTask: public Yieldin
bool result() { return _result; }
void reset(HeapWord* ra) {
+ assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)");
+ assert(_global_finger >= _perm_space->end(), "Postcondition of ::work(i)");
+ assert(ra < _perm_space->end(), "ra too large");
+ _restart_addr = _global_finger = ra;
_term.reset_for_reuse();
}
@@ -3842,16 +3848,24 @@ void CMSConcMarkingTask::do_scan_and_mar
int n_tasks = pst->n_tasks();
// We allow that there may be no tasks to do here because
// we are restarting after a stack overflow.
- assert(pst->valid() || n_tasks == 0, "Uninitializd use?");
+ assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
int nth_task = 0;
- HeapWord* start = sp->bottom();
+ HeapWord* aligned_start = sp->bottom();
+ if (sp->used_region().contains(_restart_addr)) {
+ // Align down to a card boundary for the start of 0th task
+ // for this space.
+ aligned_start =
+ (HeapWord*)align_size_down((uintptr_t)_restart_addr,
+ CardTableModRefBS::card_size);
+ }
+
size_t chunk_size = sp->marking_task_size();
while (!pst->is_task_claimed(/* reference */ nth_task)) {
// Having claimed the nth task in this space,
// compute the chunk that it corresponds to:
- MemRegion span = MemRegion(start + nth_task*chunk_size,
- start + (nth_task+1)*chunk_size);
+ MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
+ aligned_start + (nth_task+1)*chunk_size);
// Try and bump the global finger via a CAS;
// note that we need to do the global finger bump
// _before_ taking the intersection below, because
@@ -3866,26 +3880,40 @@ void CMSConcMarkingTask::do_scan_and_mar
// beyond the "top" address of the space.
span = span.intersection(sp->used_region());
if (!span.is_empty()) { // Non-null task
- // We want to skip the first object because
- // the protocol is to scan any object in its entirety
- // that _starts_ in this span; a fortiori, any
- // object starting in an earlier span is scanned
- // as part of an earlier claimed task.
- // Below we use the "careful" version of block_start
- // so we do not try to navigate uninitialized objects.
- HeapWord* prev_obj = sp->block_start_careful(span.start());
- // Below we use a variant of block_size that uses the
- // Printezis bits to avoid waiting for allocated
- // objects to become initialized/parsable.
- while (prev_obj < span.start()) {
- size_t sz = sp->block_size_no_stall(prev_obj, _collector);
- if (sz > 0) {
- prev_obj += sz;
+ HeapWord* prev_obj;
+ assert(!span.contains(_restart_addr) || nth_task == 0,
+ "Inconsistency");
+ if (nth_task == 0) {
+ // For the 0th task, we'll not need to compute a block_start.
+ if (span.contains(_restart_addr)) {
+ // In the case of a restart because of stack overflow,
+ // we might additionally skip a chunk prefix.
+ prev_obj = _restart_addr;
} else {
- // In this case we may end up doing a bit of redundant
- // scanning, but that appears unavoidable, short of
- // locking the free list locks; see bug 6324141.
- break;
+ prev_obj = span.start();
+ }
+ } else {
+ // We want to skip the first object because
+ // the protocol is to scan any object in its entirety
+ // that _starts_ in this span; a fortiori, any
+ // object starting in an earlier span is scanned
+ // as part of an earlier claimed task.
+ // Below we use the "careful" version of block_start
+ // so we do not try to navigate uninitialized objects.
+ prev_obj = sp->block_start_careful(span.start());
+ // Below we use a variant of block_size that uses the
+ // Printezis bits to avoid waiting for allocated
+ // objects to become initialized/parsable.
+ while (prev_obj < span.start()) {
+ size_t sz = sp->block_size_no_stall(prev_obj, _collector);
+ if (sz > 0) {
+ prev_obj += sz;
+ } else {
+ // In this case we may end up doing a bit of redundant
+ // scanning, but that appears unavoidable, short of
+ // locking the free list locks; see bug 6324141.
+ break;
+ }
}
}
if (prev_obj < span.end()) {
@@ -3938,12 +3966,14 @@ class Par_ConcMarkingClosure: public Oop
void handle_stack_overflow(HeapWord* lost);
};
-// Grey object rescan during work stealing phase --
-// the salient assumption here is that stolen oops must
-// always be initialized, so we do not need to check for
-// uninitialized objects before scanning here.
+// Grey object scanning during work stealing phase --
+// the salient assumption here is that any references
+// that are in these stolen objects being scanned must
+// already have been initialized (else they would not have
+// been published), so we do not need to check for
+// uninitialized objects before pushing here.
void Par_ConcMarkingClosure::do_oop(oop obj) {
- assert(obj->is_oop_or_null(), "expected an oop or NULL");
+ assert(obj->is_oop_or_null(true), "expected an oop or NULL");
HeapWord* addr = (HeapWord*)obj;
// Check if oop points into the CMS generation
// and is not marked
@@ -4001,7 +4031,7 @@ void Par_ConcMarkingClosure::trim_queue(
// in CMSCollector's _restart_address.
void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
// We need to do this under a mutex to prevent other
- // workers from interfering with the expansion below.
+ // workers from interfering with the work done below.
MutexLockerEx ml(_overflow_stack->par_lock(),
Mutex::_no_safepoint_check_flag);
// Remember the least grey address discarded
@@ -4640,8 +4670,11 @@ size_t CMSCollector::preclean_card_table
startTimer();
sample_eden();
// Get and clear dirty region from card table
- dirtyRegion = _ct->ct_bs()->dirty_card_range_after_preclean(
- MemRegion(nextAddr, endAddr));
+ dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
+ MemRegion(nextAddr, endAddr),
+ true,
+ CardTableModRefBS::precleaned_card_val());
+
assert(dirtyRegion.start() >= nextAddr,
"returned region inconsistent?");
}
@@ -5409,8 +5442,8 @@ void CMSCollector::do_remark_non_paralle
&mrias_cl);
{
TraceTime t("grey object rescan", PrintGCDetails, false, gclog_or_tty);
- // Iterate over the dirty cards, marking them precleaned, and
- // setting the corresponding bits in the mod union table.
+ // Iterate over the dirty cards, setting the corresponding bits in the
+ // mod union table.
{
ModUnionClosure modUnionClosure(&_modUnionTable);
_ct->ct_bs()->dirty_card_iterate(
@@ -6182,7 +6215,7 @@ HeapWord* CMSCollector::next_card_start_
// bit vector itself. That is done by a separate call CMSBitMap::allocate()
// further below.
CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
- _bm(NULL,0),
+ _bm(),
_shifter(shifter),
_lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
{
@@ -6207,7 +6240,7 @@ bool CMSBitMap::allocate(MemRegion mr) {
}
assert(_virtual_space.committed_size() == brs.size(),
"didn't reserve backing store for all of CMS bit map?");
- _bm.set_map((uintptr_t*)_virtual_space.low());
+ _bm.set_map((BitMap::bm_word_t*)_virtual_space.low());
assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
_bmWordSize, "inconsistency in bit map sizing");
_bm.set_size(_bmWordSize >> _shifter);
@@ -6554,7 +6587,7 @@ void Par_MarkRefsIntoAndScanClosure::do_
if (obj != NULL) {
// Ignore mark word because this could be an already marked oop
// that may be chained at the end of the overflow list.
- assert(obj->is_oop(), "expected an oop");
+ assert(obj->is_oop(true), "expected an oop");
HeapWord* addr = (HeapWord*)obj;
if (_span.contains(addr) &&
!_bit_map->isMarked(addr)) {
@@ -6845,10 +6878,10 @@ void MarkFromRootsClosure::reset(HeapWor
// Should revisit to see if this should be restructured for
// greater efficiency.
-void MarkFromRootsClosure::do_bit(size_t offset) {
+bool MarkFromRootsClosure::do_bit(size_t offset) {
if (_skipBits > 0) {
_skipBits--;
- return;
+ return true;
}
// convert offset into a HeapWord*
HeapWord* addr = _bitMap->startWord() + offset;
@@ -6886,10 +6919,11 @@ void MarkFromRootsClosure::do_bit(size_t
} // ...else the setting of klass will dirty the card anyway.
}
DEBUG_ONLY(})
- return;
+ return true;
}
}
scanOopsInOop(addr);
+ return true;
}
// We take a break if we've been at this for a while,
@@ -7023,10 +7057,10 @@ Par_MarkFromRootsClosure::Par_MarkFromRo
// Should revisit to see if this should be restructured for
// greater efficiency.
-void Par_MarkFromRootsClosure::do_bit(size_t offset) {
+bool Par_MarkFromRootsClosure::do_bit(size_t offset) {
if (_skip_bits > 0) {
_skip_bits--;
- return;
+ return true;
}
// convert offset into a HeapWord*
HeapWord* addr = _bit_map->startWord() + offset;
@@ -7041,10 +7075,11 @@ void Par_MarkFromRootsClosure::do_bit(si
if (p->klass_or_null() == NULL || !p->is_parsable()) {
// in the case of Clean-on-Enter optimization, redirty card
// and avoid clearing card by increasing the threshold.
- return;
+ return true;
}
}
scan_oops_in_oop(addr);
+ return true;
}
void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
@@ -7167,7 +7202,7 @@ void MarkFromRootsVerifyClosure::reset(H
// Should revisit to see if this should be restructured for
// greater efficiency.
-void MarkFromRootsVerifyClosure::do_bit(size_t offset) {
+bool MarkFromRootsVerifyClosure::do_bit(size_t offset) {
// convert offset into a HeapWord*
HeapWord* addr = _verification_bm->startWord() + offset;
assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
@@ -7195,6 +7230,7 @@ void MarkFromRootsVerifyClosure::do_bit(
new_oop->oop_iterate(&_pam_verify_closure);
}
assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
+ return true;
}
PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
@@ -7289,6 +7325,8 @@ Par_PushOrMarkClosure::Par_PushOrMarkClo
_should_remember_klasses(collector->should_unload_classes())
{ }
+// Assumes thread-safe access by callers, who are
+// responsible for mutual exclusion.
void CMSCollector::lower_restart_addr(HeapWord* low) {
assert(_span.contains(low), "Out of bounds addr");
if (_restart_addr == NULL) {
@@ -7314,7 +7352,7 @@ void PushOrMarkClosure::handle_stack_ove
// in CMSCollector's _restart_address.
void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
// We need to do this under a mutex to prevent other
- // workers from interfering with the expansion below.
+ // workers from interfering with the work done below.
MutexLockerEx ml(_overflow_stack->par_lock(),
Mutex::_no_safepoint_check_flag);
// Remember the least grey address discarded
@@ -7438,8 +7476,12 @@ PushAndMarkClosure::PushAndMarkClosure(C
// Grey object rescan during pre-cleaning and second checkpoint phases --
// the non-parallel version (the parallel version appears further below.)
void PushAndMarkClosure::do_oop(oop obj) {
- // If _concurrent_precleaning, ignore mark word verification
- assert(obj->is_oop_or_null(_concurrent_precleaning),
+ // Ignore mark word verification. If during concurrent precleaning,
+ // the object monitor may be locked. If during the checkpoint
+ // phases, the object may already have been reached by a different
+ // path and may be at the end of the global overflow list (so
+ // the mark word may be NULL).
+ assert(obj->is_oop_or_null(true /* ignore mark word */),
"expected an oop or NULL");
HeapWord* addr = (HeapWord*)obj;
// Check if oop points into the CMS generation
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -1327,7 +1327,7 @@ class MarkFromRootsClosure: public BitMa
CMSMarkStack* markStack,
CMSMarkStack* revisitStack,
bool should_yield, bool verifying = false);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
void reset(HeapWord* addr);
inline void do_yield_check();
@@ -1363,7 +1363,7 @@ class Par_MarkFromRootsClosure: public B
CMSMarkStack* overflow_stack,
CMSMarkStack* revisit_stack,
bool should_yield);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
inline void do_yield_check();
private:
@@ -1411,7 +1411,7 @@ class MarkFromRootsVerifyClosure: public
CMSBitMap* verification_bm,
CMSBitMap* cms_bm,
CMSMarkStack* mark_stack);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
void reset(HeapWord* addr);
};
@@ -1420,8 +1420,9 @@ class MarkFromRootsVerifyClosure: public
// "empty" (i.e. the bit vector doesn't have any 1-bits).
class FalseBitMapClosure: public BitMapClosure {
public:
- void do_bit(size_t offset) {
+ bool do_bit(size_t offset) {
guarantee(false, "Should not have a 1 bit");
+ return true;
}
};
--- a/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Tue Sep 30 12:24:27 2008 -0400
@@ -124,17 +124,6 @@ compactibleFreeListSpace.hpp
compactibleFreeListSpace.hpp space.hpp
compactingPermGenGen.cpp concurrentMarkSweepGeneration.inline.hpp
-
-concurrentGCThread.cpp concurrentGCThread.hpp
-concurrentGCThread.cpp init.hpp
-concurrentGCThread.cpp instanceRefKlass.hpp
-concurrentGCThread.cpp interfaceSupport.hpp
-concurrentGCThread.cpp java.hpp
-concurrentGCThread.cpp javaCalls.hpp
-concurrentGCThread.cpp oop.inline.hpp
-concurrentGCThread.cpp systemDictionary.hpp
-
-concurrentGCThread.hpp thread.hpp
concurrentMarkSweepGeneration.cpp cardTableRS.hpp
concurrentMarkSweepGeneration.cpp cmsAdaptiveSizePolicy.hpp
@@ -167,7 +156,7 @@ concurrentMarkSweepGeneration.cpp
concurrentMarkSweepGeneration.cpp vmCMSOperations.hpp
concurrentMarkSweepGeneration.cpp vmThread.hpp
-concurrentMarkSweepGeneration.hpp bitMap.hpp
+concurrentMarkSweepGeneration.hpp bitMap.inline.hpp
concurrentMarkSweepGeneration.hpp freeBlockDictionary.hpp
concurrentMarkSweepGeneration.hpp gSpaceCounters.hpp
concurrentMarkSweepGeneration.hpp gcStats.hpp
--- a/src/share/vm/gc_implementation/includeDB_gc_parallelScavenge Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/includeDB_gc_parallelScavenge Tue Sep 30 12:24:27 2008 -0400
@@ -150,7 +150,6 @@ parallelScavengeHeap.hpp
parallelScavengeHeap.hpp psYoungGen.hpp
parallelScavengeHeap.hpp ostream.hpp
-parMarkBitMap.cpp bitMap.hpp
parMarkBitMap.cpp bitMap.inline.hpp
parMarkBitMap.cpp oop.inline.hpp
parMarkBitMap.cpp os.hpp
@@ -159,7 +158,6 @@ parMarkBitMap.cpp parMarkBitMap.inline
parMarkBitMap.cpp parMarkBitMap.inline.hpp
parMarkBitMap.cpp psParallelCompact.hpp
-parMarkBitMap.hpp bitMap.hpp
parMarkBitMap.hpp bitMap.inline.hpp
parMarkBitMap.hpp psVirtualspace.hpp
--- a/src/share/vm/gc_implementation/includeDB_gc_shared Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/includeDB_gc_shared Tue Sep 30 12:24:27 2008 -0400
@@ -24,6 +24,23 @@
// NOTE: DO NOT CHANGE THIS COPYRIGHT TO NEW STYLE - IT WILL BREAK makeDeps!
+concurrentGCThread.cpp concurrentGCThread.hpp
+concurrentGCThread.cpp init.hpp
+concurrentGCThread.cpp instanceRefKlass.hpp
+concurrentGCThread.cpp interfaceSupport.hpp
+concurrentGCThread.cpp java.hpp
+concurrentGCThread.cpp javaCalls.hpp
+concurrentGCThread.cpp oop.inline.hpp
+concurrentGCThread.cpp systemDictionary.hpp
+
+concurrentGCThread.hpp thread.hpp
+
+coTracker.hpp globalDefinitions.hpp
+coTracker.hpp numberSeq.hpp
+
+coTracker.cpp coTracker.hpp
+coTracker.cpp os.hpp
+
allocationStats.cpp allocationStats.hpp
allocationStats.cpp ostream.hpp
@@ -36,6 +53,13 @@ gcAdaptivePolicyCounters.hpp
gcAdaptivePolicyCounters.cpp resourceArea.hpp
gcAdaptivePolicyCounters.cpp gcAdaptivePolicyCounters.hpp
+
+gcOverheadReporter.cpp allocation.inline.hpp
+gcOverheadReporter.cpp concurrentGCThread.hpp
+gcOverheadReporter.cpp coTracker.hpp
+gcOverheadReporter.cpp gcOverheadReporter.hpp
+gcOverheadReporter.cpp ostream.hpp
+gcOverheadReporter.cpp thread_<os_family>.inline.hpp
gSpaceCounters.cpp generation.hpp
gSpaceCounters.cpp resourceArea.hpp
@@ -75,3 +99,5 @@ spaceCounters.hpp
spaceCounters.hpp mutableSpace.hpp
spaceCounters.hpp perfData.hpp
spaceCounters.hpp generationCounters.hpp
+
+vmGCOperations.cpp g1CollectedHeap.hpp
--- a/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -192,16 +192,16 @@ private:
};
inline ParMarkBitMap::ParMarkBitMap():
- _beg_bits(NULL, 0),
- _end_bits(NULL, 0)
+ _beg_bits(),
+ _end_bits()
{
_region_start = 0;
_virtual_space = 0;
}
inline ParMarkBitMap::ParMarkBitMap(MemRegion covered_region):
- _beg_bits(NULL, 0),
- _end_bits(NULL, 0)
+ _beg_bits(),
+ _end_bits()
{
initialize(covered_region);
}
@@ -325,7 +325,7 @@ ParMarkBitMap::obj_size(HeapWord* beg_ad
inline size_t ParMarkBitMap::obj_size(idx_t beg_bit) const
{
- const idx_t end_bit = _end_bits.find_next_one_bit(beg_bit, size());
+ const idx_t end_bit = _end_bits.get_next_one_offset_inline(beg_bit, size());
assert(is_marked(beg_bit), "obj not marked");
assert(end_bit < size(), "end bit missing");
return obj_size(beg_bit, end_bit);
@@ -384,13 +384,13 @@ inline ParMarkBitMap::idx_t
inline ParMarkBitMap::idx_t
ParMarkBitMap::find_obj_beg(idx_t beg, idx_t end) const
{
- return _beg_bits.find_next_one_bit(beg, end);
+ return _beg_bits.get_next_one_offset_inline_aligned_right(beg, end);
}
inline ParMarkBitMap::idx_t
ParMarkBitMap::find_obj_end(idx_t beg, idx_t end) const
{
- return _end_bits.find_next_one_bit(beg, end);
+ return _end_bits.get_next_one_offset_inline_aligned_right(beg, end);
}
inline HeapWord*
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -184,6 +184,20 @@ class ParallelScavengeHeap : public Coll
size_t tlab_capacity(Thread* thr) const;
size_t unsafe_max_tlab_alloc(Thread* thr) const;
+ // Can a compiler initialize a new object without store barriers?
+ // This permission only extends from the creation of a new object
+ // via a TLAB up to the first subsequent safepoint.
+ virtual bool can_elide_tlab_store_barriers() const {
+ return true;
+ }
+
+ // Can a compiler elide a store barrier when it writes
+ // a permanent oop into the heap? Applies when the compiler
+ // is storing x to the heap, where x->is_perm() is true.
+ virtual bool can_elide_permanent_oop_store_barriers() const {
+ return true;
+ }
+
void oop_iterate(OopClosure* cl);
void object_iterate(ObjectClosure* cl);
void permanent_oop_iterate(OopClosure* cl);
--- a/src/share/vm/gc_implementation/shared/vmGCOperations.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_implementation/shared/vmGCOperations.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -74,6 +74,7 @@ bool VM_GC_Operation::doit_prologue() {
// If the GC count has changed someone beat us to the collection
// Get the Heap_lock after the pending_list_lock.
Heap_lock->lock();
+
// Check invocations
if (skip_operation()) {
// skip collection
@@ -82,6 +83,8 @@ bool VM_GC_Operation::doit_prologue() {
_prologue_succeeded = false;
} else {
_prologue_succeeded = true;
+ SharedHeap* sh = SharedHeap::heap();
+ if (sh != NULL) sh->_thread_holds_heap_lock_for_gc = true;
}
return _prologue_succeeded;
}
@@ -90,6 +93,8 @@ void VM_GC_Operation::doit_epilogue() {
void VM_GC_Operation::doit_epilogue() {
assert(Thread::current()->is_Java_thread(), "just checking");
// Release the Heap_lock first.
+ SharedHeap* sh = SharedHeap::heap();
+ if (sh != NULL) sh->_thread_holds_heap_lock_for_gc = false;
Heap_lock->unlock();
release_and_notify_pending_list_lock();
}
@@ -148,12 +153,27 @@ void VM_GenCollectForPermanentAllocation
void VM_GenCollectForPermanentAllocation::doit() {
JvmtiGCForAllocationMarker jgcm;
notify_gc_begin(true);
- GenCollectedHeap* gch = GenCollectedHeap::heap();
- GCCauseSetter gccs(gch, _gc_cause);
- gch->do_full_collection(gch->must_clear_all_soft_refs(),
- gch->n_gens() - 1);
- _res = gch->perm_gen()->allocate(_size, false);
- assert(gch->is_in_reserved_or_null(_res), "result not in heap");
+ SharedHeap* heap = (SharedHeap*)Universe::heap();
+ GCCauseSetter gccs(heap, _gc_cause);
+ switch (heap->kind()) {
+ case (CollectedHeap::GenCollectedHeap): {
+ GenCollectedHeap* gch = (GenCollectedHeap*)heap;
+ gch->do_full_collection(gch->must_clear_all_soft_refs(),
+ gch->n_gens() - 1);
+ break;
+ }
+#ifndef SERIALGC
+ case (CollectedHeap::G1CollectedHeap): {
+ G1CollectedHeap* g1h = (G1CollectedHeap*)heap;
+ g1h->do_full_collection(_gc_cause == GCCause::_last_ditch_collection);
+ break;
+ }
+#endif // SERIALGC
+ default:
+ ShouldNotReachHere();
+ }
+ _res = heap->perm_gen()->allocate(_size, false);
+ assert(heap->is_in_reserved_or_null(_res), "result not in heap");
if (_res == NULL && GC_locker::is_active_and_needs_gc()) {
set_gc_locked();
}
--- a/src/share/vm/gc_interface/collectedHeap.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_interface/collectedHeap.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -138,13 +138,6 @@ oop CollectedHeap::new_store_barrier(oop
return new_obj;
}
-bool CollectedHeap::can_elide_permanent_oop_store_barriers() const {
- // %%% This needs refactoring. (It was gating logic from the server compiler.)
- guarantee(kind() < CollectedHeap::G1CollectedHeap, "");
- return !UseConcMarkSweepGC;
-}
-
-
HeapWord* CollectedHeap::allocate_new_tlab(size_t size) {
guarantee(false, "thread-local allocation buffers not supported");
return NULL;
--- a/src/share/vm/gc_interface/collectedHeap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_interface/collectedHeap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -364,10 +364,8 @@ class CollectedHeap : public CHeapObj {
// Can a compiler initialize a new object without store barriers?
// This permission only extends from the creation of a new object
// via a TLAB up to the first subsequent safepoint.
- virtual bool can_elide_tlab_store_barriers() const {
- guarantee(kind() < CollectedHeap::G1CollectedHeap, "else change or refactor this");
- return true;
- }
+ virtual bool can_elide_tlab_store_barriers() const = 0;
+
// If a compiler is eliding store barriers for TLAB-allocated objects,
// there is probably a corresponding slow path which can produce
// an object allocated anywhere. The compiler's runtime support
@@ -379,12 +377,10 @@ class CollectedHeap : public CHeapObj {
// Can a compiler elide a store barrier when it writes
// a permanent oop into the heap? Applies when the compiler
// is storing x to the heap, where x->is_perm() is true.
- virtual bool can_elide_permanent_oop_store_barriers() const;
+ virtual bool can_elide_permanent_oop_store_barriers() const = 0;
// Does this heap support heap inspection (+PrintClassHistogram?)
- virtual bool supports_heap_inspection() const {
- return false; // Until RFE 5023697 is implemented
- }
+ virtual bool supports_heap_inspection() const = 0;
// Perform a collection of the heap; intended for use in implementing
// "System.gc". This probably implies as full a collection as the
--- a/src/share/vm/gc_interface/collectedHeap.inline.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -122,7 +122,7 @@ HeapWord* CollectedHeap::common_mem_allo
return result;
}
}
- bool gc_overhead_limit_was_exceeded;
+ bool gc_overhead_limit_was_exceeded = false;
result = Universe::heap()->mem_allocate(size,
is_noref,
false,
--- a/src/share/vm/gc_interface/gcCause.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/gc_interface/gcCause.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -60,6 +60,8 @@ class GCCause : public AllStatic {
_old_generation_too_full_to_scavenge,
_adaptive_size_policy,
+ _g1_inc_collection_pause, _g1_pop_region_collection_pause,
+
_last_ditch_collection,
_last_gc_cause
};
@@ -68,12 +70,14 @@ class GCCause : public AllStatic {
return (cause == GCCause::_java_lang_system_gc ||
cause == GCCause::_jvmti_force_gc);
}
+
inline static bool is_serviceability_requested_gc(GCCause::Cause
cause) {
return (cause == GCCause::_jvmti_force_gc ||
cause == GCCause::_heap_inspection ||
cause == GCCause::_heap_dump);
}
+
// Return a string describing the GCCause.
static const char* to_string(GCCause::Cause cause);
// Return true if the GCCause is for a full collection.
--- a/src/share/vm/includeDB_compiler1 Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/includeDB_compiler1 Tue Sep 30 12:24:27 2008 -0400
@@ -36,6 +36,9 @@ c1_CFGPrinter.hpp
c1_CFGPrinter.hpp c1_Compilation.hpp
c1_CFGPrinter.hpp c1_Instruction.hpp
+cardTableModRefBS.cpp c1_LIR.hpp
+cardTableModRefBS.cpp c1_LIRGenerator.hpp
+
c1_Canonicalizer.cpp c1_Canonicalizer.hpp
c1_Canonicalizer.cpp c1_InstructionPrinter.hpp
c1_Canonicalizer.cpp ciArray.hpp
@@ -55,6 +58,7 @@ c1_CodeStubs_<arch>.cpp
c1_CodeStubs_<arch>.cpp c1_LIRAssembler.hpp
c1_CodeStubs_<arch>.cpp c1_MacroAssembler.hpp
c1_CodeStubs_<arch>.cpp c1_Runtime1.hpp
+c1_CodeStubs_<arch>.cpp g1SATBCardTableModRefBS.hpp
c1_CodeStubs_<arch>.cpp nativeInst_<arch>.hpp
c1_CodeStubs_<arch>.cpp sharedRuntime.hpp
c1_CodeStubs_<arch>.cpp vmreg_<arch>.inline.hpp
@@ -141,6 +145,7 @@ c1_globals_<os_family>.hpp
c1_globals_<os_family>.hpp globalDefinitions.hpp
c1_globals_<os_family>.hpp macros.hpp
+c1_GraphBuilder.cpp bitMap.inline.hpp
c1_GraphBuilder.cpp bytecode.hpp
c1_GraphBuilder.cpp c1_CFGPrinter.hpp
c1_GraphBuilder.cpp c1_Canonicalizer.hpp
@@ -158,6 +163,7 @@ c1_GraphBuilder.hpp
c1_GraphBuilder.hpp ciMethodData.hpp
c1_GraphBuilder.hpp ciStreams.hpp
+c1_IR.cpp bitMap.inline.hpp
c1_IR.cpp c1_Compilation.hpp
c1_IR.cpp c1_FrameMap.hpp
c1_IR.cpp c1_GraphBuilder.hpp
@@ -232,20 +238,22 @@ c1_LIRAssembler_<arch>.cpp
c1_LIRAssembler_<arch>.hpp generate_platform_dependent_include
-c1_LIRGenerator.cpp c1_Compilation.hpp
-c1_LIRGenerator.cpp c1_FrameMap.hpp
-c1_LIRGenerator.cpp c1_Instruction.hpp
-c1_LIRGenerator.cpp c1_LIRAssembler.hpp
-c1_LIRGenerator.cpp c1_LIRGenerator.hpp
-c1_LIRGenerator.cpp c1_ValueStack.hpp
-c1_LIRGenerator.cpp ciArrayKlass.hpp
-c1_LIRGenerator.cpp ciInstance.hpp
-c1_LIRGenerator.cpp sharedRuntime.hpp
-
-c1_LIRGenerator.hpp c1_Instruction.hpp
-c1_LIRGenerator.hpp c1_LIR.hpp
-c1_LIRGenerator.hpp ciMethodData.hpp
-c1_LIRGenerator.hpp sizes.hpp
+c1_LIRGenerator.cpp bitMap.inline.hpp
+c1_LIRGenerator.cpp c1_Compilation.hpp
+c1_LIRGenerator.cpp c1_FrameMap.hpp
+c1_LIRGenerator.cpp c1_Instruction.hpp
+c1_LIRGenerator.cpp c1_LIRAssembler.hpp
+c1_LIRGenerator.cpp c1_LIRGenerator.hpp
+c1_LIRGenerator.cpp c1_ValueStack.hpp
+c1_LIRGenerator.cpp ciArrayKlass.hpp
+c1_LIRGenerator.cpp ciInstance.hpp
+c1_LIRGenerator.cpp heapRegion.hpp
+c1_LIRGenerator.cpp sharedRuntime.hpp
+
+c1_LIRGenerator.hpp c1_Instruction.hpp
+c1_LIRGenerator.hpp c1_LIR.hpp
+c1_LIRGenerator.hpp ciMethodData.hpp
+c1_LIRGenerator.hpp sizes.hpp
c1_LIRGenerator_<arch>.cpp c1_Compilation.hpp
c1_LIRGenerator_<arch>.cpp c1_FrameMap.hpp
@@ -260,6 +268,7 @@ c1_LIRGenerator_<arch>.cpp s
c1_LIRGenerator_<arch>.cpp sharedRuntime.hpp
c1_LIRGenerator_<arch>.cpp vmreg_<arch>.inline.hpp
+c1_LinearScan.cpp bitMap.inline.hpp
c1_LinearScan.cpp c1_CFGPrinter.hpp
c1_LinearScan.cpp c1_Compilation.hpp
c1_LinearScan.cpp c1_FrameMap.hpp
@@ -276,6 +285,7 @@ c1_LinearScan.hpp
c1_LinearScan.hpp c1_LIR.hpp
c1_LinearScan.hpp c1_LIRGenerator.hpp
+c1_LinearScan_<arch>.cpp bitMap.inline.hpp
c1_LinearScan_<arch>.cpp c1_Instruction.hpp
c1_LinearScan_<arch>.cpp c1_LinearScan.hpp
@@ -298,6 +308,7 @@ c1_MacroAssembler_<arch>.cpp
c1_MacroAssembler_<arch>.hpp generate_platform_dependent_include
+c1_Optimizer.cpp bitMap.inline.hpp
c1_Optimizer.cpp c1_Canonicalizer.hpp
c1_Optimizer.cpp c1_Optimizer.hpp
c1_Optimizer.cpp c1_ValueMap.hpp
@@ -363,6 +374,7 @@ c1_Runtime1_<arch>.cpp
c1_Runtime1_<arch>.cpp vframeArray.hpp
c1_Runtime1_<arch>.cpp vmreg_<arch>.inline.hpp
+c1_ValueMap.cpp bitMap.inline.hpp
c1_ValueMap.cpp c1_Canonicalizer.hpp
c1_ValueMap.cpp c1_IR.hpp
c1_ValueMap.cpp c1_ValueMap.hpp
@@ -433,4 +445,3 @@ top.hpp
top.hpp c1_globals.hpp
vmStructs.hpp c1_Runtime1.hpp
-
--- a/src/share/vm/includeDB_compiler2 Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/includeDB_compiler2 Tue Sep 30 12:24:27 2008 -0400
@@ -461,10 +461,13 @@ graphKit.cpp
graphKit.cpp addnode.hpp
graphKit.cpp barrierSet.hpp
graphKit.cpp cardTableModRefBS.hpp
+graphKit.cpp g1SATBCardTableModRefBS.hpp
graphKit.cpp collectedHeap.hpp
graphKit.cpp compileLog.hpp
graphKit.cpp deoptimization.hpp
graphKit.cpp graphKit.hpp
+graphKit.cpp heapRegion.hpp
+graphKit.cpp idealKit.hpp
graphKit.cpp locknode.hpp
graphKit.cpp machnode.hpp
graphKit.cpp parse.hpp
@@ -484,6 +487,7 @@ idealKit.cpp
idealKit.cpp callnode.hpp
idealKit.cpp cfgnode.hpp
idealKit.cpp idealKit.hpp
+idealKit.cpp runtime.hpp
idealKit.hpp connode.hpp
idealKit.hpp mulnode.hpp
@@ -915,9 +919,11 @@ runtime.cpp
runtime.cpp connode.hpp
runtime.cpp copy.hpp
runtime.cpp fprofiler.hpp
+runtime.cpp g1SATBCardTableModRefBS.hpp
runtime.cpp gcLocker.inline.hpp
runtime.cpp graphKit.hpp
runtime.cpp handles.inline.hpp
+runtime.cpp heapRegion.hpp
runtime.cpp icBuffer.hpp
runtime.cpp interfaceSupport.hpp
runtime.cpp interpreter.hpp
--- a/src/share/vm/includeDB_core Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/includeDB_core Tue Sep 30 12:24:27 2008 -0400
@@ -288,6 +288,10 @@ attachListener.hpp
attachListener.hpp debug.hpp
attachListener.hpp ostream.hpp
+barrierSet.cpp barrierSet.hpp
+barrierSet.cpp collectedHeap.hpp
+barrierSet.cpp universe.hpp
+
barrierSet.hpp memRegion.hpp
barrierSet.hpp oopsHierarchy.hpp
@@ -295,7 +299,7 @@ barrierSet.inline.hpp
barrierSet.inline.hpp cardTableModRefBS.hpp
bcEscapeAnalyzer.cpp bcEscapeAnalyzer.hpp
-bcEscapeAnalyzer.cpp bitMap.hpp
+bcEscapeAnalyzer.cpp bitMap.inline.hpp
bcEscapeAnalyzer.cpp bytecode.hpp
bcEscapeAnalyzer.cpp ciConstant.hpp
bcEscapeAnalyzer.cpp ciField.hpp
@@ -320,13 +324,12 @@ biasedLocking.hpp
biasedLocking.hpp growableArray.hpp
biasedLocking.hpp handles.hpp
-bitMap.cpp bitMap.hpp
+bitMap.cpp allocation.inline.hpp
bitMap.cpp bitMap.inline.hpp
bitMap.cpp copy.hpp
bitMap.cpp os_<os_family>.inline.hpp
bitMap.hpp allocation.hpp
-bitMap.hpp ostream.hpp
bitMap.hpp top.hpp
bitMap.inline.hpp atomic.hpp
@@ -645,6 +648,7 @@ ciMethod.cpp
ciMethod.cpp abstractCompiler.hpp
ciMethod.cpp allocation.inline.hpp
ciMethod.cpp bcEscapeAnalyzer.hpp
+ciMethod.cpp bitMap.inline.hpp
ciMethod.cpp ciCallProfile.hpp
ciMethod.cpp ciExceptionHandler.hpp
ciMethod.cpp ciInstanceKlass.hpp
@@ -1759,7 +1763,7 @@ genRemSet.cpp
genRemSet.hpp oop.hpp
-generateOopMap.cpp bitMap.hpp
+generateOopMap.cpp bitMap.inline.hpp
generateOopMap.cpp bytecodeStream.hpp
generateOopMap.cpp generateOopMap.hpp
generateOopMap.cpp handles.inline.hpp
@@ -1807,6 +1811,8 @@ generation.inline.hpp
generation.inline.hpp genCollectedHeap.hpp
generation.inline.hpp generation.hpp
generation.inline.hpp space.hpp
+
+genOopClosures.hpp oop.hpp
generationSpec.cpp compactPermGen.hpp
generationSpec.cpp defNewGeneration.hpp
@@ -2218,6 +2224,11 @@ invocationCounter.hpp
invocationCounter.hpp allocation.hpp
invocationCounter.hpp exceptions.hpp
invocationCounter.hpp handles.hpp
+
+intHisto.cpp intHisto.hpp
+
+intHisto.hpp allocation.hpp
+intHisto.hpp growableArray.hpp
iterator.cpp iterator.hpp
iterator.cpp oop.inline.hpp
@@ -2818,6 +2829,7 @@ methodKlass.hpp
methodKlass.hpp methodOop.hpp
methodLiveness.cpp allocation.inline.hpp
+methodLiveness.cpp bitMap.inline.hpp
methodLiveness.cpp bytecode.hpp
methodLiveness.cpp bytecodes.hpp
methodLiveness.cpp ciMethod.hpp
@@ -2963,6 +2975,11 @@ nmethod.cpp
nmethod.hpp codeBlob.hpp
nmethod.hpp pcDesc.hpp
+
+numberSeq.cpp debug.hpp
+numberSeq.cpp numberSeq.hpp
+numberSeq.cpp globalDefinitions.hpp
+numberSeq.cpp allocation.inline.hpp
objArrayKlass.cpp collectedHeap.inline.hpp
objArrayKlass.cpp copy.hpp
@@ -3406,8 +3423,6 @@ referencePolicy.cpp
referencePolicy.cpp referencePolicy.hpp
referencePolicy.cpp universe.hpp
-referencePolicy.hpp oop.hpp
-
referenceProcessor.cpp collectedHeap.hpp
referenceProcessor.cpp collectedHeap.inline.hpp
referenceProcessor.cpp java.hpp
@@ -3758,6 +3773,8 @@ specialized_oop_closures.cpp
specialized_oop_closures.cpp ostream.hpp
specialized_oop_closures.cpp specialized_oop_closures.hpp
+specialized_oop_closures.hpp atomic.hpp
+
stackMapFrame.cpp globalDefinitions.hpp
stackMapFrame.cpp handles.inline.hpp
stackMapFrame.cpp oop.inline.hpp
@@ -4000,7 +4017,6 @@ taskqueue.cpp
taskqueue.hpp allocation.hpp
taskqueue.hpp allocation.inline.hpp
-taskqueue.hpp debug.hpp
taskqueue.hpp mutex.hpp
taskqueue.hpp orderAccess_<os_arch>.inline.hpp
@@ -4038,6 +4054,7 @@ templateInterpreter_<arch>.hpp
templateInterpreterGenerator_<arch>.hpp generate_platform_dependent_include
+templateTable.cpp collectedHeap.hpp
templateTable.cpp templateTable.hpp
templateTable.cpp timer.hpp
@@ -4542,6 +4559,7 @@ vm_operations.cpp
vm_operations.cpp compilerOracle.hpp
vm_operations.cpp deoptimization.hpp
vm_operations.cpp interfaceSupport.hpp
+vm_operations.cpp isGCActiveMark.hpp
vm_operations.cpp resourceArea.hpp
vm_operations.cpp threadService.hpp
vm_operations.cpp thread_<os_family>.inline.hpp
--- a/src/share/vm/includeDB_gc_parallel Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/includeDB_gc_parallel Tue Sep 30 12:24:27 2008 -0400
@@ -21,6 +21,10 @@
// have any questions.
//
+assembler_<arch>.cpp g1SATBCardTableModRefBS.hpp
+assembler_<arch>.cpp g1CollectedHeap.inline.hpp
+assembler_<arch>.cpp heapRegion.hpp
+
collectorPolicy.cpp cmsAdaptiveSizePolicy.hpp
collectorPolicy.cpp cmsGCAdaptivePolicyCounters.hpp
@@ -37,6 +41,9 @@ heapDumper.cpp
heapInspection.cpp parallelScavengeHeap.hpp
+instanceKlass.cpp heapRegionSeq.inline.hpp
+instanceKlass.cpp g1CollectedHeap.inline.hpp
+instanceKlass.cpp g1OopClosures.inline.hpp
instanceKlass.cpp oop.pcgc.inline.hpp
instanceKlass.cpp psPromotionManager.inline.hpp
instanceKlass.cpp psScavenge.inline.hpp
@@ -48,6 +55,9 @@ instanceKlassKlass.cpp
instanceKlassKlass.cpp psScavenge.inline.hpp
instanceKlassKlass.cpp parOopClosures.inline.hpp
+instanceRefKlass.cpp heapRegionSeq.inline.hpp
+instanceRefKlass.cpp g1CollectedHeap.inline.hpp
+instanceRefKlass.cpp g1OopClosures.inline.hpp
instanceRefKlass.cpp oop.pcgc.inline.hpp
instanceRefKlass.cpp psPromotionManager.inline.hpp
instanceRefKlass.cpp psScavenge.inline.hpp
@@ -70,6 +80,7 @@ memoryPool.hpp
memoryService.cpp cmsPermGen.hpp
memoryService.cpp concurrentMarkSweepGeneration.hpp
+memoryService.cpp g1CollectedHeap.inline.hpp
memoryService.cpp parNewGeneration.hpp
memoryService.cpp parallelScavengeHeap.hpp
memoryService.cpp psMemoryPool.hpp
@@ -80,6 +91,9 @@ methodDataKlass.cpp
methodDataKlass.cpp oop.pcgc.inline.hpp
methodDataKlass.cpp psScavenge.inline.hpp
+objArrayKlass.cpp heapRegionSeq.inline.hpp
+objArrayKlass.cpp g1CollectedHeap.inline.hpp
+objArrayKlass.cpp g1OopClosures.inline.hpp
objArrayKlass.cpp oop.pcgc.inline.hpp
objArrayKlass.cpp psPromotionManager.inline.hpp
objArrayKlass.cpp psScavenge.inline.hpp
@@ -122,6 +136,9 @@ thread.cpp
thread.cpp concurrentMarkSweepThread.hpp
thread.cpp pcTasks.hpp
+thread.hpp dirtyCardQueue.hpp
+thread.hpp satbQueue.hpp
+
universe.cpp parallelScavengeHeap.hpp
universe.cpp cmsCollectorPolicy.hpp
universe.cpp cmsAdaptiveSizePolicy.hpp
--- a/src/share/vm/includeDB_jvmti Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/includeDB_jvmti Tue Sep 30 12:24:27 2008 -0400
@@ -209,6 +209,7 @@ jvmtiManageCapabilities.hpp
jvmtiManageCapabilities.hpp allocation.hpp
jvmtiManageCapabilities.hpp jvmti.h
+jvmtiRedefineClasses.cpp bitMap.inline.hpp
jvmtiRedefineClasses.cpp codeCache.hpp
jvmtiRedefineClasses.cpp deoptimization.hpp
jvmtiRedefineClasses.cpp gcLocker.hpp
--- a/src/share/vm/interpreter/templateTable.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/interpreter/templateTable.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -172,6 +172,7 @@ Template TemplateTable
Template* TemplateTable::_desc;
InterpreterMacroAssembler* TemplateTable::_masm;
+BarrierSet* TemplateTable::_bs;
void TemplateTable::def(Bytecodes::Code code, int flags, TosState in, TosState out, void (*gen)(), char filler) {
@@ -243,6 +244,8 @@ void TemplateTable::initialize() {
// Initialize table
TraceTime timer("TemplateTable initialization", TraceStartupTime);
+
+ _bs = Universe::heap()->barrier_set();
// For better readability
const char _ = ' ';
--- a/src/share/vm/interpreter/templateTable.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/interpreter/templateTable.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -82,6 +82,7 @@ class TemplateTable: AllStatic {
static Template* _desc; // the current template to be generated
static Bytecodes::Code bytecode() { return _desc->bytecode(); }
+ static BarrierSet* _bs; // Cache the barrier set.
public:
//%note templates_1
static InterpreterMacroAssembler* _masm; // the assembler used when generating templates
--- a/src/share/vm/memory/allocation.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/allocation.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -338,6 +338,12 @@ class ResourceObj ALLOCATION_SUPER_CLASS
DEBUG_ONLY(((ResourceObj *)res)->_allocation = RESOURCE_AREA;)
return res;
}
+ void* operator new(size_t size, void* where, allocation_type type) {
+ void* res = where;
+ // Set allocation type in the resource object
+ DEBUG_ONLY(((ResourceObj *)res)->_allocation = type;)
+ return res;
+ }
void operator delete(void* p);
};
--- a/src/share/vm/memory/barrierSet.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/barrierSet.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -32,6 +32,8 @@ public:
ModRef,
CardTableModRef,
CardTableExtension,
+ G1SATBCT,
+ G1SATBCTLogging,
Other,
Uninit
};
@@ -42,14 +44,16 @@ protected:
public:
+ BarrierSet() { _kind = Uninit; }
// To get around prohibition on RTTI.
- virtual BarrierSet::Name kind() { return _kind; }
+ BarrierSet::Name kind() { return _kind; }
virtual bool is_a(BarrierSet::Name bsn) = 0;
// These operations indicate what kind of barriers the BarrierSet has.
virtual bool has_read_ref_barrier() = 0;
virtual bool has_read_prim_barrier() = 0;
virtual bool has_write_ref_barrier() = 0;
+ virtual bool has_write_ref_pre_barrier() = 0;
virtual bool has_write_prim_barrier() = 0;
// These functions indicate whether a particular access of the given
@@ -57,7 +61,8 @@ public:
virtual bool read_ref_needs_barrier(void* field) = 0;
virtual bool read_prim_needs_barrier(HeapWord* field, size_t bytes) = 0;
virtual bool write_ref_needs_barrier(void* field, oop new_val) = 0;
- virtual bool write_prim_needs_barrier(HeapWord* field, size_t bytes, juint val1, juint val2) = 0;
+ virtual bool write_prim_needs_barrier(HeapWord* field, size_t bytes,
+ juint val1, juint val2) = 0;
// The first four operations provide a direct implementation of the
// barrier set. An interpreter loop, for example, could call these
@@ -75,6 +80,13 @@ public:
// (For efficiency reasons, this operation is specialized for certain
// barrier types. Semantically, it should be thought of as a call to the
// virtual "_work" function below, which must implement the barrier.)
+ // First the pre-write versions...
+ inline void write_ref_field_pre(void* field, oop new_val);
+protected:
+ virtual void write_ref_field_pre_work(void* field, oop new_val) {};
+public:
+
+ // ...then the post-write version.
inline void write_ref_field(void* field, oop new_val);
protected:
virtual void write_ref_field_work(void* field, oop new_val) = 0;
@@ -92,6 +104,7 @@ public:
// the particular barrier.
virtual bool has_read_ref_array_opt() = 0;
virtual bool has_read_prim_array_opt() = 0;
+ virtual bool has_write_ref_array_pre_opt() { return true; }
virtual bool has_write_ref_array_opt() = 0;
virtual bool has_write_prim_array_opt() = 0;
@@ -104,7 +117,13 @@ public:
virtual void read_ref_array(MemRegion mr) = 0;
virtual void read_prim_array(MemRegion mr) = 0;
+ virtual void write_ref_array_pre(MemRegion mr) {}
inline void write_ref_array(MemRegion mr);
+
+ // Static versions, suitable for calling from generated code.
+ static void static_write_ref_array_pre(HeapWord* start, size_t count);
+ static void static_write_ref_array_post(HeapWord* start, size_t count);
+
protected:
virtual void write_ref_array_work(MemRegion mr) = 0;
public:
@@ -119,33 +138,6 @@ protected:
protected:
virtual void write_region_work(MemRegion mr) = 0;
public:
-
- // The remaining sets of operations are called by compilers or other code
- // generators to insert barriers into generated code. There may be
- // several such code generators; the signatures of these
- // barrier-generating functions may differ from generator to generator.
- // There will be a set of four function signatures for each code
- // generator, which accomplish the generation of barriers of the four
- // kinds listed above.
-
-#ifdef TBD
- // Generates code to invoke the barrier, if any, necessary when reading
- // the ref field at "offset" in "obj".
- virtual void gen_read_ref_field() = 0;
-
- // Generates code to invoke the barrier, if any, necessary when reading
- // the primitive field of "bytes" bytes at offset" in "obj".
- virtual void gen_read_prim_field() = 0;
-
- // Generates code to invoke the barrier, if any, necessary when writing
- // "new_val" into the ref field at "offset" in "obj".
- virtual void gen_write_ref_field() = 0;
-
- // Generates code to invoke the barrier, if any, necessary when writing
- // the "bytes"-byte value "new_val" into the primitive field at "offset"
- // in "obj".
- virtual void gen_write_prim_field() = 0;
-#endif
// Some barrier sets create tables whose elements correspond to parts of
// the heap; the CardTableModRefBS is an example. Such barrier sets will
--- a/src/share/vm/memory/barrierSet.inline.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/barrierSet.inline.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -26,6 +26,14 @@
// performance-critical calls when when the barrier is the most common
// card-table kind.
+void BarrierSet::write_ref_field_pre(void* field, oop new_val) {
+ if (kind() == CardTableModRef) {
+ ((CardTableModRefBS*)this)->inline_write_ref_field_pre(field, new_val);
+ } else {
+ write_ref_field_pre_work(field, new_val);
+ }
+}
+
void BarrierSet::write_ref_field(void* field, oop new_val) {
if (kind() == CardTableModRef) {
((CardTableModRefBS*)this)->inline_write_ref_field(field, new_val);
--- a/src/share/vm/memory/blockOffsetTable.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/blockOffsetTable.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -184,7 +184,7 @@ BlockOffsetArray::set_remainder_to_point
"Offset card has an unexpected value");
size_t start_card_for_region = start_card;
u_char offset = max_jubyte;
- for (int i = 0; i <= N_powers-1; i++) {
+ for (int i = 0; i < N_powers; i++) {
// -1 so that the the card with the actual offset is counted. Another -1
// so that the reach ends in this region and not at the start
// of the next.
--- a/src/share/vm/memory/blockOffsetTable.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/blockOffsetTable.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -214,6 +214,7 @@ public:
//////////////////////////////////////////////////////////////////////////
class BlockOffsetArray: public BlockOffsetTable {
friend class VMStructs;
+ friend class G1BlockOffsetArray; // temp. until we restructure and cleanup
protected:
// The following enums are used by do_block_internal() below
enum Action {
--- a/src/share/vm/memory/cardTableModRefBS.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/cardTableModRefBS.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -343,6 +343,17 @@ void CardTableModRefBS::write_ref_field_
inline_write_ref_field(field, newVal);
}
+
+bool CardTableModRefBS::claim_card(size_t card_index) {
+ jbyte val = _byte_map[card_index];
+ if (val != claimed_card_val()) {
+ jbyte res = Atomic::cmpxchg((jbyte) claimed_card_val(), &_byte_map[card_index], val);
+ if (res == val)
+ return true;
+ else return false;
+ }
+ return false;
+}
void CardTableModRefBS::non_clean_card_iterate(Space* sp,
MemRegion mr,
@@ -443,7 +454,7 @@ void CardTableModRefBS::dirty_MemRegion(
}
}
-void CardTableModRefBS::invalidate(MemRegion mr) {
+void CardTableModRefBS::invalidate(MemRegion mr, bool whole_heap) {
for (int i = 0; i < _cur_covered_regions; i++) {
MemRegion mri = mr.intersection(_covered[i]);
if (!mri.is_empty()) dirty_MemRegion(mri);
@@ -471,11 +482,15 @@ void CardTableModRefBS::clear(MemRegion
}
}
+void CardTableModRefBS::dirty(MemRegion mr) {
+ jbyte* first = byte_for(mr.start());
+ jbyte* last = byte_after(mr.last());
+ memset(first, dirty_card, last-first);
+}
+
// NOTES:
// (1) Unlike mod_oop_in_space_iterate() above, dirty_card_iterate()
// iterates over dirty cards ranges in increasing address order.
-// (2) Unlike, e.g., dirty_card_range_after_preclean() below,
-// this method does not make the dirty cards prelceaned.
void CardTableModRefBS::dirty_card_iterate(MemRegion mr,
MemRegionClosure* cl) {
for (int i = 0; i < _cur_covered_regions; i++) {
@@ -501,7 +516,9 @@ void CardTableModRefBS::dirty_card_itera
}
}
-MemRegion CardTableModRefBS::dirty_card_range_after_preclean(MemRegion mr) {
+MemRegion CardTableModRefBS::dirty_card_range_after_reset(MemRegion mr,
+ bool reset,
+ int reset_val) {
for (int i = 0; i < _cur_covered_regions; i++) {
MemRegion mri = mr.intersection(_covered[i]);
if (!mri.is_empty()) {
@@ -518,8 +535,10 @@ MemRegion CardTableModRefBS::dirty_card_
dirty_cards++, next_entry++);
MemRegion cur_cards(addr_for(cur_entry),
dirty_cards*card_size_in_words);
- for (size_t i = 0; i < dirty_cards; i++) {
- cur_entry[i] = precleaned_card;
+ if (reset) {
+ for (size_t i = 0; i < dirty_cards; i++) {
+ cur_entry[i] = reset_val;
+ }
}
return cur_cards;
}
--- a/src/share/vm/memory/cardTableModRefBS.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/cardTableModRefBS.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -54,6 +54,7 @@ class CardTableModRefBS: public ModRefBa
clean_card = -1,
dirty_card = 0,
precleaned_card = 1,
+ claimed_card = 3,
last_card = 4,
CT_MR_BS_last_reserved = 10
};
@@ -148,17 +149,6 @@ class CardTableModRefBS: public ModRefBa
// for loops iterating through the card table.
jbyte* byte_after(const void* p) const {
return byte_for(p) + 1;
- }
-
- // Mapping from card marking array entry to address of first word
- HeapWord* addr_for(const jbyte* p) const {
- assert(p >= _byte_map && p < _byte_map + _byte_map_size,
- "out of bounds access to card marking array");
- size_t delta = pointer_delta(p, byte_map_base, sizeof(jbyte));
- HeapWord* result = (HeapWord*) (delta << card_shift);
- assert(_whole_heap.contains(result),
- "out of bounds accessor from card marking array");
- return result;
}
// Iterate over the portion of the card-table which covers the given
@@ -263,15 +253,21 @@ public:
card_size_in_words = card_size / sizeof(HeapWord)
};
+ static int clean_card_val() { return clean_card; }
+ static int dirty_card_val() { return dirty_card; }
+ static int claimed_card_val() { return claimed_card; }
+ static int precleaned_card_val() { return precleaned_card; }
+
// For RTTI simulation.
- BarrierSet::Name kind() { return BarrierSet::CardTableModRef; }
bool is_a(BarrierSet::Name bsn) {
- return bsn == BarrierSet::CardTableModRef || bsn == BarrierSet::ModRef;
+ return bsn == BarrierSet::CardTableModRef || ModRefBarrierSet::is_a(bsn);
}
CardTableModRefBS(MemRegion whole_heap, int max_covered_regions);
// *** Barrier set functions.
+
+ bool has_write_ref_pre_barrier() { return false; }
inline bool write_ref_needs_barrier(void* field, oop new_val) {
// Note that this assumes the perm gen is the highest generation
@@ -315,9 +311,31 @@ public:
// *** Card-table-barrier-specific things.
+ inline void inline_write_ref_field_pre(void* field, oop newVal) {}
+
inline void inline_write_ref_field(void* field, oop newVal) {
jbyte* byte = byte_for(field);
*byte = dirty_card;
+ }
+
+ // These are used by G1, when it uses the card table as a temporary data
+ // structure for card claiming.
+ bool is_card_dirty(size_t card_index) {
+ return _byte_map[card_index] == dirty_card_val();
+ }
+
+ void mark_card_dirty(size_t card_index) {
+ _byte_map[card_index] = dirty_card_val();
+ }
+
+ bool is_card_claimed(size_t card_index) {
+ return _byte_map[card_index] == claimed_card_val();
+ }
+
+ bool claim_card(size_t card_index);
+
+ bool is_card_clean(size_t card_index) {
+ return _byte_map[card_index] == clean_card_val();
}
// Card marking array base (adjusted for heap low boundary)
@@ -344,8 +362,9 @@ public:
}
// ModRefBS functions.
- void invalidate(MemRegion mr);
+ virtual void invalidate(MemRegion mr, bool whole_heap = false);
void clear(MemRegion mr);
+ void dirty(MemRegion mr);
void mod_oop_in_space_iterate(Space* sp, OopClosure* cl,
bool clear = false,
bool before_save_marks = false);
@@ -375,17 +394,38 @@ public:
static uintx ct_max_alignment_constraint();
- // Apply closure cl to the dirty cards lying completely
- // within MemRegion mr, setting the cards to precleaned.
- void dirty_card_iterate(MemRegion mr, MemRegionClosure* cl);
+ // Apply closure "cl" to the dirty cards containing some part of
+ // MemRegion "mr".
+ void dirty_card_iterate(MemRegion mr, MemRegionClosure* cl);
// Return the MemRegion corresponding to the first maximal run
- // of dirty cards lying completely within MemRegion mr, after
- // marking those cards precleaned.
- MemRegion dirty_card_range_after_preclean(MemRegion mr);
+ // of dirty cards lying completely within MemRegion mr.
+ // If reset is "true", then sets those card table entries to the given
+ // value.
+ MemRegion dirty_card_range_after_reset(MemRegion mr, bool reset,
+ int reset_val);
// Set all the dirty cards in the given region to precleaned state.
void preclean_dirty_cards(MemRegion mr);
+
+ // Provide read-only access to the card table array.
+ const jbyte* byte_for_const(const void* p) const {
+ return byte_for(p);
+ }
+ const jbyte* byte_after_const(const void* p) const {
+ return byte_after(p);
+ }
+
+ // Mapping from card marking array entry to address of first word
+ HeapWord* addr_for(const jbyte* p) const {
+ assert(p >= _byte_map && p < _byte_map + _byte_map_size,
+ "out of bounds access to card marking array");
+ size_t delta = pointer_delta(p, byte_map_base, sizeof(jbyte));
+ HeapWord* result = (HeapWord*) (delta << card_shift);
+ assert(_whole_heap.contains(result),
+ "out of bounds accessor from card marking array");
+ return result;
+ }
// Mapping from address to card marking array index.
int index_for(void* p) {
@@ -402,6 +442,7 @@ public:
static size_t par_chunk_heapword_alignment() {
return CardsPerStrideChunk * card_size_in_words;
}
+
};
class CardTableRS;
--- a/src/share/vm/memory/cardTableRS.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/cardTableRS.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -27,10 +27,25 @@
CardTableRS::CardTableRS(MemRegion whole_heap,
int max_covered_regions) :
- GenRemSet(&_ct_bs),
- _ct_bs(whole_heap, max_covered_regions),
- _cur_youngergen_card_val(youngergenP1_card)
+ GenRemSet(),
+ _cur_youngergen_card_val(youngergenP1_card),
+ _regions_to_iterate(max_covered_regions - 1)
{
+#ifndef SERIALGC
+ if (UseG1GC) {
+ if (G1RSBarrierUseQueue) {
+ _ct_bs = new G1SATBCardTableLoggingModRefBS(whole_heap,
+ max_covered_regions);
+ } else {
+ _ct_bs = new G1SATBCardTableModRefBS(whole_heap, max_covered_regions);
+ }
+ } else {
+ _ct_bs = new CardTableModRefBSForCTRS(whole_heap, max_covered_regions);
+ }
+#else
+ _ct_bs = new CardTableModRefBSForCTRS(whole_heap, max_covered_regions);
+#endif
+ set_bs(_ct_bs);
_last_cur_val_in_gen = new jbyte[GenCollectedHeap::max_gens + 1];
if (_last_cur_val_in_gen == NULL) {
vm_exit_during_initialization("Could not last_cur_val_in_gen array.");
@@ -38,20 +53,19 @@ CardTableRS::CardTableRS(MemRegion whole
for (int i = 0; i < GenCollectedHeap::max_gens + 1; i++) {
_last_cur_val_in_gen[i] = clean_card_val();
}
- _ct_bs.set_CTRS(this);
+ _ct_bs->set_CTRS(this);
}
void CardTableRS::resize_covered_region(MemRegion new_region) {
- _ct_bs.resize_covered_region(new_region);
+ _ct_bs->resize_covered_region(new_region);
}
jbyte CardTableRS::find_unused_youngergenP_card_value() {
- GenCollectedHeap* gch = GenCollectedHeap::heap();
for (jbyte v = youngergenP1_card;
v < cur_youngergen_and_prev_nonclean_card;
v++) {
bool seen = false;
- for (int g = 0; g < gch->n_gens()+1; g++) {
+ for (int g = 0; g < _regions_to_iterate; g++) {
if (_last_cur_val_in_gen[g] == v) {
seen = true;
break;
@@ -221,11 +235,11 @@ void CardTableRS::write_ref_field_gc_par
void CardTableRS::younger_refs_in_space_iterate(Space* sp,
OopsInGenClosure* cl) {
- DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, _ct_bs.precision(),
+ DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, _ct_bs->precision(),
cl->gen_boundary());
ClearNoncleanCardWrapper clear_cl(dcto_cl, this);
- _ct_bs.non_clean_card_iterate(sp, sp->used_region_at_save_marks(),
+ _ct_bs->non_clean_card_iterate(sp, sp->used_region_at_save_marks(),
dcto_cl, &clear_cl, false);
}
@@ -549,7 +563,7 @@ void CardTableRS::verify() {
if (ch->kind() == CollectedHeap::GenCollectedHeap) {
GenCollectedHeap::heap()->generation_iterate(&blk, false);
- _ct_bs.verify();
+ _ct_bs->verify();
// If the old gen collections also collect perm, then we are only
// interested in perm-to-young pointers, not perm-to-old pointers.
--- a/src/share/vm/memory/cardTableRS.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/cardTableRS.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -44,7 +44,7 @@ class CardTableRS: public GenRemSet {
return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv);
}
- CardTableModRefBSForCTRS _ct_bs;
+ CardTableModRefBSForCTRS* _ct_bs;
virtual void younger_refs_in_space_iterate(Space* sp, OopsInGenClosure* cl);
@@ -73,6 +73,8 @@ class CardTableRS: public GenRemSet {
jbyte _cur_youngergen_card_val;
+ int _regions_to_iterate;
+
jbyte cur_youngergen_card_val() {
return _cur_youngergen_card_val;
}
@@ -96,7 +98,7 @@ public:
CardTableRS* as_CardTableRS() { return this; }
- CardTableModRefBS* ct_bs() { return &_ct_bs; }
+ CardTableModRefBS* ct_bs() { return _ct_bs; }
// Override.
void prepare_for_younger_refs_iterate(bool parallel);
@@ -107,7 +109,7 @@ public:
void younger_refs_iterate(Generation* g, OopsInGenClosure* blk);
void inline_write_ref_field_gc(void* field, oop new_val) {
- jbyte* byte = _ct_bs.byte_for(field);
+ jbyte* byte = _ct_bs->byte_for(field);
*byte = youngergen_card;
}
void write_ref_field_gc_work(void* field, oop new_val) {
@@ -122,25 +124,27 @@ public:
void resize_covered_region(MemRegion new_region);
bool is_aligned(HeapWord* addr) {
- return _ct_bs.is_card_aligned(addr);
+ return _ct_bs->is_card_aligned(addr);
}
void verify();
void verify_aligned_region_empty(MemRegion mr);
- void clear(MemRegion mr) { _ct_bs.clear(mr); }
+ void clear(MemRegion mr) { _ct_bs->clear(mr); }
void clear_into_younger(Generation* gen, bool clear_perm);
- void invalidate(MemRegion mr) { _ct_bs.invalidate(mr); }
+ void invalidate(MemRegion mr, bool whole_heap = false) {
+ _ct_bs->invalidate(mr, whole_heap);
+ }
void invalidate_or_clear(Generation* gen, bool younger, bool perm);
static uintx ct_max_alignment_constraint() {
return CardTableModRefBS::ct_max_alignment_constraint();
}
- jbyte* byte_for(void* p) { return _ct_bs.byte_for(p); }
- jbyte* byte_after(void* p) { return _ct_bs.byte_after(p); }
- HeapWord* addr_for(jbyte* p) { return _ct_bs.addr_for(p); }
+ jbyte* byte_for(void* p) { return _ct_bs->byte_for(p); }
+ jbyte* byte_after(void* p) { return _ct_bs->byte_after(p); }
+ HeapWord* addr_for(jbyte* p) { return _ct_bs->addr_for(p); }
bool is_prev_nonclean_card_val(jbyte v) {
return
--- a/src/share/vm/memory/collectorPolicy.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/collectorPolicy.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -31,11 +31,11 @@ void CollectorPolicy::initialize_flags()
if (PermSize > MaxPermSize) {
MaxPermSize = PermSize;
}
- PermSize = align_size_down(PermSize, min_alignment());
+ PermSize = MAX2(min_alignment(), align_size_down_(PermSize, min_alignment()));
MaxPermSize = align_size_up(MaxPermSize, max_alignment());
- MinPermHeapExpansion = align_size_down(MinPermHeapExpansion, min_alignment());
- MaxPermHeapExpansion = align_size_down(MaxPermHeapExpansion, min_alignment());
+ MinPermHeapExpansion = MAX2(min_alignment(), align_size_down_(MinPermHeapExpansion, min_alignment()));
+ MaxPermHeapExpansion = MAX2(min_alignment(), align_size_down_(MaxPermHeapExpansion, min_alignment()));
MinHeapDeltaBytes = align_size_up(MinHeapDeltaBytes, min_alignment());
@@ -55,25 +55,21 @@ void CollectorPolicy::initialize_flags()
void CollectorPolicy::initialize_size_info() {
// User inputs from -mx and ms are aligned
- _initial_heap_byte_size = align_size_up(Arguments::initial_heap_size(),
- min_alignment());
- set_min_heap_byte_size(align_size_up(Arguments::min_heap_size(),
- min_alignment()));
- set_max_heap_byte_size(align_size_up(MaxHeapSize, max_alignment()));
-
- // Check validity of heap parameters from launcher
+ set_initial_heap_byte_size(Arguments::initial_heap_size());
if (initial_heap_byte_size() == 0) {
set_initial_heap_byte_size(NewSize + OldSize);
- } else {
- Universe::check_alignment(initial_heap_byte_size(), min_alignment(),
- "initial heap");
- }
+ }
+ set_initial_heap_byte_size(align_size_up(_initial_heap_byte_size,
+ min_alignment()));
+
+ set_min_heap_byte_size(Arguments::min_heap_size());
if (min_heap_byte_size() == 0) {
set_min_heap_byte_size(NewSize + OldSize);
- } else {
- Universe::check_alignment(min_heap_byte_size(), min_alignment(),
- "initial heap");
- }
+ }
+ set_min_heap_byte_size(align_size_up(_min_heap_byte_size,
+ min_alignment()));
+
+ set_max_heap_byte_size(align_size_up(MaxHeapSize, max_alignment()));
// Check heap parameter properties
if (initial_heap_byte_size() < M) {
@@ -121,8 +117,6 @@ GenRemSet* CollectorPolicy::create_rem_s
int max_covered_regions) {
switch (rem_set_name()) {
case GenRemSet::CardTable: {
- if (barrier_set_name() != BarrierSet::CardTableModRef)
- vm_exit_during_initialization("Mismatch between RS and BS.");
CardTableRS* res = new CardTableRS(whole_heap, max_covered_regions);
return res;
}
@@ -345,7 +339,7 @@ void GenCollectorPolicy::initialize_size
// At this point all three sizes have been checked against the
// maximum sizes but have not been checked for consistency
- // amoung the three.
+ // among the three.
// Final check min <= initial <= max
set_min_gen0_size(MIN2(_min_gen0_size, _max_gen0_size));
--- a/src/share/vm/memory/collectorPolicy.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/collectorPolicy.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -39,10 +39,12 @@
// Forward declarations.
class GenCollectorPolicy;
class TwoGenerationCollectorPolicy;
+class AdaptiveSizePolicy;
#ifndef SERIALGC
class ConcurrentMarkSweepPolicy;
-#endif // SERIALGC
-class AdaptiveSizePolicy;
+class G1CollectorPolicy;
+#endif // SERIALGC
+
class GCPolicyCounters;
class PermanentGenerationSpec;
class MarkSweepPolicy;
@@ -55,7 +57,7 @@ class CollectorPolicy : public CHeapObj
// Requires that the concrete subclass sets the alignment constraints
// before calling.
virtual void initialize_flags();
- virtual void initialize_size_info() = 0;
+ virtual void initialize_size_info();
// Initialize "_permanent_generation" to a spec for the given kind of
// Perm Gen.
void initialize_perm_generation(PermGen::Name pgnm);
@@ -91,17 +93,18 @@ class CollectorPolicy : public CHeapObj
enum Name {
CollectorPolicyKind,
TwoGenerationCollectorPolicyKind,
- TrainPolicyKind,
ConcurrentMarkSweepPolicyKind,
- ASConcurrentMarkSweepPolicyKind
+ ASConcurrentMarkSweepPolicyKind,
+ G1CollectorPolicyKind
};
// Identification methods.
- virtual GenCollectorPolicy* as_generation_policy() { return NULL; }
+ virtual GenCollectorPolicy* as_generation_policy() { return NULL; }
virtual TwoGenerationCollectorPolicy* as_two_generation_policy() { return NULL; }
virtual MarkSweepPolicy* as_mark_sweep_policy() { return NULL; }
#ifndef SERIALGC
virtual ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return NULL; }
+ virtual G1CollectorPolicy* as_g1_policy() { return NULL; }
#endif // SERIALGC
// Note that these are not virtual.
bool is_generation_policy() { return as_generation_policy() != NULL; }
@@ -109,9 +112,12 @@ class CollectorPolicy : public CHeapObj
bool is_mark_sweep_policy() { return as_mark_sweep_policy() != NULL; }
#ifndef SERIALGC
bool is_concurrent_mark_sweep_policy() { return as_concurrent_mark_sweep_policy() != NULL; }
+ bool is_g1_policy() { return as_g1_policy() != NULL; }
#else // SERIALGC
bool is_concurrent_mark_sweep_policy() { return false; }
-#endif // SERIALGC
+ bool is_g1_policy() { return false; }
+#endif // SERIALGC
+
virtual PermanentGenerationSpec *permanent_generation() {
assert(_permanent_generation != NULL, "Sanity check");
--- a/src/share/vm/memory/genCollectedHeap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genCollectedHeap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -252,6 +252,21 @@ public:
virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;
virtual HeapWord* allocate_new_tlab(size_t size);
+ // Can a compiler initialize a new object without store barriers?
+ // This permission only extends from the creation of a new object
+ // via a TLAB up to the first subsequent safepoint.
+ virtual bool can_elide_tlab_store_barriers() const {
+ return true;
+ }
+
+ // Can a compiler elide a store barrier when it writes
+ // a permanent oop into the heap? Applies when the compiler
+ // is storing x to the heap, where x->is_perm() is true.
+ virtual bool can_elide_permanent_oop_store_barriers() const {
+ // CMS needs to see all, even intra-generational, ref updates.
+ return !UseConcMarkSweepGC;
+ }
+
// The "requestor" generation is performing some garbage collection
// action for which it would be useful to have scratch space. The
// requestor promises to allocate no more than "max_alloc_words" in any
--- a/src/share/vm/memory/genMarkSweep.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genMarkSweep.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -191,8 +191,10 @@ void GenMarkSweep::allocate_stacks() {
void GenMarkSweep::deallocate_stacks() {
- GenCollectedHeap* gch = GenCollectedHeap::heap();
- gch->release_scratch();
+ if (!UseG1GC) {
+ GenCollectedHeap* gch = GenCollectedHeap::heap();
+ gch->release_scratch();
+ }
if (_preserved_oop_stack) {
delete _preserved_mark_stack;
--- a/src/share/vm/memory/genMarkSweep.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genMarkSweep.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -24,6 +24,7 @@
class GenMarkSweep : public MarkSweep {
friend class VM_MarkSweep;
+ friend class G1MarkSweep;
public:
static void invoke_at_safepoint(int level, ReferenceProcessor* rp,
bool clear_all_softrefs);
--- a/src/share/vm/memory/genOopClosures.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genOopClosures.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -55,6 +55,9 @@ class OopsInGenClosure : public OopClosu
// Derived classes that modify oops so that they might be old-to-young
// pointers must call the method below.
template <class T> void do_barrier(T* p);
+
+ // Version for use by closures that may be called in parallel code.
+ void par_do_barrier(oop* p);
public:
OopsInGenClosure() : OopClosure(NULL),
--- a/src/share/vm/memory/genOopClosures.inline.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genOopClosures.inline.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -45,6 +45,16 @@ template <class T> inline void OopsInGen
// If p points to a younger generation, mark the card.
if ((HeapWord*)obj < _gen_boundary) {
_rs->inline_write_ref_field_gc(p, obj);
+ }
+}
+
+inline void OopsInGenClosure::par_do_barrier(oop* p) {
+ assert(generation()->is_in_reserved(p), "expected ref in generation");
+ oop obj = *p;
+ assert(obj != NULL, "expected non-null object");
+ // If p points to a younger generation, mark the card.
+ if ((HeapWord*)obj < gen_boundary()) {
+ rs()->write_ref_field_gc_par(p, obj);
}
}
--- a/src/share/vm/memory/genRemSet.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/genRemSet.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -42,6 +42,7 @@ public:
};
GenRemSet(BarrierSet * bs) : _bs(bs) {}
+ GenRemSet() : _bs(NULL) {}
virtual Name rs_kind() = 0;
@@ -52,6 +53,9 @@ public:
// Return the barrier set associated with "this."
BarrierSet* bs() { return _bs; }
+
+ // Set the barrier set.
+ void set_bs(BarrierSet* bs) { _bs = bs; }
// Do any (sequential) processing necessary to prepare for (possibly
// "parallel", if that arg is true) calls to younger_refs_iterate.
@@ -116,7 +120,10 @@ public:
// Informs the RS that refs in the given "mr" may have changed
// arbitrarily, and therefore may contain old-to-young pointers.
- virtual void invalidate(MemRegion mr) = 0;
+ // If "whole heap" is true, then this invalidation is part of an
+ // invalidation of the whole heap, which an implementation might
+ // handle differently than that of a sub-part of the heap.
+ virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0;
// Informs the RS that refs in this generation
// may have changed arbitrarily, and therefore may contain
--- a/src/share/vm/memory/heapInspection.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/heapInspection.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -238,11 +238,14 @@ void HeapInspection::heap_inspection(out
HeapWord* ref;
CollectedHeap* heap = Universe::heap();
+ bool is_shared_heap = false;
switch (heap->kind()) {
+ case CollectedHeap::G1CollectedHeap:
case CollectedHeap::GenCollectedHeap: {
- GenCollectedHeap* gch = (GenCollectedHeap*)heap;
- gch->gc_prologue(false /* !full */); // get any necessary locks
- ref = gch->perm_gen()->used_region().start();
+ is_shared_heap = true;
+ SharedHeap* sh = (SharedHeap*)heap;
+ sh->gc_prologue(false /* !full */); // get any necessary locks, etc.
+ ref = sh->perm_gen()->used_region().start();
break;
}
#ifndef SERIALGC
@@ -284,9 +287,9 @@ void HeapInspection::heap_inspection(out
}
st->flush();
- if (Universe::heap()->kind() == CollectedHeap::GenCollectedHeap) {
- GenCollectedHeap* gch = GenCollectedHeap::heap();
- gch->gc_epilogue(false /* !full */); // release all acquired locks
+ if (is_shared_heap) {
+ SharedHeap* sh = (SharedHeap*)heap;
+ sh->gc_epilogue(false /* !full */); // release all acquired locks, etc.
}
}
--- a/src/share/vm/memory/iterator.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/iterator.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -26,9 +26,23 @@
class ReferenceProcessor;
+// Closure provides abortability.
+
+class Closure : public StackObj {
+ protected:
+ bool _abort;
+ void set_abort() { _abort = true; }
+ public:
+ Closure() : _abort(false) {}
+ // A subtype can use this mechanism to indicate to some iterator mapping
+ // functions that the iteration should cease.
+ bool abort() { return _abort; }
+ void clear_abort() { _abort = false; }
+};
+
// OopClosure is used for iterating through roots (oop*)
-class OopClosure : public StackObj {
+class OopClosure : public Closure {
public:
ReferenceProcessor* _ref_processor;
OopClosure(ReferenceProcessor* rp) : _ref_processor(rp) { }
@@ -55,11 +69,16 @@ class OopClosure : public StackObj {
Prefetch::style prefetch_style() { // Note that this is non-virtual.
return Prefetch::do_none;
}
+
+ // True iff this closure may be safely applied more than once to an oop
+ // location without an intervening "major reset" (like the end of a GC).
+ virtual bool idempotent() { return false; }
+ virtual bool apply_to_weak_ref_discovered_field() { return false; }
};
// ObjectClosure is used for iterating through an object space
-class ObjectClosure : public StackObj {
+class ObjectClosure : public Closure {
public:
// Called for each object.
virtual void do_object(oop obj) = 0;
--- a/src/share/vm/memory/modRefBarrierSet.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/modRefBarrierSet.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -31,6 +31,13 @@ class Generation;
class ModRefBarrierSet: public BarrierSet {
public:
+
+ ModRefBarrierSet() { _kind = BarrierSet::ModRef; }
+
+ bool is_a(BarrierSet::Name bsn) {
+ return bsn == BarrierSet::ModRef;
+ }
+
// Barriers only on ref writes.
bool has_read_ref_barrier() { return false; }
bool has_read_prim_barrier() { return false; }
@@ -85,8 +92,10 @@ public:
bool clear = false,
bool before_save_marks = false) = 0;
- // Causes all refs in "mr" to be assumed to be modified.
- virtual void invalidate(MemRegion mr) = 0;
+ // Causes all refs in "mr" to be assumed to be modified. If "whole_heap"
+ // is true, the caller asserts that the entire heap is being invalidated,
+ // which may admit an optimized implementation for some barriers.
+ virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0;
// The caller guarantees that "mr" contains no references. (Perhaps it's
// objects have been moved elsewhere.)
--- a/src/share/vm/memory/referenceProcessor.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/referenceProcessor.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -91,7 +91,8 @@ ReferenceProcessor::create_ref_processor
bool mt_discovery,
BoolObjectClosure* is_alive_non_header,
int parallel_gc_threads,
- bool mt_processing) {
+ bool mt_processing,
+ bool dl_needs_barrier) {
int mt_degree = 1;
if (parallel_gc_threads > 1) {
mt_degree = parallel_gc_threads;
@@ -99,7 +100,8 @@ ReferenceProcessor::create_ref_processor
ReferenceProcessor* rp =
new ReferenceProcessor(span, atomic_discovery,
mt_discovery, mt_degree,
- mt_processing && (parallel_gc_threads > 0));
+ mt_processing && (parallel_gc_threads > 0),
+ dl_needs_barrier);
if (rp == NULL) {
vm_exit_during_initialization("Could not allocate ReferenceProcessor object");
}
@@ -111,10 +113,13 @@ ReferenceProcessor::ReferenceProcessor(M
bool atomic_discovery,
bool mt_discovery,
int mt_degree,
- bool mt_processing) :
+ bool mt_processing,
+ bool discovered_list_needs_barrier) :
_discovering_refs(false),
_enqueuing_is_done(false),
_is_alive_non_header(NULL),
+ _discovered_list_needs_barrier(discovered_list_needs_barrier),
+ _bs(NULL),
_processing_is_mt(mt_processing),
_next_id(0)
{
@@ -134,6 +139,10 @@ ReferenceProcessor::ReferenceProcessor(M
for (int i = 0; i < _num_q * subclasses_of_ref; i++) {
_discoveredSoftRefs[i].set_head(sentinel_ref());
_discoveredSoftRefs[i].set_length(0);
+ }
+ // If we do barreirs, cache a copy of the barrier set.
+ if (discovered_list_needs_barrier) {
+ _bs = Universe::heap()->barrier_set();
}
}
@@ -727,10 +736,15 @@ ReferenceProcessor::abandon_partial_disc
refs_list.set_length(0);
}
-void
-ReferenceProcessor::abandon_partial_discovered_list_arr(DiscoveredList refs_lists[]) {
- for (int i = 0; i < _num_q; i++) {
- abandon_partial_discovered_list(refs_lists[i]);
+void ReferenceProcessor::abandon_partial_discovery() {
+ // loop over the lists
+ for (int i = 0; i < _num_q * subclasses_of_ref; i++) {
+ if (TraceReferenceGC && PrintGCDetails && ((i % _num_q) == 0)) {
+ gclog_or_tty->print_cr(
+ "\nAbandoning %s discovered list",
+ list_name(i));
+ }
+ abandon_partial_discovered_list(_discoveredSoftRefs[i]);
}
}
@@ -994,7 +1008,16 @@ ReferenceProcessor::add_to_discovered_li
assert(_discovery_is_mt, "!_discovery_is_mt should have been handled by caller");
// First we must make sure this object is only enqueued once. CAS in a non null
// discovered_addr.
- oop retest = oopDesc::atomic_compare_exchange_oop(refs_list.head(), discovered_addr,
+ oop current_head = refs_list.head();
+
+ // Note: In the case of G1, this pre-barrier is strictly
+ // not necessary because the only case we are interested in
+ // here is when *discovered_addr is NULL, so this will expand to
+ // nothing. As a result, I am just manually eliding this out for G1.
+ if (_discovered_list_needs_barrier && !UseG1GC) {
+ _bs->write_ref_field_pre((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
+ }
+ oop retest = oopDesc::atomic_compare_exchange_oop(current_head, discovered_addr,
NULL);
if (retest == NULL) {
// This thread just won the right to enqueue the object.
@@ -1002,6 +1025,10 @@ ReferenceProcessor::add_to_discovered_li
// is necessary.
refs_list.set_head(obj);
refs_list.set_length(refs_list.length() + 1);
+ if (_discovered_list_needs_barrier) {
+ _bs->write_ref_field((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
+ }
+
} else {
// If retest was non NULL, another thread beat us to it:
// The reference has already been discovered...
@@ -1073,8 +1100,8 @@ bool ReferenceProcessor::discover_refere
}
}
- HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj);
- oop discovered = java_lang_ref_Reference::discovered(obj);
+ HeapWord* const discovered_addr = java_lang_ref_Reference::discovered_addr(obj);
+ const oop discovered = java_lang_ref_Reference::discovered(obj);
assert(discovered->is_oop_or_null(), "bad discovered field");
if (discovered != NULL) {
// The reference has already been discovered...
@@ -1094,7 +1121,7 @@ bool ReferenceProcessor::discover_refere
// discovered twice except by concurrent collectors that potentially
// trace the same Reference object twice.
assert(UseConcMarkSweepGC,
- "Only possible with a concurrent collector");
+ "Only possible with an incremental-update concurrent collector");
return true;
}
}
@@ -1122,12 +1149,24 @@ bool ReferenceProcessor::discover_refere
return false; // nothing special needs to be done
}
- // We do a raw store here, the field will be visited later when
- // processing the discovered references.
if (_discovery_is_mt) {
add_to_discovered_list_mt(*list, obj, discovered_addr);
} else {
- oop_store_raw(discovered_addr, list->head());
+ // If "_discovered_list_needs_barrier", we do write barriers when
+ // updating the discovered reference list. Otherwise, we do a raw store
+ // here: the field will be visited later when processing the discovered
+ // references.
+ oop current_head = list->head();
+ // As in the case further above, since we are over-writing a NULL
+ // pre-value, we can safely elide the pre-barrier here for the case of G1.
+ assert(discovered == NULL, "control point invariant");
+ if (_discovered_list_needs_barrier && !UseG1GC) { // safe to elide for G1
+ _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
+ }
+ oop_store_raw(discovered_addr, current_head);
+ if (_discovered_list_needs_barrier) {
+ _bs->write_ref_field((oop*)discovered_addr, current_head);
+ }
list->set_head(obj);
list->set_length(list->length() + 1);
}
--- a/src/share/vm/memory/referenceProcessor.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/referenceProcessor.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -54,6 +54,14 @@ class ReferenceProcessor : public CHeapO
bool _discovery_is_atomic; // if discovery is atomic wrt
// other collectors in configuration
bool _discovery_is_mt; // true if reference discovery is MT.
+ // If true, setting "next" field of a discovered refs list requires
+ // write barrier(s). (Must be true if used in a collector in which
+ // elements of a discovered list may be moved during discovery: for
+ // example, a collector like Garbage-First that moves objects during a
+ // long-term concurrent marking phase that does weak reference
+ // discovery.)
+ bool _discovered_list_needs_barrier;
+ BarrierSet* _bs; // Cached copy of BarrierSet.
bool _enqueuing_is_done; // true if all weak references enqueued
bool _processing_is_mt; // true during phases when
// reference processing is MT.
@@ -196,7 +204,6 @@ class ReferenceProcessor : public CHeapO
void verify_ok_to_handle_reflists() PRODUCT_RETURN;
void abandon_partial_discovered_list(DiscoveredList& refs_list);
- void abandon_partial_discovered_list_arr(DiscoveredList refs_lists[]);
// Calculate the number of jni handles.
unsigned int count_jni_refs();
@@ -217,6 +224,8 @@ class ReferenceProcessor : public CHeapO
_discovery_is_atomic(true),
_enqueuing_is_done(false),
_discovery_is_mt(false),
+ _discovered_list_needs_barrier(false),
+ _bs(NULL),
_is_alive_non_header(NULL),
_num_q(0),
_processing_is_mt(false),
@@ -224,8 +233,10 @@ class ReferenceProcessor : public CHeapO
{}
ReferenceProcessor(MemRegion span, bool atomic_discovery,
- bool mt_discovery, int mt_degree = 1,
- bool mt_processing = false);
+ bool mt_discovery,
+ int mt_degree = 1,
+ bool mt_processing = false,
+ bool discovered_list_needs_barrier = false);
// Allocates and initializes a reference processor.
static ReferenceProcessor* create_ref_processor(
@@ -234,8 +245,8 @@ class ReferenceProcessor : public CHeapO
bool mt_discovery,
BoolObjectClosure* is_alive_non_header = NULL,
int parallel_gc_threads = 1,
- bool mt_processing = false);
-
+ bool mt_processing = false,
+ bool discovered_list_needs_barrier = false);
// RefDiscoveryPolicy values
enum {
ReferenceBasedDiscovery = 0,
@@ -295,6 +306,11 @@ class ReferenceProcessor : public CHeapO
public:
// Enqueue references at end of GC (called by the garbage collector)
bool enqueue_discovered_references(AbstractRefProcTaskExecutor* task_executor = NULL);
+
+ // If a discovery is in process that is being superceded, abandon it: all
+ // the discovered lists will be empty, and all the objects on them will
+ // have NULL discovered fields. Must be called only at a safepoint.
+ void abandon_partial_discovery();
// debugging
void verify_no_references_recorded() PRODUCT_RETURN;
--- a/src/share/vm/memory/sharedHeap.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/sharedHeap.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -57,15 +57,24 @@ SharedHeap::SharedHeap(CollectorPolicy*
}
_sh = this; // ch is static, should be set only once.
if ((UseParNewGC ||
- (UseConcMarkSweepGC && CMSParallelRemarkEnabled)) &&
+ (UseConcMarkSweepGC && CMSParallelRemarkEnabled) ||
+ UseG1GC) &&
ParallelGCThreads > 0) {
- _workers = new WorkGang("Parallel GC Threads", ParallelGCThreads, true);
+ _workers = new WorkGang("Parallel GC Threads", ParallelGCThreads,
+ /* are_GC_task_threads */true,
+ /* are_ConcurrentGC_threads */false);
if (_workers == NULL) {
vm_exit_during_initialization("Failed necessary allocation.");
}
}
}
+bool SharedHeap::heap_lock_held_for_gc() {
+ Thread* t = Thread::current();
+ return Heap_lock->owned_by_self()
+ || ( (t->is_GC_task_thread() || t->is_VM_thread())
+ && _thread_holds_heap_lock_for_gc);
+}
void SharedHeap::set_par_threads(int t) {
_n_par_threads = t;
@@ -280,10 +289,11 @@ void SharedHeap::fill_region_with_object
}
// Some utilities.
-void SharedHeap::print_size_transition(size_t bytes_before,
+void SharedHeap::print_size_transition(outputStream* out,
+ size_t bytes_before,
size_t bytes_after,
size_t capacity) {
- tty->print(" %d%s->%d%s(%d%s)",
+ out->print(" %d%s->%d%s(%d%s)",
byte_size_in_proper_unit(bytes_before),
proper_unit_for_byte_size(bytes_before),
byte_size_in_proper_unit(bytes_after),
--- a/src/share/vm/memory/sharedHeap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/sharedHeap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -44,6 +44,9 @@ class SharedHeap : public CollectedHeap
class SharedHeap : public CollectedHeap {
friend class VMStructs;
+ friend class VM_GC_Operation;
+ friend class VM_CGC_Operation;
+
private:
// For claiming strong_roots tasks.
SubTasksDone* _process_strong_tasks;
@@ -82,6 +85,14 @@ protected:
// function.
SharedHeap(CollectorPolicy* policy_);
+ // Returns true if the calling thread holds the heap lock,
+ // or the calling thread is a par gc thread and the heap_lock is held
+ // by the vm thread doing a gc operation.
+ bool heap_lock_held_for_gc();
+ // True if the heap_lock is held by the a non-gc thread invoking a gc
+ // operation.
+ bool _thread_holds_heap_lock_for_gc;
+
public:
static SharedHeap* heap() { return _sh; }
@@ -97,8 +108,8 @@ public:
void set_perm(PermGen* perm_gen) { _perm_gen = perm_gen; }
- // A helper function that fills an allocated-but-not-yet-initialized
- // region with a garbage object.
+ // A helper function that fills a region of the heap with
+ // with a single object.
static void fill_region_with_object(MemRegion mr);
// Minimum garbage fill object size
@@ -214,13 +225,12 @@ public:
// "SharedHeap" can use in the implementation of its virtual
// functions.
-protected:
+public:
// Do anything common to GC's.
virtual void gc_prologue(bool full) = 0;
virtual void gc_epilogue(bool full) = 0;
-public:
//
// New methods from CollectedHeap
//
@@ -266,7 +276,8 @@ public:
}
// Some utilities.
- void print_size_transition(size_t bytes_before,
+ void print_size_transition(outputStream* out,
+ size_t bytes_before,
size_t bytes_after,
size_t capacity);
};
--- a/src/share/vm/memory/space.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/space.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -105,7 +105,7 @@ void DirtyCardToOopClosure::do_MemRegion
"Only ones we deal with for now.");
assert(_precision != CardTableModRefBS::ObjHeadPreciseArray ||
- _last_bottom == NULL ||
+ _cl->idempotent() || _last_bottom == NULL ||
top <= _last_bottom,
"Not decreasing");
NOT_PRODUCT(_last_bottom = mr.start());
@@ -144,7 +144,14 @@ void DirtyCardToOopClosure::do_MemRegion
walk_mem_region(mr, bottom_obj, top);
}
- _min_done = bottom;
+ // An idempotent closure might be applied in any order, so we don't
+ // record a _min_done for it.
+ if (!_cl->idempotent()) {
+ _min_done = bottom;
+ } else {
+ assert(_min_done == _last_explicit_min_done,
+ "Don't update _min_done for idempotent cl");
+ }
}
DirtyCardToOopClosure* Space::new_dcto_cl(OopClosure* cl,
@@ -250,7 +257,8 @@ void Space::clear(bool mangle_space) {
}
}
-ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(NULL) {
+ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(NULL),
+ _concurrent_iteration_safe_limit(NULL) {
_mangler = new GenSpaceMangler(this);
}
@@ -263,17 +271,17 @@ void ContiguousSpace::initialize(MemRegi
bool mangle_space)
{
CompactibleSpace::initialize(mr, clear_space, mangle_space);
- _concurrent_iteration_safe_limit = top();
+ set_concurrent_iteration_safe_limit(top());
}
void ContiguousSpace::clear(bool mangle_space) {
set_top(bottom());
set_saved_mark();
- Space::clear(mangle_space);
+ CompactibleSpace::clear(mangle_space);
}
bool Space::is_in(const void* p) const {
- HeapWord* b = block_start(p);
+ HeapWord* b = block_start_const(p);
return b != NULL && block_is_obj(b);
}
@@ -342,8 +350,13 @@ void CompactibleSpace::initialize(MemReg
bool clear_space,
bool mangle_space) {
Space::initialize(mr, clear_space, mangle_space);
+ set_compaction_top(bottom());
+ _next_compaction_space = NULL;
+}
+
+void CompactibleSpace::clear(bool mangle_space) {
+ Space::clear(mangle_space);
_compaction_top = bottom();
- _next_compaction_space = NULL;
}
HeapWord* CompactibleSpace::forward(oop q, size_t size,
@@ -520,8 +533,8 @@ void ContiguousSpace::verify(bool allow_
}
guarantee(p == top(), "end of last object must match end of space");
if (top() != end()) {
- guarantee(top() == block_start(end()-1) &&
- top() == block_start(top()),
+ guarantee(top() == block_start_const(end()-1) &&
+ top() == block_start_const(top()),
"top should be start of unallocated block, if it exists");
}
}
@@ -753,7 +766,7 @@ ALL_SINCE_SAVE_MARKS_CLOSURES(ContigSpac
#undef ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN
// Very general, slow implementation.
-HeapWord* ContiguousSpace::block_start(const void* p) const {
+HeapWord* ContiguousSpace::block_start_const(const void* p) const {
assert(MemRegion(bottom(), end()).contains(p), "p not in space");
if (p >= top()) {
return top();
@@ -957,7 +970,8 @@ void OffsetTableContigSpace::verify(bool
// For a sampling of objects in the space, find it using the
// block offset table.
if (blocks == BLOCK_SAMPLE_INTERVAL) {
- guarantee(p == block_start(p + (size/2)), "check offset computation");
+ guarantee(p == block_start_const(p + (size/2)),
+ "check offset computation");
blocks = 0;
} else {
blocks++;
--- a/src/share/vm/memory/space.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/space.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -105,7 +105,7 @@ class Space: public CHeapObj {
virtual void set_bottom(HeapWord* value) { _bottom = value; }
virtual void set_end(HeapWord* value) { _end = value; }
- HeapWord* saved_mark_word() const { return _saved_mark_word; }
+ virtual HeapWord* saved_mark_word() const { return _saved_mark_word; }
void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; }
MemRegionClosure* preconsumptionDirtyCardClosure() const {
@@ -131,9 +131,15 @@ class Space: public CHeapObj {
return MemRegion(bottom(), saved_mark_word());
}
- // Initialization. These may be run to reset an existing
- // Space.
+ // Initialization.
+ // "initialize" should be called once on a space, before it is used for
+ // any purpose. The "mr" arguments gives the bounds of the space, and
+ // the "clear_space" argument should be true unless the memory in "mr" is
+ // known to be zeroed.
virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space);
+
+ // The "clear" method must be called on a region that may have
+ // had allocation performed in it, but is now to be considered empty.
virtual void clear(bool mangle_space);
// For detecting GC bugs. Should only be called at GC boundaries, since
@@ -218,7 +224,13 @@ class Space: public CHeapObj {
// "block" that contains "p". We say "block" instead of "object" since
// some heaps may not pack objects densely; a chunk may either be an
// object or a non-object. If "p" is not in the space, return NULL.
- virtual HeapWord* block_start(const void* p) const = 0;
+ virtual HeapWord* block_start_const(const void* p) const = 0;
+
+ // The non-const version may have benevolent side effects on the data
+ // structure supporting these calls, possibly speeding up future calls.
+ // The default implementation, however, is simply to call the const
+ // version.
+ inline virtual HeapWord* block_start(const void* p);
// Requires "addr" to be the start of a chunk, and returns its size.
// "addr + size" is required to be the start of a new chunk, or the end
@@ -284,12 +296,13 @@ protected:
CardTableModRefBS::PrecisionStyle _precision;
HeapWord* _boundary; // If non-NULL, process only non-NULL oops
// pointing below boundary.
- HeapWord* _min_done; // ObjHeadPreciseArray precision requires
+ HeapWord* _min_done; // ObjHeadPreciseArray precision requires
// a downwards traversal; this is the
// lowest location already done (or,
// alternatively, the lowest address that
// shouldn't be done again. NULL means infinity.)
NOT_PRODUCT(HeapWord* _last_bottom;)
+ NOT_PRODUCT(HeapWord* _last_explicit_min_done;)
// Get the actual top of the area on which the closure will
// operate, given where the top is assumed to be (the end of the
@@ -313,13 +326,15 @@ public:
HeapWord* boundary) :
_sp(sp), _cl(cl), _precision(precision), _boundary(boundary),
_min_done(NULL) {
- NOT_PRODUCT(_last_bottom = NULL;)
+ NOT_PRODUCT(_last_bottom = NULL);
+ NOT_PRODUCT(_last_explicit_min_done = NULL);
}
void do_MemRegion(MemRegion mr);
void set_min_done(HeapWord* min_done) {
_min_done = min_done;
+ NOT_PRODUCT(_last_explicit_min_done = _min_done);
}
#ifndef PRODUCT
void set_last_bottom(HeapWord* last_bottom) {
@@ -356,7 +371,11 @@ private:
CompactibleSpace* _next_compaction_space;
public:
+ CompactibleSpace() :
+ _compaction_top(NULL), _next_compaction_space(NULL) {}
+
virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space);
+ virtual void clear(bool mangle_space);
// Used temporarily during a compaction phase to hold the value
// top should have when compaction is complete.
@@ -513,7 +532,7 @@ protected:
/* prefetch beyond q */ \
Prefetch::write(q, interval); \
/* size_t size = oop(q)->size(); changing this for cms for perm gen */\
- size_t size = block_size(q); \
+ size_t size = block_size(q); \
compact_top = cp->space->forward(oop(q), size, cp, compact_top); \
q += size; \
end_of_live = q; \
@@ -577,156 +596,158 @@ protected:
cp->space->set_compaction_top(compact_top); \
}
-#define SCAN_AND_ADJUST_POINTERS(adjust_obj_size) { \
- /* adjust all the interior pointers to point at the new locations of objects \
- * Used by MarkSweep::mark_sweep_phase3() */ \
- \
- HeapWord* q = bottom(); \
- HeapWord* t = _end_of_live; /* Established by "prepare_for_compaction". */ \
- \
- assert(_first_dead <= _end_of_live, "Stands to reason, no?"); \
- \
- if (q < t && _first_dead > q && \
+#define SCAN_AND_ADJUST_POINTERS(adjust_obj_size) { \
+ /* adjust all the interior pointers to point at the new locations of objects \
+ * Used by MarkSweep::mark_sweep_phase3() */ \
+ \
+ HeapWord* q = bottom(); \
+ HeapWord* t = _end_of_live; /* Established by "prepare_for_compaction". */ \
+ \
+ assert(_first_dead <= _end_of_live, "Stands to reason, no?"); \
+ \
+ if (q < t && _first_dead > q && \
!oop(q)->is_gc_marked()) { \
/* we have a chunk of the space which hasn't moved and we've \
* reinitialized the mark word during the previous pass, so we can't \
- * use is_gc_marked for the traversal. */ \
+ * use is_gc_marked for the traversal. */ \
HeapWord* end = _first_dead; \
\
- while (q < end) { \
- /* I originally tried to conjoin "block_start(q) == q" to the \
- * assertion below, but that doesn't work, because you can't \
- * accurately traverse previous objects to get to the current one \
- * after their pointers (including pointers into permGen) have been \
- * updated, until the actual compaction is done. dld, 4/00 */ \
- assert(block_is_obj(q), \
- "should be at block boundaries, and should be looking at objs"); \
+ while (q < end) { \
+ /* I originally tried to conjoin "block_start(q) == q" to the \
+ * assertion below, but that doesn't work, because you can't \
+ * accurately traverse previous objects to get to the current one \
+ * after their pointers (including pointers into permGen) have been \
+ * updated, until the actual compaction is done. dld, 4/00 */ \
+ assert(block_is_obj(q), \
+ "should be at block boundaries, and should be looking at objs"); \
\
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::track_interior_pointers(oop(q))); \
\
- /* point all the oops to the new location */ \
- size_t size = oop(q)->adjust_pointers(); \
- size = adjust_obj_size(size); \
+ /* point all the oops to the new location */ \
+ size_t size = oop(q)->adjust_pointers(); \
+ size = adjust_obj_size(size); \
\
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers()); \
- \
+ \
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::validate_live_oop(oop(q), size)); \
- \
+ \
q += size; \
- } \
- \
- if (_first_dead == t) { \
- q = t; \
- } else { \
- /* $$$ This is funky. Using this to read the previously written \
- * LiveRange. See also use below. */ \
+ } \
+ \
+ if (_first_dead == t) { \
+ q = t; \
+ } else { \
+ /* $$$ This is funky. Using this to read the previously written \
+ * LiveRange. See also use below. */ \
q = (HeapWord*)oop(_first_dead)->mark()->decode_pointer(); \
- } \
- } \
+ } \
+ } \
\
const intx interval = PrefetchScanIntervalInBytes; \
\
- debug_only(HeapWord* prev_q = NULL); \
- while (q < t) { \
- /* prefetch beyond q */ \
+ debug_only(HeapWord* prev_q = NULL); \
+ while (q < t) { \
+ /* prefetch beyond q */ \
Prefetch::write(q, interval); \
- if (oop(q)->is_gc_marked()) { \
- /* q is alive */ \
+ if (oop(q)->is_gc_marked()) { \
+ /* q is alive */ \
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::track_interior_pointers(oop(q))); \
- /* point all the oops to the new location */ \
- size_t size = oop(q)->adjust_pointers(); \
- size = adjust_obj_size(size); \
- VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers()); \
+ /* point all the oops to the new location */ \
+ size_t size = oop(q)->adjust_pointers(); \
+ size = adjust_obj_size(size); \
+ VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers()); \
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::validate_live_oop(oop(q), size)); \
- debug_only(prev_q = q); \
+ debug_only(prev_q = q); \
q += size; \
- } else { \
- /* q is not a live object, so its mark should point at the next \
- * live object */ \
- debug_only(prev_q = q); \
- q = (HeapWord*) oop(q)->mark()->decode_pointer(); \
- assert(q > prev_q, "we should be moving forward through memory"); \
- } \
- } \
- \
- assert(q == t, "just checking"); \
+ } else { \
+ /* q is not a live object, so its mark should point at the next \
+ * live object */ \
+ debug_only(prev_q = q); \
+ q = (HeapWord*) oop(q)->mark()->decode_pointer(); \
+ assert(q > prev_q, "we should be moving forward through memory"); \
+ } \
+ } \
+ \
+ assert(q == t, "just checking"); \
}
-#define SCAN_AND_COMPACT(obj_size) { \
+#define SCAN_AND_COMPACT(obj_size) { \
/* Copy all live objects to their new location \
- * Used by MarkSweep::mark_sweep_phase4() */ \
- \
- HeapWord* q = bottom(); \
- HeapWord* const t = _end_of_live; \
- debug_only(HeapWord* prev_q = NULL); \
- \
- if (q < t && _first_dead > q && \
+ * Used by MarkSweep::mark_sweep_phase4() */ \
+ \
+ HeapWord* q = bottom(); \
+ HeapWord* const t = _end_of_live; \
+ debug_only(HeapWord* prev_q = NULL); \
+ \
+ if (q < t && _first_dead > q && \
!oop(q)->is_gc_marked()) { \
- debug_only( \
+ debug_only( \
/* we have a chunk of the space which hasn't moved and we've reinitialized \
* the mark word during the previous pass, so we can't use is_gc_marked for \
* the traversal. */ \
- HeapWord* const end = _first_dead; \
- \
- while (q < end) { \
+ HeapWord* const end = _first_dead; \
+ \
+ while (q < end) { \
size_t size = obj_size(q); \
assert(!oop(q)->is_gc_marked(), \
"should be unmarked (special dense prefix handling)"); \
- VALIDATE_MARK_SWEEP_ONLY(MarkSweep::live_oop_moved_to(q, size, q)); \
- debug_only(prev_q = q); \
+ VALIDATE_MARK_SWEEP_ONLY(MarkSweep::live_oop_moved_to(q, size, q)); \
+ debug_only(prev_q = q); \
q += size; \
- } \
- ) /* debug_only */ \
- \
- if (_first_dead == t) { \
- q = t; \
- } else { \
- /* $$$ Funky */ \
- q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); \
- } \
- } \
- \
- const intx scan_interval = PrefetchScanIntervalInBytes; \
- const intx copy_interval = PrefetchCopyIntervalInBytes; \
- while (q < t) { \
- if (!oop(q)->is_gc_marked()) { \
- /* mark is pointer to next marked oop */ \
- debug_only(prev_q = q); \
- q = (HeapWord*) oop(q)->mark()->decode_pointer(); \
- assert(q > prev_q, "we should be moving forward through memory"); \
- } else { \
- /* prefetch beyond q */ \
+ } \
+ ) /* debug_only */ \
+ \
+ if (_first_dead == t) { \
+ q = t; \
+ } else { \
+ /* $$$ Funky */ \
+ q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); \
+ } \
+ } \
+ \
+ const intx scan_interval = PrefetchScanIntervalInBytes; \
+ const intx copy_interval = PrefetchCopyIntervalInBytes; \
+ while (q < t) { \
+ if (!oop(q)->is_gc_marked()) { \
+ /* mark is pointer to next marked oop */ \
+ debug_only(prev_q = q); \
+ q = (HeapWord*) oop(q)->mark()->decode_pointer(); \
+ assert(q > prev_q, "we should be moving forward through memory"); \
+ } else { \
+ /* prefetch beyond q */ \
Prefetch::read(q, scan_interval); \
\
/* size and destination */ \
size_t size = obj_size(q); \
HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); \
\
- /* prefetch beyond compaction_top */ \
+ /* prefetch beyond compaction_top */ \
Prefetch::write(compaction_top, copy_interval); \
\
- /* copy object and reinit its mark */ \
+ /* copy object and reinit its mark */ \
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::live_oop_moved_to(q, size, \
compaction_top)); \
- assert(q != compaction_top, "everything in this pass should be moving"); \
- Copy::aligned_conjoint_words(q, compaction_top, size); \
- oop(compaction_top)->init_mark(); \
- assert(oop(compaction_top)->klass() != NULL, "should have a class"); \
- \
- debug_only(prev_q = q); \
+ assert(q != compaction_top, "everything in this pass should be moving"); \
+ Copy::aligned_conjoint_words(q, compaction_top, size); \
+ oop(compaction_top)->init_mark(); \
+ assert(oop(compaction_top)->klass() != NULL, "should have a class"); \
+ \
+ debug_only(prev_q = q); \
q += size; \
- } \
- } \
- \
+ } \
+ } \
+ \
+ /* Let's remember if we were empty before we did the compaction. */ \
+ bool was_empty = used_region().is_empty(); \
/* Reset space after compaction is complete */ \
- reset_after_compaction(); \
+ reset_after_compaction(); \
/* We do this clear, below, since it has overloaded meanings for some */ \
/* space subtypes. For example, OffsetTableContigSpace's that were */ \
/* compacted into will have had their offset table thresholds updated */ \
/* continuously, but those that weren't need to have their thresholds */ \
/* re-initialized. Also mangles unused area for debugging. */ \
- if (is_empty()) { \
- clear(SpaceDecorator::Mangle); \
+ if (used_region().is_empty()) { \
+ if (!was_empty) clear(SpaceDecorator::Mangle); \
} else { \
if (ZapUnusedHeapArea) mangle_unused_area(); \
} \
@@ -752,20 +773,18 @@ class ContiguousSpace: public Compactibl
inline HeapWord* par_allocate_impl(size_t word_size, HeapWord* end_value);
public:
-
ContiguousSpace();
~ContiguousSpace();
virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space);
+ virtual void clear(bool mangle_space);
// Accessors
HeapWord* top() const { return _top; }
void set_top(HeapWord* value) { _top = value; }
- void set_saved_mark() { _saved_mark_word = top(); }
- void reset_saved_mark() { _saved_mark_word = bottom(); }
-
- virtual void clear(bool mangle_space);
+ virtual void set_saved_mark() { _saved_mark_word = top(); }
+ void reset_saved_mark() { _saved_mark_word = bottom(); }
WaterMark bottom_mark() { return WaterMark(this, bottom()); }
WaterMark top_mark() { return WaterMark(this, top()); }
@@ -874,7 +893,7 @@ class ContiguousSpace: public Compactibl
virtual void object_iterate_from(WaterMark mark, ObjectClosure* blk);
// Very inefficient implementation.
- virtual HeapWord* block_start(const void* p) const;
+ virtual HeapWord* block_start_const(const void* p) const;
size_t block_size(const HeapWord* p) const;
// If a block is in the allocated area, it is an object.
bool block_is_obj(const HeapWord* p) const { return p < top(); }
@@ -979,7 +998,8 @@ class EdenSpace : public ContiguousSpace
HeapWord* _soft_end;
public:
- EdenSpace(DefNewGeneration* gen) : _gen(gen) { _soft_end = NULL; }
+ EdenSpace(DefNewGeneration* gen) :
+ _gen(gen), _soft_end(NULL) {}
// Get/set just the 'soft' limit.
HeapWord* soft_end() { return _soft_end; }
@@ -1033,7 +1053,7 @@ class OffsetTableContigSpace: public Con
void clear(bool mangle_space);
- inline HeapWord* block_start(const void* p) const;
+ inline HeapWord* block_start_const(const void* p) const;
// Add offset table update.
virtual inline HeapWord* allocate(size_t word_size);
--- a/src/share/vm/memory/space.inline.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/space.inline.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -21,6 +21,10 @@
* have any questions.
*
*/
+
+inline HeapWord* Space::block_start(const void* p) {
+ return block_start_const(p);
+}
inline HeapWord* OffsetTableContigSpace::allocate(size_t size) {
HeapWord* res = ContiguousSpace::allocate(size);
@@ -50,7 +54,8 @@ inline HeapWord* OffsetTableContigSpace:
return res;
}
-inline HeapWord* OffsetTableContigSpace::block_start(const void* p) const {
+inline HeapWord*
+OffsetTableContigSpace::block_start_const(const void* p) const {
return _offsets.block_start(p);
}
--- a/src/share/vm/memory/specialized_oop_closures.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/specialized_oop_closures.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -59,6 +59,12 @@ class CMSInnerParMarkAndPushClosure;
// This is split into several because of a Visual C++ 6.0 compiler bug
// where very long macros cause the compiler to crash
+// Some other heap might define further specialized closures.
+#ifndef FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES
+#define FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES(f) \
+ /* None */
+#endif
+
#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(f) \
f(ScanClosure,_nv) \
f(FastScanClosure,_nv) \
@@ -77,7 +83,7 @@ class CMSInnerParMarkAndPushClosure;
SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f)
#ifndef SERIALGC
-#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(f) \
+#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) \
f(MarkRefsIntoAndScanClosure,_nv) \
f(Par_MarkRefsIntoAndScanClosure,_nv) \
f(PushAndMarkClosure,_nv) \
@@ -85,10 +91,12 @@ class CMSInnerParMarkAndPushClosure;
f(PushOrMarkClosure,_nv) \
f(Par_PushOrMarkClosure,_nv) \
f(CMSKeepAliveClosure,_nv) \
- f(CMSInnerParMarkAndPushClosure,_nv)
-#else // SERIALGC
-#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(f)
-#endif // SERIALGC
+ f(CMSInnerParMarkAndPushClosure,_nv) \
+ FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES(f)
+#else // SERIALGC
+#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f)
+#endif // SERIALGC
+
// We separate these out, because sometime the general one has
// a different definition from the specialized ones, and sometimes it
@@ -98,8 +106,8 @@ class CMSInnerParMarkAndPushClosure;
f(OopClosure,_v) \
SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(f)
-#define ALL_OOP_OOP_ITERATE_CLOSURES_3(f) \
- SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(f)
+#define ALL_OOP_OOP_ITERATE_CLOSURES_2(f) \
+ SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f)
#ifndef SERIALGC
// This macro applies an argument macro to all OopClosures for which we
@@ -125,6 +133,13 @@ class CMSInnerParMarkAndPushClosure;
// The "root_class" is the most general class to define; this may be
// "OopClosure" in some applications and "OopsInGenClosure" in others.
+
+// Some other heap might define further specialized closures.
+#ifndef FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES
+#define FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f) \
+ /* None */
+#endif
+
#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_S(f) \
f(ScanClosure,_nv) \
f(FastScanClosure,_nv)
@@ -132,7 +147,8 @@ class CMSInnerParMarkAndPushClosure;
#ifndef SERIALGC
#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) \
f(ParScanWithBarrierClosure,_nv) \
- f(ParScanWithoutBarrierClosure,_nv)
+ f(ParScanWithoutBarrierClosure,_nv) \
+ FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f)
#else // SERIALGC
#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f)
#endif // SERIALGC
@@ -179,13 +195,15 @@ public:
#if ENABLE_SPECIALIZATION_STATS
private:
- static int _numCallsAll;
-
- static int _numCallsTotal[NUM_Kinds];
- static int _numCalls_nv[NUM_Kinds];
-
- static int _numDoOopCallsTotal[NUM_Kinds];
- static int _numDoOopCalls_nv[NUM_Kinds];
+ static bool _init;
+ static bool _wrapped;
+ static jint _numCallsAll;
+
+ static jint _numCallsTotal[NUM_Kinds];
+ static jint _numCalls_nv[NUM_Kinds];
+
+ static jint _numDoOopCallsTotal[NUM_Kinds];
+ static jint _numDoOopCalls_nv[NUM_Kinds];
public:
#endif
static void clear() PRODUCT_RETURN;
@@ -203,22 +221,22 @@ public:
#if ENABLE_SPECIALIZATION_STATS
inline void SpecializationStats::record_call() {
- _numCallsAll++;;
+ Atomic::inc(&_numCallsAll);
}
inline void SpecializationStats::record_iterate_call_v(Kind k) {
- _numCallsTotal[k]++;
+ Atomic::inc(&_numCallsTotal[k]);
}
inline void SpecializationStats::record_iterate_call_nv(Kind k) {
- _numCallsTotal[k]++;
- _numCalls_nv[k]++;
+ Atomic::inc(&_numCallsTotal[k]);
+ Atomic::inc(&_numCalls_nv[k]);
}
inline void SpecializationStats::record_do_oop_call_v(Kind k) {
- _numDoOopCallsTotal[k]++;
+ Atomic::inc(&_numDoOopCallsTotal[k]);
}
inline void SpecializationStats::record_do_oop_call_nv(Kind k) {
- _numDoOopCallsTotal[k]++;
- _numDoOopCalls_nv[k]++;
+ Atomic::inc(&_numDoOopCallsTotal[k]);
+ Atomic::inc(&_numDoOopCalls_nv[k]);
}
#else // !ENABLE_SPECIALIZATION_STATS
--- a/src/share/vm/memory/universe.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/memory/universe.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -739,6 +739,15 @@ jint Universe::initialize_heap() {
fatal("UseParallelGC not supported in java kernel vm.");
#endif // SERIALGC
+ } else if (UseG1GC) {
+#ifndef SERIALGC
+ G1CollectorPolicy* g1p = new G1CollectorPolicy_BestRegionsFirst();
+ G1CollectedHeap* g1h = new G1CollectedHeap(g1p);
+ Universe::_collectedHeap = g1h;
+#else // SERIALGC
+ fatal("UseG1GC not supported in java kernel vm.");
+#endif // SERIALGC
+
} else {
GenCollectorPolicy *gc_policy;
@@ -938,7 +947,10 @@ bool universe_post_init() {
// This needs to be done before the first scavenge/gc, since
// it's an input to soft ref clearing policy.
- Universe::update_heap_info_at_gc();
+ {
+ MutexLocker x(Heap_lock);
+ Universe::update_heap_info_at_gc();
+ }
// ("weak") refs processing infrastructure initialization
Universe::heap()->post_initialize();
@@ -1194,10 +1206,11 @@ uintptr_t Universe::verify_klass_mask()
// ???: What if a CollectedHeap doesn't have a permanent generation?
ShouldNotReachHere();
break;
- case CollectedHeap::GenCollectedHeap: {
- GenCollectedHeap* gch = (GenCollectedHeap*) Universe::heap();
- permanent_reserved = gch->perm_gen()->reserved();
- break;
+ case CollectedHeap::GenCollectedHeap:
+ case CollectedHeap::G1CollectedHeap: {
+ SharedHeap* sh = (SharedHeap*) Universe::heap();
+ permanent_reserved = sh->perm_gen()->reserved();
+ break;
}
#ifndef SERIALGC
case CollectedHeap::ParallelScavengeHeap: {
--- a/src/share/vm/oops/generateOopMap.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/generateOopMap.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -370,21 +370,8 @@ void GenerateOopMap ::initialize_bb() {
void GenerateOopMap ::initialize_bb() {
_gc_points = 0;
_bb_count = 0;
- int size = binsToHold(method()->code_size());
- _bb_hdr_bits = NEW_RESOURCE_ARRAY(uintptr_t,size);
- memset(_bb_hdr_bits, 0, size*sizeof(uintptr_t));
-}
-
-void GenerateOopMap ::set_bbmark_bit(int bci) {
- int idx = bci >> LogBitsPerWord;
- uintptr_t bit = (uintptr_t)1 << (bci & (BitsPerWord-1));
- _bb_hdr_bits[idx] |= bit;
-}
-
-void GenerateOopMap ::clear_bbmark_bit(int bci) {
- int idx = bci >> LogBitsPerWord;
- uintptr_t bit = (uintptr_t)1 << (bci & (BitsPerWord-1));
- _bb_hdr_bits[idx] &= (~bit);
+ _bb_hdr_bits.clear();
+ _bb_hdr_bits.resize(method()->code_size());
}
void GenerateOopMap::bb_mark_fct(GenerateOopMap *c, int bci, int *data) {
@@ -952,39 +939,6 @@ void GenerateOopMap::init_basic_blocks()
_basic_blocks[bbNo-1]._end_bci = prev_bci;
- _max_monitors = monitor_count;
-
- // Now that we have a bound on the depth of the monitor stack, we can
- // initialize the CellTypeState-related information.
- init_state();
-
- // We allocate space for all state-vectors for all basicblocks in one huge chuck.
- // Then in the next part of the code, we set a pointer in each _basic_block that
- // points to each piece.
- CellTypeState *basicBlockState = NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
- memset(basicBlockState, 0, bbNo * _state_len * sizeof(CellTypeState));
-
- // Make a pass over the basicblocks and assign their state vectors.
- for (int blockNum=0; blockNum < bbNo; blockNum++) {
- BasicBlock *bb = _basic_blocks + blockNum;
- bb->_state = basicBlockState + blockNum * _state_len;
-
-#ifdef ASSERT
- if (blockNum + 1 < bbNo) {
- address bcp = _method->bcp_from(bb->_end_bci);
- int bc_len = Bytecodes::java_length_at(bcp);
- assert(bb->_end_bci + bc_len == bb[1]._bci, "unmatched bci info in basicblock");
- }
-#endif
- }
-#ifdef ASSERT
- { BasicBlock *bb = &_basic_blocks[bbNo-1];
- address bcp = _method->bcp_from(bb->_end_bci);
- int bc_len = Bytecodes::java_length_at(bcp);
- assert(bb->_end_bci + bc_len == _method->code_size(), "wrong end bci");
- }
-#endif
-
// Check that the correct number of basicblocks was found
if (bbNo !=_bb_count) {
if (bbNo < _bb_count) {
@@ -996,6 +950,39 @@ void GenerateOopMap::init_basic_blocks()
}
}
+ _max_monitors = monitor_count;
+
+ // Now that we have a bound on the depth of the monitor stack, we can
+ // initialize the CellTypeState-related information.
+ init_state();
+
+ // We allocate space for all state-vectors for all basicblocks in one huge chuck.
+ // Then in the next part of the code, we set a pointer in each _basic_block that
+ // points to each piece.
+ CellTypeState *basicBlockState = NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
+ memset(basicBlockState, 0, bbNo * _state_len * sizeof(CellTypeState));
+
+ // Make a pass over the basicblocks and assign their state vectors.
+ for (int blockNum=0; blockNum < bbNo; blockNum++) {
+ BasicBlock *bb = _basic_blocks + blockNum;
+ bb->_state = basicBlockState + blockNum * _state_len;
+
+#ifdef ASSERT
+ if (blockNum + 1 < bbNo) {
+ address bcp = _method->bcp_from(bb->_end_bci);
+ int bc_len = Bytecodes::java_length_at(bcp);
+ assert(bb->_end_bci + bc_len == bb[1]._bci, "unmatched bci info in basicblock");
+ }
+#endif
+ }
+#ifdef ASSERT
+ { BasicBlock *bb = &_basic_blocks[bbNo-1];
+ address bcp = _method->bcp_from(bb->_end_bci);
+ int bc_len = Bytecodes::java_length_at(bcp);
+ assert(bb->_end_bci + bc_len == _method->code_size(), "wrong end bci");
+ }
+#endif
+
// Mark all alive blocks
mark_reachable_code();
}
@@ -1022,21 +1009,22 @@ void GenerateOopMap::update_basic_blocks
int new_method_size) {
assert(new_method_size >= method()->code_size() + delta,
"new method size is too small");
- int newWords = binsToHold(new_method_size);
-
- uintptr_t * new_bb_hdr_bits = NEW_RESOURCE_ARRAY(uintptr_t, newWords);
-
- BitMap bb_bits(new_bb_hdr_bits, new_method_size);
- bb_bits.clear();
+
+ BitMap::bm_word_t* new_bb_hdr_bits =
+ NEW_RESOURCE_ARRAY(BitMap::bm_word_t,
+ BitMap::word_align_up(new_method_size));
+ _bb_hdr_bits.set_map(new_bb_hdr_bits);
+ _bb_hdr_bits.set_size(new_method_size);
+ _bb_hdr_bits.clear();
+
for(int k = 0; k < _bb_count; k++) {
if (_basic_blocks[k]._bci > bci) {
_basic_blocks[k]._bci += delta;
_basic_blocks[k]._end_bci += delta;
}
- bb_bits.at_put(_basic_blocks[k]._bci, true);
- }
- _bb_hdr_bits = new_bb_hdr_bits ;
+ _bb_hdr_bits.at_put(_basic_blocks[k]._bci, true);
+ }
}
//
--- a/src/share/vm/oops/generateOopMap.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/generateOopMap.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -341,16 +341,22 @@ class GenerateOopMap VALUE_OBJ_CLASS_SPE
BasicBlock * _basic_blocks; // Array of basicblock info
int _gc_points;
int _bb_count;
- uintptr_t * _bb_hdr_bits;
+ BitMap _bb_hdr_bits;
// Basicblocks methods
void initialize_bb ();
void mark_bbheaders_and_count_gc_points();
- bool is_bb_header (int bci) const { return (_bb_hdr_bits[bci >> LogBitsPerWord] & ((uintptr_t)1 << (bci & (BitsPerWord-1)))) != 0; }
+ bool is_bb_header (int bci) const {
+ return _bb_hdr_bits.at(bci);
+ }
int gc_points () const { return _gc_points; }
int bb_count () const { return _bb_count; }
- void set_bbmark_bit (int bci);
- void clear_bbmark_bit (int bci);
+ void set_bbmark_bit (int bci) {
+ _bb_hdr_bits.at_put(bci, true);
+ }
+ void clear_bbmark_bit (int bci) {
+ _bb_hdr_bits.at_put(bci, false);
+ }
BasicBlock * get_basic_block_at (int bci) const;
BasicBlock * get_basic_block_containing (int bci) const;
void interp_bb (BasicBlock *bb);
--- a/src/share/vm/oops/instanceKlass.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/instanceKlass.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -1515,10 +1515,9 @@ void instanceKlass::oop_follow_contents(
// closure's do_header() method dicates whether the given closure should be
// applied to the klass ptr in the object header.
-#define InstanceKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \
- \
-int instanceKlass::oop_oop_iterate##nv_suffix(oop obj, \
- OopClosureType* closure) {\
+#define InstanceKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \
+ \
+int instanceKlass::oop_oop_iterate##nv_suffix(oop obj, OopClosureType* closure) { \
SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\
/* header */ \
if (closure->do_header()) { \
@@ -1532,6 +1531,26 @@ int instanceKlass::oop_oop_iterate##nv_s
assert_is_in_closed_subset) \
return size_helper(); \
}
+
+#ifndef SERIALGC
+#define InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN(OopClosureType, nv_suffix) \
+ \
+int instanceKlass::oop_oop_iterate_backwards##nv_suffix(oop obj, \
+ OopClosureType* closure) { \
+ SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik); \
+ /* header */ \
+ if (closure->do_header()) { \
+ obj->oop_iterate_header(closure); \
+ } \
+ /* instance variables */ \
+ InstanceKlass_OOP_MAP_REVERSE_ITERATE( \
+ obj, \
+ SpecializationStats::record_do_oop_call##nv_suffix(SpecializationStats::ik);\
+ (closure)->do_oop##nv_suffix(p), \
+ assert_is_in_closed_subset) \
+ return size_helper(); \
+}
+#endif // !SERIALGC
#define InstanceKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \
\
@@ -1550,9 +1569,13 @@ int instanceKlass::oop_oop_iterate##nv_s
}
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN)
-ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceKlass_OOP_OOP_ITERATE_DEFN)
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN_m)
-ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN_m)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceKlass_OOP_OOP_ITERATE_DEFN_m)
+#ifndef SERIALGC
+ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN)
+#endif // !SERIALGC
void instanceKlass::iterate_static_fields(OopClosure* closure) {
InstanceKlass_OOP_ITERATE( \
--- a/src/share/vm/oops/instanceKlass.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/instanceKlass.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -655,13 +655,21 @@ class instanceKlass: public Klass {
return oop_oop_iterate_v_m(obj, blk, mr);
}
-#define InstanceKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \
- int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \
- int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \
+#define InstanceKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \
+ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \
+ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \
MemRegion mr);
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DECL)
- ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DECL)
+ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceKlass_OOP_OOP_ITERATE_DECL)
+
+#ifndef SERIALGC
+#define InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DECL(OopClosureType, nv_suffix) \
+ int oop_oop_iterate_backwards##nv_suffix(oop obj, OopClosureType* blk);
+
+ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceKlass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+#endif // !SERIALGC
void iterate_static_fields(OopClosure* closure);
void iterate_static_fields(OopClosure* closure, MemRegion mr);
--- a/src/share/vm/oops/instanceRefKlass.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/instanceRefKlass.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -176,6 +176,11 @@ int instanceRefKlass::oop_adjust_pointer
}
#define InstanceRefKlass_SPECIALIZED_OOP_ITERATE(T, nv_suffix, contains) \
+ if (closure->apply_to_weak_ref_discovered_field()) { \
+ T* disc_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); \
+ closure->do_oop##nv_suffix(disc_addr); \
+ } \
+ \
T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); \
oop referent = oopDesc::load_decode_heap_oop(referent_addr); \
if (referent != NULL && contains(referent_addr)) { \
@@ -219,6 +224,25 @@ oop_oop_iterate##nv_suffix(oop obj, OopC
} \
}
+#ifndef SERIALGC
+#define InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN(OopClosureType, nv_suffix) \
+ \
+int instanceRefKlass:: \
+oop_oop_iterate_backwards##nv_suffix(oop obj, OopClosureType* closure) { \
+ /* Get size before changing pointers */ \
+ SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::irk);\
+ \
+ int size = instanceKlass::oop_oop_iterate_backwards##nv_suffix(obj, closure); \
+ \
+ if (UseCompressedOops) { \
+ InstanceRefKlass_SPECIALIZED_OOP_ITERATE(narrowOop, nv_suffix, contains); \
+ } else { \
+ InstanceRefKlass_SPECIALIZED_OOP_ITERATE(oop, nv_suffix, contains); \
+ } \
+}
+#endif // !SERIALGC
+
+
#define InstanceRefKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \
\
int instanceRefKlass:: \
@@ -236,9 +260,13 @@ oop_oop_iterate##nv_suffix##_m(oop obj,
}
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceRefKlass_OOP_OOP_ITERATE_DEFN)
-ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceRefKlass_OOP_OOP_ITERATE_DEFN)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceRefKlass_OOP_OOP_ITERATE_DEFN)
+#ifndef SERIALGC
+ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN)
+#endif // SERIALGC
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceRefKlass_OOP_OOP_ITERATE_DEFN_m)
-ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceRefKlass_OOP_OOP_ITERATE_DEFN_m)
+ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceRefKlass_OOP_OOP_ITERATE_DEFN_m)
#ifndef SERIALGC
template <class T>
@@ -423,7 +451,7 @@ void instanceRefKlass::oop_verify_on(oop
// Verify next field
oop next = java_lang_ref_Reference::next(obj);
if (next != NULL) {
- guarantee(next->is_oop(), "next field verify fa iled");
+ guarantee(next->is_oop(), "next field verify failed");
guarantee(next->is_instanceRef(), "next field verify failed");
if (gch != NULL && !gch->is_in_youngest(obj)) {
// We do a specific remembered set check here since the next field is
--- a/src/share/vm/oops/instanceRefKlass.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/instanceRefKlass.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -72,7 +72,15 @@ class instanceRefKlass: public instanceK
int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, MemRegion mr);
ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceRefKlass_OOP_OOP_ITERATE_DECL)
- ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceRefKlass_OOP_OOP_ITERATE_DECL)
+ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceRefKlass_OOP_OOP_ITERATE_DECL)
+
+#ifndef SERIALGC
+#define InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DECL(OopClosureType, nv_suffix) \
+ int oop_oop_iterate_backwards##nv_suffix(oop obj, OopClosureType* blk);
+
+ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceRefKlass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+#endif // !SERIALGC
static void release_and_notify_pending_list_lock(BasicLock *pending_list_basic_lock);
static void acquire_pending_list_lock(BasicLock *pending_list_basic_lock);
--- a/src/share/vm/oops/klass.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/klass.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -134,14 +134,14 @@ class Klass_vtbl {
// Every subclass on which vtbl_value is called must include this macro.
// Delay the installation of the klassKlass pointer until after the
// the vtable for a new klass has been installed (after the call to new()).
-#define DEFINE_ALLOCATE_PERMANENT(thisKlass) \
+#define DEFINE_ALLOCATE_PERMANENT(thisKlass) \
void* allocate_permanent(KlassHandle& klass_klass, int size, TRAPS) const { \
- void* result = new(klass_klass, size, THREAD) thisKlass(); \
- if (HAS_PENDING_EXCEPTION) return NULL; \
- klassOop new_klass = ((Klass*) result)->as_klassOop(); \
- OrderAccess::storestore(); \
- post_new_init_klass(klass_klass, new_klass, size); \
- return result; \
+ void* result = new(klass_klass, size, THREAD) thisKlass(); \
+ if (HAS_PENDING_EXCEPTION) return NULL; \
+ klassOop new_klass = ((Klass*) result)->as_klassOop(); \
+ OrderAccess::storestore(); \
+ post_new_init_klass(klass_klass, new_klass, size); \
+ return result; \
}
bool null_vtbl() { return *(intptr_t*)this == 0; }
@@ -694,6 +694,14 @@ class Klass : public Klass_vtbl {
return oop_oop_iterate(obj, blk);
}
+#ifndef SERIALGC
+ // In case we don't have a specialized backward scanner use forward
+ // iteration.
+ virtual int oop_oop_iterate_backwards_v(oop obj, OopClosure* blk) {
+ return oop_oop_iterate_v(obj, blk);
+ }
+#endif // !SERIALGC
+
// Iterates "blk" over all the oops in "obj" (of type "this") within "mr".
// (I don't see why the _m should be required, but without it the Solaris
// C++ gives warning messages about overridings of the "oop_oop_iterate"
@@ -722,7 +730,19 @@ class Klass : public Klass_vtbl {
}
SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(Klass_OOP_OOP_ITERATE_DECL)
- SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(Klass_OOP_OOP_ITERATE_DECL)
+ SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(Klass_OOP_OOP_ITERATE_DECL)
+
+#ifndef SERIALGC
+#define Klass_OOP_OOP_ITERATE_BACKWARDS_DECL(OopClosureType, nv_suffix) \
+ virtual int oop_oop_iterate_backwards##nv_suffix(oop obj, \
+ OopClosureType* blk) { \
+ /* Default implementation reverts to general version. */ \
+ return oop_oop_iterate_backwards_v(obj, blk); \
+ }
+
+ SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(Klass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+ SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(Klass_OOP_OOP_ITERATE_BACKWARDS_DECL)
+#endif // !SERIALGC
virtual void array_klasses_do(void f(klassOop k)) {}
virtual void with_array_klasses_do(void f(klassOop k));
--- a/src/share/vm/oops/markOop.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/markOop.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -222,11 +222,7 @@ class markOopDesc: public oopDesc {
static markOop INFLATING() { return (markOop) 0; } // inflate-in-progress
// Should this header be preserved during GC?
- bool must_be_preserved(oop obj_containing_mark) const {
- if (!UseBiasedLocking)
- return (!is_unlocked() || !has_no_hash());
- return must_be_preserved_with_bias(obj_containing_mark);
- }
+ inline bool must_be_preserved(oop obj_containing_mark) const;
inline bool must_be_preserved_with_bias(oop obj_containing_mark) const;
// Should this header (including its age bits) be preserved in the
@@ -246,22 +242,14 @@ class markOopDesc: public oopDesc {
// observation is that promotion failures are quite rare and
// reducing the number of mark words preserved during them isn't a
// high priority.
- bool must_be_preserved_for_promotion_failure(oop obj_containing_mark) const {
- if (!UseBiasedLocking)
- return (this != prototype());
- return must_be_preserved_with_bias_for_promotion_failure(obj_containing_mark);
- }
+ inline bool must_be_preserved_for_promotion_failure(oop obj_containing_mark) const;
inline bool must_be_preserved_with_bias_for_promotion_failure(oop obj_containing_mark) const;
// Should this header be preserved during a scavenge where CMS is
// the old generation?
// (This is basically the same body as must_be_preserved_for_promotion_failure(),
// but takes the klassOop as argument instead)
- bool must_be_preserved_for_cms_scavenge(klassOop klass_of_obj_containing_mark) const {
- if (!UseBiasedLocking)
- return (this != prototype());
- return must_be_preserved_with_bias_for_cms_scavenge(klass_of_obj_containing_mark);
- }
+ inline bool must_be_preserved_for_cms_scavenge(klassOop klass_of_obj_containing_mark) const;
inline bool must_be_preserved_with_bias_for_cms_scavenge(klassOop klass_of_obj_containing_mark) const;
// WARNING: The following routines are used EXCLUSIVELY by
--- a/src/share/vm/oops/markOop.inline.hpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/markOop.inline.hpp Tue Sep 30 12:24:27 2008 -0400
@@ -39,6 +39,12 @@ inline bool markOopDesc::must_be_preserv
return (!is_unlocked() || !has_no_hash());
}
+inline bool markOopDesc::must_be_preserved(oop obj_containing_mark) const {
+ if (!UseBiasedLocking)
+ return (!is_unlocked() || !has_no_hash());
+ return must_be_preserved_with_bias(obj_containing_mark);
+}
+
// Should this header (including its age bits) be preserved in the
// case of a promotion failure during scavenge?
inline bool markOopDesc::must_be_preserved_with_bias_for_promotion_failure(oop obj_containing_mark) const {
@@ -59,6 +65,13 @@ inline bool markOopDesc::must_be_preserv
return (this != prototype());
}
+inline bool markOopDesc::must_be_preserved_for_promotion_failure(oop obj_containing_mark) const {
+ if (!UseBiasedLocking)
+ return (this != prototype());
+ return must_be_preserved_with_bias_for_promotion_failure(obj_containing_mark);
+}
+
+
// Should this header (including its age bits) be preserved in the
// case of a scavenge in which CMS is the old generation?
inline bool markOopDesc::must_be_preserved_with_bias_for_cms_scavenge(klassOop klass_of_obj_containing_mark) const {
@@ -70,6 +83,11 @@ inline bool markOopDesc::must_be_preserv
}
return (this != prototype());
}
+inline bool markOopDesc::must_be_preserved_for_cms_scavenge(klassOop klass_of_obj_containing_mark) const {
+ if (!UseBiasedLocking)
+ return (this != prototype());
+ return must_be_preserved_with_bias_for_cms_scavenge(klass_of_obj_containing_mark);
+}
inline markOop markOopDesc::prototype_for_object(oop obj) {
#ifdef ASSERT
--- a/src/share/vm/oops/objArrayKlass.cpp Fri Sep 26 13:33:15 2008 -0400
+++ b/src/share/vm/oops/objArrayKlass.cpp Tue Sep 30 12:24:27 2008 -0400
@@ -86,14 +86,18 @@ template <class T> void objArrayKlass::d
const size_t word_len = objArrayOopDesc::array_size(length);
- // For performance reasons, we assume we are using a card marking write
- // barrier. The assert will fail if this is not the case.
BarrierSet* bs = Universe::heap()->barrier_set();
+ // For performance reasons, we assume we are that the write barrier we
+ // are using has optimized modes for arrays of references. At least one
+ // of the asserts below will fail if this is not the case.
assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt");
-
+ assert(bs->has_write_ref_array_pre_opt(), "For pre-barrier as well.");
+
+ MemRegion dst_mr = MemRegion((HeapWord*)dst, word_len);
if (s == d) {
// since source and destination are equal we do not need conversion checks.
assert(length > 0, "sanity check");
+ bs->write_ref_array_pre(dst_mr);
Copy::conjoint_oops_atomic(src, dst, length);
} else {
// We have to make sure all elements conform to the destination array
@@ -101,6 +105,7 @@ template <class T> void objArrayKlass::d
klassOop stype = objArrayKlass::cast(s->klass())->element_klass();
if (stype == bound || Klass::cast(stype)->is_subtype_of(bound)) {
// elements are guaranteed to be subtypes, so no check necessary
+ bs->write_ref_array_pre(dst_mr);
Copy::conjoint_oops_atomic(src, dst, length);
} else {
// slow case: need individual subtype checks
@@ -110,8 +115,13 @@ template <class T> void objArrayKlass::d
for (T* p = dst; from < end; from++, p++) {
// XXX this is going to be slow.
T element = *from;
- if (oopDesc::is_null(element) ||
- Klass::cast(oopDesc::decode_heap_oop_not_null(element)->klass())->is_subtype_of(bound)) {