annotate src/share/vm/runtime/sweeper.cpp @ 1202:5f24d0319e54

4360113: Evict nmethods when code cache gets full Summary: Speculatively unload the oldest nmethods when code cache gets full. Reviewed-by: never, kvn Contributed-by: eric.caspole@amd.com
author kvn
date Fri, 29 Jan 2010 09:27:22 -0800
parents a1423fe86a18
children bfe29ec02863
rev   line source
duke@0 1 /*
xdono@948 2 * Copyright 1997-2009 Sun Microsystems, Inc. All Rights Reserved.
duke@0 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@0 4 *
duke@0 5 * This code is free software; you can redistribute it and/or modify it
duke@0 6 * under the terms of the GNU General Public License version 2 only, as
duke@0 7 * published by the Free Software Foundation.
duke@0 8 *
duke@0 9 * This code is distributed in the hope that it will be useful, but WITHOUT
duke@0 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@0 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@0 12 * version 2 for more details (a copy is included in the LICENSE file that
duke@0 13 * accompanied this code).
duke@0 14 *
duke@0 15 * You should have received a copy of the GNU General Public License version
duke@0 16 * 2 along with this work; if not, write to the Free Software Foundation,
duke@0 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@0 18 *
duke@0 19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@0 20 * CA 95054 USA or visit www.sun.com if you need additional information or
duke@0 21 * have any questions.
duke@0 22 *
duke@0 23 */
duke@0 24
duke@0 25 # include "incls/_precompiled.incl"
duke@0 26 # include "incls/_sweeper.cpp.incl"
duke@0 27
duke@0 28 long NMethodSweeper::_traversals = 0; // No. of stack traversals performed
duke@0 29 CodeBlob* NMethodSweeper::_current = NULL; // Current nmethod
duke@0 30 int NMethodSweeper::_seen = 0 ; // No. of blobs we have currently processed in current pass of CodeCache
duke@0 31 int NMethodSweeper::_invocations = 0; // No. of invocations left until we are completed with this pass
duke@0 32
duke@0 33 jint NMethodSweeper::_locked_seen = 0;
duke@0 34 jint NMethodSweeper::_not_entrant_seen_on_stack = 0;
duke@0 35 bool NMethodSweeper::_rescan = false;
kvn@1202 36 bool NMethodSweeper::_was_full = false;
kvn@1202 37 jint NMethodSweeper::_advise_to_sweep = 0;
kvn@1202 38 jlong NMethodSweeper::_last_was_full = 0;
kvn@1202 39 uint NMethodSweeper::_highest_marked = 0;
kvn@1202 40 long NMethodSweeper::_was_full_traversal = 0;
duke@0 41
jrose@989 42 class MarkActivationClosure: public CodeBlobClosure {
jrose@989 43 public:
jrose@989 44 virtual void do_code_blob(CodeBlob* cb) {
jrose@989 45 // If we see an activation belonging to a non_entrant nmethod, we mark it.
jrose@989 46 if (cb->is_nmethod() && ((nmethod*)cb)->is_not_entrant()) {
jrose@989 47 ((nmethod*)cb)->mark_as_seen_on_stack();
jrose@989 48 }
jrose@989 49 }
jrose@989 50 };
jrose@989 51 static MarkActivationClosure mark_activation_closure;
jrose@989 52
duke@0 53 void NMethodSweeper::sweep() {
duke@0 54 assert(SafepointSynchronize::is_at_safepoint(), "must be executed at a safepoint");
duke@0 55 if (!MethodFlushing) return;
duke@0 56
duke@0 57 // No need to synchronize access, since this is always executed at a
duke@0 58 // safepoint. If we aren't in the middle of scan and a rescan
duke@0 59 // hasn't been requested then just return.
duke@0 60 if (_current == NULL && !_rescan) return;
duke@0 61
duke@0 62 // Make sure CompiledIC_lock in unlocked, since we might update some
duke@0 63 // inline caches. If it is, we just bail-out and try later.
duke@0 64 if (CompiledIC_lock->is_locked() || Patching_lock->is_locked()) return;
duke@0 65
duke@0 66 // Check for restart
duke@0 67 assert(CodeCache::find_blob_unsafe(_current) == _current, "Sweeper nmethod cached state invalid");
duke@0 68 if (_current == NULL) {
duke@0 69 _seen = 0;
duke@0 70 _invocations = NmethodSweepFraction;
duke@0 71 _current = CodeCache::first();
duke@0 72 _traversals += 1;
duke@0 73 if (PrintMethodFlushing) {
duke@0 74 tty->print_cr("### Sweep: stack traversal %d", _traversals);
duke@0 75 }
jrose@989 76 Threads::nmethods_do(&mark_activation_closure);
duke@0 77
duke@0 78 // reset the flags since we started a scan from the beginning.
duke@0 79 _rescan = false;
duke@0 80 _locked_seen = 0;
duke@0 81 _not_entrant_seen_on_stack = 0;
duke@0 82 }
duke@0 83
duke@0 84 if (PrintMethodFlushing && Verbose) {
duke@0 85 tty->print_cr("### Sweep at %d out of %d. Invocations left: %d", _seen, CodeCache::nof_blobs(), _invocations);
duke@0 86 }
duke@0 87
duke@0 88 // We want to visit all nmethods after NmethodSweepFraction invocations.
duke@0 89 // If invocation is 1 we do the rest
duke@0 90 int todo = CodeCache::nof_blobs();
duke@0 91 if (_invocations != 1) {
duke@0 92 todo = (CodeCache::nof_blobs() - _seen) / _invocations;
duke@0 93 _invocations--;
duke@0 94 }
duke@0 95
duke@0 96 for(int i = 0; i < todo && _current != NULL; i++) {
duke@0 97 CodeBlob* next = CodeCache::next(_current); // Read next before we potentially delete current
duke@0 98 if (_current->is_nmethod()) {
duke@0 99 process_nmethod((nmethod *)_current);
duke@0 100 }
duke@0 101 _seen++;
duke@0 102 _current = next;
duke@0 103 }
duke@0 104 // Because we could stop on a codeBlob other than an nmethod we skip forward
duke@0 105 // to the next nmethod (if any). codeBlobs other than nmethods can be freed
duke@0 106 // async to us and make _current invalid while we sleep.
duke@0 107 while (_current != NULL && !_current->is_nmethod()) {
duke@0 108 _current = CodeCache::next(_current);
duke@0 109 }
duke@0 110
duke@0 111 if (_current == NULL && !_rescan && (_locked_seen || _not_entrant_seen_on_stack)) {
duke@0 112 // we've completed a scan without making progress but there were
duke@0 113 // nmethods we were unable to process either because they were
duke@0 114 // locked or were still on stack. We don't have to aggresively
duke@0 115 // clean them up so just stop scanning. We could scan once more
duke@0 116 // but that complicates the control logic and it's unlikely to
duke@0 117 // matter much.
duke@0 118 if (PrintMethodFlushing) {
duke@0 119 tty->print_cr("### Couldn't make progress on some nmethods so stopping sweep");
duke@0 120 }
duke@0 121 }
kvn@1202 122
kvn@1202 123 if (UseCodeCacheFlushing) {
kvn@1202 124 if (!CodeCache::needs_flushing()) {
kvn@1202 125 // In a safepoint, no race with setters
kvn@1202 126 _advise_to_sweep = 0;
kvn@1202 127 }
kvn@1202 128
kvn@1202 129 if (was_full()) {
kvn@1202 130 // There was some progress so attempt to restart the compiler
kvn@1202 131 jlong now = os::javaTimeMillis();
kvn@1202 132 jlong max_interval = (jlong)MinCodeCacheFlushingInterval * (jlong)1000;
kvn@1202 133 jlong curr_interval = now - _last_was_full;
kvn@1202 134 if ((!CodeCache::needs_flushing()) && (curr_interval > max_interval)) {
kvn@1202 135 CompileBroker::set_should_compile_new_jobs(CompileBroker::run_compilation);
kvn@1202 136 set_was_full(false);
kvn@1202 137
kvn@1202 138 // Update the _last_was_full time so we can tell how fast the
kvn@1202 139 // code cache is filling up
kvn@1202 140 _last_was_full = os::javaTimeMillis();
kvn@1202 141
kvn@1202 142 if (PrintMethodFlushing) {
kvn@1202 143 tty->print_cr("### sweeper: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes, restarting compiler",
kvn@1202 144 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 145 }
kvn@1202 146 if (LogCompilation && (xtty != NULL)) {
kvn@1202 147 ttyLocker ttyl;
kvn@1202 148 xtty->begin_elem("restart_compiler live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
kvn@1202 149 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 150 xtty->stamp();
kvn@1202 151 xtty->end_elem();
kvn@1202 152 }
kvn@1202 153 }
kvn@1202 154 }
kvn@1202 155 }
duke@0 156 }
duke@0 157
duke@0 158
duke@0 159 void NMethodSweeper::process_nmethod(nmethod *nm) {
duke@0 160 // Skip methods that are currently referenced by the VM
duke@0 161 if (nm->is_locked_by_vm()) {
duke@0 162 // But still remember to clean-up inline caches for alive nmethods
duke@0 163 if (nm->is_alive()) {
duke@0 164 // Clean-up all inline caches that points to zombie/non-reentrant methods
duke@0 165 nm->cleanup_inline_caches();
duke@0 166 } else {
duke@0 167 _locked_seen++;
duke@0 168 }
duke@0 169 return;
duke@0 170 }
duke@0 171
duke@0 172 if (nm->is_zombie()) {
duke@0 173 // If it is first time, we see nmethod then we mark it. Otherwise,
duke@0 174 // we reclame it. When we have seen a zombie method twice, we know that
duke@0 175 // there are no inline caches that referes to it.
duke@0 176 if (nm->is_marked_for_reclamation()) {
duke@0 177 assert(!nm->is_locked_by_vm(), "must not flush locked nmethods");
ysr@941 178 if (PrintMethodFlushing && Verbose) {
kvn@1202 179 tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (marked for reclamation) being flushed", nm->compile_id(), nm);
ysr@941 180 }
duke@0 181 nm->flush();
duke@0 182 } else {
ysr@941 183 if (PrintMethodFlushing && Verbose) {
kvn@1202 184 tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (zombie) being marked for reclamation", nm->compile_id(), nm);
ysr@941 185 }
duke@0 186 nm->mark_for_reclamation();
duke@0 187 _rescan = true;
duke@0 188 }
duke@0 189 } else if (nm->is_not_entrant()) {
duke@0 190 // If there is no current activations of this method on the
duke@0 191 // stack we can safely convert it to a zombie method
duke@0 192 if (nm->can_not_entrant_be_converted()) {
ysr@941 193 if (PrintMethodFlushing && Verbose) {
kvn@1202 194 tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (not entrant) being made zombie", nm->compile_id(), nm);
ysr@941 195 }
duke@0 196 nm->make_zombie();
duke@0 197 _rescan = true;
duke@0 198 } else {
duke@0 199 // Still alive, clean up its inline caches
duke@0 200 nm->cleanup_inline_caches();
duke@0 201 // we coudn't transition this nmethod so don't immediately
duke@0 202 // request a rescan. If this method stays on the stack for a
duke@0 203 // long time we don't want to keep rescanning at every safepoint.
duke@0 204 _not_entrant_seen_on_stack++;
duke@0 205 }
duke@0 206 } else if (nm->is_unloaded()) {
duke@0 207 // Unloaded code, just make it a zombie
ysr@941 208 if (PrintMethodFlushing && Verbose)
kvn@1202 209 tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (unloaded) being made zombie", nm->compile_id(), nm);
ysr@941 210 if (nm->is_osr_method()) {
duke@0 211 // No inline caches will ever point to osr methods, so we can just remove it
duke@0 212 nm->flush();
duke@0 213 } else {
duke@0 214 nm->make_zombie();
duke@0 215 _rescan = true;
duke@0 216 }
duke@0 217 } else {
duke@0 218 assert(nm->is_alive(), "should be alive");
kvn@1202 219
kvn@1202 220 if (UseCodeCacheFlushing) {
kvn@1202 221 if ((nm->method()->code() != nm) && !(nm->is_locked_by_vm()) && !(nm->is_osr_method()) &&
kvn@1202 222 (_traversals > _was_full_traversal+2) && (((uint)nm->compile_id()) < _highest_marked) &&
kvn@1202 223 CodeCache::needs_flushing()) {
kvn@1202 224 // This method has not been called since the forced cleanup happened
kvn@1202 225 nm->make_not_entrant();
kvn@1202 226 }
kvn@1202 227 }
kvn@1202 228
duke@0 229 // Clean-up all inline caches that points to zombie/non-reentrant methods
duke@0 230 nm->cleanup_inline_caches();
duke@0 231 }
duke@0 232 }
kvn@1202 233
kvn@1202 234 // Code cache unloading: when compilers notice the code cache is getting full,
kvn@1202 235 // they will call a vm op that comes here. This code attempts to speculatively
kvn@1202 236 // unload the oldest half of the nmethods (based on the compile job id) by
kvn@1202 237 // saving the old code in a list in the CodeCache. Then
kvn@1202 238 // execution resumes. If a method so marked is not called by the second
kvn@1202 239 // safepoint from the current one, the nmethod will be marked non-entrant and
kvn@1202 240 // got rid of by normal sweeping. If the method is called, the methodOop's
kvn@1202 241 // _code field is restored and the methodOop/nmethod
kvn@1202 242 // go back to their normal state.
kvn@1202 243 void NMethodSweeper::handle_full_code_cache(bool is_full) {
kvn@1202 244 // Only the first one to notice can advise us to start early cleaning
kvn@1202 245 if (!is_full){
kvn@1202 246 jint old = Atomic::cmpxchg( 1, &_advise_to_sweep, 0 );
kvn@1202 247 if (old != 0) {
kvn@1202 248 return;
kvn@1202 249 }
kvn@1202 250 }
kvn@1202 251
kvn@1202 252 if (is_full) {
kvn@1202 253 // Since code cache is full, immediately stop new compiles
kvn@1202 254 bool did_set = CompileBroker::set_should_compile_new_jobs(CompileBroker::stop_compilation);
kvn@1202 255 if (!did_set) {
kvn@1202 256 // only the first to notice can start the cleaning,
kvn@1202 257 // others will go back and block
kvn@1202 258 return;
kvn@1202 259 }
kvn@1202 260 set_was_full(true);
kvn@1202 261
kvn@1202 262 // If we run out within MinCodeCacheFlushingInterval of the last unload time, give up
kvn@1202 263 jlong now = os::javaTimeMillis();
kvn@1202 264 jlong max_interval = (jlong)MinCodeCacheFlushingInterval * (jlong)1000;
kvn@1202 265 jlong curr_interval = now - _last_was_full;
kvn@1202 266 if (curr_interval < max_interval) {
kvn@1202 267 _rescan = true;
kvn@1202 268 if (PrintMethodFlushing) {
kvn@1202 269 tty->print_cr("### handle full too often, turning off compiler");
kvn@1202 270 }
kvn@1202 271 if (LogCompilation && (xtty != NULL)) {
kvn@1202 272 ttyLocker ttyl;
kvn@1202 273 xtty->begin_elem("disable_compiler flushing_interval='" UINT64_FORMAT "' live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
kvn@1202 274 curr_interval/1000, CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 275 xtty->stamp();
kvn@1202 276 xtty->end_elem();
kvn@1202 277 }
kvn@1202 278 return;
kvn@1202 279 }
kvn@1202 280 }
kvn@1202 281
kvn@1202 282 VM_HandleFullCodeCache op(is_full);
kvn@1202 283 VMThread::execute(&op);
kvn@1202 284
kvn@1202 285 // rescan again as soon as possible
kvn@1202 286 _rescan = true;
kvn@1202 287 }
kvn@1202 288
kvn@1202 289 void NMethodSweeper::speculative_disconnect_nmethods(bool is_full) {
kvn@1202 290 // If there was a race in detecting full code cache, only run
kvn@1202 291 // one vm op for it or keep the compiler shut off
kvn@1202 292
kvn@1202 293 debug_only(jlong start = os::javaTimeMillis();)
kvn@1202 294
kvn@1202 295 if ((!was_full()) && (is_full)) {
kvn@1202 296 if (!CodeCache::needs_flushing()) {
kvn@1202 297 if (PrintMethodFlushing) {
kvn@1202 298 tty->print_cr("### sweeper: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes, restarting compiler",
kvn@1202 299 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 300 }
kvn@1202 301 if (LogCompilation && (xtty != NULL)) {
kvn@1202 302 ttyLocker ttyl;
kvn@1202 303 xtty->begin_elem("restart_compiler live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
kvn@1202 304 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 305 xtty->stamp();
kvn@1202 306 xtty->end_elem();
kvn@1202 307 }
kvn@1202 308 CompileBroker::set_should_compile_new_jobs(CompileBroker::run_compilation);
kvn@1202 309 return;
kvn@1202 310 }
kvn@1202 311 }
kvn@1202 312
kvn@1202 313 // Traverse the code cache trying to dump the oldest nmethods
kvn@1202 314 uint curr_max_comp_id = CompileBroker::get_compilation_id();
kvn@1202 315 uint flush_target = ((curr_max_comp_id - _highest_marked) >> 1) + _highest_marked;
kvn@1202 316 if (PrintMethodFlushing && Verbose) {
kvn@1202 317 tty->print_cr("### Cleaning code cache: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes",
kvn@1202 318 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 319 }
kvn@1202 320 if (LogCompilation && (xtty != NULL)) {
kvn@1202 321 ttyLocker ttyl;
kvn@1202 322 xtty->begin_elem("start_cleaning_code_cache live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
kvn@1202 323 CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 324 xtty->stamp();
kvn@1202 325 xtty->end_elem();
kvn@1202 326 }
kvn@1202 327
kvn@1202 328 nmethod* nm = CodeCache::alive_nmethod(CodeCache::first());
kvn@1202 329 jint disconnected = 0;
kvn@1202 330 jint made_not_entrant = 0;
kvn@1202 331 while ((nm != NULL)){
kvn@1202 332 uint curr_comp_id = nm->compile_id();
kvn@1202 333
kvn@1202 334 // OSR methods cannot be flushed like this. Also, don't flush native methods
kvn@1202 335 // since they are part of the JDK in most cases
kvn@1202 336 if (nm->is_in_use() && (!nm->is_osr_method()) && (!nm->is_locked_by_vm()) &&
kvn@1202 337 (!nm->is_native_method()) && ((curr_comp_id < flush_target))) {
kvn@1202 338
kvn@1202 339 if ((nm->method()->code() == nm)) {
kvn@1202 340 // This method has not been previously considered for
kvn@1202 341 // unloading or it was restored already
kvn@1202 342 CodeCache::speculatively_disconnect(nm);
kvn@1202 343 disconnected++;
kvn@1202 344 } else if (nm->is_speculatively_disconnected()) {
kvn@1202 345 // This method was previously considered for preemptive unloading and was not called since then
kvn@1202 346 nm->method()->invocation_counter()->decay();
kvn@1202 347 nm->method()->backedge_counter()->decay();
kvn@1202 348 nm->make_not_entrant();
kvn@1202 349 made_not_entrant++;
kvn@1202 350 }
kvn@1202 351
kvn@1202 352 if (curr_comp_id > _highest_marked) {
kvn@1202 353 _highest_marked = curr_comp_id;
kvn@1202 354 }
kvn@1202 355 }
kvn@1202 356 nm = CodeCache::alive_nmethod(CodeCache::next(nm));
kvn@1202 357 }
kvn@1202 358
kvn@1202 359 if (LogCompilation && (xtty != NULL)) {
kvn@1202 360 ttyLocker ttyl;
kvn@1202 361 xtty->begin_elem("stop_cleaning_code_cache disconnected='" UINT32_FORMAT "' made_not_entrant='" UINT32_FORMAT "' live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
kvn@1202 362 disconnected, made_not_entrant, CodeCache::nof_blobs(), CodeCache::unallocated_capacity());
kvn@1202 363 xtty->stamp();
kvn@1202 364 xtty->end_elem();
kvn@1202 365 }
kvn@1202 366
kvn@1202 367 // Shut off compiler. Sweeper will run exiting from this safepoint
kvn@1202 368 // and turn it back on if it clears enough space
kvn@1202 369 if (was_full()) {
kvn@1202 370 _last_was_full = os::javaTimeMillis();
kvn@1202 371 CompileBroker::set_should_compile_new_jobs(CompileBroker::stop_compilation);
kvn@1202 372 }
kvn@1202 373
kvn@1202 374 // After two more traversals the sweeper will get rid of unrestored nmethods
kvn@1202 375 _was_full_traversal = _traversals;
kvn@1202 376 #ifdef ASSERT
kvn@1202 377 jlong end = os::javaTimeMillis();
kvn@1202 378 if(PrintMethodFlushing && Verbose) {
kvn@1202 379 tty->print_cr("### sweeper: unload time: " INT64_FORMAT, end-start);
kvn@1202 380 }
kvn@1202 381 #endif
kvn@1202 382 }