Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -946,12 +946,27 @@ ;; we are going to reset all the counters for test retries by setting a new hash table ;; this means they will increment only when nothing can be run (set! *max-tries-hash* (make-hash-table)) ;; well, first lets see if cpu load throttling is enabled. If so wait around until the ;; average cpu load is under the threshold before continuing - (if (configf:lookup *configdat* "jobtools" "maxload") ;; only gate if maxload is specified - (common:wait-for-cpuload maxload numcpus waitdelay)) + (let ((hh (common:get-homehost)) + (maxload (configf:lookup *configdat* "jobtools" "maxload"))) + (if maxload ;; only gate if maxload is specified + (let loadloop ((load-dat #f)) + (common:wait-for-cpuload maxload numcpus waitdelay) ;; first wait for local load to decrease if it happens to be high + (if (and hh + (not (common:on-homehost?))) + (let* ((hh-load-dat (common:get-normalized-cpu-load hh)) + (hh-load (if hh-load-dat + (alist-ref 'adj-cpu-load hh-load-dat) + #f))) + (cond + ((not hh-load)(debug:print-info 0 *default-log-port* "Could not determine load on homehost. Proceeding as if load is fine ...")) + ((> hh-load maxload) + (debug:print-info 0 *default-log-port "Load too high on homehost, delaying before launching tests.") + + (run:test run-id run-info keyvals runname test-record flags #f test-registry all-tests-registry) (runs:incremental-print-results run-id) (hash-table-set! test-registry (db:test-make-full-name test-name item-path) 'running) (runs:shrink-can-run-more-tests-count runsdat) ;; DELAY TWEAKER (still needed?) ;; (thread-sleep! *global-delta*) Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -457,10 +457,11 @@ (thread-sleep! 0.05) ;; delay for startup (let ((legacy-sync (common:run-sync?)) (debug-mode (debug:debug-mode 1)) (last-time (current-seconds)) (no-sync-db (db:open-no-sync-db)) + (sync-delay (string->number (or (configf:get *configdat* "server" "sync-delay") "5"))) (this-wd-num (begin (mutex-lock! *wdnum*mutex) (let ((x *wdnum*)) (set! *wdnum* (add1 *wdnum*)) (mutex-unlock! *wdnum*mutex) x)))) (set! *no-sync-db* no-sync-db) ;; make the no sync db available to api calls (debug:print-info 2 *default-log-port* "Periodic sync thread started.") (debug:print-info 3 *default-log-port* "watchdog starting. legacy-sync is " legacy-sync" pid="(current-process-id)" this-wd-num="this-wd-num) (if (and legacy-sync (not *time-to-exit*)) @@ -470,18 +471,25 @@ (debug:print-info 0 *default-log-port* "Server running, periodic sync started.") (let loop () ;; sync for filesystem local db writes ;; (mutex-lock! *db-multi-sync-mutex*) - (let* ((need-sync (>= *db-last-access* *db-last-sync*)) ;; no sync since last write + (let* ((load-too-high (> (common:get-normalized-cpu-load #f) 1)) + (need-sync (>= *db-last-access* *db-last-sync*)) ;; no sync since last write (sync-in-progress *db-sync-in-progress*) (should-sync (and (not *time-to-exit*) - (> (- (current-seconds) *db-last-sync*) 5))) ;; sync every five seconds minimum + (> (- (current-seconds) *db-last-sync*) + (or (if load-too-high ;; if load is high increase delay to bigger of sync-delay or 15 sec + (max sync-delay 15) + #f) + sync-delay)))) ;; sync every five seconds minimum (start-time (current-seconds)) (mt-mod-time (file-modification-time mtpath)) (recently-synced (< (- start-time mt-mod-time) 4)) - (will-sync (and (or need-sync should-sync) + (will-sync (and (if load-too-high + should-sync + (or need-sync should-sync)) (not sync-in-progress) (not recently-synced)))) (debug:print-info 13 *default-log-port* "WD writable-watchdog top of loop. need-sync="need-sync" sync-in-progress="sync-in-progress" should-sync="should-sync" start-time="start-time" mt-mod-time="mt-mod-time" recently-synced="recently-synced" will-sync="will-sync) ;; (if recently-synced (debug:print-info 0 *default-log-port* "Skipping sync due to recently-synced flag=" recently-synced)) ;; (debug:print-info 0 *default-log-port* "need-sync: " need-sync " sync-in-progress: " sync-in-progress " should-sync: " should-sync " will-sync: " will-sync)