Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -464,30 +464,49 @@
 
 ;;======================================================================
 ;; M A I N T E N A N C E
 ;;======================================================================
 
-(define (db:find-and-mark-incomplete db)
+;;  select end_time-now from
+;;      (select testname,item_path,event_time+run_duration as
+;;                          end_time,strftime('%s','now') as now from tests where state in
+;;      ('RUNNING','REMOTEHOSTSTART','LAUNCED'));
+
+
+(define (db:find-and-mark-incomplete db #!key (ovr-deadtime #f))
   (let* ((incompleted '())
 	 (deadtime-str (configf:lookup *configdat* "setup" "deadtime"))
 	 (deadtime     (if (and deadtime-str
 				(string->number deadtime-str))
 			   (string->number deadtime-str)
-			   1800)))
-    (sqlite3:for-each-row 
-     (lambda (test-id)
-       (set! incompleted (cons test-id incompleted)))
-     db
-     "SELECT id FROM tests WHERE event_time<? AND state IN ('RUNNING','REMOTEHOSTSTART');"
-     (- (current-seconds) deadtime)) ;; in RUNNING or REMOTEHOSTSTART for more than 10 minutes
-    (sqlite3:for-each-row
-     (lambda (test-id)
-       (set! incompleted (cons test-id incompleted)))
-     db
-     "SELECT id FROM tests WHERE event_time<? AND state IN ('LAUNCHED');"
-     (- (current-seconds)(* 60 60 24))) ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config
+			   7200)) ;; two hours
+	 (run-ids      (db:get-run-ids db))) ;; iterate over runs to divy up the calls
+    (if (number? ovr-deadtime)(set! deadtime ovr-deadtime))
+    (for-each
+     (lambda (run-id)
+
+       ;; in RUNNING or REMOTEHOSTSTART for more than 10 minutes
+       ;;
+       (sqlite3:for-each-row 
+	(lambda (test-id)
+	  (set! incompleted (cons test-id incompleted)))
+	db
+	"SELECT id FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time - run_duration) > ? AND state IN ('RUNNING','REMOTEHOSTSTART');"
+	run-id deadtime)
+
+       ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config
+       ;;
+       (sqlite3:for-each-row
+	(lambda (test-id)
+	  (set! incompleted (cons test-id incompleted)))
+	db
+	"SELECT id FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time - run_duration) > ? AND state IN ('LAUNCHED');"
+	run-id (* 60 60 24)))
+     run-ids)
+       
     ;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE.
+    ;;
     (if (> (length incompleted) 0)
 	(begin
 	  (debug:print 0 "WARNING: Marking test(s); " (string-intersperse (map conc incompleted) ", ") " as INCOMPLETE")
 	  (sqlite3:execute 
 	   db
@@ -877,10 +896,18 @@
     (sqlite3:execute db "UPDATE runs SET state=? WHERE id=?;" newlockval run-id)
     (sqlite3:execute db "INSERT INTO access_log (user,accessed,args) VALUES(?,strftime('%s','now'),?);"
 		     user (conc newlockval " " run-id))
     (debug:print-info 1 "" newlockval " run number " run-id)))
 
+(define (db:get-run-ids db)
+  (let ((res '()))
+    (sqlite3:for-each-row
+     (lambda (id)
+       (set! res (cons id res)))
+     db 
+     "SELECT id FROM runs;")))
+
 ;;======================================================================
 ;; K E Y S
 ;;======================================================================
 
 ;; get key val pairs for a given run-id

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -723,10 +723,15 @@
 ;; test-records is a hash table testname:item_path => vector < testname testconfig waitons priority items-info ... >
 (define (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests reglen-in all-tests-registry)
   ;; At this point the list of parent tests is expanded 
   ;; NB// Should expand items here and then insert into the run queue.
   (debug:print 5 "test-records: " test-records ", flags: " (hash-table->alist flags))
+
+  ;; Do mark-and-find clean up of db before starting runing of quue
+  ;;
+  (cdb:remote-run db:find-and-mark-incomplete #f)
+
   (let ((run-info              (cdb:remote-run db:get-run-info #f run-id))
 	(tests-info            (mt:get-tests-for-run run-id #f '() '())) ;;  qryvals: "id,testname,item_path"))
 	(sorted-test-names     (tests:sort-by-priority-and-waiton test-records))
 	(test-registry         (make-hash-table))
 	(registry-mutex        (make-mutex))
@@ -735,11 +740,11 @@
 	(max-concurrent-jobs   (let ((mcj (config-lookup *configdat* "setup"     "max_concurrent_jobs")))
 				 (if (and mcj (string->number mcj))
 				     (string->number mcj)
 				     1))) ;; length of the register queue ahead
 	(reglen                (if (number? reglen-in) reglen-in 1))
-	(last-time-incomplete  (- (current-seconds) 610)))
+	(last-time-incomplete  (current-seconds)))
 
     ;; Initialize the test-registery hash with tests that already have a record
     ;; convert state to symbol and use that as the hash value
     (for-each (lambda (trec)
 		(let ((id (db:test-get-id        trec))
@@ -755,12 +760,12 @@
 	       (tal         (cdr sorted-test-names))
 	       (reg         '()) ;; registered, put these at the head of tal 
 	       (reruns      '()))
       (if (not (null? reruns))(debug:print-info 4 "reruns=" reruns))
 
-      ;; Here we mark any old defunct tests as incomplete. Do this every five minutes
-      (if (> (current-seconds)(+ last-time-incomplete 300))
+      ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes
+      (if (> (current-seconds)(+ last-time-incomplete 900))
 	  (begin
 	    (set! last-time-incomplete (current-seconds))
 	    (cdb:remote-run db:find-and-mark-incomplete #f)))
 
       ;; (print "Top of loop, hed=" hed ", tal=" tal " ,reruns=" reruns)