Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -460,10 +460,38 @@
        (db:set-var db "MEGATEST_VERSION" 1.37)
        (sqlite3:execute db "ALTER TABLE tests ADD COLUMN archived INTEGER DEFAULT 0;")) 
       ((< mver megatest-version)
        (db:set-var db "MEGATEST_VERSION" megatest-version))))))
 
+;;======================================================================
+;; M A I N T E N A N C E
+;;======================================================================
+
+(define (db:find-and-mark-incomplete db)
+  (let ((incompleted '()))
+    (sqlite3:for-each-row 
+     (lambda (test-id)
+       (set! incompleted (cons test-id incompleted)))
+     db
+     "SELECT id FROM tests WHERE event_time<? AND state IN ('RUNNING','REMOTEHOSTSTART');"
+     (- (current-seconds) 600)) ;; in RUNNING or REMOTEHOSTSTART for more than 10 minutes
+    (sqlite3:for-each-row
+     (lambda (test-id)
+       (set! incompleted (cons test-id incompleted)))
+     db
+     "SELECT id FROM tests WHERE event_time<? AND state IN ('LAUNCHED');"
+     (- (current-seconds)(* 60 60 24))) ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config
+    ;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE.
+    (if (> (length incompleted) 0)
+	(begin
+	  (debug:print 0 "WARNING: Marking test(s); " (string-intersperse (map conc incompleted) ", ") " as INCOMPLETE")
+	  (sqlite3:execute 
+	   db
+	   (conc "UPDATE tests SET state='INCOMPLETE' WHERE id IN (" 
+		 (string-intersperse (map conc incompleted) ",")
+		 ");"))))))
+		     
 ;; Clean out old junk and vacuum the database
 ;;
 ;; Ultimately do something like this:
 ;;
 ;; 1. Look at test records either deleted or part of deleted run:
@@ -500,10 +528,11 @@
        (sqlite3:for-each-row (lambda (tot)
 			       (debug:print-info 0 "Records count after  clean: " tot))
 			     count-stmt)))
     (map sqlite3:finalize! statements)
     (sqlite3:finalize! count-stmt)
+    (db:find-and-mark-incomplete db)
     (sqlite3:execute db "VACUUM;")))
 
 ;; (define (db:report-junk-records db)
 
 

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -734,11 +734,12 @@
 	(max-retries           (config-lookup *configdat* "setup" "maxretries"))
 	(max-concurrent-jobs   (let ((mcj (config-lookup *configdat* "setup"     "max_concurrent_jobs")))
 				 (if (and mcj (string->number mcj))
 				     (string->number mcj)
 				     1))) ;; length of the register queue ahead
-	(reglen                (if (number? reglen-in) reglen-in 1)))
+	(reglen                (if (number? reglen-in) reglen-in 1))
+	(last-time-incomplete  (- (current-seconds) 610)))
 
     ;; Initialize the test-registery hash with tests that already have a record
     ;; convert state to symbol and use that as the hash value
     (for-each (lambda (trec)
 		(let ((id (db:test-get-id        trec))
@@ -753,10 +754,16 @@
     (let loop ((hed         (car sorted-test-names))
 	       (tal         (cdr sorted-test-names))
 	       (reg         '()) ;; registered, put these at the head of tal 
 	       (reruns      '()))
       (if (not (null? reruns))(debug:print-info 4 "reruns=" reruns))
+
+      ;; Here we mark any old defunct tests as incomplete. Do this every five minutes
+      (if (> (current-seconds)(+ last-time-incomplete 300))
+	  (begin
+	    (set! last-time-incomplete (current-seconds))
+	    (cdb:remote-run db:find-and-mark-incomplete #f)))
 
       ;; (print "Top of loop, hed=" hed ", tal=" tal " ,reruns=" reruns)
       (let* ((test-record (hash-table-ref test-records hed))
 	     (test-name   (tests:testqueue-get-testname test-record))
 	     (tconfig     (tests:testqueue-get-testconfig test-record))