Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -45,11 +45,13 @@
     get-prev-run-ids
     get-run-ids-matching-target
     get-runs-by-patt
     get-steps-data
     login
-    testmeta-get-record))
+    testmeta-get-record
+    have-incompletes?
+    ))
 
 (define api:write-queries
   '(
     ;; SERVERS
     start-server
@@ -68,11 +70,11 @@
     register-run
     set-tests-state-status
     delete-run
     lock/unlock-run
     update-run-event_time
-    find-and-mark-incomplete
+    mark-incomplete
 
     ;; STEPS
     teststep-set-status!
 
     ;; TEST DATA
@@ -134,12 +136,10 @@
 	    ((register-run)                 (apply db:register-run dbstruct params))
 	    ((set-tests-state-status)       (apply db:set-tests-state-status dbstruct params))
 	    ((delete-run)                   (apply db:delete-run dbstruct params))
 	    ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
 	    ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
-	    ((find-and-mark-incomplete)     (apply db:find-and-mark-incomplete dbstruct params))
-
 
 	    ;; STEPS
 	    ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
 
 	    ;; TEST DATA
@@ -147,10 +147,11 @@
 	    ((csv->test-data)               (apply db:csv->test-data dbstruct params))
 
 	    ;; MISC
 	    ((sync-inmem->db)               (let ((run-id (car params)))
 					      (db:sync-touched dbstruct run-id force-sync: #t)))
+	    ((mark-incomplete)              (apply db:find-and-mark-incomplete dbstruct params))
 
 	    ;; TESTMETA
 	    ((testmeta-add-record)       (apply db:testmeta-add-record dbstruct params))
 	    ((testmeta-update-field)     (apply db:testmeta-update-field dbstruct params))
 
@@ -199,10 +200,11 @@
 
 	    ;; STEPS
 	    ((get-steps-data)               (apply db:get-steps-data dbstruct params))
 
 	    ;; MISC
+	    ((have-incompletes?)            (apply db:have-incompletes? dbstruct params))
 	    ((login)                        (apply db:login dbstruct params))
 	    ((general-call)                 (let ((stmtname   (car params))
 						  (run-id     (cadr params))
 						  (realparams (cddr params)))
 					      (db:with-db dbstruct run-id #t ;; these are all for modifying the db

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -935,10 +935,64 @@
 ;;======================================================================
 
 ;;======================================================================
 ;; M A I N T E N A N C E
 ;;======================================================================
+
+(define (db:have-incompletes? dbstruct run-id ovr-deadtime)
+  (let* ((dbdat        (db:get-db dbstruct run-id))
+	 (db           (db:dbdat-get-db dbdat))
+	 (incompleted '())
+	 (oldlaunched '())
+	 (toplevels   '())
+	 (deadtime-str (configf:lookup *configdat* "setup" "deadtime"))
+	 (deadtime     (if (and deadtime-str
+				(string->number deadtime-str))
+			   (string->number deadtime-str)
+			   7200))) ;; two hours
+    (if (number? ovr-deadtime)(set! deadtime ovr-deadtime))
+    
+    ;; in RUNNING or REMOTEHOSTSTART for more than 10 minutes
+    ;;
+    ;; HOWEVER: this code in run:test seems to work fine
+    ;;              (> (- (current-seconds)(+ (db:test-get-event_time testdat)
+    ;;                     (db:test-get-run_duration testdat)))
+    ;;                    600) 
+    (db:delay-if-busy dbdat)
+    (sqlite3:for-each-row 
+     (lambda (test-id run-dir uname testname item-path)
+       (if (and (equal? uname "n/a")
+		(equal? item-path "")) ;; this is a toplevel test
+	   ;; what to do with toplevel? call rollup?
+	   (begin
+	     (set! toplevels   (cons (list test-id run-dir uname testname item-path run-id) toplevels))
+	     (debug:print-info 0 "Found old toplevel test in RUNNING state, test-id=" test-id))
+	   (set! incompleted (cons (list test-id run-dir uname testname item-path run-id) incompleted))))
+     db
+     "SELECT id,rundir,uname,testname,item_path FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time) > (run_duration + ?) AND state IN ('RUNNING','REMOTEHOSTSTART');"
+     run-id deadtime)
+
+    ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config
+    ;;
+    (db:delay-if-busy dbdat)
+    (sqlite3:for-each-row
+     (lambda (test-id run-dir uname testname item-path)
+       (if (and (equal? uname "n/a")
+		(equal? item-path "")) ;; this is a toplevel test
+	   ;; what to do with toplevel? call rollup?
+	   (set! toplevels   (cons (list test-id run-dir uname testname item-path run-id) toplevels))
+	   (set! oldlaunched (cons (list test-id run-dir uname testname item-path run-id) oldlaunched))))
+     db
+     "SELECT id,rundir,uname,testname,item_path FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time) > 86400 AND state IN ('LAUNCHED');"
+     run-id)
+    
+    (debug:print-info 18 "Found " (length oldlaunched) " old LAUNCHED items, " (length toplevels) " old LAUNCHED toplevel tests and " (length incompleted) " tests marked RUNNING but apparently dead.")
+    (if (and (null? incompleted)
+	     (null? oldlaunched)
+	     (null? toplevels))
+	#f
+	#t)))
 
 ;;  select end_time-now from
 ;;      (select testname,item_path,event_time+run_duration as
 ;;                          end_time,strftime('%s','now') as now from tests where state in
 ;;      ('RUNNING','REMOTEHOSTSTART','LAUNCED'));

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -517,11 +517,12 @@
 
 (define (rmt:get-runs-by-patt  keys runnamepatt targpatt offset limit)
   (rmt:send-receive 'get-runs-by-patt #f (list keys runnamepatt targpatt offset limit)))
 
 (define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
-  (rmt:send-receive 'find-and-mark-incomplete run-id (list run-id ovr-deadtime)))
+  (if (rmt:send-receive 'have-incompletes? run-id (list run-id ovr-deadtime))
+      (rmt:send-receive 'mark-incomplete run-id (list run-id ovr-deadtime))))
 
 ;;======================================================================
 ;; M U L T I R U N   Q U E R I E S
 ;;======================================================================