Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -51,10 +51,12 @@
     ((get-tests-for-runs-mindata)   (apply db:get-tests-for-runs-mindata dbstruct params))
     ((get-run-name-from-id)         (apply db:get-run-name-from-id dbstruct params))
     ((delete-run)                   (apply db:delete-run dbstruct params))
     ((get-runs)                     (apply db:get-runs dbstruct params))
     ((get-all-run-ids)              (db:get-all-run-ids dbstruct))
+    ((get-prev-run-ids)             (apply db:get-prev-run-ids dbstruct params))
+    ((get-run-ids-matching-target)  (apply db:get-run-ids-matching-target dbstruct params))
     ((get-runs-by-patt)             (apply db:get-runs-by-patt dbstruct params))
     ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
     ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
 
     ;; STEPS

Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -69,16 +69,16 @@
 	      (if start-res ;; sucessful login?
 		  start-res
 		  (if (and (< remaining-tries 10)
 			   (odd? remaining-tries))
 		      (begin    ;; login failed
-			(hash-table-delete! *runremote* run-id)
-			(open-run-close tasks:server-force-clean-run-record
-					tasks:open-db
-					run-id 
-					(car  server-dat)
-					(cadr server-dat))
+			;; (hash-table-delete! *runremote* run-id)
+			;; (open-run-close tasks:server-force-clean-run-record
+			;; 		tasks:open-db
+			;; 		run-id 
+			;; 		(car  server-dat)
+			;; 		(cadr server-dat))
 			(thread-sleep! 5)
 			(client:setup run-id remaining-tries: (- remaining-tries 1)))
 		      (begin
 			(thread-sleep! 5)
 			(client:setup run-id remaining-tries: (- remaining-tries 1))))))

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -77,14 +77,21 @@
 
 ;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;")
 ;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
 ;;
 (define (db:with-db dbstruct run-id r/w proc . params)
-  (let* ((db (db:get-db dbstruct run-id)))
-    (let ((res (apply proc db params)))
-      (db:done-with dbstruct run-id r/w)
-      res)))
+  (let* ((db    (db:get-db dbstruct run-id))
+	 (proc  (lambda ()
+		  (let ((res (apply proc db params)))
+		    (db:done-with dbstruct run-id r/w)
+		    res))))
+    (handle-exceptions
+     exn
+     (begin
+       (thread-sleep! 10)
+       (proc))
+     (proc))))
 
 ;;======================================================================
 ;; K E E P   F I L E D B   I N   dbstruct
 ;;======================================================================
 
@@ -642,11 +649,11 @@
 	 (deadtime-str (configf:lookup *configdat* "setup" "deadtime"))
 	 (deadtime     (if (and deadtime-str
 				(string->number deadtime-str))
 			   (string->number deadtime-str)
 			   7200)) ;; two hours
-	 (run-ids      (db:get-run-ids db))) ;; iterate over runs to divy up the calls
+	 (run-ids      (db:get-all-run-ids db))) ;; iterate over runs to divy up the calls
     (if (number? ovr-deadtime)(set! deadtime ovr-deadtime))
     (for-each
      (lambda (run-id)
 
        ;; in RUNNING or REMOTEHOSTSTART for more than 10 minutes
@@ -1011,11 +1018,11 @@
     (sqlite3:for-each-row
      (lambda (run-id)
        (set! run-ids (cons run-id run-ids)))
      (db:get-db dbstruct #f)
      "SELECT id FROM runs WHERE state != 'deleted';")
-    run-ids))
+    (reverse run-ids)))
 
 ;; get some basic run stats
 ;;
 ;; ( (runname (( state  count ) ... ))
 ;;   (   ...  
@@ -1138,27 +1145,10 @@
     (sqlite3:execute (db:get-db dbstruct #f) "UPDATE runs SET state=? WHERE id=?;" newlockval run-id)
     (sqlite3:execute (db:get-db dbstruct #f) "INSERT INTO access_log (user,accessed,args) VALUES(?,strftime('%s','now'),?);"
 		     user (conc newlockval " " run-id))
     (debug:print-info 1 "" newlockval " run number " run-id)))
 
-(define (db:get-all-run-ids dbstruct)
-  (let ((res '()))
-    (sqlite3:for-each-row
-     (lambda (run-id)
-       (set! res (cons run-id res)))
-     (db:get-db dbstruct #f)
-     "SELECT id FROM runs;")
-    (reverse res)))
-
-(define (db:get-run-ids db)
-  (let ((res '()))
-    (sqlite3:for-each-row
-     (lambda (id)
-       (set! res (cons id res)))
-     db 
-     "SELECT id FROM runs;")))
-
 ;;======================================================================
 ;; K E Y S
 ;;======================================================================
 
 ;; get key val pairs for a given run-id
@@ -1177,36 +1167,46 @@
      keys)
     (reverse res)))
 
 ;; get key vals for a given run-id
 (define (db:get-key-vals dbstruct run-id)
-  (let ((mykeyvals (hash-table-ref/default *keyvals* run-id #f)))
-    (if mykeyvals 
-	mykeyvals
-	(let* ((keys (db:get-keys dbstruct))
-	       (res  '()))
-	  (for-each 
-	   (lambda (key)
-	     (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
-	       (sqlite3:for-each-row 
-		(lambda (key-val)
-		  (set! res (cons key-val res)))
-		(db:get-db dbstruct #f) qry run-id)))
-	   keys)
-	  (let ((final-res (reverse res)))
-	    (hash-table-set! *keyvals* run-id final-res)
-	    final-res)))))
+  (let* ((keys (db:get-keys dbstruct))
+	 (res  '()))
+    (for-each 
+     (lambda (key)
+       (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
+	 (sqlite3:for-each-row 
+	  (lambda (key-val)
+	    (set! res (cons key-val res)))
+	  (db:get-db dbstruct #f) qry run-id)))
+     keys)
+    (let ((final-res (reverse res)))
+      (hash-table-set! *keyvals* run-id final-res)
+      final-res)))
 
 ;; The target is keyval1/keyval2..., cached in *target* as it is used often
 (define (db:get-target dbstruct run-id)
-  (let ((mytarg (hash-table-ref/default *target* run-id #f)))
-    (if mytarg
-	mytarg
-	(let* ((keyvals (db:get-key-vals dbstruct run-id))
-	       (thekey  (string-intersperse (map (lambda (x)(if x x "-na-")) keyvals) "/")))
-	  (hash-table-set! *target* run-id thekey)
-	  thekey))))
+  (let* ((keyvals (db:get-key-vals dbstruct run-id))
+	 (thekey  (string-intersperse (map (lambda (x)(if x x "-na-")) keyvals) "/")))
+    thekey))
+
+;; Get run-ids for runs with same target but different runnames and NOT run-id
+;;
+(define (db:get-prev-run-ids dbstruct run-id)
+  (let* ((keyvals (rmt:get-key-val-pairs run-id))
+	 (kvalues (make cdr keyvals))
+	 (keys    (rmt:get-keys))
+	 (qrystr  (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")))
+    (let ((prev-run-ids '()))
+      (db:with-db dbstruct #f #f ;; #f means work with the zeroth db - i.e. the runs db
+       (lambda ()
+	 (apply sqlite3:for-each-row
+		(lambda (id)
+		  (set! prev-run-ids (cons id prev-run-ids)))
+		db
+		(conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append kvalues (list run-id)))))
+      prev-run-ids)))
 
 ;;======================================================================
 ;;  T E S T S
 ;;======================================================================
 
@@ -1683,34 +1683,35 @@
 
 ;;======================================================================
 ;; Misc. test related queries
 ;;======================================================================
 
-(define (db:test-get-paths-matching-keynames-target-new dbstruct keynames target res testpatt statepatt statuspatt runname)
+(define (db:get-run-ids-matching-target dbstruct keynames target res testpatt statepatt statuspatt)
   (let* ((row-ids '())
 	 (keystr (string-intersperse 
 		  (map (lambda (key val)
 			 (conc key " like '" val "'"))
 		       keynames 
 		       (string-split target "/"))
 		  " AND "))
 	 (testqry (tests:match->sqlqry testpatt))
-	 (runsqry (sqlite3:prepare (db:get-db dbstruct #f)(conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';")))
-	 (tstsqry (conc "SELECT rundir FROM tests WHERE " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")))
+	 (runsqry (sqlite3:prepare (db:get-db dbstruct #f)(conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';"))))
     (debug:print 8 "db:test-get-paths-matching-keynames-target-new\n  runsqry=" runsqry "\n  tstsqry=" tstsqry)
     (sqlite3:for-each-row
      (lambda (rid)
        (set! row-ids (cons rid row-ids)))
      runsqry)
     (sqlite3:finalize! runsqry)
-    (for-each (lambda (rid)
-		(sqlite3:for-each-row 
-		 (lambda (p)
-		   (set! res (cons p res)))
-		 (db:get-db dbstruct rid)
-		 tstsqry))
-	      row-ids)
+    row-ids))
+
+(define (db:test-get-paths-matching-keynames-target-new dbstruct run-id keynames target res testpatt statepatt statuspatt runname)
+  (let ((tstsqry (conc "SELECT rundir FROM tests WHERE " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")))
+    (sqlite3:for-each-row 
+     (lambda (p)
+       (set! res (cons p res)))
+     (db:get-db dbstruct run-id)
+     tstsqry)
     res))
 
 ;;======================================================================
 ;; QUEUE UP META, TEST STATUS AND STEPS REMOTE ACCESS
 ;;======================================================================
@@ -1878,49 +1879,10 @@
 				   db:queries)))
  		 (if q (car q) #f))))
     (apply sqlite3:execute db query params)
     #t))
 
-;; get the previous record for when this test was run where all keys match but runname
-;; returns #f if no such test found, returns a single test record if found
-;; 
-;; Run this server-side
-;;
-(define (db:get-previous-test-run-record dbstruct run-id test-name item-path)
-  (let* ((db      (db:get-db dbstruct #f)) ;; 
-	 (keys    (db:get-keys db))
-	 (selstr  (string-intersperse  keys ","))
-	 (qrystr  (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND "))
-	 (keyvals #f))
-    ;; first look up the key values from the run selected by run-id
-    (sqlite3:for-each-row 
-     (lambda (a . b)
-       (set! keyvals (cons a b)))
-     db
-     (conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id)
-    (if (not keyvals)
-	#f
-	(let ((prev-run-ids '()))
-	  (apply sqlite3:for-each-row
-		 (lambda (id)
-		   (set! prev-run-ids (cons id prev-run-ids)))
-		 db
-		 (conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append keyvals (list run-id)))
-	  ;; for each run starting with the most recent look to see if there is a matching test
-	  ;; if found then return that matching test record
-	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids)
-	  (if (null? prev-run-ids) #f
-	      (let loop ((hed (car prev-run-ids))
-			 (tal (cdr prev-run-ids)))
-		(let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f)))
-		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results)
-		  (if (and (null? results)
-			   (not (null? tal)))
-		      (loop (car tal)(cdr tal))
-		      (if (null? results) #f
-			  (car results))))))))))
-
 ;; get the previous records for when these tests were run where all keys match but runname
 ;; NB// Merge this with test:get-previous-test-run-records? This one looks for all matching tests
 ;; can use wildcards. Also can likely be factored in with get test paths?
 ;;
 ;; Run this remotely!!

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -1159,11 +1159,11 @@
 
 (if (args:get-arg "-import-megatest.db")
     (let* ((toppath  (setup-for-run))
 	   (dbstruct (if toppath (make-dbr:dbstruct path: toppath) #f))
 	   (mtdb     (if toppath (db:open-megatest-db)))
-	   (run-ids  (if toppath (db:get-run-ids mtdb))))
+	   (run-ids  (if toppath (db:get-all-run-ids mtdb))))
       ;; sync runs, test_meta etc.
       (db:sync-tables (db:sync-main-list mtdb) mtdb (db:get-db dbstruct #f))
       (for-each 
        (lambda (run-id)
 	 (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id)))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -120,12 +120,12 @@
 
 ;;======================================================================
 ;;  K E Y S 
 ;;======================================================================
 
-;; These should not require run-id but it is more consistent to have it. 
-;; run-id can theoretically be #f but how to handle that is not yet done.
+;; These require run-id because the values come from the run!
+;;
 (define (rmt:get-key-val-pairs run-id)
   (rmt:send-receive 'get-key-val-pairs run-id (list run-id)))
 
 (define (rmt:get-keys)
   (rmt:send-receive 'get-keys #f '()))
@@ -182,12 +182,12 @@
   (rmt:send-receive 'delete-test-records run-id (list run-id test-id)))
 
 (define (rmt:test-set-status-state run-id test-id status state msg)
   (rmt:send-receive 'test-set-status-state run-id (list run-id test-id status state msg)))
 
-(define (rmt:get-previous-test-run-record run-id test-name item-path)
-  (rmt:send-receive 'get-previous-test-run-record run-id (list run-id test-name item-path)))
+;; (define (rmt:get-previous-test-run-record run-id test-name item-path)
+;;   (rmt:send-receive 'get-previous-test-run-record run-id (list run-id test-name item-path)))
 
 (define (rmt:get-matching-previous-test-run-records run-id test-name item-path)
   (rmt:send-receive 'get-matching-previous-test-run-records run-id (list run-id test-name item-path)))
 
 (define (rmt:test-get-logfile-info run-id test-name)
@@ -200,16 +200,19 @@
   (rmt:send-receive 'get-testinfo-state-status run-id (list run-id test-id)))
 
 (define (rmt:test-set-log! run-id test-id logf)
   (if (string? logf)(rmt:general-call 'test-set-log run-id logf test-id)))
 
+(define (rmt:get-run-ids-matching-target keynames target res testpatt statepatt statuspatt)
+  (rmt:send-receive 'get-run-ids-matching-target #f (list keynames target res testpatt statepatt statuspatt)))
+
 ;; NOTE: This will open and access ALL run databases. 
 ;;
 (define (rmt:test-get-paths-matching-keynames-target-new keynames target res testpatt statepatt statuspatt runname)
-  (let ((run-ids (rmt:get-all-run-ids))) ;; (rmt:get-run-ids-matching keynames target res)))
+  (let ((run-ids (rmt:get-run-ids-matching-target keynames target res testpatt statepatt statuspatt)))
     (apply append (lambda (run-id)
-		    (rmt:send-receive 'test-get-paths-matching-keynames-target-new (list keynames target res testpatt statepatt statuspatt runname)))
+		    (rmt:send-receive 'test-get-paths-matching-keynames-target-new run-id (list keynames target res testpatt statepatt statuspatt runname)))
 	   run-ids)))
 
 (define (rmt:get-run-ids-matching keynames target res)
   (rmt:send-receive #f 'get-run-ids-matching (list keynames target res)))
 
@@ -260,18 +263,52 @@
   (rmt:send-receive 'get-runs #f (list runpatt count offset keypatts)))
 
 (define (rmt:get-all-run-ids)
   (rmt:send-receive 'get-all-run-ids #f '()))
 
+(define (rmt:get-prev-run-ids run-id)
+  (rmt:send-receive 'get-prev-run-ids #f (list run-id)))
+
 (define (rmt:lock/unlock-run run-id lock unlock user)
   (rmt:send-receive 'lock/unlock-run #f (list run-id lock unlock user)))
 
 (define (rmt:update-run-event_time run-id)
   (rmt:send-receive 'update-run-event_time #f (list run-id)))
 
 (define (rmt:get-runs-by-patt  keys runnamepatt targpatt offset limit)
   (rmt:send-receive 'get-runs-by-patt #f (list keys runnamepatt targpatt offset limit)))
+
+;;======================================================================
+;; M U L T I R U N   Q U E R I E S
+;;======================================================================
+
+;; get the previous record for when this test was run where all keys match but runname
+;; returns #f if no such test found, returns a single test record if found
+;; 
+;; Run this at the client end since we have to connect to multiple run-id dbs
+;;
+(define (rmt:get-previous-test-run-record run-id test-name item-path)
+  (let* ((keyvals (rmt:get-key-val-pairs run-id))
+	 (keys    (rmt:get-keys))
+	 (selstr  (string-intersperse  keys ","))
+	 (qrystr  (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")))
+    (if (not keyvals)
+	#f
+	(let ((prev-run-ids (rmt:get-prev-run-ids run-id)))
+	  ;; for each run starting with the most recent look to see if there is a matching test
+	  ;; if found then return that matching test record
+	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids)
+	  (if (null? prev-run-ids) #f
+	      (let loop ((hed (car prev-run-ids))
+			 (tal (cdr prev-run-ids)))
+		(let ((results (rmt:get-tests-for-run hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f)))
+		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results)
+		  (if (and (null? results)
+			   (not (null? tal)))
+		      (loop (car tal)(cdr tal))
+		      (if (null? results) #f
+			  (car results))))))))))
 
 ;;======================================================================
 ;;  S T E P S
 ;;======================================================================
 

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -64,13 +64,13 @@
 
 # NOTE: Only one instance can be a server
 test5 : cleanprep
 	@echo "WARNING: No longer running fullprep, test converage may be lessened"
 	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log &
-	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ab -debug $(DEBUG) $(LOGGING) > ab.log 2> ab.log &
-	cd fullrun;sleep 5;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ac -debug $(DEBUG) $(LOGGING) > ac.log 2> ac.log &
-	cd fullrun;sleep 8;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ad -debug $(DEBUG) $(LOGGING) > ad.log 2> ad.log &	
+	cd fullrun;sleep 5;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ab -debug $(DEBUG) $(LOGGING) > ab.log 2> ab.log &
+	cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ac -debug $(DEBUG) $(LOGGING) > ac.log 2> ac.log &
+#	cd fullrun;sleep 15;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ad -debug $(DEBUG) $(LOGGING) > ad.log 2> ad.log &	
 #	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ae -debug $(DEBUG) $(LOGGING) > ae.log 2> ae.log &	
 #	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_af -debug $(DEBUG) $(LOGGING) > af.log 2> af.log &	a
 
 # MUST ADD THIS BACK IN ASAP!!!!
 	# cd fullrun;sleep 10;$(MEGATEST) -run-wait  -target $(TARGET) :runname % -testpatt % :state RUNNING,LAUNCHED,NOT_STARTED,REMOTEHOSTSTART;echo ALL DONE

Index: tests/fullrun/config/mt_include_1.config
==================================================================
--- tests/fullrun/config/mt_include_1.config
+++ tests/fullrun/config/mt_include_1.config
@@ -1,8 +1,8 @@
 [setup]
 # exectutable /path/to/megatest
-max_concurrent_jobs 500
+max_concurrent_jobs 150
 
 linktree #{getenv MT_RUN_AREA_HOME}/tmp/mt_links
 
 [jobtools]
 useshell yes

Index: tests/fullrun/megatest.config
==================================================================
--- tests/fullrun/megatest.config
+++ tests/fullrun/megatest.config
@@ -115,11 +115,12 @@
 # it succeeds
 port 8080
 
 # This server will keep running this number of hours after last access. 
 # Three minutes is 0.05 hours
-timeout 0.025
+# timeout 0.025
+timeout 1
 
 ## disks are:
 ## name host:/path/to/area
 ## -or-
 ## name /path/to/area