Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -33,15 +33,17 @@
 (define *configinfo* #f)
 (define *configdat*  #f)
 (define *toppath*    #f)
 (define *already-seen-runconfig-info* #f)
 (define *waiting-queue* (make-hash-table))
+(define *test-meta-updated* (make-hash-table))
 (define *globalexitstatus* 0) ;; attempt to work around possible thread issues
 (define *passnum*     0) ;; when running track calls to run-tests or similar
 (define *verbosity*   1)
 (define *rpc:listener* #f) ;; if set up for server communication this will hold the tcp port
 (define *runremote*    #f) ;; if set up for server communication this will hold <host port>
+(define *last-db-access* 0) ;; update when db is accessed via server
 
 (define (get-with-default val default)
   (let ((val (args:get-arg val)))
     (if val val default)))
 

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -78,11 +78,11 @@
       (exit 1)))
 
 (define *db* (open-db))
 
 ;; HACK ALERT: this is a hack, please fix.
-(define *read-only* (file-read-access? (conc *toppath* "/megatest.db")))
+(define *read-only* (not (file-read-access? (conc *toppath* "/megatest.db"))))
 ;; (server:client-setup *db*)
 
 (define toplevel #f)
 (define dlg      #f)
 (define max-test-num 0)

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -38,11 +38,13 @@
 
 (define (open-db) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
   (let* ((dbpath    (conc *toppath* "/megatest.db")) ;; fname)
 	 (dbexists  (file-exists? dbpath))
 	 (db        (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath))
-	 (handler   (make-busy-timeout 36000)))
+	 (handler   (make-busy-timeout (if (args:get-arg "-override-timeout")
+					   (string->number (args:get-arg "-override-timeout"))
+					   36000)))) ;; 136000)))
     (sqlite3:set-busy-handler! db handler)
     (if (not dbexists)
 	(db:initialize db))
     db))
 
@@ -213,11 +215,11 @@
 	                        tol REAL,
                                 units TEXT,
                                 comment TEXT DEFAULT '',
                                 status TEXT DEFAULT 'n/a',
                               CONSTRAINT test_data UNIQUE (test_id,category,variable));")
-       (print "WARNING: Table test_data and test_meta where recreated. Please do megatest -update-meta")
+       (print "WARNING: Table test_data and test_meta were recreated. Please do megatest -update-meta")
        (patch-db))
       ((< mver 1.27)
        (db:set-var db "MEGATEST_VERSION" 1.27)
        (sqlite3:execute db "ALTER TABLE test_data ADD COLUMN type TEXT DEFAULT '';")
        (patch-db))
@@ -445,11 +447,13 @@
 			  db
 			  "SELECT id FROM tests WHERE run_id=? AND testname=? AND item_path=?;"
 			  run-id test-name (item-list->path itemdat))
     (for-each (lambda (id)
 		(sqlite3:execute db "DELETE FROM test_steps WHERE test_id=?;" id)
-		(thread-sleep! 0.1)) ;; give others access to the db
+		(thread-sleep! 0.1) ;; give others access to the db
+                (sqlite3:execute db "DELETE FROM test_data WHERE test_id=?;" id)
+                (thread-sleep! 0.1)) ;; give others access to the db
 	      ids)))
 ;;"DELETE FROM test_steps WHERE test_id in (SELECT id FROM tests WHERE run_id=? AND testname=? AND item_path=?);" 
 		   
 ;; 
 (define (db:delete-test-records db test-id)
@@ -536,26 +540,28 @@
      "SELECT id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment FROM tests WHERE id=?;"
      test-id)
     res))
 
 
-(define (db:test-set-comment db run-id test-name item-path comment)
+(define (db:test-set-comment db test-id comment)
   (sqlite3:execute 
    db 
-   "UPDATE tests SET comment=? WHERE run_id=? AND testname=? AND item_path=?;"
-   comment run-id test-name item-path))
+   "UPDATE tests SET comment=? WHERE id=?;"
+   comment test-id))
 
 ;;
 (define (db:test-set-rundir! db run-id test-name item-path rundir)
   (sqlite3:execute 
    db 
    "UPDATE tests SET rundir=? WHERE run_id=? AND testname=? AND item_path=?;"
    rundir run-id test-name item-path))
 
-(define (db:test-set-log! db run-id test-name item-path logf)
-  (sqlite3:execute db "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path=?;" 
-		   logf run-id test-name item-path))
+(define (db:test-set-log! db test-id logf)
+  (if (string? logf)
+      (sqlite3:execute db "UPDATE tests SET final_logf=? WHERE id=?;"
+		   logf test-id)
+      (debug:print 0 "ERROR: db:test-set-log! called with non-string log file name " logf)))
 
 ;;======================================================================
 ;; Misc. test related queries
 ;;======================================================================
 
@@ -678,14 +684,14 @@
 		     ((step-status)
 		      (apply sqlite3:execute step-stmt (vector-ref entry 2)))
 		     (else
 		      (debug:print 0 "ERROR: Queued entry not recognised " entry))))
 		 data)))
+    (sqlite3:finalize! meta-stmt) ;; sqlite is the bottleneck, clear the statements asap?
+    (sqlite3:finalize! step-stmt)
     (set! *incoming-data* '())
-    (mutex-unlock! *incoming-mutex*)
-    (sqlite3:finalize! meta-stmt)
-    (sqlite3:finalize! step-stmt)))
+    (mutex-unlock! *incoming-mutex*)))
 
 (define (db:roll-up-pass-fail-counts db run-id test-name item-path status)
   (if (and (not (equal? item-path ""))
 	   (or (equal? status "PASS")
 	       (equal? status "WARN")
@@ -698,10 +704,11 @@
 	 "UPDATE tests 
              SET fail_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status='FAIL'),
                  pass_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND (status='PASS' OR status='WARN' OR status='WAIVED'))
              WHERE run_id=? AND testname=? AND item_path='';"
 	 run-id test-name run-id test-name run-id test-name)
+        (thread-sleep! 0.1) ;; give other processes a chance here
 	(if (equal? status "RUNNING") ;; running takes priority over all other states, force the test state to RUNNING
 	    (sqlite3:execute db "UPDATE tests SET state=? WHERE run_id=? AND testname=? AND item_path='';" "RUNNING" run-id test-name)
 	    (sqlite3:execute
 	     db
 	     "UPDATE tests
@@ -708,11 +715,13 @@
                        SET state=CASE WHEN (SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND state in ('RUNNING','NOT_STARTED')) > 0 THEN 
                           'RUNNING'
                        ELSE 'COMPLETED' END,
                           status=CASE WHEN fail_count > 0 THEN 'FAIL' WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' ELSE 'UNKNOWN' END
                        WHERE run_id=? AND testname=? AND item_path='';"
-	     run-id test-name run-id test-name)))))
+	     run-id test-name run-id test-name))
+	#f)
+      #f))
 
 
 ;;======================================================================
 ;; Tests meta data
 ;;======================================================================
@@ -804,26 +813,20 @@
        (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
      db
      "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
     (reverse res)))
 
-(define (db:load-test-data db run-id test-name itemdat)
-  (let* ((item-path (item-list->path itemdat))
-	 (testdat (db:get-test-info db run-id test-name item-path))
-	 (test-id (if testdat (db:test-get-id testdat) #f)))
-    ;; (debug:print 1 "Enter records to insert in the test_data table, seven fields, comma separated per line")
-    (debug:print 4 "itemdat: " itemdat ", test-name: " test-name ", test-id: " test-id)
-    (if test-id
-	(let loop ((lin (read-line)))
-	  (if (not (eof-object? lin))
-	      (begin
-		(debug:print 4 lin)
-		(db:csv->test-data db test-id lin)
-		(loop (read-line))))))
-    ;; roll up the current results.
-    ;; FIXME: Add the status to 
-    (db:test-data-rollup db test-id #f)))
+(define (db:load-test-data db test-id)
+  (let loop ((lin (read-line)))
+    (if (not (eof-object? lin))
+	(begin
+	  (debug:print 4 lin)
+	  (rdb:csv->test-data db test-id lin)
+	  (loop (read-line)))))
+  ;; roll up the current results.
+  ;; FIXME: Add the status to 
+  (rdb:test-data-rollup db test-id #f))
 
 ;; WARNING: Do NOT call this for the parent test on an iterated test
 ;; Roll up test_data pass/fail results
 ;; look at the test_data status field, 
 ;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
@@ -974,32 +977,26 @@
 	     ;; if the test is not found then clearly the waiton is not met...
 	     (if (not ever-seen)(set! result (cons waitontest-name result)))))
 	waitons)
       (delete-duplicates result))))
 
-(define (db:teststep-set-status! db run-id test-name teststep-name state-in status-in item-path comment logfile)
-  (debug:print 4 "run-id: " run-id " test-name: " test-name)
+(define (db:teststep-set-status! db test-id teststep-name state-in status-in item-path comment logfile)
+  (debug:print 4 "test-id: " test-id " teststep-name: " teststep-name)
   (let* ((state     (check-valid-items "state" state-in))
-	 (status    (check-valid-items "status" status-in))
-	 (testdat   (db:get-test-info db run-id test-name item-path)))
-    (debug:print 5 "testdat: " testdat)
-    (if (and testdat ;; if the section exists then force specification BUG, I don't like how this works.
-	     (or (not state)(not status)))
+	 (status    (check-valid-items "status" status-in)))
+    (if (or (not state)(not status))
 	(debug:print 0 "WARNING: Invalid " (if status "status" "state")
-	       " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
-    (if testdat
-	(let ((test-id (test:get-id testdat)))
-	  (mutex-lock! *incoming-mutex*)
-	  (set! *incoming-data* (cons (vector 'step-status
-					      (current-seconds)
-					      ;; FIXME - this should not update the logfile unless it is specified.
-					      (list test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile "")))
-				      *incoming-data*))
-	  (mutex-unlock! *incoming-mutex*)
-	  (if (not *cache-on*)(db:write-cached-data db))
-	  #t)
-	(debug:print 0 "ERROR: Can't update " test-name " for run " run-id " -> no such test in db"))))
+		     " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
+    (mutex-lock! *incoming-mutex*)
+    (set! *incoming-data* (cons (vector 'step-status
+					(current-seconds)
+					;; FIXME - this should not update the logfile unless it is specified.
+					(list test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile "")))
+				*incoming-data*))
+    (mutex-unlock! *incoming-mutex*)
+    (if (not *cache-on*)(db:write-cached-data db))
+    #t))
 
 ;;======================================================================
 ;; Extract ods file from the db
 ;;======================================================================
 
@@ -1135,18 +1132,18 @@
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:set-tests-state-status host port)
 	 run-id testnames currstate currstatus newstate newstatus))
       (db:set-tests-state-status db run-id testnames currstate currstatus newstate newstatus)))
 
-(define (rdb:teststep-set-status! db run-id test-name teststep-name state-in status-in itemdat comment logfile)
+(define (rdb:teststep-set-status! db test-id teststep-name state-in status-in itemdat comment logfile)
   (let ((item-path (item-list->path itemdat)))
     (if *runremote*
 	(let ((host (vector-ref *runremote* 0))
 	      (port (vector-ref *runremote* 1)))
 	  ((rpc:procedure 'rdb:teststep-set-status! host port)
-	   run-id test-name teststep-name state-in status-in item-path comment logfile))
-	(db:teststep-set-status! db run-id test-name teststep-name state-in status-in item-path comment logfile))))
+	   test-id teststep-name state-in status-in item-path comment logfile))
+	(db:teststep-set-status! db test-id teststep-name state-in status-in item-path comment logfile))))
 
 (define (rdb:test-update-meta-info db run-id test-name itemdat minutes cpuload diskfree tmpfree)
   (let ((item-path (item-list->path itemdat)))
     (if *runremote*
 	(let ((host (vector-ref *runremote* 0))
@@ -1177,25 +1174,24 @@
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:roll-up-pass-fail-counts host port)
 	 run-id test-name item-path status))
       (db:roll-up-pass-fail-counts db run-id test-name item-path status)))
 
-(define (rdb:test-set-comment db run-id test-name item-path comment)
+(define (rdb:test-set-comment db test-id comment)
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:test-set-comment host port)
-	 run-id test-name item-path comment))
-      (db:test-set-comment db run-id test-name item-path comment)))
+	 test-id comment))
+      (db:test-set-comment db test-id comment)))
 
-(define (rdb:test-set-log! db run-id test-name item-path logf)
+(define (rdb:test-set-log! db test-id logf)
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
-	((rpc:procedure 'rdb:test-set-log! host port)
-	 run-id test-name item-path logf))
-      (db:test-set-log! db run-id test-name item-path logf)))
+	((rpc:procedure 'rdb:test-set-log! host port) test-id logf))
+      (db:test-set-log! db test-id logf)))
 
 (define (rdb:get-runs db runnamepatt numruns startrunoffset keypatts)
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
@@ -1311,5 +1307,12 @@
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:delete-test-records host port) test-id))
       (db:delete-test-records db test-id)))
+
+(define (rdb:test-data-rollup db test-id status)
+    (if *runremote*
+      (let ((host (vector-ref *runremote* 0))
+	    (port (vector-ref *runremote* 1)))
+	((rpc:procedure 'rdb:test-data-rollup host port) test-id status))
+      (db:test-data-rollup db test-id status)))

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -44,18 +44,20 @@
       (and logpro (eq? exitcode 2))))
 
 (define (launch:execute encoded-cmd)
   (let* ((cmdinfo   (read (open-input-string (base64:base64-decode encoded-cmd)))))
     (setenv "MT_CMDINFO" encoded-cmd)
-    (if (list? cmdinfo) ;; ((testpath /tmp/mrwellan/jazzmind/src/example_run/tests/sqlitespeed) (test-name sqlitespeed) (runscript runscript.rb) (db-host localhost) (run-id 1))
+    (if (list? cmdinfo) ;; ((testpath /tmp/mrwellan/jazzmind/src/example_run/tests/sqlitespeed)
+                        ;; (test-name sqlitespeed) (runscript runscript.rb) (db-host localhost) (run-id 1))
 	(let* ((testpath  (assoc/default 'testpath  cmdinfo))
 	       (work-area (assoc/default 'work-area cmdinfo))
 	       (test-name (assoc/default 'test-name cmdinfo))
 	       (runscript (assoc/default 'runscript cmdinfo))
 	       (ezsteps   (assoc/default 'ezsteps   cmdinfo))
 	       (db-host   (assoc/default 'db-host   cmdinfo))
 	       (run-id    (assoc/default 'run-id    cmdinfo))
+	       (test-id   (assoc/default 'test-id   cmdinfo))
 	       (itemdat   (assoc/default 'itemdat   cmdinfo))
 	       (env-ovrd  (assoc/default 'env-ovrd  cmdinfo))
 	       (set-vars  (assoc/default 'set-vars  cmdinfo)) ;; pre-overrides from -setvar
 	       (runname   (assoc/default 'runname   cmdinfo))
 	       (megatest  (assoc/default 'megatest  cmdinfo))
@@ -102,11 +104,11 @@
 	  (alist->env-vars env-ovrd)
 	  (set-megatest-env-vars db run-id)
 	  (set-item-env-vars itemdat)
 	  (save-environment-as-files "megatest")
 	  (test-set-meta-info db run-id test-name itemdat)
-	  (test-set-status! db run-id test-name "REMOTEHOSTSTART" "n/a" itemdat (args:get-arg "-m") #f)
+	  (test-set-status! db test-id "REMOTEHOSTSTART" "n/a" (args:get-arg "-m") #f)
 	  (if (args:get-arg "-xterm")
 	      (set! fullrunscript "xterm")
 	      (if (and fullrunscript (not (file-execute-access? fullrunscript)))
 		  (system (conc "chmod ug+x " fullrunscript))))
 	  ;; We are about to actually kick off the test
@@ -184,11 +186,11 @@
 						   ;; call the command using mt_ezstep
 						   (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd))
 
 						   (debug:print 4 "script: " script)
 
-						   (rdb:teststep-set-status! db run-id test-name stepname "start" "-" itemdat #f #f)
+						   (rdb:teststep-set-status! db test-id stepname "start" "-" itemdat #f #f)
 						   ;; now launch
 						   (let ((pid (process-run script)))
 						     (let processloop ((i 0))
 						       (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
 								   (mutex-lock! m)
@@ -199,13 +201,16 @@
 								   (if (eq? pid-val 0)
 								       (begin
 									 (thread-sleep! 2)
 									 (processloop (+ i 1))))
 								   ))
-						     (rdb:teststep-set-status! db run-id test-name stepname "end" (vector-ref exit-info 2) itemdat #f (if logpro-used (conc stepname ".html") ""))
+                                                     (let ((exinfo (vector-ref exit-info 2))
+                                                           (logfna (if logpro-used (conc stepname ".html") "")))
+                                                        ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect)
+						        (rdb:teststep-set-status! db test-id stepname "end" exinfo itemdat #f logfna))
 						     (if logpro-used
-							 (test-set-log! db run-id test-name itemdat (conc stepname ".html")))
+							 (rdb:test-set-log! db test-id (conc stepname ".html")))
 						     ;; set the test final status
 						     (let* ((this-step-status (cond
 									       ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)
 									       ((eq? (vector-ref exit-info 2) 0)                   'pass)
 									       (else 'fail)))
@@ -223,18 +228,18 @@
 								    " next-status: " next-status " rollup-status: " rollup-status)
 						       (case next-status
 							 ((warn)
 							  (set! rollup-status 2)
 							  ;; NB// test-set-status! does rdb calls under the hood
-							  (test-set-status! db run-id test-name "RUNNING" "WARN" itemdat 
+							  (test-set-status! db test-id "RUNNING" "WARN" 
 									    (if (eq? this-step-status 'warn) "Logpro warning found" #f)
 									    #f))
 							 ((pass)
-							  (test-set-status! db run-id test-name "RUNNING" "PASS" itemdat #f #f))
+							  (test-set-status! db test-id "RUNNING" "PASS" #f #f))
 							 (else ;; 'fail
 							  (set! rollup-status 1) ;; force fail
-							  (test-set-status! db run-id test-name "RUNNING" "FAIL" itemdat (conc "Failed at step " stepname) #f)
+							  (test-set-status! db test-id "RUNNING" "FAIL" (conc "Failed at step " stepname) #f)
 							  ))))
 						   (if (and (steprun-good? logpro-used (vector-ref exit-info 2))
 							    (not (null? tal)))
 						       (loop (car tal) (cdr tal) stepname)))
 					     (debug:print 4 "WARNING: a prior step failed, stopping at " ezstep))))))))
@@ -278,12 +283,12 @@
 								  (system (conc "kill -9 " p-id))))))
 							(car processes))
 						       (system (conc "kill -9 " pid))))
 						   (begin
 						     (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process")
-						     (test-set-status! db run-id test-name "KILLED"  "FAIL"
-								       itemdat (args:get-arg "-m") #f)
+						     (test-set-status! db test-id "KILLED"  "FAIL"
+								       (args:get-arg "-m") #f)
 						     (sqlite3:finalize! db)
 						     (exit 1))))
 					     (set! kill-tries (+ 1 kill-tries))
 					     (mutex-unlock! m)))
 				       (sqlite3:finalize! db)
@@ -298,15 +303,15 @@
 	    (mutex-lock! m)
 	    (set! db (open-db))
 	    (if (not (args:get-arg "-server"))
 		(server:client-setup db))
 	    (let* ((item-path (item-list->path itemdat))
-		   (testinfo  (db:get-test-info db run-id test-name item-path)))
+		   (testinfo  (rdb:get-test-info db run-id test-name item-path)))
 	      (if (not (equal? (db:test-get-state testinfo) "COMPLETED"))
 		  (begin
 		    (debug:print 2 "Test NOT logged as COMPLETED, (state=" (db:test-get-state testinfo) "), updating result, rollup-status is " rollup-status)
-		    (test-set-status! db run-id test-name
+		    (test-set-status! db test-id 
 				      (if kill-job? "KILLED" "COMPLETED")
 				      ;; Old logic:
 				      ;; (if (vector-ref exit-info 1) ;; look at the exit-status, #t means it at least ran
 				      ;;     (if (and (not kill-job?) 
 				      ;;         (eq? (vector-ref exit-info 2) 0)) ;; we can now use rollup-status instead
@@ -322,11 +327,11 @@
 				       ((eq? rollup-status 1) "FAIL")
 				       ((eq? rollup-status 2)
 					;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
 					(if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
 				       (else "FAIL"))
-				      itemdat (args:get-arg "-m") #f)))
+				      (args:get-arg "-m") #f)))
 	      ;; for automated creation of the rollup html file this is a good place...
 	      (if (not (equal? item-path ""))
 		  (tests:summarize-items db run-id test-name #f)) ;; don't force - just update if no
 	      )
 	    (mutex-unlock! m)
@@ -430,36 +435,42 @@
 ;;    - could be ssh to host from hosts table (update regularly with load)
 ;;    - could be netbatch
 ;;      (launch-test db (cadr status) test-conf))
 (define (launch-test db run-id runname test-conf keyvallst test-name test-path itemdat params)
   (change-directory *toppath*)
-  (let ((useshell   (config-lookup *configdat* "jobtools"     "useshell"))
+  (let* ((useshell   (config-lookup *configdat* "jobtools"     "useshell"))
 	(launcher   (config-lookup *configdat* "jobtools"     "launcher"))
 	(runscript  (config-lookup test-conf   "setup"        "runscript"))
 	(ezsteps    (> (length (hash-table-ref/default test-conf "ezsteps" '())) 0)) ;; don't send all the steps, could be big
 	(diskspace  (config-lookup test-conf   "requirements" "diskspace"))
 	(memory     (config-lookup test-conf   "requirements" "memory"))
 	(hosts      (config-lookup *configdat* "jobtools"     "workhosts"))
 	(remote-megatest (config-lookup *configdat* "setup" "executable"))
 	;; FIXME SOMEDAY: not good how this is so obtuse, this hack is to 
-	;;                allow running from dashboard 
+	;;                allow running from dashboard. Extract the path
+        ;;                from the called megatest and convert dashboard
+  	;;             	  or dboard to megatest
 	(local-megatest  (let* ((lm  (car (argv)))
 				(dir (pathname-directory lm))
 				(exe (pathname-strip-directory lm)))
 			   (conc (if dir (conc dir "/") "")
 				 (case (string->symbol exe)
 				   ((dboard) "megatest")
 				   ((dashboard) "megatest")
 				   (else exe)))))
-	(test-sig   (conc "=" test-name ":" (item-list->path itemdat) "=")) ;; test-path is the full path including the item-path
+	(test-sig   (conc test-name ":" (item-list->path itemdat))) ;; test-path is the full path including the item-path
 	(work-area  #f)
 	(toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
 	(diskpath   #f)
 	(cmdparms   #f)
 	(fullcmd    #f) ;; (define a (with-output-to-string (lambda ()(write x))))
-	(mt-bindir-path #f))
-    (if hosts (set! hosts (string-split hosts)))
+	(mt-bindir-path #f)
+	(item-path (item-list->path itemdat))
+	(testinfo   (rdb:get-test-info db run-id test-name item-path))
+	(test-id    (db:test-get-id testinfo)))
+  (if hosts (set! hosts (string-split hosts)))
+    ;; set the megatest to be called on the remote host
     (if (not remote-megatest)(set! remote-megatest local-megatest)) ;; "megatest"))
     (set! mt-bindir-path (pathname-directory remote-megatest))
     (if launcher (set! launcher (string-split launcher)))
     ;; set up the run work area for this test
     (set! diskpath (get-best-disk *configdat*))
@@ -476,10 +487,11 @@
 				      (write (list (list 'testpath  test-path)
 						   (list 'work-area work-area)
 						   (list 'test-name test-name) 
 						   (list 'runscript runscript) 
 						   (list 'run-id    run-id   )
+						   (list 'test-id   test-id  )
 						   (list 'itemdat   itemdat  )
 						   (list 'megatest  remote-megatest)
 						   (list 'ezsteps   ezsteps) 
  						   (list 'env-ovrd  (hash-table-ref/default *configdat* "env-override" '())) 
 						   (list 'set-vars  (if params (hash-table-ref/default params "-setvars" #f)))
@@ -496,12 +508,11 @@
      (else
       (if (not useshell)(debug:print 0 "WARNING: internal launching will not work well without \"useshell yes\" in your [jobtools] section"))
       (set! fullcmd (list remote-megatest test-sig "-execute" cmdparms (if useshell "&" "")))))
     (if (args:get-arg "-xterm")(set! fullcmd (append fullcmd (list "-xterm"))))
     (debug:print 1 "Launching megatest for test " test-name " in " work-area" ...")
-    (test-set-status! db run-id test-name "LAUNCHED" "n/a" itemdat #f #f) ;; (if launch-results launch-results "FAILED"))
-    ;; set 
+    (test-set-status! db test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED"))
     ;; set pre-launch-env-vars before launching, keep the vars in prevvals and put the envionment back when done
     (debug:print 4 "fullcmd: " fullcmd)
     (let* ((commonprevvals (alist->env-vars
 			    (hash-table-ref/default *configdat* "env-override" '())))
 	   (testprevvals   (alist->env-vars
@@ -518,11 +529,11 @@
 				  print
 				  (if useshell
 				      '()
 				      (cdr fullcmd))))) ;;  launcher fullcmd)));; (apply cmd-run-proc-each-line launcher print fullcmd))) ;; (cmd-run->list fullcmd))
       (debug:print 2 "Launching completed, updating db")
-      (debug:print 4 "Launch results: " launch-results)
+      (debug:print 2 "Launch results: " launch-results)
       (if (not launch-results)
 	  (begin
 	    (print "ERROR: Failed to run " (string-intersperse fullcmd " ") ", exiting now")
 	    (sqlite3:finalize! db)
 	    ;; good ole "exit" seems not to work

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -8,20 +8,21 @@
 ;;  PURPOSE.
 
 ;; (include "common.scm")
 ;; (include "megatest-version.scm")
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 format readline apropos)
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 format readline apropos) ;; (srfi 18) extras)
 (import (prefix sqlite3 sqlite3:))
 (import (prefix base64 base64:))
 
 (declare (uses common))
 (declare (uses megatest-version))
 (declare (uses margs))
 (declare (uses runs))
 (declare (uses launch))
 (declare (uses server))
+(declare (uses tests))
 
 (define *db* #f) ;; this is only for the repl, do not use in general!!!!
 
 (include "common_records.scm")
 (include "key_records.scm")
@@ -161,10 +162,11 @@
 			"-extract-ods"
 			"-pathmod"
 			"-env2file"
 			"-setvars"
 			"-debug" ;; for *verbosity* > 2
+			"-override-timeout"
 			) 
 		 (list  "-h"
 		        "-force"
 		        "-xterm"
 		        "-showkeys"
@@ -337,12 +339,18 @@
 (if (and (args:get-arg "-server")
 	 (not (or (args:get-arg "-runall")
 		  (args:get-arg "-runtests"))))
     (let* ((toppath (setup-for-run))
 	   (db      (if toppath (open-db) #f)))
+      (debug:print 0 "INFO: Starting the standalone server")
       (if db 
-	  (server:start db (args:get-arg "-server"))
+	  (let* ((host:port (db:get-var db "SERVER")) ;; this doen't support multiple servers BUG!!!!
+		 (th2 (server:start db (args:get-arg "-server")))
+		 (th3 (make-thread (lambda ()
+				      (server:keep-running db)))))
+	    (thread-start! th3)
+	    (thread-join! th3))
 	  (debug:print 0 "ERROR: Failed to setup for megatest"))))
 
 ;;======================================================================
 ;; full run
 ;;======================================================================
@@ -455,11 +463,11 @@
 	  (if (not (args:get-arg "-server"))
 	      (server:client-setup db))
 	  (let* ((itempatt (args:get-arg "-itempatt"))
 		 (keys     (rdb:get-keys db))
 		 (keynames (map key:get-fieldname keys))
-		 (paths    (db:test-get-paths-matching db keynames target)))
+		 (paths    (rdb:test-get-paths-matching db keynames target)))
 	    (set! *didsomething* #t)
 	    (for-each (lambda (path)
 			(print path))
 		      paths)))
 	;; else do a general-run-call
@@ -466,11 +474,11 @@
 	(general-run-call 
 	 "-test-paths"
 	 "Get paths to tests"
 	 (lambda (db target runname keys keynames keyvallst)
 	   (let* ((itempatt (args:get-arg "-itempatt"))
-		  (paths    (db:test-get-paths-matching db keynames target)))
+		  (paths    (rdb:test-get-paths-matching db keynames target)))
 	     (for-each (lambda (path)
 			 (print path))
 		       paths))))))
 
 ;;======================================================================
@@ -562,10 +570,11 @@
 	       (testpath  (assoc/default 'testpath  cmdinfo))
 	       (test-name (assoc/default 'test-name cmdinfo))
 	       (runscript (assoc/default 'runscript cmdinfo))
 	       (db-host   (assoc/default 'db-host   cmdinfo))
 	       (run-id    (assoc/default 'run-id    cmdinfo))
+	       (test-id   (assoc/default 'test-id   cmdinfo))
 	       (itemdat   (assoc/default 'itemdat   cmdinfo))
 	       (db        #f)
 	       (state    (args:get-arg ":state"))
 	       (status   (args:get-arg ":status"))
 	       (logfile  (args:get-arg "-setlog")))
@@ -576,11 +585,11 @@
 		(exit 1)))
 	  (set! db (open-db))
 	  (if (not (args:get-arg "-server"))
 	      (server:client-setup db))
 	  (if (and state status)
-	      (rdb:teststep-set-status! db run-id test-name step state status itemdat (args:get-arg "-m") logfile)
+	      (rdb:teststep-set-status! db test-id step state status itemdat (args:get-arg "-m") logfile)
 	      (begin
 		(debug:print 0 "ERROR: You must specify :state and :status with every call to -step")
 		(exit 6)))
 	  (sqlite3:finalize! db)
 	  (set! *didsomething* #t))))
@@ -601,10 +610,11 @@
 	       (testpath  (assoc/default 'testpath  cmdinfo))
 	       (test-name (assoc/default 'test-name cmdinfo))
 	       (runscript (assoc/default 'runscript cmdinfo))
 	       (db-host   (assoc/default 'db-host   cmdinfo))
 	       (run-id    (assoc/default 'run-id    cmdinfo))
+	       (test-id   (assoc/default 'test-id   cmdinfo))
 	       (itemdat   (assoc/default 'itemdat   cmdinfo))
 	       (db        #f)
 	       (state     (args:get-arg ":state"))
 	       (status    (args:get-arg ":status")))
 	  (change-directory testpath)
@@ -614,17 +624,18 @@
 		(exit 1)))
 	  (set! db (open-db))
 	  (if (not (args:get-arg "-server"))
 	      (server:client-setup db))
 	  (if (args:get-arg "-load-test-data")
-	      (db:load-test-data db run-id test-name itemdat))
+	      ;; has sub commands that are rdb:
+	      (db:load-test-data db test-id))
 	  (if (args:get-arg "-setlog")
-	      (test-set-log! db run-id test-name itemdat (args:get-arg "-setlog")))
+	      (rtests:test-set-log! db test-id (args:get-arg "-setlog")))
 	  (if (args:get-arg "-set-toplog")
-	      (test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
+	      (rtests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
 	  (if (args:get-arg "-summarize-items")
-	      (tests:summarize-items db run-id test-name #t)) ;; do force here
+	      (rdb:tests:summarize-items db run-id test-name #t)) ;; do force here
 	  (if (args:get-arg "-runstep")
 	      (if (null? remargs)
 		  (begin
 		    (debug:print 0 "ERROR: nothing specified to run!")
 		    (sqlite3:finalize! db)
@@ -636,28 +647,29 @@
 			 (params     (if cmd (cdr remargs) '()))
 			 (exitstat   #f)
 			 (shell      (last (string-split (get-environment-variable "SHELL") "/")))
 			 (redir      (case (string->symbol shell)
 				       ((tcsh csh ksh)    ">&")
-				       ((zsh bash sh ash) "2>&1 >")))
+				       ((zsh bash sh ash) "2>&1 >")
+				       (else ">&")))
 			 (fullcmd    (conc "(" (string-intersperse 
 						(cons cmd params) " ")
 					   ") " redir " " logfile)))
 		    ;; mark the start of the test
-		    (rdb:teststep-set-status! db run-id test-name stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
+		    (rdb:teststep-set-status! db test-id stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
 		    ;; close the db
-		    (sqlite3:finalize! db)
+		    ;; (sqlite3:finalize! db)
 		    ;; run the test step
 		    (debug:print 2 "INFO: Running \"" fullcmd "\"")
 		    (change-directory startingdir)
 		    (set! exitstat (system fullcmd)) ;; cmd params))
 		    (set! *globalexitstatus* exitstat)
 		    (change-directory testpath)
 		    ;; re-open the db
-		    (set! db (open-db))
-		    (if (not (args:get-arg "-server"))
-			(server:client-setup db))
+		    ;; (set! db (open-db))
+		    ;; (if (not (args:get-arg "-server"))
+		    ;;     (server:client-setup db))
 		    ;; run logpro if applicable ;; (process-run "ls" (list "/foo" "2>&1" "blah.log"))
 		    (if logprofile
 			(let* ((htmllogfile (conc stepname ".html"))
 			       (oldexitstat exitstat)
 			       (cmd         (string-intersperse (list "logpro" logprofile htmllogfile "<" logfile ">" (conc stepname "_logpro.log")) " ")))
@@ -664,15 +676,16 @@
 			  (debug:print 2 "INFO: running \"" cmd "\"")
 			  (change-directory startingdir)
 			  (set! exitstat (system cmd))
 			  (set! *globalexitstatus* exitstat) ;; no necessary
 			  (change-directory testpath)
-			  (test-set-log! db run-id test-name itemdat htmllogfile)))
-		    (rdb:teststep-set-status! db run-id test-name stepname "end" exitstat itemdat (args:get-arg "-m") logfile)
-		    (sqlite3:finalize! db)
-		    (if (not (eq? exitstat 0))
-			(exit 254)) ;; (exit exitstat) doesn't work?!?
+			  (rdb:test-set-log! db test-id htmllogfile)))
+		    (let ((msg (args:get-arg "-m")))
+		      (rdb:teststep-set-status! db test-id stepname "end" exitstat itemdat msg logfile))
+		    ;; (sqlite3:finalize! db)
+		    ;;(if (not (eq? exitstat 0))
+		    ;;	(exit 254)) ;; (exit exitstat) doesn't work?!?
 		  ;; open the db
 		  ;; mark the end of the test
 		  )))
 	  (if (or (args:get-arg "-test-status")
 		  (args:get-arg "-set-values"))
@@ -694,11 +707,12 @@
 			     (not status)))
 		    (begin
 		      (debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help)
 		      (sqlite3:finalize! db)
 		      (exit 6)))
-		(test-set-status! db run-id test-name state newstatus itemdat (args:get-arg "-m") otherdata)))
+		(let ((msg (args:get-arg "-m")))
+		  (rtests:test-set-status! db test-id state newstatus msg otherdata))))
 	  (sqlite3:finalize! db)
 	  (set! *didsomething* #t))))
 
 (if (args:get-arg "-showkeys")
     (let ((db #f)

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -8,11 +8,11 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 ;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking)
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18))
 (import (prefix sqlite3 sqlite3:))
 
 (declare (unit runs))
 (declare (uses db))
 (declare (uses common))
@@ -259,10 +259,11 @@
 
     (if (not (null? required-tests))
 	(debug:print 1 "INFO: Adding " required-tests " to the run queue"))
     ;; NOTE: these are all parent tests, items are not expanded yet.
     (runs:run-tests-queue db run-id runname test-records keyvallst flags)
+    (if *rpc:listener* (server:keep-running db))
     (debug:print 4 "INFO: All done by here")))
 
 (define (runs:run-tests-queue db run-id runname test-records keyvallst flags)
     ;; At this point the list of parent tests is expanded 
     ;; NB// Should expand items here and then insert into the run queue.
@@ -362,12 +363,13 @@
 	
       ;; we get here on "drop through" - loop for next test in queue
       (if (null? tal)
 	  (begin
 	    ;; FIXME!!!! THIS SHOULD NOT REQUIRE AN EXIT!!!!!!!
-	    (debug:print 1 "INFO: All tests launched, exiting")
-	    (exit 0))
+	    (debug:print 1 "INFO: All tests launched")
+	    ;; (exit 0)
+	    )
 	  (loop (car tal)(cdr tal))))))
 
 ;; parent-test is there as a placeholder for when parent-tests can be run as a setup step
 (define (run:test db run-id runname keyvallst test-record flags parent-test)
   ;; All these vars might be referenced by the testconfig file reader
@@ -392,23 +394,29 @@
     (setenv "MT_RUNNAME"   runname)
     (set-megatest-env-vars db run-id) ;; these may be needed by the launching process
     (change-directory *toppath*)
 
     ;; Here is where the test_meta table is best updated
-    (runs:update-test_meta db test-name test-conf)
+    ;; Yes, another use of a global for caching. Need a better way?
+    (if (not (hash-table-ref/default *test-meta-updated* test-name #f))
+        (begin
+	   (hash-table-set! *test-meta-updated* test-name #t)
+           (runs:update-test_meta db test-name test-conf)))
     
     ;; (lambda (itemdat) ;;; ((ripeness "overripe") (temperature "cool") (season "summer"))
     (let* ((new-test-path (string-intersperse (cons test-path (map cadr itemdat)) "/"))
 	   (new-test-name (if (equal? item-path "") test-name (conc test-name "/" item-path))) ;; just need it to be unique
-	   (testdat       (db:get-test-info db run-id test-name item-path)))
+	   (testdat       (db:get-test-info db run-id test-name item-path))
+	   (test-id       #f))
       (if (not testdat)
 	  (begin
 	    ;; ensure that the path exists before registering the test
 	    ;; NOPE: Cannot! Don't know yet which disk area will be assigned....
 	    ;; (system (conc "mkdir -p " new-test-path))
 	    (rtests:register-test db run-id test-name item-path)
 	    (set! testdat (db:get-test-info db run-id test-name item-path))))
+      (set! test-id (db:test-get-id testdat))
       (change-directory test-path)
       (case (if force ;; (args:get-arg "-force")
 		'NOT_STARTED
 		(if testdat
 		    (string->symbol (test:get-state testdat))
@@ -449,11 +457,12 @@
 	    (else (set! runflag #f)))
 	   (debug:print 6 "RUNNING => runflag: " runflag " STATE: " (test:get-state testdat) " STATUS: " (test:get-status testdat))
 	   (if (not runflag)
 	       (if (not parent-test)
 		   (debug:print 1 "NOTE: Not starting test " new-test-name " as it is state \"" (test:get-state testdat) 
-				"\" and status \"" (test:get-status testdat) "\", use -rerun \"" (test:get-status testdat) "\" or -force to override"))
+				"\" and status \"" (test:get-status testdat) "\", use -rerun \"" (test:get-status testdat)
+                                "\" or -force to override"))
 	       ;; NOTE: No longer be checking prerequisites here! Will never get here unless prereqs are
 	       ;;       already met.
 	       (if (not (launch-test db run-id runname test-conf keyvallst test-name test-path itemdat flags))
 		   (begin
 		     (print "ERROR: Failed to launch the test. Exiting as soon as possible")
@@ -465,11 +474,11 @@
 	 (if (> (- (current-seconds)(+ (db:test-get-event_time testdat)
 				       (db:test-get-run_duration testdat)))
 		600) ;; i.e. no update for more than 600 seconds
 	     (begin
 	       (debug:print 0 "WARNING: Test " test-name " appears to be dead. Forcing it to state INCOMPLETE and status STUCK/DEAD")
-	       (test-set-status! db run-id test-name "INCOMPLETE" "STUCK/DEAD" itemdat "Test is stuck or dead" #f))
+	       (test-set-status! db test-id "INCOMPLETE" "STUCK/DEAD" "Test is stuck or dead" #f))
 	     (debug:print 2 "NOTE: " test-name " is already running")))
 	(else       (debug:print 0 "ERROR: Failed to launch test " new-test-name ". Unrecognised state " (test:get-state testdat)))))))
 
 ;;======================================================================
 ;; END OF NEW STUFF
@@ -571,11 +580,12 @@
 ;; this wrapper is used to reduce the replication of code
 (define (general-run-call switchname action-desc proc)
   (let ((runname (args:get-arg ":runname"))
 	(target  (if (args:get-arg "-target")
 		     (args:get-arg "-target")
-		     (args:get-arg "-reqtarg"))))
+		     (args:get-arg "-reqtarg")))
+	(th1     #f))
     (cond
      ((not target)
       (debug:print 0 "ERROR: Missing required parameter for " switchname ", you must specify the target with -target")
       (exit 3))
      ((not runname)
@@ -587,12 +597,15 @@
 	(if (not (setup-for-run))
 	    (begin 
 	      (debug:print 0 "Failed to setup, exiting")
 	      (exit 1)))
 	(set! db   (open-db))
-	(if (not (args:get-arg "-server"))
-	    (server:client-setup db))
+	(if (args:get-arg "-server")
+	    (server:start db (args:get-arg "-server"))
+	    (if (not (or (args:get-arg "-runall")
+			  (args:get-arg "-runtests")))
+		(server:client-setup db)))
 	(set! keys (rdb:get-keys db))
 	;; have enough to process -target or -reqtarg here
 	(if (args:get-arg "-reqtarg")
 	    (let* ((runconfigf (conc  *toppath* "/runconfigs.config")) ;; DO NOT EVALUATE ALL 
 		   (runconfig  (read-config runconfigf #f #f environ-patt: #f))) 
@@ -611,10 +624,11 @@
 	    ;; Extract out stuff needed in most or many calls
 	    ;; here then call proc
 	    (let* ((keynames   (map key:get-fieldname keys))
 		   (keyvallst  (keys->vallist keys #t)))
 	      (proc db target runname keys keynames keyvallst)))
+	(if th1 (thread-join! th1))
 	(sqlite3:finalize! db)
 	(set! *didsomething* #t))))))
 
 ;;======================================================================
 ;; Rollup runs

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -62,153 +62,222 @@
     ;;======================================================================
     ;; ** set-tests-state-status
     (rpc:publish-procedure!
      'rdb:set-tests-state-status 
      (lambda (run-id testnames currstate currstatus newstate newstatus)
+       (set! *last-db-access* (current-seconds))
        (db:set-tests-state-status db run-id testnames currstate currstatus newstate newstatus)))
 
     (rpc:publish-procedure!
      'rdb:teststep-set-status!
-     (lambda (run-id test-name teststep-name state-in status-in item-path comment logfile)
-       (db:teststep-set-status! db run-id test-name teststep-name state-in status-in item-path comment logfile)))
+     (lambda (test-id teststep-name state-in status-in item-path comment logfile)
+       (set! *last-db-access* (current-seconds))
+       (db:teststep-set-status! db test-id teststep-name state-in status-in item-path comment logfile)))
 
     (rpc:publish-procedure!
      'rdb:test-update-meta-info
      (lambda (run-id testname item-path minutes cpuload diskfree tmpfree)
+       (set! *last-db-access* (current-seconds))
        (db:test-update-meta-info db run-id testname item-path minutes cpuload diskfree tmpfree)))
      
     (rpc:publish-procedure!
      'rdb:test-set-state-status-by-run-id-testname
      (lambda (run-id test-name item-path status state)
+       (set! *last-db-access* (current-seconds))
        (db:test-set-state-status-by-run-id-testname db run-id test-name item-path status state)))
 
     (rpc:publish-procedure!
      'rdb:csv->test-data 
      (lambda (test-id csvdata)
-       (db:csv->data db test-id csvdata)))
+       (set! *last-db-access* (current-seconds))
+       (db:csv->test-data db test-id csvdata)))
 
     (rpc:publish-procedure!
      'rdb:roll-up-pass-fail-counts
      (lambda (run-id test-name item-path status)
+       (set! *last-db-access* (current-seconds))
        (db:roll-up-pass-fail-counts db run-id test-name item-path status)))
 
     (rpc:publish-procedure!
      'rdb:test-set-comment 
      (lambda (run-id test-name item-path comment)
+       (set! *last-db-access* (current-seconds))
        (db:test-set-comment db run-id test-name item-path comment)))
     
     (rpc:publish-procedure!
      'rdb:test-set-log!
-     (lambda (run-id test-name item-path logf)
-       (db:test-set-log! db run-id test-name item-path logf)))
+     (lambda (test-id logf)
+       (set! *last-db-access* (current-seconds))
+       (db:test-set-log! db test-id logf)))
     
     (rpc:publish-procedure!
-     'rpc:get-test-data-by-id
+     'rdb:get-test-data-by-id
      (lambda (test-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-test-data-by-id db test-id)))
 
     (rpc:publish-procedure!
      'serve:get-toppath
      (lambda ()
+       (set! *last-db-access* (current-seconds))
        *toppath*))
 
     (rpc:publish-procedure!
      'serve:login
      (lambda (toppath)
+       (set! *last-db-access* (current-seconds))
        (if (equal? *toppath* toppath)
 	   (begin
 	     (debug:print 2 "INFO: login successful")
 	     #t)
 	   #f)))	     
     
     (rpc:publish-procedure!
      'rdb:get-runs
      (lambda (runnamepatt numruns startrunoffset keypatts)
+       (set! *last-db-access* (current-seconds))
        (db:get-runs db runnamepatt numruns startrunoffset keypatts)))
 
     (rpc:publish-procedure!
      'rdb:get-tests-for-run 
      (lambda (run-id testpatt itempatt states statuses)
+       (set! *last-db-access* (current-seconds))
        (db:get-tests-for-run db run-id testpatt itempatt states statuses)))
 
     (rpc:publish-procedure!
      'rdb:get-keys
      (lambda ()
+       (set! *last-db-access* (current-seconds))
        (db:get-keys db)))
 
     (rpc:publish-procedure!
      'rdb:get-num-runs
      (lambda (runpatt)
+       (set! *last-db-access* (current-seconds))
        (db:get-num-runs db runpatt)))
 
     (rpc:publish-procedure!
      'rdb:test-set-state-status-by-id
      (lambda (test-id newstate newstatus newcomment)
+       (set! *last-db-access* (current-seconds))
        (db:test-set-state-status-by-id db test-id newstate newstatus newcomment)))
 
     (rpc:publish-procedure!
      'rdb:get-key-val-pairs
      (lambda (run-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-key-val-pairs db run-id)))
 
     (rpc:publish-procedure!
      'rdb:get-key-vals
      (lambda (run-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-key-vals db run-id)))
 
     (rpc:publish-procedure!
      'rdb:testmeta-get-record
      (lambda (run-id)
+       (set! *last-db-access* (current-seconds))
        (db:testmeta-get-record db run-id)))
 
     (rpc:publish-procedure!
      'rdb:get-test-data-by-id
      (lambda (test-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-test-data-by-id db test-id)))
 
     (rpc:publish-procedure!
      'rdb:get-run-info
      (lambda (run-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-run-info db run-id)))
 
     (rpc:publish-procedure!
      'rdb:get-steps-for-test
      (lambda (test-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-steps-for-test db test-id)))
 
     (rpc:publish-procedure!
      'rdb:get-steps-table
      (lambda (test-id)
+       (set! *last-db-access* (current-seconds))
        (db:get-steps-table db test-id)))
 
     (rpc:publish-procedure!
      'rdb:read-test-data
      (lambda (test-id categorypatt)
+       (set! *last-db-access* (current-seconds))
        (db:read-test-data db test-id categorypatt)))
 
     (rpc:publish-procedure!
      'rdb:get-test-info
      (lambda (run-id testname item-path)
+       (set! *last-db-access* (current-seconds))
        (db:get-test-info db  run-id testname item-path)))
 
     (rpc:publish-procedure!
      'rdb:delete-test-records
      (lambda (test-id)
+       (set! *last-db-access* (current-seconds))
        (db:delete-test-records db test-id)))
 
     (rpc:publish-procedure!
      'rtests:register-test
      (lambda (run-id test-name item-path)
+       (set! *last-db-access* (current-seconds))
        (tests:register-test db run-id test-name item-path)))
 
+    (rpc:publish-procedure!
+     'rdb:test-data-rollup
+     (lambda (test-id status)
+       (set! *last-db-access* (current-seconds))
+       (db:test-data-rollup db test-id status)))
+    
+    (rpc:publish-procedure!
+     'rtests:test-set-status!
+     (lambda (test-id state status comment dat)
+       (set! *last-db-access* (current-seconds))
+       (test-set-status! db test-id state status comment dat)))
+
+    (rpc:publish-procedure!
+     'rtests:test-set-toplog!
+     (lambda (run-id test-name logf)
+        (set! *last-db-access* (current-seconds))
+        (test-set-toplog! db run-id test-name logf)))
+
+    ;;======================================================================
+    ;; end of publish-procedure section
+    ;;======================================================================
+
     (set! *rpc:listener* rpc:listener)
     (on-exit (lambda ()
 	       (sqlite3:execute db "DELETE FROM metadat WHERE var='SERVER' and val=?;" host:port)
 	       (sqlite3:finalize! db)))
     (thread-start! th1)
     (thread-start! th2)
-    (thread-join!  th2))) ;; rpc:server)))
+    ;; (thread-join!  th2)
+    ;; return th2 for the calling process to do a join with 
+    th2
+    )) ;; rpc:server)))
+
+(define (server:keep-running db)
+  ;; if none running or if > 20 seconds since 
+  ;; server last used then start shutdown
+  (let loop ((count 0))
+    (thread-sleep! 20) ;; no need to do this very often
+    (let ((numrunning (db:get-count-tests-running db)))
+      (if (or (not (> numrunning 0))
+	      (> *last-db-access* (+ (current-seconds) 20)))
+	  (begin
+	    (debug:print 0 "INFO: Starting to shutdown the server side")
+	    (sqlite3:execute db "DELETE FROM metadat WHERE var='SERVER'"); ;;  AND val like ?;"
+			  ;; host:port) ;; need to delete only *my* server entry (future use)
+	    (thread-sleep! 10)
+	    (debug:print 0 "INFO: Server shutdown complete. Exiting")
+	    (exit))))
+    (loop (+ 1 count))))
 
 (define (server:find-free-port-and-open port)
   (handle-exceptions
    exn
    (begin
@@ -217,11 +286,13 @@
    (rpc:default-server-port port)
    (tcp-listen (rpc:default-server-port))))
 
 (define (server:client-setup db)
   (if *runremote*
-      (debug:print 0 "ERROR: Attempt to connect to server but already connected")
+      (begin
+	(debug:print 0 "ERROR: Attempt to connect to server but already connected")
+	#f)
       (let* ((hostinfo (db:get-var db "SERVER"))
 	     (hostdat  (if hostinfo (string-split hostinfo ":")))
 	     (host     (if hostinfo (car hostdat)))
 	     (port     (if (and hostinfo (> (length hostdat) 1))(cadr hostdat) #f)))
 	(if (and port

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -51,11 +51,11 @@
 	  ;; if found then return that matching test record
 	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids)
 	  (if (null? prev-run-ids) #f
 	      (let loop ((hed (car prev-run-ids))
 			 (tal (cdr prev-run-ids)))
-		(let ((results (rdb:get-tests-for-run db hed test-name item-path '() '())))
+		(let ((results (db:get-tests-for-run db hed test-name item-path '() '())))
 		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results)
 		  (if (and (null? results)
 			   (not (null? tal)))
 		      (loop (car tal)(cdr tal))
 		      (if (null? results) #f
@@ -108,16 +108,17 @@
 		  (if (null? tal)
 		      (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests
 		      (loop (car tal)(cdr tal))))))))))
 
 ;; 
-(define (test-set-status! db run-id test-name state status itemdat-or-path comment dat)
+(define (test-set-status! db test-id state status comment dat)
   (let* ((real-status status)
-	 (item-path   (if (string? itemdat-or-path) itemdat-or-path (item-list->path itemdat-or-path)))
-	 (testdat     (db:get-test-info db run-id test-name item-path))
-	 (test-id     (if testdat (db:test-get-id testdat) #f))
 	 (otherdat    (if dat dat (make-hash-table)))
+	 (testdat     (db:get-test-data-by-id db test-id))
+	 (run-id      (db:test-get-run_id testdat))
+	 (test-name   (db:test-get-testname   testdat))
+	 (item-path   (db:test-get-item-path testdat))
 	 ;; before proceeding we must find out if the previous test (where all keys matched except runname)
 	 ;; was WAIVED if this test is FAIL
 	 (waived   (if (equal? status "FAIL")
 		       (let ((prev-test (test:get-previous-test-run-record db run-id test-name item-path)))
 			 (if prev-test ;; true if we found a previous test in this run series
@@ -165,33 +166,31 @@
 	  (dcomment (hash-table-ref/default otherdat ":comment"  "")))
       (debug:print 4 
 		   "category: " category ", variable: " variable ", value: " value
 		   ", expected: " expected ", tol: " tol ", units: " units)
       (if (and value expected tol) ;; all three required
-	  (rdb:csv->test-data db test-id 
-			     (conc category ","
-				   variable ","
-				   value    ","
-				   expected ","
-				   tol      ","
-				   units    ","
-				   dcomment ",," ;; extra comma for status
-				   type     ))))
-				   
+	  (let ((dat (conc category ","
+			   variable ","
+			   value    ","
+			   expected ","
+			   tol      ","
+			   units    ","
+			   dcomment ",," ;; extra comma for status
+			   type     )))
+	    (rdb:csv->test-data db test-id
+				dat))))
+      
     ;; need to update the top test record if PASS or FAIL and this is a subtest
     (rdb:roll-up-pass-fail-counts db run-id test-name item-path status)
 
     (if (or (and (string? comment)
 		 (string-match (regexp "\\S+") comment))
 	    waived)
-	(rdb:test-set-comment db  run-id test-name item-path (if waived waived comment)))
+	(let ((cmt  (if waived waived comment)))
+	  (rdb:test-set-comment db test-id cmt)))
     ))
 
-(define (test-set-log! db run-id test-name itemdat logf) 
-  (let ((item-path (item-list->path itemdat)))
-    (rdb:test-set-log! db run-id test-name item-path logf)))
-
 (define (test-set-toplog! db run-id test-name logf) 
   (sqlite3:execute db "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path='';" 
 		   logf run-id test-name))
 
 (define (tests:summarize-items db run-id test-name force)
@@ -387,5 +386,21 @@
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rtests:register-test host port) run-id test-name item-path))
       (tests:register-test db run-id test-name item-path)))
+
+(define (rtests:test-set-status!  db test-id state status comment dat)
+  (if *runremote*
+      (let ((host (vector-ref *runremote* 0))
+	    (port (vector-ref *runremote* 1)))
+	((rpc:procedure 'rtests:test-set-status! host port) test-id state status comment dat))
+      (test-set-status! db test-id state status comment dat)))
+
+(define (rtests:test-set-toplog! db run-id test-name logf)
+  (if *runremote*
+      (let ((host (vector-ref *runremote* 0))
+            (port (vector-ref *runremote* 1)))
+        ((rpc:procedure 'rtests:test-set-toplog! host port) run-id test-name logf))
+      (test-set-toplog! db run-id test-name logf)))
+
+

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -1,16 +1,39 @@
 # run some tests
 
 BINPATH=$(shell realpath ../bin)
 MEGATEST=$(BINPATH)/megatest
 PATH := $(BINPATH):$(PATH)
+RUNNAME := $(shell date +w%V.%u.%H)
+IPADDR :="-"
+
+runall : test1 test2
+
+test1 : cleanprep
+	$(MEGATEST) -runtests ez_pass -target ubuntu/nfs/none :runname $(RUNNAME)_a -server $(IPADDR)
+
+test2 : cleanprep
+	$(MEGATEST) -runtests runfirst -target ubuntu/nfs/none :runname $(RUNNAME)_b  -server $(IPADDR) -debug 10
+
+test3 : cleanprep
+	$(MEGATEST) -runall -target ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v -server $(IPADDR) 
 
-runall :
-	cd ../;make install
+cleanprep : ../*.scm
+	sqlite3 megatest.db "delete from metadat where var='SERVER';"
 	mkdir -p /tmp/mt_runs /tmp/mt_links
+	cd ..;make
+	@sleep 1
+	@if ps -def |awk '{print $8}'|grep megatest; then \
+	   echo WARNING: These tests will kill megatest and dashboard!; \
+	   sleep 3; \
+	   killall -9 dboard || true; \
+  	   killall -9 megatest || true; \
+	fi
+	cd ../;make install
+	$(MEGATEST) -remove-runs :runname $(RUNNAME)% -target %/%/% -testpatt % -itempatt %
 	$(BINPATH)/dboard -rows 15 &
-	$(MEGATEST) -runall -target ubuntu/nfs/none :runname `date +w%V.%u.%H` -m "This is a comment specific to a run" -v
+	touch cleanprep
 
 test :
 	csi -b -I .. ../megatest.scm -- -runall -target ubuntu/afs/tmp :runname blah
 	cd ../;make test
 	make runall
@@ -20,7 +43,10 @@
 	$(BINPATH)/dboard &
 
 remove :
 	(cd ../;make);$(MEGATEST) -remove-runs :runname $(RUN)  -testpatt % -itempatt % :sysname % :fsname % :datapath %
 
+clean  : 
+	rm cleanprep
+
 runforever :
 	while(ls); do runname=`date +%F-%R:%S`;$(MEGATEST) -runall -target ubuntu/nfs/none :runname $$runname;/home/matt/data/megatest/megatest -runall -target ubuntu/nfs/none :runname $$runname;/home/matt/data/megatest/megatest -runall -target ubuntu/nfs/none :runname $$runname;done

Index: tests/megatest.config
==================================================================
--- tests/megatest.config
+++ tests/megatest.config
@@ -7,11 +7,11 @@
 # exectutable /path/to/megatest
 max_concurrent_jobs 200
 linktree /tmp/mt_links
 
 [jobtools]
-# useshell yes
+useshell yes
 # ## launcher launches jobs, the job is managed on the target host
 ## by megatest, comment out launcher to run local
 # workhosts localhost hermes
 launcher nbfake
 # launcher nodanggood

Index: utils/mt_ezstep
==================================================================
--- utils/mt_ezstep
+++ utils/mt_ezstep
@@ -27,14 +27,18 @@
 
 # source the environment from the previous step if it exists
 
 # if a logpro file exists then use it otherwise just run the command, nb// was using 2>&1
 if [ -e ${stepname}.logpro ];then
-   $command 2>&1| logpro ${stepname}.logpro ${stepname}.html &> ${stepname}.log 
+   # could do:
+   $command 2>&1| tee ${stepname}.log | logpro ${stepname}.logpro ${stepname}.html &> /dev/null
+   logprostatus=$?
+   # $command 2>&1| logpro ${stepname}.logpro ${stepname}.html &> ${stepname}.log 
+   # allstatus=(${PIPESTATUS[0]} ${PIPESTATUS[1]}) 
    allstatus=(${PIPESTATUS[0]} ${PIPESTATUS[1]})
    runstatus=${allstatus[0]}
-   logprostatus=${allstatus[1]}
+   # logprostatus=${allstatus[1]}
 else
    $command &> ${stepname}.log
    runstatus=$?
    logprostatus=$runstatus
 fi