Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -36,11 +36,12 @@
     ((get-previous-test-run-record)    (apply db:get-previous-test-run-record params))
     ((get-matching-previous-test-run-records)(map vector->list (apply db:get-matching-previous-test-run-records db params)))
     ((db:test-get-logfile-info)        (apply db:test-get-logfile-info db params))
     ((test-get-records-for-index-file  (apply db:test-get-records-for-index-file db params)))
     ((get-testinfo-state-status)       (apply db:get-testinfo-state-status db params))
-    ((update-testdat-meta-info)        (apply db:update-testdat-meta-info db params))
+    ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new params))
+    ((get-prereqs-not-met)             (apply db:get-prereqs-not-met params))
 
     ;; RUNS
     ((get-run-info)                 (let ((res (apply db:get-run-info db params)))
 				      (list (vector-ref res 0)
 					    (vector->list (vector-ref res 1)))))
@@ -53,16 +54,38 @@
     ((delete-run)                   (apply db:delete-run db params))
     ((get-runs)                     (let* ((res  (apply db:get-runs db params))
 					   (hedr (vector-ref res 0))
 					   (data (vector-ref res 1)))
 				      (list hedr (map vector->list data))))
+    ((get-runs-by-patt)             (let* ((res  (apply db:get-runs-by-patt db params))
+					   (hedr (vector-ref res 0))
+					   (data (vector-ref res 1)))
+				      (list hedr (map vector->list data))))
 
     ;; MISC
     ((login)                        (apply db:login db params))
     ((general-call)                 (let ((stmtname   (car params))
 					  (realparams (cdr params)))
 				      (db:general-call db stmtname realparams)))
+    ((kill-server)
+     (db:sync-to *inmemdb* *db*)
+     (let ((hostname (car  *runremote*))
+	   (port     (cadr *runremote*))
+	   (pid      (if (null? params) #f (car params)))
+	   (th1      (make-thread (lambda ()(thread-sleep! 3)(debug:print 0 "Server exiting!")(exit 0)) "Server exit thread")))
+       (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!")
+       (debug:print-info 1 "current pid=" (current-process-id))
+       (open-run-close tasks:server-deregister tasks:open-db 
+		       hostname
+		       port: port)
+       (set! *server-run* #f)
+       (thread-sleep! 3)
+       (if pid 
+	   (process-signal pid signal/kill)
+	   (thread-start! th1))
+       '(#t "exit process started")))
+
     (else
      (list "ERROR" 0))))
 
 ;; http-server  send-response
 ;;                 api:process-request

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -376,110 +376,10 @@
     ;; Must do this *after* running patch db !! No more. 
     (db:set-var db "MEGATEST_VERSION" megatest-version)
     (debug:print-info 11 "db:initialize END")
     ))
 
-;;======================================================================
-;; T E S T   S P E C I F I C   D B 
-;;======================================================================
-
-;; Create the sqlite db for the individual test(s)
-(define (open-test-db work-area) 
-  (debug:print-info 11 "open-test-db " work-area)
-  (if (and work-area 
-	   (directory? work-area)
-	   (file-read-access? work-area))
-      (let* ((dbpath    (conc work-area "/testdat.db"))
-	     (tdb-writeable (file-write-access? dbpath))
-	     (dbexists  (file-exists? dbpath))
-	     (handler   (make-busy-timeout (if (args:get-arg "-override-timeout")
-					       (string->number (args:get-arg "-override-timeout"))
-					       136000))))
-	(handle-exceptions
-	 exn
-	 (begin
-	   (debug:print 2 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test"
-			((condition-property-accessor 'exn 'message) exn))
-	   (set! db (sqlite3:open-database ":memory:"))) ;; open an in-memory db to allow readonly access 
-	 (set! db (sqlite3:open-database dbpath)))
-	(if *db-write-access* (sqlite3:set-busy-handler! db handler))
-	(if (not dbexists)
-	    (begin
-	      (sqlite3:execute db "PRAGMA synchronous = FULL;")
-	      (debug:print-info 11 "Initialized test database " dbpath)
-	      (db:testdb-initialize db)))
-	;; (sqlite3:execute db "PRAGMA synchronous = 0;")
-	(debug:print-info 11 "open-test-db END (sucessful)" work-area)
-	;; now let's test that everything is correct
-	(handle-exceptions
-	 exn
-	 (begin
-	   (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test"
-			((condition-property-accessor 'exn 'message) exn))
-	   #f)
-	 ;; Is there a cheaper single line operation that will check for existance of a table
-	 ;; and raise an exception ?
-	 (sqlite3:execute db "SELECT id FROM test_data LIMIT 1;"))
-	db)
-      (begin
-	(debug:print-info 11 "open-test-db END (unsucessful)" work-area)
-	#f)))
-
-;; find and open the testdat.db file for an existing test
-(define (db:open-test-db-by-test-id db test-id #!key (work-area #f))
-  (let* ((test-path (if work-area
-			work-area
-			(db:test-get-rundir-from-test-id db test-id))))
-    (debug:print 3 "TEST PATH: " test-path)
-    (open-test-db test-path)))
-
-(define (db:testdb-initialize db)
-  (debug:print 11 "db:testdb-initialize START")
-  (for-each
-   (lambda (sqlcmd)
-     (sqlite3:execute db sqlcmd))
-   (list "CREATE TABLE IF NOT EXISTS test_rundat (
-              id INTEGER PRIMARY KEY,
-              update_time TIMESTAMP,
-              cpuload INTEGER DEFAULT -1,
-              diskfree INTEGER DEFAULT -1,
-              diskusage INTGER DEFAULT -1,
-              run_duration INTEGER DEFAULT 0);"
-	 "CREATE TABLE IF NOT EXISTS test_data (
-              id INTEGER PRIMARY KEY,
-              test_id INTEGER,
-              category TEXT DEFAULT '',
-              variable TEXT,
-	      value REAL,
-	      expected REAL,
-	      tol REAL,
-              units TEXT,
-              comment TEXT DEFAULT '',
-              status TEXT DEFAULT 'n/a',
-              type TEXT DEFAULT '',
-              CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable));"
-	 "CREATE TABLE IF NOT EXISTS test_steps (
-              id INTEGER PRIMARY KEY,
-              test_id INTEGER, 
-              stepname TEXT, 
-              state TEXT DEFAULT 'NOT_STARTED', 
-              status TEXT DEFAULT 'n/a',
-              event_time TIMESTAMP,
-              comment TEXT DEFAULT '',
-              logfile TEXT DEFAULT '',
-              CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state));"
-	 ;; test_meta can be used for handing commands to the test
-	 ;; e.g. KILLREQ
-	 ;;      the ackstate is set to 1 once the command has been completed
-	 "CREATE TABLE IF NOT EXISTS test_meta (
-              id INTEGER PRIMARY KEY,
-              var TEXT,
-              val TEXT,
-              ackstate INTEGER DEFAULT 0,
-              CONSTRAINT metadat_constraint UNIQUE (var));"))
-  (debug:print 11 "db:testdb-initialize END"))
-
 ;;======================================================================
 ;; L O G G I N G    D B 
 ;;======================================================================
 
 (define (open-logging-db) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
@@ -1268,28 +1168,16 @@
      db 
      qry
      )
     res))
 
-;; this one is a bit broken BUG FIXME
-(define (db:delete-test-step-records db test-id #!key (work-area #f))
-  ;; Breaking it into two queries for better file access interleaving
-  (let* ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area)))
-    ;; test db's can go away - must check every time
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:execute tdb "DELETE FROM test_steps;")
-	  (sqlite3:execute tdb "DELETE FROM test_data;")
-	  (sqlite3:finalize! tdb)))))
-
-;; 
 (define (db:delete-test-records db tdb test-id #!key (force #f))
   (if tdb 
       (begin
 	(sqlite3:execute tdb "DELETE FROM test_steps;")
 	(sqlite3:execute tdb "DELETE FROM test_data;"))
-      (db:delete-test-step-records db test-id))
+      (tdb:delete-test-step-records db test-id))
   (if db 
       (begin
 	(sqlite3:execute db "DELETE FROM test_steps WHERE test_id=?;" test-id)
 	(sqlite3:execute db "DELETE FROM test_data  WHERE test_id=?;" test-id)
 	(if force
@@ -1330,19 +1218,10 @@
     (if newstate   (sqlite3:execute db "UPDATE tests SET state=?   WHERE id=?;" newstate   test-id))
     (if newstatus  (sqlite3:execute db "UPDATE tests SET status=?  WHERE id=?;" newstatus  test-id))
     (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment test-id))))
   (mt:process-triggers test-id newstate newstatus))
 
-(define (db:update-testdat-meta-info db test-id work-area cpuload diskfree minutes)
-  (let ((tdb         (db:open-test-db-by-test-id db test-id work-area: work-area)))
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
-			   cpuload diskfree minutes)
-	  (sqlite3:finalize! tdb))
-	(debug:print 2 "Can't update testdat.db for test " test-id " read-only or non-existant"))))
-    
 ;; Never used, but should be?
 (define (db:test-set-state-status-by-run-id-testname db run-id test-name item-path status state)
   (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" 
  		   state status run-id test-name item-path))
 
@@ -1414,54 +1293,10 @@
      "SELECT id FROM tests WHERE run_id=? AND testname=? AND item_path=?;"
      run-id testname item-path)
     res))
 
 (define db:get-test-id db:get-test-id-not-cached)
-
-;; given a test-info record, patch in the latest data from the testdat.db file
-;; found in the test run directory
-;;
-;; NOT USED
-;;
-(define (db:patch-tdb-data-into-test-info db test-id res #!key (work-area #f))
-  (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area)))
-    ;; get state and status from megatest.db in real time
-    ;; other fields that perhaps should be updated:
-    ;;   fail_count
-    ;;   pass_count
-    ;;   final_logf
-    (sqlite3:for-each-row
-     (lambda (state status final_logf)
-       (db:test-set-state!        res state)
-       (db:test-set-status!       res status)
-       (db:test-set-final_logf!   res final_logf))
-     db
-     "SELECT state,status,final_logf FROM tests WHERE id=?;"
-     test-id)
-    (if tdb
-	(begin
-	  (sqlite3:for-each-row
-	   (lambda (update_time cpuload disk_free run_duration)
-	     (db:test-set-cpuload!      res cpuload)
-	     (db:test-set-diskfree!     res disk_free)
-	     (db:test-set-run_duration! res run_duration))
-	   tdb
-	   "SELECT update_time,cpuload,diskfree,run_duration FROM test_rundat ORDER BY id DESC LIMIT 1;")
-	  (sqlite3:finalize! tdb))
-	;; if the test db is not found what to do?
-	;; 1. set state to DELETED
-	;; 2. set status to n/a
-	(begin
-	  (db:test-set-state!  res "NOT_STARTED")
-	  (db:test-set-status! res "n/a")))))
-
-(define *last-test-cache-delete* (current-seconds))
-
-(define (db:clean-all-caches)
-  (set! *test-info* (make-hash-table))
-  (set! *test-id-cache* (make-hash-table)))
-
 
 (define (db:get-all-tests-info-by-run-id db run-id)
   (let ((res '()))
     (sqlite3:for-each-row
      (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment)
@@ -1544,14 +1379,14 @@
   (let* ((testpatt   (if (args:get-arg "-testpatt")(args:get-arg "-testpatt") "%"))
 	 (statepatt  (if (args:get-arg ":state")   (args:get-arg ":state")    "%"))
 	 (statuspatt (if (args:get-arg ":status")  (args:get-arg ":status")   "%"))
 	 (runname    (if (args:get-arg ":runname") (args:get-arg ":runname")  "%"))
 	 (paths-from-db (cdb:remote-run db:test-get-paths-matching-keynames-target-new db keynames target res
-					testpatt:   testpatt
-					statepatt:  statepatt
-					statuspatt: statuspatt
-					runname:    runname)))
+					testpatt
+					statepatt
+					statuspatt
+					runname)))
     (if fnamepatt
 	(apply append 
 	       (map (lambda (p)
 		      (if (directory-exists? p)
 			  (glob (conc p "/" fnamepatt))
@@ -1582,15 +1417,14 @@
      db 
      qrystr)
     res))
 
 (define (db:test-get-paths-matching-keynames-target-new db keynames target res 
-							#!key
-							(testpatt   "%")
-							(statepatt  "%")
-							(statuspatt "%")
-							(runname    "%"))
+							testpatt  
+							statepatt 
+							statuspatt
+							runname)
   (let* ((row-ids '())
 	 (keystr (string-intersperse 
 		  (map (lambda (key val)
 			 (conc key " like '" val "'"))
 		       keynames 
@@ -1661,16 +1495,16 @@
 	   (lambda ()(deserialize)))
 	 (vector #f #f #f))) ;; crude reply for when things go awry
     ((zmq)(with-input-from-string msg (lambda ()(deserialize))))
     (else msg)))
 
-(define (cdb:use-non-blocking-mode proc)
-  (set! *client-non-blocking-mode* #t)
-  (let ((res (proc)))
-    (set! *client-non-blocking-mode* #f)
-    res))
-
+;; (define (cdb:use-non-blocking-mode proc)
+;;   (set! *client-non-blocking-mode* #t)
+;;   (let ((res (proc)))
+;;     (set! *client-non-blocking-mode* #f)
+;;     res))
+;; 
 ;; params = 'target cached remparams
 ;;
 ;; make-vector-record cdb packet client-sig qtype immediate query-sig params qtime
 ;;
 ;; cdb:client-call is the unified interface to all the transports. It dispatches the
@@ -1677,134 +1511,127 @@
 ;;                 query to a server routine (e.g. server:client-send-recieve) that 
 ;;                 transports the data to the server where it is passed to db:process-queue-item
 ;;                 which either returns the data to the calling server routine or 
 ;;                 directly calls the returning procedure (e.g. zmq).
 ;;
-(define (cdb:client-call serverdat qtype immediate numretries . params)
-  (debug:print-info 11 "cdb:client-call serverdat=" serverdat ", qtype=" qtype ", immediate=" immediate ", numretries=" numretries ", params=" params)
-  (case *transport-type* 
-    ((fs)
-     (let ((packet (vector "na" qtype immediate "na" params 0)))
-       (fs:process-queue-item packet)))
-    ((http)
-     (let* ((client-sig  (client:get-signature))
-	    (query-sig   (message-digest-string (md5-primitive) (conc qtype immediate params)))
-	    (zdat        (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds))))) ;; (with-output-to-string (lambda ()(serialize params))))
-       (debug:print-info 11 "zdat=" zdat)
-       (let* ((res  #f)
-	      (rawdat      (http-transport:client-send-receive serverdat zdat))
-	      (tmp         #f))
-	 (debug:print-info 11 "Sent " zdat ", received " rawdat)
-	 (if rawdat
-	     (begin
-	       (set! tmp (db:string->obj rawdat))
-	       (vector-ref tmp 2))
-	     (begin
-	       (debug:print 0 "ERROR: Communication with the server failed. Exiting if possible")
-	       (exit 1))))))
-    ((zmq)
-     (handle-exceptions
-      exn
-      (begin
-	(debug:print-info 0 "cdb:client-call timeout or error. Trying again in 5 seconds")
-	(thread-sleep! 5) 
-	(if (> numretries 0)(apply cdb:client-call serverdat qtype immediate (- numretries 1) params)))
-      (let* ((push-socket (vector-ref serverdat 0))
-	     (sub-socket  (vector-ref serverdat 1))
-	     (client-sig  (client:get-signature))
-	     (query-sig   (message-digest-string (md5-primitive) (conc qtype immediate params)))
-	     (zdat        (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds)))) ;; (with-output-to-string (lambda ()(serialize params))))
-	     (res  #f)
-	     (send-receive (lambda ()
-			     (debug:print-info 11 "sending message")
-			     (send-message push-socket zdat)
-			     (debug:print-info 11 "message sent")
-			     (let loop ()
-			       ;; get the sender info
-			       ;; this should match (client:get-signature)
-			       ;; we will need to process "all" messages here some day
-			       (receive-message* sub-socket)
-			       ;; now get the actual message
-			       (let ((myres (db:string->obj (receive-message* sub-socket))))
-				 (if (equal? query-sig (vector-ref myres 1))
-				     (set! res (vector-ref myres 2))
-				     (loop)))))))
-	;; (timeout (lambda ()
-	;;     	(let loop ((n numretries))
-	;;     	  (thread-sleep! 15)
-	;;     	  (if (not res)
-	;;     	      (if (> numretries 0)
-	;;     		  (begin
-	;;     		    (debug:print 2 "WARNING: no reply to query " params ", trying resend")
-	;;     		    (debug:print-info 11 "re-sending message")
-	;;     		    (send-message push-socket zdat)
-	;;     		    (debug:print-info 11 "message re-sent")
-	;;     		    (loop (- n 1)))
-	;;     		  ;; (apply cdb:client-call *runremote* qtype immediate (- numretries 1) params))
-	;;     		  (begin
-	;;     		    (debug:print 0 "ERROR: cdb:client-call timed out " params ", exiting.")
-	;;     		    (exit 5))))))))
-	(debug:print-info 11 "Starting threads")
-	(let ((th1 (make-thread send-receive "send receive"))
-	      ;; (th2 (make-thread timeout      "timeout"))
-	      )
-	  (thread-start! th1)
-	  ;; (thread-start! th2)
-	  (thread-join!  th1)
-	  (debug:print-info 11 "cdb:client-call returning res=" res)
-	  res))))))
-
-(define (cdb:set-verbosity serverdat val)
-  (cdb:client-call serverdat 'set-verbosity #f *default-numtries* val))
-
-(define (cdb:num-clients serverdat)
-  (cdb:client-call serverdat 'numclients #t *default-numtries*))
-
-(define (db:test-set-status-state db test-id status state msg)
-  (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
-      (db:general-call db 'set-test-start-time (list test-id)))
-  (if msg
-      (db:general-call db 'state-status-msg (list state status msg test-id))
-      (db:general-call db 'state-status     (list state status test-id))))
-
-(define (cdb:test-rollup-test_data-pass-fail serverdat test-id)
-  (cdb:client-call serverdat 'test_data-pf-rollup #t *default-numtries* test-id test-id test-id test-id))
-
-(define (cdb:pass-fail-counts serverdat test-id fail-count pass-count)
-  (cdb:client-call serverdat 'pass-fail-counts #t *default-numtries* fail-count pass-count test-id))
-
-(define (cdb:tests-register-test serverdat run-id test-name item-path)
-  (cdb:client-call serverdat 'register-test #t *default-numtries* run-id test-name item-path))
-
-;; more transactioned calls, these for roll-up-pass-fail stuff
-(define (cdb:update-pass-fail-counts serverdat run-id test-name)
-  (cdb:client-call serverdat 'update-fail-pass-counts #t *default-numtries* run-id test-name run-id test-name run-id test-name))
-
-(define (cdb:top-test-set-running serverdat run-id test-name)
-  (cdb:client-call serverdat 'top-test-set-running #t *default-numtries* run-id test-name))
-
-(define (cdb:top-test-set-per-pf-counts serverdat run-id test-name)
-  (cdb:client-call serverdat 'top-test-set-per-pf-counts #t *default-numtries* run-id test-name run-id test-name run-id test-name))
-
-;;=
-
-(define (cdb:flush-queue serverdat)
-  (cdb:client-call serverdat 'flush #f *default-numtries*))
-
-(define (cdb:kill-server serverdat pid)
-  (cdb:client-call serverdat 'killserver #t *default-numtries* pid))
-
-(define (cdb:roll-up-pass-fail-counts serverdat run-id test-name item-path status)
-  (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:roll-up-pass-fail-counts #f run-id test-name item-path status))
-
-(define (cdb:get-test-info serverdat run-id test-name item-path)
-  (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info #f run-id test-name item-path))
-
-(define (cdb:get-test-info-by-id serverdat test-id)
-  (let ((test-dat (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info-by-id #f test-id)))
-    (hash-table-set! *test-info* test-id (vector (current-seconds) test-dat)) ;; cached for use where up-to-date info is not needed
-    test-dat))
+;; (define (cdb:client-call serverdat qtype immediate numretries . params)
+;;   (debug:print-info 11 "cdb:client-call serverdat=" serverdat ", qtype=" qtype ", immediate=" immediate ", numretries=" numretries ", params=" params)
+;;   (case *transport-type* 
+;;     ((fs)
+;;      (let ((packet (vector "na" qtype immediate "na" params 0)))
+;;        (fs:process-queue-item packet)))
+;;     ((http)
+;;      (let* ((client-sig  (client:get-signature))
+;; 	    (query-sig   (message-digest-string (md5-primitive) (conc qtype immediate params)))
+;; 	    (zdat        (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds))))) ;; (with-output-to-string (lambda ()(serialize params))))
+;;        (debug:print-info 11 "zdat=" zdat)
+;;        (let* ((res  #f)
+;; 	      (rawdat      (http-transport:client-send-receive serverdat zdat))
+;; 	      (tmp         #f))
+;; 	 (debug:print-info 11 "Sent " zdat ", received " rawdat)
+;; 	 (if rawdat
+;; 	     (begin
+;; 	       (set! tmp (db:string->obj rawdat))
+;; 	       (vector-ref tmp 2))
+;; 	     (begin
+;; 	       (debug:print 0 "ERROR: Communication with the server failed. Exiting if possible")
+;; 	       (exit 1))))))
+;;     ((zmq)
+;;      (handle-exceptions
+;;       exn
+;;       (begin
+;; 	(debug:print-info 0 "cdb:client-call timeout or error. Trying again in 5 seconds")
+;; 	(thread-sleep! 5) 
+;; 	(if (> numretries 0)(apply cdb:client-call serverdat qtype immediate (- numretries 1) params)))
+;;       (let* ((push-socket (vector-ref serverdat 0))
+;; 	     (sub-socket  (vector-ref serverdat 1))
+;; 	     (client-sig  (client:get-signature))
+;; 	     (query-sig   (message-digest-string (md5-primitive) (conc qtype immediate params)))
+;; 	     (zdat        (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds)))) ;; (with-output-to-string (lambda ()(serialize params))))
+;; 	     (res  #f)
+;; 	     (send-receive (lambda ()
+;; 			     (debug:print-info 11 "sending message")
+;; 			     (send-message push-socket zdat)
+;; 			     (debug:print-info 11 "message sent")
+;; 			     (let loop ()
+;; 			       ;; get the sender info
+;; 			       ;; this should match (client:get-signature)
+;; 			       ;; we will need to process "all" messages here some day
+;; 			       (receive-message* sub-socket)
+;; 			       ;; now get the actual message
+;; 			       (let ((myres (db:string->obj (receive-message* sub-socket))))
+;; 				 (if (equal? query-sig (vector-ref myres 1))
+;; 				     (set! res (vector-ref myres 2))
+;; 				     (loop)))))))
+;; 	;; (timeout (lambda ()
+;; 	;;     	(let loop ((n numretries))
+;; 	;;     	  (thread-sleep! 15)
+;; 	;;     	  (if (not res)
+;; 	;;     	      (if (> numretries 0)
+;; 	;;     		  (begin
+;; 	;;     		    (debug:print 2 "WARNING: no reply to query " params ", trying resend")
+;; 	;;     		    (debug:print-info 11 "re-sending message")
+;; 	;;     		    (send-message push-socket zdat)
+;; 	;;     		    (debug:print-info 11 "message re-sent")
+;; 	;;     		    (loop (- n 1)))
+;; 	;;     		  ;; (apply cdb:client-call *runremote* qtype immediate (- numretries 1) params))
+;; 	;;     		  (begin
+;; 	;;     		    (debug:print 0 "ERROR: cdb:client-call timed out " params ", exiting.")
+;; 	;;     		    (exit 5))))))))
+;; 	(debug:print-info 11 "Starting threads")
+;; 	(let ((th1 (make-thread send-receive "send receive"))
+;; 	      ;; (th2 (make-thread timeout      "timeout"))
+;; 	      )
+;; 	  (thread-start! th1)
+;; 	  ;; (thread-start! th2)
+;; 	  (thread-join!  th1)
+;; 	  (debug:print-info 11 "cdb:client-call returning res=" res)
+;; 	  res))))))
+;; 
+;; ;; (define (cdb:set-verbosity serverdat val)
+;;   (cdb:client-call serverdat 'set-verbosity #f *default-numtries* val))
+;; 
+;; (define (cdb:num-clients serverdat)
+;;   (cdb:client-call serverdat 'numclients #t *default-numtries*))
+;; 
+;; (define (db:test-set-status-state db test-id status state msg)
+;;   (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
+;;       (db:general-call db 'set-test-start-time (list test-id)))
+;;   (if msg
+;;       (db:general-call db 'state-status-msg (list state status msg test-id))
+;;       (db:general-call db 'state-status     (list state status test-id))))
+;; 
+;; (define (cdb:test-rollup-test_data-pass-fail serverdat test-id)
+;;   (cdb:client-call serverdat 'test_data-pf-rollup #t *default-numtries* test-id test-id test-id test-id))
+;; 
+;; (define (cdb:tests-register-test serverdat run-id test-name item-path)
+;;   (cdb:client-call serverdat 'register-test #t *default-numtries* run-id test-name item-path))
+;; 
+;; (define (cdb:top-test-set-running serverdat run-id test-name)
+;;   (cdb:client-call serverdat 'top-test-set-running #t *default-numtries* run-id test-name))
+;; 
+;; (define (cdb:top-test-set-per-pf-counts serverdat run-id test-name)
+;;   (cdb:client-call serverdat 'top-test-set-per-pf-counts #t *default-numtries* run-id test-name run-id test-name run-id test-name))
+;; 
+;; ;;=
+;; 
+;; (define (cdb:flush-queue serverdat)
+;;   (cdb:client-call serverdat 'flush #f *default-numtries*))
+;; 
+;; (define (cdb:kill-server serverdat pid)
+;;   (cdb:client-call serverdat 'killserver #t *default-numtries* pid))
+;; 
+;; (define (cdb:roll-up-pass-fail-counts serverdat run-id test-name item-path status)
+;;   (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:roll-up-pass-fail-counts #f run-id test-name item-path status))
+;; 
+;; (define (cdb:get-test-info serverdat run-id test-name item-path)
+;;   (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info #f run-id test-name item-path))
+;; 
+;; (define (cdb:get-test-info-by-id serverdat test-id)
+;;   (let ((test-dat (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info-by-id #f test-id)))
+;;     (hash-table-set! *test-info* test-id (vector (current-seconds) test-dat)) ;; cached for use where up-to-date info is not needed
+;;     test-dat))
 
 ;; ;; db should be db open proc or #f
 ;; (define (cdb:remote-run proc db . params)
 ;;   (if (or *db-write-access*
 ;; 	  (not (member proc *db:all-write-procs*)))
@@ -1984,45 +1811,45 @@
 
 ;; The queue is a list of vectors where the zeroth slot indicates the type of query to
 ;; apply and the second slot is the time of the query and the third entry is a list of 
 ;; values to be applied
 ;;
-(define (db:queue-write-and-wait db qry-sig query params)
-  (let ((queue-len  0)
-	(res        #f)
-	(got-it     #f)
-	(qry-pkt    (vector qry-sig query params))
-	(start-time (current-milliseconds))
-	(timeout    (+ 10 (current-seconds)))) ;; set the time out to 10 secs in future
-
-    ;; Put the item in the queue *incoming-writes* 
-    (mutex-lock! *incoming-mutex*)
-    (set! *incoming-writes* (cons qry-pkt *incoming-writes*))
-    (set! queue-len (length *incoming-writes*))
-    (mutex-unlock! *incoming-mutex*)
-
-    (debug:print-info 7 "Current write queue length is " queue-len)
-
-    ;; poll for the write to complete, timeout after 10 seconds
-    ;; periodic flushing of the queue is taken care of by 
-    ;; db:flush-queue
-    (let loop ()
-      (thread-sleep! 0.001)
-      (mutex-lock! *completed-mutex*)
-      (if (hash-table-ref/default *completed-writes* qry-sig #f)
-	  (begin
-	    (hash-table-delete! *completed-writes* qry-sig)
-	    (set! got-it #t)))
-      (mutex-unlock! *completed-mutex*)
-      (if (and (not got-it)
-	       (< (current-seconds) timeout))
-	  (begin
-	    (thread-sleep! 0.01)
-	    (loop))))
-    (set! *number-of-writes*   (+ *number-of-writes*   1))
-    (set! *writes-total-delay* (+ *writes-total-delay* (- (current-milliseconds) start-time)))
-    got-it))
+;; (define (db:queue-write-and-wait db qry-sig query params)
+;;   (let ((queue-len  0)
+;; 	(res        #f)
+;; 	(got-it     #f)
+;; 	(qry-pkt    (vector qry-sig query params))
+;; 	(start-time (current-milliseconds))
+;; 	(timeout    (+ 10 (current-seconds)))) ;; set the time out to 10 secs in future
+;; 
+;;     ;; Put the item in the queue *incoming-writes* 
+;;     (mutex-lock! *incoming-mutex*)
+;;     (set! *incoming-writes* (cons qry-pkt *incoming-writes*))
+;;     (set! queue-len (length *incoming-writes*))
+;;     (mutex-unlock! *incoming-mutex*)
+;; 
+;;     (debug:print-info 7 "Current write queue length is " queue-len)
+;; 
+;;     ;; poll for the write to complete, timeout after 10 seconds
+;;     ;; periodic flushing of the queue is taken care of by 
+;;     ;; db:flush-queue
+;;     (let loop ()
+;;       (thread-sleep! 0.001)
+;;       (mutex-lock! *completed-mutex*)
+;;       (if (hash-table-ref/default *completed-writes* qry-sig #f)
+;; 	  (begin
+;; 	    (hash-table-delete! *completed-writes* qry-sig)
+;; 	    (set! got-it #t)))
+;;       (mutex-unlock! *completed-mutex*)
+;;       (if (and (not got-it)
+;; 	       (< (current-seconds) timeout))
+;; 	  (begin
+;; 	    (thread-sleep! 0.01)
+;; 	    (loop))))
+;;     (set! *number-of-writes*   (+ *number-of-writes*   1))
+;;     (set! *writes-total-delay* (+ *writes-total-delay* (- (current-milliseconds) start-time)))
+;;     got-it))
 
 (define (db:general-call db stmtname params)
   (let ((query (let ((q (alist-ref (if (string? stmtname)
 				       (string->symbol stmtname)
 				       stmtname)
@@ -2119,87 +1946,87 @@
 		  (if (null? tal)
 		      (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests
 		      (loop (car tal)(cdr tal))))))))))
 
 
-(define (db:process-queue-item db item)
-  (let* ((stmt-key       (cdb:packet-get-qtype item))
-	 (qry-sig        (cdb:packet-get-query-sig item))
-	 (return-address (cdb:packet-get-client-sig item))
-	 (params         (cdb:packet-get-params item))
-	 (query          (let ((q (alist-ref stmt-key db:queries)))
-			   (if q (car q) #f))))
-    (debug:print-info 11 "Special queries/requests stmt-key=" stmt-key ", return-address=" return-address ", query=" query ", params=" params)
-    (if query
-	;; hand queries off to the write queue
-	(let ((response (case *transport-type*
-			  ((http)
-			   (debug:print-info 7 "Queuing item " item " for wrapped write")
-			   (db:queue-write-and-wait db qry-sig query params))
-			  (else 
-			   (apply sqlite3:execute db query params)
-			   #t))))
-	  (debug:print-info 7 "Received " response " from wrapped write")
-	  (server:reply return-address qry-sig response response))
-	;; otherwise if appropriate flush the queue (this is a read or complex query)
-	(begin
-	  (cond
-	   ((member stmt-key db:special-queries)
-	    (let ((starttime (current-milliseconds)))
-	      (debug:print-info 9 "Handling special statement " stmt-key)
-	      (case stmt-key
-		((immediate)
-		 (debug:print 0 "WARNING: Immediate calls are verboten now!")
-		 (let* ((proc      (car params))
-			(remparams (cdr params))
-			;; we are being handed a procedure so call it
-			(result (server:reply return-address qry-sig #t (apply proc remparams))))
-		   (debug:print-info 11 "Ran (apply " proc " " remparams ")")
-		   ;; (set! *total-non-write-delay* (+ *total-non-write-delay* (- (current-milliseconds) starttime))) 
-		   ;; (set! *number-non-write-queries* (+ *number-non-write-queries* 1))
-		   result))
-		((login)
-		 (if (< (length params) 3) ;; should get toppath, version and signature
-		     (server:reply return-address qry-sig '(#f "login failed due to missing params")) ;; missing params
-		     (let ((calling-path (car   params))
-			   (calling-vers (cadr  params))
-			   (client-key   (caddr params)))
-		       (if (and (equal? calling-path *toppath*)
-				(equal? megatest-version calling-vers))
-			   (begin
-			     (hash-table-set! *logged-in-clients* client-key (current-seconds))
-			     (server:reply return-address qry-sig #t '(#t "successful login")))      ;; path matches - pass! Should vet the caller at this time ...
-			   (server:reply return-address qry-sig #f (list #f (conc "Login failed due to mismatch paths: " calling-path ", " *toppath*)))))))
-		((flush sync)
-		 (server:reply return-address qry-sig #t 1)) ;; (length data)))
-		((set-verbosity)
-		 (set! *verbosity* (car params))
-		 (server:reply return-address qry-sig #t (list #t *verbosity*)))
-		((killserver)
-		 (db:sync-to *inmemdb* *db*)
-		 (let ((hostname (car  *runremote*))
-		       (port     (cadr *runremote*))
-		       (pid      (car params))
-		       (th1      (make-thread (lambda ()(thread-sleep! 3)(debug:print 0 "Server exiting!")(exit 0)) "Server exit thread")))
-		   (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!")
-		   (debug:print-info 1 "current pid=" (current-process-id))
-		   (open-run-close tasks:server-deregister tasks:open-db 
-				   hostname
-				   port: port)
-		   (set! *server-run* #f)
-		   (thread-sleep! 3)
-		   (if pid 
-		       (process-signal pid signal/kill)
-		       (thread-start! th1))
-		   (server:reply return-address qry-sig #t '(#t "exit process started"))))
-		(else ;; not a command, i.e. is a query
-		 (debug:print 0 "ERROR: Unrecognised query/command " stmt-key)
-		 (server:reply return-address qry-sig #f 'failed)))))
-	   (else
-	    (debug:print-info 11 "Executing " stmt-key " for " params)
-	    (apply sqlite3:execute (hash-table-ref queries stmt-key) params)
-	    (server:reply return-address qry-sig #t #t)))))))
+;; (define (db:process-queue-item db item)
+;;   (let* ((stmt-key       (cdb:packet-get-qtype item))
+;; 	 (qry-sig        (cdb:packet-get-query-sig item))
+;; 	 (return-address (cdb:packet-get-client-sig item))
+;; 	 (params         (cdb:packet-get-params item))
+;; 	 (query          (let ((q (alist-ref stmt-key db:queries)))
+;; 			   (if q (car q) #f))))
+;;     (debug:print-info 11 "Special queries/requests stmt-key=" stmt-key ", return-address=" return-address ", query=" query ", params=" params)
+;;     (if query
+;; 	;; hand queries off to the write queue
+;; 	(let ((response (case *transport-type*
+;; 			  ((http)
+;; 			   (debug:print-info 7 "Queuing item " item " for wrapped write")
+;; 			   (db:queue-write-and-wait db qry-sig query params))
+;; 			  (else 
+;; 			   (apply sqlite3:execute db query params)
+;; 			   #t))))
+;; 	  (debug:print-info 7 "Received " response " from wrapped write")
+;; 	  (server:reply return-address qry-sig response response))
+;; 	;; otherwise if appropriate flush the queue (this is a read or complex query)
+;; 	(begin
+;; 	  (cond
+;; 	   ((member stmt-key db:special-queries)
+;; 	    (let ((starttime (current-milliseconds)))
+;; 	      (debug:print-info 9 "Handling special statement " stmt-key)
+;; 	      (case stmt-key
+;; 		((immediate)
+;; 		 (debug:print 0 "WARNING: Immediate calls are verboten now!")
+;; 		 (let* ((proc      (car params))
+;; 			(remparams (cdr params))
+;; 			;; we are being handed a procedure so call it
+;; 			(result (server:reply return-address qry-sig #t (apply proc remparams))))
+;; 		   (debug:print-info 11 "Ran (apply " proc " " remparams ")")
+;; 		   ;; (set! *total-non-write-delay* (+ *total-non-write-delay* (- (current-milliseconds) starttime))) 
+;; 		   ;; (set! *number-non-write-queries* (+ *number-non-write-queries* 1))
+;; 		   result))
+;; 		((login)
+;; 		 (if (< (length params) 3) ;; should get toppath, version and signature
+;; 		     (server:reply return-address qry-sig '(#f "login failed due to missing params")) ;; missing params
+;; 		     (let ((calling-path (car   params))
+;; 			   (calling-vers (cadr  params))
+;; 			   (client-key   (caddr params)))
+;; 		       (if (and (equal? calling-path *toppath*)
+;; 				(equal? megatest-version calling-vers))
+;; 			   (begin
+;; 			     (hash-table-set! *logged-in-clients* client-key (current-seconds))
+;; 			     (server:reply return-address qry-sig #t '(#t "successful login")))      ;; path matches - pass! Should vet the caller at this time ...
+;; 			   (server:reply return-address qry-sig #f (list #f (conc "Login failed due to mismatch paths: " calling-path ", " *toppath*)))))))
+;; 		((flush sync)
+;; 		 (server:reply return-address qry-sig #t 1)) ;; (length data)))
+;; 		((set-verbosity)
+;; 		 (set! *verbosity* (car params))
+;; 		 (server:reply return-address qry-sig #t (list #t *verbosity*)))
+;; 		((killserver)
+;; 		 (db:sync-to *inmemdb* *db*)
+;; 		 (let ((hostname (car  *runremote*))
+;; 		       (port     (cadr *runremote*))
+;; 		       (pid      (car params))
+;; 		       (th1      (make-thread (lambda ()(thread-sleep! 3)(debug:print 0 "Server exiting!")(exit 0)) "Server exit thread")))
+;; 		   (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!")
+;; 		   (debug:print-info 1 "current pid=" (current-process-id))
+;; 		   (open-run-close tasks:server-deregister tasks:open-db 
+;; 				   hostname
+;; 				   port: port)
+;; 		   (set! *server-run* #f)
+;; 		   (thread-sleep! 3)
+;; 		   (if pid 
+;; 		       (process-signal pid signal/kill)
+;; 		       (thread-start! th1))
+;; 		   (server:reply return-address qry-sig #t '(#t "exit process started"))))
+;; 		(else ;; not a command, i.e. is a query
+;; 		 (debug:print 0 "ERROR: Unrecognised query/command " stmt-key)
+;; 		 (server:reply return-address qry-sig #f 'failed)))))
+;; 	   (else
+;; 	    (debug:print-info 11 "Executing " stmt-key " for " params)
+;; 	    (apply sqlite3:execute (hash-table-ref queries stmt-key) params)
+;; 	    (server:reply return-address qry-sig #t #t)))))))
 
 (define (db:test-get-records-for-index-file db run-id test-name)
   (let ((res '()))
     (sqlite3:for-each-row 
      (lambda (id itempath state status run_duration logf comment)
@@ -2238,328 +2065,10 @@
        (set! res (cons (apply vector a b) res)))
      db
      "SELECT id,testname,author,owner,description,reviewed,iterated,avg_runtime,avg_disk,tags,jobgroup FROM test_meta;")
     res))
 
-;;======================================================================
-;; T E S T   D A T A 
-;;======================================================================
-
-(define (db:csv->test-data db test-id csvdata #!key (work-area #f))
-  (debug:print 4 "test-id " test-id ", csvdata: " csvdata)
-  (let ((tdb     (db:open-test-db-by-test-id db test-id work-area: work-area)))
-    (if (sqlite3:database? tdb)
-	(let ((csvlist (csv->list (make-csv-reader
-				   (open-input-string csvdata)
-				   '((strip-leading-whitespace? #t)
-				     (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata)))
-	  (for-each 
-	   (lambda (csvrow)
-	     (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
-		    (category    (list-ref padded-row 0))
-		    (variable    (list-ref padded-row 1))
-		    (value       (any->number-if-possible (list-ref padded-row 2)))
-		    (expected    (any->number-if-possible (list-ref padded-row 3)))
-		    (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
-		    (units       (list-ref padded-row 5))
-		    (comment     (list-ref padded-row 6))
-		    (status      (let ((s (list-ref padded-row 7)))
-				   (if (and (string? s)(or (string-match (regexp "^\\s*$") s)
-							   (string-match (regexp "^n/a$") s)))
-				       #f
-				       s))) ;; if specified on the input then use, else calculate
-		    (type        (list-ref padded-row 8)))
-	       ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
-	       (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
-
-	       (if (and (or (not expected)(equal? expected ""))
-			(or (not tol)     (equal? expected ""))
-			(or (not units)   (equal? expected "")))
-		   (let-values (((new-expected new-tol new-units)(db:get-prev-tol-for-test db test-id category variable)))
-			       (set! expected new-expected)
-			       (set! tol      new-tol)
-			       (set! units    new-units)))
-
-	       (debug:print 4 "AFTER:  category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	       ;; calculate status if NOT specified
-	       (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
-		   (if (number? tol) ;; if tol is a number then we do the standard comparison
-		       (let* ((max-val (+ expected tol))
-			      (min-val (- expected tol))
-			      (result  (and (>=  value min-val)(<= value max-val))))
-			 (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result)
-			 (set! status (if result "pass" "fail")))
-		       (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
-			     (case (string->symbol tol) ;; tol should be >, <, >=, <=
-			       ((>)  (if (>  value expected) "pass" "fail"))
-			       ((<)  (if (<  value expected) "pass" "fail"))
-			       ((>=) (if (>= value expected) "pass" "fail"))
-			       ((<=) (if (<= value expected) "pass" "fail"))
-			       (else (conc "ERROR: bad tol comparator " tol))))))
-	       (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	       (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
-				test-id category variable value expected tol units (if comment comment "") status type)))
-	   csvlist)
-	  (sqlite3:finalize! tdb)))))
-
-;; get a list of test_data records matching categorypatt
-(define (db:read-test-data db test-id categorypatt #!key (work-area #f))
-  (let ((tdb  (db:open-test-db-by-test-id db test-id work-area: work-area)))
-    (if (sqlite3:database? tdb)
-	(let ((res '()))
-	  (sqlite3:for-each-row 
-	   (lambda (id test_id category variable value expected tol units comment status type)
-	     (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
-	   tdb
-	   "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
-	  (sqlite3:finalize! tdb)
-	  (reverse res))
-	'())))
-
-;; NOTE: Run this local with #f for db !!!
-(define (db:load-test-data db test-id #!key (work-area #f))
-  (let loop ((lin (read-line)))
-    (if (not (eof-object? lin))
-	(begin
-	  (debug:print 4 lin)
-	  (db:csv->test-data db test-id lin work-area: work-area)
-	  (loop (read-line)))))
-  ;; roll up the current results.
-  ;; FIXME: Add the status to 
-  (db:test-data-rollup db test-id #f work-area: work-area))
-
-;; WARNING: Do NOT call this for the parent test on an iterated test
-;; Roll up test_data pass/fail results
-;; look at the test_data status field, 
-;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
-;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
-(define (db:test-data-rollup db test-id status #!key (work-area #f))
-  (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))
-	(fail-count 0)
-	(pass-count 0))
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:for-each-row
-	   (lambda (fcount pcount)
-	     (set! fail-count fcount)
-	     (set! pass-count pcount))
-	   tdb 
-	   "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
-                   (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;"
-	   test-id test-id)
-	  (sqlite3:finalize! tdb)
-
-	  ;; Now rollup the counts to the central megatest.db
-	  (cdb:pass-fail-counts *runremote* test-id fail-count pass-count)
-	  ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" 
-	  ;;                     fail-count pass-count test-id)
-
-	  ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines
-	  ;; next time you read this!
-	  ;;
-	  ;; (cdb:flush-queue *runremote*)
-	  ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set
-	  
-	  ;; if the test is not FAIL then set status based on the fail and pass counts.
-	  (cdb:test-rollup-test_data-pass-fail *runremote* test-id)
-	  ;; (sqlite3:execute
-	  ;;  db   ;;; NOTE: Should this be WARN,FAIL? A WARN is not a FAIL????? BUG FIXME
-	  ;;  "UPDATE tests
-	  ;;             SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 
-	  ;;                THEN 'FAIL'
-	  ;;             WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND 
-	  ;;                  (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL')
-	  ;;             THEN 'PASS'
-	  ;;             ELSE status
-	  ;;         END WHERE id=?;"
-	  ;;  test-id test-id test-id test-id)
-	  ))))
-
-(define (db:get-prev-tol-for-test db test-id category variable)
-  ;; Finish me?
-  (values #f #f #f))
-
-;;======================================================================
-;; S T E P S 
-;;======================================================================
-
-(define (db:step-get-time-as-string vec)
-  (seconds->time-string (db:step-get-event_time vec)))
-
-;; db-get-test-steps-for-run
-(define (db:get-steps-for-test db test-id #!key (work-area #f))
-  (let* ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))
-	 (res '()))
-    (if (sqlite3:database? tdb)
-	(handle-exceptions
-	 exn
-	 (begin
-	   (debug:print 0 "ERROR: error on access to testdat for test with id " test-id)
-	   '())
-	 (begin
-	   (sqlite3:for-each-row 
-	    (lambda (id test-id stepname state status event-time logfile)
-	      (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
-	    tdb
-	    "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
-	    test-id)
-	   (sqlite3:finalize! tdb)
-	   (reverse res)))
-	'())))
-
-;; get a pretty table to summarize steps
-;;
-(define (db:get-steps-table db test-id #!key (work-area #f))
-  (let ((steps   (db:get-steps-for-test db test-id work-area: work-area)))
-    ;; organise the steps for better readability
-    (let ((res (make-hash-table)))
-      (for-each 
-       (lambda (step)
-	 (debug:print 6 "step=" step)
-	 (let ((record (hash-table-ref/default 
-			res 
-			(db:step-get-stepname step) 
-			;;        stepname                start end status Duration  Logfile 
-			(vector (db:step-get-stepname step) ""   "" ""     ""        ""))))
-	   (debug:print 6 "record(before) = " record 
-			"\nid:       " (db:step-get-id step)
-			"\nstepname: " (db:step-get-stepname step)
-			"\nstate:    " (db:step-get-state step)
-			"\nstatus:   " (db:step-get-status step)
-			"\ntime:     " (db:step-get-event_time step))
-	   (case (string->symbol (db:step-get-state step))
-	     ((start)(vector-set! record 1 (db:step-get-event_time step))
-	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
-					(db:step-get-status step)))
-	      (if (> (string-length (db:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (db:step-get-logfile step))))
-	     ((end)  
-	      (vector-set! record 2 (any->number (db:step-get-event_time step)))
-	      (vector-set! record 3 (db:step-get-status step))
-	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
-					  (endt   (any->number (vector-ref record 2))))
-				      (debug:print 4 "record[1]=" (vector-ref record 1) 
-						   ", startt=" startt ", endt=" endt
-						   ", get-status: " (db:step-get-status step))
-				      (if (and (number? startt)(number? endt))
-					  (seconds->hr-min-sec (- endt startt)) "-1")))
-	      (if (> (string-length (db:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (db:step-get-logfile step))))
-	     (else
-	      (vector-set! record 2 (db:step-get-state step))
-	      (vector-set! record 3 (db:step-get-status step))
-	      (vector-set! record 4 (db:step-get-event_time step))))
-	   (hash-table-set! res (db:step-get-stepname step) record)
-	   (debug:print 6 "record(after)  = " record 
-			"\nid:       " (db:step-get-id step)
-			"\nstepname: " (db:step-get-stepname step)
-			"\nstate:    " (db:step-get-state step)
-			"\nstatus:   " (db:step-get-status step)
-			"\ntime:     " (db:step-get-event_time step))))
-       ;; (else   (vector-set! record 1 (db:step-get-event_time step)))
-       (sort steps (lambda (a b)
-		     (cond
-		      ((<   (db:step-get-event_time a)(db:step-get-event_time b)) #t)
-		      ((eq? (db:step-get-event_time a)(db:step-get-event_time b)) 
-		       (<   (db:step-get-id a)        (db:step-get-id b)))
-		      (else #f)))))
-      res)))
-
-;; get a pretty table to summarize steps
-;;
-(define (db:get-steps-table-list db test-id #!key (work-area #f))
-  (let ((steps   (db:get-steps-for-test db test-id work-area: work-area)))
-    ;; organise the steps for better readability
-    (let ((res (make-hash-table)))
-      (for-each 
-       (lambda (step)
-	 (debug:print 6 "step=" step)
-	 (let ((record (hash-table-ref/default 
-			res 
-			(db:step-get-stepname step) 
-			;;        stepname                start end status    
-			(vector (db:step-get-stepname step) ""   "" ""     "" ""))))
-	   (debug:print 6 "record(before) = " record 
-			"\nid:       " (db:step-get-id step)
-			"\nstepname: " (db:step-get-stepname step)
-			"\nstate:    " (db:step-get-state step)
-			"\nstatus:   " (db:step-get-status step)
-			"\ntime:     " (db:step-get-event_time step))
-	   (case (string->symbol (db:step-get-state step))
-	     ((start)(vector-set! record 1 (db:step-get-event_time step))
-	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
-					(db:step-get-status step)))
-	      (if (> (string-length (db:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (db:step-get-logfile step))))
-	     ((end)  
-	      (vector-set! record 2 (any->number (db:step-get-event_time step)))
-	      (vector-set! record 3 (db:step-get-status step))
-	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
-					  (endt   (any->number (vector-ref record 2))))
-				      (debug:print 4 "record[1]=" (vector-ref record 1) 
-						   ", startt=" startt ", endt=" endt
-						   ", get-status: " (db:step-get-status step))
-				      (if (and (number? startt)(number? endt))
-					  (seconds->hr-min-sec (- endt startt)) "-1")))
-	      (if (> (string-length (db:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (db:step-get-logfile step))))
-	     (else
-	      (vector-set! record 2 (db:step-get-state step))
-	      (vector-set! record 3 (db:step-get-status step))
-	      (vector-set! record 4 (db:step-get-event_time step))))
-	   (hash-table-set! res (db:step-get-stepname step) record)
-	   (debug:print 6 "record(after)  = " record 
-			"\nid:       " (db:step-get-id step)
-			"\nstepname: " (db:step-get-stepname step)
-			"\nstate:    " (db:step-get-state step)
-			"\nstatus:   " (db:step-get-status step)
-			"\ntime:     " (db:step-get-event_time step))))
-       ;; (else   (vector-set! record 1 (db:step-get-event_time step)))
-       (sort steps (lambda (a b)
-		     (cond
-		      ((<   (db:step-get-event_time a)(db:step-get-event_time b)) #t)
-		      ((eq? (db:step-get-event_time a)(db:step-get-event_time b)) 
-		       (<   (db:step-get-id a)        (db:step-get-id b)))
-		      (else #f)))))
-      res)))
-
-(define (db:get-compressed-steps test-id #!key (work-area #f)(tdb #f))
-  (if (or (not work-area)
-	  (file-exists? (conc work-area "/testdat.db")))
-      (let* ((comprsteps (open-run-close db:get-steps-table tdb test-id work-area: work-area)))
-	(map (lambda (x)
-	       ;; take advantage of the \n on time->string
-	       (vector
-		(vector-ref x 0)
-		(let ((s (vector-ref x 1)))
-		  (if (number? s)(seconds->time-string s) s))
-		(let ((s (vector-ref x 2)))
-		  (if (number? s)(seconds->time-string s) s))
-		(vector-ref x 3)    ;; status
-		(vector-ref x 4)
-		(vector-ref x 5)))  ;; time delta
-	     (sort (hash-table-values comprsteps)
-		   (lambda (a b)
-		     (let ((time-a (vector-ref a 1))
-			   (time-b (vector-ref b 1)))
-		       (if (and (number? time-a)(number? time-b))
-			   (if (< time-a time-b)
-			       #t
-			       (if (eq? time-a time-b)
-				   (string<? (conc (vector-ref a 2))
-					     (conc (vector-ref b 2)))
-				   #f))
-			   (string<? (conc time-a)(conc time-b))))))))
-      '()))
-
 ;;======================================================================
 ;; M I S C   M A N A G E M E N T   I T E M S 
 ;;======================================================================
 
 ;; the new prereqs calculation, looks also at itempath if specified
@@ -2569,11 +2078,11 @@
 ;;
 ;; Note: mode 'normal means that tests must be COMPLETED and ok (i.e. PASS, WARN, CHECK, SKIP or WAIVED)
 ;;       mode 'toplevel means that tests must be COMPLETED only
 ;;       mode 'itemmatch or 'itemwait means that tests items must be COMPLETED and (PASS|WARN|WAIVED|CHECK) [[ NB// NOT IMPLEMENTED YET ]]
 ;; 
-(define (db:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal))
+(define (db:get-prereqs-not-met db run-id waitons ref-item-path mode)
   (if (or (not waitons)
 	  (null? waitons))
       '()
       (let* ((unmet-pre-reqs '())
 	     (result         '()))
@@ -2580,11 +2089,11 @@
 	(for-each 
 	 (lambda (waitontest-name)
 	   ;; by getting the tests with matching name we are looking only at the matching test 
 	   ;; and related sub items
 	   ;; next should be using mt:get-tests-for-run?
-	   (let ((tests             (cdb:remote-run db:get-tests-for-run-state-status #f run-id waitontest-name)) ;; (mt:get-tests-for-run run-id waitontest-name '() '()))
+	   (let ((tests             (db:get-tests-for-run-state-status db run-id waitontest-name))
 		 (ever-seen         #f)
 		 (parent-waiton-met #f)
 		 (item-waiton-met   #f))
 	     (for-each 
 	      (lambda (test)
@@ -2634,28 +2143,10 @@
 	     (if (not ever-seen)
 		 (set! result (append (if (null? tests)(list waitontest-name) tests) result)))))
 	 waitons)
 	(delete-duplicates result))))
 
-(define (db:teststep-set-status! db test-id teststep-name state-in status-in comment logfile #!key (work-area #f))
-  ;;                 db:open-test-db-by-test-id does cdb:remote-run
-  (let* ((tdb       (db:open-test-db-by-test-id db test-id work-area: work-area))
-	 (state     (items:check-valid-items "state" state-in))
-	 (status    (items:check-valid-items "status" status-in)))
-    (if (or (not state)(not status))
-	(debug:print 3 "WARNING: Invalid " (if status "status" "state")
-		     " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:execute 
-	   tdb
-	   "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);"
-	   test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile ""))
-	  (sqlite3:finalize! tdb)
-	  #t)
-	#f)))
-
 ;;======================================================================
 ;; Extract ods file from the db
 ;;======================================================================
 
 ;; runspatt is a comma delimited list of run patterns
@@ -2779,26 +2270,25 @@
 
 ;; (db:extract-ods-file db "outputfile.ods" '(("sysname" "%")("fsname" "%")("datapath" "%")) "%")
 
 ;; This is a list of all procs that write to the db
 ;;
-(define *db:all-write-procs*
-  (list 
-   db:set-var 
-   db:del-var
-   db:register-run
-   db:set-comment-for-run
-   db:delete-run
-   db:update-run-event_time
-   db:lock/unlock-run 
-   db:delete-test-step-records
-   db:delete-test-records
-   db:delete-tests-for-run
-   db:delete-old-deleted-test-records
-   db:set-tests-state-status
-   db:test-set-state-status-by-id
-   db:test-set-state-status-by-run-id-testname
-   db:testmeta-add-record
-   db:csv->test-data
-   db:test-data-rollup
-   db:teststep-set-status! ))
+;; (define *db:all-write-procs*
+;;   (list 
+;;    db:set-var 
+;;    db:del-var
+;;    db:register-run
+;;    db:set-comment-for-run
+;;    db:delete-run
+;;    db:update-run-event_time
+;;    db:lock/unlock-run 
+;;    db:delete-test-step-records
+;;    db:delete-test-records
+;;    db:delete-tests-for-run
+;;    db:delete-old-deleted-test-records
+;;    db:set-tests-state-status
+;;    db:test-set-state-status-by-id
+;;    db:test-set-state-status-by-run-id-testname
+;;    db:testmeta-add-record
+;;    db:csv->test-data
+;;    ))
 

Index: db_records.scm
==================================================================
--- db_records.scm
+++ db_records.scm
@@ -97,38 +97,38 @@
 ;; S T E P S 
 ;;======================================================================
 ;; Run steps
 ;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time    
 (define (make-db:step)(make-vector 7))
-(define-inline (db:step-get-id              vec)    (vector-ref  vec 0))
-(define-inline (db:step-get-test_id         vec)    (vector-ref  vec 1))
-(define-inline (db:step-get-stepname        vec)    (vector-ref  vec 2))
-(define-inline (db:step-get-state           vec)    (vector-ref  vec 3))
-(define-inline (db:step-get-status          vec)    (vector-ref  vec 4))
-(define-inline (db:step-get-event_time      vec)    (vector-ref  vec 5))
-(define-inline (db:step-get-logfile         vec)    (vector-ref  vec 6))
-(define-inline (db:step-set-id!             vec val)(vector-set! vec 0 val))
-(define-inline (db:step-set-test_id!        vec val)(vector-set! vec 1 val))
-(define-inline (db:step-set-stepname!       vec val)(vector-set! vec 2 val))
-(define-inline (db:step-set-state!          vec val)(vector-set! vec 3 val))
-(define-inline (db:step-set-status!         vec val)(vector-set! vec 4 val))
-(define-inline (db:step-set-event_time!     vec val)(vector-set! vec 5 val))
-(define-inline (db:step-set-logfile!        vec val)(vector-set! vec 6 val))
+(define-inline (tdb:step-get-id              vec)    (vector-ref  vec 0))
+(define-inline (tdb:step-get-test_id         vec)    (vector-ref  vec 1))
+(define-inline (tdb:step-get-stepname        vec)    (vector-ref  vec 2))
+(define-inline (tdb:step-get-state           vec)    (vector-ref  vec 3))
+(define-inline (tdb:step-get-status          vec)    (vector-ref  vec 4))
+(define-inline (tdb:step-get-event_time      vec)    (vector-ref  vec 5))
+(define-inline (tdb:step-get-logfile         vec)    (vector-ref  vec 6))
+(define-inline (tdb:step-set-id!             vec val)(vector-set! vec 0 val))
+(define-inline (tdb:step-set-test_id!        vec val)(vector-set! vec 1 val))
+(define-inline (tdb:step-set-stepname!       vec val)(vector-set! vec 2 val))
+(define-inline (tdb:step-set-state!          vec val)(vector-set! vec 3 val))
+(define-inline (tdb:step-set-status!         vec val)(vector-set! vec 4 val))
+(define-inline (tdb:step-set-event_time!     vec val)(vector-set! vec 5 val))
+(define-inline (tdb:step-set-logfile!        vec val)(vector-set! vec 6 val))
 
 
 ;; The steps table
 (define (make-db:steps-table)(make-vector 5))
-(define-inline (db:steps-table-get-stepname   vec)    (vector-ref  vec 0))
-(define-inline (db:steps-table-get-start      vec)    (vector-ref  vec 1))
-(define-inline (db:steps-table-get-end        vec)    (vector-ref  vec 2))
-(define-inline (db:steps-table-get-status     vec)    (vector-ref  vec 3))
-(define-inline (db:steps-table-get-runtime    vec)    (vector-ref  vec 4))
-(define-inline (db:step-stable-set-stepname!  vec val)(vector-set! vec 0 val))
-(define-inline (db:step-stable-set-start!     vec val)(vector-set! vec 1 val))
-(define-inline (db:step-stable-set-end!       vec val)(vector-set! vec 2 val))
-(define-inline (db:step-stable-set-status!    vec val)(vector-set! vec 3 val))
-(define-inline (db:step-stable-set-runtime!   vec val)(vector-set! vec 4 val))
+(define-inline (tdb:steps-table-get-stepname   vec)    (vector-ref  vec 0))
+(define-inline (tdb:steps-table-get-start      vec)    (vector-ref  vec 1))
+(define-inline (tdb:steps-table-get-end        vec)    (vector-ref  vec 2))
+(define-inline (tdb:steps-table-get-status     vec)    (vector-ref  vec 3))
+(define-inline (tdb:steps-table-get-runtime    vec)    (vector-ref  vec 4))
+(define-inline (tdb:step-stable-set-stepname!  vec val)(vector-set! vec 0 val))
+(define-inline (tdb:step-stable-set-start!     vec val)(vector-set! vec 1 val))
+(define-inline (tdb:step-stable-set-end!       vec val)(vector-set! vec 2 val))
+(define-inline (tdb:step-stable-set-status!    vec val)(vector-set! vec 3 val))
+(define-inline (tdb:step-stable-set-runtime!   vec val)(vector-set! vec 4 val))
 
 ;; use this one for db-get-run-info
 (define-inline (db:get-row    vec)(vector-ref vec 1))
 
 ;; The data structure for handing off requests via wire

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -19,10 +19,11 @@
 
 (declare (unit launch))
 (declare (uses common))
 (declare (uses configf))
 (declare (uses db))
+(declare (uses tdb))
 
 (include "common_records.scm")
 (include "key_records.scm")
 (include "db_records.scm")
 
@@ -235,11 +236,11 @@
 						   ;; call the command using mt_ezstep
 						   (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd))
 
 						   (debug:print 4 "script: " script)
 						   ;; DO NOT remote
-						   (db:teststep-set-status! #f test-id stepname "start" "-" #f #f work-area: work-area)
+						   (tdb:teststep-set-status! test-id stepname "start" "-" #f #f work-area: work-area)
 						   ;; now launch
 						   (let ((pid (process-run script)))
 						     (let processloop ((i 0))
 						       (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
 								   (mutex-lock! m)
@@ -253,11 +254,11 @@
 									 (processloop (+ i 1))))
 								   ))
                                                      (let ((exinfo (vector-ref exit-info 2))
                                                            (logfna (if logpro-used (conc stepname ".html") "")))
 						       ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect)
-						       (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: work-area))
+						       (tdb:teststep-set-status! test-id stepname "end" exinfo #f logfna work-area: work-area))
 						     (if logpro-used
 							 (rmt:test-set-log! test-id (conc stepname ".html")))
 						     ;; set the test final status
 						     (let* ((this-step-status (cond
 									       ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)
@@ -305,11 +306,11 @@
 							  (round 
 							   (- 
 							    (current-seconds) 
 							    start-seconds)))))
 					(kill-tries 0))
-				   (tests:set-full-meta-info #f test-id run-id (calc-minutes) work-area)
+				   (tests:set-full-meta-info test-id run-id (calc-minutes) work-area)
 				   (let loop ((minutes   (calc-minutes)))
 				     (begin
 				       (set! kill-job? (or (test-get-kill-request test-id) ;; run-id test-name itemdat))
 							   (and runtlim (let* ((run-seconds   (- (current-seconds) start-seconds))
 									       (time-exceeded (> run-seconds runtlim)))
@@ -317,11 +318,11 @@
 									      (begin
 										(debug:print-info 0 "KILLING TEST DUE TO TIME LIMIT EXCEEDED! Runtime=" run-seconds " seconds, limit=" runtlim)
 										#t)
 									      #f)))))
 				       ;; open-run-close not needed for test-set-meta-info
-				       (tests:set-partial-meta-info #f test-id run-id minutes work-area)
+				       (tests:set-partial-meta-info test-id run-id minutes work-area)
 				       (if kill-job? 
 					   (begin
 					     (mutex-lock! m)
 					     ;; NOTE: The pid can change as different steps are run. Do we need handshaking between this
 					     ;;       section and the runit section? Or add a loop that tries three times with a 1/4 second
@@ -346,16 +347,14 @@
 						   ;;     (system (conc "kill -9 -" pid))))
 						   (begin
 						     (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process")
 						     (tests:test-set-status! test-id "KILLED"  "FAIL"
 								     (args:get-arg "-m") #f)
-						     (sqlite3:finalize! tdb)
 						     (exit 1) ;; IS THIS NECESSARY OR WISE???
 						     )))
 					     (set! kill-tries (+ 1 kill-tries))
 					     (mutex-unlock! m)))
-				       ;; (sqlite3:finalize! db)
 				       (if keep-going
 					   (begin
 					     (thread-sleep! 3) ;; (+ 3 (random 6))) ;; add some jitter to the call home time to spread out the db accesses
 					     (if keep-going
 						 (loop (calc-minutes)))))))
@@ -366,14 +365,11 @@
 	    (thread-start! th1)
 	    (thread-start! th2)
 	    (thread-join! th2)
 	    (set! keep-going #f)
 	    (thread-join! th1)
-	    ;; (thread-sleep! 1)
-	    ;; (thread-terminate! th1) ;; Not sure if this is a good idea
 	    (thread-sleep! 1)       ;; give thread th1 a chance to be done TODO: Verify this is needed. At 0.1 I was getting fail to stop, increased to total of 1.1 sec.
-	    ;; (tests:update-central-meta-info test-id cpuload diskfree minutes #f #f)
 	    (mutex-lock! m)
 	    (let* ((item-path (item-list->path itemdat))
 		   ;; only state and status needed - use lazy routine
 		   (testinfo  (rmt:get-testinfo-state-status test-id))) ;;;(cdb:get-test-info-by-id *runremote* test-id))) ;; )) ;; run-id test-name item-path)))
 	      ;; Am I completed?
@@ -404,11 +400,11 @@
 		    ;;       (thread-sleep! 0.1) ;; give other processes an opportunity to access the db as rollup is lower priority
 		    ;;       (cdb:roll-up-pass-fail-counts *runremote* run-id test-name item-path new-status)))
 		    ))
 	      ;; for automated creation of the rollup html file this is a good place...
 	      (if (not (equal? item-path ""))
-		  (tests:summarize-items #f run-id test-id test-name #f))) ;; don't force - just update if no
+		  (tests:summarize-items run-id test-id test-name #f))) ;; don't force - just update if no
 	    (mutex-unlock! m)
 	    (debug:print 2 "Output from running " fullrunscript ", pid " (vector-ref exit-info 0) " in work area " 
 			 work-area ":\n====\n exit code " (vector-ref exit-info 2) "\n" "====\n")
 	    (if (not (vector-ref exit-info 1))
 		(exit 4)))))))
@@ -591,19 +587,10 @@
 	     (debug:print 0 "ERROR:  Failed to re-create link " linktarget ((condition-property-accessor 'exn 'message) exn) ", exiting")
 	     (exit))
 	   (if (symbolic-link? lnktarget)     (delete-file lnktarget))
 	   (if (not (file-exists? lnktarget)) (create-symbolic-link test-path lnktarget)))))
 
-    ;; I suspect this section was deleting test directories under some 
-    ;; wierd sitations? This doesn't make sense - reenabling the rm -f 
-    ;; I honestly don't remember *why* this chunk was needed...
-    ;; (let ((testlink (conc lnkpath "/" testname)))
-    ;;   (if (and (file-exists? testlink)
-    ;;            (or (regular-file? testlink)
-    ;;     	   (symbolic-link? testlink)))
-    ;;       (system (conc "rm -f " testlink)))
-    ;;   (system  (conc "ln -sf " test-path " " testlink)))
     (if (directory? test-path)
 	(begin
 	  (let* ((ovrcmd (let ((cmd (config-lookup *configdat* "setup" "testcopycmd")))
 			   (if cmd
 			       ;; substitute the TEST_SRC_PATH and TEST_TARG_PATH

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -604,11 +604,10 @@
 			 ;; (print "[" targetstr "]"))))
 			 (print targetstr))))
 	       (if (not db-targets)
 		   (let* ((run-id (db:get-value-by-header run header "id"))
 			  (tests  (db:get-tests-for-run db run-id testpatt '() '() #f #f #f 'testname 'asc #f)))
-		     ;; (db:get-tests-for-run db run-id testpatt '() '())))
 		     (print "Run: " targetstr "/" (db:get-value-by-header run header "runname") 
 			    " status: " (db:get-value-by-header run header "state")
 			    " run-id: " run-id ", number tests: " (length tests))
 		     (for-each 
 		      (lambda (test)
@@ -768,11 +767,11 @@
 		(exit 1)))
 	  (if (not (setup-for-run))
 	      (begin
 		(debug:print 0 "Failed to setup, giving up on -test-paths or -test-files, exiting")
 		(exit 1)))
-	  (let* ((keys     (cdb:remote-run db:get-keys db))
+	  (let* ((keys     (rmt:get-keys))
 		 ;; db:test-get-paths must not be run remote
 		 (paths    (db:test-get-paths-matching db keys target (args:get-arg "-test-files"))))
 	    (set! *didsomething* #t)
 	    (for-each (lambda (path)
 			(print path))
@@ -804,11 +803,11 @@
 	       (test-name (assoc/default 'test-name cmdinfo))
 	       (runscript (assoc/default 'runscript cmdinfo))
 	       (db-host   (assoc/default 'db-host   cmdinfo))
 	       (run-id    (assoc/default 'run-id    cmdinfo))
 	       (itemdat   (assoc/default 'itemdat   cmdinfo))
-	       (db        #f)
+	       (db        (open-db))
 	       (state     (args:get-arg ":state"))
 	       (status    (args:get-arg ":status"))
 	       (target    (args:get-arg "-target")))
 	  (change-directory testpath)
 	  ;; (set! *runremote* runremote)
@@ -819,28 +818,30 @@
 		(exit 1)))
 	  (if (not (setup-for-run))
 	      (begin
 		(debug:print 0 "Failed to setup, giving up on -archive, exiting")
 		(exit 1)))
-	  (let* ((keys     (cdb:remote-run db:get-keys db))
+	  (let* ((keys     (db:get-keys db))
 		 ;; DO NOT run remote
 		 (paths    (db:test-get-paths-matching db keys target)))
 	    (set! *didsomething* #t)
 	    (for-each (lambda (path)
 			(print path))
-		      paths)))
+		      paths))
+	  (if (sqlite3:database? db)(sqlite3:finalize! db)))
 	;; else do a general-run-call
 	(general-run-call 
 	 "-test-paths"
 	 "Get paths to tests"
 	 (lambda (target runname keys keyvals)
-	   (let* ((db       #f)
+	   (let* ((db       (open-db))
 		  ;; DO NOT run remote
 		  (paths    (db:test-get-paths-matching db keys target)))
 	     (for-each (lambda (path)
 			 (print path))
-		       paths))))))
+		       paths)
+	     (sqlite3:finalize! db))))))
 
 ;;======================================================================
 ;; Extract a spreadsheet from the runs database
 ;;======================================================================
 
@@ -847,17 +848,19 @@
 (if (args:get-arg "-extract-ods")
     (general-run-call
      "-extract-ods"
      "Make ods spreadsheet"
      (lambda (target runname keys keyvals)
-       (let ((db         #f)
+       (let ((db         (open-db))
 	     (outputfile (args:get-arg "-extract-ods"))
 	     (runspatt   (args:get-arg ":runname"))
 	     (pathmod    (args:get-arg "-pathmod")))
 	     ;; (keyvalalist (keys->alist keys "%")))
 	 (debug:print 2 "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvals: " keyvals)
-	 (cdb:remote-run db:extract-ods-file db outputfile keyvals (if runspatt runspatt "%") pathmod)))))
+	 (db:extract-ods-file db outputfile keyvals (if runspatt runspatt "%") pathmod)
+	 (sqlite3:finalize! db)
+	 (set! *didsomething* #t)))))
 
 ;;======================================================================
 ;; execute the test
 ;;    - gets called on remote host
 ;;    - receives info from the -execute param

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -39,11 +39,11 @@
 ;;
 ;; Use: (db-get-value-by-header (db:get-header runinfo)(db:get-row runinfo))
 ;;  to extract info from the structure returned
 ;;
 (define (mt:get-runs-by-patt keys runnamepatt targpatt)
-  (let loop ((runsdat  (cdb:remote-run db:get-runs-by-patt #f keys runnamepatt targpatt 0 500))
+  (let loop ((runsdat  (rmt:get-runs-by-patt keys runnamepatt targpatt 0 500))
 	     (res      '())
 	     (offset   0)
 	     (limit    500))
     ;; (print "runsdat: " runsdat)
     (let* ((header    (vector-ref runsdat 0))
@@ -51,11 +51,11 @@
 	   (full-list (append res runslst))
 	   (have-more (eq? (length runslst) limit)))
       ;; (debug:print 0 "header: " header " runslst: " runslst " have-more: " have-more)
       (if have-more 
 	  (let ((new-offset (+ offset limit))
-		(next-batch (cdb:remote-run db:get-runs-by-patt #f keys runnamepatt targpatt offset limit)))
+		(next-batch (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit)))
 	    (debug:print-info 4 "More than " limit " runs, have " (length full-list) " runs so far.")
 	    (debug:print-info 0 "next-batch: " next-batch)
 	    (loop next-batch
 		  full-list
 		  new-offset
@@ -80,13 +80,10 @@
 		  full-list
 		  new-offset
 		  limit))
 	  full-list))))
 
-(define (mt:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal))
-  (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode))
-
 (define (mt:lazy-get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal))
   (let* ((key    (list run-id waitons ref-item-path mode))
 	 (res    (hash-table-ref/default *pre-reqs-met-cache* key #f))
 	 (useres (let ((last-time (if (vector? res) (vector-ref res 0) #f)))
 		   (if last-time
@@ -94,16 +91,17 @@
 		       #f))))
     (if useres
 	(let ((result (vector-ref res 1)))
 	  (debug:print 4 "Using lazy value res: " result)
 	  result)
-	(let ((newres (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode)))
+	(let ((newres (rmt:get-prereqs-not-met run-id waitons ref-item-path mode: mode)))
 	  (hash-table-set! *pre-reqs-met-cache* key (vector (current-seconds) newres))
 	  newres))))
 
+;;  Get run stats from local access, move this ... but where?
 (define (mt:get-run-stats)
-  (cdb:remote-run db:get-run-stats #f))
+  (db:get-run-stats #f))
 
 (define (mt:discard-blocked-tests run-id failed-test tests test-records)
   (if (null? tests)
       tests
       (begin
@@ -126,11 +124,11 @@
 ;;======================================================================
 ;;  T R I G G E R S
 ;;======================================================================
 
 (define (mt:process-triggers test-id newstate newstatus)
-  (let* ((test-dat      (mt:lazy-get-test-info-by-id test-id))
+  (let* ((test-dat      (rmt:get-test-info-by-id test-id))
 	 (test-rundir   (db:test-get-rundir test-dat))
 	 (test-name     (db:test-get-testname test-dat))
 	 (tconfig       #f)
 	 (state         (if newstate  newstate  (db:test-get-state  test-dat)))
 	 (status        (if newstatus newstatus (db:test-get-status test-dat))))
@@ -157,21 +155,10 @@
 
 ;;======================================================================
 ;;  S T A T E   A N D   S T A T U S   F O R   T E S T S 
 ;;======================================================================
 
-(define (mt:roll-up-pass-fail-counts run-id test-name item-path status)
-  (if (and (not (equal? item-path ""))
-	   (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP")))
-      (begin
-	(cdb:update-pass-fail-counts *runremote* run-id test-name)
-	(if (equal? status "RUNNING")
-	    (cdb:top-test-set-running *runremote* run-id test-name)
-	    (cdb:top-test-set-per-pf-counts *runremote* run-id test-name))
-	#f)
-      #f))
-
 ;; speed up for common cases with a little logic
 (define (mt:test-set-state-status-by-id test-id newstate newstatus newcomment)
   (cond
    ((and newstate newstatus newcomment)
     (cdb:client-call *runremote* 'state-status-msg #t *default-numtries* newstate newstatus newcomment test-id))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -67,10 +67,13 @@
 ;;======================================================================
 
 (define (rmt:login)
   (rmt:send-receive 'login (list *toppath* megatest-version *my-client-signature*)))
 
+(define (rmt:kill-server)
+  (rmt:send-receive 'kill-server '()))
+
 ;; hand off a call to one of the db:queries statements
 (define (rmt:general-call stmtname . params)
   (rmt:send-receive 'general-call (append (list stmtname) params)))
 
 ;;======================================================================
@@ -144,24 +147,41 @@
   (rmt:send-receive 'test-get-records-for-index-file (list  run-id test-name)))
 
 (define (rmt:get-testinfo-state-status test-id)
   (rmt:send-receive 'get-testinfo-state-status (list test-id)))
 
-(define (rmt:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
-  (rmt:send-receive 'update-testdat-meta-info (list test-id work-area cpuload diskfree minutes)))
-
 (define (rmt:test-set-log! test-id logf)
   (if (string? logf)(rmt:general-call 'test-set-log logf test-id)))
+
+(define (rmt:test-get-paths-matching-keynames-target-new keynames target res testpatt statepatt statuspatt runname)
+  (rmt:send-receive 'test-get-paths-matching-keynames-target-new (list keynames target res testpatt statepatt statuspatt runname)))
+
+(define (rmt:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal))
+  (rmt:send-receive 'get-prereqs-not-met (list run-id waitons ref-item-path mode)))
 
 ;; Statistical queries
 
 (define (rmt:get-count-tests-running)
   (rmt:send-receive 'get-count-tests-running '()))
 
 (define (rmt:get-count-tests-running-in-jobgroup jobgroup)
   (rmt:send-receive 'get-count-tests-running-in-jobgroup (list jobgroup)))
 
+(define (rmt:roll-up-pass-fail-counts run-id test-name item-path status)
+  (if (and (not (equal? item-path ""))
+	   (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP")))
+      (begin
+	(cdb:update-pass-fail-counts *runremote* run-id test-name)
+	(if (equal? status "RUNNING")
+	    (cdb:top-test-set-running *runremote* run-id test-name)
+	    (cdb:top-test-set-per-pf-counts *runremote* run-id test-name))
+	#f)
+      #f))
+
+(define (rmt:update-pass-fail-counts run-id test-name)
+  (rmt:general-call 'update-fail-pass-counts run-id test-name run-id test-name run-id test-name))
+
 ;;======================================================================
 ;;  R U N S
 ;;======================================================================
 
 (define (rmt:get-run-info run-id)
@@ -185,10 +205,16 @@
   (let* ((res  (rmt:send-receive 'get-runs (list runpatt count offset keypatts)))
 	 (hedr (car res))
 	 (data (cadr res)))
     (vector hedr (map list->vector data))))
 
+(define (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit)
+  (let* ((res  (rmt:send-receive 'get-runs-by-patt (list runpatt count offset keypatts)))
+	 (hedr (car res))
+	 (data (cadr res)))
+    (vector hedr (map list->vector data))))
+  
 ;;======================================================================
 ;;  S T E P S
 ;;======================================================================
 
 ;; Getting steps is more complicated.

Index: tdb.scm
==================================================================
--- tdb.scm
+++ tdb.scm
@@ -40,10 +40,110 @@
 ;;
 ;; T E S T   D A T A B A S E S
 ;;
 ;;======================================================================
 
+;;======================================================================
+;; T E S T   S P E C I F I C   D B 
+;;======================================================================
+
+;; Create the sqlite db for the individual test(s)
+(define (open-test-db work-area) 
+  (debug:print-info 11 "open-test-db " work-area)
+  (if (and work-area 
+	   (directory? work-area)
+	   (file-read-access? work-area))
+      (let* ((dbpath    (conc work-area "/testdat.db"))
+	     (tdb-writeable (file-write-access? dbpath))
+	     (dbexists  (file-exists? dbpath))
+	     (handler   (make-busy-timeout (if (args:get-arg "-override-timeout")
+					       (string->number (args:get-arg "-override-timeout"))
+					       136000))))
+	(handle-exceptions
+	 exn
+	 (begin
+	   (debug:print 2 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test"
+			((condition-property-accessor 'exn 'message) exn))
+	   (set! db (sqlite3:open-database ":memory:"))) ;; open an in-memory db to allow readonly access 
+	 (set! db (sqlite3:open-database dbpath)))
+	(if *db-write-access* (sqlite3:set-busy-handler! db handler))
+	(if (not dbexists)
+	    (begin
+	      (sqlite3:execute db "PRAGMA synchronous = FULL;")
+	      (debug:print-info 11 "Initialized test database " dbpath)
+	      (tdb:testdb-initialize db)))
+	;; (sqlite3:execute db "PRAGMA synchronous = 0;")
+	(debug:print-info 11 "open-test-db END (sucessful)" work-area)
+	;; now let's test that everything is correct
+	(handle-exceptions
+	 exn
+	 (begin
+	   (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test"
+			((condition-property-accessor 'exn 'message) exn))
+	   #f)
+	 ;; Is there a cheaper single line operation that will check for existance of a table
+	 ;; and raise an exception ?
+	 (sqlite3:execute db "SELECT id FROM test_data LIMIT 1;"))
+	db)
+      (begin
+	(debug:print-info 11 "open-test-db END (unsucessful)" work-area)
+	#f)))
+
+;; find and open the testdat.db file for an existing test
+(define (tdb:open-test-db-by-test-id test-id #!key (work-area #f))
+  (let* ((test-path (if work-area
+			work-area
+			(rmt:test-get-rundir-from-test-id test-id))))
+    (debug:print 3 "TEST PATH: " test-path)
+    (open-test-db test-path)))
+
+(define (tdb:testdb-initialize db)
+  (debug:print 11 "db:testdb-initialize START")
+  (for-each
+   (lambda (sqlcmd)
+     (sqlite3:execute db sqlcmd))
+   (list "CREATE TABLE IF NOT EXISTS test_rundat (
+              id INTEGER PRIMARY KEY,
+              update_time TIMESTAMP,
+              cpuload INTEGER DEFAULT -1,
+              diskfree INTEGER DEFAULT -1,
+              diskusage INTGER DEFAULT -1,
+              run_duration INTEGER DEFAULT 0);"
+	 "CREATE TABLE IF NOT EXISTS test_data (
+              id INTEGER PRIMARY KEY,
+              test_id INTEGER,
+              category TEXT DEFAULT '',
+              variable TEXT,
+	      value REAL,
+	      expected REAL,
+	      tol REAL,
+              units TEXT,
+              comment TEXT DEFAULT '',
+              status TEXT DEFAULT 'n/a',
+              type TEXT DEFAULT '',
+              CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable));"
+	 "CREATE TABLE IF NOT EXISTS test_steps (
+              id INTEGER PRIMARY KEY,
+              test_id INTEGER, 
+              stepname TEXT, 
+              state TEXT DEFAULT 'NOT_STARTED', 
+              status TEXT DEFAULT 'n/a',
+              event_time TIMESTAMP,
+              comment TEXT DEFAULT '',
+              logfile TEXT DEFAULT '',
+              CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state));"
+	 ;; test_meta can be used for handing commands to the test
+	 ;; e.g. KILLREQ
+	 ;;      the ackstate is set to 1 once the command has been completed
+	 "CREATE TABLE IF NOT EXISTS test_meta (
+              id INTEGER PRIMARY KEY,
+              var TEXT,
+              val TEXT,
+              ackstate INTEGER DEFAULT 0,
+              CONSTRAINT metadat_constraint UNIQUE (var));"))
+  (debug:print 11 "db:testdb-initialize END"))
+
 (define (tdb:get-steps-data tdb test-id)
   (let ((res '()))
     (sqlite3:for-each-row 
      (lambda (id test-id stepname state status event-time logfile)
        (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
@@ -60,5 +160,362 @@
        (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
      tdb
      "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
     (sqlite3:finalize! tdb)
     (reverse res)))
+
+;;======================================================================
+;; T E S T   D A T A 
+;;======================================================================
+
+(define (tdb:csv->test-data test-id csvdata #!key (work-area #f))
+  (debug:print 4 "test-id " test-id ", csvdata: " csvdata)
+  (let ((tdb     (tdb:open-test-db-by-test-id test-id work-area: work-area)))
+    (if (sqlite3:database? tdb)
+	(let ((csvlist (csv->list (make-csv-reader
+				   (open-input-string csvdata)
+				   '((strip-leading-whitespace? #t)
+				     (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata)))
+	  (for-each 
+	   (lambda (csvrow)
+	     (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
+		    (category    (list-ref padded-row 0))
+		    (variable    (list-ref padded-row 1))
+		    (value       (any->number-if-possible (list-ref padded-row 2)))
+		    (expected    (any->number-if-possible (list-ref padded-row 3)))
+		    (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
+		    (units       (list-ref padded-row 5))
+		    (comment     (list-ref padded-row 6))
+		    (status      (let ((s (list-ref padded-row 7)))
+				   (if (and (string? s)(or (string-match (regexp "^\\s*$") s)
+							   (string-match (regexp "^n/a$") s)))
+				       #f
+				       s))) ;; if specified on the input then use, else calculate
+		    (type        (list-ref padded-row 8)))
+	       ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
+	       (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value 
+			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
+
+	       (if (and (or (not expected)(equal? expected ""))
+			(or (not tol)     (equal? expected ""))
+			(or (not units)   (equal? expected "")))
+		   (let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test db test-id category variable)))
+			       (set! expected new-expected)
+			       (set! tol      new-tol)
+			       (set! units    new-units)))
+
+	       (debug:print 4 "AFTER:  category: " category " variable: " variable " value: " value 
+			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	       ;; calculate status if NOT specified
+	       (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
+		   (if (number? tol) ;; if tol is a number then we do the standard comparison
+		       (let* ((max-val (+ expected tol))
+			      (min-val (- expected tol))
+			      (result  (and (>=  value min-val)(<= value max-val))))
+			 (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result)
+			 (set! status (if result "pass" "fail")))
+		       (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
+			     (case (string->symbol tol) ;; tol should be >, <, >=, <=
+			       ((>)  (if (>  value expected) "pass" "fail"))
+			       ((<)  (if (<  value expected) "pass" "fail"))
+			       ((>=) (if (>= value expected) "pass" "fail"))
+			       ((<=) (if (<= value expected) "pass" "fail"))
+			       (else (conc "ERROR: bad tol comparator " tol))))))
+	       (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value 
+			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	       (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
+				test-id category variable value expected tol units (if comment comment "") status type)))
+	   csvlist)
+	  (sqlite3:finalize! tdb)))))
+
+;; get a list of test_data records matching categorypatt
+(define (tdb:read-test-data test-id categorypatt #!key (work-area #f))
+  (let ((tdb  (tdb:open-test-db-by-test-id test-id work-area: work-area)))
+    (if (sqlite3:database? tdb)
+	(let ((res '()))
+	  (sqlite3:for-each-row 
+	   (lambda (id test_id category variable value expected tol units comment status type)
+	     (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
+	   tdb
+	   "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
+	  (sqlite3:finalize! tdb)
+	  (reverse res))
+	'())))
+
+;; NOTE: Run this local with #f for db !!!
+(define (tdb:load-test-data test-id #!key (work-area #f))
+  (let loop ((lin (read-line)))
+    (if (not (eof-object? lin))
+	(begin
+	  (debug:print 4 lin)
+	  (tdb:csv->test-data db test-id lin work-area: work-area)
+	  (loop (read-line)))))
+  ;; roll up the current results.
+  ;; FIXME: Add the status to 
+  (tdb:test-data-rollup db test-id #f work-area: work-area))
+
+;; WARNING: Do NOT call this for the parent test on an iterated test
+;; Roll up test_data pass/fail results
+;; look at the test_data status field, 
+;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
+;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
+(define (tdb:test-data-rollup test-id status #!key (work-area #f))
+  (let ((tdb (tdb:open-test-db-by-test-id test-id work-area: work-area))
+	(fail-count 0)
+	(pass-count 0))
+    (if (sqlite3:database? tdb)
+	(begin
+	  (sqlite3:for-each-row
+	   (lambda (fcount pcount)
+	     (set! fail-count fcount)
+	     (set! pass-count pcount))
+	   tdb 
+	   "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
+                   (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;"
+	   test-id test-id)
+	  (sqlite3:finalize! tdb)
+
+	  ;; Now rollup the counts to the central megatest.db
+	  (rmt:general-call 'pass-fail-counts fail-count pass-count test-id)
+	  ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" 
+	  ;;                     fail-count pass-count test-id)
+
+	  ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines
+	  ;; next time you read this!
+	  ;;
+	  ;; (cdb:flush-queue *runremote*)
+	  ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set
+	  
+	  ;; if the test is not FAIL then set status based on the fail and pass counts.
+	  (rmt:general-call 'test-rollup-test_data-pass-fail test-id)
+	  ;; (sqlite3:execute
+	  ;;  db   ;;; NOTE: Should this be WARN,FAIL? A WARN is not a FAIL????? BUG FIXME
+	  ;;  "UPDATE tests
+	  ;;             SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 
+	  ;;                THEN 'FAIL'
+	  ;;             WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND 
+	  ;;                  (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL')
+	  ;;             THEN 'PASS'
+	  ;;             ELSE status
+	  ;;         END WHERE id=?;"
+	  ;;  test-id test-id test-id test-id)
+	  ))))
+
+(define (tdb:get-prev-tol-for-test test-id category variable)
+  ;; Finish me?
+  (values #f #f #f))
+
+;;======================================================================
+;; S T E P S 
+;;======================================================================
+
+(define (tdb:step-get-time-as-string vec)
+  (seconds->time-string (tdb:step-get-event_time vec)))
+
+;; db-get-test-steps-for-run
+(define (tdb:get-steps-for-test test-id #!key (work-area #f))
+  (let* ((tdb (tdb:open-test-db-by-test-id test-id work-area: work-area))
+	 (res '()))
+    (if (sqlite3:database? tdb)
+	(handle-exceptions
+	 exn
+	 (begin
+	   (debug:print 0 "ERROR: error on access to testdat for test with id " test-id)
+	   '())
+	 (begin
+	   (sqlite3:for-each-row 
+	    (lambda (id test-id stepname state status event-time logfile)
+	      (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
+	    tdb
+	    "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
+	    test-id)
+	   (sqlite3:finalize! tdb)
+	   (reverse res)))
+	'())))
+
+;; get a pretty table to summarize steps
+;;
+(define (tdb:get-steps-table test-id #!key (work-area #f))
+  (let ((steps   (tdb:get-steps-for-test test-id work-area: work-area)))
+    ;; organise the steps for better readability
+    (let ((res (make-hash-table)))
+      (for-each 
+       (lambda (step)
+	 (debug:print 6 "step=" step)
+	 (let ((record (hash-table-ref/default 
+			res 
+			(tdb:step-get-stepname step) 
+			;;        stepname                start end status Duration  Logfile 
+			(vector (tdb:step-get-stepname step) ""   "" ""     ""        ""))))
+	   (debug:print 6 "record(before) = " record 
+			"\nid:       " (tdb:step-get-id step)
+			"\nstepname: " (tdb:step-get-stepname step)
+			"\nstate:    " (tdb:step-get-state step)
+			"\nstatus:   " (tdb:step-get-status step)
+			"\ntime:     " (tdb:step-get-event_time step))
+	   (case (string->symbol (tdb:step-get-state step))
+	     ((start)(vector-set! record 1 (tdb:step-get-event_time step))
+	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
+					(tdb:step-get-status step)))
+	      (if (> (string-length (tdb:step-get-logfile step))
+		     0)
+		  (vector-set! record 5 (tdb:step-get-logfile step))))
+	     ((end)  
+	      (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
+	      (vector-set! record 3 (tdb:step-get-status step))
+	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
+					  (endt   (any->number (vector-ref record 2))))
+				      (debug:print 4 "record[1]=" (vector-ref record 1) 
+						   ", startt=" startt ", endt=" endt
+						   ", get-status: " (tdb:step-get-status step))
+				      (if (and (number? startt)(number? endt))
+					  (seconds->hr-min-sec (- endt startt)) "-1")))
+	      (if (> (string-length (tdb:step-get-logfile step))
+		     0)
+		  (vector-set! record 5 (tdb:step-get-logfile step))))
+	     (else
+	      (vector-set! record 2 (tdb:step-get-state step))
+	      (vector-set! record 3 (tdb:step-get-status step))
+	      (vector-set! record 4 (tdb:step-get-event_time step))))
+	   (hash-table-set! res (tdb:step-get-stepname step) record)
+	   (debug:print 6 "record(after)  = " record 
+			"\nid:       " (tdb:step-get-id step)
+			"\nstepname: " (tdb:step-get-stepname step)
+			"\nstate:    " (tdb:step-get-state step)
+			"\nstatus:   " (tdb:step-get-status step)
+			"\ntime:     " (tdb:step-get-event_time step))))
+       ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
+       (sort steps (lambda (a b)
+		     (cond
+		      ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
+		      ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
+		       (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
+		      (else #f)))))
+      res)))
+
+;; get a pretty table to summarize steps
+;;
+(define (tdb:get-steps-table-list test-id #!key (work-area #f))
+  (let ((steps   (tdb:get-steps-for-test test-id work-area: work-area)))
+    ;; organise the steps for better readability
+    (let ((res (make-hash-table)))
+      (for-each 
+       (lambda (step)
+	 (debug:print 6 "step=" step)
+	 (let ((record (hash-table-ref/default 
+			res 
+			(tdb:step-get-stepname step) 
+			;;        stepname                start end status    
+			(vector (tdb:step-get-stepname step) ""   "" ""     "" ""))))
+	   (debug:print 6 "record(before) = " record 
+			"\nid:       " (tdb:step-get-id step)
+			"\nstepname: " (tdb:step-get-stepname step)
+			"\nstate:    " (tdb:step-get-state step)
+			"\nstatus:   " (tdb:step-get-status step)
+			"\ntime:     " (tdb:step-get-event_time step))
+	   (case (string->symbol (tdb:step-get-state step))
+	     ((start)(vector-set! record 1 (tdb:step-get-event_time step))
+	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
+					(tdb:step-get-status step)))
+	      (if (> (string-length (tdb:step-get-logfile step))
+		     0)
+		  (vector-set! record 5 (tdb:step-get-logfile step))))
+	     ((end)  
+	      (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
+	      (vector-set! record 3 (tdb:step-get-status step))
+	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
+					  (endt   (any->number (vector-ref record 2))))
+				      (debug:print 4 "record[1]=" (vector-ref record 1) 
+						   ", startt=" startt ", endt=" endt
+						   ", get-status: " (tdb:step-get-status step))
+				      (if (and (number? startt)(number? endt))
+					  (seconds->hr-min-sec (- endt startt)) "-1")))
+	      (if (> (string-length (tdb:step-get-logfile step))
+		     0)
+		  (vector-set! record 5 (tdb:step-get-logfile step))))
+	     (else
+	      (vector-set! record 2 (tdb:step-get-state step))
+	      (vector-set! record 3 (tdb:step-get-status step))
+	      (vector-set! record 4 (tdb:step-get-event_time step))))
+	   (hash-table-set! res (tdb:step-get-stepname step) record)
+	   (debug:print 6 "record(after)  = " record 
+			"\nid:       " (tdb:step-get-id step)
+			"\nstepname: " (tdb:step-get-stepname step)
+			"\nstate:    " (tdb:step-get-state step)
+			"\nstatus:   " (tdb:step-get-status step)
+			"\ntime:     " (tdb:step-get-event_time step))))
+       ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
+       (sort steps (lambda (a b)
+		     (cond
+		      ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
+		      ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
+		       (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
+		      (else #f)))))
+      res)))
+
+(define (tdb:get-compressed-steps test-id #!key (work-area #f)(tdb #f))
+  (if (or (not work-area)
+	  (file-exists? (conc work-area "/testdat.db")))
+      (let* ((comprsteps (tdb:get-steps-table test-id work-area: work-area)))
+	(map (lambda (x)
+	       ;; take advantage of the \n on time->string
+	       (vector
+		(vector-ref x 0)
+		(let ((s (vector-ref x 1)))
+		  (if (number? s)(seconds->time-string s) s))
+		(let ((s (vector-ref x 2)))
+		  (if (number? s)(seconds->time-string s) s))
+		(vector-ref x 3)    ;; status
+		(vector-ref x 4)
+		(vector-ref x 5)))  ;; time delta
+	     (sort (hash-table-values comprsteps)
+		   (lambda (a b)
+		     (let ((time-a (vector-ref a 1))
+			   (time-b (vector-ref b 1)))
+		       (if (and (number? time-a)(number? time-b))
+			   (if (< time-a time-b)
+			       #t
+			       (if (eq? time-a time-b)
+				   (string<? (conc (vector-ref a 2))
+					     (conc (vector-ref b 2)))
+				   #f))
+			   (string<? (conc time-a)(conc time-b))))))))
+      '()))
+
+(define (tdb:teststep-set-status! test-id teststep-name state-in status-in comment logfile #!key (work-area #f))
+  (let* ((tdb       (tdb:open-test-db-by-test-id test-id work-area: work-area))
+	 (state     (items:check-valid-items "state" state-in))
+	 (status    (items:check-valid-items "status" status-in)))
+    (if (or (not state)(not status))
+	(debug:print 3 "WARNING: Invalid " (if status "status" "state")
+		     " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
+    (if (sqlite3:database? tdb)
+	(begin
+	  (sqlite3:execute 
+	   tdb
+	   "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);"
+	   test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile ""))
+	  (sqlite3:finalize! tdb)
+	  #t)
+	#f)))
+
+;; this one is a bit broken BUG FIXME
+(define (tdb:delete-test-step-records test-id #!key (work-area #f))
+  ;; Breaking it into two queries for better file access interleaving
+  (let* ((tdb (tdb:open-test-db-by-test-id test-id work-area: work-area)))
+    ;; test db's can go away - must check every time
+    (if (sqlite3:database? tdb)
+	(begin
+	  (sqlite3:execute tdb "DELETE FROM test_steps;")
+	  (sqlite3:execute tdb "DELETE FROM test_data;")
+	  (sqlite3:finalize! tdb)))))
+
+;; 
+(define (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
+  (let ((tdb         (tdb:open-test-db-by-test-id test-id work-area: work-area)))
+    (if (sqlite3:database? tdb)
+	(begin
+	  (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
+			   cpuload diskfree minutes)
+	  (sqlite3:finalize! tdb))
+	(debug:print 2 "Can't update testdat.db for test " test-id " read-only or non-existant"))))
+    

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -17,10 +17,11 @@
 (import (prefix sqlite3 sqlite3:))
 
 (declare (unit tests))
 (declare (uses lock-queue))
 (declare (uses db))
+(declare (uses tdb))
 (declare (uses common))
 (declare (uses items))
 (declare (uses runconfig))
 
 (include "common_records.scm")
@@ -195,13 +196,11 @@
   (rmt:test-set-status-state test-id status state #f)
   (mt:process-triggers test-id state status))
 
 ;; Do not rpc this one, do the underlying calls!!!
 (define (tests:test-set-status! test-id state status comment dat #!key (work-area #f))
-  (debug:print-info 4 "tests:test-set-status! test-id=" test-id ", state=" state ", status=" status ", dat=" dat)
-  (let* ((db          #f)
-	 (real-status status)
+  (let* ((real-status status)
 	 (otherdat    (if dat dat (make-hash-table)))
 	 (testdat     (rmt:get-test-info-by-id test-id))
 	 (run-id      (db:test-get-run_id testdat))
 	 (test-name   (db:test-get-testname   testdat))
 	 (item-path   (db:test-get-item-path testdat))
@@ -290,15 +289,14 @@
 		 (string-match (regexp "\\S+") comment))
 	    waived)
 	(let ((cmt  (if waived waived comment)))
 	  (rmt:general-call 'set-test-comment (list cmt test-id))))))
 
-
-(define (tests:test-set-toplog! db run-id test-name logf) 
+(define (tests:test-set-toplog! run-id test-name logf) 
   (rmt:general-call 'tests:test-set-toplog logf run-id test-name))
 
-(define (tests:summarize-items db run-id test-id test-name force)
+(define (tests:summarize-items run-id test-id test-name force)
   ;; if not force then only update the record if one of these is true:
   ;;   1. logf is "log/final.log
   ;;   2. logf is same as outputfilename
   (let* ((outputfilename (conc "megatest-rollup-" test-name ".html"))
 	 (orig-dir       (current-directory))
@@ -316,12 +314,11 @@
     (debug:print 4 "summarize-items with logf " logf ", outputfilename " outputfilename " and force " force)
     (if (or (equal? logf "logs/final.log")
 	    (equal? logf outputfilename)
 	    force)
 	(begin
-	  (if ;; (not (obtain-dot-lock outputfilename 1 5 7)) ;; retry every second for 20 seconds, call it dead after 30 seconds and steal the lock
-              (not (lock-queue:wait-turn outputfilename test-id))
+	  (if (not (lock-queue:wait-turn outputfilename test-id))
 	      (print "Not updating " outputfilename " as another test item has signed up for the job")
 	      (begin
 		(print "Obtained lock for " outputfilename)
 		(let ((oup    (open-output-file outputfilename))
 		      (counts (make-hash-table))
@@ -377,16 +374,17 @@
 		      (print "</td></td></tr></table>")
 		      
 		      (print "<table cellspacing=\"0\" border=\"1\">" 
 			     "<tr><td>Item</td><td>State</td><td>Status</td><td>Comment</td>"
 			     outtxt "</table></body></html>")
-		      (release-dot-lock outputfilename)))
+		      ;; (release-dot-lock outputfilename)
+		      ))
 		  (close-output-port oup)
 		  (lock-queue:release-lock outputfilename test-id)
 		  (change-directory orig-dir)
 		  ;; NB// tests:test-set-toplog! is remote internal...
-		  (tests:test-set-toplog! db run-id test-name outputfilename)
+		  (tests:test-set-toplog! run-id test-name outputfilename)
 		  )))))))
 
 ;;======================================================================
 ;; Gather data from test/task specifications
 ;;======================================================================
@@ -606,37 +604,29 @@
 	 "SELECT count(id) FROM test_rundat;")
 	res))
   0)
 
 (define (tests:update-central-meta-info test-id cpuload diskfree minutes uname hostname)
-  ;; This is a good candidate for threading the requests to enable
-  ;; transactionized write at the server
   (rmt:general-call 'update-cpuload-diskfree cpuload diskfree test-id)
   (if minutes 
       (rmt:general-call 'update-run-duration minutes test-id))
   (if (and uname hostname)
       (rmt:general-call 'update-uname-host uname hostname test-id)))
   
-(define (tests:set-full-meta-info db test-id run-id minutes work-area)
-  ;; DOES cdb:remote-run under the hood!
-  (let* ((num-records 0) ;; (test:tdb-get-rundat-count tdb))
+(define (tests:set-full-meta-info test-id run-id minutes work-area)
+  (let* ((num-records 0)
 	 (cpuload  (get-cpu-load))
 	 (diskfree (get-df (current-directory)))
 	 (uname    (get-uname "-srvpio"))
 	 (hostname (get-host-name)))
-    (rmt:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
+    (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
     (tests:update-central-meta-info test-id cpuload diskfree minutes uname hostname)))
 	  
-(define (tests:set-partial-meta-info db test-id run-id minutes work-area)
-  ;; DOES cdb:remote-run under the hood!
+(define (tests:set-partial-meta-info test-id run-id minutes work-area)
   (let* ((cpuload  (get-cpu-load))
 	 (diskfree (get-df (current-directory))))
-    (rmt:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
-    ;; Update central with uname and hostname = #f
-    ;; Is this one of the performance problems? This info should come from testdat-meta anyway
-    ;; (tests:update-central-meta-info test-id cpuload diskfree minutes #f #f)
-  ))
+    (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes)))
 	 
 ;;======================================================================
 ;; A R C H I V I N G
 ;;======================================================================
 

Index: tests/unittests/server.scm
==================================================================
--- tests/unittests/server.scm
+++ tests/unittests/server.scm
@@ -75,7 +75,7 @@
 
 ;;======================================================================
 ;; D B
 ;;======================================================================
 
-(test #f '(#t "exit process started") (cdb:kill-server *runremote* #f)) ;; *toppath* *my-client-signature* #f)))
+(test #f '(#t "exit process started") (rmt:kill-server)) ;; *toppath* *my-client-signature* #f)))