Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -6,10 +6,12 @@
 ;; 
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 ;;======================================================================
+
+(use srfi-69 posix)
 
 (declare (unit api))
 (declare (uses rmt))
 (declare (uses db))
 (declare (uses tasks))
@@ -72,13 +74,13 @@
 
     ;; TESTS
     test-set-state-status-by-id
     delete-test-records
     delete-old-deleted-test-records
-    test-set-status-state
+    test-set-state-status
     test-set-top-process-pid
-    roll-up-pass-fail-counts
+    set-state-status-and-roll-up-items
     update-pass-fail-counts
     top-test-set-per-pf-counts ;; (db:top-test-set-per-pf-counts (db:get-db *db* 5) 5 "runfirst")
 
     ;; RUNS
     register-run
@@ -122,165 +124,171 @@
      (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
    (if (not (vector? dat))                    ;; it is an error to not receive a vector
        (vector #f #f "remote must be called with a vector")       
        (vector                                   ;; return a vector + the returned data structure
 	#t 
-	(let ((cmd    (vector-ref dat 0))
-	      (params (vector-ref dat 1)))
-	  (case (if (symbol? cmd)
-		    cmd
-		    (string->symbol cmd))
-
-	    ;;===============================================
-	    ;; READ/WRITE QUERIES
-	    ;;===============================================
-
-	    ;; SERVERS
-	    ((start-server)                    (apply server:kind-run params))
-	    ((kill-server)                     (set! *server-run* #f))
-
-	    ;; TESTS
-	    ((test-set-state-status-by-id)     (apply db:test-set-state-status-by-id dbstruct params))
-	    ((delete-test-records)             (apply db:delete-test-records dbstruct params))
-	    ((delete-old-deleted-test-records) (apply db:delete-old-deleted-test-records dbstruct params))
-	    ((test-set-status-state)           (apply db:test-set-status-state dbstruct params))
-	    ((test-set-top-process-pid)        (apply db:test-set-top-process-pid dbstruct params))
-	    ((roll-up-pass-fail-counts)        (apply db:roll-up-pass-fail-counts dbstruct params))
-	    ;; ((update-pass-fail-counts)         (apply db:general-call dbstruct 'update-pass-fail-counts params))
-	    ((top-test-set-per-pf-counts)      (apply db:top-test-set-per-pf-counts dbstruct params))
-	    ((test-set-archive-block-id)       (apply db:test-set-archive-block-id dbstruct params))
-
-	    ;; RUNS
-	    ((register-run)                 (apply db:register-run dbstruct params))
-	    ((set-tests-state-status)       (apply db:set-tests-state-status dbstruct params))
-	    ((delete-run)                   (apply db:delete-run dbstruct params))
-	    ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
-	    ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
-	    ((update-run-stats)             (apply db:update-run-stats dbstruct params))
-	    ((set-var)                      (apply db:set-var dbstruct params))
-
-	    ;; STEPS
-	    ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
-
-	    ;; TEST DATA
-	    ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
-	    ((csv->test-data)               (apply db:csv->test-data dbstruct params))
-
-	    ;; MISC
-	    ((sync-inmem->db)               (let ((run-id (car params)))
-					      (db:sync-touched dbstruct run-id force-sync: #t)))
-	    ((mark-incomplete)              (apply db:find-and-mark-incomplete dbstruct params))
-
-	    ;; TESTMETA
-	    ((testmeta-add-record)       (apply db:testmeta-add-record dbstruct params))
-	    ((testmeta-update-field)     (apply db:testmeta-update-field dbstruct params))
-            ((get-tests-tags)            (db:get-tests-tags dbstruct))
-
-	    ;; TASKS
-	    ((tasks-add)                 (apply tasks:add dbstruct params))   
-	    ((tasks-set-state-given-param-key) (apply tasks:set-state-given-param-key dbstruct params))
-	    ((tasks-get-last)            (apply tasks:get-last dbstruct params))
-
-	    ;; ARCHIVES
-	    ;; ((archive-get-allocations)   
-	    ((archive-register-disk)     (apply db:archive-register-disk dbstruct params))
-	    ((archive-register-block-name)(apply db:archive-register-block-name dbstruct params))
-	    ((archive-allocate-testsuite/area-to-block)(apply db:archive-allocate-testsuite/area-to-block dbstruct block-id testsuite-name areakey))
-
-	    ;;======================================================================
-	    ;; READ ONLY QUERIES
-	    ;;======================================================================
-
-	    ;; KEYS
-	    ((get-key-val-pairs)               (apply db:get-key-val-pairs dbstruct params))
-	    ((get-keys)                        (db:get-keys dbstruct))
-            ((get-keys-write)                        (db:get-keys dbstruct)) ;; force a dummy "write" query to force server
-	    ((get-key-vals)                    (apply db:get-key-vals dbstruct params))
-	    ((get-target)                      (apply db:get-target dbstruct params))
-	    ((get-targets)                     (db:get-targets dbstruct))
-
-	    ;; ARCHIVES
-	    ((test-get-archive-block-info)     (apply db:test-get-archive-block-info dbstruct params))
-	    
-	    ;; TESTS
-	    ((test-toplevel-num-items)         (apply db:test-toplevel-num-items dbstruct params))
-	    ((get-test-info-by-id)	       (apply db:get-test-info-by-id dbstruct params))
-	    ((test-get-rundir-from-test-id)    (apply db:test-get-rundir-from-test-id dbstruct params))
-	    ((get-count-tests-running-for-testname) (apply db:get-count-tests-running-for-testname dbstruct params))
-	    ((get-count-tests-running)         (apply db:get-count-tests-running dbstruct params))
-	    ((get-count-tests-running-in-jobgroup) (apply db:get-count-tests-running-in-jobgroup dbstruct params))
-	    ;; ((delete-test-step-records)        (apply db:delete-test-step-records dbstruct params))
-	    ((get-previous-test-run-record)    (apply db:get-previous-test-run-record dbstruct params))
-	    ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records dbstruct params))
-	    ((test-get-logfile-info)           (apply db:test-get-logfile-info dbstruct params))
-	    ((test-get-records-for-index-file)  (apply db:test-get-records-for-index-file dbstruct params))
-	    ((get-testinfo-state-status)       (apply db:get-testinfo-state-status dbstruct params))
-	    ((test-get-top-process-pid)        (apply db:test-get-top-process-pid dbstruct params))
-	    ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new dbstruct params))
-	    ((get-prereqs-not-met)             (apply db:get-prereqs-not-met dbstruct params))
-	    ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id dbstruct params))
-	    ((synchash-get)                    (apply synchash:server-get dbstruct params))
-	    ((get-raw-run-stats)               (apply db:get-raw-run-stats dbstruct params))
-
-	    ;; RUNS
-	    ((get-run-info)                 (apply db:get-run-info dbstruct params))
-	    ((get-run-status)               (apply db:get-run-status dbstruct params))
-	    ((set-run-status)               (apply db:set-run-status dbstruct params))
-	    ((get-tests-for-run)            (apply db:get-tests-for-run dbstruct params))
-	    ((get-test-id)                  (apply db:get-test-id dbstruct params))
-	    ((get-tests-for-run-mindata)    (apply db:get-tests-for-run-mindata dbstruct params))
-	    ((get-runs)                     (apply db:get-runs dbstruct params))
-	    ((get-num-runs)                 (apply db:get-num-runs dbstruct params))
-	    ((get-all-run-ids)              (db:get-all-run-ids dbstruct))
-	    ((get-prev-run-ids)             (apply db:get-prev-run-ids dbstruct params))
-	    ((get-run-ids-matching-target)  (apply db:get-run-ids-matching-target dbstruct params))
-	    ((get-runs-by-patt)             (apply db:get-runs-by-patt dbstruct params))
-	    ((get-run-name-from-id)         (apply db:get-run-name-from-id dbstruct params))
-	    ((get-main-run-stats)           (apply db:get-main-run-stats dbstruct params))
-	    ((get-var)                      (apply db:get-var dbstruct params))
-	    ((get-run-stats)                (apply db:get-run-stats dbstruct params))
-
-	    ;; STEPS
-	    ((get-steps-data)               (apply db:get-steps-data dbstruct params))
-	    ((get-steps-for-test)           (apply db:get-steps-for-test dbstruct params))
-
-	    ;; TEST DATA
-	    ((read-test-data)               (apply db:read-test-data dbstruct params))
-
-	    ;; MISC
-            ((get-latest-host-load)         (apply db:get-latest-host-load dbstruct params))
-	    ((have-incompletes?)            (apply db:have-incompletes? dbstruct params))
-	    ((login)                        (apply db:login dbstruct params))
-	    ((general-call)                 (let ((stmtname   (car params))
-						  (run-id     (cadr params))
-						  (realparams (cddr params)))
-					      (db:with-db dbstruct run-id #t ;; these are all for modifying the db
-							  (lambda (db)
-							    (db:general-call db stmtname realparams)))))
-	    ((sdb-qry)                      (apply sdb:qry params))
-	    ((ping)                         (current-process-id))
-
-	    ;; TESTMETA
-	    ((testmeta-get-record)       (apply db:testmeta-get-record dbstruct params))
-
-	    ;; TASKS 
-	    ((find-task-queue-records)   (apply tasks:find-task-queue-records dbstruct params))))))))
-
+	(let* ((cmd-in (vector-ref dat 0))
+	       (cmd    (if (symbol? cmd-in)
+			  cmd-in
+			  (string->symbol cmd-in)))
+	       (params (vector-ref dat 1))
+	       (start-t (current-milliseconds))
+	       (res    
+		(case cmd
+		  ;;===============================================
+		  ;; READ/WRITE QUERIES
+		  ;;===============================================
+
+		  ;; SERVERS
+		  ((start-server)                    (apply server:kind-run params))
+		  ((kill-server)                     (set! *server-run* #f))
+
+		  ;; TESTS
+		  ((test-set-state-status-by-id)     (apply db:test-set-state-status-by-id dbstruct params))
+		  ((delete-test-records)             (apply db:delete-test-records dbstruct params))
+		  ((delete-old-deleted-test-records) (apply db:delete-old-deleted-test-records dbstruct params))
+		  ((test-set-state-status)           (apply db:test-set-state-status dbstruct params))
+		  ((test-set-top-process-pid)        (apply db:test-set-top-process-pid dbstruct params))
+		  ((set-state-status-and-roll-up-items) (apply db:set-state-status-and-roll-up-items dbstruct params))
+		  ((top-test-set-per-pf-counts)      (apply db:top-test-set-per-pf-counts dbstruct params))
+		  ((test-set-archive-block-id)       (apply db:test-set-archive-block-id dbstruct params))
+
+		  ;; RUNS
+		  ((register-run)                 (apply db:register-run dbstruct params))
+		  ((set-tests-state-status)       (apply db:set-tests-state-status dbstruct params))
+		  ((delete-run)                   (apply db:delete-run dbstruct params))
+		  ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
+		  ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
+		  ((update-run-stats)             (apply db:update-run-stats dbstruct params))
+		  ((set-var)                      (apply db:set-var dbstruct params))
+
+		  ;; STEPS
+		  ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
+
+		  ;; TEST DATA
+		  ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
+		  ((csv->test-data)               (apply db:csv->test-data dbstruct params))
+
+		  ;; MISC
+		  ((sync-inmem->db)               (let ((run-id (car params)))
+						    (db:sync-touched dbstruct run-id force-sync: #t)))
+		  ((mark-incomplete)              (apply db:find-and-mark-incomplete dbstruct params))
+
+		  ;; TESTMETA
+		  ((testmeta-add-record)       (apply db:testmeta-add-record dbstruct params))
+		  ((testmeta-update-field)     (apply db:testmeta-update-field dbstruct params))
+		  ((get-tests-tags)            (db:get-tests-tags dbstruct))
+
+		  ;; TASKS
+		  ((tasks-add)                 (apply tasks:add dbstruct params))   
+		  ((tasks-set-state-given-param-key) (apply tasks:set-state-given-param-key dbstruct params))
+		  ((tasks-get-last)            (apply tasks:get-last dbstruct params))
+
+		  ;; ARCHIVES
+		  ;; ((archive-get-allocations)   
+		  ((archive-register-disk)     (apply db:archive-register-disk dbstruct params))
+		  ((archive-register-block-name)(apply db:archive-register-block-name dbstruct params))
+		  ((archive-allocate-testsuite/area-to-block)(apply db:archive-allocate-testsuite/area-to-block dbstruct block-id testsuite-name areakey))
+
+		  ;;======================================================================
+		  ;; READ ONLY QUERIES
+		  ;;======================================================================
+
+		  ;; KEYS
+		  ((get-key-val-pairs)               (apply db:get-key-val-pairs dbstruct params))
+		  ((get-keys)                        (db:get-keys dbstruct))
+		  ((get-keys-write)                        (db:get-keys dbstruct)) ;; force a dummy "write" query to force server
+		  ((get-key-vals)                    (apply db:get-key-vals dbstruct params))
+		  ((get-target)                      (apply db:get-target dbstruct params))
+		  ((get-targets)                     (db:get-targets dbstruct))
+
+		  ;; ARCHIVES
+		  ((test-get-archive-block-info)     (apply db:test-get-archive-block-info dbstruct params))
+		  
+		  ;; TESTS
+		  ((test-toplevel-num-items)         (apply db:test-toplevel-num-items dbstruct params))
+		  ((get-test-info-by-id)	       (apply db:get-test-info-by-id dbstruct params))
+		  ((test-get-rundir-from-test-id)    (apply db:test-get-rundir-from-test-id dbstruct params))
+		  ((get-count-tests-running-for-testname) (apply db:get-count-tests-running-for-testname dbstruct params))
+		  ((get-count-tests-running)         (apply db:get-count-tests-running dbstruct params))
+		  ((get-count-tests-running-in-jobgroup) (apply db:get-count-tests-running-in-jobgroup dbstruct params))
+		  ;; ((delete-test-step-records)        (apply db:delete-test-step-records dbstruct params))
+		  ((get-previous-test-run-record)    (apply db:get-previous-test-run-record dbstruct params))
+		  ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records dbstruct params))
+		  ((test-get-logfile-info)           (apply db:test-get-logfile-info dbstruct params))
+		  ((test-get-records-for-index-file)  (apply db:test-get-records-for-index-file dbstruct params))
+		  ((get-testinfo-state-status)       (apply db:get-testinfo-state-status dbstruct params))
+		  ((test-get-top-process-pid)        (apply db:test-get-top-process-pid dbstruct params))
+		  ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new dbstruct params))
+		  ((get-prereqs-not-met)             (apply db:get-prereqs-not-met dbstruct params))
+		  ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id dbstruct params))
+		  ((synchash-get)                    (apply synchash:server-get dbstruct params))
+		  ((get-raw-run-stats)               (apply db:get-raw-run-stats dbstruct params))
+
+		  ;; RUNS
+		  ((get-run-info)                 (apply db:get-run-info dbstruct params))
+		  ((get-run-status)               (apply db:get-run-status dbstruct params))
+		  ((set-run-status)               (apply db:set-run-status dbstruct params))
+		  ((get-tests-for-run)            (apply db:get-tests-for-run dbstruct params))
+		  ((get-test-id)                  (apply db:get-test-id dbstruct params))
+		  ((get-tests-for-run-mindata)    (apply db:get-tests-for-run-mindata dbstruct params))
+		  ((get-runs)                     (apply db:get-runs dbstruct params))
+		  ((get-num-runs)                 (apply db:get-num-runs dbstruct params))
+		  ((get-all-run-ids)              (db:get-all-run-ids dbstruct))
+		  ((get-prev-run-ids)             (apply db:get-prev-run-ids dbstruct params))
+		  ((get-run-ids-matching-target)  (apply db:get-run-ids-matching-target dbstruct params))
+		  ((get-runs-by-patt)             (apply db:get-runs-by-patt dbstruct params))
+		  ((get-run-name-from-id)         (apply db:get-run-name-from-id dbstruct params))
+		  ((get-main-run-stats)           (apply db:get-main-run-stats dbstruct params))
+		  ((get-var)                      (apply db:get-var dbstruct params))
+		  ((get-run-stats)                (apply db:get-run-stats dbstruct params))
+
+		  ;; STEPS
+		  ((get-steps-data)               (apply db:get-steps-data dbstruct params))
+		  ((get-steps-for-test)           (apply db:get-steps-for-test dbstruct params))
+
+		  ;; TEST DATA
+		  ((read-test-data)               (apply db:read-test-data dbstruct params))
+
+		  ;; MISC
+		  ((get-latest-host-load)         (apply db:get-latest-host-load dbstruct params))
+		  ((have-incompletes?)            (apply db:have-incompletes? dbstruct params))
+		  ((login)                        (apply db:login dbstruct params))
+		  ((general-call)                 (let ((stmtname   (car params))
+							(run-id     (cadr params))
+							(realparams (cddr params)))
+						    (db:general-call dbstruct stmtname realparams)))
+		  ((sdb-qry)                      (apply sdb:qry params))
+		  ((ping)                         (current-process-id))
+
+		  ;; TESTMETA
+		  ((testmeta-get-record)       (apply db:testmeta-get-record dbstruct params))
+
+		  ;; TASKS 
+		  ((find-task-queue-records)   (apply tasks:find-task-queue-records dbstruct params)))))
+	  (let ((delta-t (- (current-milliseconds)
+			    start-t)))
+	    (hash-table-set! *db-api-call-time* cmd
+			     (cons delta-t (hash-table-ref/default *db-api-call-time* cmd '()))))
+	  res)))))
 
 ;; http-server  send-response
 ;;                 api:process-request
 ;;                    db:*
 ;;
 ;; NB// Runs on the server as part of the server loop
 ;;
 (define (api:process-request dbstruct $) ;; the $ is the request vars proc
+  (set! *api-process-request-count* (+ *api-process-request-count* 1))
   (let* ((cmd     ($ 'cmd))
 	 (paramsj ($ 'params))
 	 (params  (db:string->obj paramsj transport: 'http)) ;; (rmt:json-str->dat paramsj))
 	 (resdat  (api:execute-requests dbstruct (vector cmd params))) ;; #( flag result )
 	 (res     (vector-ref resdat 1)))
-
+    (if (> *api-process-request-count* *max-api-process-requests*)
+	(set! *max-api-process-requests* *api-process-request-count*))
+    (set! *api-process-request-count* (- *api-process-request-count* 1))
     ;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds
     ;; (rmt:dat->json-str
     ;;  (if (or (string? res)
     ;;          (list?   res)
     ;;          (number? res)

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -97,11 +97,14 @@
 (define *db-multi-sync-mutex* (make-mutex))      ;; protect access to *db-sync-in-progress*, *db-last-sync*
 ;; task db
 (define *task-db*             #f) ;; (vector db path-to-db)
 (define *db-access-allowed*   #t) ;; flag to allow access
 (define *db-access-mutex*     (make-mutex))
+(define *db-transaction-mutex* (make-mutex))
 (define *db-cache-path*       #f)
+(define *db-with-db-mutex*    (make-mutex))
+(define *db-api-call-time*    (make-hash-table)) ;; hash of command => (list of times)
 
 ;; SERVER
 (define *my-client-signature* #f)
 (define *transport-type*    'http)             ;; override with [server] transport http|rpc|nmsg
 (define *runremote*         #f)                ;; if set up for server communication this will hold <host port>
@@ -114,10 +117,12 @@
 (define *run-id*            #f)
 (define *server-kind-run*   (make-hash-table))
 (define *home-host*         #f)
 (define *total-non-write-delay* 0)
 (define *heartbeat-mutex*   (make-mutex))
+(define *api-process-request-count* 0)
+(define *max-api-process-requests* 0)
 
 ;; client
 (define *rmt-mutex*         (make-mutex))     ;; remote access calls mutex 
 
 ;; RPC transport
@@ -960,10 +965,20 @@
       #f ;; better than an exception for my needs
       (fold (lambda (a b)
 	      (if (comp a b) a b))
 	    (car lst)
 	    lst)))
+
+;; get min or max, use > for max and < for min, this works around the limits on apply
+;;
+(define (common:sum lst)
+  (if (null? lst)
+      0
+      (fold (lambda (a b)
+	      (+ a b))
+	    (car lst)
+	    lst)))
 
 ;; path list to hash-table tree
 ;;   ((a b c)(a b d)(e b c)) => ((a (b (d) (c))) (e (b (c))))
 ;;
 (define (common:list->htree lst)

Index: dashboard-tests.scm
==================================================================
--- dashboard-tests.scm
+++ dashboard-tests.scm
@@ -271,13 +271,14 @@
     (iup:frame
      #:title "Set fields"
      (iup:vbox
       (iup:hbox (iup:label "Comment:")
 		(let ((txtbox (iup:textbox #:action (lambda (val a b)
-						      (rmt:test-set-state-status-by-id run-id test-id #f #f b)
+						      ;; (rmt:test-set-state-status-by-id run-id test-id #f #f b)
+						      (rmt:test-set-state-status run-id test-id #f #f b)
 						      ;; IDEA: Just set a variable with the proc to call?
-						      (rmt:test-set-state-status-by-id run-id test-id #f #f b)
+						      ;; (rmt:test-set-state-status-by-id run-id test-id #f #f b)
 						      (set! newcomment b))
 					   #:value (db:test-get-comment testdat)
 					   #:expand "HORIZONTAL")))
 		  (set! wtxtbox txtbox)
 		  txtbox))
@@ -287,11 +288,11 @@
 	     (let* ((btns  (map (lambda (state)
 				  (let ((btn (iup:button state
 							 #:expand "HORIZONTAL" #:size "50x" #:font "Courier New, -10"
 							 #:action (lambda (x)
 								    ;; (rmt:test-set-state-status-by-id run-id test-id state #f #f)
-								    (rmt:roll-up-pass-fail-counts run-id test-id #f state #f #f) ;; test-name passed in as test-id is respected
+								    (rmt:set-state-status-and-roll-up-items run-id test-id #f state #f #f) ;; test-name passed in as test-id is respected
 								    (db:test-set-state! testdat state)))))
 				    btn))
 				(map cadr *common:std-states*)))) ;; (list "COMPLETED" "NOT_STARTED" "RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "KILLED" "KILLREQ"))))
 	       (vector-set! *state-status* 0
 			    (lambda (state color)
@@ -321,11 +322,11 @@
 														    (if (not *dashboard-comment-share-slot*)
 															(set! *dashboard-comment-share-slot* wtxtbox)))
 														  ))))
 									  (begin
 									    ;; (rmt:test-set-state-status-by-id run-id test-id #f status #f)
-									    (rmt:roll-up-pass-fail-counts run-id test-id #f #f status #f) ;; test-name passed in as test-id is respected
+									    (rmt:set-state-status-and-roll-up-items run-id test-id #f #f status #f) ;; test-name passed in as test-id is respected
 									    (db:test-set-status! testdat status))))))))
 				    btn))
 				(map cadr *common:std-statuses*)))) ;; (list  "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP"))))
 	       (vector-set! *state-status* 1
 			    (lambda (status color)
@@ -402,11 +403,12 @@
 					 (let ((comment (iup:attribute comnt "VALUE"))
 					       (test-id (db:test-get-id testdat)))
 					   (if (or (not wpatt)
 						   (string-match wregx comment))
 					       (begin
-						 (rmt:test-set-state-status-by-id run-id test-id #f "WAIVED" comment)
+						 ;; (rmt:test-set-state-status-by-id run-id test-id #f "WAIVED" comment)
+						 (rmt:test-set-state-status-by run-id test-id #f "WAIVED" comment)
 						 (db:test-set-status! testdat "WAIVED")
 						 (cmtcmd comment)
 						 (iup:destroy! dlog))))))
 		  (iup:button "Cancel"
 			      #:expand "HORIZONTAL" 

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -13,11 +13,11 @@
 ;; Database access
 ;;======================================================================
 
 ;; dbstruct vector containing all the relevant dbs like main.db, megatest.db, run.db etc
 
-(require-extension (srfi 18) extras tcp) ;; RADT => use of require-extension?
+(use (srfi 18) extras tcp) ;; RADT => use of require-extension?
 (use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3 typed-records)
 (import (prefix sqlite3 sqlite3:))
 (import (prefix base64 base64:)) ;; RADT => prefix??
 
 (declare (unit db))
@@ -131,17 +131,26 @@
 		    (db:get-db dbstruct run-id)
 		    (begin
 		      (print-call-chain)
 		      (print "db:with-db called with dbdat instead of dbstruct, FIXME!!")
 		      dbstruct))) ;; cheat, allow for passing in a dbdat
-	 (db    (db:dbdat-get-db dbdat))) 
+	 (db    (db:dbdat-get-db dbdat))
+	 (use-mutex (> *api-process-request-count* 50)))
+    (if (and use-mutex
+	     (common:low-noise-print 120 "over-50-parallel-api-requests"))
+	(debug:print-info 0 *default-log-port* *api-process-request-count* " parallel api requests being processed in process " (current-process-id) ", throttling access"))
+    (if (common:low-noise-print 120 "parallel-api-requests")
+	(debug:print-info 0 *default-log-port* "Parallel api request count: " *api-process-request-count* " max parallel requests: " *max-api-process-requests*))
     (handle-exceptions
      exn
      (begin
+       (print-call-chain (current-error-port))
        (debug:print-error 0 *default-log-port* "sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn))
-       (print-call-chain (current-error-port)))
+       )
+     (if use-mutex (mutex-lock! *db-with-db-mutex*))
      (let ((res (apply proc db params)))
+       (if use-mutex (mutex-unlock! *db-with-db-mutex*))
        ;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
        res))))
 
 ;;======================================================================
 ;; K E E P   F I L E D B   I N   dbstruct
@@ -626,11 +635,11 @@
 	    ;; first pass implementation, just insert all changed rows
 	    (for-each 
 	     (lambda (targdb)
 	       (let* ((db     (db:dbdat-get-db targdb))
 		      (stmth  (sqlite3:prepare db full-ins)))
-		 ;; (db:delay-if-busy targdb) ;; NO WAITING
+		 (db:delay-if-busy targdb) ;; NO WAITING
 		 (for-each
 		  (lambda (fromdat-lst)
 		    (sqlite3:with-transaction
 		     db
 		     (lambda ()
@@ -1139,11 +1148,11 @@
                      fail_count   INTEGER   DEFAULT 0,
                      pass_count   INTEGER   DEFAULT 0,
                      archived     INTEGER   DEFAULT 0, -- 0=no, > 1=archive block id where test data can be found
                      last_update  INTEGER DEFAULT (strftime('%s','now')),
                         CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path));")
-     (sqlite3:execute db "CREATE INDEX IF NOT EXISTS tests_index ON tests (run_id, testname, item_path);")
+     (sqlite3:execute db "CREATE INDEX IF NOT EXISTS tests_index ON tests (run_id, testname, item_path, uname);")
      (sqlite3:execute db "CREATE TRIGGER  IF NOT EXISTS update_tests_trigger AFTER UPDATE ON tests
                              FOR EACH ROW
                                BEGIN 
                                  UPDATE tests SET last_update=(strftime('%s','now'))
                                    WHERE id=old.id;
@@ -1500,11 +1509,11 @@
       (if (> (length all-ids) 0)
 	  (begin
 	    (debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ") " as INCOMPLETE")
             (for-each
              (lambda (test-id)
-               (db:test-set-status-state dbstruct run-id test-id "COMPLETE" "DEAD" "Test failed to complete"))
+               (db:test-set-state-status dbstruct run-id test-id "COMPLETE" "DEAD" "Test failed to complete"))
              all-ids))))))
 
 ;; ALL REPLACED BY THE BLOCK ABOVE
 ;;
 ;; 	    (sqlite3:execute 
@@ -1525,13 +1534,12 @@
 ;;      toplevels)))
 
 ;; BUG: Probably broken - does not explicitly use run-id in the query
 ;;
 (define (db:top-test-set-per-pf-counts dbstruct run-id test-name)
-  (db:general-call (db:get-db dbstruct run-id) 'top-test-set-per-pf-counts (list test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name))) 
- 
-		     
+  (db:general-call dbstruct 'top-test-set-per-pf-counts (list test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name)))
+
 ;; Clean out old junk and vacuum the database
 ;;
 ;; Ultimately do something like this:
 ;;
 ;; 1. Look at test records either deleted or part of deleted run:
@@ -1699,16 +1707,15 @@
 ;; 	(begin
 ;; 	  (debug:print-info 4 *default-log-port* "launch throttle factor=" *global-delta*)
 ;; 	  (set! *last-global-delta-printed* *global-delta*)))
 
 (define (db:set-var dbstruct var val)
-  (let* ((dbdat (db:get-db dbstruct #f))
-	 (db    (db:dbdat-get-db dbdat)))
-    (sqlite3:execute db "INSERT OR REPLACE INTO metadat (var,val) VALUES (?,?);" var val)))
+  (db:with-db dbstruct #f #t 
+	      (lambda (db)
+		(sqlite3:execute db "INSERT OR REPLACE INTO metadat (var,val) VALUES (?,?);" var val))))
 
 (define (db:del-var dbstruct var)
-  ;; (db:delay-if-busy)
   (db:with-db dbstruct #f #t 
 	      (lambda (db)
 		(sqlite3:execute db "DELETE FROM metadat WHERE var=?;" var))))
 
 ;; use a global for some primitive caching, it is just silly to
@@ -1817,26 +1824,28 @@
 	 (qryvals   (append (list runname) (map cadr keyvals)))
 	 (key=?str  (string-intersperse (map (lambda (k)(conc k "=?")) keys) " AND ")))
     (debug:print 3 *default-log-port* "keys: " keys " allvals: " allvals " keyvals: " keyvals " key=?str is " key=?str)
     (debug:print 2 *default-log-port* "NOTE: using target " (string-intersperse (map cadr keyvals) "/") " for this run")
     (if (and runname (null? (filter (lambda (x)(not x)) keyvals))) ;; there must be a better way to "apply and"
-	(let ((res #f))
-	  ;; (db:delay-if-busy dbdat)
-	  (apply sqlite3:execute db (conc "INSERT OR IGNORE INTO runs (runname,state,status,owner,event_time" comma keystr ") VALUES (?,?,?,?,strftime('%s','now')" comma valslots ");")
-		 allvals)
-	  ;; (db:delay-if-busy dbdat)
-	  (apply sqlite3:for-each-row 
-		 (lambda (id)
-		   (set! res id))
-		 db
-		 (let ((qry (conc "SELECT id FROM runs WHERE (runname=? " andstr key=?str ");")))
+	(db:with-db
+	 dbstruct #f #f
+	 (lambda (db)
+	   (let ((res #f))
+	     ;; (db:delay-if-busy dbdat)
+	     (apply sqlite3:execute db (conc "INSERT OR IGNORE INTO runs (runname,state,status,owner,event_time" comma keystr ") VALUES (?,?,?,?,strftime('%s','now')" comma valslots ");")
+		    allvals)
+	     ;; (db:delay-if-busy dbdat)
+	     (apply sqlite3:for-each-row 
+		    (lambda (id)
+		      (set! res id))
+		    db
+		    (let ((qry (conc "SELECT id FROM runs WHERE (runname=? " andstr key=?str ");")))
 					;(debug:print 4 *default-log-port* "qry: " qry) 
-		   qry)
-		 qryvals)
-	  ;; (db:delay-if-busy dbdat)
-	  (sqlite3:execute db "UPDATE runs SET state=?,status=?,event_time=strftime('%s','now') WHERE id=? AND state='deleted';" state status res)
-	  res) 
+		      qry)
+		    qryvals)
+	     (sqlite3:execute db "UPDATE runs SET state=?,status=?,event_time=strftime('%s','now') WHERE id=? AND state='deleted';" state status res)
+	     res))) 
 	(begin
 	  (debug:print-error 0 *default-log-port* "Called without all necessary keys")
 	  #f))))
 
 ;; replace header and keystr with a call to runs:get-std-run-fields
@@ -1954,23 +1963,26 @@
      (sqlite3:fold-row
 	(lambda (res state status count)
 	  (cons (list state status count) res))
 	'()
 	db
-	"SELECT state,status,count(id) AS count FROM tests WHERE run_id=? AND NOT(uname='n/a' AND item_path='') GROUP BY state,status;;"
+	"SELECT state,status,count(id) AS count FROM tests WHERE run_id=? AND NOT(uname='n/a' AND item_path='') GROUP BY state,status;"
 	run-id))))
 
 ;; Update run_stats for given run_id
 ;; input data is a list (state status count)
 ;;
 (define (db:update-run-stats dbstruct run-id stats)
+  ;; (mutex-lock! *db-transaction-mutex*)
   (db:with-db
    dbstruct
    #f
    #f
+
    (lambda (db)
      ;; remove previous data
+     
      (let* ((stmt1 (sqlite3:prepare db "DELETE FROM run_stats WHERE run_id=? AND state=? AND status=?;"))
 	    (stmt2 (sqlite3:prepare db "INSERT INTO run_stats (run_id,state,status,count) VALUES (?,?,?,?);"))
 	    (res
 	     (sqlite3:with-transaction
 	      db
@@ -1980,10 +1992,11 @@
 		   (sqlite3:execute stmt1 run-id (car dat)(cadr dat))
 		   (apply sqlite3:execute stmt2 run-id dat))
 		 stats)))))
        (sqlite3:finalize! stmt1)
        (sqlite3:finalize! stmt2)
+       ;; (mutex-unlock! *db-transaction-mutex*)
        res))))
 
 (define (db:get-main-run-stats dbstruct run-id)
   (db:with-db
    dbstruct
@@ -1995,10 +2008,25 @@
 	  (cons (list state status count) res))
 	'()
 	db
 	"SELECT state,status,count FROM run_stats WHERE run_id=? AND run_id IN (SELECT id FROM runs WHERE state NOT IN ('DELETED','deleted'));"
 	run-id))))
+
+(define (db:print-current-query-stats)
+  ;; generate stats from *db-api-call-time*
+  (let ((ordered-keys (sort (hash-table-keys *db-api-call-time*)
+			    (lambda (a b)
+			      (let ((sum-a (common:sum (hash-table-ref *db-api-call-time* a)))
+				    (sum-b (common:sum (hash-table-ref *db-api-call-time* b))))
+				(> sum-a sum-b))))))
+    (for-each
+     (lambda (cmd-key)
+       (let* ((dat  (hash-table-ref *db-api-call-time* cmd-key))
+	      (avg  (if (> (length dat) 0)
+			(/ (common:sum dat)(length dat)))))
+	 (debug:print-info 0 *default-log-port* cmd-key "\tavg: " avg " max: " (common:max dat) " min: " (common:min-max < dat) " num: " (length dat))))
+     ordered-keys)))
 
 (define (db:get-all-run-ids dbstruct)
   (db:with-db
    dbstruct
    #f
@@ -2015,23 +2043,23 @@
 ;; get some basic run stats
 ;;
 ;; ( (runname (( state  count ) ... ))
 ;;   (   ...  
 (define (db:get-run-stats dbstruct)
-  (let* ((dbdat        (db:get-db dbstruct #f))
-	 (db           (db:dbdat-get-db dbdat))
-	 (totals       (make-hash-table))
+  (let* ((totals       (make-hash-table))
 	 (curr         (make-hash-table))
 	 (res          '())
 	 (runs-info    '()))
     ;; First get all the runname/run-ids
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row
-     (lambda (run-id runname)
-       (set! runs-info (cons (list run-id runname) runs-info)))
-     db
-     "SELECT id,runname FROM runs WHERE state != 'deleted' ORDER BY event_time DESC;") ;; If you change this to the more logical ASC please adjust calls to db:get-run-stats
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row
+	(lambda (run-id runname)
+	  (set! runs-info (cons (list run-id runname) runs-info)))
+	db
+	"SELECT id,runname FROM runs WHERE state != 'deleted' ORDER BY event_time DESC;"))) ;; If you change this to the more logical ASC please adjust calls to db:get-run-stats
     ;; for each run get stats data
     (for-each
      (lambda (run-info)
        ;; get the net state/status counts for this run
        (let* ((run-id   (car  run-info))
@@ -2111,26 +2139,26 @@
 
 ;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
 (define (db:get-run-info dbstruct run-id)
   ;;(if (hash-table-ref/default *run-info-cache* run-id #f)
   ;;    (hash-table-ref *run-info-cache* run-id)
-  (let* ((dbdat     (db:get-db dbstruct #f))
-	 (db        (db:dbdat-get-db dbdat))
-	 (res       (vector #f #f #f #f))
+  (let* ((res       (vector #f #f #f #f))
 	 (keys      (db:get-keys dbstruct))
 	 (remfields (list "id" "runname" "state" "status" "owner" "event_time"))
 	 (header    (append keys remfields))
 	 (keystr    (conc (keys->keystr keys) ","
 			  (string-intersperse remfields ","))))
     (debug:print-info 11 *default-log-port* "db:get-run-info run-id: " run-id " header: " header " keystr: " keystr)
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row
-     (lambda (a . x)
-       (set! res (apply vector a x)))
-     db 
-     (conc "SELECT " keystr " FROM runs WHERE id=? AND state != 'deleted';")
-     run-id)
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row
+	(lambda (a . x)
+	  (set! res (apply vector a x)))
+	db 
+	(conc "SELECT " keystr " FROM runs WHERE id=? AND state != 'deleted';")
+	run-id)))
     (debug:print-info 11 *default-log-port* "db:get-run-info run-id: " run-id " header: " header " keystr: " keystr)
     (let ((finalres (vector header res)))
       ;; (hash-table-set! *run-info-cache* run-id finalres)
       finalres)))
 
@@ -2212,39 +2240,40 @@
 
 ;; get key val pairs for a given run-id
 ;; ( (FIELDNAME1 keyval1) (FIELDNAME2 keyval2) ... )
 (define (db:get-key-val-pairs dbstruct run-id)
   (let* ((keys (db:get-keys dbstruct))
-	 (res  '())
-	 (dbdat  (db:get-db dbstruct #f))
-	 (db     (db:dbdat-get-db dbdat)))
-    (for-each 
-     (lambda (key)
-       (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
-	 ;; (db:delay-if-busy dbdat)
-	 (sqlite3:for-each-row 
-	  (lambda (key-val)
-	    (set! res (cons (list key key-val) res)))
-	  db qry run-id)))
-     keys)
-    (reverse res)))
+	 (res  '()))
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (for-each 
+	(lambda (key)
+	  (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
+	    (sqlite3:for-each-row 
+	     (lambda (key-val)
+	       (set! res (cons (list key key-val) res)))
+	     db qry run-id)))
+	keys)))
+       (reverse res)))
 
 ;; get key vals for a given run-id
 (define (db:get-key-vals dbstruct run-id)
   (let* ((keys (db:get-keys dbstruct))
-	 (res  '())
-	 (dbdat  (db:get-db dbstruct #f))
-	 (db     (db:dbdat-get-db dbdat)))
-    (for-each 
-     (lambda (key)
-       (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
-	 ;; (db:delay-if-busy dbdat)
-	 (sqlite3:for-each-row 
-	  (lambda (key-val)
-	    (set! res (cons key-val res)))
-	  db qry run-id)))
-     keys)
+	 (res  '()))
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (for-each 
+	(lambda (key)
+	  (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
+	    ;; (db:delay-if-busy dbdat)
+	    (sqlite3:for-each-row 
+	     (lambda (key-val)
+	       (set! res (cons key-val res)))
+	     db qry run-id)))
+	keys)))
     (let ((final-res (reverse res)))
       (hash-table-set! *keyvals* run-id final-res)
       final-res)))
 
 ;; The target is keyval1/keyval2..., cached in *target* as it is used often
@@ -2437,16 +2466,15 @@
 
 ;; Convert calling routines to get list of run-ids and loop, do not use the get-tests-for-runs
 ;;
 
 (define (db:delete-test-records dbstruct run-id test-id)
-  (let* ((dbdat (db:get-db dbstruct run-id))
-	 (db    (db:dbdat-get-db dbdat)))
-    (db:general-call dbdat 'delete-test-step-records (list test-id))
-    ;; (db:delay-if-busy)
-    (db:general-call dbdat 'delete-test-data-records (list test-id))
-    (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id)))
+  (db:general-call dbstruct 'delete-test-step-records (list test-id))
+  (db:general-call dbstruct 'delete-test-data-records (list test-id))
+  (db:with-db
+   dbstruct #f #f
+   (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id)))
 
 ;; 
 (define (db:delete-old-deleted-test-records dbstruct)
   (let (;; (run-ids  (db:get-all-run-ids dbstruct))
 	(targtime (- (current-seconds)(* 30 24 60 60)))) ;; one month in the past
@@ -2475,26 +2503,25 @@
 (define (db:set-tests-state-status dbstruct run-id testnames currstate currstatus newstate newstatus)
   (for-each (lambda (testname)
 	      (let ((qry (conc "UPDATE tests SET state=?,status=? WHERE "
 			       (if currstate  (conc "state='" currstate "' AND ") "")
 			       (if currstatus (conc "status='" currstatus "' AND ") "")
-			       " run_id=? AND testname LIKE ?;")))
+			       " run_id=? AND testname LIKE ?;"))
+		    (test-id (db:get-test-id dbstruct run-id testname "")))
 		(db:with-db
 		 dbstruct
 		 run-id
 		 #t
 		 (lambda (db)
-		   (let ((test-id (db:get-test-id dbstruct run-id testname "")))
-		     (sqlite3:execute db qry newstate newstatus run-id testname)
-		     (if test-id (mt:process-triggers run-id test-id newstate newstatus)))
-		   ))))
+		   (sqlite3:execute db qry newstate newstatus run-id testname)))
+		(if test-id (mt:process-triggers dbstruct run-id test-id newstate newstatus))))
 	    testnames))
 
-;; speed up for common cases with a little logic
-;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id
-;;
-(define (db:test-set-state-status-by-id dbstruct run-id test-id newstate newstatus newcomment)
+;; ;; speed up for common cases with a little logic
+;; ;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id
+;; ;;
+(define (db:test-set-state-status dbstruct run-id test-id newstate newstatus newcomment)
   (db:with-db
    dbstruct
    run-id
    #t
    (lambda (db)
@@ -2506,12 +2533,12 @@
        (sqlite3:execute db "UPDATE tests SET state=?,status=? WHERE id=?;" newstate newstatus test-id))
       (else
        (if newstate   (sqlite3:execute db "UPDATE tests SET state=?   WHERE id=?;" newstate   test-id))
        (if newstatus  (sqlite3:execute db "UPDATE tests SET status=?  WHERE id=?;" newstatus  test-id))
        (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment ;; (sdb:qry 'getid newcomment)
-				       test-id))))
-     (mt:process-triggers run-id test-id newstate newstatus))))
+				       test-id))))))
+  (mt:process-triggers dbstruct run-id test-id newstate newstatus))
 
 ;; NEW BEHAVIOR: Count tests running in all runs!
 ;;
 (define (db:get-count-tests-running dbstruct run-id)
   (db:with-db
@@ -2567,23 +2594,23 @@
      (sqlite3:first-result
       db
       "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=? AND NOT (uname = 'n/a' AND item_path = '') AND testname=?;" run-id testname))))
 
 (define (db:get-count-tests-running-in-jobgroup dbstruct run-id jobgroup)
-  (let* ((dbdat (db:get-db dbstruct #f))
-	 (db    (db:dbdat-get-db dbdat)))
   (if (not jobgroup)
       0 ;; 
       (let ((testnames '()))
 	;; get the testnames
-	;; (db:delay-if-busy dbdat)
-	(sqlite3:for-each-row
-	 (lambda (testname)
-	   (set! testnames (cons testname testnames)))
-	 db
-	 "SELECT testname FROM test_meta WHERE jobgroup=?"
-	 jobgroup)
+	(db:with-db
+	 dbstruct #f #f
+	 (lambda (db)
+	   (sqlite3:for-each-row
+	    (lambda (testname)
+	      (set! testnames (cons testname testnames)))
+	    db
+	    "SELECT testname FROM test_meta WHERE jobgroup=?"
+	    jobgroup)))
 	;; get the jobcount NB// EXTEND THIS TO OPPERATE OVER ALL RUNS?
 	(if (not (null? testnames))
 	    (db:with-db
 	     dbstruct
 	     run-id
@@ -2593,14 +2620,11 @@
 		db
 		(conc "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND testname in ('"
 		      (string-intersperse testnames "','")
 		      "') AND NOT (uname = 'n/a' AND item_path='');")) ;; should this include the (uname = 'n/a' ...) ???
 	       ))
-	    0)))))
-             ;; DEBUG FIXME - need to merge this v.155 query correctly   
-             ;; AND testname in (SELECT testname FROM test_meta WHERE jobgroup=?)
-             ;; AND NOT (uname = 'n/a' AND item_path = '');"
+	    0))))
 
 ;; tags: '("tag%" "tag2" "%ag6")
 ;;
 
 ;; done with run when:
@@ -2674,24 +2698,22 @@
 
 
 ;; NOTE: Use db:test-get* to access records
 ;; NOTE: This needs rundir decoding? Decide, decode here or where used? For the moment decode where used.
 (define (db:get-all-tests-info-by-run-id dbstruct run-id)
-  (let* ((dbdat (if (vector? dbstruct)
-		    (db:get-db dbstruct run-id)
-		    dbstruct)) ;; still settling on when to use dbstruct or dbdat
-	 (db    (db:dbdat-get-db dbdat))
-	 (res '()))
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row
-     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum archived)
-       ;;                 0    1       2      3      4        5       6      7        8     9     10      11          12          13       14     15        16
-       (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum archived)
-		       res)))
-     db
-     (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE state != 'DELETED' AND run_id=?;")
-     run-id)
+  (let* ((res '()))
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row
+	(lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum archived)
+	  ;;                 0    1       2      3      4        5       6      7        8     9     10      11          12          13       14     15        16
+	  (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum archived)
+			  res)))
+	db
+	(conc "SELECT " db:test-record-qry-selector " FROM tests WHERE state != 'DELETED' AND run_id=?;")
+	run-id)))
     res))
 
 (define (db:replace-test-records dbstruct run-id testrecs)
   (db:with-db dbstruct run-id #t 
 	      (lambda (db)
@@ -2874,23 +2896,25 @@
 (define (db:test-data-rollup dbstruct run-id test-id status)
   (let* ((dbdat      (db:get-db dbstruct run-id))
 	 (db         (db:dbdat-get-db dbdat))
 	 (fail-count 0)
 	 (pass-count 0))
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row
-     (lambda (fcount pcount)
-       (set! fail-count fcount)
-       (set! pass-count pcount))
-     db 
-     "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row
+	(lambda (fcount pcount)
+	  (set! fail-count fcount)
+	  (set! pass-count pcount))
+	db 
+	"SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
              (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;"
-     test-id test-id)
-    ;; Now rollup the counts to the central megatest.db
-    (db:general-call dbdat 'pass-fail-counts (list pass-count fail-count test-id))
-    ;; if the test is not FAIL then set status based on the fail and pass counts.
-    (db:general-call dbdat 'test_data-pf-rollup (list test-id test-id test-id test-id))))
+	test-id test-id)
+       ;; Now rollup the counts to the central megatest.db
+       (db:general-call dbstruct 'pass-fail-counts (list pass-count fail-count test-id))
+       ;; if the test is not FAIL then set status based on the fail and pass counts.
+       (db:general-call dbstruct 'test_data-pf-rollup (list test-id test-id test-id test-id))))))
 
 ;; each section is a rule except "final" which is the final result
 ;;
 ;; [rule-5]
 ;; operator in
@@ -2971,105 +2995,107 @@
 ;; faz,bar,    10,  8mA,     ,     ,"this is a comment"
 ;; EOF
 
 (define (db:csv->test-data dbstruct run-id test-id csvdata)
   (debug:print 4 *default-log-port* "test-id " test-id ", csvdata: " csvdata)
-  (let* ((dbdat   (db:get-db dbstruct run-id))
-	 (db      (db:dbdat-get-db dbdat))
-	 (csvlist (csv->list (make-csv-reader
-			      (open-input-string csvdata)
-			      '((strip-leading-whitespace? #t)
-				(strip-trailing-whitespace? #t)))))) ;; (csv->list csvdata)))
-    (for-each
-     (lambda (csvrow)
-       (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
-	      (category    (list-ref padded-row 0))
-	      (variable    (list-ref padded-row 1))
-	      (value       (any->number-if-possible (list-ref padded-row 2)))
-	      (expected    (any->number-if-possible (list-ref padded-row 3)))
-	      (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
-	      (units       (list-ref padded-row 5))
-	      (comment     (list-ref padded-row 6))
-	      (status      (let ((s (list-ref padded-row 7)))
-			     (if (and (string? s)(or (string-match (regexp "^\\s*$") s)
-						     (string-match (regexp "^n/a$") s)))
-				 #f
-				 s))) ;; if specified on the input then use, else calculate
-	      (type        (list-ref padded-row 8)))
-	 ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
-	 (debug:print 4 *default-log-port* "BEFORE: category: " category " variable: " variable " value: " value 
-		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
-	 
-	 (if (and (or (not expected)(equal? expected ""))
-		  (or (not tol)     (equal? expected ""))
-		  (or (not units)   (equal? expected "")))
-	     (let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test #f test-id category variable)))
-			 (set! expected new-expected)
-			 (set! tol      new-tol)
-			 (set! units    new-units)))
-	 
-	 (debug:print 4 *default-log-port* "AFTER:  category: " category " variable: " variable " value: " value 
-		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	 ;; calculate status if NOT specified
-	 (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
-	     (if (number? tol) ;; if tol is a number then we do the standard comparison
-		 (let* ((max-val (+ expected tol))
-			(min-val (- expected tol))
-			(result  (and (>=  value min-val)(<= value max-val))))
-		   (debug:print 4 *default-log-port* "max-val: " max-val " min-val: " min-val " result: " result)
-		   (set! status (if result "pass" "fail")))
-		 (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
-		       (case (string->symbol tol) ;; tol should be >, <, >=, <=
-			 ((>)  (if (>  value expected) "pass" "fail"))
-			 ((<)  (if (<  value expected) "pass" "fail"))
-			 ((>=) (if (>= value expected) "pass" "fail"))
-			 ((<=) (if (<= value expected) "pass" "fail"))
-			 (else (conc "ERROR: bad tol comparator " tol))))))
-	 (debug:print 4 *default-log-port* "AFTER2: category: " category " variable: " variable " value: " value 
-		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	 ;; (db:delay-if-busy dbdat)
-	 (sqlite3:execute db "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
-			  test-id category variable value expected tol units (if comment comment "") status type)))
-     csvlist)))
+  (db:with-db
+   dbstruct #f #f
+   (lambda (db)
+     (let* ((csvlist (csv->list (make-csv-reader
+				 (open-input-string csvdata)
+				 '((strip-leading-whitespace? #t)
+				   (strip-trailing-whitespace? #t)))))) ;; (csv->list csvdata)))
+       (for-each
+	(lambda (csvrow)
+	  (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
+		 (category    (list-ref padded-row 0))
+		 (variable    (list-ref padded-row 1))
+		 (value       (any->number-if-possible (list-ref padded-row 2)))
+		 (expected    (any->number-if-possible (list-ref padded-row 3)))
+		 (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
+		 (units       (list-ref padded-row 5))
+		 (comment     (list-ref padded-row 6))
+		 (status      (let ((s (list-ref padded-row 7)))
+				(if (and (string? s)(or (string-match (regexp "^\\s*$") s)
+							(string-match (regexp "^n/a$") s)))
+				    #f
+				    s))) ;; if specified on the input then use, else calculate
+		 (type        (list-ref padded-row 8)))
+	    ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
+	    (debug:print 4 *default-log-port* "BEFORE: category: " category " variable: " variable " value: " value 
+			 ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
+	    
+	    (if (and (or (not expected)(equal? expected ""))
+		     (or (not tol)     (equal? expected ""))
+		     (or (not units)   (equal? expected "")))
+		(let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test #f test-id category variable)))
+		  (set! expected new-expected)
+		  (set! tol      new-tol)
+		  (set! units    new-units)))
+	    
+	    (debug:print 4 *default-log-port* "AFTER:  category: " category " variable: " variable " value: " value 
+			 ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	    ;; calculate status if NOT specified
+	    (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
+		(if (number? tol) ;; if tol is a number then we do the standard comparison
+		    (let* ((max-val (+ expected tol))
+			   (min-val (- expected tol))
+			   (result  (and (>=  value min-val)(<= value max-val))))
+		      (debug:print 4 *default-log-port* "max-val: " max-val " min-val: " min-val " result: " result)
+		      (set! status (if result "pass" "fail")))
+		    (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
+		      (case (string->symbol tol) ;; tol should be >, <, >=, <=
+			((>)  (if (>  value expected) "pass" "fail"))
+			((<)  (if (<  value expected) "pass" "fail"))
+			((>=) (if (>= value expected) "pass" "fail"))
+			((<=) (if (<= value expected) "pass" "fail"))
+			(else (conc "ERROR: bad tol comparator " tol))))))
+	    (debug:print 4 *default-log-port* "AFTER2: category: " category " variable: " variable " value: " value 
+			 ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	    ;; (db:delay-if-busy dbdat)
+	    (sqlite3:execute db "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
+			     test-id category variable value expected tol units (if comment comment "") status type)))
+	csvlist)))))
 
 ;; This routine moved from tdb.scm, tdb:read-test-data
 ;;
 (define (db:read-test-data dbstruct run-id test-id categorypatt)
-  (let* ((dbdat      (db:get-db dbstruct run-id))
-	 (db         (db:dbdat-get-db dbdat))
-	 (res '()))
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row 
-     (lambda (id test_id category variable value expected tol units comment status type)
-       (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
-     db
-     "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
-    (reverse res)))
+  (let* ((res '()))
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row 
+	(lambda (id test_id category variable value expected tol units comment status type)
+	  (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
+	db
+	"SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt)
+       (reverse res)))))
 
 ;;======================================================================
 ;; Misc. test related queries
 ;;======================================================================
 
 (define (db:get-run-ids-matching-target dbstruct keynames target res runname testpatt statepatt statuspatt)
-  (let* ((dbdat    (db:get-db dbstruct #f))
-	 (db       (db:dbdat-get-db dbdat))
-	 (row-ids '())
-	 (keystr (string-intersperse 
-		  (map (lambda (key val)
-			 (conc key " like '" val "'"))
-		       keynames 
-		       (string-split target "/"))
-		  " AND "))
-	 ;; (testqry (tests:match->sqlqry testpatt))
-	 (runsqry (sqlite3:prepare db (conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';"))))
-    ;; (debug:print 8 *default-log-port* "db:test-get-paths-matching-keynames-target-new\n  runsqry=" runsqry "\n  tstsqry=" testqry)
-    (sqlite3:for-each-row
-     (lambda (rid)
-       (set! row-ids (cons rid row-ids)))
-     runsqry)
-    (sqlite3:finalize! runsqry)
-    row-ids))
+  (db:with-db
+   dbstruct #f #f
+   (lambda (db)
+     (let* ((row-ids '())
+	    (keystr (string-intersperse 
+		     (map (lambda (key val)
+			    (conc key " like '" val "'"))
+			  keynames 
+			  (string-split target "/"))
+		     " AND "))
+	    ;; (testqry (tests:match->sqlqry testpatt))
+	    (runsqry (sqlite3:prepare db (conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';"))))
+       ;; (debug:print 8 *default-log-port* "db:test-get-paths-matching-keynames-target-new\n  runsqry=" runsqry "\n  tstsqry=" testqry)
+       (sqlite3:for-each-row
+	(lambda (rid)
+	  (set! row-ids (cons rid row-ids)))
+	runsqry)
+       (sqlite3:finalize! runsqry)
+       row-ids))))
 
 ;; finds latest matching all patts for given run-id
 ;;
 (define (db:test-get-paths-matching-keynames-target-new dbstruct run-id keynames target res testpatt statepatt statuspatt runname)
   (let* ((testqry (tests:match->sqlqry testpatt))
@@ -3138,140 +3164,109 @@
            (print-call-chain (current-error-port))
 	   msg))) ;; crude reply for when things go awry
     ((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize))))
     (else msg))) ;; rpc
 
-;; This is to be the big daddy call
-
-(define (db:test-set-status-state dbstruct run-id test-id status state msg)
-  (let ((dbdat  (db:get-db dbstruct run-id)))
-    (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
-	(db:general-call dbdat 'set-test-start-time (list test-id)))
-    ;; (if msg
-    ;; 	(db:general-call dbdat 'state-status-msg (list state status msg test-id))
-    ;; 	(db:general-call dbdat 'state-status     (list state status test-id)))
-    (db:set-state-status-and-roll-up-items dbstruct run-id test-id #f state status msg)
-    ;; process the test_data table
-    (if (and test-id state status (equal? status "AUTO")) 
-	(db:test-data-rollup dbstruct run-id test-id status))
-    (mt:process-triggers run-id test-id state status)))
+;; ; This is to be the big daddy call NOPE: Replaced by db:set-state-status-and-roll-up-items
+;; ;
+;; define (db:test-set-state-status dbstruct run-id test-id state status msg)
+;;  (let ((dbdat  (db:get-db dbstruct run-id)))
+;;    (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
+;; 	(db:general-call dbdat 'set-test-start-time (list test-id)))
+;;    ;; (if msg
+;;    ;; 	(db:general-call dbdat 'state-status-msg (list state status msg test-id))
+;;    ;; 	(db:general-call dbdat 'state-status     (list state status test-id)))
+;;    (db:set-state-status-and-roll-up-items dbstruct run-id test-id #f state status msg)
+;;    ;; process the test_data table
+;;    (if (and test-id state status (equal? status "AUTO")) 
+;; 	(db:test-data-rollup dbstruct run-id test-id status))
+;;    (mt:process-triggers dbstruct run-id test-id state status)))
 
 ;; state is the priority rollup of all states
 ;; status is the priority rollup of all completed statesfu
 ;;
 ;; if test-name is an integer work off that instead of test-name test-path
 ;;
 (define (db:set-state-status-and-roll-up-items dbstruct run-id test-name item-path state status comment)
   ;; establish info on incoming test followed by info on top level test
-  (let* ((db           (db:dbdat-get-db (dbr:dbstruct-tmpdb dbstruct)))
-	 (testdat      (if (number? test-name)
+  (let* ((testdat      (if (number? test-name)
 			   (db:get-test-info-by-id dbstruct run-id test-name) ;; test-name is actually a test-id
 			   (db:get-test-info       dbstruct run-id test-name item-path)))
 	 (test-id      (db:test-get-id testdat))
 	 (test-name    (if (number? test-name)
 			   (db:test-get-testname testdat)
 			   test-name))
 	 (item-path    (db:test-get-item-path testdat))
          (tl-testdat   (db:get-test-info dbstruct run-id test-name ""))
-         (tl-test-id   (db:test-get-id tl-testdat)))
-    (sqlite3:with-transaction
-     db
-     (lambda ()
-       (db:test-set-state-status-by-id dbstruct run-id test-id state status comment)
-       (if (not (equal? item-path "")) ;; only roll up IF incoming test is an item
-	   (let* ((state-status-counts  (db:get-all-state-status-counts-for-test db run-id test-name item-path)) ;; item-path is used to exclude current state/status of THIS test
-		  (running              (length (filter (lambda (x)
-							  (member (dbr:counts-state x) *common:running-states*))
-							state-status-counts)))
-		  (bad-not-started      (length (filter (lambda (x)
-							  (and (equal? (dbr:counts-state x) "NOT_STARTED")
-							       (not (member (dbr:counts-status x)
-									    *common:not-started-ok-statuses*))))
-							state-status-counts)))
-		  (all-curr-states   (common:special-sort  ;; worst -> best (sort of)
-                                      (delete-duplicates
-                                       (cons state (map dbr:counts-state state-status-counts)))
-                                      *common:std-states* >))
-                  (all-curr-statuses (common:special-sort  ;; worst -> best
-                                      (delete-duplicates
-				       (cons status (map dbr:counts-status state-status-counts)))
-				      *common:std-statuses* >))
-		  (newstate          (if (> running 0)
-					 "RUNNING"
-					 (if (> bad-not-started 0)
-					     "COMPLETED"
-					     (car all-curr-states))))
-		  (newstatus         (if (> bad-not-started 0)
-					 "CHECK"
-					 (car all-curr-statuses))))
-	     ;; (print "Setting toplevel to: " newstate "/" newstatus)
-	     (db:test-set-state-status-by-id dbstruct run-id tl-test-id newstate newstatus #f)))))))
-
-(define db:roll-up-pass-fail-counts db:set-state-status-and-roll-up-items)
-
-;; call with state = #f to roll up with out accounting for state/status of this item
-;;
-;;    (define (db:roll-up-pass-fail-counts dbstruct run-id test-name item-path state status)
-;;      (if (not (equal? item-path "")) ;; if asked to do this for a specific item then do an incremental update
-;;          (let* ((dbdat         (db:get-db dbstruct run-id))
-;;                 (toptestdat    (db:get-test-info dbstruct run-id test-name item-path))
-;;                 (currtopstate  (db:test-get-state toptestdat))
-;;                 (currtopstatus (db:test-get-status toptestdat))
-;;                 (nextss        (common:apply-state-status currtopstate currtopstatus state status))
-;;                 (newtopstate   (car nextss))  ;; #f or a symbol
-;;                 (newtopstatus  (cdr nextss))) ;; #f or a symbol
-;;            (if (not newtopstate) ;; need to calculate it
-;;                
-;;            ;; We rely on the toplevel to track status as state varies. I.e. preserve an ABORT
-;;            
-;;                 
-;;    	;;	(db    (db:dbdat-get-db dbdat)))
-;;    	(db:general-call dbdat 'update-pass-fail-counts (list test-name test-name test-name))
-;;    	(db:top-test-set-per-pf-counts dbstruct run-id test-name))))
-;;      
-;;    ;;     (case (string->symbol status)
-;;    ;;       ((RUNNING)  (db:general-call dbdat 'top-test-set-running (list test-name)))
-;;    ;;       ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name)))
-;;    ;;       ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name))))
-;;        
-;;    ;;     (if (or (not state)
-;;    ;; 	    (not (equal? item-path "")))
-;;    ;; 	;; just do a rollup
-;;    ;; 	(begin
-;;    ;; 	  (db:top-test-set-per-pf-counts dbdat run-id test-name)
-;;    ;; 	  #f)
-;;    ;; 	(begin
-;;    ;; 	  ;; NOTE: No else clause needed for this case
-;;    ;; 	  (case (string->symbol status)
-;;    ;; 	    ((RUNNING)  (db:general-call dbdat 'top-test-set-running (list test-name)))
-;;    ;; 	    ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name)))
-;;    ;; 	    ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name))))
-;;    ;; 	  #f)
-;;    ;; 	)))
-
-(define (db:get-all-state-status-counts-for-test db run-id test-name item-path)
-  (sqlite3:map-row
-   (lambda (state status count)
-     (make-dbr:counts state: state status: status count: count))
-   db
-   "SELECT state,status,count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND item_path !=? GROUP BY state,status;"
-   run-id test-name item-path))
-
-
-(define (db:get-all-item-states db run-id test-name)
-  (sqlite3:map-row 
-   (lambda (a) a)
-   db
-   "SELECT DISTINCT state FROM tests WHERE item_path != '' AND state != 'DELETED' AND run_id=? AND testname=?"
-   run-id test-name))
-
-(define (db:get-all-item-statuses db run-id test-name)
-  (sqlite3:map-row 
-   (lambda (a) a)
-   db
-   "SELECT DISTINCT status FROM tests WHERE item_path != '' AND state != 'DELETED' AND state='COMPLETED' AND run_id=? AND testname=?"
-   run-id test-name))
+         (tl-test-id   (db:test-get-id tl-testdat))
+	 (db           (db:dbdat-get-db  (db:get-db dbstruct #f))))
+    (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
+	(db:general-call dbstruct 'set-test-start-time (list test-id)))
+    (mutex-lock! *db-transaction-mutex*)
+    (let ((tr-res
+	   (sqlite3:with-transaction
+	    db
+	    (lambda ()
+	      ;; (db:test-set-state-status-by-id dbstruct run-id test-id state status comment)
+	      (db:test-set-state-status dbstruct run-id test-id state status comment)
+	      (if (not (equal? item-path "")) ;; only roll up IF incoming test is an item
+		  (let* ((state-status-counts  (db:get-all-state-status-counts-for-test dbstruct run-id test-name item-path)) ;; item-path is used to exclude current state/status of THIS test
+			 (running              (length (filter (lambda (x)
+								 (member (dbr:counts-state x) *common:running-states*))
+							       state-status-counts)))
+			 (bad-not-started      (length (filter (lambda (x)
+								 (and (equal? (dbr:counts-state x) "NOT_STARTED")
+								      (not (member (dbr:counts-status x)
+										   *common:not-started-ok-statuses*))))
+								  state-status-counts)))
+			 (all-curr-states   (common:special-sort  ;; worst -> best (sort of)
+					     (delete-duplicates
+					      (cons state (map dbr:counts-state state-status-counts)))
+					     *common:std-states* >))
+			 (all-curr-statuses (common:special-sort  ;; worst -> best
+					     (delete-duplicates
+					      (cons status (map dbr:counts-status state-status-counts)))
+					     *common:std-statuses* >))
+			 (newstate          (if (> running 0)
+						"RUNNING"
+						(if (> bad-not-started 0)
+						    "COMPLETED"
+						    (car all-curr-states))))
+			 (newstatus         (if (> bad-not-started 0)
+						"CHECK"
+						(car all-curr-statuses))))
+		    ;; (print "Setting toplevel to: " newstate "/" newstatus)
+		    (db:test-set-state-status dbstruct run-id tl-test-id newstate newstatus #f)))))))
+      (mutex-unlock! *db-transaction-mutex*)
+      (if (and test-id state status (equal? status "AUTO")) 
+	  (db:test-data-rollup dbstruct run-id test-id status))
+      tr-res)))
+
+(define (db:get-all-state-status-counts-for-test dbstruct run-id test-name item-path)
+  (db:with-db
+   dbstruct #f #f
+   (lambda (db)
+     (sqlite3:map-row
+      (lambda (state status count)
+	(make-dbr:counts state: state status: status count: count))
+      db
+      "SELECT state,status,count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND item_path !=? GROUP BY state,status;"
+      run-id test-name item-path))))
+
+;; (define (db:get-all-item-states db run-id test-name)
+;;   (sqlite3:map-row 
+;;    (lambda (a) a)
+;;    db
+;;    "SELECT DISTINCT state FROM tests WHERE item_path != '' AND state != 'DELETED' AND run_id=? AND testname=?"
+;;    run-id test-name))
+;; 
+;; (define (db:get-all-item-statuses db run-id test-name)
+;;   (sqlite3:map-row 
+;;    (lambda (a) a)
+;;    db
+;;    "SELECT DISTINCT status FROM tests WHERE item_path != '' AND state != 'DELETED' AND state='COMPLETED' AND run_id=? AND testname=?"
+;;    run-id test-name))
 
 (define (db:test-get-logfile-info dbstruct run-id test-name)
   (db:with-db
    dbstruct
    run-id
@@ -3329,11 +3324,11 @@
 	'(update-cpuload-diskfree "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;") ;; DONE
 	'(update-uname-host       "UPDATE tests SET uname=?,host=? WHERE id=?;")       ;; DONE
         '(update-test-rundat      "INSERT INTO test_rundat (test_id,update_time,cpuload,diskfree,diskusage,run_duration) VALUES (?,?,?,?,?,?);")
 	'(update-test-state       "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
 	'(update-test-status      "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
-	;; stuff for roll-up-pass-fail-counts
+	;; stuff for set-state-status-and-roll-up-items
 	'(update-pass-fail-counts "UPDATE tests 
              SET fail_count=(SELECT count(id) FROM tests WHERE testname=? AND item_path != '' AND status IN ('FAIL','CHECK','INCOMPLETE','ABORT')),
                  pass_count=(SELECT count(id) FROM tests WHERE testname=? AND item_path != '' AND status IN ('PASS','WARN','WAIVED'))
              WHERE testname=? AND item_path='' AND run_id=?;") ;; DONE  ;; BROKEN!!! NEEDS run-id
 	'(top-test-set-running  "UPDATE tests SET state='RUNNING' WHERE testname=? AND item_path='' AND run_id=?;") ;; DONE   ;; BROKEN!!! NEEDS run-id
@@ -3437,11 +3432,11 @@
   (let ((q (alist-ref qry-name db:queries)))
     (if q (car q) #f)))
 
 ;; do not run these as part of the transaction
 (define db:special-queries   '(rollup-tests-pass-fail
-			       ;; db:roll-up-pass-fail-counts  ;; WHY NOT!?
+			       ;; db:set-state-status-and-roll-up-items  ;; WHY NOT!?
 			       login
 			       immediate
 			       flush
 			       sync
 			       set-verbosity
@@ -3458,33 +3453,35 @@
     (list #f "Login failed due to mismatch megatest version: " calling-version ", " megatest-version))
    (else
     (hash-table-set! *logged-in-clients* client-signature (current-seconds))
     '(#t "successful login"))))
 
-(define (db:general-call dbdat stmtname params)
+(define (db:general-call dbstruct stmtname params)
   (let ((query (let ((q (alist-ref (if (string? stmtname)
 				       (string->symbol stmtname)
 				       stmtname)
 				   db:queries)))
  		 (if q (car q) #f))))
-    ;; (db:delay-if-busy dbdat)
-    (apply sqlite3:execute (db:dbdat-get-db dbdat) query params)
-    #t))
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (apply sqlite3:execute db query params)
+       #t))))
 
 ;; get a summary of state and status counts to calculate a rollup
 ;;
-;; NOTE: takes a db, not a dbstruct
-;;
-(define (db:get-state-status-summary db run-id testname)
+(define (db:get-state-status-summary dbstruct run-id testname)
   (let ((res   '()))
-    (sqlite3:for-each-row
-     (lambda (state status count)
-       (set! res (cons (vector state status count) res)))
-     db
-     "SELECT state,status,count(state) FROM tests WHERE run_id=? AND testname=? AND item_path='' GROUP BY state,status;"
-     run-id testname)
-    res))
+    (db:with-db
+     dbstruct #f #f
+     (sqlite3:for-each-row
+      (lambda (state status count)
+	(set! res (cons (vector state status count) res)))
+      db
+      "SELECT state,status,count(state) FROM tests WHERE run_id=? AND testname=? AND item_path='' GROUP BY state,status;"
+      run-id testname)
+     res)))
 
 
 (define (db:get-latest-host-load dbstruct raw-hostname)
   (let* ((hostname (string-substitute "\\..*$" "" raw-hostname))
         (res  (cons -1 0))
@@ -3501,13 +3498,11 @@
         "SELECT tr.cpuload, tr.update_time FROM test_rundat tr, tests t WHERE t.host=? AND tr.cpuload != -1  AND tr.test_id=t.id ORDER BY tr.update_time DESC LIMIT 1;"
         hostname))) res ))
 
 
 (define (db:set-top-level-from-items dbstruct run-id testname)
-  (let* ((dbdat (db:get-db dbstruct run-id))
-	 (db    (db:dbdat-get-db dbdat))
-	 (summ  (db:get-state-status-summary db run-id testname))
+  (let* ((summ  (db:get-state-status-summary dbstruct run-id testname))
 	 (find  (lambda (state status)
 		  (if (null? summ) 
 		      #f
 		      (let loop ((hed (car summ))
 				 (tal (cdr summ)))
@@ -3532,32 +3527,35 @@
 ;; can use wildcards. Also can likely be factored in with get test paths?
 ;;
 ;; Run this remotely!!
 ;;
 (define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path)
-  (let* ((dbdat   (db:get-db dbstruct #f))
-	 (db      (db:dbdat-get-db dbdat))
-	 (keys    (db:get-keys dbstruct))
+  (let* ((keys    (db:get-keys dbstruct))
 	 (selstr  (string-intersperse keys ","))
 	 (qrystr  (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND "))
 	 (keyvals #f)
 	 (tests-hash (make-hash-table)))
     ;; first look up the key values from the run selected by run-id
-    ;; (db:delay-if-busy dbdat)
-    (sqlite3:for-each-row 
-     (lambda (a . b)
-       (set! keyvals (cons a b)))
-     db
-     (conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id)
+    (db:with-db
+     dbstruct #f #f
+     (lambda (db)
+       (sqlite3:for-each-row 
+	(lambda (a . b)
+	  (set! keyvals (cons a b)))
+	db
+	(conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id)))
     (if (not keyvals)
 	'()
 	(let ((prev-run-ids '()))
-	  (apply sqlite3:for-each-row
-		 (lambda (id)
-		   (set! prev-run-ids (cons id prev-run-ids)))
-		 db
-		 (conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append keyvals (list run-id)))
+	  (db:with-db
+	   dbstruct #f #f
+	   (lambda (db)
+	     (apply sqlite3:for-each-row
+		    (lambda (id)
+		      (set! prev-run-ids (cons id prev-run-ids)))
+		    db
+		    (conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append keyvals (list run-id)))))
 	  ;; collect all matching tests for the runs then
 	  ;; extract the most recent test and return that.
 	  (debug:print 4 *default-log-port* "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals 
 		       ", previous run ids found: " prev-run-ids)
 	  (if (null? prev-run-ids) '()  ;; no previous runs? return null
@@ -3643,25 +3641,26 @@
 ;;======================================================================
 
 ;; returns a hash table of tags to tests
 ;;
 (define (db:get-tests-tags dbstruct)
-  (let* ((dbdat   (db:get-db dbstruct #f))
-	 (db      (db:dbdat-get-db dbdat))
-         (res     (make-hash-table)))
-    (sqlite3:for-each-row
-     (lambda (testname tags-in)
-       (let ((tags (string-split tags-in ",")))
-         (for-each
-          (lambda (tag)
-            (hash-table-set! res tag
-                             (delete-duplicates
-                              (cons testname (hash-table-ref/default res tag '())))))
-          tags)))
-     db
-     "SELECT testname,tags FROM test_meta")
-    res))
+  (db:with-db
+   dbstruct #f #f
+   (lambda (db)
+     (let* ((res     (make-hash-table)))
+       (sqlite3:for-each-row
+	(lambda (testname tags-in)
+	  (let ((tags (string-split tags-in ",")))
+	    (for-each
+	     (lambda (tag)
+	       (hash-table-set! res tag
+				(delete-duplicates
+				 (cons testname (hash-table-ref/default res tag '())))))
+	     tags)))
+	db
+	"SELECT testname,tags FROM test_meta")
+       res))))
 
 ;; read the record given a testname
 (define (db:testmeta-get-record dbstruct testname)
   (let ((res   #f))
     (db:with-db

Index: ezsteps.scm
==================================================================
--- ezsteps.scm
+++ ezsteps.scm
@@ -164,11 +164,11 @@
 					  new-state
 					  new-status
 					  (args:get-arg "-m") #f)
 		  ;; need to update the top test record if PASS or FAIL and this is a subtest
 		  (if (not (equal? item-path ""))
-		      (cdb:roll-up-pass-fail-counts *runremote* run-id test-name item-path new-status))))
+		      (cdb:set-state-status-and-roll-up-items *runremote* run-id test-name item-path new-status))))
 	    ;; for automated creation of the rollup html file this is a good place...
 	    (if (not (equal? item-path ""))
 		(tests:summarize-items #f run-id test-id test-name #f)) ;; don't force - just update if no
 	    )))
     (pop-directory)

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -505,25 +505,28 @@
     ;;
     (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "shutting-down")
     (set! *time-to-exit* #t) ;; tell on-exit to be fast as we've already cleaned up
     (portlogger:open-run-close portlogger:set-port port "released")
     (thread-sleep! 5)
-    (debug:print-info 0 *default-log-port* "Max cached queries was    " *max-cache-size*)
-    (debug:print-info 0 *default-log-port* "Number of cached writes   " *number-of-writes*)
-    (debug:print-info 0 *default-log-port* "Average cached write time "
-		      (if (eq? *number-of-writes* 0)
-			  "n/a (no writes)"
-			  (/ *writes-total-delay*
-			     *number-of-writes*))
-		      " ms")
-    (debug:print-info 0 *default-log-port* "Number non-cached queries "  *number-non-write-queries*)
-    (debug:print-info 0 *default-log-port* "Average non-cached time   "
-		      (if (eq? *number-non-write-queries* 0)
-			  "n/a (no queries)"
-			  (/ *total-non-write-delay* 
-			     *number-non-write-queries*))
-		      " ms")
+;; (debug:print-info 0 *default-log-port* "Max cached queries was    " *max-cache-size*)
+;; (debug:print-info 0 *default-log-port* "Number of cached writes   " *number-of-writes*)
+;; (debug:print-info 0 *default-log-port* "Average cached write time "
+;; 		      (if (eq? *number-of-writes* 0)
+;; 			  "n/a (no writes)"
+;; 			  (/ *writes-total-delay*
+;; 			     *number-of-writes*))
+;; 		      " ms")
+;; (debug:print-info 0 *default-log-port* "Number non-cached queries "  *number-non-write-queries*)
+;; (debug:print-info 0 *default-log-port* "Average non-cached time   "
+;; 		      (if (eq? *number-non-write-queries* 0)
+;; 			  "n/a (no queries)"
+;; 			  (/ *total-non-write-delay* 
+;; 			     *number-non-write-queries*))
+    ;; 		      " ms")
+
+    (db:print-current-query-stats)
+    
     (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting")
     (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running complete")
     ;; if the .server file contained :myport then we can remove it
     (server:remove-dotserver-file *toppath* port)
     ;;(BB> "http-transport:server-shutdown -> exit")

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -64,18 +64,19 @@
 (define (launch:load-logpro-dat run-id test-id stepname)
   (let ((cname (conc stepname ".dat")))
     (if (file-exists? cname)
 	(let* ((dat  (read-config cname #f #f))
 	       (csvr (db:logpro-dat->csv dat stepname))
-	       (csvt (let-values (( (fmt-cell fmt-record fmt-csv) (make-format ",")))
-				 (fmt-csv (map list->csv-record csvr))))
+	       (csvt (let-values (((fmt-cell fmt-record fmt-csv) (make-format ",")))
+		       (fmt-csv (map list->csv-record csvr))))
 	       (status (configf:lookup dat "final" "exit-status"))
 	       (msg     (configf:lookup dat "final" "message")))
-          ;;(if csvt  ;; this if blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
+          (if csvt  ;; this if blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
               (rmt:csv->test-data run-id test-id csvt)
-            ;;  (BB> "Error: run-id/test-id/stepname="run-id"/"test-id"/"stepname" => bad csvr="csvr)
-            ;;  )
+	      (debug:print 0 *default-log-port* "ERROR: no csvdat exists for run-id: " run-id " test-id: " test-id " stepname: " stepname ", check that logpro version is 1.15 or newer"))
+	  ;;  (BB> "Error: run-id/test-id/stepname="run-id"/"test-id"/"stepname" => bad csvr="csvr)
+	  ;;  )
 	  (cond
 	   ((equal? status "PASS") "PASS") ;; skip the message part if status is pass
 	   (status (conc (configf:lookup dat "final" "exit-status") ": " (if msg msg "no message")))
 	   (else #f)))
 	#f)))
@@ -253,11 +254,11 @@
   ;; any of the other stuff that tests:test-set-status! does. Let's just 
   ;; force RUNNING/n/a
 
   ;; (thread-sleep! 0.3)
   ;; (tests:test-force-state-status! run-id test-id "RUNNING" "n/a")
-  (rmt:roll-up-pass-fail-counts run-id test-name item-path #f "RUNNING" #f) 
+  (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "RUNNING" #f) 
   ;; (thread-sleep! 0.3) ;; NFS slowness has caused grief here
 
   ;; if there is a runscript do it first
   (if fullrunscript
       (let ((pid (process-run fullrunscript)))
@@ -457,11 +458,11 @@
 			   (if (eq? signum signal/stop)
 			       (debug:print-error 0 *default-log-port* "attempt to STOP process. Exiting."))
 			   (set! *time-to-exit* #t)
 			   (print "Received signal " signum ", cleaning up before exit. Please wait...")
 			   (let ((th1 (make-thread (lambda ()
-						     (tests:test-force-state-status! run-id test-id "INCOMPLETE" "KILLED")
+						     (rmt:test-set-state-status run-id test-id "INCOMPLETE" "KILLED" #f)
 						     (print "Killed by signal " signum ". Exiting")
 						     (thread-sleep! 1)
 						     (exit 1))))
 				 (th2 (make-thread (lambda ()
 						     (thread-sleep! 2)
@@ -481,17 +482,23 @@
 		 (test-host (db:test-get-host        test-info))
 		 (test-pid  (db:test-get-process_id  test-info)))
 	    (cond
 	     ((member (db:test-get-state test-info) '("INCOMPLETE" "KILLED" "UNKNOWN" "KILLREQ" "STUCK")) ;; prior run of this test didn't complete, go ahead and try to rerun
 	      (debug:print 0 *default-log-port* "INFO: test is INCOMPLETE or KILLED, treat this execute call as a rerun request")
-	      (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a")) ;; prime it for running
+	      ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a")
+	      (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f)
+	      ) ;; prime it for running
 	     ((member (db:test-get-state test-info) '("RUNNING" "REMOTEHOSTSTART"))
 	      (if (process:alive-on-host? test-host test-pid)
 		  (debug:print-error 0 *default-log-port* "test state is "  (db:test-get-state test-info) " and process " test-pid " is still running on host " test-host ", cannot proceed")
-		  (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a")))
+		  ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a")
+		  (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f)
+		  ))
 	     ((not (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ")))
-	      (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a"))
+	      ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a")
+	      (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f)
+	      )
 	     (else ;; (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ"))
 	      (debug:print-error 0 *default-log-port* "test state is " (db:test-get-state test-info) ", cannot proceed")
 	      (exit))))
 	  
 	  (debug:print 2 *default-log-port* "Exectuing " test-name " (id: " test-id ") on " (get-host-name))
@@ -651,11 +658,11 @@
 					    test-id 
 					    new-state
 					    new-status
 					    (args:get-arg "-m") #f)
 		    ;; need to update the top test record if PASS or FAIL and this is a subtest
-		    ;; NO NEED TO CALL roll-up-pass-fail-counts HERE, THIS IS DONE IN roll-up-pass-fail-counts called by tests:test-set-status!
+		    ;; NO NEED TO CALL set-state-status-and-roll-up-items HERE, THIS IS DONE IN set-state-status-and-roll-up-items called by tests:test-set-status!
 		    ))
 	      ;; for automated creation of the rollup html file this is a good place...
 	      (if (not (equal? item-path ""))
 		  (tests:summarize-items run-id test-id test-name #f))
 	      (tests:summarize-test run-id test-id)  ;; don't force - just update if no
@@ -1121,12 +1128,12 @@
 				      ((dboard)    "../megatest")
 				      ((mtest)     "../megatest")
 				      ((dashboard) "megatest")
 				      (else exe)))))
 	   (launcher        (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools"     "launcher"))
-	   (test-sig   (conc (common:get-testsuite-name) ":" test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
-	   (work-area  #f)
+	   (test-sig        (conc (common:get-testsuite-name) ":" test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
+	   (work-area       #f)
 	   (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
 	   (diskpath   #f)
 	   (cmdparms   #f)
 	   (fullcmd    #f) ;; (define a (with-output-to-string (lambda ()(write x))))
 	   (mt-bindir-path #f)
@@ -1148,11 +1155,11 @@
       
       ;; prevent overlapping actions - set to LAUNCHED as early as possible
       ;;
       ;; the following call handles waiver propogation. cannot yet condense into roll-up-pass-fail
       (tests:test-set-status! run-id test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED"))
-      (rmt:roll-up-pass-fail-counts run-id test-name item-path #f "LAUNCHED" #f)
+      (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "LAUNCHED" #f)
       ;; (pp (hash-table->alist tconfig))
       (set! diskpath (get-best-disk *configdat* tconfig))
       (if diskpath
 	  (let ((dat  (create-work-area run-id run-info keyvals test-id test-path diskpath test-name itemdat)))
 	    (set! work-area (car dat))
@@ -1227,11 +1234,11 @@
 					process-run)
 				    (if useshell
 					(let ((cmdstr (string-intersperse fullcmd " ")))
 					  (if launchwait
 					      cmdstr
-					      (conc cmdstr " >> mt_launch.log 2>&1")))
+					      (conc cmdstr " >> mt_launch.log 2>&1 &")))
 					(car fullcmd))
 				    (if useshell
 					'()
 					(cdr fullcmd)))))
         (mutex-unlock! *launch-setup-mutex*) ;; yes, really should mutex all the way to here. Need to put this entire process into a fork.

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -128,12 +128,12 @@
 
 ;;======================================================================
 ;;  T R I G G E R S
 ;;======================================================================
 
-(define (mt:process-triggers run-id test-id newstate newstatus)
-  (let* ((test-dat      (rmt:get-test-info-by-id run-id test-id)))
+(define (mt:process-triggers dbstruct run-id test-id newstate newstatus)
+  (let* ((test-dat      (db:get-test-info-by-id dbstruct run-id test-id)))
     (if test-dat
 	(let* ((test-rundir   ;; (rmt:sdb-qry 'getstr ;; (filedb:get-path *fdb*
 		(db:test-get-rundir test-dat)) ;; ) ;; )
 	       (test-name     (db:test-get-testname test-dat))
 	       (tconfig       #f)
@@ -186,18 +186,18 @@
 	;;  (rmt:general-call 'state-status run-id newstate newstatus test-id))
 	;; (else
 	;;  (if newstate   (rmt:general-call 'set-test-state   run-id newstate   test-id))
 	;;  (if newstatus  (rmt:general-call 'set-test-status  run-id newstatus  test-id))
 	;;  (if newcomment (rmt:general-call 'set-test-comment run-id newcomment test-id))))
-	(rmt:roll-up-pass-fail-counts run-id test-id #f newstate newstatus newcomment)
-	(mt:process-triggers run-id test-id newstate newstatus)
+	(rmt:set-state-status-and-roll-up-items run-id test-id #f newstate newstatus newcomment)
+	;; (mt:process-triggers run-id test-id newstate newstatus)
 	#t)))
 
 (define (mt:test-set-state-status-by-testname run-id test-name item-path new-state new-status new-comment)
   (let ((test-id (rmt:get-test-id run-id test-name item-path)))
-    (rmt:roll-up-pass-fail-counts run-id test-name item-path new-state new-status new-comment)
-    (mt:process-triggers run-id test-id new-state new-status)
+    (rmt:set-state-status-and-roll-up-items run-id test-name item-path new-state new-status new-comment)
+    ;; (mt:process-triggers run-id test-id new-state new-status)
     #t))
 	;;(mt:test-set-state-status-by-id run-id test-id new-state new-status new-comment)))
 
 (define (mt:lazy-read-test-config test-name)
   (let ((tconf (hash-table-ref/default *testconfigs* test-name #f)))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -75,11 +75,11 @@
       (set! *runremote* (make-remote))
       (mutex-unlock! *rmt-mutex*)
       (debug:print-info 12 *default-log-port* "rmt:send-receive, case  1")
       (rmt:send-receive cmd rid params attemptnum: attemptnum))
      ;; ensure we have a homehost record
-     ((not (pair? (remote-hh-dat *runremote*)))  ;; have a homehost record?
+     ((not (pair? (remote-hh-dat *runremote*)))  ;; not on homehost
       (thread-sleep! 0.1) ;; since we shouldn't get here, delay a little
       (remote-hh-dat-set! *runremote* (common:get-homehost))
       (mutex-unlock! *rmt-mutex*)
       (debug:print-info 12 *default-log-port* "rmt:send-receive, case  2")
       (rmt:send-receive cmd rid params attemptnum: attemptnum))
@@ -141,18 +141,20 @@
             ;; (rmt:send-receive cmd rid params attemptnum: attemptnum))   ;)  ;)
 ;;;;
      
      ;; if not on homehost ensure we have a connection to a live server
      ;; NOTE: we *have* a homehost record by now
-     ((and (not (cdr (remote-hh-dat *runremote*)))        ;; not on a homehost 
-           (not (remote-conndat *runremote*))             ;; and no connection
-           (server:read-dotserver *toppath*))             ;; .server file exists
-      ;; something caused the server entry in tdb to disappear, but the server is still running
-      (server:remove-dotserver-file *toppath* ".*")
-      (mutex-unlock! *rmt-mutex*)
-      (debug:print-info 12 *default-log-port* "rmt:send-receive, case  20")
-      (rmt:send-receive cmd rid params attemptnum: (add1 attemptnum)))
+
+     ;; ((and (not (cdr (remote-hh-dat *runremote*)))        ;; not on a homehost 
+     ;;       (not (remote-conndat *runremote*))             ;; and no connection
+     ;;       (server:read-dotserver *toppath*))             ;; .server file exists
+     ;;  ;; something caused the server entry in tdb to disappear, but the server is still running
+     ;;  (server:remove-dotserver-file *toppath* ".*")
+     ;;  (mutex-unlock! *rmt-mutex*)
+     ;;  (debug:print-info 12 *default-log-port* "rmt:send-receive, case  20")
+     ;;  (rmt:send-receive cmd rid params attemptnum: (add1 attemptnum)))
+
      ((and (not (cdr (remote-hh-dat *runremote*)))        ;; not on a homehost 
            (not (remote-conndat *runremote*)))            ;; and no connection
       (debug:print-info 12 *default-log-port* "rmt:send-receive, case  6  hh-dat: " (remote-hh-dat *runremote*) " conndat: " (remote-conndat *runremote*))
       (mutex-unlock! *rmt-mutex*)
       (tasks:start-and-wait-for-server (tasks:open-db) 0 15)
@@ -166,10 +168,11 @@
 
      ;; not on homehost, do server query
      (else
       (mutex-unlock! *rmt-mutex*)
       (debug:print-info 12 *default-log-port* "rmt:send-receive, case  9")
+      (mutex-lock! *rmt-mutex*)
       (let* ((conninfo (remote-conndat *runremote*))
 	     (dat      (case (remote-transport *runremote*)
 			 ((http) (condition-case ;; handling here has caused a lot of problems. However it is needed to deal with attemtped communication to servers that have gone away
                                   (http-transport:client-api-send-receive 0 conninfo cmd params)
                                   ((commfail)(vector #f "communications fail"))
@@ -178,22 +181,27 @@
 			  (debug:print 0 *default-log-port* "ERROR: transport " (remote-transport *runremote*) " not supported")
 			  (exit))))
 	     (success  (if (vector? dat) (vector-ref dat 0) #f))
 	     (res      (if (vector? dat) (vector-ref dat 1) #f)))
 	(if (vector? conninfo)(http-transport:server-dat-update-last-access conninfo)) ;; refresh access time
-        (debug:print-info 12 *default-log-port* "rmt:send-receive, case  9. conninfo=" conninfo " dat=" dat)
+	;; (mutex-unlock! *rmt-mutex*)
+        (debug:print-info 12 *default-log-port* "rmt:send-receive, case  9. conninfo=" conninfo " dat=" dat " *runremote* = "*runremote*)
 	(if success
 	    (case (remote-transport *runremote*)
-	      ((http) res)
+	      ((http)
+	       (mutex-unlock! *rmt-mutex*)
+	       res)
 	      (else
 	       (debug:print 0 *default-log-port* "ERROR: transport " (remote-transport *runremote*) " is unknown")
+	       (mutex-unlock! *rmt-mutex*)
 	       (exit 1)))
 	    (begin
 	      (debug:print 0 *default-log-port* "WARNING: communication failed. Trying again, try num: " attemptnum)
 	      (remote-conndat-set!    *runremote* #f)
 	      (remote-server-url-set! *runremote* #f)
               (debug:print-info 12 *default-log-port* "rmt:send-receive, case  9.1")
+	      (mutex-unlock! *rmt-mutex*)
 	      (tasks:start-and-wait-for-server (tasks:open-db) 0 15)
 	      (rmt:send-receive cmd rid params attemptnum: (+ attemptnum 1)))))))))
 
 (define (rmt:update-db-stats run-id rawcmd params duration)
   (mutex-lock! *db-stats-mutex*)
@@ -499,12 +507,12 @@
 ;; This is not needed as test steps are deleted on test delete call
 ;;
 ;; (define (rmt:delete-test-step-records run-id test-id)
 ;;   (rmt:send-receive 'delete-test-step-records run-id (list run-id test-id)))
 
-(define (rmt:test-set-status-state run-id test-id status state msg)
-  (rmt:send-receive 'test-set-status-state run-id (list run-id test-id status state msg)))
+(define (rmt:test-set-state-status run-id test-id state status msg)
+  (rmt:send-receive 'test-set-state-status run-id (list run-id test-id state status msg)))
 
 (define (rmt:test-toplevel-num-items run-id test-name)
   (rmt:send-receive 'test-toplevel-num-items run-id (list run-id test-name)))
 
 ;; (define (rmt:get-previous-test-run-record run-id test-name item-path)
@@ -563,12 +571,12 @@
 (define (rmt:get-count-tests-running-in-jobgroup run-id jobgroup)
   (rmt:send-receive 'get-count-tests-running-in-jobgroup run-id (list run-id jobgroup)))
 
 ;; state and status are extra hints not usually used in the calculation
 ;;
-(define (rmt:roll-up-pass-fail-counts run-id test-name item-path state status comment)
-  (rmt:send-receive 'roll-up-pass-fail-counts run-id (list run-id test-name item-path state status comment)))
+(define (rmt:set-state-status-and-roll-up-items run-id test-name item-path state status comment)
+  (rmt:send-receive 'set-state-status-and-roll-up-items run-id (list run-id test-name item-path state status comment)))
 
 (define (rmt:update-pass-fail-counts run-id test-name)
   (rmt:general-call 'update-pass-fail-counts run-id test-name test-name test-name))
 
 (define (rmt:top-test-set-per-pf-counts run-id test-name)

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -928,11 +928,11 @@
 			  (debug:print 0 *default-log-port* "WARNING: test " hed " has FAILED prerequitests and we've tried at least 10 times to run it. Giving up now."))
 		      ;; (debug:print 0 *default-log-port* "         prereqs: " prereqs-not-met)
 		      (hash-table-set! test-registry hed 'removed)
 		      (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "TEN_STRIKES" #f)
 		      ;; I'm unclear on if this roll up is needed - it may be the root cause of the "all set to FAIL" bug.
-		      (rmt:roll-up-pass-fail-counts run-id test-name item-path #f "FAIL" #f) ;; treat as FAIL
+		      (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "FAIL" #f) ;; treat as FAIL
 		      (list (if (null? tal)(car newtal)(car tal))
 			    tal
 			    reg
 			    reruns)))))
 	      ;; can't drop this - maybe running? Just keep trying
@@ -1673,10 +1673,11 @@
 		   ((remove-runs)
 		    ;; (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
 		    ;; seek and kill in flight -runtests with % as testpatt here
 		    ;; (if (equal? testpatt "%")
 		    (tasks:kill-runner target run-name testpatt)
+		    
 		    ;; (debug:print 0 *default-log-port* "not attempting to kill any run launcher processes as testpatt is " testpatt))
 		    (debug:print 1 *default-log-port* "Removing tests for run: " runkey " " (db:get-value-by-header run header "runname")))
 		   ((set-state-status)
 		    (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
 		    (debug:print 1 *default-log-port* "Modifying state and staus for tests for run: " runkey " " (db:get-value-by-header run header "runname")))

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -64,11 +64,16 @@
   
   (case transport-type
     ((http)(http-transport:launch run-id))
     ;;((nmsg)(nmsg-transport:launch run-id))
     ((rpc)  (rpc-transport:launch run-id))
-    (else (debug:print-error 0 *default-log-port* "unknown server type " transport-type))))
+    (else (debug:print-error 0 *default-log-port* "unknown server type " transport-type)))
+
+  ;; is this a good place to print server exit stats?
+  (debug:print 0 "SERVER: max parallel api requests: " *max-api-process-requests*)
+  
+  )
 ;;       (else   (debug:print-error 0 *default-log-port* "No known transport set, transport=" transport ", using rpc")
 ;; 	      (rpc-transport:launch run-id)))))
 
 ;;======================================================================
 ;; S E R V E R   U T I L I T I E S 

Index: tasks.scm
==================================================================
--- tasks.scm
+++ tasks.scm
@@ -448,14 +448,16 @@
     (reverse res)))
 
 ;; no elegance here ...
 ;;
 (define (tasks:kill-server hostname pid #!key (kill-switch ""))
+  (server:remove-dotserver-file *toppath* ".*")
   (debug:print-info 0 *default-log-port* "Attempting to kill server process " pid " on host " hostname)
   (setenv "TARGETHOST" hostname)
   (setenv "TARGETHOST_LOGF" "server-kills.log")
   (system (conc "nbfake kill "kill-switch" "pid))
+
   (unsetenv "TARGETHOST_LOGF")
   (unsetenv "TARGETHOST"))
  
 ;; look up a server by run-id and send it a kill, also delete the record for that server
 ;;
@@ -466,10 +468,11 @@
 	(let ((hostname (vector-ref sdat 6))
 	      (pid      (vector-ref sdat 5))
 	      (server-id (vector-ref sdat 0)))
 	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "killed")
 	  (debug:print-info 0 *default-log-port* "Killing server " server-id " for run-id " run-id " on host " hostname " with pid " pid)
+          (server:remove-dotserver-file *toppath* ".*")
 	  (tasks:kill-server hostname pid)
 	  (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id tag) )
 	(debug:print-info 0 *default-log-port* "No server found for run-id " run-id ", nothing to kill"))
     ;; (sqlite3:finalize! tdb)
     ))

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -351,15 +351,10 @@
 					(loop (car tal)(cdr tal)))
 				    #f))))))
 	    (pop-directory)
 	    result)))))
 
-(define (tests:test-force-state-status! run-id test-id state status)
-  (rmt:test-set-status-state run-id test-id status state #f)
-  ;; (rmt:roll-up-pass-fail-counts run-id test-name item
-  (mt:process-triggers run-id test-id state status))
-
 ;; Do not rpc this one, do the underlying calls!!!
 (define (tests:test-set-status! run-id test-id state status comment dat #!key (work-area #f))
   (let* ((real-status status)
 	 (otherdat    (if dat dat (make-hash-table)))
 	 (testdat     (rmt:get-test-info-by-id run-id test-id))
@@ -396,12 +391,12 @@
     (debug:print 4 *default-log-port* "real-status " real-status ", waived " waived ", status " status)
 
     ;; update the primary record IF state AND status are defined
     (if (and state status)
 	(begin
-	  (rmt:test-set-status-state run-id test-id real-status state (if waived waived comment))
-	  ;; (mt:process-triggers run-id test-id state real-status) ;; triggers are called in test-set-status-state
+	  (rmt:set-state-status-and-roll-up-items run-id test-id item-path state real-status (if waived waived comment))
+	  ;; (mt:process-triggers run-id test-id state real-status) ;; triggers are called in test-set-state-status
 	  ))
     
     ;; if status is "AUTO" then call rollup (note, this one modifies data in test
     ;; run area, it does remote calls under the hood.
     ;; (if (and test-id state status (equal? status "AUTO")) 
@@ -442,12 +437,12 @@
 	    ;; This was run remote, don't think that makes sense. Perhaps not, but that is the easiest path for the moment.
 	    (rmt:csv->test-data run-id test-id
 				dat))))
       
     ;; need to update the top test record if PASS or FAIL and this is a subtest
-    (if (not (equal? item-path ""))
-	(rmt:roll-up-pass-fail-counts run-id test-name item-path state status #f))
+    ;;;;;; (if (not (equal? item-path ""))
+    ;;;;;;     (rmt:set-state-status-and-roll-up-items run-id test-name item-path state status #f) ;;;;;)
 
     (if (or (and (string? comment)
 		 (string-match (regexp "\\S+") comment))
 	    waived)
 	(let ((cmt  (if waived waived comment)))
@@ -481,12 +476,11 @@
 	      (lockf         (conc outputfilename ".lock")))
 	  (let loop ((have-lock  (common:simple-file-lock lockf)))
 	    (if have-lock
 		(let ((script (configf:lookup *configdat* "testrollup" test-name)))
 		  (print "Obtained lock for " outputfilename)
-		  (rmt:roll-up-pass-fail-counts run-id test-name "" #f #f #f)
-		  ;; (rmt:test-set-status-state run-id test-name #f #f #f) ;; (rmt:top-test-set-per-pf-counts run-id test-name)
+		  (rmt:set-state-status-and-roll-up-items run-id test-name "" #f #f #f)
 		  (if script
 		      (system (conc script " > " outputfilename " & "))
 		      (tests:generate-html-summary-for-iterated-test run-id test-id test-name outputfilename))
 		  (common:simple-file-release-lock lockf)
 		  (change-directory orig-dir)

Index: tests/fdktestqa/testqa/Makefile
==================================================================
--- tests/fdktestqa/testqa/Makefile
+++ tests/fdktestqa/testqa/Makefile
@@ -5,16 +5,16 @@
 NEWDASHBOARD = $(BINDIR)/newdashboard
 RUNNAME   = a
 NUMTESTS  = 20
 
 all :
-	$(MEGATEST) -remove-runs -target a/b :runname c -testpatt %/%
-	$(MEGATEST) -runtests % -target a/b :runname c
+	$(MEGATEST) -remove-runs -target a/b -runname c -testpatt %/%
+	$(MEGATEST) -run -testpatt % -target a/b -runname c
 
 bigbig :
 	for tn in a b c d;do \
-	   ($(MEGATEST) -runtests % -target a/b :runname $tn & ) ; \
+	   ($(MEGATEST) -run -testpatt % -target a/b -runname $tn & ) ; \
 	done
 
 waitonpatt :
 	megatest -remove-runs -runname waitonpatt -target a/b -testpatt %
 	NUMTESTS=15 megatest -run -target a/b -runname waitonpatt -testpatt bigrun3/%8
@@ -22,17 +22,17 @@
 waitonall :
 	megatest -remove-runs -runname waitonall -target a/b -testpatt %
 	NUMTESTS=20 megatest -run -target a/b -runname waitonall -testpatt alltop
 
 bigrun :
-	NUMTESTS=$(NUMTESTS) $(MEGATEST) -runtests bigrun -target a/bigrun :runname a$(shell date +%V)
+	NUMTESTS=$(NUMTESTS) $(MEGATEST) -run -testpatt bigrun -target a/bigrun -runname a$(shell date +%V)
 
 bigrun2 :
-	NUMTESTS=$(NUMTESTS) $(MEGATEST) -runtests bigrun2 -target a/bigrun2 :runname a$(shell date +%V)
+	NUMTESTS=$(NUMTESTS) $(MEGATEST) -run -testpatt bigrun2 -target a/bigrun2 -runname a$(shell date +%V)
 
 bigrun3 :
-	NUMTESTS=$(NUMTESTS) $(MEGATEST) -runtests bigrun3 -target a/bigrun3 :runname $(RUNNAME)
+	NUMTESTS=$(NUMTESTS) $(MEGATEST) -run -testpatt bigrun3 -target a/bigrun3 -runname $(RUNNAME)
 
 dashboard : 
 	mkdir -p ../simpleruns
 	$(DASHBOARD) -rows 20 &
 
@@ -41,6 +41,6 @@
 
 compile :
 	(cd ../../..;make -j && make install)
 
 clean :
-	rm -rf ../simple*/*/* megatest.db db/* ../simple*/.db/* logs/* monitor.db
+	rm -rf ../simple*/*/* megatest.db db/* ../simple*/.db/* logs/* monitor.db /tmp/$(USER)/megatest_localdb/testqa .server

ADDED   tests/fdktestqa/testqa/local.config.example
Index: tests/fdktestqa/testqa/local.config.example
==================================================================
--- /dev/null
+++ tests/fdktestqa/testqa/local.config.example
@@ -0,0 +1,15 @@
+[host-types]
+general #MTLOWESTLOAD xena zeus
+
+[jobtools]
+launcher nbfake
+maxload 1.5
+flexi-launcher yes
+# useshell no
+
+[setup]
+launch-delay 1
+launchwait no
+
+[launchers]
+% general

Index: tests/fdktestqa/testqa/megatest.config
==================================================================
--- tests/fdktestqa/testqa/megatest.config
+++ tests/fdktestqa/testqa/megatest.config
@@ -1,11 +1,13 @@
 [setup]
 testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log
 # launchwait no
+launch-delay 0
 
 # All these are overridden in ../fdk.config
 # [jobtools]
 # launcher nbfake
 # launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log 
 
 [include ../fdk.config]
 
+[include local.config]

Index: utils/nbfake
==================================================================
--- utils/nbfake
+++ utils/nbfake
@@ -70,7 +70,7 @@
 if [[ -z "$MY_NBFAKE_HOST" ]]; then
   # Run locally
   sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &"
 else
   # run remotely
-  ssh -n -f $MY_NBFAKE_HOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &\""
+  ssh -X -n -f $MY_NBFAKE_HOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &\""
 fi

Index: utils/plot-code.scm
==================================================================
--- utils/plot-code.scm
+++ utils/plot-code.scm
@@ -8,17 +8,19 @@
 ;; third param is list of files to scan
 
 (use regex srfi-69 srfi-13)
 
 (define targs #f) 
-(define files (cddddr (argv)))
+(define files (cdr (cddddr (argv))))
 
 (let ((targdat (cadddr (argv))))
   (if (equal? targdat "-")
       (set! targs files)
       (set! targs (string-split targdat ","))))
 
+(define function-patt (car (cdr (cdddr (argv)))))
+(define function-rx   (regexp function-patt))
 (define filedat-defns (make-hash-table))
 (define filedat-usages (make-hash-table))
 
 (define defn-rx (regexp "^\\s*\\(define\\s+\\(([^\\s\\)]+).*"))
 (define all-regexs (make-hash-table))
@@ -32,10 +34,11 @@
     (lambda ()
       (apply print data))))
 
 (print-err "Making graph for files: " (string-intersperse targs ", "))
 (print-err "Looking at files: " (string-intersperse files ", "))
+(print-err "Function regex: " function-patt)
 
 ;; Gather the functions
 ;;
 (for-each 
  (lambda (fname)
@@ -46,16 +49,18 @@
 	 (if (not (eof-object? inl))
 	     (let ((match (string-match defn-rx inl)))
 	       (if match 
 		   (let ((fnname (cadr match)))
 		     ;; (print "   " fnname)
-		     (set! all-fns (cons fnname all-fns))
-		     (hash-table-set! 
-		      filedat-defns 
-		      fname
-		      (cons fnname (hash-table-ref/default filedat-defns fname '())))
-		     ))
+		     (if (string-match function-rx fnname)
+			 (begin
+			   (set! all-fns (cons fnname all-fns)))
+			 (hash-table-set! 
+			  filedat-defns 
+			  fname
+			  (cons fnname (hash-table-ref/default filedat-defns fname '())))
+			 )))
 	       (loop (read-line))))))))
  files)
 
 ;; fill up the regex hash
 (print-err "Make the huge regex hash")