Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -26,10 +26,11 @@
     get-previous-test-run-record
     get-matching-previous-test-run-records
     test-get-logfile-info
     test-get-records-for-index-file
     get-testinfo-state-status
+    test-get-top-process-pid
     test-get-paths-matching-keynames-target-new
     get-prereqs-not-met
     get-count-tests-running-for-run-id
     get-run-info
     get-run-status
@@ -73,10 +74,12 @@
     ((get-previous-test-run-record)    (apply db:get-previous-test-run-record dbstruct params))
     ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records dbstruct params))
     ((test-get-logfile-info)           (apply db:test-get-logfile-info dbstruct params))
     ((test-get-records-for-index-file)  (apply db:test-get-records-for-index-file dbstruct params))
     ((get-testinfo-state-status)       (apply db:get-testinfo-state-status dbstruct params))
+    ((test-set-top-process-pid)        (apply db:test-set-top-process-pid dbstruct params))
+    ((test-get-top-process-pid)        (apply db:test-get-top-process-pid dbstruct params))
     ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new dbstruct params))
     ((get-prereqs-not-met)             (apply db:get-prereqs-not-met dbstruct params))
     ((roll-up-pass-fail-counts)        (apply db:roll-up-pass-fail-counts dbstruct params))
     ((update-fail-pass-counts)         (apply db:general-call dbstruct 'update-pass-fail-counts params))
     ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id dbstruct params))

Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -126,10 +126,11 @@
   (http-transport:client-connect (tasks:hostinfo-get-interface server-info)
 				 (tasks:hostinfo-get-port server-info)))
 
 ;; client:signal-handler
 (define (client:signal-handler signum)
+  (signal-mask! signum)
   (handle-exceptions
    exn
    (debug:print " ... exiting ...")
    (let ((th1 (make-thread (lambda ()
 			     "") ;; do nothing for now (was flush out last call if applicable)

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -50,20 +50,22 @@
 (define *alt-log-file* #f)  ;; used by -log
 (define *common:denoise*    (make-hash-table)) ;; for low noise printing
 
 ;; DATABASE
 (define *dbstruct-db*  #f)
-(define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total >
+(define *db-stats*            (make-hash-table)) ;; hash of vectors < count duration-total >
 (define *db-stats-mutex*      (make-mutex))
 (define *db-sync-mutex*       (make-mutex))
 (define *db-multi-sync-mutex* (make-mutex))
 (define *db-local-sync*       (make-hash-table)) ;; used to record last touch of db
 (define *megatest-db*         #f)
 (define *last-db-access*      (current-seconds))  ;; update when db is accessed via server
 (define *db-write-access*     #t)
 (define *inmemdb*             #f)
 (define *task-db*             #f) ;; (vector db path-to-db)
+(define *db-access-allowed*   #t) ;; flag to allow access
+(define *db-access-mutex*     (make-mutex))
 
 ;; SERVER
 (define *my-client-signature* #f)
 (define *transport-type*    'http)
 (define *rpc:listener*      #f) ;; if set up for server communication this will hold the tcp port
@@ -112,14 +114,31 @@
   (set! *test-info*          (make-hash-table))
   (set! *run-info-cache*     (make-hash-table))
   (set! *env-vars-by-run-id* (make-hash-table))
   (set! *test-id-cache*      (make-hash-table)))
 
-;; Generic string database (normalization of sorts)
+;; Generic string database
 (define sdb:qry #f) ;; (make-sdb:qry)) ;;  'init #f)
-;; Generic path database (normalization of sorts)
+;; Generic path database
 (define *fdb* #f)
+
+;;======================================================================
+;; L O C K E R S   A N D   B L O C K E R S 
+;;======================================================================
+
+;; block further accesses to databases. Call this before shutting db down
+(define (common:db-block-further-queries)
+  (mutex-lock! *db-access-mutex*)
+  (set! *db-access-allowed* #f)
+  (mutex-unlock! *db-access-mutex*))
+
+(define (common:db-access-allowed?)
+  (let ((val (begin
+	       (mutex-lock! *db-access-mutex*)
+	       *db-access-allowed*
+	       (mutex-unlock! *db-access-mutex*))))
+    val))
 
 ;;======================================================================
 ;; U S E F U L   S T U F F
 ;;======================================================================
 
@@ -137,11 +156,17 @@
   (if (getenv "MT_MEGATEST") (getenv "MT_MEGATEST") "megatest"))
 
 (define (common:read-encoded-string instr)
   (handle-exceptions
    exn
-   (read (open-input-string (base64:base64-decode instr)))
+   (handle-exceptions
+    exn
+    (begin
+      (debug:print 0 "ERROR: received bad encoded string \"" instr "\", message: " ((condition-property-accessor 'exn 'message) exn))
+      (print-call-chain)
+      #f)
+    (read (open-input-string (base64:base64-decode instr))))
    (read (open-input-string (z3:decode-buffer (base64:base64-decode instr))))))
 
 ;;======================================================================
 ;; S T A T E S   A N D   S T A T U S E S
 ;;======================================================================
@@ -183,10 +208,14 @@
     (if val val default)))
 
 (define (assoc/default key lst . default)
   (let ((res (assoc key lst)))
     (if res (cadr res)(if (null? default) #f (car default)))))
+
+(define (common:get-testsuite-name)
+  (or (configf:lookup *configdat* "server" "testsuite" )
+       (pathname-file *toppath*)))
 
 ;;======================================================================
 ;; Misc utils
 ;;======================================================================
 
@@ -527,10 +556,21 @@
    (seconds->local-time sec) "%yww%V.%w"))
 
 (define (seconds->year-work-week/day-time sec)
   (time->string
    (seconds->local-time sec) "%yww%V.%w %H:%M"))
+
+(define (seconds->quarter sec)
+  (case (string->number
+	 (time->string 
+	  (seconds->local-time sec)
+	  "%m"))
+    ((1 2 3) 1)
+    ((4 5 6) 2)
+    ((7 8 9) 3)
+    ((10 11 12) 4)
+    (else #f)))
 
 ;;======================================================================
 ;; Colors
 ;;======================================================================
       

Index: configf.scm
==================================================================
--- configf.scm
+++ configf.scm
@@ -61,11 +61,11 @@
 (define configf:cont-ln-rx (regexp "^(\\s+)(\\S+.*)$"))
 
 ;; read a line and process any #{ ... } constructs
 
 (define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget)\\s+([^\\}\\{]*)\\}(.*)"))
-(define (configf:process-line l ht)
+(define (configf:process-line l ht allow-system)
   (let loop ((res l))
     (if (string? res)
 	(let ((matchdat (string-search configf:var-expand-regex res)))
 	  (if matchdat
 	      (let* ((prestr  (list-ref matchdat 1))
@@ -85,14 +85,16 @@
 				   (conc "(lambda (ht)(config-lookup ht \"" sect "\" \"" var "\"))")))
 				((runconfigs-get) (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
 				((rget)           (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
 				(else "(lambda (ht)(print \"ERROR\") \"ERROR\")"))))
 		;; (print "fullcmd=" fullcmd)
-		(with-input-from-string fullcmd
-		  (lambda ()
-		    (set! result ((eval (read)) ht))))
-		(loop (conc prestr result poststr)))
+		(if (or allow-system
+			(not (member cmdtype '("system" "shell"))))
+		    (with-input-from-string fullcmd
+		      (lambda ()
+			(set! result ((eval (read)) ht))))
+		    (set! result (conc "#{(" cmdtype ") "  cmd "}")))		(loop (conc prestr result poststr)))
 	      res))
 	res)))
 
 ;; Run a shell command and return the output as a string
 (define (shell cmd)
@@ -117,11 +119,13 @@
     (if targ
 	(or (configf:lookup config targ var)
 	    (configf:lookup config "default" var))
 	(configf:lookup config "default" var))))
 
-(define-inline (configf:read-line p ht allow-processing)
+;; this was inline but I'm pretty sure that is a hold over from when it was *very* simple ...
+;;
+(define (configf:read-line p ht allow-processing)
   (let loop ((inl (read-line p)))
     (let ((cont-line (and (string? inl)
 			  (not (string-null? inl))
 			  (equal? "\\" (string-take-right inl 1)))))
       (if cont-line ;; last character is \ 
@@ -129,14 +133,18 @@
 	    (if (not (eof-object? nextl))
 		(loop (string-append (if cont-line 
 					 (string-take inl (- (string-length inl) 1))
 					 inl)
 				     nextl))))
-	  (if (and allow-processing 
-		   (not (eq? allow-processing 'return-string)))
-	      (configf:process-line inl ht)
-	      inl)))))
+	  (case allow-processing ;; if (and allow-processing 
+	    ;;	   (not (eq? allow-processing 'return-string)))
+	    ((#t #f)
+	     (configf:process-line inl ht allow-processing))
+	    ((return-string)
+	     inl)
+	    (else
+	     (configf:process-line inl ht allow-processing)))))))
 
 ;; read a config file, returns hash table of alists
 
 ;; read a config file, returns hash table of alists
 ;; adds to ht if given (must be #f otherwise)

Index: dashboard-tests.scm
==================================================================
--- dashboard-tests.scm
+++ dashboard-tests.scm
@@ -128,12 +128,11 @@
 				     ))
 			(list "Author: "
 			      "Owner: "
 			      "Reviewed: "
 			      "Tags: "
-			      "Description: "
-			      ))
+			      "Description: "))
 		   (list (iup:label "" #:expand "VERTICAL"))))
     (apply iup:vbox  ; #:expand "YES"
 	   (list 
 	    (store-meta "author"
 			 (iup:label (db:testmeta-get-author testmeta) #:expand "HORIZONTAL")
@@ -197,11 +196,12 @@
 			(list "Hostname: "
 			      "Uname -a: "
 			      "Disk free: "
 			      "CPU Load: "
 			      "Run duration: "
-			      "Logfile: "))
+			      "Logfile: "
+			      "Top process id: "))
 		   (iup:label "" #:expand "VERTICAL")))
     (apply iup:vbox ; #:expand "YES"
 	   (list
 	    ;; NOTE: Yes, the host can change!
 	    (store-label "HostName"
@@ -220,13 +220,17 @@
 			 (iup:label (conc (db:test-get-cpuload testdat)) #:expand "HORIZONTAL")
 			 (lambda (testdat)(conc (db:test-get-cpuload testdat))))
 	    (store-label "RunDuration"
 			 (iup:label (conc (seconds->hr-min-sec (db:test-get-run_duration testdat))) #:expand "HORIZONTAL")
 			 (lambda (testdat)(conc (seconds->hr-min-sec (db:test-get-run_duration testdat)))))
-	    (store-label "CPULoad"
+	    (store-label "LogFile"
 			 (iup:label (conc (db:test-get-final_logf testdat)) #:expand "HORIZONTAL")
-			 (lambda (testdat)(conc (db:test-get-final_logf testdat)))))))))
+			 (lambda (testdat)(conc (db:test-get-final_logf testdat))))
+	    (store-label "ProcessId"
+			 (iup:label (conc (db:test-get-process_id testdat)) #:expand "HORIZONTAL")
+			 (lambda (testdat)(conc (db:test-get-process_id testdat))))
+	    )))))
 
 ;; use a global for setting the buttons colors
 ;;                           state status teststeps
 (define *state-status* (vector #f #f #f))
 (define (update-state-status-buttons testdat)
@@ -557,11 +561,11 @@
 						       request-update))
 				    (newtestdat (if need-update 
 						    ;; NOTE: BUG HIDER, try to eliminate this exception handler
 						    (handle-exceptions
 						     exn 
-						     (debug:print-info 0 "test db access issue: " ((condition-property-accessor 'exn 'message) exn))
+						     (debug:print-info 0 "test db access issue in examine test for run-id " run-id ", test-id " test-id ": " ((condition-property-accessor 'exn 'message) exn))
 						     (db:get-test-info-by-id dbstruct run-id test-id )))))
 			       ;; (debug:print-info 0 "need-update= " need-update " curr-mod-time = " curr-mod-time)
 			       (cond
 				((and need-update newtestdat)
 				 (set! testdat newtestdat)

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -976,11 +976,11 @@
 ;; S U M M A R Y 
 ;;======================================================================
 ;;
 ;; General info about the run(s) and megatest area
 (define (dashboard:summary db)
-  (let ((rawconfig        (read-config (conc *toppath* "/megatest.config") #f 'return-string)))
+  (let ((rawconfig        (read-config (conc *toppath* "/megatest.config") #f #f))) ;; changed to #f since I want #{} to be expanded by [system ...] to NOT be expanded. WAS: 'return-string)))
     (iup:vbox
      (iup:split
       #:value 500
       (iup:frame 
        #:title "General Info"
@@ -1455,13 +1455,18 @@
 ;; Force creation of the db in case it isn't already there.
 (let ((db (tasks:open-db)))
   (sqlite3:finalize! db))
 
 (define (dashboard:get-youngest-run-db-mod-time)
-  (apply max (map (lambda (filen)
-		    (file-modification-time filen))
-		  (glob (conc *dbdir* "/*.db")))))
+  (handle-exceptions
+   exn
+   (begin
+     (debug:print 0 "WARNING: error in accessing databases in get-youngest-run-db-mod-time: " ((condition-property-accessor 'exn 'message) exn))
+     (current-seconds)) ;; something went wrong - just print an error and return current-seconds
+   (apply max (map (lambda (filen)
+		     (file-modification-time filen))
+		   (glob (conc *dbdir* "/*.db"))))))
 
 (define (dashboard:run-update x)
   (let* ((modtime         (dashboard:get-youngest-run-db-mod-time)) ;; (file-modification-time *db-file-path*))
 	 (monitor-modtime (if (file-exists? *monitor-db-path*)
 			      (file-modification-time *monitor-db-path*)

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -12,11 +12,11 @@
 ;;======================================================================
 ;; Database access
 ;;======================================================================
 
 (require-extension (srfi 18) extras tcp)
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking)
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3)
 (import (prefix sqlite3 sqlite3:))
 (import (prefix base64 base64:))
 
 (declare (unit db))
 (declare (uses common))
@@ -278,29 +278,44 @@
 
 ;; close all opened run-id dbs
 (define (db:close-all dbstruct)
   ;; finalize main.db
   (db:sync-touched dbstruct 0 force-sync: #t)
+  ;;(common:db-block-further-queries)
+  ;; (mutex-lock! *db-sync-mutex*) ;; with this perhaps it isn't necessary to use the block-further-queries mechanism?
   (sqlite3:finalize! (db:get-db dbstruct #f))
   (let* ((local (dbr:dbstruct-get-local dbstruct))
 	 (rundb (dbr:dbstruct-get-rundb dbstruct)))
     (if local
 	(for-each
 	 (lambda (db)
 	   (if (sqlite3:database? db)
-	       (sqlite3:finalize! db)))
+	       (begin
+		 (sqlite3:interrupt! db)
+		 (sqlite3:finalize! db #t))))
 	 (hash-table-values (dbr:dbstruct-get-locdbs dbstruct))))
-    (if rundb
-	(if (sqlite3:database? rundb)
-	    (sqlite3:finalize! rundb)
-	    (debug:print 2 "WARNING: attempting to close databases but got " rundb " instead of a database")))))
+    (thread-sleep! 3)
+    (if (and rundb
+	     (sqlite3:database? rundb))
+	(handle-exceptions
+	 exn
+	 (begin 
+	   (debug:print 0 "WARNING: database files may not have been closed correctly. Consider running -cleanup-db")
+	   (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+	   (debug:print 0 " db: " rundb)
+	   (print-call-chain)
+	   #f)
+	 (sqlite3:interrupt! rundb)
+	 (sqlite3:finalize! rundb #t))))
+  ;; (mutex-unlock! *db-sync-mutex*)
+  )
 
 (define (db:open-inmem-db)
   (let* ((db      (sqlite3:open-database ":memory:"))
 	 (handler (make-busy-timeout 3600)))
-    (db:initialize-run-id-db db)
     (sqlite3:set-busy-handler! db handler)
+    (db:initialize-run-id-db db)
     db))
 
 ;; just tests, test_steps and test_data tables
 (define db:sync-tests-only
   (list
@@ -847,11 +862,11 @@
 ;; 2. Look at run records
 ;;    a. If have tests that are not deleted, set state='unknown'
 ;;    b. ....
 ;;
 (define (db:clean-up db)
-  (debug:print 0 "WARNING: db clean up not ported to v1.60, cleanup action will be on megatest.db")
+  (debug:print 0 "WARNING: db clean up not fully ported to v1.60, cleanup action will be on megatest.db")
   (let* (;; (db         (db:get-db dbstruct #f))
 	 (count-stmt (sqlite3:prepare db "SELECT (SELECT count(id) FROM tests)+(SELECT count(id) FROM runs);"))
 	(statements
 	 (map (lambda (stmt)
 		(sqlite3:prepare db stmt))
@@ -1163,11 +1178,11 @@
   (let ((run-ids '()))
     (sqlite3:for-each-row
      (lambda (run-id)
        (set! run-ids (cons run-id run-ids)))
      (db:get-db dbstruct #f)
-     "SELECT id FROM runs WHERE state != 'deleted';")
+     "SELECT id FROM runs WHERE state != 'deleted' ORDER BY event_time DESC;")
     (reverse run-ids)))
 
 ;; get some basic run stats
 ;;
 ;; ( (runname (( state  count ) ... ))
@@ -1541,19 +1556,21 @@
 
 ;; set tests with state currstate and status currstatus to newstate and newstatus
 ;; use currstate = #f and or currstatus = #f to apply to any state or status respectively
 ;; WARNING: SQL injection risk. NB// See new but not yet used "faster" version below
 ;;
+		;;  AND NOT (item_path='' AND testname in (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")))
+		;;(debug:print 0 "QRY: " qry)
+		;; (db:delay-if-busy)
+
 (define (db:set-tests-state-status dbstruct run-id testnames currstate currstatus newstate newstatus)
   (for-each (lambda (testname)
 	      (let ((qry (conc "UPDATE tests SET state=?,status=? WHERE "
 			       (if currstate  (conc "state='" currstate "' AND ") "")
 			       (if currstatus (conc "status='" currstatus "' AND ") "")
-			       " run_id=? AND testname=? AND NOT (item_path='' AND testname in (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")))
-		;;(debug:print 0 "QRY: " qry)
-		;; (db:delay-if-busy)
-		(sqlite3:execute (db:get-db dbstruct run-id) qry run-id newstate newstatus testname testname)))
+			       " run_id=? AND testname LIKE ?;")))
+		(sqlite3:execute (db:get-db dbstruct run-id) qry newstate newstatus run-id testname)))
 	    testnames))
 
 ;; speed up for common cases with a little logic
 ;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id
 ;;
@@ -1662,14 +1679,25 @@
     (db:first-result-default
      (db:get-db dbstruct run-id)
      "SELECT id FROM tests WHERE testname=? AND item_path=?;"
      #f ;; the default
      testname item-path)))
+
+;; overload the unused attemptnum field for the process id of the runscript or 
+;; ezsteps step script in progress
+;;
+(define (db:test-set-top-process-pid dbstruct run-id test-id pid)
+  (sqlite3:execute (db:get-db dbstruct run-id) "UPDATE tests SET attemptnum=? WHERE id=?;"
+		   pid test-id))
+
+(define (db:test-get-top-process-pid dbstruct run-id test-id)
+  (sqlite3:first-result (db:get-db dbstruct run-id) "SELECT attemptnum FROM tests WHERE id=?;"
+			test-id))
 
 (define db:test-record-fields '("id"           "run_id"        "testname"  "state"      "status"      "event_time"
-				"host"         "cpuload"       "diskfree"  "uname"      "rundir"   "item_path"
-                                "run_duration" "final_logf" "comment"   "shortdir"))
+				"host"         "cpuload"       "diskfree"  "uname"      "rundir"      "item_path"
+                                "run_duration" "final_logf"    "comment"   "shortdir"   "attemptnum"))
 
 ;; fields *must* be a non-empty list
 ;;
 (define (db:field->number fieldname fields)
   (if (null? fields)
@@ -1682,20 +1710,21 @@
 	    (if (null? tal)
 		#f
 		(loop (car tal)(cdr tal)(+ indx 1)))))))
 
 (define db:test-record-qry-selector (string-intersperse db:test-record-fields ","))
+
 
 ;; NOTE: Use db:test-get* to access records
 ;; NOTE: This needs rundir decoding? Decide, decode here or where used? For the moment decode where used.
 (define (db:get-all-tests-info-by-run-id dbstruct run-id)
   (let ((db (db:get-db dbstruct run-id))
 	(res '()))
     (sqlite3:for-each-row
-     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir)
-       ;;                 0    1       2      3      4        5       6      7        8     9     10      11          12          13       14
-       (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir)
+     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum)
+       ;;                 0    1       2      3      4        5       6      7        8     9     10      11          12          13       14     15
+       (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir attemptnum)
 		       res)))
      (db:get-db dbstruct run-id)
      (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE state != 'DELETED' AND run_id=?;")
      run-id)
     res))
@@ -1761,14 +1790,14 @@
 
 ;; Get test data using test_id
 (define (db:get-test-info-by-id dbstruct run-id test-id)
   (let ((db (db:get-db dbstruct run-id))
 	(res #f))
-    (sqlite3:for-each-row
-     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id)
-	   ;;                 0    1       2      3      4        5       6      7        8     9     10      11          12          13       14
-       (set! res (vector id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id)))
+    (sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test
+     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum)
+	   ;;             0    1       2      3      4        5       6      7        8     9     10      11          12          13           14         15          16
+       (set! res (vector id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum)))
      (db:get-db dbstruct run-id)
      (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id=?;")
 	 test-id)
     res))
 
@@ -1980,11 +2009,14 @@
   (case *transport-type*
     ;; ((fs) obj)
     ((http fs)
      (string-substitute
       (regexp "=") "_"
-      (base64:base64-encode (with-output-to-string (lambda ()(serialize obj))))
+      (base64:base64-encode 
+       (z3:encode-buffer
+	(with-output-to-string
+	  (lambda ()(serialize obj)))))
       #t))
     ((zmq)(with-output-to-string (lambda ()(serialize obj))))
     (else obj)))
 
 (define (db:string->obj msg)
@@ -1991,13 +2023,14 @@
   (case *transport-type*
     ;; ((fs) msg)
     ((http fs)
      (if (string? msg)
 	 (with-input-from-string 
-	     (base64:base64-decode
-	      (string-substitute 
-	       (regexp "_") "=" msg #t))
+	     (z3:decode-buffer
+	      (base64:base64-decode
+	       (string-substitute 
+		(regexp "_") "=" msg #t)))
 	   (lambda ()(deserialize)))
 	 (vector #f #f #f))) ;; crude reply for when things go awry
     ((zmq)(with-input-from-string msg (lambda ()(deserialize))))
     (else msg)))
 
@@ -2409,11 +2442,11 @@
    (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
      ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
      (if (eq? err-status 'done)
 	 default
 	 (begin
-	   (debug:print 0 "ERROR:  query " stmt " failed " ((condition-property-accessor 'exn 'message) exn))
+	   (debug:print 0 "ERROR:  query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
 	   (print-call-chain)
 	   default)))
    (apply sqlite3:first-result db stmt params)))
 
 ;;======================================================================

Index: db_records.scm
==================================================================
--- db_records.scm
+++ db_records.scm
@@ -76,12 +76,13 @@
 (define-inline (db:test-get-rundir       vec) (vector-ref vec 10))
 (define-inline (db:test-get-item-path    vec) (vector-ref vec 11))
 (define-inline (db:test-get-run_duration vec) (vector-ref vec 12))
 (define-inline (db:test-get-final_logf   vec) (vector-ref vec 13))
 (define-inline (db:test-get-comment      vec) (vector-ref vec 14))
-(define-inline (db:test-get-pass_count   vec) (vector-ref vec 15))
-(define-inline (db:test-get-fail_count   vec) (vector-ref vec 16))
+(define-inline (db:test-get-process_id   vec) (vector-ref vec 16))
+;; (define-inline (db:test-get-pass_count   vec) (vector-ref vec 15))
+;; (define-inline (db:test-get-fail_count   vec) (vector-ref vec 16))
 (define-inline (db:test-get-fullname     vec)
   (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))
 
 (define-inline (db:test-get-first_err    vec) (printable (vector-ref vec 15)))
 (define-inline (db:test-get-first_warn   vec) (printable (vector-ref vec 16)))

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,11 +1,11 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
     "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
 <head>
 <meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
-<meta name="generator" content="AsciiDoc 8.6.7" />
+<meta name="generator" content="AsciiDoc 8.6.9" />
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -85,14 +85,20 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-pre {
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
   padding: 0;
   margin: 0;
 }
+pre {
+  white-space: pre-wrap;
+}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -217,11 +223,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; }
+span.image img { border-style: none; vertical-align: text-bottom; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -413,16 +419,10 @@
 /*
  * xhtml11 specific
  *
  * */
 
-tt {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 div.tableblock > table {
@@ -452,16 +452,10 @@
 /*
  * html5 specific
  *
  * */
 
-.monospaced {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 thead, p.tableblock.header {
@@ -537,10 +531,12 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
+
+
 </style>
 <script type="text/javascript">
 /*<![CDATA[*/
 var asciidoc = {  // Namespace.
 
@@ -737,11 +733,11 @@
 </head>
 <body class="book">
 <div id="header">
 <h1>The Megatest Users Manual</h1>
 <span id="author">Matt Welland</span><br />
-<span id="email"><tt>&lt;<a href="mailto:matt@kiatoa.com">matt@kiatoa.com</a>&gt;</tt></span><br />
+<span id="email"><code>&lt;<a href="mailto:matt@kiatoa.com">matt@kiatoa.com</a>&gt;</code></span><br />
 <span id="revnumber">version 1.0,</span>
 <span id="revdate">April 2012</span>
 </div>
 <div id="content">
 <div class="sect1">
@@ -959,43 +955,26 @@
 <div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
 configuring and generally getting the most out of Megatest.</p></div>
 </div>
 </div>
 <div class="sect1">
-<h2 id="_debugging_tricks">Debugging Tricks</h2>
-<div class="sectionbody">
-<div class="sect2">
-<h3 id="_examining_the_environment">Examining The Environment</h3>
-<div class="sect3">
-<h4 id="_during_config_file_processing">During Config File Processing</h4>
-</div>
-<div class="sect3">
-<h4 id="_organising_your_tests_and_tasks">Organising Your Tests and Tasks</h4>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[tests-paths]
-1 #{get misc parent}/simplerun/tests</tt></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[setup]</tt></pre>
-</div></div>
-<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
-user is responsible for setting STATE and STATUS</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>runscript main.csh</tt></pre>
-</div></div>
-</div>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_tricks">Tricks</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
-configuring and generally getting the most out of Megatest.</p></div>
+<h2 id="_limiting_your_running_jobs">Limiting your running jobs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
+<div class="paragraph"><p>In your testconfig:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[test_meta]
+jobgroup group1</code></pre>
+</div></div>
+<div class="paragraph"><p>In your megatest.config:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[jobgroups]
+group1 10
+custdes 4</code></pre>
+</div></div>
 </div>
 </div>
 <div class="sect1">
 <h2 id="_debugging_tricks">Debugging Tricks</h2>
 <div class="sectionbody">
@@ -1004,107 +983,153 @@
 <div class="sect3">
 <h4 id="_during_config_file_processing">During Config File Processing</h4>
 </div>
 <div class="sect3">
 <h4 id="_organising_your_tests_and_tasks">Organising Your Tests and Tasks</h4>
-<div class="paragraph"><p>/nfs/ch/disks/ch_unienv_disk005/qa_mrwellan/interim/src/megatest/tests/fdktestqa/testqa</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[tests-paths]
-1 #{get misc parent}/simplerun/tests</tt></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[setup]</tt></pre>
-</div></div>
-<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
-user is responsible for setting STATE and STATUS</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>runscript main.csh</tt></pre>
-</div></div>
-<div class="paragraph"><p>ww30.2
-cellname/LVS/cellname.LAYOUT_ERRORS</p></div>
-<div class="paragraph"><p>Error: text open</p></div>
-<div class="paragraph"><p>ww31.3
-cellname/LVS/cellname.LAYOUT_ERRORS</p></div>
-<div class="paragraph"><p>Error: text open
-Reference</p></div>
-<div class="exampleblock">
-<div class="content">
-<div class="paragraph"><p>Chapters grouped into book parts are at level 1 and can contain
-sub-sections.</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[setup]</tt></pre>
-</div></div>
-<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
-user is responsible for setting STATE and STATUS</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>runscript main.csh</tt></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>[requirements]</tt></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt># A normal waiton waits for the prior tests to be COMPLETED
-# and PASS, CHECK or WAIVED
-waiton test1 test2</tt></pre>
-</div></div>
-<div class="paragraph"><p>The default (i.e. if mode is not specified) is normal. All pre-dependent tests
-must be COMPLETED and PASS, CHECK or WAIVED before the test will start</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>mode   normal</tt></pre>
-</div></div>
-<div class="paragraph"><p>The toplevel mode requires only that the prior tests are COMPLETED.</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt>mode toplevel</tt></pre>
+<div class="listingblock">
+<div class="content">
+<pre><code>[tests-paths]
+1 #{get misc parent}/simplerun/tests</code></pre>
+</div></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[setup]</code></pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>runscript main.csh</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_debugging_server_problems">Debugging Server Problems</h3>
+<div class="listingblock">
+<div class="content">
+<pre><code>sudo lsof -i
+sudo netstat -lptu
+sudo netstat -tulpn</code></pre>
+</div></div>
+</div>
+</div>
+</div>
+<h1 id="_reference">Reference</h1>
+<div class="sect1">
+<h2 id="_the_first_chapter_of_the_second_part_2">The First Chapter of the Second Part</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Chapters grouped into book parts are at level 1 and can contain
+sub-sections.</p></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_the_testconfig_file">The testconfig File</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_setup_section">Setup section</h3>
+<div class="sect3">
+<h4 id="_header">Header</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[setup]</code></pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>runscript main.csh</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_requirements_section">Requirements section</h3>
+<div class="sect3">
+<h4 id="_header_2">Header</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[requirements]</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_wait_on_other_tests">Wait on Other Tests</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code># A normal waiton waits for the prior tests to be COMPLETED
+# and PASS, CHECK or WAIVED
+waiton test1 test2</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_mode">Mode</h4>
+<div class="paragraph"><p>The default (i.e. if mode is not specified) is normal. All pre-dependent tests
+must be COMPLETED and PASS, CHECK or WAIVED before the test will start</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>mode   normal</code></pre>
+</div></div>
+<div class="paragraph"><p>The toplevel mode requires only that the prior tests are COMPLETED.</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>mode toplevel</code></pre>
 </div></div>
 <div class="paragraph"><p>A item based waiton will start items in a test when the
 same-named item is COMPLETED and PASS, CHECK or WAIVED
 in the prior test</p></div>
 <div class="listingblock">
 <div class="content">
-<pre><tt>mode itemmatch</tt></pre>
+<pre><code>mode itemmatch</code></pre>
 </div></div>
 <div class="listingblock">
 <div class="content">
-<pre><tt># With a toplevel test you may wish to generate your list
+<pre><code># With a toplevel test you may wish to generate your list
 # of tests to run dynamically
 #
-# waiton #{shell get-valid-tests-to-run.sh}</tt></pre>
+# waiton #{shell get-valid-tests-to-run.sh}</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_run_time_limit">Run time limit</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s</code></pre>
 </div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip">Skip</h4>
+</div>
+<div class="sect3">
+<h4 id="_header_3">Header</h4>
 <div class="listingblock">
 <div class="content">
-<pre><tt>runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s</tt></pre>
+<pre><code>[skip]</code></pre>
 </div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip_on_still_running_tests">Skip on Still-running Tests</h4>
 <div class="listingblock">
 <div class="content">
-<pre><tt>[skip]</tt></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><tt># NB// If the prevrunning line exists with *any* value the test will
+<pre><code># NB// If the prevrunning line exists with *any* value the test will
 # automatically SKIP if the same-named test is currently RUNNING
 
-prevrunning x</tt></pre>
+prevrunning x</code></pre>
 </div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip_if_a_file_exists">Skip if a File Exists</h4>
 <div class="listingblock">
 <div class="content">
-<pre><tt>fileexists /path/to/a/file # skip if /path/to/a/file exists</tt></pre>
+<pre><code>fileexists /path/to/a/file # skip if /path/to/a/file exists</code></pre>
 </div></div>
+</div>
+<div class="sect3">
+<h4 id="_controlled_waiver_propagation">Controlled waiver propagation</h4>
 <div class="paragraph"><p>If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
 If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED</p></div>
 <div class="paragraph"><p>Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)</p></div>
 <div class="listingblock">
 <div class="content">
-<pre><tt>###### EXAMPLE FROM testconfig #########
+<pre><code>###### EXAMPLE FROM testconfig #########
 # matching file(s) will be diff'd with previous run and logpro applied
 # if PASS or WARN result from logpro then WAIVER state is set
 #
 [waivers]
 # logpro_file    rulename      input_glob
@@ -1114,26 +1139,71 @@
 
 # This builtin rule is the default if there is no &lt;waivername&gt;.logpro file
 # diff   diff %file1% %file2%
 
 # This builtin rule is applied if a &lt;waivername&gt;.logpro file exists
-# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html</tt></pre>
+# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html</code></pre>
 </div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_ezsteps">Ezsteps</h3>
 <div class="paragraph"><p>To transfer the environment to the next step you can do the following:</p></div>
 <div class="listingblock">
 <div class="content">
-<pre><tt>$MT_MEGATEST -env2file .ezsteps/${stepname}</tt></pre>
+<pre><code>$MT_MEGATEST -env2file .ezsteps/${stepname}</code></pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_triggers">Triggers</h3>
+<div class="paragraph"><p>In your testconfig triggers can be specified</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[triggers]
+
+# Call script running.sh when test goes to state=RUNNING, status=PASS
+RUNNING/PASS running.sh
+
+# Call script running.sh any time state goes to RUNNING
+RUNNING/ running.sh
+
+# Call script onpass.sh any time status goes to PASS
+PASS/ onpass.sh</code></pre>
+</div></div>
+<div class="paragraph"><p>Scripts called will have; test-id test-rundir trigger, added to the commandline.</p></div>
+<div class="paragraph"><p>HINT</p></div>
+<div class="paragraph"><p>To start an xterm (useful for debugging), use a command line like the following:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[triggers]
+COMPLETED/ xterm -e bash -s --</code></pre>
 </div></div>
+<div class="admonitionblock">
+<table><tr>
+<td class="icon">
+<div class="title">Note</div>
+</td>
+<td class="content">There is a trailing space after the --</td>
+</tr></table>
+</div>
+</div>
 <div class="sect2">
 <h3 id="_megatest_internals">Megatest Internals</h3>
 <div class="imageblock graphviz">
 <div class="content">
 <img src="server.png" alt="server.png" />
 </div>
 </div>
 </div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_appendix">Appendix A: Example Appendix</h2>
+<div class="sectionbody">
 <div class="paragraph"><p>One or more optional appendixes go here at section level zero.</p></div>
+<div class="sect2">
+<h3 id="_appendix_sub_section">Appendix Sub-section</h3>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
 <div class="title">Note</div>
 </td>
@@ -1140,10 +1210,16 @@
 <td class="content">Preface and appendix subsections start out of sequence at level
 2 (level 1 is skipped).  This only applies to multi-part book
 documents.</td>
 </tr></table>
 </div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_bibliography">Example Bibliography</h2>
+<div class="sectionbody">
 <div class="paragraph"><p>The bibliography list is a style of AsciiDoc bulleted list.</p></div>
 <div class="ulist bibliography"><ul>
 <li>
 <p>
 <a id="taoup"></a>[taoup] Eric Steven Raymond. <em>The Art of Unix
@@ -1156,10 +1232,15 @@
   <em>DocBook - The Definitive Guide</em>. O&#8217;Reilly &amp; Associates. 1999.
   ISBN 1-56592-580-7.
 </p>
 </li>
 </ul></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_glossary">Example Glossary</h2>
+<div class="sectionbody">
 <div class="paragraph"><p>Glossaries are optional. Glossaries entries are an example of a style
 of AsciiDoc labeled lists.</p></div>
 <div class="dlist glossary"><dl>
 <dt>
 A glossary term
@@ -1176,21 +1257,28 @@
 <p>
   The corresponding (indented) definition.
 </p>
 </dd>
 </dl></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_colophon">Example Colophon</h2>
+<div class="sectionbody">
 <div class="paragraph"><p>Text at the end of a book describing facts about its production.</p></div>
-</div></div>
 </div>
 </div>
+<div class="sect1">
+<h2 id="_example_index">Example Index</h2>
+<div class="sectionbody">
 </div>
 </div>
 </div>
 <div id="footnotes"><hr /></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.0<br />
-Last updated 2014-02-18 07:24:48 MST
+Last updated 2014-10-08 23:02:21 MST
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -148,7 +148,38 @@
 
 ----------------------------
 $MT_MEGATEST -env2file .ezsteps/${stepname}	  
 ----------------------------
 
+Triggers
+~~~~~~~~
+
+In your testconfig triggers can be specified 
+
+-----------------
+[triggers]
+
+# Call script running.sh when test goes to state=RUNNING, status=PASS
+RUNNING/PASS running.sh
+
+# Call script running.sh any time state goes to RUNNING
+RUNNING/ running.sh
+
+# Call script onpass.sh any time status goes to PASS
+PASS/ onpass.sh
+-----------------
+
+Scripts called will have; test-id test-rundir trigger, added to the commandline.
+
+HINT
+
+To start an xterm (useful for debugging), use a command line like the following:
+
+-----------------
+[triggers]
+COMPLETED/ xterm -e bash -s -- 
+-----------------
+
+NOTE: There is a trailing space after the --
+
 :numbered!:
 

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -248,15 +248,19 @@
      exn
      (if (> numretries 0)
 	 (begin
 	   (mutex-unlock! *http-mutex*)
 	   (thread-sleep! 1)
-	   (close-all-connections!)
+	   (handle-exceptions
+	    exn
+	    (debug:print 0 "WARNING: closing connections failed. Server at " fullurl " almost certainly dead")
+	    (close-all-connections!))
 	   (debug:print 0 "WARNING: Failed to communicate with server, trying again, numretries left: " numretries)
 	   (http-transport:client-api-send-receive run-id serverdat cmd params numretries: (- numretries 1)))
 	 (begin
 	   (mutex-unlock! *http-mutex*)
+	   (tasks:kill-server-run-id run-id)
 	   #f))
      (begin
        (debug:print-info 11 "fullurl=" fullurl ", cmd=" cmd ", params=" params ", run-id=" run-id "\n")
        ;; set up the http-client here
        (max-retry-attempts 1)
@@ -268,16 +272,22 @@
        ;; process and return it.
        (let* ((send-recieve (lambda ()
 			      (mutex-lock! *http-mutex*)
 			      ;; (condition-case (with-input-from-request "http://localhost"; #f read-lines)
 			      ;;					       ((exn http client-error) e (print e)))
-			      (set! res (with-input-from-request ;; was dat
-					 fullurl 
-					 (list (cons 'key "thekey")
-					       (cons 'cmd cmd)
-					       (cons 'params params))
-					 read-string))
+			      (set! res (handle-exceptions
+					 exn
+					 (begin
+					   (debug:print 0 "ERROR: failure in with-input-from-request. Giving up.")
+					   (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+					   #f)
+					 (with-input-from-request ;; was dat
+					  fullurl 
+					  (list (cons 'key "thekey")
+						(cons 'cmd cmd)
+						(cons 'params params))
+					  read-string)))
 			      ;; Shouldn't this be a call to the managed call-all-connections stuff above?
 			      (close-all-connections!)
 			      (mutex-unlock! *http-mutex*)
 			      ))
 	      (time-out     (lambda ()
@@ -307,10 +317,11 @@
 (define (http-transport:server-dat-get-iface         vec)    (vector-ref  vec 0))
 (define (http-transport:server-dat-get-port          vec)    (vector-ref  vec 1))
 (define (http-transport:server-dat-get-api-uri       vec)    (vector-ref  vec 2))
 (define (http-transport:server-dat-get-api-url       vec)    (vector-ref  vec 3))
 (define (http-transport:server-dat-get-api-req       vec)    (vector-ref  vec 4))
+(define (http-transport:server-dat-get-last-access   vec)    (vector-ref  vec 5))
 
 (define (http-transport:server-dat-make-url vec)
   (if (and (http-transport:server-dat-get-iface vec)
 	   (http-transport:server-dat-get-port  vec))
       (conc "http://" 
@@ -317,18 +328,21 @@
 	    (http-transport:server-dat-get-iface vec)
 	    ":"
 	    (http-transport:server-dat-get-port  vec))
       #f))
 
+(define (http-transport:server-dat-update-last-access vec)
+  (vector-set! vec 5 (current-seconds)))
+
 ;;
 ;; connect
 ;;
 (define (http-transport:client-connect iface port)
   (let* ((api-url      (conc "http://" iface ":" port "/api"))
 	 (api-uri      (uri-reference (conc "http://" iface ":" port "/api")))
 	 (api-req      (make-request method: 'POST uri: api-uri))
-	 (server-dat   (vector iface port api-uri api-url api-req)))
+	 (server-dat   (vector iface port api-uri api-url api-req (current-seconds))))
     server-dat))
 
 ;; run http-transport:keep-running in a parallel thread to monitor that the db is being 
 ;; used and to shutdown after sometime if it is not.
 ;;
@@ -524,10 +538,11 @@
 	  (set! *didsomething* #t)
 	  (thread-join! th2)
 	  (exit)))))
 
 (define (http-transport:server-signal-handler signum)
+  (signal-mask! signum)
   (handle-exceptions
    exn
    (debug:print " ... exiting ...")
    (let ((th1 (make-thread (lambda ()
 			     (thread-sleep! 1))

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -91,10 +91,12 @@
                                               fulln
                                               runscript))))) ;; assume it is on the path
 	       (rollup-status 0))
 	  (change-directory top-path)
 
+	  ;; (set-signal-handler! signal/int (lambda ()
+					    
 	  ;; Do not run the test if it is REMOVING, RUNNING, KILLREQ or REMOTEHOSTSTART,
 	  ;; Mark the test as REMOTEHOSTSTART *IMMEDIATELY*
 	  ;;
 	  (let ((test-info (rmt:get-testinfo-state-status run-id test-id)))
 	    (if (not (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ")))
@@ -207,10 +209,11 @@
 				 (thread-sleep! 0.3) ;; NFS slowness has caused grief here
 
 				 ;; if there is a runscript do it first
 				 (if fullrunscript
 				     (let ((pid (process-run fullrunscript)))
+				       (rmt:test-set-top-process-pid run-id test-id pid)
 				       (let loop ((i 0))
 					 (let-values
 					  (((pid-val exit-status exit-code) (process-wait pid #t)))
 					  (mutex-lock! m)
 					  (vector-set! exit-info 0 pid)
@@ -262,10 +265,11 @@
 
 						   (debug:print 4 "script: " script)
 						   (rmt:teststep-set-status! run-id test-id stepname "start" "-" #f #f)
 						   ;; now launch
 						   (let ((pid (process-run script)))
+						     (rmt:test-set-top-process-pid run-id test-id pid)
 						     (let processloop ((i 0))
 						       (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
 								   (mutex-lock! m)
 								   (vector-set! exit-info 0 pid)
 								   (vector-set! exit-info 1 exit-status)
@@ -341,52 +345,45 @@
 									  (if time-exceeded
 									      (begin
 										(debug:print-info 0 "KILLING TEST DUE TO TIME LIMIT EXCEEDED! Runtime=" run-seconds " seconds, limit=" runtlim)
 										#t)
 									      #f)))))
-				       ;; open-run-close not needed for test-set-meta-info
-				       ;; (tests:set-partial-meta-info #f test-id run-id minutes work-area)
 				       (tests:update-central-meta-info run-id test-id (get-cpu-load) (get-df (current-directory))(calc-minutes) #f #f)
-				       ;; (tests:set-partial-meta-info #f test-id run-id minutes work-area 10)
-				       ;; (tests:set-partial-meta-info test-id run-id minutes work-area)
 				       (if kill-job? 
 					   (begin
 					     (mutex-lock! m)
 					     ;; NOTE: The pid can change as different steps are run. Do we need handshaking between this
 					     ;;       section and the runit section? Or add a loop that tries three times with a 1/4 second
 					     ;;       between tries?
-					     (let* ((pid (vector-ref exit-info 0)))
-					       (if (number? pid)
-						   (handle-exceptions
-						    exn
-						    (debug:print-info 0 "Unable to kill process with pid " pid ", possibly already killed.")
-						    ;;(process-signal pid signal/kill))
-						    (begin
-						      (debug:print 0 "WARNING: Request received to kill job (attempt # " kill-tries ")")
-						      (let ((processes (cmd-run->list (conc "pgrep -l -P " pid))))
-							(for-each 
-							 (lambda (p)
-							   (let* ((parts  (string-split p))
-								  (p-id   (if (> (length parts) 0)
-									      (string->number (car parts))
-									      #f)))
-							     (if p-id
-								 (begin
-								   (debug:print 0 "Killing " (cadr parts) "; kill -9  " p-id)
-								   ;; (process-signal pid signal/kill))))) ;; 
-								   (system (conc "kill -9 " p-id))))))
-							 (car processes)))
-						      (system (conc "kill -9 -" pid))
-						      (tests:test-set-status! test-id "KILLED"  "KILLED" (args:get-arg "-m") #f)))
-						   (begin
-						     (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process")
-						     (tests:test-set-status! test-id "KILLED"  "KILLED" (args:get-arg "-m") #f)
-						     (tests:test-set-status! run-id test-id "KILLED"  "FAIL" (args:get-arg "-m") #f)
-						     (exit 1) ;; IS THIS NECESSARY OR WISE???
-						     )))
-					     (set! kill-tries (+ 1 kill-tries))
-					     (mutex-unlock! m)))
+					     (let* ((pid1 (vector-ref exit-info 0))
+						    (pid2 (rmt:test-get-top-process-pid run-id test-id))
+						    (pids (delete-duplicates (filter number? (list pid1 pid2)))))
+					       (if (not (null? pids))
+						   (begin
+						     (for-each
+						      (lambda (pid)
+							(handle-exceptions
+							 exn
+							 (begin
+							   (debug:print-info 0 "Unable to kill process with pid " pid ", possibly already killed.")
+							   (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn)))
+							 (debug:print 0 "WARNING: Request received to kill job " pid) ;;  " (attempt # " kill-tries ")")
+							 (if (process:alive? pid)
+							     (begin
+							       (process-signal pid signal/int)
+							       (thread-sleep! 5)
+							       (if (process:process-alive? pid)
+								   (process-signal pid signal/kill))))))
+						      pids)
+						     (tests:test-set-status! run-id test-id "KILLED"  "KILLED" (args:get-arg "-m") #f))
+						   (begin
+						     (debug:print 0 "ERROR: Nothing to kill, pid1=" pid1 ", pid2=" pid2)
+						     (tests:test-set-status! run-id test-id "KILLED"  "FAILED TO KILL" (args:get-arg "-m") #f)
+						     )))
+					     (mutex-unlock! m)
+					     ;; no point in sticking around. Exit now.
+					     (exit)))
 				       (if keep-going
 					   (begin
 					     (thread-sleep! 3) ;; (+ 3 (random 6))) ;; add some jitter to the call home time to spread out the db accesses
 					     (if keep-going
 						 (loop (calc-minutes)))))))
@@ -395,10 +392,11 @@
 		 (th2          (make-thread runit "run job")))
 	    (set! job-thread th2)
 	    (thread-start! th1)
 	    (thread-start! th2)
 	    (thread-join! th2)
+	    (debug:print-info 0 "Megatest exectute of test " test-name ", item path " item-path " complete. Notifying the db ...")
 	    (set! keep-going #f)
 	    (thread-join! th1)
 	    (thread-sleep! 1)       ;; give thread th1 a chance to be done TODO: Verify this is needed. At 0.1 I was getting fail to stop, increased to total of 1.1 sec.
 	    (mutex-lock! m)
 	    (let* ((item-path (item-list->path itemdat))
@@ -459,20 +457,23 @@
 		  (begin
 		    (handle-exceptions
 		     exn
 		     (begin
 		       (debug:print 0 "ERROR: Something went wrong when trying to create linktree dir at " linktree)
+		       (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
 		       (exit 1))
 		     (create-directory linktree #t))))
 	      (begin
 		(debug:print 0 "ERROR: linktree not defined in [setup] section of megatest.config")
 		(exit 1)))
 	  (if linktree
 	      (let ((dbdir (conc linktree "/.db")))
 		(handle-exceptions
 		 exn
-		 (debug:print 0 "ERROR: failed to create the " dbdir " area for your database files")
+		 (begin
+		   (debug:print 0 "ERROR: failed to create the " dbdir " area for your database files")
+		   (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn)))
 		 (if (not (directory-exists? dbdir))(create-directory dbdir)))
 		(setenv "MT_LINKTREE" linktree))
 	      (begin
 		(debug:print 0 "ERROR: linktree is required in your megatest.config [setup] section")
 		(exit 1)))

Index: lock-queue.scm
==================================================================
--- lock-queue.scm
+++ lock-queue.scm
@@ -36,34 +36,38 @@
 	   (begin
 	     (thread-sleep! 10)
 	     (if (> count 0)
 		 (lock-queue:open-db fname count: (- count 1))
 		 db))
-	   (sqlite3:execute 
-	    db
-	    "CREATE TABLE IF NOT EXISTS queue (
-  	      id         INTEGER PRIMARY KEY,
-              test_id    INTEGER,
-              start_time INTEGER,
-              state      TEXT,
-              CONSTRAINT queue_constraint UNIQUE (test_id));")
-	   (sqlite3:execute
-	    db
-	    "CREATE TABLE IF NOT EXISTS runlocks (
-              id         INTEGER PRIMARY KEY,
-              test_id    INTEGER,
-              run_lock   TEXT,
-              CONSTRAINT runlock_constraint UNIQUE (run_lock));"))))
+	   (sqlite3:with-transaction
+	    db
+	    (lambda ()
+	      (sqlite3:execute 
+	       db
+	       "CREATE TABLE IF NOT EXISTS queue (
+     	         id         INTEGER PRIMARY KEY,
+                 test_id    INTEGER,
+                 start_time INTEGER,
+                 state      TEXT,
+                 CONSTRAINT queue_constraint UNIQUE (test_id));")
+	      (sqlite3:execute
+	       db
+	       "CREATE TABLE IF NOT EXISTS runlocks (
+                 id         INTEGER PRIMARY KEY,
+                 test_id    INTEGER,
+                 run_lock   TEXT,
+                 CONSTRAINT runlock_constraint UNIQUE (run_lock));"))))))
     (sqlite3:set-busy-handler! db handler)
     db))
 
 (define (lock-queue:set-state db test-id newstate #!key (remtries 10))
   (handle-exceptions
    exn
    (if (> remtries 0)
        (begin
 	 (debug:print 0 "WARNING: exception on lock-queue:set-state. Trying again in 30 seconds.")
+	 (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
 	 (thread-sleep! 30)
 	 (lock-queue:set-state db test-id newstate remtries: (- remtries 1)))
        (begin
 	 (debug:print 0 "ERROR:  Failed to set lock state for test with id " test-id ", error: " ((condition-property-accessor 'exn 'message) exn) ", giving up.")
 	 #f))
@@ -75,10 +79,11 @@
   (handle-exceptions
    exn
    (if (> remtries 0)
        (begin
 	 (debug:print 0 "WARNING: exception on lock-queue:any-younger. Trying again in 30 seconds.")
+	 (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
 	 (thread-sleep! 30)
 	 (lock-queue:any-younger? db mystart test-id remtries: (- remtries 1)))
        (begin
 	 (debug:print 0 "ERROR:  Failed to find younger locks for test with id " test-id ", error: " ((condition-property-accessor 'exn 'message) exn) ", giving up.")
 	 #f))
@@ -98,10 +103,12 @@
 	(mklckqry  (sqlite3:prepare db "INSERT INTO runlocks (test_id,run_lock) VALUES (?,'locked');")))
     (let ((result 
 	   (handle-exceptions
 	    exn
 	    (begin
+	      (debug:print 0 "WARNING: failed to get queue lock. Will try again in a few seconds")
+	      (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
 	      (thread-sleep! 10)
 	      (if (> count 0)
 		  (lock-queue:get-lock db test-id count: (- count 1)))
 	      #f)
 	    (sqlite3:with-transaction
@@ -125,10 +132,12 @@
 (define (lock-queue:release-lock fname test-id #!key (count 10))
   (let ((db (lock-queue:open-db fname)))
     (handle-exceptions
      exn
      (begin
+       (debug:print 0 "WARNING: Failed to release queue lock. Will try again in few seconds")
+       (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
        (thread-sleep! 10)
        (if (> count 0)
 	   (lock-queue:release-lock fname test-id count: (- count 1))
 	   #f))
      (sqlite3:execute db "DELETE FROM runlocks WHERE test_id=?;" test-id)
@@ -136,10 +145,12 @@
 
 (define (lock-queue:steal-lock db test-id #!key (count 10))
   (handle-exceptions
    exn
    (begin
+     (debug:print 0 "WARNING: Failed to steal queue lock. Will try again in few seconds")
+     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
      (thread-sleep! 10)
      (if (> count 0)
 	 (lock-queue:steal-lock db test-id count: (- count 1))
 	 #f))
    (sqlite3:execute db "DELETE FROM runlocks WHERE run_lock='locked';"))
@@ -153,10 +164,12 @@
   (let ((db      (lock-queue:open-db fname))
 	(mystart (current-seconds)))
     (handle-exceptions
      exn
      (begin
+       (debug:print 0 "WARNING: Failed to find out if it is ok to skip the wait queue. Will try again in few seconds")
+       (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
        (thread-sleep! 10)
        (if (> count 0)
 	   (lock-queue:wait-turn fname test-id count: (- count 1))
 	   #f))
      (sqlite3:execute

Index: megatest-version.scm
==================================================================
--- megatest-version.scm
+++ megatest-version.scm
@@ -1,7 +1,7 @@
 ;; Always use two or four digit decimal
 ;; 1.01, 1.02...1.10,1.11,1,1101 ... 1.99,2.00..
 
 (declare (unit megatest-version))
 
-(define megatest-version 1.6003)
+(define megatest-version 1.6005)
 

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -8,11 +8,13 @@
 ;;  PURPOSE.
 
 ;; (include "common.scm")
 ;; (include "megatest-version.scm")
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 format readline apropos json http-client directory-utils z3) ;; (srfi 18) extras)
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 format readline apropos json 
+     http-client directory-utils z3 srfi-18) ;;  extras)
+
 (import (prefix sqlite3 sqlite3:))
 (import (prefix base64 base64:))
 
 ;; (use zmq)
 
@@ -279,10 +281,66 @@
 			"-q" ;; quiet 0, errors/warnings only
 		       )
 		 args:arg-hash
 		 0))
 
+;; The watchdog is to keep an eye on things like db sync etc.
+;;
+(define *watchdog*
+  (make-thread 
+   (lambda ()
+     (let loop ()
+       (thread-sleep! 5) ;; five second resolution is only a minor burden and should be tolerable 
+
+       ;; sync for filesystem local db writes
+       ;;
+       (let ((start-time (current-seconds)))
+	 (mutex-lock! *db-multi-sync-mutex*)
+	 (for-each 
+	  (lambda (run-id)
+	    (let ((last-write (hash-table-ref/default *db-local-sync* run-id 0)))
+	      (if ;; (and 
+	       (> (- start-time last-write) 5) ;; every five seconds
+	       ;;      (common:db-access-allowed?))
+	       (begin
+		 (db:multi-db-sync (list run-id) 'new2old)
+		 (if (common:low-noise-print 30 "sync new to old")
+		     (debug:print-info 0 "Sync of newdb to olddb for run-id " run-id " completed in " (- (current-seconds) start-time) " seconds"))
+		 (hash-table-delete! *db-local-sync* run-id)))))
+	  (hash-table-keys *db-local-sync*))
+	 (mutex-unlock! *db-multi-sync-mutex*))
+       
+       ;; keep going unless time to exit
+       ;;
+       (if (not *time-to-exit*)
+	   (loop))))
+   "Watchdog thread"))
+
+(thread-start! *watchdog*)
+
+(define (std-exit-procedure)
+  (rmt:print-db-stats)
+  (let ((run-ids (hash-table-keys *db-local-sync*)))
+    (if (not (null? run-ids))
+	(db:multi-db-sync run-ids 'new2old)))
+  (if *dbstruct-db* (db:close-all *dbstruct-db*))
+  (if *megatest-db* (begin
+		      (sqlite3:interrupt! *megatest-db*)
+		      (sqlite3:finalize! *megatest-db* #t)))
+  (if *task-db*     (let ((db (vector-ref *task-db* 0)))
+		      (sqlite3:interrupt! db)
+		      (sqlite3:finalize! db #t))))
+
+(define (std-signal-handler signum)
+  (signal-mask! signum)
+  (debug:print 0 "ERROR: Received signal " signum " exiting promptly")
+  (std-exit-procedure)
+  (exit))
+
+(set-signal-handler! signal/int std-signal-handler)
+(set-signal-handler! signal/term std-signal-handler)
+
 (if (args:get-arg "-log")
     (let ((oup (open-output-file (args:get-arg "-log"))))
       (debug:print-info 0 "Sending log output to " (args:get-arg "-log"))
       (current-error-port oup)
       (current-output-port oup)))
@@ -347,18 +405,11 @@
     (let ((newval (conc (args:get-arg "-testpatt") "/" (args:get-arg "-itempatt"))))
       (debug:print 0 "WARNING: -itempatt has been deprecated, please use -testpatt testpatt/itempatt method, new testpatt is "newval)
       (hash-table-set! args:arg-hash "-testpatt" newval)
       (hash-table-delete! args:arg-hash "-itempatt")))
 
-(on-exit (lambda ()
-	   (rmt:print-db-stats)
-	   (let ((run-ids (hash-table-keys *db-local-sync*)))
-	     (if (not (null? run-ids))
-		 (db:multi-db-sync run-ids 'new2old)))
-	   (if *dbstruct-db* (db:close-all *dbstruct-db*))
-	   (if *megatest-db* (sqlite3:finalize! *megatest-db*))
-	   (if *task-db*     (sqlite3:finalize! (vector-ref *task-db* 0)))))
+;; (on-exit std-exit-procedure)
 
 ;;======================================================================
 ;; Misc general calls
 ;;======================================================================
 
@@ -1204,11 +1255,20 @@
       (if (not (launch:setup-for-run))
 	  (begin
 	    (debug:print 0 "Failed to setup, exiting") 
 	    (exit 1)))
       ;; keep this one local
-      (open-run-close db:clean-up #f)
+      ;; (open-run-close db:clean-up #f)
+      (db:multi-db-sync 
+       #f ;; do all run-ids
+       'new2old
+       'killservers
+       'dejunk
+       'adj-testids
+       'old2new
+       'new2old
+       )
       (set! *didsomething* #t)))
 
 (if (args:get-arg "-mark-incompletes")
     (begin
       (if (not (launch:setup-for-run))
@@ -1329,10 +1389,13 @@
 (if *runremote* (close-all-connections!))
 
 (if (not *didsomething*)
     (debug:print 0 help))
 
+(set! *time-to-exit* #t)
+(thread-join! *watchdog*)
+
 (if (not (eq? *globalexitstatus* 0))
     (if (or (args:get-arg "-runtests")(args:get-arg "-runall"))
         (begin
            (debug:print 0 "NOTE: Subprocesses with non-zero exit code detected: " *globalexitstatus*)
            (exit 0))

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -165,21 +165,27 @@
 ;;  S T A T E   A N D   S T A T U S   F O R   T E S T S 
 ;;======================================================================
 
 ;; speed up for common cases with a little logic
 (define (mt:test-set-state-status-by-id run-id test-id newstate newstatus newcomment)
-  (cond
-   ((and newstate newstatus newcomment)
-    (rmt:general-call 'state-status-msg run-id newstate newstatus newcomment test-id))
-   ((and newstate newstatus)
-    (rmt:general-call 'state-status run-id newstate newstatus test-id))
-   (else
-    (if newstate   (rmt:general-call 'set-test-state   run-id newstate   test-id))
-    (if newstatus  (rmt:general-call 'set-test-status  run-id newstatus  test-id))
-    (if newcomment (rmt:general-call 'set-test-comment run-id newcomment test-id))))
-   (mt:process-triggers run-id test-id newstate newstatus)
-   #t)
+  (if (not (and run-id test-id))
+      (begin
+	(debug:print 0 "ERROR: bad data handed to mt:test-set-state-status-by-id, run-id=" run-id ", test-id=" test-id ", newstate=" newstate)
+	(print-call-chain)
+	#f)
+      (begin
+	(cond
+	 ((and newstate newstatus newcomment)
+	  (rmt:general-call 'state-status-msg run-id newstate newstatus newcomment test-id))
+	 ((and newstate newstatus)
+	  (rmt:general-call 'state-status run-id newstate newstatus test-id))
+	 (else
+	  (if newstate   (rmt:general-call 'set-test-state   run-id newstate   test-id))
+	  (if newstatus  (rmt:general-call 'set-test-status  run-id newstatus  test-id))
+	  (if newcomment (rmt:general-call 'set-test-comment run-id newcomment test-id))))
+	(mt:process-triggers run-id test-id newstate newstatus)
+	#t)))
 
 (define (mt:test-set-state-status-by-testname run-id test-name item-path new-state new-status new-comment)
   (let ((test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path)))
     (mt:test-set-state-status-by-id test-id new-state new-status new-comment)))
 

Index: portlogger.scm
==================================================================
--- portlogger.scm
+++ portlogger.scm
@@ -31,11 +31,11 @@
     (sqlite3:set-busy-handler! db handler)
     (sqlite3:execute db "PRAGMA synchronous = 0;")
     (if (not exists)
 	(sqlite3:execute 
 	 db
-	 "CREATE TABLE ports (
+	 "CREATE TABLE IF NOT EXISTS ports (
             port INTEGER PRIMARY KEY,
             state TEXT DEFAULT 'not-used',
             fail_count INTEGER DEFAULT 0,
             update_time TIMESTAMP DEFAULT (strftime('%s','now')) );"))
     db))
@@ -88,16 +88,25 @@
     (sqlite3:finalize! qry2)
     (sqlite3:finalize! qry3)
     res))
 
 (define (portlogger:get-prev-used-port db)
-  (sqlite3:fold-row
-   (lambda (var curr)
-     (or curr var curr))
-   #f
-   db
-   "SELECT (port) FROM ports WHERE state='released' LIMIT 1;"))
+  (handle-exceptions
+   exn
+   (begin
+     (debug:print 0 "EXCEPTION: portlogger database probably overloaded or unreadable. If you see this message again remove /tmp/.$USER-portlogger.db")
+     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+     (debug:print 0 "exn=" (condition->list exn))
+     (print-call-chain)
+     (debug:print 0 "Continuing anyway.")
+     #f)
+   (sqlite3:fold-row
+    (lambda (var curr)
+      (or curr var curr))
+    #f
+    db
+    "SELECT (port) FROM ports WHERE state='released' LIMIT 1;")))
 
 (define (portlogger:find-port db)
   (let* ((lowport (let ((val (configf:lookup *configdat* "server" "lowport")))
 		    (if (and val 
 			     (string->number val))
@@ -104,11 +113,19 @@
 			(string->number val)
 			32768)))
 	 (portnum (or (portlogger:get-prev-used-port db)
 		      (+ lowport ;; top of registered ports is 49152 but lets use ports in the registered range
 			 (random (- 64000 lowport))))))
-    (portlogger:take-port db portnum)
+    (handle-exceptions
+     exn
+     (begin
+       (debug:print 0 "EXCEPTION: portlogger database probably overloaded or unreadable. If you see this message again remove /tmp/.$USER-portlogger.db")
+       (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+       (debug:print 0 "exn=" (condition->list exn))
+       (print-call-chain)
+       (debug:print 0 "Continuing anyway."))
+     (portlogger:take-port db portnum))
     portnum))
 
 ;; set port to "released", "failed" etc.
 ;; 
 (define (portlogger:set-port db portnum value)
@@ -122,20 +139,30 @@
 ;;======================================================================
 ;; MAIN
 ;;======================================================================
 
 (define (portlogger:main . args)
-  (let* ((db      (portlogger:open-db (conc "/tmp/." (current-user-name) "-portlogger.db")))
+  (let* ((dbfname (conc "/tmp/." (current-user-name) "-portlogger.db"))
+	 (db      (portlogger:open-db dbfname))
 	 (numargs (length args))
-	 (result  (cond
-		   ((> numargs 1) ;; most commands
-		    (case (string->symbol (car args)) ;; commands with two or more params
-		      ((take)(portlogger:take-port db (string->number (cadr args))))
-		      ((set) (portlogger:set-port db 
-						  (string->number (cadr args))
-						  (caddr args))
-		       (caddr args))
-		      ((failed)(portlogger:set-failed db (string->number (cadr args))) 'failed))))))
+	 (result  
+	  (handle-exceptions
+	   exn
+	   (begin
+	     (debug:print 0 "EXCEPTION: portlogger database at " dbfname " probably overloaded or unreadable. Try removing it.")
+	     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+	     (print "exn=" (condition->list exn))
+	     (debug:print 0 " status:  " ((condition-property-accessor 'sqlite3 'status) exn))
+	     (print-call-chain))
+	   (cond
+	    ((> numargs 1) ;; most commands
+	     (case (string->symbol (car args)) ;; commands with two or more params
+	       ((take)(portlogger:take-port db (string->number (cadr args))))
+	       ((set) (portlogger:set-port db 
+					   (string->number (cadr args))
+					   (caddr args))
+		(caddr args))
+	       ((failed)(portlogger:set-failed db (string->number (cadr args))) 'failed)))))))
     (sqlite3:finalize! db)
     result))
      
 ;; (print (apply portlogger:main (cdr (argv))))

Index: process.scm
==================================================================
--- process.scm
+++ process.scm
@@ -51,10 +51,12 @@
   ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
   (handle-exceptions
    exn
    (begin
      (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
+     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+     (print "exn=" (condition->list exn))
      #f)
    (let-values (((fh fho pid) (if (null? params)
 				  (process cmd)
 				  (process cmd params))))
        (let loop ((curr (read-line fh))
@@ -124,6 +126,15 @@
        (if (eof-object? inl)
 	   (reverse res)
 	   (let ((pid (string->number inl)))
 	     (if proc (proc pid))
 	     (loop (read-line) (cons pid res))))))))
-       
+
+(define (process:alive? pid)
+  (handle-exceptions
+   exn
+   ;; possibly pid is a process not a child, look in /proc to see if it is running still
+   (file-exists? (conc "/proc/" pid))
+   (let-values (((rpid exit-type exit-signal)(process-wait pid #t)))
+       (and (number? rpid)
+	    (equal? rpid pid)))))
+	 

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -57,10 +57,23 @@
 
 ;; cmd is a symbol
 ;; vars is a json string encoding the parameters for the call
 ;;
 (define (rmt:send-receive cmd rid params)
+  ;; clean out old connections
+  (mutex-lock! *db-multi-sync-mutex*)
+  (let ((expire-time (- (current-seconds) 60)))
+    (for-each 
+     (lambda (run-id)
+       (let ((connection (hash-table-ref/default *runremote* run-id #f)))
+	 (if ;; (and connection 
+		  (< (http-transport:server-dat-get-last-access connection) expire-time) ; )
+	     (begin
+	       (debug:print-info 0 "Discarding connection to server for run-id " run-id ", too long between accesses")
+	       (hash-table-delete! *runremote* run-id)))))
+     (hash-table-keys *runremote*)))
+  (mutex-unlock! *db-multi-sync-mutex*)
   (let* ((run-id          (if rid rid 0))
 	 (connection-info (let ((cinfo (hash-table-ref/default *runremote* run-id #f)))
 			    (if cinfo
 				cinfo
 				;; NB// can cache the answer for server running for 10 seconds ...
@@ -72,16 +85,17 @@
 					  #f))
 				    #f))))
 	 (jparams         (db:obj->string params)))
     (if connection-info
 	(let ((res             (http-transport:client-api-send-receive run-id connection-info cmd jparams)))
+	  (http-transport:server-dat-update-last-access connection-info)
 	  (if res
 	      (db:string->obj res)
 	      (let ((new-connection-info (client:setup run-id)))
 		(debug:print 0 "WARNING: Communication failed, trying call to http-transport:client-api-send-receive again.")
 		(rmt:send-receive cmd run-id params))))
-	(let ((max-avg-qry (string->number (or (configf:lookup *configdat* "server" "server-query-threshold") "800"))))
+	(let ((max-avg-qry (string->number (or (configf:lookup *configdat* "server" "server-query-threshold") "-1"))))
 	  (debug:print-info 4 "no server and read-only query, bypassing normal channel")
 	  ;; (if (rmt:write-frequency-over-limit? cmd run-id)(server:kind-run run-id))
 	  (let ((curr-max (rmt:get-max-query-average)))
 	    (if (> (cdr curr-max) max-avg-qry)
 		(begin
@@ -89,18 +103,25 @@
 		  (server:kind-run run-id))))
 	  (rmt:open-qry-close-locally cmd run-id params)))))
 
 (define (rmt:update-db-stats rawcmd params duration)
   (mutex-lock! *db-stats-mutex*)
-  (let* ((cmd      (if (eq? rawcmd 'general-call) (car params) rawcmd))
-	 (stat-vec (hash-table-ref/default *db-stats* cmd #f)))
-    (if (not stat-vec)
-	(let ((newvec (vector 0 0)))
-	  (hash-table-set! *db-stats* cmd newvec)
-	  (set! stat-vec newvec)))
-    (vector-set! stat-vec 0 (+ (vector-ref stat-vec 0) 1))
-    (vector-set! stat-vec 1 (+ (vector-ref stat-vec 1) duration)))
+  (handle-exceptions
+   exn
+   (begin
+     (debug:print 0 "WARNING: stats collection failed in update-db-stats")
+     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+     (print "exn=" (condition->list exn))
+     #f) ;; if this fails we don't care, it is just stats
+   (let* ((cmd      (if (eq? rawcmd 'general-call) (car params) rawcmd))
+	  (stat-vec (hash-table-ref/default *db-stats* cmd #f)))
+     (if (not stat-vec)
+	 (let ((newvec (vector 0 0)))
+	   (hash-table-set! *db-stats* cmd newvec)
+	   (set! stat-vec newvec)))
+     (vector-set! stat-vec 0 (+ (vector-ref stat-vec 0) 1))
+     (vector-set! stat-vec 1 (+ (vector-ref stat-vec 1) duration))))
   (mutex-unlock! *db-stats-mutex*))
 
 
 (define (rmt:print-db-stats)
   (let ((fmtstr "~40a~7-d~9-d~20,2-f")) ;; "~20,2-f"
@@ -135,14 +156,14 @@
 			   (loop (car tal)(cdr tal) newmax-cmd currmax)))))))
     (mutex-unlock! *db-stats-mutex*)
     res))
 	  
 (define (rmt:open-qry-close-locally cmd run-id params)
-  (let* ((dbdir (conc    (configf:lookup *configdat* "setup" "linktree") "/.db"))
-	 (dbstruct-local (if *dbstruct-db*
+  (let* ((dbstruct-local (if *dbstruct-db*
 			     *dbstruct-db*
-			     (let ((db (make-dbr:dbstruct path:  dbdir local: #t)))
+			     (let* ((dbdir (conc    (configf:lookup *configdat* "setup" "linktree") "/.db"))
+				    (db (make-dbr:dbstruct path:  dbdir local: #t)))
 			       (set! *dbstruct-db* db)
 			       db)))
 	 (db-file-path   (db:dbfile-path 0)))
     ;; (read-only      (not (file-read-access? db-file-path)))
     (let* ((start         (current-milliseconds))
@@ -151,16 +172,12 @@
       (rmt:update-db-stats cmd params duration)
       ;; mark this run as dirty if this was a write
       (if (not (member cmd api:read-only-queries))
 	  (let ((start-time (current-seconds)))
 	    (mutex-lock! *db-multi-sync-mutex*)
-	    (let ((last-sync (hash-table-ref/default *db-local-sync* run-id 0)))
-	      (if (> (- start-time last-sync) 5) ;; every five seconds
-		  (begin
-		    (db:multi-db-sync (list run-id) 'new2old)
-		    (debug:print-info 0 "Sync of newdb to olddb for run-id " run-id " completed in " (- (current-seconds) start-time) " seconds")
-		    (hash-table-set! *db-local-sync* run-id start-time))))
+	    (if (not (hash-table-ref/default *db-local-sync* run-id #f))
+		(hash-table-set! *db-local-sync* run-id start-time)) ;; the oldest "write"
 	    (mutex-unlock! *db-multi-sync-mutex*)))
       res)))
 
 (define (rmt:send-receive-no-auto-client-setup connection-info cmd run-id params)
   (let* ((run-id   (if run-id run-id 0))
@@ -247,11 +264,11 @@
 
 (define (rmt:get-test-info-by-id run-id test-id)
   (if (and (number? run-id)(number? test-id))
       (rmt:send-receive 'get-test-info-by-id run-id (list run-id test-id))
       (begin
-	(debug:print 0 "ERROR: Bad data handed to rmt:get-test-info-by-id run-id=" run-id ", test-id=" test-id)
+	(debug:print 0 "WARNING: Bad data handed to rmt:get-test-info-by-id run-id=" run-id ", test-id=" test-id)
 	(print-call-chain)
 	#f)))
 
 (define (rmt:test-get-rundir-from-test-id run-id test-id)
   (rmt:send-receive 'test-get-rundir-from-test-id run-id (list run-id test-id)))
@@ -316,10 +333,16 @@
   (rmt:send-receive 'get-testinfo-state-status run-id (list run-id test-id)))
 
 (define (rmt:test-set-log! run-id test-id logf)
   (if (string? logf)(rmt:general-call 'test-set-log run-id logf test-id)))
 
+(define (rmt:test-set-top-process-pid run-id test-id pid)
+  (rmt:send-receive 'test-set-top-process-pid run-id (list run-id test-id pid)))
+
+(define (rmt:test-get-top-process-pid run-id test-id)
+  (rmt:send-receive 'test-get-top-process-pid run-id (list run-id test-id)))
+
 (define (rmt:get-run-ids-matching-target keynames target res runname testpatt statepatt statuspatt)
   (rmt:send-receive 'get-run-ids-matching-target #f (list keynames target res runname testpatt statepatt statuspatt)))
 
 ;; NOTE: This will open and access ALL run databases. 
 ;;

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -217,14 +217,16 @@
 	 (task-key           (conc (hash-table->alist flags) " " (get-host-name) " " (current-process-id)))
 	 (tasks-db           (tasks:open-db)))
 
     (set-signal-handler! signal/int
 			 (lambda (signum)
+			   (signal-mask! signum)
 			   (let ((tdb (tasks:open-db)))
 			     (tasks:set-state-given-param-key tdb task-key "killed")
+			     ;; (sqlite3:interrupt! tdb) ;; seems silly?
 			     (sqlite3:finalize! tdb))
-			   (print "Killed by sigint. Exiting")
+			   (print "Killed by signal " signum ". Exiting")
 			   (exit)))
 
     ;; register this run in monitor.db
     (tasks:add tasks-db "run-tests" user target runname test-patts task-key) ;; params)
     (tasks:set-state-given-param-key tasks-db task-key "running")
@@ -249,18 +251,25 @@
 
     ;; on the first pass or call to run-tests set FAILS to NOT_STARTED if
     ;; -keepgoing is specified
     (if (eq? *passnum* 0)
 	(begin
+	  ;; Is this still necessary? I think not. Unreachable tests are marked as such and 
+	  ;; should not cause problems here.
+	  ;;
 	  ;; have to delete test records where NOT_STARTED since they can cause -keepgoing to 
 	  ;; get stuck due to becoming inaccessible from a failed test. I.e. if test B depends 
 	  ;; on test A but test B reached the point on being registered as NOT_STARTED and test
 	  ;; A failed for some reason then on re-run using -keepgoing the run can never complete.
+	  ;;
+	  ;; (rmt:general-call 'delete-tests-in-state run-id "NOT_STARTED")
+	  
+	  ;; Now convert FAIL and anything in allow-auto-rerun to NOT_STARTED
+	  ;;
 	  (for-each (lambda (state)
-		      (rmt:general-call 'delete-tests-in-state run-id state))
-		    (cons "NOT_STARTED" (string-split (or (configf:lookup *configdat* "setup" "allow-auto-rerun") ""))))
-	  (rmt:set-tests-state-status run-id test-names #f "FAIL" "NOT_STARTED" "FAIL")))
+		      (rmt:set-tests-state-status run-id test-names state #f "NOT_STARTED" state))
+		    (string-split (or (configf:lookup *configdat* "setup" "allow-auto-rerun") "")))))
 
     ;; Ensure all tests are registered in the test_meta table
     (runs:update-all-test_meta #f)
 
     ;; now add non-directly referenced dependencies (i.e. waiton)
@@ -356,12 +365,27 @@
 	(debug:print-info 1 "Adding " required-tests " to the run queue"))
     ;; NOTE: these are all parent tests, items are not expanded yet.
     (debug:print-info 4 "test-records=" (hash-table->alist test-records))
     (let ((reglen (configf:lookup *configdat* "setup" "runqueue")))
       (if (> (length (hash-table-keys test-records)) 0)
-	  (begin
-	    (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry)
+	  (let* ((keep-going #t)
+		 (th1        (make-thread (lambda ()
+					    (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry))
+					  "runs:run-tests-queue"))
+		 (th2        (make-thread (lambda ()				    
+					    ;; (rmt:find-and-mark-incomplete-all-runs))))) CAN'T INTERRUPT IT ...
+					    (let ((run-ids (rmt:get-all-run-ids)))
+					      (for-each (lambda (run-id)
+							  (if keep-going
+							      (rmt:find-and-mark-incomplete run-id #f))) ;; ovr-deadtime)))
+							run-ids)))
+					  "runs: mark-incompletes")))
+	    (thread-start! th1)
+	    (thread-start! th2)
+	    (thread-join! th1)
+	    (set! keep-going #f)
+	    (thread-join! th2)
 	    ;; if run-count > 0 call, set -preclean and -rerun STUCK/DEAD
 	    (if (> run-count 0)
 		(begin
 		  (if (not (hash-table-ref/default flags "-preclean" #f))
 		      (hash-table-set! flags "-preclean" #t))
@@ -475,11 +499,11 @@
 	(let ((items-list (items:get-items-from-config tconfig)))
 	  (if (list? items-list)
 	      (begin
 		(if (null? items-list)
 		    (let ((test-id (rmt:get-test-id run-id test-name "")))
-		      (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "ZERO_ITEMS" "Failed to run due to failed prerequisites")))
+		      (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "ZERO_ITEMS" "Failed to run due to failed prerequisites"))))
 		(tests:testqueue-set-items! test-record items-list)
 		(list hed tal reg reruns))
 	      (begin
 		(debug:print 0 "ERROR: The proc from reading the items table did not yield a list - please report this")
 		(exit 1))))))
@@ -513,11 +537,11 @@
 	    (let ((trimmed-tal (mt:discard-blocked-tests run-id hed tal test-records))
 		  (trimmed-reg (mt:discard-blocked-tests run-id hed reg test-records)))
 	      (debug:print 1 "WARNING: test " hed " has discarded prerequisites, removing it from the queue")
 
 	      (let ((test-id (rmt:get-test-id run-id hed "")))
-		(mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to discarded prerequisites"))
+		(if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to discarded prerequisites")))
 	      
 	      (if (and (null? trimmed-tal)
 		       (null? trimmed-reg))
 		  #f
 		  (list (runs:queue-next-hed trimmed-tal trimmed-reg reglen regfull)
@@ -537,11 +561,11 @@
 	    ;; we use this opportunity to move contents of reg to tal
 	    (list (car newtal)(append (cdr newtal) reg) '() reruns)) ;; an issue with prereqs not yet met?
 	  (begin
 	    (debug:print-info 1 "no fails in prerequisites for " hed " but nothing seen running in a while, dropping test " hed " from the run queue")
 	    (let ((test-id (rmt:get-test-id run-id hed "")))
-	      (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "TIMED_OUT" "Nothing seen running in a while."))
+	      (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "TIMED_OUT" "Nothing seen running in a while.")))
 	    (list (runs:queue-next-hed tal reg reglen regfull)
 		  (runs:queue-next-tal tal reg reglen regfull)
 		  (runs:queue-next-reg tal reg reglen regfull)
 		  reruns))))
 
@@ -551,13 +575,14 @@
        (member 'normal testmode))
       (debug:print-info 1 "test "  hed " (mode=" testmode ") has failed prerequisite(s); "
 			(string-intersperse (map (lambda (t)(conc (db:test-get-testname t) ":" (db:test-get-state t)"/"(db:test-get-status t))) fails) ", ")
 			", removing it from to-do list")
       (let ((test-id (rmt:get-test-id run-id hed "")))
-	(if (not (null? prereq-fails))
-	    (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites")
-	    (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL"      "Failed to run due to failed prerequisites")))
+	(if test-id
+	    (if (not (null? prereq-fails))
+		(mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites")
+		(mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL"      "Failed to run due to failed prerequisites"))))
       (if (or (not (null? reg))(not (null? tal)))
 	  (begin
 	    (hash-table-set! test-registry hed 'CANNOTRUN)
 	    (list (runs:queue-next-hed tal reg reglen regfull)
 		  (runs:queue-next-tal tal reg reglen regfull)
@@ -644,11 +669,11 @@
      
      ;; Register tests 
      ;;
      ((not (hash-table-ref/default test-registry (runs:make-full-test-name test-name item-path) #f))
       (debug:print-info 4 "Pre-registering test " test-name "/" item-path " to create placeholder" )
-      (if (eq? *transport-type* 'fs) ;; no point in parallel registration if use fs
+      (if #t ;; always do firm registration now in v1.60 and greater ;; (eq? *transport-type* 'fs) ;; no point in parallel registration if use fs
 	  (begin
 	    (rmt:general-call 'register-test run-id run-id test-name item-path)
 	    (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'done))
 	  (let ((th (make-thread (lambda ()
 				   (mutex-lock! registry-mutex)
@@ -748,11 +773,11 @@
 	      (if (vector? hed)
 		  (begin
 		    (debug:print 1 "WARNING: Dropping test " test-name "/" item-path
 				 " from the launch list as it has prerequistes that are FAIL")
 		    (let ((test-id (rmt:get-test-id run-id hed "")))
-		      (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL" "Failed to run due to failed prerequisites"))
+		      (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL" "Failed to run due to failed prerequisites")))
 		    (runs:shrink-can-run-more-tests-count) ;; DELAY TWEAKER (still needed?)
 		    ;; (thread-sleep! *global-delta*)
 		    ;; This next is for the items
 		    (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "BLOCKED" #f)
 		    (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'removed)
@@ -887,14 +912,17 @@
 	       (reruns      '()))
 
       (if (not (null? reruns))(debug:print-info 4 "reruns=" reruns))
 
       ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes
+      ;; moving this to a parallel thread and just run it once.
+      ;;
       (if (> (current-seconds)(+ last-time-incomplete 900))
           (begin
             (set! last-time-incomplete (current-seconds))
-            (rmt:find-and-mark-incomplete-all-runs)))
+            ;; (rmt:find-and-mark-incomplete-all-runs)
+	    ))
 
       ;; (print "Top of loop, hed=" hed ", tal=" tal " ,reruns=" reruns)
       (let* ((test-record (hash-table-ref test-records hed))
 	     (test-name   (tests:testqueue-get-testname test-record))
 	     (tconfig     (tests:testqueue-get-testconfig test-record))

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -83,16 +83,17 @@
 ;;
 (define  (server:run run-id)
   (let* ((curr-host   (get-host-name))
 	 (curr-ip     (server:get-best-guess-address curr-host))
 	 (target-host (configf:lookup *configdat* "server" "homehost" ))
+	 (testsuite   (common:get-testsuite-name))
 	 (logfile     (conc *toppath* "/logs/" run-id ".log"))
 	 (cmdln (conc (common:get-megatest-exe)
 		      " -server " (or target-host "-") " -run-id " run-id (if (equal? (configf:lookup *configdat* "server" "daemonize") "yes")
 									      (conc " -daemonize -log " logfile)
 									      "")
-		      " -debug 4 "))) ;; (conc " >> " logfile " 2>&1 &")))))
+		      " -debug 4 testsuite:" testsuite))) ;; (conc " >> " logfile " 2>&1 &")))))
     (debug:print 0 "INFO: Starting server (" cmdln ") as none running ...")
     (push-directory *toppath*)
     (if (not (directory-exists? "logs"))(create-directory "logs"))
     ;; host.domain.tld match host?
     (if (and target-host 
@@ -103,10 +104,12 @@
 	(begin
 	  (debug:print-info 0 "Starting server on " target-host ", logfile is " logfile)
 	  (setenv "TARGETHOST" target-host)))
     (setenv "TARGETHOST_LOGF" logfile)
     (system (conc "nbfake " cmdln))
+    (unsetenv "TARGETHOST_LOGF")
+    (if (get-environment-variable "TARGETHOST")(unsetenv "TARGETHOST"))
     ;; (system cmdln)
     (pop-directory)))
 
 ;; kind start up of servers, wait 40 seconds before allowing another server for a given
 ;; run-id to be launched

Index: tasks.scm
==================================================================
--- tasks.scm
+++ tasks.scm
@@ -24,22 +24,25 @@
 
 ;; wait up to aprox n seconds for a journal to go away
 ;;
 (define (tasks:wait-on-journal path n #!key (remove #f))
   (let ((fullpath (conc path "-journal")))
-    (let loop ((journal-exists (file-exists? fullpath))
-	       (count          n)) ;; wait ten times ...
-      (if journal-exists
-	  (if (> count 0)
-	      (begin
-		(thread-sleep! 1)
-		(loop (file-exists? fullpath)
-		      (- count 1)))
-	      (begin
-		(if remove (system (conc "rm -rf " path)))
-		#f))
-	  #t))))
+    (handle-exceptions
+     exn
+     #t ;; if stuff goes wrong just allow it to move on
+     (let loop ((journal-exists (file-exists? fullpath))
+		(count          n)) ;; wait ten times ...
+       (if journal-exists
+	   (if (> count 0)
+	       (begin
+		 (thread-sleep! 1)
+		 (loop (file-exists? fullpath)
+		       (- count 1)))
+	       (begin
+		 (if remove (system (conc "rm -rf " path)))
+		 #f))
+	   #t)))))
 
 (define (tasks:get-task-db-path)
   (if *task-db*
       (vector-ref *task-db* 1)
       (let* ((linktree     (configf:lookup *configdat* "setup" "linktree"))
@@ -321,12 +324,28 @@
 ;; no elegance here ...
 ;;
 (define (tasks:kill-server hostname pid)
   (debug:print-info 0 "Attempting to kill server process " pid " on host " hostname)
   (setenv "TARGETHOST" hostname)
-  (system (conc "nbfake kill " pid)))
+  (setenv "TARGETHOST_LOGF" "server-kills.log")
+  (system (conc "nbfake kill " pid))
+  (unsetenv "TARGETHOST_LOGF")
+  (unsetenv "TARGETHOST"))
  
+;; look up a server by run-id and send it a kill, also delete the record for that server
+;;
+(define (tasks:kill-server-run-id run-id)
+  (let* ((tdb  (tasks:open-db))
+	 (sdat (tasks:get-server mdb run-id)))
+    (if sdat
+	(let ((hostname (vector-ref sdat 6))
+	      (pid      (vector-ref sdat 5)))
+	  (debug:print-info 0 "Killing server for run-id " run-id " on host " hostname " with pid " pid)
+	  (tasks:kill-server hostname pid)
+	  (tasks:server-delete-record mdb server-id tag) )
+	(debug:print-info 0 "No server found for run-id " run-id ", nothing to kill"))))
+    
 ;;   (if status ;; #t means alive
 ;;       (begin
 ;; 	(if (equal? hostname (get-host-name))
 ;; 	    (handle-exceptions
 ;; 	     exn
@@ -628,27 +647,35 @@
 	(debug:print 0 "No run launching processes found for " target " / " run-name)
 	(debug:print 0 "Found " (length records) " run(s) to kill."))
     (for-each 
      (lambda (record)
        (let* ((param-key (list-ref record 8))
-	      (match-dat (string-search hostpid-rx param-key))
-	      (hostname  (cadr match-dat))
-	      (pid       (caddr match-dat)))
-	 (debug:print 0 "Sending SIGINT to process " pid " on host " hostname)
-	 (if (equal? (get-host-name) hostname)
-	     (begin
-	       (process-signal (string->number pid) signal/int)
-	       (thread-sleep! 5)
-	       (handle-exceptions
-		exn
-		#t
-		(process-signal (string->number pid) signal/kill)))
-	     ;;  (call-with-environment-variables
-	     (let ((old-targethost (getenv "TARGETHOST")))
-	       (set-environment-variable "TARGETHOST" hostname)
-	       (system (conc "nbfake " kill " " pid))
-	       (if old-targethost (set-environment-variable "TARGETHOST" old-targethost))))))
+	      (match-dat (string-search hostpid-rx param-key)))
+	 (if match-dat
+	     (let ((hostname  (cadr match-dat))
+		   (pid       (string->number (caddr match-dat))))
+	       (debug:print 0 "Sending SIGINT to process " pid " on host " hostname)
+	       (if (equal? (get-host-name) hostname)
+		   (if (process:alive? pid)
+		       (begin
+			 (handle-exceptions
+			  exn
+			  (begin
+			    (debug:print 0 "Kill of process " pid " on host " hostname " failed.")
+			    (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+			    #t)
+			  (process-signal pid signal/int)
+			  (thread-sleep! 5)
+			  (if (process:alive? pid)
+			      (process-signal pid signal/kill)))))
+		   ;;  (call-with-environment-variables
+		   (let ((old-targethost (getenv "TARGETHOST")))
+		     (setenv "TARGETHOST" hostname)
+		     (system (conc "nbfake kill " pid))
+		     (if old-targethost (setenv "TARGETHOST" old-targethost))
+		     (unsetenv "TARGETHOST"))))
+	     (debug:print 0 "ERROR: no record or improper record for " target "/" run-name " in tasks_queue in monitor.db"))))
      records)))
 
 
 ;;======================================================================
 ;; The routines to process tasks

Index: tdb.scm
==================================================================
--- tdb.scm
+++ tdb.scm
@@ -70,11 +70,12 @@
 	(debug:print-info 11 "open-test-db END (sucessful)" work-area)
 	;; now let's test that everything is correct
 	(handle-exceptions
 	 exn
 	 (begin
-	   (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test"
+	   (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test or remove the file " 
+			dbpath ".\n  "
 			((condition-property-accessor 'exn 'message) exn))
 	   #f)
 	 ;; Is there a cheaper single line operation that will check for existance of a table
 	 ;; and raise an exception ?
 	 (sqlite3:execute db "SELECT id FROM test_data LIMIT 1;"))

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -174,11 +174,11 @@
 clean  : 
 	rm cleanprep
 
 kill :
 	killall -v mtest main.sh dboard || true
-	rm -rf *run/db/* */megatest.db */logging.db */monitor.db fullrun/tmp/mt_*/* fullrun/tmp/mt_*/.db* fullrun/logs/*.log || true
+	rm -rf *run/db/* */megatest.db */logging.db */monitor.db fullrun/tmp/mt_*/* fullrun/tmp/mt_*/.db* fullrun/logs/*.log fullrun/*.log || true
 	killall -v mtest dboard || true
 
 hardkill : kill
 	sleep 2;killall -v mtest main.sh dboard -9
 

Index: tests/fullrun/runconfigs.config
==================================================================
--- tests/fullrun/runconfigs.config
+++ tests/fullrun/runconfigs.config
@@ -17,10 +17,11 @@
 WACKYVAR2 #{runconfigs-get CURRENT}
 
 [ubuntu/nfs/none]
 WACKYVAR2 #{runconfigs-get CURRENT}
 SOMEVAR2  This should show up in SOMEVAR4 if the target is ubuntu/nfs/none
+VARWITHDOLLARSIGNS The$USER/signs/should/be/replaced/with/variable
 
 [default]
 SOMEVAR3 #{rget SOMEVAR}
 SOMEVAR4 #{rget SOMEVAR2}
 SOMEVAR5 #{runconfigs-get SOMEVAR2}

Index: tests/fullrun/tests/exit_0/testconfig
==================================================================
--- tests/fullrun/tests/exit_0/testconfig
+++ tests/fullrun/tests/exit_0/testconfig
@@ -6,5 +6,10 @@
 owner  bob
 description This test checks that a multi-lineitem test with mix of pass and non-fail rolls up a PASS
 
 tags first,single
 reviewed 09/10/2011, by Matt
+
+[triggers]
+NOT_STARTED/ xterm -e bash -s -- 
+RUNNING/ xterm -e bash -s -- 
+

ADDED   tests/fullrun/tests/test_mt_vars/eval_vars.sh
Index: tests/fullrun/tests/test_mt_vars/eval_vars.sh
==================================================================
--- /dev/null
+++ tests/fullrun/tests/test_mt_vars/eval_vars.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if env | grep VARWITHDOLLARSIGNS | grep USER;then
+    exit 1 # fails!
+else
+    exit 0 # good!
+fi

Index: tests/fullrun/tests/test_mt_vars/testconfig
==================================================================
--- tests/fullrun/tests/test_mt_vars/testconfig
+++ tests/fullrun/tests/test_mt_vars/testconfig
@@ -18,10 +18,13 @@
 # VACKYVAR should be set to a path
 vackyvar vackyvar.sh
 
 # test-path and test-file
 test-path test-path-file.sh
+
+# verify that vars with $ signs get expanded
+varwithdollar eval_vars.sh
 
 [requirements]
 waiton runfirst
 priority 0
 

Index: utils/nbfake
==================================================================
--- utils/nbfake
+++ utils/nbfake
@@ -1,20 +1,76 @@
 #!/bin/bash
+###############################################################################
+#
+# nbfake - capture command output in a logfile
+#
+# nbfake behavior can be changed by setting the following env vars:
+#   NBFAKE_HOST       SSH to $NBFAKE_HOST and run command
+#   NBFAKE_LOG        Logfile for nbfake output
+#
+###############################################################################
+
+if [[ -z "$@" ]]; then
+  cat <<__EOF
+
+nbfake usage:
+
+nbfake <command to run>
+
+nbfake behavior can be changed by setting the following env vars:
+   NBFAKE_HOST       SSH to \$NBFAKE_HOST and run command
+   NBFAKE_LOG        Logfile for nbfake output
+
+__EOF
+  exit
+fi
+
+#==============================================================================
+# Setup
+#==============================================================================
 
 # Can't always trust $PWD
-CURRWD=`pwd`
-if [[ $TARGETHOST_LOGF == "" ]]; then
-    TARGETHOST_LOGF=NBFAKE-`date +%GWW%V.%u_%T`
-fi
-echo "#======================================================================"
-echo "# NBFAKE Running command:"
-echo "#     \"$*\""
-echo "#======================================================================"
-
-if [[ $TARGETHOST == ""  ]]; then
+CURRWD=$(pwd)
+
+# Make sure nbfake host and logfile are set. Fall back to old-style variable names
+
+if [[ -z "$NBFAKE_HOST" && -n "$TARGETHOST" ]]; then 
+  MY_NBFAKE_HOST=$TARGETHOST     
   unset TARGETHOST
-  TARGETHOST_LOGF_TEMP=$TARGETHOST_LOGF
+else
+  MY_NBFAKE_HOST=$NBFAKE_HOST
+  unset NBFAKE_HOST
+fi
+
+
+if [[ -z "$NBFAKE_LOG" && -n "$TARGETHOST_LOGF" ]]; then 
+  MY_NBFAKE_LOG=$TARGETHOST_LOGF
   unset TARGETHOST_LOGF
-  sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF_TEMP 2>&1 &"
+else
+  MY_NBFAKE_LOG=$NBFAKE_LOG
+  unset NBFAKE_LOG
+fi
+
+# Set default nbfake log
+
+if [[ -z "$MY_NBFAKE_LOG" ]]; then
+  MY_NBFAKE_LOG=NBFAKE-$(date +%GWW%V.%u_%T)
+fi
+
+#==============================================================================
+# Run and log
+#==============================================================================
+
+cat <<__EOF >&2
+#======================================================================
+# NBFAKE logging command to: $MY_NBFAKE_LOG
+#     $*
+#======================================================================
+__EOF
+
+if [[ -z "$MY_NBFAKE_HOST" ]]; then
+  # Run locally
+  sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &"
 else
-  ssh -n -f $TARGETHOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF 2>&1 &\""
+  # run remotely
+  ssh -n -f $MY_NBFAKE_HOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &\""
 fi