Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -131,10 +131,14 @@
 	chmod a+x $@
 
 $(PREFIX)/bin/nbfake : utils/nbfake
 	$(INSTALL) $< $@
 	chmod a+x $@
+
+$(PREFIX)/bin/remrun : utils/remrun
+	$(INSTALL) $< $@
+	chmod a+x $@
 
 $(PREFIX)/bin/viewscreen : utils/viewscreen
 	$(INSTALL) $< $@
 	chmod a+x $@
 
@@ -169,11 +173,11 @@
 	$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard
 
 install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
           $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
 	  $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \
-	  $(PREFIX)/share/docs/megatest_manual.html 
+	  $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun
 
 $(PREFIX)/bin/.$(ARCHSTR) : 
 	mkdir -p $(PREFIX)/bin/.$(ARCHSTR)
 
 test: tests/tests.scm
@@ -211,11 +215,11 @@
 #	chicken-install -prefix deploytarg -deploy $$i;done
 
 # deploytarg/libsqlite3.so : 
 # 	CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3
 
-deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/viewsceen deploytarg/nbfind deploytarg/apropos.so
+deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/remrun deploytarg/viewsceen deploytarg/nbfind deploytarg/apropos.so
 
 # deploytarg/libiupcd.so : $(CKPATH)/lib/libiupcd.so
 # 	for i in iup im cd av call sqlite; do \
 # 	  cp $(CKPATH)/lib/lib$$i* deploytarg/ ; \
 # 	done

Index: NOTES
==================================================================
--- NOTES
+++ NOTES
@@ -1,13 +1,19 @@
+=====================================================================
+NOTES from looking at branch v1.62-rpc
+=====================================================================
+
+*last-db-access* or *db-last-access* ==> which is it to be?
+seen in singletest: ERROR: Unrecognised arguments: :first_err This is the first error
 
 ======================================================================
 New way of launching needed to accomodate different target hosttypes
 for items
 ======================================================================
 
 [flavors]
-general ssh #{getbgesthost general}
+general ssh #{getbesthost general}
 nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
 
 [hosts]
 general cubian xena
 

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -131,28 +131,37 @@
 (define *toptest-paths*     (make-hash-table)) ;; cache toptest path settings here
 (define *test-paths*        (make-hash-table)) ;; cache test-id to test run paths here
 (define *test-ids*          (make-hash-table)) ;; cache run-id, testname, and item-path => test-id
 (define *test-info*         (make-hash-table)) ;; cache the test info records, update the state, status, run_duration etc. from testdat.db
 
-(define *run-info-cache*    (make-hash-table)) ;; run info is stable, no need to reget
+(define *run-info-cache*     (make-hash-table)) ;; run info is stable, no need to reget
 (define *launch-setup-mutex* (make-mutex))     ;; need to be able to call launch:setup often so mutex it and re-call the real deal only if *toppath* not set
 (define *homehost-mutex*     (make-mutex))
 
+;; launching and hosts
+(defstruct host
+  (reachable    #f)
+  (last-update  0)
+  (last-used    0)
+  (last-cpuload 1))
+
+(define *host-loads*         (make-hash-table))
+
 ;; cache environment vars for each run here
 (define *env-vars-by-run-id* (make-hash-table))
 
 ;; Testconfig and runconfig caches. 
-(define *testconfigs*       (make-hash-table)) ;; test-name => testconfig
-(define *runconfigs*        (make-hash-table)) ;; target    => runconfig
+(define *testconfigs*        (make-hash-table)) ;; test-name => testconfig
+(define *runconfigs*         (make-hash-table)) ;; target    => runconfig
 
 ;; This is a cache of pre-reqs met, don't re-calc in cases where called with same params less than
 ;; five seconds ago
 (define *pre-reqs-met-cache* (make-hash-table))
 
 ;; cache of verbosity given string
 ;;
-(define *verbosity-cache* (make-hash-table))
+(define *verbosity-cache*    (make-hash-table))
 
 (define (common:clear-caches)
   (set! *target*             (make-hash-table))
   (set! *keys*               (make-hash-table))
   (set! *keyvals*            (make-hash-table))
@@ -1073,10 +1082,109 @@
 	   (with-input-from-pipe 
 	    (conc "ssh " remote-host " cat /proc/loadavg")
 	    (lambda ()(list (read)(read)(read)))))
       (with-input-from-file "/proc/loadavg" 
 	(lambda ()(list (read)(read)(read))))))
+
+;; get normalized cpu load by reading from /proc/loadavg and /proc/cpuinfo return all three values and the number of real cpus and the number of threads
+;; returns list (normalized-proc-load normalized-core-load 1m 5m 15m ncores nthreads)
+;;
+(define (common:get-normalized-cpu-load remote-host)
+  (let ((data (if remote-host
+                  (with-input-from-pipe 
+                   (conc "ssh " remote-host " cat /proc/loadavg;cat /proc/cpuinfo;echo end")
+                   read-lines)
+                  (append 
+                   (with-input-from-file "/proc/loadavg" 
+                     read-lines)
+                   (with-input-from-file "/proc/cpuinfo"
+                     read-lines)
+                   (list "end"))))
+        (load-rx  (regexp "^([\\d\\.]+)\\s+([\\d\\.]+)\\s+([\\d\\.]+)\\s+.*$"))
+        (proc-rx  (regexp "^processor\\s+:\\s+(\\d+)\\s*$"))
+        (core-rx  (regexp "^core id\\s+:\\s+(\\d+)\\s*$"))
+        (phys-rx  (regexp "^physical id\\s+:\\s+(\\d+)\\s*$"))
+        (max-num  (lambda (p n)(max (string->number p) n))))
+    ;; (print "data=" data)
+    (if (null? data) ;; something went wrong
+        #f
+        (let loop ((hed      (car data))
+                   (tal      (cdr data))
+                   (loads    #f)
+                   (proc-num 0)  ;; processor includes threads
+                   (phys-num 0)  ;; physical chip on motherboard
+                   (core-num 0)) ;; core
+          ;; (print hed ", " loads ", " proc-num ", " phys-num ", " core-num)
+          (if (null? tal) ;; have all our data, calculate normalized load and return result
+              (let* ((act-proc (+ proc-num 1))
+                     (act-phys (+ phys-num 1))
+                     (act-core (+ core-num 1))
+                     (adj-proc-load (/ (car loads) act-proc))
+                     (adj-core-load (/ (car loads) act-core)))
+                (append (list (cons 'adj-proc-load adj-proc-load)
+                              (cons 'adj-core-load adj-core-load))
+                        (list (cons '1m-load (car loads))
+                              (cons '5m-load (cadr loads))
+                              (cons '15m-load (caddr loads)))
+                        (list (cons 'proc act-proc)
+                              (cons 'core act-core)
+                              (cons 'phys act-phys))))
+              (regex-case
+               hed
+               (load-rx  ( x l1 l5 l15 ) (loop (car tal)(cdr tal)(map string->number (list l1 l5 l15)) proc-num phys-num core-num))
+               (proc-rx  ( x p         ) (loop (car tal)(cdr tal) loads           (max-num p proc-num) phys-num core-num))
+               (phys-rx  ( x p         ) (loop (car tal)(cdr tal) loads           proc-num (max-num p phys-num) core-num))
+               (core-rx  ( x c         ) (loop (car tal)(cdr tal) loads           proc-num phys-num (max-num c core-num)))
+               (else 
+                (begin
+                  ;; (print "NO MATCH: " hed)
+                  (loop (car tal)(cdr tal) loads proc-num phys-num core-num)))))))))
+
+(define (common:unix-ping hostname)
+  (let ((res (system (conc "ping -c 1 " hostname " > /dev/null"))))
+    (eq? res 0)))
+
+;; ideally put all this info into the db, no need to preserve it across moving homehost
+;;
+(define (common:get-least-loaded-host hosts)
+  (if (null? hosts)
+      #f
+      ;;
+      ;; stategy:
+      ;;    sort by last-used and normalized-load
+      ;;    if last-updated > 15 seconds then re-update
+      ;;    take the host with the lowest load with the lowest last-used (i.e. not used for longest time)
+      ;;
+      (let ((best-host #f)
+	    (curr-time (current-seconds)))
+	(for-each
+	 (lambda (hostname)
+	   (let* ((rec       (let ((h (hash-table-ref/default *host-loads* hostname #f)))
+			       (if h
+				   h
+				   (let ((h (make-host)))
+				     (hash-table-set! *host-loads* hostname h)
+				     h))))
+		  ;; if host hasn't been pinged in 15 sec update it's data
+		  (ping-good (if (< (- curr-time (host-last-update rec)) 15)
+				 (host-reachable rec)
+				 (or (host-reachable rec)
+				     (begin
+				       (host-reachable-set! rec (common:unix-ping hostname))
+				       (host-last-update-set! rec curr-time)
+				       (host-last-cpuload-set! rec (common:get-normalized-cpu-load hostname))
+				       (host-reachable rec))))))
+	     (cond
+	      ((not best-host)
+	       (set! best-host hostname))
+	      ((and ping-good
+		    (< (alist-ref 'adj-core-load (host-last-cpuload rec))
+		       (alist-ref 'adj-core-load
+				  (host-last-cpuload (hash-table-ref *host-loads* best-host)))))
+	       (set! best-host rec)))))
+	 hosts)
+	best-host)))
 
 (define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000) (msg #f)(remote-host #f))
   (let* ((loadavg (common:get-cpu-load remote-host))
 	 (first   (car loadavg))
 	 (next    (cadr loadavg))
@@ -1575,28 +1683,30 @@
     
 ;;======================================================================
 ;;  T E S T   L A U N C H I N G   P E R   I T E M   W I T H   H O S T   T Y P E S
 ;;======================================================================
 ;; 
-;; [host-types]
-;; general ssh #{getbgesthost general}
-;; nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+;; [hosts]
+;; arm cubie01 cubie02
+;; x86_64 zeus xena myth01
+;; allhosts #{g hosts arm} #{g hosts x86_64}
 ;; 
-;; [hosts]
-;; general cubian xena
+;; [host-types]
+;; general #MTLOWESTLOAD #{g hosts allhosts}
+;; arm     #MTLOWESTLOAD #{g hosts arm}
+;; nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
 ;; 
 ;; [launchers]
 ;; envsetup general
 ;; xor/%/n 4C16G
 ;; % nbgeneral
 ;; 
 ;; [jobtools]
-;; launcher bsub
-;; # if defined and not "no" flexi-launcher will bypass launcher unless there is no
-;; # match.
+;; # if defined and not "no" flexi-launcher will bypass "launcher" unless no match.
 ;; flexi-launcher yes  
-
+;; launcher nbfake
+;;
 (define (common:get-launcher configdat testname itempath)
   (let ((fallback-launcher (configf:lookup configdat "jobtools" "launcher")))
     (if (and (configf:lookup configdat "jobtools" "flexi-launcher") ;; overrides launcher
 	     (not (equal? (configf:lookup configdat "jobtools" "flexi-launcher") "no")))
 	(let* ((launchers         (hash-table-ref/default configdat "launchers" '())))
@@ -1609,11 +1719,16 @@
 		  (if (tests:match patt testname itempath)
 		      (begin
 			(debug:print-info 2 *default-log-port* "Have flexi-launcher match for " testname "/" itempath " = " host-type)
 			(let ((launcher (configf:lookup configdat "host-types" host-type)))
 			  (if launcher
-			      launcher
+			      (let* ((launcher-parts (string-split launcher))
+				     (launcher-exe   (car launcher-parts)))
+				(if (equal? launcher-exe "#MTLOWESTLOAD") ;; this is our special case, we will find the lowest load and craft a nbfake commandline
+				    (let ((targ-host (common:get-least-loaded-host (cdr launcher-parts))))
+				      (conc "remrun " targ-host))
+				    launcher))
 			      (begin
 				(debug:print-info 0 *default-log-port* "WARNING: no launcher found for host-type " host-type)
 				(if (null? tal)
 				    fallback-launcher
 				    (loop (car tal)(cdr tal)))))))

Index: configf.scm
==================================================================
--- configf.scm
+++ configf.scm
@@ -57,10 +57,11 @@
 ;;======================================================================
 ;; Make the regexp's needed globally available
 ;;======================================================================
 
 (define configf:include-rx (regexp "^\\[include\\s+(.*)\\]\\s*$"))
+(define configf:script-rx  (regexp "^\\[scriptinc\\s+(.*)\\]\\s*$")) ;; include output from a script
 (define configf:section-rx (regexp "^\\[(.*)\\]\\s*$"))
 (define configf:blank-l-rx (regexp "^\\s*$"))
 (define configf:key-sys-pr (regexp "^(\\S+)\\s+\\[system\\s+(\\S+.*)\\]\\s*$"))
 (define configf:key-val-pr (regexp "^(\\S+)(\\s+(.*)|())$"))
 (define configf:key-no-val (regexp "^(\\S+)(\\s*)$"))
@@ -68,11 +69,11 @@
 (define configf:cont-ln-rx (regexp "^(\\s+)(\\S+.*)$"))
 (define configf:settings   (regexp "^\\[configf:settings\\s+(\\S+)\\s+(\\S+)]\\s*$"))
 
 ;; read a line and process any #{ ... } constructs
 
-(define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget)\\s+([^\\}\\{]*)\\}(.*)"))
+(define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget|scm|sh|rp|gv|g|mtrah)\\s+([^\\}\\{]*)\\}(.*)"))
 
 (define (configf:process-line l ht allow-system #!key (linenum #f))
   (let loop ((res l))
     (if (string? res)
 	(let ((matchdat (string-search configf:var-expand-regex res)))
@@ -83,36 +84,42 @@
 		     (poststr (list-ref matchdat 4))
 		     (result  #f)
 		     (start-time (current-seconds))
 		     (cmdsym  (string->symbol cmdtype))
 		     (fullcmd (case cmdsym
-				((scheme)(conc "(lambda (ht)" cmd ")"))
-				((system)(conc "(lambda (ht)(system \"" cmd "\"))"))
-				((shell) (conc "(lambda (ht)(shell \""  cmd "\"))"))
-				((getenv)(conc "(lambda (ht)(get-environment-variable \"" cmd "\"))"))
-				((get)   
+				((scheme scm) (conc "(lambda (ht)" cmd ")"))
+				((system)     (conc "(lambda (ht)(system \"" cmd "\"))"))
+				((shell sh)   (conc "(lambda (ht)(string-translate (shell \""  cmd "\") \"\n\" \" \"))"))
+				((realpath rp)(conc "(lambda (ht)(common:nice-path \"" cmd "\"))"))
+				((getenv gv)  (conc "(lambda (ht)(get-environment-variable \"" cmd "\"))"))
+				((mtrah)      (conc "(lambda (ht)"
+                                                    "    (let ((extra \"" cmd "\"))"
+						    "       (conc (or *toppath* (get-environment-variable \"MT_RUN_AREA_HOME\"))"
+						    "             (if (string-null? extra) \"\" \"/\")"
+						    "             extra)))"))
+				((get g)   
 				 (let* ((parts (string-split cmd))
 					(sect  (car parts))
 					(var   (cadr parts)))
 				   (conc "(lambda (ht)(config-lookup ht \"" sect "\" \"" var "\"))")))
-				((runconfigs-get) (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
-				((rget)           (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
+				((runconfigs-get rget) (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
+				;; ((rget)           (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))"))
 				(else "(lambda (ht)(print \"ERROR\") \"ERROR\")"))))
 		;; (print "fullcmd=" fullcmd)
 		(handle-exceptions
 		 exn
 		 (begin
 		   (debug:print 0 *default-log-port* "WARNING: failed to process config input \"" l "\"")
 		   (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
 		   ;; (print "exn=" (condition->list exn))
-		   (set! result (conc "#{( " cmdtype ") " cmd"}")))
+		   (set! result (conc "#{( " cmdtype ") " cmd "}, full expansion: " fullcmd)))
 		 (if (or allow-system
-			 (not (member cmdtype '("system" "shell"))))
+			 (not (member cmdtype '("system" "shell" "sh"))))
 		     (with-input-from-string fullcmd
 		       (lambda ()
 			 (set! result ((eval (read)) ht))))
-		    (set! result (conc "#{(" cmdtype ") "  cmd "}"))))
+		     (set! result (conc "#{(" cmdtype ") "  cmd "}"))))
 		(case cmdsym
 		  ((system shell scheme)
 		   (let ((delta (- (current-seconds) start-time)))
 		     (if (> delta 2)
 			 (debug:print-info 0 *default-log-port* "for line \"" l "\"\n command:  " cmd " took " delta " seconds to run with output:\n   " result)
@@ -182,16 +189,19 @@
 ;; post-section-procs alist of section-pattern => proc, where: (proc section-name next-section-name ht curr-path)
 ;;
 (define (read-config path ht allow-system #!key (environ-patt #f)(curr-section #f)(sections #f)(settings (make-hash-table))(keep-filenames #f)(post-section-procs '()))
   (debug:print-info 5 *default-log-port* "read-config " path " allow-system " allow-system " environ-patt " environ-patt " curr-section: " curr-section " sections: " sections " pwd: " (current-directory))
   (debug:print 9 *default-log-port* "START: " path)
-  (if (not (file-exists? path))
+  (if (and (not (port? path))
+	   (not (file-exists? path))) ;; for case where we are handed a port
       (begin 
 	(debug:print-info 1 *default-log-port* "read-config - file not found " path " current path: " (current-directory))
 	;; WARNING: This is a risky change but really, we should not return an empty hash table if no file read?
 	#f) ;; (if (not ht)(make-hash-table) ht))
-      (let ((inp        (open-input-file path))
+      (let ((inp        (if (string? path)
+			    (open-input-file path)
+			      path)) ;; we can be handed a port
 	    (res        (if (not ht)(make-hash-table) ht))
 	    (metapath   (if (or (debug:debug-mode 9)
 				keep-filenames)
 			    path #f)))
 	(let loop ((inl               (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings)) ;; (read-line inp))
@@ -199,11 +209,12 @@
 		   (var-flag #f);; turn on for key-var-pr and cont-ln-rx, turn off elsewhere
 		   (lead     #f))
 	  (debug:print-info 8 *default-log-port* "curr-section-name: " curr-section-name " var-flag: " var-flag "\n   inl: \"" inl "\"")
 	  (if (eof-object? inl) 
 	      (begin
-		(close-input-port inp)
+		(if (string? path) ;; we received a path, not a port, thus we are responsible for closing it.
+		    (close-input-port inp))
 		(hash-table-delete! res "") ;; we are using "" as a dumping ground and must remove it before returning the ht
 		(debug:print 9 *default-log-port* "END: " path)
 		res)
 	      (regex-case 
 	       inl 
@@ -229,10 +240,26 @@
 							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
 							    (begin
 							      (debug:print '(2 9) #f "INFO: include file " include-file " not found (called from " path ")")
 							      (debug:print 2 *default-log-port* "        " full-conf)
 							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))))
+	       (configf:script-rx ( x include-script );; handle-exceptions
+						      ;;    exn
+						      ;;    (begin
+						      ;;      (debug:print '(0 2 9) #f "INFO: include from script " include-script " failed.")
+						      ;;      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
+							 (if (and (file-exists? include-script)(file-execute-access? include-script))
+							     (let* ((new-inp-port (open-input-pipe include-script)))
+							       (debug:print '(2 9) *default-log-port* "Including from script output: " include-script)
+							      ;;  (print "We got here, calling read-config next. Port is: " new-inp-port)
+							       (read-config new-inp-port res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
+							       (close-input-port new-inp-port)
+							       (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
+							     (begin
+							       (debug:print 0 *default-log-port* "Script not found or not exectutable: " include-script)
+							       (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))
+							 ) ;; )
 	       (configf:section-rx ( x section-name ) (begin
 							;; call post-section-procs
 							(for-each 
 							 (lambda (dat)
 							   (let ((patt (car dat))

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -193,11 +193,11 @@
 ;;  (or (configf:lookup *configdat* "setup" "dbdir")
 ;;      (conc (configf:lookup *configdat* "setup" "linktree") "/.db")))
 	       
 (define (db:set-sync db)
   (let ((syncprag (configf:lookup *configdat* "setup" "sychronous")))
-    (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 1) ";")))) 
+    (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 0) ";")))) 
 
 ;; open an sql database inside a file lock
 ;; returns: db existed-prior-to-opening
 ;; RA => Returns a db handler; sets the lock if opened in writable mode
 ;;
@@ -211,11 +211,11 @@
     (if file-write ;; dir-writable
 	(let (;; (lock    (obtain-dot-lock fname 1 5 10))
 	      (db      (sqlite3:open-database fname)))
 	  (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
 	  ;; (db:set-sync db)
-	  (sqlite3:execute db "PRAGMA synchronous = NORMAL;")
+	  (sqlite3:execute db "PRAGMA synchronous = 0;")
 	  (if (not file-exists)
 	      (begin
 		(if (string-match "^/tmp/.*" fname) ;; this is a file in /tmp
 		    (sqlite3:execute db "PRAGMA journal_mode=WAL;")
 		    (print "Creating " fname " in NON-WAL mode."))
@@ -739,11 +739,11 @@
 
 ;; Add db direct
 ;;
 (define (db:dispatch-query access-mode rmt-cmd db-cmd . params)
   (if (eq? access-mode 'cached)
-      (print "not doing cached calls right now"))
+      (debug:print 2 *default-log-port* "not doing cached calls right now"))
 ;;      (apply db:call-with-cached-db db-cmd params)
       (apply rmt-cmd params))
 ;;)
 
 ;; return the target db handle so it can be used
@@ -1499,13 +1499,14 @@
       (if (> (length all-ids) 0)
 	  (begin
 	    (debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ") " as INCOMPLETE")
 	    (sqlite3:execute 
 	     db
-	     (conc "UPDATE tests SET state='INCOMPLETE' WHERE id IN (" 
+	     (conc "UPDATE tests SET state='INCOMPLETE' WHERE run_id=? AND id IN (" 
 		   (string-intersperse (map conc all-ids) ",")
-		   ");")))))
+		   ");")
+             run-id))))
 
     ;; Now do rollups for the toplevel tests
     ;;
     ;; (db:delay-if-busy dbdat)
     (for-each
@@ -3054,23 +3055,26 @@
        (set! row-ids (cons rid row-ids)))
      runsqry)
     (sqlite3:finalize! runsqry)
     row-ids))
 
+;; finds latest matching all patts for given run-id
+;;
 (define (db:test-get-paths-matching-keynames-target-new dbstruct run-id keynames target res testpatt statepatt statuspatt runname)
   (let* ((testqry (tests:match->sqlqry testpatt))
-	 (tstsqry (conc "SELECT rundir FROM tests WHERE " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")))
+	 (tstsqry (conc "SELECT rundir FROM tests WHERE run_id=? AND " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")))
     (db:with-db
      dbstruct
      run-id
      #f
      (lambda (db)
        (sqlite3:for-each-row 
 	(lambda (p)
 	  (set! res (cons p res)))
 	db
-	tstsqry)
+	tstsqry
+	run-id)
        res))))
 
 (define (db:test-toplevel-num-items dbstruct run-id testname)
   (db:with-db
    dbstruct
@@ -3310,10 +3314,11 @@
 	'(delete-tests-in-state   ;; "DELETE FROM tests WHERE state=?;")                  ;; DONE
 	  "UPDATE tests SET state='DELETED' WHERE state=?")
 	'(tests:test-set-toplog   "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path='';")
 	'(update-cpuload-diskfree "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;") ;; DONE
 	'(update-uname-host       "UPDATE tests SET uname=?,host=? WHERE id=?;")       ;; DONE
+        '(update-test-rundat      "INSERT INTO test_rundat (test_id,update_time,cpuload,diskfree,diskusage,run_duration) VALUES (?,?,?,?,?,?);")
 	'(update-test-state       "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
 	'(update-test-status      "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
 	;; stuff for roll-up-pass-fail-counts
 	'(update-pass-fail-counts "UPDATE tests 
              SET fail_count=(SELECT count(id) FROM tests WHERE testname=? AND item_path != '' AND status IN ('FAIL','CHECK','INCOMPLETE','ABORT')),

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="generator" content="AsciiDoc 8.6.7">
+<meta name="generator" content="AsciiDoc 8.6.9">
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -84,14 +84,20 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-pre {
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
   padding: 0;
   margin: 0;
 }
+pre {
+  white-space: pre-wrap;
+}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -216,11 +222,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; }
+span.image img { border-style: none; vertical-align: text-bottom; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -412,16 +418,10 @@
 /*
  * xhtml11 specific
  *
  * */
 
-tt {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 div.tableblock > table {
@@ -451,16 +451,10 @@
 /*
  * html5 specific
  *
  * */
 
-.monospaced {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 thead, p.tableblock.header {
@@ -536,10 +530,12 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
+
+
 @media screen {
   body {
     max-width: 50em; /* approximately 80 characters wide */
     margin-left: 16em;
   }
@@ -1325,11 +1321,78 @@
 </div>
 <div class="sect1">
 <h2 id="_reference">Reference</h2>
 <div class="sectionbody">
 <div class="sect2">
-<h3 id="_megatest_config_file_settings">Megatest Config File Settings</h3>
+<h3 id="_config_file_helpers">Config File Helpers</h3>
+<div class="paragraph"><p>Various helpers for more advanced config files.</p></div>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:80%;
+">
+<caption class="title">Table 2. Helpers</caption>
+<col style="width:14%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<thead>
+<tr>
+<th class="tableblock halign-center valign-top" >Helper                      </th>
+<th class="tableblock halign-left valign-top" > Purpose                       </th>
+<th class="tableblock halign-left valign-top" > Valid values            </th>
+<th class="tableblock halign-left valign-top" > Comments</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{scheme (scheme code&#8230;)}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute arbitrary scheme code</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid scheme</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Value returned from the call is converted to a string and processed as part of the config file</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{system command}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute program, inserts exit code</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid Unix command</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Discards the output from the program</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{shell  command} or #{sh &#8230;}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute program, inserts result from stdout</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid Unix command</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Value returned from the call is converted to a string and processed as part of the config file</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{realpath path} or #{rp &#8230;}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with normalized path</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Must be a valid path</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{getenv VAR} or #{gv VAR}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with content of env variable</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Must be a valid var</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{get s v} or #{g s v}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with variable v from section s</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Variable must be defined before use</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{rget v}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with variable v from target or default of runconfigs file</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect2">
+<h3 id="_config_file_settings">Config File Settings</h3>
+<div class="paragraph"><p>Settings in megatest.config</p></div>
 <div class="sect3">
 <h4 id="_disk_space_checks">Disk Space Checks</h4>
 <div class="paragraph"><p>Some parameters you can put in the [setup] section of megatest.config:</p></div>
 <div class="listingblock">
 <div class="content monospaced">
@@ -1448,11 +1511,11 @@
 <h3 id="_database_settings">Database settings</h3>
 <table class="tableblock frame-topbot grid-all"
 style="
 width:70%;
 ">
-<caption class="title">Table 2. Database config settings in [setup] section of megatest.config</caption>
+<caption class="title">Table 3. Database config settings in [setup] section of megatest.config</caption>
 <col style="width:14%;">
 <col style="width:28%;">
 <col style="width:28%;">
 <col style="width:28%;">
 <thead>
@@ -1824,11 +1887,11 @@
 COMPLETED/ xterm -e bash -s --</pre>
 </div></div>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">There is a trailing space after the --</td>
 </tr></table>
 </div>
 </div>
@@ -1917,11 +1980,11 @@
 <div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
 <table class="tableblock frame-topbot grid-all"
 style="
 width:70%;
 ">
-<caption class="title">Table 3. API Keys Related Calls</caption>
+<caption class="title">Table 4. API Keys Related Calls</caption>
 <col style="width:14%;">
 <col style="width:28%;">
 <col style="width:28%;">
 <col style="width:28%;">
 <thead>
@@ -1969,10 +2032,11 @@
 </div>
 <div id="footnotes"><hr></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.0<br>
-Last updated 2016-10-19 10:23:07 PDT
+Last updated
+ 2016-11-27 13:47:38 MST
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -1,11 +1,32 @@
 
 Reference
 ---------
 
-Megatest Config File Settings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Config File Helpers
+~~~~~~~~~~~~~~~~~~~
+
+Various helpers for more advanced config files.
+
+.Helpers
+[width="80%",cols="^,2m,2m,2m",frame="topbot",options="header"]
+|======================
+|Helper                      | Purpose                       | Valid values            | Comments
+| #{scheme (scheme code...)} | Execute arbitrary scheme code | Any valid scheme        | Value returned from the call is converted to a string and processed as part of the config file
+| #{system command}          | Execute program, inserts exit code  | Any valid Unix command  | Discards the output from the program
+| #{shell  command} or #{sh ...}  | Execute program, inserts result from stdout | Any valid Unix command | Value returned from the call is converted to a string and processed as part of the config file
+| #{realpath path} or #{rp ...}   | Replace with normalized path | Must be a valid path |
+| #{getenv VAR} or #{gv VAR}      | Replace with content of env variable | Must be a valid var |
+| #{get s v} or #{g s v}     | Replace with variable v from section s | Variable must be defined before use |
+| #{rget v}                  | Replace with variable v from target or default of runconfigs file | |
+| #{mtrah}                   | Replace with the path to the megatest testsuite area | | 
+|======================
+
+Config File Settings
+~~~~~~~~~~~~~~~~~~~~
+
+Settings in megatest.config
 
 Disk Space Checks
 ^^^^^^^^^^^^^^^^^
 
 Some parameters you can put in the [setup] section of megatest.config:

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -861,11 +861,11 @@
 	  (if res
 	      (cdr res)
 	      (begin
 		(if (common:low-noise-print 20 "No valid disks or no disk with enough space")
 		    (debug:print-error 0 *default-log-port* "No valid disks found in megatest.config. Please add some to your [disks] section and ensure the directory exists and has enough space!\n    You can change minspace in the [setup] section of megatest.config. Current setting is: " minspace))
-		(exit 1)))))))
+		(exit 1))))))) ;; TODO - move the exit to the calling location and return #f
 
 ;; Desired directory structure:
 ;;
 ;;  <linkdir> - <target> - <testname> -.
 ;;                                     |
@@ -1053,198 +1053,200 @@
 ;; 4. remotely run the test on allocated host
 ;;    - could be ssh to host from hosts table (update regularly with load)
 ;;    - could be netbatch
 ;;      (launch-test db (cadr status) test-conf))
 (define (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat params)
-  (let loop ((delta        (- (current-seconds) *last-launch*))
-	     (launch-delay (string->number (or (configf:lookup *configdat* "setup" "launch-delay") "5"))))
-    (if (> launch-delay delta)
-	(begin
-	  (debug:print-info 0 *default-log-port* "Delaying launch of " test-name " for " (- launch-delay delta) " seconds")
-	  (thread-sleep! (- launch-delay delta))
-	  (loop (- (current-seconds) *last-launch*) launch-delay))))
-  (set! *last-launch* (current-seconds))
-  (change-directory *toppath*)
-  (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
-   (list ;; (list "MT_TEST_RUN_DIR" work-area)
-    (list "MT_RUN_AREA_HOME" *toppath*)
-    (list "MT_TEST_NAME" test-name)
-    ;; (list "MT_ITEM_INFO" (conc itemdat)) 
-    (list "MT_RUNNAME"   runname)
-    ;; (list "MT_TARGET"    mt_target)
-    ))
-  (let* ((tregistry       (tests:get-all))
-	 (item-path       (let ((ip (item-list->path itemdat)))
-			    (alist->env-vars (list (list "MT_ITEMPATH" ip)))
-			    ip))
-	 (tconfig         (or (tests:get-testconfig test-name tregistry #t force-create: #t)
-			      test-conf)) ;; force re-read now that all vars are set
-	 (useshell        (let ((ush (config-lookup *configdat* "jobtools"     "useshell")))
-			    (if ush 
-				(if (equal? ush "no") ;; must use "no" to NOT use shell
-				    #f
-				    ush)
-				#t)))     ;; default is yes
-	 (runscript       (config-lookup tconfig   "setup"        "runscript"))
-	 (ezsteps         (> (length (hash-table-ref/default tconfig "ezsteps" '())) 0)) ;; don't send all the steps, could be big
-	 (diskspace       (config-lookup tconfig   "requirements" "diskspace"))
-	 (memory          (config-lookup tconfig   "requirements" "memory"))
-	 (hosts           (config-lookup *configdat* "jobtools"     "workhosts"))
-	 (remote-megatest (config-lookup *configdat* "setup" "executable"))
-	 (run-time-limit  (or (configf:lookup  tconfig   "requirements" "runtimelim")
-			      (configf:lookup  *configdat* "setup" "runtimelim")))
-	 ;; FIXME SOMEDAY: not good how this is so obtuse, this hack is to 
-	 ;;                allow running from dashboard. Extract the path
-	 ;;                from the called megatest and convert dashboard
-	 ;;             	  or dboard to megatest
-	 (local-megatest  (let* ((lm  (car (argv)))
-				 (dir (pathname-directory lm))
-				 (exe (pathname-strip-directory lm)))
-			    (conc (if dir (conc dir "/") "")
-				  (case (string->symbol exe)
-				    ((dboard)    "../megatest")
-				    ((mtest)     "../megatest")
-				    ((dashboard) "megatest")
-				    (else exe)))))
-	 (launcher        (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools"     "launcher"))
-	 (test-sig   (conc (common:get-testsuite-name) ":" test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
-	 (work-area  #f)
-	 (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
-	 (diskpath   #f)
-	 (cmdparms   #f)
-	 (fullcmd    #f) ;; (define a (with-output-to-string (lambda ()(write x))))
-	 (mt-bindir-path #f)
-	 (testinfo   (rmt:get-test-info-by-id run-id test-id))
-	 (mt_target  (string-intersperse (map cadr keyvals) "/"))
-	 (debug-param (append (if (args:get-arg "-debug")  (list "-debug" (args:get-arg "-debug")) '())
-			      (if (args:get-arg "-logging")(list "-logging") '()))))
-
-    (setenv "MT_ITEMPATH" item-path)
-    (if hosts (set! hosts (string-split hosts)))
-    ;; set the megatest to be called on the remote host
-    (if (not remote-megatest)(set! remote-megatest local-megatest)) ;; "megatest"))
-    (set! mt-bindir-path (pathname-directory remote-megatest))
-    (if launcher (set! launcher (string-split launcher)))
-    ;; set up the run work area for this test
-    (if (and (args:get-arg "-preclean") ;; user has requested to preclean for this run
-	     (not (member (db:test-get-rundir testinfo)(list "n/a" "/tmp/badname")))) ;; n/a is a placeholder and thus not a read dir
-	(begin
-	  (debug:print-info 0 *default-log-port* "attempting to preclean directory " (db:test-get-rundir testinfo) " for test " test-name "/" item-path)
-	  (runs:remove-test-directory testinfo 'remove-data-only))) ;; remove data only, do not perturb the record
-
-    ;; prevent overlapping actions - set to LAUNCHED as early as possible
-    ;;
-    ;; the following call handles waiver propogation. cannot yet condense into roll-up-pass-fail
-    (tests:test-set-status! run-id test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED"))
-    (rmt:roll-up-pass-fail-counts run-id test-name item-path #f "LAUNCHED" #f)
-    (set! diskpath (get-best-disk *configdat* tconfig))
-    (if diskpath
-	(let ((dat  (create-work-area run-id run-info keyvals test-id test-path diskpath test-name itemdat)))
-	  (set! work-area (car dat))
-	  (set! toptest-work-area (cadr dat))
-	  (debug:print-info 2 *default-log-port* "Using work area " work-area))
-	(begin
-	  (set! work-area (conc test-path "/tmp_run"))
-	  (create-directory work-area #t)
-	  (debug:print 0 *default-log-port* "WARNING: No disk work area specified - running in the test directory under tmp_run")))
-    (set! cmdparms (base64:base64-encode 
-		    (z3:encode-buffer 
-		     (with-output-to-string
-		       (lambda () ;; (list 'hosts     hosts)
-			 (write (list (list 'testpath  test-path)
-				      (list 'transport (conc *transport-type*))
-				      ;; (list 'serverinf *server-info*)
-				      (list 'toppath   *toppath*)
-				      (list 'work-area work-area)
-				      (list 'test-name test-name) 
-				      (list 'runscript runscript) 
-				      (list 'run-id    run-id   )
-				      (list 'test-id   test-id  )
-				      ;; (list 'item-path item-path )
-				      (list 'itemdat   itemdat  )
-				      (list 'megatest  remote-megatest)
-				      (list 'ezsteps   ezsteps) 
-				      (list 'target    mt_target)
-				      (list 'runtlim   (if run-time-limit (common:hms-string->seconds run-time-limit) #f))
-				      (list 'env-ovrd  (hash-table-ref/default *configdat* "env-override" '())) 
-				      (list 'set-vars  (if params (hash-table-ref/default params "-setvars" #f)))
-				      (list 'runname   runname)
-				      (list 'mt-bindir-path mt-bindir-path))))))))
-
-    ;; clean out step records from previous run if they exist
-    ;; (rmt:delete-test-step-records run-id test-id)
-    ;; if the dir does not exist we may have a itempath where individual variables are a path, launch anyway
-    (if (file-exists? work-area)
-	(change-directory work-area)) ;; so that log files from the launch process don't clutter the test dir
-    (cond
-     ((and launcher hosts) ;; must be using ssh hostname
-      (set! fullcmd (append launcher (car hosts)(list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param)))
-     ;; (set! fullcmd (append launcher (car hosts)(list remote-megatest test-sig "-execute" cmdparms))))
-     (launcher
-      (set! fullcmd (append launcher (list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param)))
-     ;; (set! fullcmd (append launcher (list remote-megatest test-sig "-execute" cmdparms))))
-     (else
-      (if (not useshell)(debug:print 0 *default-log-port* "WARNING: internal launching will not work well without \"useshell yes\" in your [jobtools] section"))
-      (set! fullcmd (append (list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param (list (if useshell "&" ""))))))
-    ;; (set! fullcmd (list remote-megatest test-sig "-execute" cmdparms (if useshell "&" "")))))
-    (if (args:get-arg "-xterm")(set! fullcmd (append fullcmd (list "-xterm"))))
-    (debug:print 1 *default-log-port* "Launching " work-area)
-    ;; set pre-launch-env-vars before launching, keep the vars in prevvals and put the envionment back when done
-    (debug:print 4 *default-log-port* "fullcmd: " fullcmd)
-    (let* ((commonprevvals (alist->env-vars
-			    (hash-table-ref/default *configdat* "env-override" '())))
-	   (testprevvals   (alist->env-vars
-			    (hash-table-ref/default tconfig "pre-launch-env-overrides" '())))
-	   (miscprevvals   (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
-			    (append (list (list "MT_TEST_RUN_DIR" work-area)
-					  (list "MT_TEST_NAME" test-name)
-					  (list "MT_ITEM_INFO" (conc itemdat)) 
-					  (list "MT_RUNNAME"   runname)
-					  (list "MT_TARGET"    mt_target)
-					  (list "MT_ITEMPATH"  item-path)
-					  )
-				    itemdat)))
-	   ;; Launchwait defaults to true, must override it to turn off wait
-	   (launchwait     (if (equal? (configf:lookup *configdat* "setup" "launchwait") "no") #f #t))
-	   (launch-results (apply (if launchwait
-				      process:cmd-run-with-stderr->list
-				      process-run)
-				  (if useshell
-				      (let ((cmdstr (string-intersperse fullcmd " ")))
-					(if launchwait
-					    cmdstr
-					    (conc cmdstr " >> mt_launch.log 2>&1")))
-				      (car fullcmd))
-				  (if useshell
-				      '()
-				      (cdr fullcmd)))))
-      (if (not launchwait) ;; give the OS a little time to allow the process to start
-	  (thread-sleep! 0.01))
-      (with-output-to-file "mt_launch.log"
-	(lambda ()
-	  (print "LAUNCHCMD: " (string-intersperse fullcmd " "))
-	  (if (list? launch-results)
-	      (apply print launch-results)
-	      (print "NOTE: launched \"" fullcmd "\"\n  but did not wait for it to proceed. Add the following to megatest.config \n[setup]\nlaunchwait yes\n  if you have problems with this"))
-	  #:append))
-      (debug:print 2 *default-log-port* "Launching completed, updating db")
-      (debug:print 2 *default-log-port* "Launch results: " launch-results)
-      (if (not launch-results)
-          (begin
-            (print "ERROR: Failed to run " (string-intersperse fullcmd " ") ", exiting now")
-            ;; (sqlite3:finalize! db)
-            ;; good ole "exit" seems not to work
-            ;; (_exit 9)
-            ;; but this hack will work! Thanks go to Alan Post of the Chicken email list
-            ;; NB// Is this still needed? Should be safe to go back to "exit" now?
-            (process-signal (current-process-id) signal/kill)
-            ))
-      (alist->env-vars miscprevvals)
-      (alist->env-vars testprevvals)
-      (alist->env-vars commonprevvals)
-      launch-results))
-  (change-directory *toppath*))
+  (let* ((item-path       (item-list->path itemdat)))
+    (let loop ((delta        (- (current-seconds) *last-launch*))
+	       (launch-delay (string->number (or (configf:lookup *configdat* "setup" "launch-delay") "5"))))
+      (if (> launch-delay delta)
+	  (begin
+	    (debug:print-info 0 *default-log-port* "Delaying launch of " test-name " for " (- launch-delay delta) " seconds")
+	    (thread-sleep! (- launch-delay delta))
+	    (loop (- (current-seconds) *last-launch*) launch-delay))))
+    (set! *last-launch* (current-seconds))
+    (change-directory *toppath*)
+    (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute", *maybe* - the longer they are set the longer each launch takes (must be non-overlapping with the vars)
+     (append
+      (list
+       (list "MT_RUN_AREA_HOME" *toppath*)
+       (list "MT_TEST_NAME" test-name)
+       (list "MT_RUNNAME"   runname)
+       (list "MT_ITEMPATH"  item-path)
+       )
+      itemdat))
+    (let* ((tregistry       (tests:get-all)) ;; third param (below) is system-allowed
+           ;; for tconfig, why do we allow fallback to test-conf?
+	   (tconfig         (or (tests:get-testconfig test-name tregistry #t force-create: #t)
+				(begin
+                                  (debug:print 0 *default-log-port* "WARNING: falling back to pre-calculated testconfig. This is likely not desired.")
+                                  test-conf))) ;; force re-read now that all vars are set
+	   (useshell        (let ((ush (config-lookup *configdat* "jobtools"     "useshell")))
+			      (if ush 
+				  (if (equal? ush "no") ;; must use "no" to NOT use shell
+				      #f
+				      ush)
+				  #t)))     ;; default is yes
+	   (runscript       (config-lookup tconfig   "setup"        "runscript"))
+	   (ezsteps         (> (length (hash-table-ref/default tconfig "ezsteps" '())) 0)) ;; don't send all the steps, could be big
+	   ;; (diskspace       (config-lookup tconfig   "requirements" "diskspace"))
+	   ;; (memory          (config-lookup tconfig   "requirements" "memory"))
+	   ;; (hosts           (config-lookup *configdat* "jobtools"     "workhosts")) ;; I'm pretty sure this was never completed
+	   (remote-megatest (config-lookup *configdat* "setup" "executable"))
+	   (run-time-limit  (or (configf:lookup  tconfig   "requirements" "runtimelim")
+				(configf:lookup  *configdat* "setup" "runtimelim")))
+	   ;; FIXME SOMEDAY: not good how this is so obtuse, this hack is to 
+	   ;;                allow running from dashboard. Extract the path
+	   ;;                from the called megatest and convert dashboard
+	   ;;             	  or dboard to megatest
+	   (local-megatest  (let* ((lm  (car (argv)))
+				   (dir (pathname-directory lm))
+				   (exe (pathname-strip-directory lm)))
+			      (conc (if dir (conc dir "/") "")
+				    (case (string->symbol exe)
+				      ((dboard)    "../megatest")
+				      ((mtest)     "../megatest")
+				      ((dashboard) "megatest")
+				      (else exe)))))
+	   (launcher        (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools"     "launcher"))
+	   (test-sig   (conc (common:get-testsuite-name) ":" test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
+	   (work-area  #f)
+	   (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
+	   (diskpath   #f)
+	   (cmdparms   #f)
+	   (fullcmd    #f) ;; (define a (with-output-to-string (lambda ()(write x))))
+	   (mt-bindir-path #f)
+	   (testinfo   (rmt:get-test-info-by-id run-id test-id))
+	   (mt_target  (string-intersperse (map cadr keyvals) "/"))
+	   (debug-param (append (if (args:get-arg "-debug")  (list "-debug" (args:get-arg "-debug")) '())
+				(if (args:get-arg "-logging")(list "-logging") '()))))
+      
+      ;; (if hosts (set! hosts (string-split hosts)))
+      ;; set the megatest to be called on the remote host
+      (if (not remote-megatest)(set! remote-megatest local-megatest)) ;; "megatest"))
+      (set! mt-bindir-path (pathname-directory remote-megatest))
+      (if launcher (set! launcher (string-split launcher)))
+      ;; set up the run work area for this test
+      (if (and (args:get-arg "-preclean") ;; user has requested to preclean for this run
+	       (not (member (db:test-get-rundir testinfo)(list "n/a" "/tmp/badname")))) ;; n/a is a placeholder and thus not a read dir
+	  (begin
+	    (debug:print-info 0 *default-log-port* "attempting to preclean directory " (db:test-get-rundir testinfo) " for test " test-name "/" item-path)
+	    (runs:remove-test-directory testinfo 'remove-data-only))) ;; remove data only, do not perturb the record
+      
+      ;; prevent overlapping actions - set to LAUNCHED as early as possible
+      ;;
+      ;; the following call handles waiver propogation. cannot yet condense into roll-up-pass-fail
+      (tests:test-set-status! run-id test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED"))
+      (rmt:roll-up-pass-fail-counts run-id test-name item-path #f "LAUNCHED" #f)
+      ;; (pp (hash-table->alist tconfig))
+      (set! diskpath (get-best-disk *configdat* tconfig))
+      (if diskpath
+	  (let ((dat  (create-work-area run-id run-info keyvals test-id test-path diskpath test-name itemdat)))
+	    (set! work-area (car dat))
+	    (set! toptest-work-area (cadr dat))
+	    (debug:print-info 2 *default-log-port* "Using work area " work-area))
+	  (begin
+	    (set! work-area (conc test-path "/tmp_run"))
+	    (create-directory work-area #t)
+	    (debug:print 0 *default-log-port* "WARNING: No disk work area specified - running in the test directory under tmp_run")))
+      (set! cmdparms (base64:base64-encode 
+		      (z3:encode-buffer 
+		       (with-output-to-string
+			 (lambda () ;; (list 'hosts     hosts)
+			   (write (list (list 'testpath  test-path)
+					(list 'transport (conc *transport-type*))
+					;; (list 'serverinf *server-info*)
+					(list 'toppath   *toppath*)
+					(list 'work-area work-area)
+					(list 'test-name test-name) 
+					(list 'runscript runscript) 
+					(list 'run-id    run-id   )
+					(list 'test-id   test-id  )
+					;; (list 'item-path item-path )
+					(list 'itemdat   itemdat  )
+					(list 'megatest  remote-megatest)
+					(list 'ezsteps   ezsteps) 
+					(list 'target    mt_target)
+					(list 'runtlim   (if run-time-limit (common:hms-string->seconds run-time-limit) #f))
+					(list 'env-ovrd  (hash-table-ref/default *configdat* "env-override" '())) 
+					(list 'set-vars  (if params (hash-table-ref/default params "-setvars" #f)))
+					(list 'runname   runname)
+					(list 'mt-bindir-path mt-bindir-path))))))))
+      
+      ;; clean out step records from previous run if they exist
+      ;; (rmt:delete-test-step-records run-id test-id)
+      ;; if the dir does not exist we may have a itempath where individual variables are a path, launch anyway
+      (if (file-exists? work-area)
+	  (change-directory work-area)) ;; so that log files from the launch process don't clutter the test dir
+      (cond
+       ;; ((and launcher hosts) ;; must be using ssh hostname
+       ;;    (set! fullcmd (append launcher (car hosts)(list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param)))
+       ;; (set! fullcmd (append launcher (car hosts)(list remote-megatest test-sig "-execute" cmdparms))))
+       (launcher
+	(set! fullcmd (append launcher (list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param)))
+       ;; (set! fullcmd (append launcher (list remote-megatest test-sig "-execute" cmdparms))))
+       (else
+	(if (not useshell)(debug:print 0 *default-log-port* "WARNING: internal launching will not work well without \"useshell yes\" in your [jobtools] section"))
+	(set! fullcmd (append (list remote-megatest "-m" test-sig "-execute" cmdparms) debug-param (list (if useshell "&" ""))))))
+      ;; (set! fullcmd (list remote-megatest test-sig "-execute" cmdparms (if useshell "&" "")))))
+      (if (args:get-arg "-xterm")(set! fullcmd (append fullcmd (list "-xterm"))))
+      (debug:print 1 *default-log-port* "Launching " work-area)
+      ;; set pre-launch-env-vars before launching, keep the vars in prevvals and put the envionment back when done
+      (debug:print 4 *default-log-port* "fullcmd: " fullcmd)
+      (let* ((commonprevvals (alist->env-vars
+			      (hash-table-ref/default *configdat* "env-override" '())))
+	     (miscprevvals   (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
+			      (append (list (list "MT_TEST_RUN_DIR" work-area)
+					    (list "MT_TEST_NAME" test-name)
+					    (list "MT_ITEM_INFO" (conc itemdat)) 
+					    (list "MT_RUNNAME"   runname)
+					    (list "MT_TARGET"    mt_target)
+					    (list "MT_ITEMPATH"  item-path)
+					    )
+				      itemdat)))
+	     (testprevvals   (alist->env-vars
+			      (hash-table-ref/default tconfig "pre-launch-env-overrides" '())))
+	     ;; Launchwait defaults to true, must override it to turn off wait
+	     (launchwait     (if (equal? (configf:lookup *configdat* "setup" "launchwait") "no") #f #t))
+	     (launch-results (apply (if launchwait
+					process:cmd-run-with-stderr->list
+					process-run)
+				    (if useshell
+					(let ((cmdstr (string-intersperse fullcmd " ")))
+					  (if launchwait
+					      cmdstr
+					      (conc cmdstr " >> mt_launch.log 2>&1")))
+					(car fullcmd))
+				    (if useshell
+					'()
+					(cdr fullcmd)))))
+	(if (not launchwait) ;; give the OS a little time to allow the process to start
+	    (thread-sleep! 0.01))
+	(with-output-to-file "mt_launch.log"
+	  (lambda ()
+	    (print "LAUNCHCMD: " (string-intersperse fullcmd " "))
+	    (if (list? launch-results)
+		(apply print launch-results)
+		(print "NOTE: launched \"" fullcmd "\"\n  but did not wait for it to proceed. Add the following to megatest.config \n[setup]\nlaunchwait yes\n  if you have problems with this"))
+	    #:append))
+	(debug:print 2 *default-log-port* "Launching completed, updating db")
+	(debug:print 2 *default-log-port* "Launch results: " launch-results)
+	(if (not launch-results)
+	    (begin
+	      (print "ERROR: Failed to run " (string-intersperse fullcmd " ") ", exiting now")
+	      ;; (sqlite3:finalize! db)
+	      ;; good ole "exit" seems not to work
+	      ;; (_exit 9)
+	      ;; but this hack will work! Thanks go to Alan Post of the Chicken email list
+	      ;; NB// Is this still needed? Should be safe to go back to "exit" now?
+	      (process-signal (current-process-id) signal/kill)
+	      ))
+	(alist->env-vars miscprevvals)
+	(alist->env-vars testprevvals)
+	(alist->env-vars commonprevvals)
+	launch-results))
+    (change-directory *toppath*)))
 
 ;; recover a test where the top controlling mtest may have died
 ;;
 (define (launch:recover-test run-id test-id)
   ;; this function is called on the test run host via ssh

Index: megatest-version.scm
==================================================================
--- megatest-version.scm
+++ megatest-version.scm
@@ -1,7 +1,7 @@
 ;; Always use two or four digit decimal
 ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..
 
 (declare (unit megatest-version))
 
-(define megatest-version 1.6208)
+(define megatest-version 1.6302)
 

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -118,11 +118,11 @@
                             fields category,variable,value,comment
 
 Queries
   -list-runs patt         : list runs matching pattern \"patt\", % is the wildcard
   -show-keys              : show the keys used in this megatest setup
-  -test-files targpatt    : get the most recent test path/file matching targpatt e.g. %/%... 
+  -test-files targpatt    : get the most recent test path/file matching targpatt e.g. %/% or '*.log'
                             returns list sorted by age ascending, see examples below
   -test-paths             : get the test paths matching target, runname, item and test
                             patterns.
   -list-disks             : list the disks available for storing runs
   -list-targets           : list the targets in runconfigs.config
@@ -790,23 +790,24 @@
 ;;======================================================================
 ;; Weird special calls that need to run *after* the server has started?
 ;;======================================================================
 
 (if (args:get-arg "-list-targets")
-    (let ((targets (common:get-runconfig-targets)))
-      (debug:print 1 *default-log-port* "Found "(length targets) " targets")
-      (case (string->symbol (or (args:get-arg "-dumpmode") "alist"))
-	((alist)
-	 (for-each (lambda (x)
-		     ;; (print "[" x "]"))
-		     (print x))
-		   targets))
-	((json)
-	 (json-write targets))
-	(else
-	 (debug:print-error 0 *default-log-port* "dump output format " (args:get-arg "-dumpmode") " not supported for -list-targets")))
-      (set! *didsomething* #t)))
+    (if (launch:setup)
+        (let ((targets (common:get-runconfig-targets)))
+          (debug:print 1 *default-log-port* "Found "(length targets) " targets")
+          (case (string->symbol (or (args:get-arg "-dumpmode") "alist"))
+            ((alist)
+             (for-each (lambda (x)
+                         ;; (print "[" x "]"))
+                         (print x))
+                       targets))
+            ((json)
+             (json-write targets))
+            (else
+             (debug:print-error 0 *default-log-port* "dump output format " (args:get-arg "-dumpmode") " not supported for -list-targets")))
+          (set! *didsomething* #t))))
 
 ;; cache the runconfigs in $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME/.runconfig
 ;;
 (define (full-runconfigs-read)
 ;; in the envprocessing branch the below code replaces the further below code
@@ -1018,29 +1019,30 @@
 	       ;;  	        "%"))
 	       (keys        (rmt:get-keys)) ;; (db:get-keys dbstruct))
 	       ;; (runsdat  (db:get-runs dbstruct runpatt #f #f '()))
 	;; (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") (common:args-get-target) ;; (db:get-runs-by-patt dbstruct keys (or runpatt "%") (common:args-get-target)
 	;; 		           	 #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
-	       (runsdat     (db:dispatch-query access-mode rmt:get-runs-by-patt db:get-runs-by-patt keys (or runpatt "%") 
-                                            (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
+	       (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") 
+                                                  (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
 	       (runstmp     (db:get-rows runsdat))
 	       (header      (db:get-header runsdat))
 	       ;; this is "-since" support. This looks at last mod times of <run-id>.db files
 	       ;; and collects those modified since the -since time.
-	       (runs        (if (and (not (null? runstmp))
-				     (args:get-arg "-since"))
-				(let ((changed-ids (db:get-changed-run-ids (string->number (args:get-arg "-since")))))
-				  (let loop ((hed (car runstmp))
-					     (tal (cdr runstmp))
-					     (res '()))
-				    (let ((new-res (if (member (db:get-value-by-header hed header "id") changed-ids)
-						       (cons hed res)
-						       res)))
-				      (if (null? tal)
-					  (reverse new-res)
-					  (loop (car tal)(cdr tal) new-res)))))
-				runstmp))
+	       (runs        runstmp)
+                        ;; (if (and (not (null? runstmp))
+			;;        (args:get-arg "-since"))
+			;;   (let ((changed-ids (db:get-changed-run-ids (string->number (args:get-arg "-since")))))
+			;;     (let loop ((hed (car runstmp))
+			;;   	     (tal (cdr runstmp))
+			;;   	     (res '()))
+			;;       (let ((new-res (if (member (db:get-value-by-header hed header "id") changed-ids)
+			;;   		       (cons hed res)
+			;;   		       res)))
+			;;         (if (null? tal)
+			;;   	  (reverse new-res)
+			;;   	  (loop (car tal)(cdr tal) new-res)))))
+			;;   runstmp))
 	       (db-targets  (args:get-arg "-list-db-targets"))
 	       (seen        (make-hash-table))
 	       (dmode       (let ((d (args:get-arg "-dumpmode")))
 			      (if d (string->symbol d) #f)))
 	       (data        (make-hash-table))
@@ -1527,11 +1529,12 @@
 	  (let* ((keys     (rmt:get-keys))
 		 ;; db:test-get-paths must not be run remote
 		 (paths    (tests:test-get-paths-matching keys target (args:get-arg "-test-files"))))
 	    (set! *didsomething* #t)
 	    (for-each (lambda (path)
-			(print path))
+			(if (file-exists? path)
+			(print path)))	
 		      paths)))
 	;; else do a general-run-call
 	(general-run-call 
 	 "-test-files"
 	 "Get paths to test"
@@ -1825,11 +1828,12 @@
     (begin
       (if (not (launch:setup))
 	  (begin
 	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
 	    (exit 1)))
-      (common:cleanup-db)
+      (let ((dbstruct (db:setup *toppath*)))
+        (common:cleanup-db dbstruct))
       (set! *didsomething* #t)))
 
 (if (args:get-arg "-mark-incompletes")
     (begin
       (if (not (launch:setup))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -588,12 +588,12 @@
 
 (define (rmt:get-runs-by-patt  keys runnamepatt targpatt offset limit fields last-runs-update) ;; fields of #f uses default
   (rmt:send-receive 'get-runs-by-patt #f (list keys runnamepatt targpatt offset limit fields last-runs-update)))
 
 (define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
-  (if (rmt:send-receive 'have-incompletes? run-id (list run-id ovr-deadtime))
-      (rmt:send-receive 'mark-incomplete run-id (list run-id ovr-deadtime))))
+  ;; (if (rmt:send-receive 'have-incompletes? run-id (list run-id ovr-deadtime))
+  (rmt:send-receive 'mark-incomplete run-id (list run-id ovr-deadtime))) ;; )
 
 (define (rmt:get-main-run-stats run-id)
   (rmt:send-receive 'get-main-run-stats #f (list run-id)))
 
 (define (rmt:get-var varname)

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -920,24 +920,30 @@
 	  
 ;; MUST BE CALLED local!
 ;;
 (define (tests:test-get-paths-matching keynames target fnamepatt #!key (res '()))
   ;; BUG: Move the values derived from args to parameters and push to megatest.scm
-  (let* ((testpatt   (if (args:get-arg "-testpatt")(args:get-arg "-testpatt") "%"))
-	 (statepatt  (if (args:get-arg ":state")   (args:get-arg ":state")    "%"))
-	 (statuspatt (if (args:get-arg ":status")  (args:get-arg ":status")   "%"))
-	 (runname    (if (args:get-arg ":runname") (args:get-arg ":runname")  "%"))
+  (let* ((testpatt   (or (args:get-arg "-testpatt")(args:get-arg "-testpatt") "%"))
+	 (statepatt  (or (args:get-arg "-state")   (args:get-arg ":state")    "%"))
+	 (statuspatt (or (args:get-arg "-status")  (args:get-arg ":status")   "%"))
+	 (runname    (or (args:get-arg "-runname") (args:get-arg ":runname")  "%"))
 	 (paths-from-db (rmt:test-get-paths-matching-keynames-target-new keynames target res
 					testpatt
 					statepatt
 					statuspatt
 					runname)))
     (if fnamepatt
 	(apply append 
 	       (map (lambda (p)
 		      (if (directory-exists? p)
-			  (glob (conc p "/" fnamepatt))
+			  (let ((glob-query (conc p "/" fnamepatt)))
+			    (handle-exceptions
+				exn
+				(with-input-from-pipe
+				    (conc "echo " glob-query)
+				  read-lines)  ;; we aren't going to try too hard. If glob breaks it is likely because someone tried to do */*/*.log or similar
+			      (glob glob-query)))
 			  '()))
 		    paths-from-db))
 	paths-from-db)))
 
 			      
@@ -1332,10 +1338,11 @@
 	 "SELECT count(id) FROM test_rundat;")
 	res))
   0)
 
 (define (tests:update-central-meta-info run-id test-id cpuload diskfree minutes uname hostname)
+  (rmt:general-call 'update-test-rundat run-id test-id (current-seconds) (or cpuload -1)(or diskfree -1) -1 (or minutes -1))
   (if (and cpuload diskfree)
       (rmt:general-call 'update-cpuload-diskfree run-id cpuload diskfree test-id))
   (if minutes 
       (rmt:general-call 'update-run-duration run-id minutes test-id))
   (if (and uname hostname)

ADDED   utils/remrun
Index: utils/remrun
==================================================================
--- /dev/null
+++ utils/remrun
@@ -0,0 +1,28 @@
+#!/bin/bash
+###############################################################################
+#
+# remrun - same behavior as nbfake but first param is a hosthane
+#          (capture command output in a logfile)
+#
+# remrun behavior can be changed by setting the following env var:
+#   NBFAKE_LOG        Logfile for nbfake output
+#
+###############################################################################
+
+if [[ -z "$@" ]]; then
+  cat <<__EOF
+
+remrun usage:
+
+remrun hostname <command to run>
+
+remrun behavior can be changed by setting the following env vars:
+   NBFAKE_LOG        Logfile for remrun output
+
+__EOF
+  exit
+fi
+
+export NBFAKE_HOST=$1
+shift
+exec nbfake $*