Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -531,17 +531,21 @@
     (6 "CHECK")
     (7 "STUCK/DEAD")
     (8 "DEAD")
     (9 "FAIL")
     (10 "PREQ_FAIL")
-    (11 "ABORT")))
+    (11 "PREQ_DISCARDED")
+    (12 "ABORT")))
 
 (define *common:ended-states*       ;; states which indicate the test is stopped and will not proceed
-  '("COMPLETED" "ARCHIVED" "KILLED" "KILLREQ" "STUCK" "INCOMPLETE"))
+  '("COMPLETED" "ARCHIVED" "KILLED" "KILLREQ" "STUCK" "INCOMPLETE" ))
 
 (define *common:badly-ended-states* ;; these roll up as CHECK, i.e. results need to be checked
   '("KILLED" "KILLREQ" "STUCK" "INCOMPLETE" "DEAD"))
+
+(define *common:well-ended-states* ;; an item's prereq in this state allows item to proceed
+  '("PASS" "WARN" "CHECK" "WAIVED" "SKIP"))
 
 ;; BBnote: *common:running-states* used from db:set-state-status-and-roll-up-items
 (define *common:running-states*     ;; test is either running or can be run
   '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "STARTED"))
 
@@ -1748,10 +1752,12 @@
 			       key "=" delim (mungeval val) delim)))
                     envvars)))))
 
 ;; set some env vars from an alist, return an alist with original values
 ;; (("VAR" "value") ...)
+;; a value of #f means "unset this var"
+;;
 (define (alist->env-vars lst)
   (if (list? lst)
       (let ((res '()))
 	(for-each (lambda (p)
 		    (let* ((var (car  p))
@@ -2410,5 +2416,43 @@
     (lambda (a b)(> (cdr a)(cdr b))))      ;; sort descending
    (lambda (a b)(equal? (car a)(car b))))) ;; remove duplicates by target
 
 
 
+;; accept an alist or hash table containing envvar/env value pairs (value of #f causes unset) 
+;;   execute thunk in context of environment modified as per this list
+;;   restore env to prior state then return value of eval'd thunk.
+;;   ** this is not thread safe **
+(define (common:with-env-vars delta-env-alist-or-hash-table thunk)
+  (let* ((delta-env-alist (if (hash-table? delta-env-alist-or-hash-table)
+                              (hash-table->alist delta-env-alist-or-hash-table)
+                              delta-env-alist-or-hash-table))
+         (restore-thunks
+          (filter
+           identity
+           (map (lambda (env-pair)
+                  (let* ((env-var     (car env-pair))
+                         (new-val     (cadr env-pair))
+                         (current-val (get-environment-variable env-var))
+                         (restore-thunk
+                          (cond
+                           ((not current-val) (lambda () (unsetenv env-var)))
+                           ((not (string? new-val)) #f)
+                           ((eq? current-val new-val) #f)
+                           (else 
+                            (lambda () (setenv env-var current-val))))))
+                    ;;(when (not (string? new-val))
+                    ;;    (debug:print 0 *default-log-port* " PROBLEM: not a string: "new-val"\n from env-alist:\n"delta-env-alist)
+                    ;;    (pp delta-env-alist)
+                    ;;    (exit 1))
+                        
+                    
+                    (cond
+                     ((not new-val)  ;; modify env here
+                      (unsetenv env-var))
+                     ((string? new-val)
+                      (setenv env-var new-val)))
+                    restore-thunk))
+                delta-env-alist))))
+    (let ((rv (thunk)))
+      (for-each (lambda (x) (x)) restore-thunks) ;; restore env to original state
+      rv)))

Index: common_records.scm
==================================================================
--- common_records.scm
+++ common_records.scm
@@ -147,15 +147,17 @@
               (temp     (string-split (->string this-loc) " "))
               (this-func (if (and (list? temp) (> (length temp) 1)) (cadr temp) "???")))
          (if (equal? this-func "BB>")
              (set! location this-loc))))
      stack)
-    (let ((dp-args
-           (append
-            (list 0 *default-log-port*
-                  (conc location "@"(/ (- (current-milliseconds) *BB-process-starttime*) 1000)"   ")  )
-            in-args)))
+    (let* ((color-on "\x1b[1m")
+           (color-off "\x1b[0m")
+           (dp-args
+            (append
+             (list 0 *default-log-port*
+                   (conc color-on location "@"(/ (- (current-milliseconds) *BB-process-starttime*) 1000) color-off "   ")  )
+             in-args)))
       (apply debug:print dp-args))))
 
 (define *BBpp_custom_expanders_list* (make-hash-table))
 
 

Index: configf.scm
==================================================================
--- configf.scm
+++ configf.scm
@@ -77,10 +77,14 @@
 (define configf:settings   (regexp "^\\[configf:settings\\s+(\\S+)\\s+(\\S+)]\\s*$"))
 
 ;; read a line and process any #{ ... } constructs
 
 (define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget|scm|sh|rp|gv|g|mtrah)\\s+([^\\}\\{]*)\\}(.*)"))
+
+(define (configf:system ht cmd)
+  (system cmd)
+  )
 
 (define (configf:process-line l ht allow-system #!key (linenum #f))
   (let loop ((res l))
     (if (string? res)
 	(let ((matchdat (string-search configf:var-expand-regex res)))
@@ -92,11 +96,11 @@
 		     (result  #f)
 		     (start-time (current-seconds))
 		     (cmdsym  (string->symbol cmdtype))
 		     (fullcmd (case cmdsym
 				((scheme scm) (conc "(lambda (ht)" cmd ")"))
-				((system)     (conc "(lambda (ht)(system \"" cmd "\"))"))
+				((system)     (conc "(lambda (ht)(configf:system ht \"" cmd "\"))"))
 				((shell sh)   (conc "(lambda (ht)(string-translate (shell \""  cmd "\") \"\n\" \" \"))"))
 				((realpath rp)(conc "(lambda (ht)(common:nice-path \"" cmd "\"))"))
 				((getenv gv)  (conc "(lambda (ht)(get-environment-variable \"" cmd "\"))"))
 				((mtrah)      (conc "(lambda (ht)"
                                                     "    (let ((extra \"" cmd "\"))"
@@ -176,11 +180,27 @@
 			(configf:process-line inl ht allow-processing)))))
 	    (if (and (string? res)
 		     (not (equal? (hash-table-ref/default settings "trim-trailing-spaces" "no") "no")))
 		(string-substitute "\\s+$" "" res)
 		res))))))
-  
+
+(define (configf:cfgdat->env-alist section cfgdat-ht allow-system)
+  (filter
+   (lambda (pair)
+     (let* ((var (car pair))
+            (val (cdr pair)))
+       (cons var
+             (cond
+              ((and allow-system (procedure? val)) ;; if we decided to use something other than #t or #f for allow-system ('return-procs or 'return-string) , this may become problematic
+               (val))
+              ((procedure? val) #f)
+              ((string? val) val)
+              (else "#f")))))
+   (append
+    (hash-table-ref/default cfgdat-ht "default" '())
+    (if (equal? section "default") '() (hash-table-ref/default cfgdat-ht section '())))))
+
 (define (calc-allow-system allow-system section sections)
   (if sections
       (and (or (equal? "default" section)
 	       (member section sections))
 	   allow-system) ;; account for sections and return allow-system as it might be a symbol such as return-strings
@@ -217,19 +237,24 @@
 
 ;; read a config file, returns hash table of alists
 
 ;; read a config file, returns hash table of alists
 ;; adds to ht if given (must be #f otherwise)
+;; allow-system:
+;;    #f - do not evaluate [system
+;;    #t - immediately evaluate [system and store result as string
+;;    'return-procs -- return a proc taking ht as an argument that may be evaulated at some future time
+;;    'return-string -- return a string representing a proc taking ht as an argument that may be evaulated at some future time
 ;; envion-patt is a regex spec that identifies sections that will be eval'd
 ;; in the environment on the fly
 ;; sections: #f => get all, else list of sections to gather
 ;; post-section-procs alist of section-pattern => proc, where: (proc section-name next-section-name ht curr-path)
 ;; apply-wildcards: #t/#f - apply vars from targets with % wildcards to all matching sections
 ;;
-(define (read-config path ht allow-system #!key (environ-patt #f)            (curr-section #f)
+(define (read-config path ht allow-system #!key (environ-patt #f)            (curr-section #f)   
 		     (sections #f)              (settings (make-hash-table)) (keep-filenames #f)
-		     (post-section-procs '())   (apply-wildcards #t))
+		     (post-section-procs '())   (apply-wildcards #t) )
   (debug:print 9 *default-log-port* "START: " path)
   (if (and (not (port? path))
 	   (not (common:file-exists? path))) ;; for case where we are handed a port
       (begin 
 	(debug:print-info 1 *default-log-port* "read-config - file not found " path " current path: " (current-directory))
@@ -265,138 +290,174 @@
 		     (lambda (section)
 		       (if (not (member section sections))
 			   (hash-table-delete! res section))) ;; we are using "" as a dumping ground and must remove it before returning the ht
 		     (hash-table-keys res)))
 		(debug:print 9 *default-log-port* "END: " path)
-		res)
+                res
+                ) ;; retval
 	      (regex-case 
 	       inl 
-	       (configf:comment-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-	       (configf:blank-l-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-	       (configf:settings   ( x setting val  ) (begin
-							(hash-table-set! settings setting val)
-							(loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))
-	       (configf:include-rx ( x include-file ) (let* ((curr-conf-dir (pathname-directory path))
-							     (full-conf     (if (absolute-pathname? include-file)
-										include-file
-										(common:nice-path 
-										 (conc (if curr-conf-dir
-											   curr-conf-dir
-											   ".")
-										       "/" include-file)))))
-							(if (common:file-exists? full-conf)
-							    (begin
-							      ;; (push-directory conf-dir)
-							      (debug:print 9 *default-log-port* "Including: " full-conf)
-							      (read-config full-conf res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
-							      ;; (pop-directory)
-							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-							    (begin
-							      (debug:print '(2 9) #f "INFO: include file " include-file " not found (called from " path ")")
-							      (debug:print 2 *default-log-port* "        " full-conf)
-							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))))
+	       (configf:comment-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
+                                                            curr-section-name #f #f))
+               
+	       (configf:blank-l-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
+                                                            curr-section-name #f #f))
+	       (configf:settings   ( x setting val  )
+                                   (begin
+                                     (hash-table-set! settings setting val)
+                                     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
+                                           curr-section-name #f #f)))
+               
+	       (configf:include-rx ( x include-file )
+                                   (let* ((curr-conf-dir (pathname-directory path))
+                                          (full-conf     (if (absolute-pathname? include-file)
+                                                             include-file
+                                                             (common:nice-path 
+                                                              (conc (if curr-conf-dir
+                                                                        curr-conf-dir
+                                                                        ".")
+                                                                    "/" include-file)))))
+                                     (if (common:file-exists? full-conf)
+                                         (begin
+                                           ;; (push-directory conf-dir)
+                                           (debug:print 9 *default-log-port* "Including: " full-conf)
+                                           (read-config full-conf res allow-system environ-patt: environ-patt
+                                                        curr-section: curr-section-name sections: sections settings: settings
+                                                        keep-filenames: keep-filenames)
+                                           ;; (pop-directory)
+                                           (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
+                                         (begin
+                                           (debug:print '(2 9) #f "INFO: include file " include-file " not found (called from " path ")")
+                                           (debug:print 2 *default-log-port* "        " full-conf)
+							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
+                                                                    curr-section-name #f #f)))))
 	       (configf:script-rx ( x include-script params);; handle-exceptions
                                   ;;    exn
                                   ;;    (begin
                                   ;;      (debug:print '(0 2 9) #f "INFO: include from script " include-script " failed.")
                                   ;;      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-							 (if (and (common:file-exists? include-script)(file-execute-access? include-script))
-							     (let* ((new-inp-port (open-input-pipe (conc include-script " " params))))
-							       (debug:print '(2 9) *default-log-port* "Including from script output: " include-script)
-							      ;;  (print "We got here, calling read-config next. Port is: " new-inp-port)
-							       (read-config new-inp-port res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
-							       (close-input-port new-inp-port)
-							       (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-							     (begin
-							       (debug:print 0 *default-log-port* "Script not found or not exectutable: " include-script)
-							       (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))
-							 ) ;; )
-	       (configf:section-rx ( x section-name ) (begin
-							;; call post-section-procs
-							(for-each 
-							 (lambda (dat)
-							   (let ((patt (car dat))
-								 (proc (cdr dat)))
-							     (if (string-match patt curr-section-name)
-								 (proc curr-section-name section-name res path))))
-							 post-section-procs)
-                                                        ;; after gathering the vars for a section and if apply-wildcards is true and if there is a wildcard in the section name process wildcards
-                                                        ;; NOTE: we are processing the curr-section-name, NOT section-name.
-                                                        (process-wildcards res curr-section-name)
-							(if (not (hash-table-ref/default res section-name #f))(hash-table-set! res section-name '())) ;; ensure that mere mention of a section is not lost
-							(loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
-							      ;; if we have the sections list then force all settings into "" and delete it later?
-							      ;; (if (or (not sections) 
-							      ;;	      (member section-name sections))
-							      ;;	  section-name "") ;; stick everything into "". NOPE: We need new strategy. Put stuff in correct sections and then delete all sections later.
-							      section-name
-							      #f #f)))
-	       (configf:key-sys-pr ( x key cmd      ) (if (calc-allow-system allow-system curr-section-name sections)
-							  (let ((alist    (hash-table-ref/default res curr-section-name '()))
-								(val-proc (lambda ()
-									    (let* ((start-time (current-seconds))
-										   (cmdres     (process:cmd-run->list cmd))
-										   (delta      (- (current-seconds) start-time))
-										   (status     (cadr cmdres))
-										   (res        (car  cmdres)))
-									      (debug:print-info 4 *default-log-port* "" inl "\n => " (string-intersperse res "\n"))
-									      (if (not (eq? status 0))
-										  (begin
-										    (debug:print-error 0 *default-log-port* "problem with " inl ", return code " status
-												 " output: " cmdres)))
-									      (if (> delta 2)
-										  (debug:print-info 0 *default-log-port* "for line \"" inl "\"\n  command: " cmd " took " delta " seconds to run with output:\n   " res)
-										  (debug:print-info 9 *default-log-port* "for line \"" inl "\"\n  command: " cmd " took " delta " seconds to run with output:\n   " res))
-									      (if (null? res)
-										  ""
-										  (string-intersperse res " "))))))
-							    (hash-table-set! res curr-section-name 
-									     (config:assoc-safe-add alist
-									   			    key 
-												    (case (calc-allow-system allow-system curr-section-name sections)
-												      ((return-procs) val-proc)
-												      ((return-string) cmd)
-												      (else (val-proc)))
-												    metadata: metapath))
-							    (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
-							  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))
-	       (configf:key-no-val ( x key val)            (let* ((alist   (hash-table-ref/default res curr-section-name '()))
-								  (fval    (or (if (string? val) val #f) ""))) ;; fval should be either "" or " " (one or more spaces)
-							     (debug:print 10 *default-log-port* "   setting: [" curr-section-name "] " key " = #t")
-							     (safe-setenv key fval)
-							     (hash-table-set! res curr-section-name 
-									      (config:assoc-safe-add alist key fval metadata: metapath))
-							     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name key #f)))
-	       (configf:key-val-pr ( x key unk1 val unk2 ) (let* ((alist   (hash-table-ref/default res curr-section-name '()))
-								  (envar   (and environ-patt (string-search (regexp environ-patt) curr-section-name)))
-								  (realval (if envar
-									       (config:eval-string-in-environment val)
-									       val)))
-							     (debug:print-info 6 *default-log-port* "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name)
-							     (if envar (safe-setenv key realval))
-							     (debug:print 10 *default-log-port* "   setting: [" curr-section-name "] " key " = " val)
-							     (hash-table-set! res curr-section-name 
-									      (config:assoc-safe-add alist key realval metadata: metapath))
-							     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name key #f)))
+                                  (if (and (common:file-exists? include-script)(file-execute-access? include-script))
+                                      (let* ((local-allow-system  (calc-allow-system allow-system curr-section-name sections))
+                                             (env-delta  (configf:cfgdat->env-alist curr-section-name res local-allow-system))
+                                             (new-inp-port
+                                              (common:with-env-vars
+                                               env-delta
+                                               (lambda ()
+                                                 (open-input-pipe (conc include-script " " params))))))
+                                        (debug:print '(2 9) *default-log-port* "Including from script output: " include-script)
+                                        ;;  (print "We got here, calling read-config next. Port is: " new-inp-port)
+                                        (read-config new-inp-port res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
+                                        (close-input-port new-inp-port)
+                                        (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
+                                      (begin
+                                        (debug:print 0 *default-log-port* "Script not found or not exectutable: " include-script)
+                                        (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f)))
+                                  ) ;; )
+	       (configf:section-rx ( x section-name )
+                                   (begin
+                                     ;; call post-section-procs
+                                     (for-each 
+                                      (lambda (dat)
+                                        (let ((patt (car dat))
+                                              (proc (cdr dat)))
+                                          (if (string-match patt curr-section-name)
+                                              (proc curr-section-name section-name res path))))
+                                      post-section-procs)
+                                     ;; after gathering the vars for a section and if apply-wildcards is true and if there is a wildcard in the section name process wildcards
+                                     ;; NOTE: we are processing the curr-section-name, NOT section-name.
+                                     (process-wildcards res curr-section-name)
+                                     (if (not (hash-table-ref/default res section-name #f))(hash-table-set! res section-name '())) ;; ensure that mere mention of a section is not lost
+                                     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings)
+                                           ;; if we have the sections list then force all settings into "" and delete it later?
+                                           ;; (if (or (not sections) 
+                                           ;;	      (member section-name sections))
+                                           ;;	  section-name "") ;; stick everything into "". NOPE: We need new strategy. Put stuff in correct sections and then delete all sections later.
+                                           section-name
+                                           #f #f)))
+	       (configf:key-sys-pr ( x key cmd      )
+                                   (if (calc-allow-system allow-system curr-section-name sections)
+                                       (let ((alist    (hash-table-ref/default res curr-section-name '()))
+                                             (val-proc (lambda ()
+                                                         (let* ((start-time (current-seconds))
+                                                                (local-allow-system  (calc-allow-system allow-system curr-section-name sections))
+                                                                (env-delta  (configf:cfgdat->env-alist curr-section-name res local-allow-system))
+                                                                (cmdres     (process:cmd-run->list cmd delta-env-alist-or-hash-table: env-delta)) ;; BB: here is where [system is exec'd.  needs to have env from other vars!
+                                                                (delta      (- (current-seconds) start-time))
+                                                                (status     (cadr cmdres))
+                                                                (res        (car  cmdres)))
+                                                           (debug:print-info 4 *default-log-port* "" inl "\n => " (string-intersperse res "\n"))
+                                                           (if (not (eq? status 0))
+                                                               (begin
+                                                                 (debug:print-error 0 *default-log-port* "problem with " inl ", return code " status
+                                                                                    " output: " cmdres)))
+                                                           (if (> delta 2)
+                                                               (debug:print-info 0 *default-log-port* "for line \"" inl "\"\n  command: " cmd " took " delta " seconds to run with output:\n   " res)
+                                                               (debug:print-info 9 *default-log-port* "for line \"" inl "\"\n  command: " cmd " took " delta " seconds to run with output:\n   " res))
+                                                           (if (null? res)
+                                                               ""
+                                                               (string-intersperse res " "))))))
+                                         (hash-table-set! res curr-section-name 
+                                                          (config:assoc-safe-add alist
+                                                                                 key 
+                                                                                 (case (calc-allow-system allow-system curr-section-name sections)
+                                                                                   ((return-procs) val-proc)
+                                                                                   ((return-string) cmd)
+                                                                                   (else (val-proc)))
+                                                                                 metadata: metapath))
+                                         (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))
+                                       (loop (configf:read-line inp res
+                                                                (calc-allow-system allow-system curr-section-name sections)
+                                                                settings)
+                                             curr-section-name #f #f)))
+               
+	       (configf:key-no-val ( x key val)
+                                   (let* ((alist   (hash-table-ref/default res curr-section-name '()))
+                                          (fval    (or (if (string? val) val #f) ""))) ;; fval should be either "" or " " (one or more spaces)
+                                     (debug:print 10 *default-log-port* "   setting: [" curr-section-name "] " key " = #t")
+                                     (safe-setenv key fval)
+                                     (hash-table-set! res curr-section-name 
+                                                      (config:assoc-safe-add alist key fval metadata: metapath))
+                                     (loop (configf:read-line inp res
+                                                              (calc-allow-system allow-system curr-section-name sections)
+                                                              settings)
+                                           curr-section-name key #f)))
+               
+	       (configf:key-val-pr ( x key unk1 val unk2 )
+                                   (let* ((alist   (hash-table-ref/default res curr-section-name '()))
+                                          (envar   (and environ-patt (string-search (regexp environ-patt) curr-section-name)))
+                                          (realval (if envar
+                                                       (config:eval-string-in-environment val)
+                                                       val)))
+                                     (debug:print-info 6 *default-log-port* "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name)
+                                     (if envar (safe-setenv key realval))
+                                     (debug:print 10 *default-log-port* "   setting: [" curr-section-name "] " key " = " val)
+                                     (hash-table-set! res curr-section-name 
+                                                      (config:assoc-safe-add alist key realval metadata: metapath))
+                                     (loop (configf:read-line inp res
+                                                              (calc-allow-system allow-system curr-section-name sections) settings)
+                                           curr-section-name key #f)))
 	       ;; if a continued line
-	       (configf:cont-ln-rx ( x whsp val     ) (let ((alist (hash-table-ref/default res curr-section-name '())))
-						(if var-flag             ;; if set to a string then we have a continued var
-						    (let ((newval (conc 
-								   (config-lookup res curr-section-name var-flag) "\n"
-								   ;; trim lead from the incoming whsp to support some indenting.
-								   (if lead
-								       (string-substitute (regexp lead) "" whsp)
-								       "")
-								   val)))
-						      ;; (print "val: " val "\nnewval: \"" newval "\"\nvarflag: " var-flag)
-						      (hash-table-set! res curr-section-name 
-								       (config:assoc-safe-add alist var-flag newval metadata: metapath))
-						      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name var-flag (if lead lead whsp)))
-						    (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
+	       (configf:cont-ln-rx ( x whsp val     )
+                                   (let ((alist (hash-table-ref/default res curr-section-name '())))
+                                     (if var-flag             ;; if set to a string then we have a continued var
+                                         (let ((newval (conc 
+                                                        (config-lookup res curr-section-name var-flag) "\n"
+                                                        ;; trim lead from the incoming whsp to support some indenting.
+                                                        (if lead
+                                                            (string-substitute (regexp lead) "" whsp)
+                                                            "")
+                                                        val)))
+                                           ;; (print "val: " val "\nnewval: \"" newval "\"\nvarflag: " var-flag)
+                                           (hash-table-set! res curr-section-name 
+                                                            (config:assoc-safe-add alist var-flag newval metadata: metapath))
+                                           (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name var-flag (if lead lead whsp)))
+                                         (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
 	       (else (debug:print-error 0 *default-log-port* "problem parsing " path ",\n   \"" inl "\"")
 		     (set! var-flag #f)
-		     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))))))
+		     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
+          ) ;; end loop
+        )))
   
 ;; pathenvvar will set the named var to the path of the config
 (define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
   (let* ((curr-dir   (current-directory))
          (configinfo (find-config fname toppath: given-toppath))
@@ -457,10 +518,18 @@
 	'()
 	(map car sectdat))))
 
 (define (configf:get-section cfgdat section)
   (hash-table-ref/default cfgdat section '()))
+
+(define (configf:set-section-var cfgdat section var val)
+  (let ((sectdat (configf:get-section cfgdat section)))
+    (hash-table-set! cfgdat section
+                     (config:assoc-safe-add sectdat var val))))
+
+    ;;(append (filter (lambda (x)(not (assoc var sectdat))) sectdat)
+    ;;	    (list var val))))
 
 (define (setup)
   (let* ((configf (find-config "megatest.config"))
 	 (config  (if configf (read-config configf #f #t) #f)))
     (if config

Index: dashboard-tests.scm
==================================================================
--- dashboard-tests.scm
+++ dashboard-tests.scm
@@ -626,15 +626,16 @@
 				     " -testpatt " (conc testname "/" (if (equal? item-path "")
 									  "%"
 									  item-path))
 				     " -v"))))
 	       (clean-run-execute  (lambda (x)
-				     (let ((cmd (conc "megatest -remove-runs -target " keystring " -runname " runname
+				     (let ((cmd (conc ;; "megatest -remove-runs -target " keystring " -runname " runname
+                                                 "megatest -set-state-status NOT_STARTED,n/a -target " keystring " -runname " runname
 						      " -testpatt " (conc testname "/" (if (equal? item-path "")
 						       					   "%"
 						       					   item-path))
-						      ";megatest -target " keystring " -runname " runname 
+                                                      ";megatest -target " keystring " -runname " runname 
 						      " -run -preclean -testpatt " (conc testname "/" (if (equal? item-path "")
 											   "%" 
 											   item-path))
 						      " -clean-cache"
 						      )))

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -1014,11 +1014,11 @@
 	      (testsdat-by-name (dboard:rundat-tests-by-name rundat))
 	      (key-val-dat      (dboard:rundat-key-vals rundat))
 	      (run-id           (db:get-value-by-header run (dboard:tabdat-header tabdat) "id"))
 	      (key-vals         (append key-val-dat
 					(list (let ((x (db:get-value-by-header run (dboard:tabdat-header tabdat) "runname")))
-						(if x x "")))))
+						(if (string? x) x "")))))
 	      (run-key          (string-intersperse key-vals "\n")))
 	 
 	 ;; fill in the run header key values
 	 ;;
 	 (let ((rown      0)
@@ -1682,11 +1682,13 @@
 	 (last-runs-update  (dboard:tabdat-last-runs-update tabdat))
 	 (runs-dat     (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)))
     (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2))
     (for-each (lambda (run-id)
 		(let* ((run-record (hash-table-ref/default runs-hash run-id #f))
-		       (key-vals   (map (lambda (key)(db:get-value-by-header run-record runs-header key))
+		       (key-vals   (map (lambda (key)
+                                          (let ((val (db:get-value-by-header run-record runs-header key)))
+                                            (if (string? val) val "")))
 					(dboard:tabdat-keys tabdat)))
 		       (run-name   (db:get-value-by-header run-record runs-header "runname"))
 		       (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
 		       (run-path   (append key-vals (list run-name))))
 		  (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f))

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -221,10 +221,11 @@
            (let ((db      (sqlite3:open-database fname)))
              (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
              (sqlite3:execute db "PRAGMA synchronous = 0;")
              (if (not file-exists)
                  (begin
+                   
                    (if (and (configf:lookup *configdat* "setup" "use-wal")
                             (string-match "^/tmp/.*" fname)) ;; this is a file in /tmp
                        (sqlite3:execute db "PRAGMA journal_mode=WAL;")
                        (debug:print 2 *default-log-port* "Creating " fname " in NON-WAL mode."))
                    (initproc db)))
@@ -334,11 +335,11 @@
           (if (and  (or (not dbfexists)
 			(and modtimedelta
 			     (> modtimedelta 10))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
 		    do-sync)
 	      (begin
-		(debug:print 4 *default-log-port* "filling db " (db:dbdat-get-path tmpdb) " with data \n    from " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta)
+		(debug:print 1 *default-log-port* "filling db " (db:dbdat-get-path tmpdb) " with data \n    from " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta)
 		(db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb refndb tmpdb)
                 (debug:print-info 13 *default-log-port* "db:sync-all-tables-list done.")
                 )
 	      (debug:print 4 *default-log-port* " db, " (db:dbdat-get-path tmpdb) " already exists or fresh enough, not propogating data from\n     " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta) )
 	  ;; (db:multi-db-sync dbstruct 'old2new))  ;; migrate data from megatest.db automatically
@@ -1581,11 +1582,11 @@
   
 
 ;;  select end_time-now from
 ;;      (select testname,item_path,event_time+run_duration as
 ;;                          end_time,strftime('%s','now') as now from tests where state in
-;;      ('RUNNING','REMOTEHOSTSTART','LAUNCED'));
+;;      ('RUNNING','REMOTEHOSTSTART','LAUNCHED'));
 
 (define (db:find-and-mark-incomplete dbstruct run-id ovr-deadtime)
   (let* ((incompleted '())
 	 (oldlaunched '())
 	 (toplevels   '())
@@ -1992,11 +1993,11 @@
 
 
 (define (db:get-run-times dbstruct run-patt target-patt)
 (let ((res `())
            (qry 	(conc "select runname, (max(end_time)-min(event_time))/60 as runtime, target from (select runname, run_id,tests.event_time,tests.event_time+run_duration AS end_time, " (string-join (db:get-keys dbstruct) " || '/' || ") " as target from tests inner join runs on tests.run_id = runs.id where runs.runname like ? and target like ?) group by run_id ;")))
-(print qry)
+;(print qry)
 (db:with-db 
    dbstruct
    #f ;; this is for the main runs db
    #f ;; does not modify db
    (lambda (db)
@@ -2536,11 +2537,11 @@
        (for-each 
 	(lambda (key)
 	  (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
 	    (sqlite3:for-each-row 
 	     (lambda (key-val)
-	       (set! res (cons (list key key-val) res)))
+	       (set! res (cons (list key (if (string? key-val) key-val "")) res))) ;; replace non-string bad values with empty string to prevent crashes. This scenario can happen when Megatest is killed on updating the db
 	     db qry run-id)))
 	keys)))
        (reverse res)))
 
 ;; get key vals for a given run-id
@@ -2554,11 +2555,11 @@
 	(lambda (key)
 	  (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;")))
 	    ;; (db:delay-if-busy dbdat)
 	    (sqlite3:for-each-row 
 	     (lambda (key-val)
-	       (set! res (cons key-val res)))
+	       (set! res (cons (if (string? key-val) key-val "") res))) ;; check that the key-val is a string for cases where a crash injected bad data in the megatest.db
 	     db qry run-id)))
 	keys)))
     (let ((final-res (reverse res)))
       (hash-table-set! *keyvals* run-id final-res)
       final-res)))
@@ -3136,10 +3137,27 @@
      (db:first-result-default
       db
       "SELECT rundir FROM tests WHERE id=?;"
       #f ;; default result
       test-id))))
+
+(define (db:get-test-times dbstruct run-name target)
+  (let ((res `())
+        (qry 	(conc "select testname, item_path, run_duration, " (string-join (db:get-keys dbstruct) " || '/' || ") " as target from tests inner join runs on tests.run_id = runs.id where runs.runname = ? and target = ?  ;")))
+   
+  (db:with-db 
+    dbstruct
+    #f ;; this is for the main runs db
+    #f ;; does not modify db
+    (lambda (db)
+            (sqlite3:for-each-row
+	(lambda (test-name item-path test-time target )
+	  (set! res (cons (vector test-name item-path test-time) res)))
+	db
+        qry 
+	run-name target)
+       res))))
 
 (define (db:get-test-times dbstruct run-name target)
   (let ((res `())
         (qry 	(conc "select testname, item_path, run_duration, " (string-join (db:get-keys dbstruct) " || '/' || ") " as target from tests inner join runs on tests.run_id = runs.id where runs.runname = ? and target = ?  ;")))
    
@@ -4200,72 +4218,125 @@
 	;;	    (conc (db:test-get-testname testdat)
 	;;		  "/"
 	;;		  (db:test-get-item-path testdat))))
 	 running-tests) ;; calling functions want the entire data
        '())
+
+   ;; collection of: for each waiton -
+   ;;   if this ref-test-name is an item in an itemized test and mode is itemwait/itemmatch:
+   ;;     if waiton is not itemized - if waiton is not both completed and in ok status, add as unmet prerequisite
+   ;;     if waiton is itemized:
+   ;;           and waiton's items are not expanded, add as unmet prerequisite
+   ;;           else if matching waiton item is not both completed and in an ok status, add as unmet prerequisite
+   ;;   else
+   ;;    if waiton toplevel is not in both completed and ok status, add as unmet prerequisite
+
    (if (or (not waitons)
 	   (null? waitons))
        '()
-       (let* ((unmet-pre-reqs '())
-	      (result         '()))
-	 (for-each 
+       (let* ((ref-test-itemized-mode (not (null? (lset-intersection eq? mode '(itemmatch itemwait)))))
+              (ref-test-toplevel-mode (not (null? (lset-intersection eq? mode '(toplevel)))))
+              (ref-test-is-toplevel   (equal? ref-item-path ""))
+              (ref-test-is-item       (not ref-test-is-toplevel))
+              (unmet-pre-reqs '())
+	      (result         '())
+              (unmet-prereq-items '())
+              )
+	 (for-each  ; waitons
 	  (lambda (waitontest-name)
 	    ;; by getting the tests with matching name we are looking only at the matching test 
 	    ;; and related sub items
 	    ;; next should be using mt:get-tests-for-run?
-	    (let ((tests             (db:get-tests-for-run-state-status dbstruct run-id waitontest-name))
+
+            (let (;(waiton-is-itemized ...)
+                  ;(waiton-items-are-expanded ...)
+                  (waiton-tests             (db:get-tests-for-run-state-status dbstruct run-id waitontest-name))
 		  (ever-seen         #f)
 		  (parent-waiton-met #f)
-		  (item-waiton-met   #f))
-	      (for-each 
-	       (lambda (test) ;; BB- this is the upstream test
-		 ;; (if (equal? waitontest-name (db:test-get-testname test)) ;; by defintion this had better be true ...
-		 (let* ((state             (db:test-get-state test))
-			(status            (db:test-get-status test))
-			(item-path         (db:test-get-item-path test)) ;; BB- this is the upstream itempath
-			(is-completed      (equal? state "COMPLETED"))
-			(is-running        (equal? state "RUNNING"))
-			(is-killed         (equal? state "KILLED"))
-			(is-ok             (member status '("PASS" "WARN" "CHECK" "WAIVED" "SKIP")))
+		  (item-waiton-met   #f)
+
+                  )
+	      (for-each ; test expanded from waiton
+	       (lambda (waiton-test) 
+		 (let* ((waiton-state             (db:test-get-state waiton-test))
+			(waiton-status            (db:test-get-status waiton-test))
+			(waiton-item-path         (db:test-get-item-path waiton-test)) ;; BB- this is the upstream itempath
+                        (waiton-is-toplevel       (equal? waiton-item-path ""))
+                        (waiton-is-item           (not waiton-is-toplevel))
+			(waiton-is-completed      (member waiton-state  *common:ended-states*))
+			(waiton-is-running        (member waiton-state  *common:running-states*))
+			(waiton-is-killed         (member waiton-state  *common:badly-ended-states*))
+			(waiton-is-ok             (member waiton-status *common:well-ended-states*))
 			;;                                       testname-b    path-a    path-b
-			(same-itempath     (db:compare-itempaths ref-test-name item-path ref-item-path itemmaps))) ;; (equal? ref-item-path item-path)))
+			(same-itempath     (db:compare-itempaths ref-test-name waiton-item-path ref-item-path itemmaps))) ;; (equal? ref-item-path waiton-item-path)))
 		   (set! ever-seen #t)
-		   (cond
-		    ;; case 1, non-item (parent test) is 
-		    ((and (equal? item-path "") ;; this is the parent test of the waiton being examined
-			  is-completed
-			  (or is-ok (not (null? (lset-intersection eq? mode '(toplevel)))))) ;;  itemmatch itemwait))))))
+                   ;;(BB> "***consider waiton "waiton-test"/"waiton-item-path"***")
+                   (cond
+                    ;; case 0 - toplevel of an itemized test, at least one item in prereq has completed
+                    ((and waiton-is-item ref-test-is-toplevel ref-test-itemized-mode waiton-is-completed)
+                     (set! parent-waiton-met #t))
+
+                    ;; case 1, non-item (parent test) is 
+		    ((and waiton-is-toplevel ;; this is the parent test of the waiton being examined
+			  waiton-is-completed
+                          ;;(BB> "cond1")
+			  (or waiton-is-ok ref-test-toplevel-mode)) ;;  itemmatch itemwait))))))
 		     (set! parent-waiton-met #t))
 		    ;; Special case for toplevel and KILLED
-		    ((and (equal? item-path "") ;; this is the parent test
-			  is-killed
+		    ((and waiton-is-toplevel ;; this is the parent test
+			  waiton-is-killed
 			  (member 'toplevel mode))
+                     ;;(BB> "cond2")
 		     (set! parent-waiton-met #t))
 		    ;; For itemwait mode IFF the previous matching item is good the set parent-waiton-met
-		    ((and (not (null? (lset-intersection eq? mode '(itemmatch itemwait)))) ;; how is that different from (member mode '(itemmatch itemwait)) ?????
-			  ;; (not (equal? item-path "")) ;; this applies to both top level (to allow launching of next batch) and items
-			  same-itempath)
-		     (if (and is-completed is-ok)
-			 (set! item-waiton-met #t))
-		     (if (and (equal? item-path "") ;; if upstream rollup test is completed, parent-waiton-met is set
-			      (or is-completed is-running));; this is the parent, set it to run if completed or running ;; BB1
+                    ((and ref-test-itemized-mode ref-test-is-item same-itempath)
+                     ;;(BB> "cond3")
+		     (if (and waiton-is-completed (or waiton-is-ok ref-test-toplevel-mode)) 
+                         (set! item-waiton-met #t)
+                         (set! unmet-prereq-items (cons waiton-test unmet-prereq-items)))
+                     (if (and waiton-is-toplevel ;; if upstream rollup test is completed, parent-waiton-met is set
+			      (or waiton-is-completed waiton-is-running))
 			 (set! parent-waiton-met #t)))
 		    ;; normal checking of parent items, any parent or parent item not ok blocks running
-		    ((and is-completed
-			  (or is-ok 
+		    ((and waiton-is-completed
+			  (or waiton-is-ok 
 			      (member 'toplevel mode))              ;; toplevel does not block on FAIL
-			  (and is-ok (member 'itemmatch mode))) ;; itemmatch blocks on not ok
-		     (set! item-waiton-met #t)))))
-	       tests)
+			  (and waiton-is-ok (member 'itemmatch mode) ;; itemmatch blocks on not ok
+                               ))
+                     ;;(BB> "cond4")
+		     (set! item-waiton-met #t))
+
+                    ((and waiton-is-completed waiton-is-ok same-itempath)
+                     ;;(BB> "cond5")
+                     (set! item-waiton-met #t))
+                    (else
+                     #t
+                     ;;(BB> "condelse")
+                     ))))
+               waiton-tests)
 	      ;; both requirements, parent and item-waiton must be met to NOT add item to
 	      ;; prereq's not met list
-	      (if (not (or parent-waiton-met item-waiton-met))
-		  (set! result (append (if (null? tests) (list waitontest-name) tests) result))) ;; appends the string if the full record is not available
+               ;; (BB>
+               ;;  "\n* waiton-tests           "waiton-tests
+               ;;  "\n* parent-waiton-met      "parent-waiton-met
+               ;;  "\n* item-waiton-met        "item-waiton-met
+               ;;  "\n* ever-seen              "ever-seen
+               ;;  "\n* ref-test-itemized-mode "ref-test-itemized-mode
+               ;;  "\n* unmet-prereq-items     "unmet-prereq-items
+               ;;  "\n* result (pre)           "result
+               ;;  "\n* ever-seen              "ever-seen
+               ;;  "\n")
+
+              (cond
+               ((and ref-test-itemized-mode ref-test-is-item (not (null? unmet-prereq-items)))
+                (set! result (append unmet-prereq-items result)))
+               ((not (or parent-waiton-met item-waiton-met))
+                (set! result (append (if (null? waiton-tests) (list waitontest-name) waiton-tests) result))) ;; appends the string if the full record is not available
 	      ;; if the test is not found then clearly the waiton is not met...
 	      ;; (if (not ever-seen)(set! result (cons waitontest-name result)))))
-	      (if (not ever-seen)
-		  (set! result (append (if (null? tests)(list waitontest-name) tests) result)))))
+               ((not ever-seen)
+                (set! result (append (if (null? waiton-tests)(list waitontest-name) waiton-tests) result))))))
 	  waitons)
 	 (delete-duplicates result)))))
 
 ;;======================================================================
 ;; Just for sync, procedures to make sync easy

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="generator" content="AsciiDoc 8.6.9">
+<meta name="generator" content="AsciiDoc 8.6.7">
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -84,20 +84,14 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-.monospaced, code, pre {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
+pre {
   padding: 0;
   margin: 0;
 }
-pre {
-  white-space: pre-wrap;
-}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -222,11 +216,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; vertical-align: text-bottom; }
+span.image img { border-style: none; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -417,10 +411,16 @@
 
 /*
  * xhtml11 specific
  *
  * */
+
+tt {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
+}
 
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
@@ -450,10 +450,16 @@
 
 /*
  * html5 specific
  *
  * */
+
+.monospaced {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
+}
 
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
@@ -530,12 +536,10 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
-
-
 @media screen {
   body {
     max-width: 50em; /* approximately 80 characters wide */
     margin-left: 16em;
   }
@@ -1943,11 +1947,11 @@
 COMPLETED/ xterm -e bash -s --</pre>
 </div></div>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/usr/images/icons/note.png" alt="Note">
+<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
 </td>
 <td class="content">There is a trailing space after the --</td>
 </tr></table>
 </div>
 <div class="paragraph"><p>There are a number of environment variables available to the trigger script
@@ -2089,11 +2093,10 @@
 <div class="paragraph"><p>Then in runconfigs.config</p></div>
 <div class="listingblock">
 <div class="title">Example of using modified.config in a testconfig</div>
 <div class="content monospaced">
 <pre>cat testconfig
-
 [pre-launch-env-vars]
 [include modified.config]</pre>
 </div></div>
 </div>
 </div>
@@ -2124,10 +2127,66 @@
 <div class="content monospaced">
 <pre># use -precmd 'sleep 5;nbfake' to limit overloading the host computer but to allow the removes to run in parallel.
 megatest -actions print,remove-runs -remove-keep 3 -target %/%/%/% -runname % -age 1w -precmd 'sleep 5;nbfake'"</pre>
 </div></div>
 </div>
+</div>
+<div class="sect1">
+<h2 id="_nested_runs">Nested Runs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>A Megatest test can run a full Megatest run in either the same
+Megatest area or in another area. This is a powerful way of chaining
+complex suites of tests and or actions.</p></div>
+<div class="paragraph"><p>If you are not using the current area you can use ezsteps to retrieve
+and setup the sub-Megatest run area.</p></div>
+<div class="paragraph"><p>In the testconfig:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[subrun]
+
+# Required: wait for the run or just launch it
+#           if no then the run will be an automatic PASS irrespective of the actual result
+runwait yes|no
+
+# Optional: where to execute the run. Default is the current runarea
+runarea /some/path/to/megatest/area
+
+# Optional: method to use to determine pass/fail status of the run
+#   auto (default) - roll up the net state/status of the sub-run
+#   logpro         - use the provided logpro rules, happens automatically if there is a logpro section
+# passfail auto|logpro
+# Example of logpro:
+passfail logpro
+
+# Optional:
+logpro ;; if this section exists then logpro is used to determine pass/fail
+  (expect:required in "LogFileBody" &gt;= 1 "At least one pass" #/PASS/)
+
+# Optional: target translator, default is to use the parent target
+target #{shell somescript.sh}
+
+# Optional: runname translator/generator, default is to use the parent runname
+runname #{somescript.sh}
+
+# Optional: testpatt spec, default is to first look for TESTPATT spec from runconfigs unless there is a contour spec
+testpatt %/item1,test2
+
+# Optional: contour spec, use the named contour from the megatest.config contour spec
+contour contourname ### NOTE: Not implemented yet! Let us know if you need this feature.
+
+# Optional: mode-patt, use this spec for testpatt from runconfigs
+mode-patt TESTPATT
+
+# Optional: tag-expr, use this tag-expr to select tests
+tag-expr quick
+
+# Optional: (not yet implemented), propagate these actions from the parent
+#           test
+#   Note// default is % for all
+propagate remove-runs archive ...</pre>
+</div></div>
+</div>
 </div>
 <div class="sect1">
 <h2 id="_programming_api">Programming API</h2>
 <div class="sectionbody">
 <div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
@@ -2185,11 +2244,10 @@
 </div>
 <div id="footnotes"><hr></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.0<br>
-Last updated
- 2017-06-25 21:04:36 MST
+Last updated 2017-08-30 10:42:37 PDT
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -608,11 +608,10 @@
 Then in runconfigs.config
 
 .Example of using modified.config in a testconfig
 ------------------------------
 cat testconfig
-
 [pre-launch-env-vars]
 [include modified.config]
 ------------------------------
 
 Managing Old Runs
@@ -628,10 +627,68 @@
 ---------------------
 # use -precmd 'sleep 5;nbfake' to limit overloading the host computer but to allow the removes to run in parallel.
 megatest -actions print,remove-runs -remove-keep 3 -target %/%/%/% -runname % -age 1w -precmd 'sleep 5;nbfake'"
 ---------------------
 
+Nested Runs
+-----------
+
+A Megatest test can run a full Megatest run in either the same
+Megatest area or in another area. This is a powerful way of chaining
+complex suites of tests and or actions.
+
+If you are not using the current area you can use ezsteps to retrieve
+and setup the sub-Megatest run area.
+
+In the testconfig:
+---------------
+[subrun]
+
+# Required: wait for the run or just launch it
+#           if no then the run will be an automatic PASS irrespective of the actual result
+runwait yes|no
+
+# Optional: where to execute the run. Default is the current runarea
+runarea /some/path/to/megatest/area
+
+# Optional: method to use to determine pass/fail status of the run
+#   auto (default) - roll up the net state/status of the sub-run
+#   logpro         - use the provided logpro rules, happens automatically if there is a logpro section
+# passfail auto|logpro
+# Example of logpro:
+passfail logpro
+
+# Optional: 
+logpro ;; if this section exists then logpro is used to determine pass/fail
+  (expect:required in "LogFileBody" >= 1 "At least one pass" #/PASS/)
+  (expect:fail     in "LogFileBody"  = 0 "No FAILs allowed"  #/FAIL/)
+
+# Optional: target translator, default is to use the parent target
+target #{shell somescript.sh}
+
+# Optional: runname translator/generator, default is to use the parent runname
+runname #{somescript.sh}
+
+# Optional: testpatt spec, default is to first look for TESTPATT spec from runconfigs unless there is a contour spec
+testpatt %/item1,test2
+
+# Optional: contour spec, use the named contour from the megatest.config contour spec
+contour contourname ### NOTE: Not implemented yet! Let us know if you need this feature.
+
+# Optional: mode-patt, use this spec for testpatt from runconfigs
+mode-patt TESTPATT
+
+# Optional: tag-expr, use this tag-expr to select tests
+tag-expr quick
+
+# Optional: (not yet implemented), propagate these actions from the parent
+#           test
+#   Note// default is % for all
+propagate remove-runs archive ...
+
+---------------
+
 Programming API
 ---------------
 
 These routines can be called from the megatest repl. 
 

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

Index: gutils.scm
==================================================================
--- gutils.scm
+++ gutils.scm
@@ -36,10 +36,11 @@
   ;; ((if get-label cadr car)
   (case (string->symbol state)
     ((COMPLETED) ;; ARCHIVED)
      (case (string->symbol status)
        ((PASS)        (list "70  249 73" status))
+       ((PREQ_FAIL PREQ_DISCARDED) (list "255 127 127" status))
        ((WARN WAIVED) (list "255 172 13" status))
        ((SKIP)        (list (gutils:get-color-spec 'SKIP) status))
        ((ABORT)       (list "198 36 166" status))
        (else (list "253 33 49" status))))
     ((ARCHIVED)

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -79,20 +79,31 @@
 	#f)))
 
 (define (launch:runstep ezstep run-id test-id exit-info m tal testconfig)
   (let* ((stepname       (car ezstep))  ;; do stuff to run the step
 	 (stepinfo       (cadr ezstep))
-	 (stepparts      (string-match (regexp "^(\\{([^\\}]*)\\}\\s*|)(.*)$") stepinfo))
-	 (stepparms      (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each 
+	;; (let ((info (cadr ezstep)))
+	;; 		   (if (proc? info) "" info)))
+	;; (stepproc       (let ((info (cadr ezstep)))
+	;; 		   (if (proc? info) info #f)))
+	 (stepparts      (string-match (regexp "^(\\{([^\\}\\{]*)\\}\\s*|)(.*)$") stepinfo))
+	 (stepparams     (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each
+	 (paramparts     (if (string? stepparams)
+			     (map (lambda (x)(string-split x "=")) (string-split-fields "[^;]*=[^;]*" stepparams))
+			     '()))
+	 (subrun         (alist-ref "subrun" paramparts equal?))
 	 (stepcmd        (list-ref stepparts 3))
 	 (script         "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!\
 	 (logpro-file    (conc stepname ".logpro"))
 	 (html-file      (conc stepname ".html"))
 	 (dat-file       (conc stepname ".dat"))
 	 (tconfig-logpro (configf:lookup testconfig "logpro" stepname))
 	 (logpro-used    (common:file-exists? logpro-file)))
 
+    (debug:print 0 *default-log-port* "stepparts: " stepparts ", stepparams: " stepparams
+                 ", paramparts: " paramparts ", subrun: " subrun ", stepcmd: " stepcmd)
+    
     (if (and tconfig-logpro
 	     (not logpro-used)) ;; no logpro file found but have a defn in the testconfig
 	(begin
 	  (with-output-to-file logpro-file
 	    (lambda ()
@@ -101,11 +112,11 @@
 	      (print tconfig-logpro)))
 	  (set! logpro-used #t)))
     
     ;; NB// can safely assume we are in test-area directory
     (debug:print 4 *default-log-port* "ezsteps:\n stepname: " stepname " stepinfo: " stepinfo " stepparts: " stepparts
-		 " stepparms: " stepparms " stepcmd: " stepcmd)
+		 " stepparams: " stepparams " stepcmd: " stepcmd)
     
     ;; ;; first source the previous environment
     ;; (let ((prev-env (conc ".ezsteps/" prevstep (if (string-search (regexp "csh") 
     ;;      							 (get-environment-variable "SHELL")) ".csh" ".sh"))))
     ;;   (if (and prevstep (common:file-exists? prev-env))
@@ -119,12 +130,19 @@
     ;; now launch the actual process
     (call-with-environment-variables 
      (list (cons "PATH" (conc (get-environment-variable "PATH") ":.")))
      (lambda () ;; (process-run "/bin/bash" "-c" "exec ls -l /tmp/foobar > /tmp/delme-more.log 2>&1")
        (let* ((cmd (conc stepcmd " > " stepname ".log 2>&1")) ;; >outfile 2>&1 
-	      (pid (process-run "/bin/bash" (list "-c" cmd))))
-
+	      (pid #f))
+	 (let ((proc (lambda ()
+		       (set! pid (process-run "/bin/bash" (list "-c" cmd))))))
+	   (if subrun
+               (begin
+                 (debug:print-info 0 *default-log-port* "Running without MT_.* environment variables.")
+                 (common:without-vars proc "^MT_.*"))
+	       (proc)))
+	 
          (with-output-to-file "Makefile.ezsteps"
            (lambda ()
              (print stepname ".log :")
              (print "\t" cmd)
              (if (common:file-exists? (conc stepname ".logpro"))
@@ -240,11 +258,11 @@
 	 (launch:einf-rollup-status-set! exit-info 1) ;; (vector-set! exit-info 3 1) ;; force fail, this used to be next-state but that doesn't make sense. should always be "COMPLETED" 
 	 (tests:test-set-status! run-id test-id "COMPLETED" "FAIL" (conc "Failed at step " stepname) #f)
 	 )))
     logpro-used))
 
-(define (launch:manage-steps run-id test-id item-path fullrunscript ezsteps test-name tconfigreg exit-info m)
+(define (launch:manage-steps run-id test-id item-path fullrunscript ezsteps subrun test-name tconfigreg exit-info m)
   ;; (let-values
   ;;  (((pid exit-status exit-code)
   ;;    (run-n-wait fullrunscript)))
   ;; (tests:test-set-status! test-id "RUNNING" "n/a" #f #f)
   ;; Since we should have a clean slate at this time there is no need to do 
@@ -274,11 +292,11 @@
 		 (thread-sleep! 2)
 		 (loop (+ i 1)))
 	       )))))
   ;; then, if runscript ran ok (or did not get called)
   ;; do all the ezsteps (if any)
-  (if ezsteps
+  (if (or ezsteps subrun)
       (let* ((testconfig ;; (read-config (conc work-area "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) ;; FIXME??? is allow-system ok here?
 	      ;; NOTE: it is tempting to turn off force-create of testconfig but dynamic
 	      ;;       ezstep names need a full re-eval here.
 	      (tests:get-testconfig test-name item-path tconfigreg #t force-create: #t)) ;; 'return-procs)))
 	     (ezstepslst (if (hash-table? testconfig)
@@ -293,29 +311,86 @@
 	;; after all that, still no testconfig? Time to abort
 	(if (not testconfig)
 	    (begin
 	      (debug:print-error 0 *default-log-port* "Failed to resolve megatest.config, runconfigs.config and testconfig issues. Giving up now")
 	      (exit 1)))
-	(if (not (common:file-exists? ".ezsteps"))(create-directory ".ezsteps"))
-	;; if ezsteps was defined then we are sure to have at least one step but check anyway
-	(if (not (> (length ezstepslst) 0))
-	    (debug:print-error 0 *default-log-port* "ezsteps defined but ezstepslst is zero length")
-	    (let loop ((ezstep (car ezstepslst))
-		       (tal    (cdr ezstepslst))
-		       (prevstep #f))
-	      ;; check exit-info (vector-ref exit-info 1)
-	      (if (launch:einf-exit-status exit-info) ;; (vector-ref exit-info 1)
-		  (let ((logpro-used (launch:runstep ezstep run-id test-id exit-info m tal testconfig))
-			(stepname    (car ezstep)))
-		    ;; if logpro-used read in the stepname.dat file
-		    (if (and logpro-used (common:file-exists? (conc stepname ".dat")))
-			(launch:load-logpro-dat run-id test-id stepname))
-		    (if (steprun-good? logpro-used (launch:einf-exit-code exit-info))
-			(if (not (null? tal))
-			    (loop (car tal) (cdr tal) stepname))
-			(debug:print 4 *default-log-port* "WARNING: step " (car ezstep) " failed. Stopping")))
-		  (debug:print 4 *default-log-port* "WARNING: a prior step failed, stopping at " ezstep)))))))
+
+	;; create a proc for the subrun if requested, save that proc in the ezsteps table as the last entry
+	;; 1. get section [runarun]
+	;; 2. unset MT_* vars
+	;; 3. fix target
+	;; 4. fix runname
+	;; 5. fix testpatt or calculate it from contour
+	;; 6. launch the run
+	;; 7. roll up the run result and or roll up the logpro processed result
+	(if (configf:lookup testconfig "subrun" "runwait") ;; we use runwait as the flag that a subrun is requested
+	    (let* ((runarea   (configf:lookup testconfig "subrun" "runarea"))
+		   (passfail  (configf:lookup testconfig "subrun" "passfail"))
+		   (target    (configf:lookup testconfig "subrun" "target"))
+		   (runname   (configf:lookup testconfig "subrun" "runname"))
+		   (contour   (configf:lookup testconfig "subrun" "contour"))
+		   (testpatt  (configf:lookup testconfig "subrun" "testpatt"))
+		   (mode-patt (configf:lookup testconfig "subrun" "mode-patt"))
+		   (tag-expr  (configf:lookup testconfig "subrun" "tag-expr"))
+		   (run-wait  (configf:lookup testconfig "subrun" "runwait"))
+		   (logpro    (configf:lookup testconfig "subrun" "logpro"))
+		   (compact-stem (string-substitute "[/*]" "_" (conc target "-" runname "-" (or testpatt mode-patt tag-expr))))
+		   (log-file (conc compact-stem ".log"))
+		   (mt-cmd    (conc "megatest -run -target " target
+				    " -runname " (or runname (get-environment-variable "MT_RUNNAME"))
+				    (conc " -start-dir " (if runarea runarea *toppath*))
+				    (if testpatt  (conc " -testpatt " testpatt)  "")
+				    (if mode-patt (conc " -modepatt " mode-patt) "")
+				    (if tag-expr  (conc " -tag-expr"  tag-expr)  "")
+				    (if (equal? run-wait "yes") " -run-wait " "")
+				    " -log " log-file)))
+	      ;; change directory to runarea, create it if needed, we do NOT create the directory 
+	;; (if runarea
+	;;     (if (directory-exists? runarea)
+	;;         (change-directory runarea)
+	;;         (begin
+	;;   	(debug:print 0 *default-log-port* "ERROR: for sub-megatest run the runarea \"" runarea "\" does not exist! EXITING.")
+	;;   	(exit 1))))
+	      ;; (let ((subrun (conc *toppath* "/subrun") #t))
+	      ;; 	 (create-directory subrun)
+	      ;; 	 (change-directory subrun)))
+	      
+	      ;; by this point we are in the right place to run the subrun and we have a Megatest command to run
+	      ;; (filter (lambda (x)(string-match "MT_.*" (car x))) (get-environment-variables))
+	      ;; (common:without-vars mt-cmd "^MT_.*")
+              (debug:print-info 0 *default-log-port* "Subrun command is \"" mt-cmd "\"")
+              (set! ezsteps #t) ;; set the needed flag
+	      (set! ezstepslst (append (or ezstepslst '())
+                                       (list (list "subrun" (conc "{subrun=true} " mt-cmd)))))
+	      (configf:set-section-var testconfig "logpro" "subrun" logpro) ;; append the logpro rules to the logpro section as stepname subrun
+              (if runarea (configf:set-section-var testconfig "setup" "submegatest" runarea))
+              (configf:write-alist testconfig "testconfig.subrun")
+	      ))
+
+	;; process the ezsteps
+	(if ezsteps
+	    (begin
+	      (if (not (common:file-exists? ".ezsteps"))(create-directory ".ezsteps"))
+	      ;; if ezsteps was defined then we are sure to have at least one step but check anyway
+	      (if (not (> (length ezstepslst) 0))
+		  (debug:print-error 0 *default-log-port* "ezsteps defined but ezstepslst is zero length")
+		  (let loop ((ezstep (car ezstepslst))
+			     (tal    (cdr ezstepslst))
+			     (prevstep #f))
+                    (debug:print-info 0 *default-log-port* "Processing ezstep \"" (string-intersperse ezstep " ") "\"")
+		    ;; check exit-info (vector-ref exit-info 1)
+		    (if (launch:einf-exit-status exit-info) ;; (vector-ref exit-info 1)
+			(let ((logpro-used (launch:runstep ezstep run-id test-id exit-info m tal testconfig))
+			      (stepname    (car ezstep)))
+			  ;; if logpro-used read in the stepname.dat file
+			  (if (and logpro-used (common:file-exists? (conc stepname ".dat")))
+			      (launch:load-logpro-dat run-id test-id stepname))
+			  (if (steprun-good? logpro-used (launch:einf-exit-code exit-info))
+			      (if (not (null? tal))
+				  (loop (car tal) (cdr tal) stepname))
+			      (debug:print 0 *default-log-port* "WARNING: step " (car ezstep) " failed. Stopping")))
+			(debug:print 0 *default-log-port* "WARNING: a prior step failed, stopping at " ezstep)))))))))
 
 (define (launch:monitor-job run-id test-id item-path fullrunscript ezsteps test-name tconfigreg exit-info m work-area runtlim misc-flags)
   (let* ((update-period (string->number (or (configf:lookup *configdat* "setup" "test-stats-update-period") "30")))
          (start-seconds (current-seconds))
 	 (calc-minutes  (lambda ()
@@ -423,10 +498,11 @@
 	       (top-path  (assoc/default 'toppath   cmdinfo))
 	       (work-area (assoc/default 'work-area cmdinfo))  ;; work-area is the test run area
 	       (test-name (assoc/default 'test-name cmdinfo))
 	       (runscript (assoc/default 'runscript cmdinfo))
 	       (ezsteps   (assoc/default 'ezsteps   cmdinfo))
+	       (subrun    (assoc/default 'subrun    cmdinfo))
 	       ;; (runremote (assoc/default 'runremote cmdinfo))
 	       ;; (transport (assoc/default 'transport cmdinfo))  ;; not used
 	       ;; (serverinf (assoc/default 'serverinf cmdinfo))
 	       ;; (port      (assoc/default 'port      cmdinfo))
 	       (serverurl (assoc/default 'serverurl cmdinfo))
@@ -500,14 +576,26 @@
 	  (setenv "MT_TESTSUITENAME" areaname)
 	  (setenv "MT_RUN_AREA_HOME" top-path)
 	  (set! *toppath* top-path)
           (change-directory *toppath*) ;; temporarily switch to the run area home
 	  (setenv "MT_TEST_RUN_DIR"  work-area)
+
+	  (launch:setup) ;; should be properly in the run area home now
+
+	  (if contour (setenv "MT_CONTOUR" contour))
+	  
+	  ;; immediated set some key variables from CMDINFO data, yes, these will be set again below ...
+	  ;;
+	  (setenv "MT_TESTSUITENAME" areaname)
+	  (setenv "MT_RUN_AREA_HOME" top-path)
+	  (set! *toppath* top-path)
+          (change-directory *toppath*) ;; temporarily switch to the run area home
+	  (setenv "MT_TEST_RUN_DIR"  work-area)
 
 	  (launch:setup) ;; should be properly in the run area home now
           
-	  (set! tconfigreg (tests:get-all))
+	  (set! tconfigreg (tests:get-all)) ;; mapping of testname => test source path
 	  (let ((sighand (lambda (signum)
 			   ;; (signal-mask! signum) ;; to mask or not? seems to cause issues in exiting
 			   (if (eq? signum signal/stop)
 			       (debug:print-error 0 *default-log-port* "attempt to STOP process. Exiting."))
 			   (set! *time-to-exit* #t)
@@ -582,20 +670,22 @@
 	  (let ((rconfig (full-runconfigs-read)) ;; (read-config (conc  *toppath* "/runconfigs.config") #f #t sections: (list "default" target))))
 		(wconfig (read-config "waivers.config" #f #t sections: `( "default" ,target )))) ;; read the waivers config if it exists
 	    ;; (setup-env-defaults (conc *toppath* "/runconfigs.config") run-id (make-hash-table) keyvals target)
 	    ;; (set-run-config-vars run-id keyvals target) ;; (db:get-target db run-id))
 	    ;; Now have runconfigs data loaded, set environment vars
-	    (for-each (lambda (section)
-			(for-each (lambda (varval)
-				    (let ((var (car varval))
-					  (val (cadr varval)))
-				      (if (and (string? var)(string? val))
-					  (begin
-					    (setenv var (config:eval-string-in-environment val))) ;; val)
-					  (debug:print-error 0 *default-log-port* "bad variable spec, " var "=" val))))
-				  (configf:get-section rconfig section)))
-		      (list "default" target)))
+	    (for-each
+	     (lambda (section)
+	       (for-each
+		(lambda (varval)
+		  (let ((var (car varval))
+			(val (cadr varval)))
+		    (if (and (string? var)(string? val))
+			(begin
+			  (setenv var (config:eval-string-in-environment val))) ;; val)
+			(debug:print-error 0 *default-log-port* "bad variable spec, " var "=" val))))
+		(configf:get-section rconfig section)))
+	     (list "default" target)))
           ;;(bb-check-path msg: "launch:execute post block 1")
 
 	  ;; NFS might not have propagated the directory meta data to the run host - give it time if needed
 	  (let loop ((count 0))
 	    (if (or (common:file-exists? work-area)
@@ -691,11 +781,11 @@
 		 ;; (keep-going   #t)
 		 (misc-flags   (let ((ht (make-hash-table)))
 				 (hash-table-set! ht 'keep-going #t)
 				 ht))
 		 (runit        (lambda ()
-				 (launch:manage-steps run-id test-id item-path fullrunscript ezsteps test-name tconfigreg exit-info m)))
+				 (launch:manage-steps run-id test-id item-path fullrunscript ezsteps subrun test-name tconfigreg exit-info m)))
 		 (monitorjob   (lambda ()
 				 (launch:monitor-job  run-id test-id item-path fullrunscript ezsteps test-name tconfigreg exit-info m work-area runtlim misc-flags)))
 		 (th1          (make-thread monitorjob "monitor job"))
 		 (th2          (make-thread runit "run job")))
 	    (set! job-thread th2)
@@ -1017,11 +1107,11 @@
         ;; one more attempt to cache the configs for future reading
         (let* ((cachefiles   (launch:get-cache-file-paths areapath toppath target mtconfig))
                (mtcachef     (car cachefiles))
                (rccachef     (cdr cachefiles)))
 
-          ;; trap exception due to stale NFS handle -- Error: (open-output-file) cannot open file - Stale NFS file handle: "/p/fdk/gwa/lefkowit/mtTesting/qa/primbeqa/links/p1222/11/PDK_r1.1.1/prim/clean/pcell_testgen/.runconfigs.cfg-1.6427-7d1e789cb3f62f9cde719a4865bb51b3c17ea853" - ticket 220546342
+          ;; trap exception due to stale NFS handle -- Error: (open-output-file) cannot open file - Stale NFS file handle: "...somepath.../.runconfigs.cfg-1.6427-7d1e789cb3f62f9cde719a4865bb51b3c17ea853" - ticket 220546342
           ;; TODO - consider 1) using simple-lock to bracket cache write
           ;;                 2) cache in hash on server, since need to do rmt: anyway to lock.
           (if (and rccachef *runconfigdat* (not (common:file-exists? rccachef)))
               (common:fail-safe
                (lambda ()
@@ -1301,11 +1391,12 @@
 				  (if (equal? ush "no") ;; must use "no" to NOT use shell
 				      #f
 				      ush)
 				  #t)))     ;; default is yes
 	   (runscript       (config-lookup tconfig   "setup"        "runscript"))
-	   (ezsteps         (> (length (hash-table-ref/default tconfig "ezsteps" '())) 0)) ;; don't send all the steps, could be big
+	   (ezsteps         (> (length (hash-table-ref/default tconfig "ezsteps" '())) 0)) ;; don't send all the steps, could be big, just send a flag
+	   (subrun          (> (length (hash-table-ref/default tconfig "subrun"  '())) 0)) ;; send a flag to process a subrun
 	   ;; (diskspace       (config-lookup tconfig   "requirements" "diskspace"))
 	   ;; (memory          (config-lookup tconfig   "requirements" "memory"))
 	   ;; (hosts           (config-lookup *configdat* "jobtools"     "workhosts")) ;; I'm pretty sure this was never completed
 	   (remote-megatest (config-lookup *configdat* "setup" "executable"))
 	   (run-time-limit  (or (configf:lookup  tconfig   "requirements" "runtimelim")
@@ -1385,11 +1476,12 @@
 					(list 'run-id    run-id   )
 					(list 'test-id   test-id  )
 					;; (list 'item-path item-path )
 					(list 'itemdat   itemdat  )
 					(list 'megatest  remote-megatest)
-					(list 'ezsteps   ezsteps) 
+					(list 'ezsteps   ezsteps)
+					(list 'subrun    subrun)
 					(list 'target    mt_target)
 					(list 'contour   contour)
 					(list 'runtlim   (if run-time-limit (common:hms-string->seconds run-time-limit) #f))
 					(list 'env-ovrd  (hash-table-ref/default *configdat* "env-override" '())) 
 					(list 'set-vars  (if params (hash-table-ref/default params "-setvars" #f)))

Index: megatest.config
==================================================================
--- megatest.config
+++ megatest.config
@@ -20,11 +20,11 @@
 #           the target translator can return: a/target OR (list/of targets/to apply/run) OR #f i.e. run nothing
 # ext-tests path=ext-tests; targtrans=prefix-contour;
 ext       path=ext-tests
 
 [contours]
-#     mode-patt/tag-expr
+#     selector=tag-expr/mode-patt
 quick areas=ext;    selector=/QUICKPATT
 quick2 areafn=check-area; selector=/QUICKPATT
 # quick areas=fullrun,ext-tests; selector=QUICKPATT/quick
 # full  areas=fullrun,ext-tests; selector=MAXPATT/
 # short areas=fullrun,ext-tests; selector=MAXPATT/

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -1618,11 +1618,11 @@
        (lambda (target runname keys keyvals)
          (if (args:get-arg "-rerun-clean") ;; first set states/statuses correct
              (let ((states   (or (configf:lookup *configdat* "validvalues" "cleanrerun-states")
                                  "KILLREQ,KILLED,UNKNOWN,INCOMPLETE,STUCK,NOT_STARTED"))
                    (statuses (or (configf:lookup *configdat* "validvalues" "cleanrerun-statuses")
-                                 "FAIL,INCOMPLETE,ABORT,CHECK,DEAD")))
+                                 "FAIL,INCOMPLETE,ABORT,CHECK,DEAD,PREQ_FAIL,PREQ_DISCARDED")))
                (hash-table-set! args:arg-hash "-preclean" #t)
                (runs:operate-on 'set-state-status
                                 target
                                 (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
                                 "%" ;; (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -177,38 +177,51 @@
 		   (event-time    (db:test-get-event_time   test-dat))
 		   (tconfig       #f)
 		   (state         (if newstate  newstate  (db:test-get-state  test-dat)))
 		   (status        (if newstatus newstatus (db:test-get-status test-dat))))
 	      ;; (mutex-lock! *triggers-mutex*)
-	      (if (and test-name
-		       test-rundir)   ;; #f means no dir set yet
-		       ;; (common:file-exists? test-rundir)
-		       ;; (directory? test-rundir))
-		  (call-with-environment-variables
-		   (list (cons "MT_TEST_NAME"    (or test-name "no such test"))
-			 (cons "MT_TEST_RUN_DIR" (or test-rundir "no test directory yet"))
-			 (cons "MT_ITEMPATH"     (or item-path "")))
-		   (lambda ()
-		     (if (directory-exists? test-rundir)
-			 (push-directory test-rundir)
-			 (push-directory *toppath*))
-		     (set! tconfig (mt:lazy-read-test-config test-name))
-		     (for-each (lambda (trigger)
-				 (let* ((munged-trigger (string-translate trigger "/ " "--"))
+              (handle-exceptions
+               exn
+               (begin
+                 (debug:print-error 0 *default-log-port* " Exception in mt:process-triggers for run-id="run-id" test-id="test-id" newstate="newstate" newstatus="newstatus
+                                    "\n   error: " ((condition-property-accessor 'exn 'message) exn)
+                                    "\n   test-rundir="test-rundir
+                                    "\n   test-name="test-name
+                                    "\n   item-path="item-path
+                                    "\n   state="state
+                                    "\n   status="status
+                                    "\n")
+                 (print-call-chain (current-error-port))
+                 #f)
+               (if (and test-name
+                        test-rundir)   ;; #f means no dir set yet
+                   ;; (common:file-exists? test-rundir)
+                   ;; (directory? test-rundir))
+                   (call-with-environment-variables
+                    (list (cons "MT_TEST_NAME"    (or test-name "no such test"))
+                          (cons "MT_TEST_RUN_DIR" (or test-rundir "no test directory yet"))
+                          (cons "MT_ITEMPATH"     (or item-path "")))
+                    (lambda ()
+                      (if (directory-exists? test-rundir)
+                          (push-directory test-rundir)
+                          (push-directory *toppath*))
+                      (set! tconfig (mt:lazy-read-test-config test-name))
+                      (for-each (lambda (trigger)
+                                  (let* ((munged-trigger (string-translate trigger "/ " "--"))
 					(logname        (conc "last-trigger-" munged-trigger ".log")))
-				   ;; first any triggers from the testconfig
-				   (let ((cmd  (configf:lookup tconfig "triggers" trigger)))
-				     (if cmd (mt:run-trigger cmd test-id test-rundir trigger (conc "tconfig-" logname) test-name item-path event-time state status)))
-				   ;; next any triggers from megatest.config
-				   (let ((cmd  (configf:lookup *configdat* "triggers" trigger)))
-				     (if cmd (mt:run-trigger cmd test-id test-rundir trigger (conc "mtconfig-" logname) test-name item-path event-time state status)))))
-			       (list
-				(conc state "/" status)
-				(conc state "/")
-				(conc "/" status)))
+                                    ;; first any triggers from the testconfig
+                                    (let ((cmd  (configf:lookup tconfig "triggers" trigger)))
+                                      (if cmd (mt:run-trigger cmd test-id test-rundir trigger (conc "tconfig-" logname) test-name item-path event-time state status)))
+                                    ;; next any triggers from megatest.config
+                                    (let ((cmd  (configf:lookup *configdat* "triggers" trigger)))
+                                      (if cmd (mt:run-trigger cmd test-id test-rundir trigger (conc "mtconfig-" logname) test-name item-path event-time state status)))))
+                                (list
+                                 (conc state "/" status)
+                                 (conc state "/")
+                                 (conc "/" status)))
 		     (pop-directory))
-		   ))
+                    )))
 	      ;; (mutex-unlock! *triggers-mutex*)
 	      )))))
 
 ;;======================================================================
 ;;  S T A T E   A N D   S T A T U S   F O R   T E S T S 

Index: process.scm
==================================================================
--- process.scm
+++ process.scm
@@ -74,16 +74,19 @@
   (let* ((fh (open-input-pipe cmd))
          (res (port-proc->list fh proc))
          (status (close-input-pipe fh)))
     (if (eq? status 0) res #f)))
 
-(define (process:cmd-run->list cmd)
-  (let* ((fh (open-input-pipe cmd))
-         (res (port->list fh))
-         (status (close-input-pipe fh)))
-    (list res status)))
-
+(define (process:cmd-run->list cmd #!key (delta-env-alist-or-hash-table '()))
+  (common:with-env-vars
+   delta-env-alist-or-hash-table
+   (lambda ()
+     (let* ((fh (open-input-pipe cmd))
+            (res (port->list fh))
+            (status (close-input-pipe fh)))
+       (list res status)))))
+   
 (define (port->list fh)
   (if (eof-object? fh) #f
       (let loop ((curr (read-line fh))
                  (result '()))
         (if (not (eof-object? curr))

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -191,11 +191,12 @@
 	   (args:get-arg "-one-pass"))
       (exit 0))
   (thread-sleep! (cond
         	  ((> (runs:dat-can-run-more-tests-count runsdat) 20)
 		   (if (runs:lownoise "waiting on tasks" 60)(debug:print-info 2 *default-log-port* "waiting for tasks to complete, sleeping briefly ..."))
-		   2);; obviously haven't had any work to do for a while
+                   (configf:lookup-number *configdat* "setup" "inter-test-delay" default: 0.1) ;; was 2
+		   );; obviously haven't had any work to do for a while
         	  (else 0)))
   (let* ((num-running             (rmt:get-count-tests-running run-id))
 	 (num-running-in-jobgroup (rmt:get-count-tests-running-in-jobgroup run-id jobgroup))
 	 (job-group-limit         (let ((jobg-count (config-lookup *configdat* "jobgroups" jobgroup)))
 				    (if (string? jobg-count)
@@ -501,11 +502,11 @@
 			  (waiton-tconfig  (if waiton-record (vector-ref waiton-record 1) #f))
 			  (waiton-itemized (and waiton-tconfig
 						(or (hash-table-ref/default waiton-tconfig "items" #f)
 						    (hash-table-ref/default waiton-tconfig "itemstable" #f))))
 			  (itemmaps        (tests:get-itemmaps config))  ;; (configf:lookup config "requirements" "itemmap"))
-			  (new-test-patts  (tests:extend-test-patts test-patts hed waiton itemmaps)))   ;; BB:  items expanded here.
+			  (new-test-patts  (tests:extend-test-patts test-patts hed waiton itemmaps))) ;; BB: items expanded here
 		     (debug:print-info 0 *default-log-port* "Test " waiton " has " (if waiton-record "a" "no") " waiton-record and" (if waiton-itemized " " " no ") "items")
 		     ;; need to account for test-patt here, if I am test "a", selected with a test-patt of "hed/b%"
 		     ;; and we are waiting on "waiton" we need to add "waiton/,waiton/b%" to test-patt
 		     ;; is this satisfied by merely appending "/" to the waiton name added to the list?
 		     ;;
@@ -603,25 +604,25 @@
 ;;   but have items in reg; loop with (car reg)(cdr reg) '() reruns
 ;;   If reg is empty => all done
 
 (define (runs:queue-next-hed tal reg n regfull)
   (if regfull
-      (car reg)
+      (if (null? reg) #f (car reg))
       (if (null? tal) ;; tal is used up, pop from reg
-	  (car reg)
+	  (if (null? reg) #f (car reg))
 	  (car tal))))
 
 (define (runs:queue-next-tal tal reg n regfull)
   (if regfull
       tal
       (if (null? tal) ;; must transfer from reg
-	  (cdr reg)
+	  (if (null? reg) '() (cdr reg))
 	  (cdr tal))))
 
 (define (runs:queue-next-reg tal reg n regfull)
   (if regfull
-      (cdr reg)
+      (if (null? reg) '() (cdr reg)) ;; EXPLORE: reorder (cdr reg) such that looping is more efficient
       (if (null? tal) ;; if tal is null and reg not full then '() as reg contents moved to tal
 	  '()
 	  reg)))
 
 ;; this is the list of parameters to the named loop "loop" near the top of runs:run-tests-queue, look around line 1216
@@ -630,12 +631,25 @@
   (list (runs:queue-next-hed tal reg reglen regfull)      ;; hed
         (runs:queue-next-tal tal reg reglen regfull)      ;; tal
         (runs:queue-next-reg tal reg reglen regfull)      ;; reg
         reruns))                                          ;; reruns
 
+;; objective - iterate thru tests
+;;    => want to prioritize tests we haven't seen before
+;;    => sometimes need to squeeze things in (added to reg)
+;;    => review of a previously seen test is higher priority of never visited test
+;; reg - list of previously visited tests
+;; tal - list of never visited tests
+;;   prefer next hed to be from reg than tal.
+
 (define runs:nothing-left-in-queue-count 0)
 
+;; return value of runs:expand-items is passed back to runs-tests-queue and is fed to named loop with this signature:
+;;    (let loop ((hed         (car sorted-test-names))
+;;	         (tal         (cdr sorted-test-names))
+;;	         (reg         '()) ;; registered, put these at the head of tal 
+;;	         (reruns      '()))
 ;; BB: for future reference - suspect target vars are not expanded to env vars at this point (item expansion using [items]\nwhatever [system echo $TARGETVAR] doesnt work right whereas [system echo #{targetvar}] does.. Tal and Randy have tix on this.  on first pass, var not set, on second pass, ok.  
 (define (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records itemmaps)
   (let* ((loop-list       (list hed tal reg reruns))
 	 (prereqs-not-met (let ((res (rmt:get-prereqs-not-met run-id waitons hed item-path mode: testmode itemmaps: itemmaps)))
 			    (if (list? res)
@@ -643,15 +657,26 @@
 				(begin
 				  (debug:print 0 *default-log-port*
 					       "ERROR: rmt:get-prereqs-not-met returned non-list!\n"
 					       "  res=" res " run-id=" run-id " waitons=" waitons " hed=" hed " item-path=" item-path " testmode=" testmode " itemmaps=" itemmaps)
 				  '()))))
+         (have-itemized (not (null? (lset-intersection eq? testmode '(itemmatch itemwait)))))
 	 ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap))
 	 (fails           (runs:calc-fails prereqs-not-met))
 	 (prereq-fails    (runs:calc-prereq-fail prereqs-not-met))
 	 (non-completed   (runs:calc-not-completed prereqs-not-met))
-	 (runnables       (runs:calc-runnable prereqs-not-met)))
+	 (runnables       (runs:calc-runnable prereqs-not-met))
+         (unexpanded-prereqs
+          (filter (lambda (testname)
+                    (let* ((test-rec (hash-table-ref test-records testname))
+                           (items       (tests:testqueue-get-items  test-rec)))
+                      ;;(BB> "HEY " testname "=>"items)
+                      (or (procedure? items)(eq? items 'have-procedure))))
+                  waitons))
+
+
+         )
     (debug:print-info 4 *default-log-port* "START OF INNER COND #2 "
 		      "\n can-run-more:    " can-run-more
 		      "\n testname:        " hed
 		      "\n prereqs-not-met: " (runs:pretty-string prereqs-not-met)
 		      "\n non-completed:   " (runs:pretty-string non-completed) 
@@ -685,11 +710,20 @@
 		  (debug:print 0 *default-log-port* "WARNING: this condition is triggered when there were no items to expand and nothing to run. Please check your run for completeness")
 		  (exit 0))
 		(set! runs:nothing-left-in-queue-count (+ runs:nothing-left-in-queue-count 1)))
 	    #f)))
 
-     ;; 
+     ;; desired result of below cond branch:
+     ;;   we want to expand items in our test of interest (hed) in the following cases:
+     ;;    case 1 - mode is itemmatch or itemwait: 
+     ;;       - all prereq tests have been expanded
+     ;;       - at least one prereq's items have completed
+     ;;    case 2 - mode is toplevel   
+     ;;       - prereqs are completed.
+     ;;       - or no prereqs can complete 
+     ;;    case 3 - mode not specified 
+     ;;       - prereqs are completed and passed (we could consider removing "and passed" -- it would change behavior from current)          
      ((or (null? prereqs-not-met)
 	  (and (member 'toplevel testmode)
 	       (null? non-completed)))
       (debug:print-info 4 *default-log-port* "runs:expand-items: (or (null? prereqs-not-met) (and (member 'toplevel testmode)(null? non-completed)))")
       (let ((test-name (tests:testqueue-get-testname test-record)))
@@ -740,11 +774,11 @@
 	    (let ((trimmed-tal (mt:discard-blocked-tests run-id hed tal test-records))
 		  (trimmed-reg (mt:discard-blocked-tests run-id hed reg test-records)))
 	      (debug:print 1 *default-log-port* "WARNING: test " hed " has discarded prerequisites, removing it from the queue")
 
 	      (let ((test-id (rmt:get-test-id run-id hed "")))
-		(if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to discarded prerequisites")))
+		(if test-id (mt:test-set-state-status-by-id run-id test-id "COMPLETED" "PREQ_DISCARDED" "Failed to run due to discarded prerequisites")))
 	      
 	      (if (and (null? trimmed-tal)
 		       (null? trimmed-reg))
 		  #f
                   (runs:loop-values trimmed-tal trimmed-reg reglen regfull reruns)
@@ -778,21 +812,21 @@
 			(string-intersperse (map (lambda (t)(conc (db:test-get-testname t) ":" (db:test-get-state t)"/"(db:test-get-status t))) fails) ", ")
 			", removing it from to-do list")
       (let ((test-id (rmt:get-test-id run-id hed "")))
 	(if test-id
 	    (if (not (null? prereq-fails))
-		(mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites")
-                (begin
-                  (debug:print 4 *default-log-port*"BB> set PREQ_FAIL on "hed)
-                  (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL"      "Failed to run due to failed prerequisites"))))) ;; BB: this works, btu equivalent for itemwait mode does not work.
+		(mt:test-set-state-status-by-id run-id test-id "COMPLETED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites")
+		(mt:test-set-state-status-by-id run-id test-id "COMPLETED" "PREQ_FAIL"      "Failed to run due to failed prerequisites"))))
+      ;; (debug:print 4 *default-log-port*"BB> set PREQ_FAIL on "hed)
+      ;; (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL"      "Failed to run due to failed prerequisites")))) ;; BB: this works, btu equivalent for itemwait mode does not work.
       (if (or (not (null? reg))(not (null? tal)))
 	  (begin
 	    (hash-table-set! test-registry hed 'CANNOTRUN)
             (runs:loop-values tal reg reglen regfull (cons hed reruns))
             )
 	  #f)) ;; #f flags do not loop
-
+      
      ((and (not (null? fails))(member 'toplevel testmode))
       (if (or (not (null? reg))(not (null? tal)))
 	   (list (car newtal)(append (cdr newtal) reg) '() reruns)
 	  #f)) 
      ((null? runnables) #f) ;; if we get here and non-completed is null then it is all over.
@@ -859,11 +893,11 @@
 	 (num-running-in-jobgroup(list-ref run-limits-info 2)) 
 	 (max-concurrent-jobs    (list-ref run-limits-info 3))
 	 (job-group-limit        (list-ref run-limits-info 4))
 	 ;; (prereqs-not-met        (rmt:get-prereqs-not-met run-id waitons hed item-path mode: testmode itemmaps: itemmaps))
 	 ;; (prereqs-not-met         (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap))
-	 (fails                  (if (list? prereqs-not-met)
+	 (fails                  (if (list? prereqs-not-met) ;; TODO: rename fails to failed-prereqs
 				      (runs:calc-fails prereqs-not-met)
 				      (begin
 					(debug:print-error 0 *default-log-port* "prereqs-not-met is not a list! " prereqs-not-met)
 					'())))
 	 (non-completed           (filter (lambda (x)             ;; remove hed from not completed list, duh, of course it is not completed!
@@ -1008,17 +1042,23 @@
 	    (if  (runs:lownoise "Waiting for more work to do..." 60)
 		 (debug:print-info 0 *default-log-port* "Waiting for more work to do..."))
 	    (thread-sleep! 1)
 	    (list (car newtal)(cdr newtal) reg reruns))
 	  ;; the waiton is FAIL so no point in trying to run hed ever again
-	  (if (or (not (null? reg))(not (null? tal)))
-	      (if (or (vector? hed)  (not (null? fails))) ;; BB: why do we need a vector?  in my case, fails is populated (prereq failed), reg is not nul, and we really want to drop this one
+	  (begin
+            (let ((my-test-id (rmt:get-test-id run-id test-name item-path)))
+              (mt:test-set-state-status-by-id run-id my-test-id "COMPLETED" "PREQ_FAIL" "Failed to run due to failed prerequisites2"))
+
+
+            
+            (if (or (not (null? reg))(not (null? tal)))
+                (if (vector? hed)
 		  (begin
 		    (debug:print 1 *default-log-port* "WARNING: Dropping test " test-name "/" item-path
 				 " from the launch list as it has prerequistes that are FAIL")
 		    (let ((test-id (rmt:get-test-id run-id hed "")))
-		      (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL" "Failed to run due to failed prerequisites")))
+		      (if test-id (mt:test-set-state-status-by-id run-id test-id "COMPLETED" "PREQ_FAIL" "Failed to run due to failed prerequisites")))
 		    (runs:shrink-can-run-more-tests-count runsdat) ;; DELAY TWEAKER (still needed?)
 		    ;; (thread-sleep! *global-delta*)
 		    ;; This next is for the items
 
                     (if (not (null? fails))
@@ -1032,15 +1072,15 @@
                     (debug:print 2 *default-log-port* "nth-try("hed")="nth-try)
 		    (cond
 		     ((member "RUNNING" (map db:test-get-state prereqs-not-met))
 		      (if (runs:lownoise (conc "possible RUNNING prerequistes " hed) 60)
 			  (debug:print 0 *default-log-port* "WARNING: test " hed " has possible RUNNING prerequisites, don't give up on it yet."))
-		      (thread-sleep! 4)
+		      (thread-sleep! 0.1)
 		      (runs:loop-values tal reg reglen regfull reruns))
 		     ((or (not nth-try) ;; BB: condition on subsequent tries, condition below fires on first try 
 			  (and (number? nth-try)
-			       (< nth-try 10)))
+			       (< nth-try 2)))
 		      (hash-table-set! test-registry hed (if (number? nth-try)
 							     (+ nth-try 1)
 							     0))
 		      (if (runs:lownoise (conc "not removing test " hed) 60)
 			  (debug:print 1 *default-log-port* "WARNING: not removing test " hed " from queue although it may not be runnable due to FAILED prerequisites"))
@@ -1053,13 +1093,16 @@
 			      #f ;; yes, really
 			      (list (car tal)(cdr tal) reg reruns))
 			  (begin
 			    (if (runs:lownoise (conc "FAILED prerequisites or other issue" hed) 60)
 				(debug:print 0 *default-log-port* "WARNING: test " hed " has FAILED prerequisites or other issue. Internal state >" nth-try "< will be overridden and we'll retry."))
-			    (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "KEEP_TRYING" #f)
-			    (hash-table-set! test-registry hed 0)
-			    (runs:loop-values newtal reg reglen regfull))))
+			    ;; was: (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "KEEP_TRYING" #f)
+                            (mt:test-set-state-status-by-testname run-id test-name item-path "COMPLETED" "PREQ_FAIL" #f)
+			    (hash-table-set! test-registry hed 'removed) ;; was 0
+                            (if (not (and (null? reg) (null? tal)))
+                                (runs:loop-values tal reg reglen regfull reruns)
+                                #f))))
 		     (else
 		      (if (runs:lownoise (conc "FAILED prerequitests and we tried" hed) 60)
 			  (debug:print 0 *default-log-port* "WARNING: test " hed " has FAILED prerequitests and we've tried at least 10 times to run it. Giving up now."))
 		      ;; (debug:print 0 *default-log-port* "         prereqs: " prereqs-not-met)
 		      (hash-table-set! test-registry hed 'removed)
@@ -1068,15 +1111,20 @@
 		      (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "FAIL" #f) ;; treat as FAIL
 		      (list (if (null? tal)(car newtal)(car tal))
 			    tal
 			    reg
 			    reruns)))))
-	      ;; can't drop this - maybe running? Just keep trying
-	      (let ((runable-tests (runs:runable-tests prereqs-not-met))) ;; SUSPICIOUS: Should look at more than just prereqs-not-met?
-		(if (null? runable-tests)
-		    #f   ;; I think we are truly done here
-		    (runs:loop-values newtal reg reglen regfull reruns)))))))))
+	      ;; ELSE: can't drop this - maybe running? Just keep trying
+
+                ;;(if (not (or (not (null? reg))(not (null? tal)))) ;; old experiment
+                (let ((runable-tests (runs:runable-tests prereqs-not-met))) ;; SUSPICIOUS: Should look at more than just prereqs-not-met?
+                  (if (null? runable-tests)
+                      #f   ;; I think we are truly done here
+                      (runs:loop-values newtal reg reglen regfull reruns)))
+                ;;) ;;from old experiment
+            ) ;; end if (or (not (null? reg))(not (null? tal)))
+            ))))))
 
 ;; scan a list of tests looking to see if any are potentially runnable
 ;;
 (define (runs:runable-tests tests)
   (filter (lambda (t)
@@ -1230,11 +1278,11 @@
 		      (hash-table-set! test-registry (db:test-make-full-name tn ip) (string->symbol st)))))
 	      tests-info)
     (set! max-retries (if (and max-retries (string->number max-retries))(string->number max-retries) 100))
 
     (let loop ((hed         (car sorted-test-names))
-	       (tal         (cdr sorted-test-names))
+	       (tal         (cdr sorted-test-names)) 
 	       (reg         '()) ;; registered, put these at the head of tal 
 	       (reruns      '()))
 
       (runs:incremental-print-results run-id)
 
@@ -1424,11 +1472,11 @@
 	 ;; EXPAND ITEMS
 	 ((or (procedure? items)(eq? items 'have-procedure)) ;; BB - target vars are env vars here? to allow expansion of [items]\nsomething [system echo $SOMETARGVAR], which is wonky
 	  (let ((can-run-more    (runs:can-run-more-tests runsdat run-id jobgroup max-concurrent-jobs)))
 	    (if (and (list? can-run-more)
 		     (car can-run-more))
-		(let ((loop-list (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records itemmaps)))
+		(let ((loop-list (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records itemmaps))) ;; itemized test expanded here
 		  (if loop-list
 		      (apply loop loop-list)))
 		;; if can't run more just loop with next possible test
 		(loop (car newtal)(cdr newtal) reg reruns))))
 	    
@@ -1484,16 +1532,16 @@
     (debug:print-info 1 *default-log-port* "All tests launched")))
 
 (define (runs:calc-fails prereqs-not-met)
   (filter (lambda (test)
 	    (and (vector? test) ;; not (string? test))
-		 (member (db:test-get-state test) '("INCOMPLETE" "COMPLETED"))
+		 (member (db:test-get-state test) '("INCOMPLETE" "COMPLETED")) ;; TODO: pull from *common:stuff...*
 		 (not (member (db:test-get-status test)
 			      '("PASS" "WARN" "CHECK" "WAIVED" "SKIP")))))
 	  prereqs-not-met))
 
-(define (runs:calc-prereq-fail prereqs-not-met)
+(define (runs:calc-prereq-fail prereqs-not-met) ;; REMOVEME since NOT_STARTED/PREQ_FAIL is now COMPLETED/PREQ_FAIL
   (filter (lambda (test)
 	    (and (vector? test) ;; not (string? test))
 		 (equal? (db:test-get-state test) "NOT_STARTED")
 		 (not (member (db:test-get-status test)
 			      '("n/a" "KEEP_TRYING")))))

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -20,10 +20,11 @@
 (declare (uses common))
 (declare (uses db))
 (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
 ;; (declare (uses synchash))
 (declare (uses http-transport))
+;;(declare (uses rpc-transport))
 (declare (uses launch))
 (declare (uses daemon))
 
 (include "common_records.scm")
 (include "db_records.scm")
@@ -54,11 +55,11 @@
 ;;
 (define (server:launch run-id transport-type)
   (case transport-type
     ((http)(http-transport:launch))
     ;;((nmsg)(nmsg-transport:launch run-id))
-    ((rpc)  (rpc-transport:launch run-id))
+    ;;((rpc)  (rpc-transport:launch run-id))
     (else (debug:print-error 0 *default-log-port* "unknown server type " transport-type))))
 
 ;;======================================================================
 ;; S E R V E R   U T I L I T I E S 
 ;;======================================================================

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -148,11 +148,11 @@
 
 
 ;; returns waitons waitors tconfigdat
 ;;
 (define (tests:get-waitons test-name all-tests-registry)
-   (let* ((config  (tests:get-testconfig test-name #f all-tests-registry 'return-procs)))
+   (let* ((config  (tests:get-testconfig test-name #f all-tests-registry 'return-procs))) ;; assuming no problems with immediate evaluation, this could be simplified ('return-procs -> #t)
      (let ((instr (if config 
 		      (config-lookup config "requirements" "waiton")
 		      (begin ;; No config means this is a non-existant test
 			(debug:print-error 0 *default-log-port* "non-existent required test \"" test-name "\"")
 			(exit 1))))

Index: utils/installall.sh
==================================================================
--- utils/installall.sh
+++ utils/installall.sh
@@ -68,94 +68,15 @@
 	IMVER=3.12
         CHICKEN_VERSION=4.12.0
         CHICKEN_BASEVER=4.12.0
 	;;
 SUSE_LINUX_11-x86_64-std)
-  KTYPE=26g4 
+	KTYPE=26g4 
 	CDVER=5.11.1
 	IUPVER=3.22
 	IMVER=3.12
-  ;;
-CentOS_5.11-x86_64-std)
-  KTYPE=24g3 
-  CDVER=5.4.1
-  IUPVER=3.5
-  IMVER=3.6.3
-  ;; 
-esac
-
-echo SYSTEM_TYPE=$SYSTEM_TYPE
-echo KTYPE=$KTYPE			  
-echo CDVER=$CDVER
-echo IUPVER=$IUPVER
-echo IMVER=$IMVER	
-echo CHICKEN_VERSION=$CHICKEN_VERSION
-echo CHICKEN_BASEVER=$CHICKEN_BASEVER
-
-# NOTES:
-#
-# Centos with security setup may need to do commands such as following as root:
-#
-# NB// fix the paths first
-#
-# for a in /localdisk/chicken/4.8.0/lib/*.so;do chcon -t textrel_shlib_t $a; done 
-
-echo ADDITIONAL_LIBPATH=$ADDITIONAL_LIBPATH
-echo  
-echo To use previous IUP libraries set USEOLDIUP to yes
-echo USEOLDIUP=$USEOLDIUP
-echo 
-echo Hit ^C now to do that
-
-# A nice way to run this script:
-#
-# script -c 'PREFIX=/tmp/delme ./installall.sh ' installall.log
-# logpro installall.logpro installall.html < installall.log
-# firefox installall.html
-
-sleep 5
-
-if [[ $proxy == "" ]]; then 
-  echo 'Please set the environment variable "proxy" to host.com:port (e.g. foo.com:1234) to use a proxy'
-  echo PROX=""
-else
-  export http_proxy=http://$proxy
-  export https_proxy=http://$proxy
-  export PROX="-proxy $proxy"
-fi
-
-if [[ $KTYPE == "" ]]; then
-  echo 'Using KTYPE=26'
-  export KTYPE=26g4
-else
-  echo Using KTYPE=$KTYPE
-fi
-
-# Put all the downloaded tar files in tgz
-mkdir -p tgz
-
-# http://code.call-cc.org/releases/4.8.0/chicken-4.8.0.5.tar.gz
-chicken_targz=chicken-${CHICKEN_VERSION}.tar.gz
-if ! [[ -e tgz/$chicken_targz ]]; then 
-    wget http://code.call-cc.org/releases/${CHICKEN_BASEVER}/${chicken_targz}
-    mv $chicken_targz tgz
-fi 
-
-BUILDHOME=$PWD
-DEPLOYTARG=$BUILDHOME/deploy
-
-if [[ $PREFIX == "" ]]; then
-   PREFIX=$PWD/inst
-fi
-
-export PATH=$PREFIX/bin:$PATH
-export LIBPATH=$PREFIX/lib:$PREFIX/lib64:$ADDITIONAL_LIBPATH
-export LD_LIBRARY_PATH=$LIBPATH
-export CHICKEN_INSTALL=$PREFIX/bin/chicken-install
-mkdir -p $PREFIX
-echo "export PATH=$PREFIX/bin:\$PATH" > $PREFIX/setup-chicken4x.sh
-echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\$LD_LIBRARY_PATH" >> $PREFIX/setup-chicken4x.sh
+  >> $PREFIX/setup-chicken4x.sh
 echo "export CHICKEN_DOC_PAGER=cat" >> $PREFIX/setup-chicken4x.sh
 
 echo "setenv PATH $PREFIX/bin:\$PATH" > $PREFIX/setup-chicken4x.csh
 echo "setenv LD_LIBRARY_PATH $LD_LIBRARY_PATH:\$LD_LIBRARY_PATH" >> $PREFIX/setup-chicken4x.csh
 echo "setenv CHICKEN_DOC_PAGER cat" >> $PREFIX/setup-chicken4x.csh