Megatest

ezsteps.scm at [99dec402d5]
Login

File ezsteps.scm artifact f78a0c389c part of check-in 99dec402d5



;; Copyright 2006-2012, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')

(use srfi-1 posix regex srfi-69 directory-utils call-with-environment-variables)

(declare (unit ezsteps))
(declare (uses db))
(declare (uses common))
(declare (uses items))
(declare (uses runconfig))
;; (declare (uses sdb))
;; (declare (uses filedb))

(include "common_records.scm")
(include "key_records.scm")
(include "db_records.scm")
(include "run_records.scm")



;;======================================================================
;; ezsteps
;;======================================================================

;; ezsteps were going to be coded as
;; stepname[,predstep1,predstep2 ...] [{VAR1=first,second,third}] command to execute
;;   BUT
;; now are
;; stepname {VAR=first,second,third ...} command ...
;; where the {VAR=first,second,third ...} is optional.

;; given an exit code and whether or not logpro was used calculate OK/BAD
;; return #t if we are ok, #f otherwise
(define (ezsteps:steprun-good? logpro exitcode)
  (or (eq? exitcode 0)
      (and logpro (eq? exitcode 2))))

(define (ezsteps:manage-steps run-id test-id item-path fullrunscript ezsteps subrun test-name tconfigreg exit-info m)
  ;; (let-values
  ;;  (((pid exit-status exit-code)
  ;;    (run-n-wait fullrunscript)))
  ;; (tests:test-set-status! test-id "RUNNING" "n/a" #f #f)
  ;; Since we should have a clean slate at this time there is no need to do 
  ;; any of the other stuff that tests:test-set-status! does. Let's just 
  ;; force RUNNING/n/a

  ;; (thread-sleep! 0.3)
  ;; (tests:test-force-state-status! run-id test-id "RUNNING" "n/a")
  (rmt:set-state-status-and-roll-up-items run-id test-name item-path "RUNNING" #f #f) 
  ;; (thread-sleep! 0.3) ;; NFS slowness has caused grief here

  ;; if there is a runscript do it first
  (if fullrunscript
      (let ((pid (process-run fullrunscript)))
	(rmt:test-set-top-process-pid run-id test-id pid)
	(let loop ((i 0))
	  (let-values
	   (((pid-val exit-status exit-code) (process-wait pid #t)))
	   (mutex-lock! m)
	   (launch:einf-pid-set!           exit-info  pid)         ;; (vector-set! exit-info 0 pid)
	   (launch:einf-exit-status-set!   exit-info  exit-status) ;; (vector-set! exit-info 1 exit-status)
	   (launch:einf-exit-code-set!     exit-info  exit-code)   ;; (vector-set! exit-info 2 exit-code)
	   (launch:einf-rollup-status-set! exit-info  exit-code)   ;; (vector-set! exit-info 3 exit-code)  ;; rollup status
	   (mutex-unlock! m)
	   (if (eq? pid-val 0)
	       (begin
		 (thread-sleep! 2)
		 (loop (+ i 1)))
	       )))))
  ;; then, if runscript ran ok (or did not get called)
  ;; do all the ezsteps (if any)
  (if (or ezsteps subrun)
      (let* ((test-run-dir (tests:get-test-path-from-environment))
             (testconfig ;; (read-config (conc work-area "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) ;; FIXME??? is allow-system ok here?
	      ;; NOTE: it is tempting to turn off force-create of testconfig but dynamic
	      ;;       ezstep names need a full re-eval here.
	      (tests:get-testconfig test-name item-path tconfigreg #t force-create: #t)) ;; 'return-procs)))
	     (ezstepslst (if (hash-table? testconfig)
			     (hash-table-ref/default testconfig "ezsteps" '())
			     #f)))
	(if testconfig
	    (hash-table-set! *testconfigs* test-name testconfig) ;; cached for lazy reads later ...
	    (begin
	      (launch:setup)
	      (debug:print 0 *default-log-port* "WARNING: no testconfig found for " test-name " in search path:\n  "
			   (string-intersperse (tests:get-tests-search-path *configdat*) "\n  "))))
	;; after all that, still no testconfig? Time to abort
	(if (not testconfig)
	    (begin
	      (debug:print-error 0 *default-log-port* "Failed to resolve megatest.config, runconfigs.config and testconfig issues. Giving up now")
	      (exit 1)))

	;; create a proc for the subrun if requested, save that proc in the ezsteps table as the last entry
	;; 1. get section [runarun]
	;; 2. unset MT_* vars
	;; 3. fix target
	;; 4. fix runname
	;; 5. fix testpatt or calculate it from contour
	;; 6. launch the run
	;; 7. roll up the run result and or roll up the logpro processed result
	(when (configf:lookup testconfig "subrun" "runwait") ;; we use runwait as the flag that a subrun is requested
            (subrun:initialize-toprun-test testconfig test-run-dir)
	    (let* ((mt-cmd (subrun:launch-cmd test-run-dir)))
              (debug:print-info 0 *default-log-port* "Subrun command is \"" mt-cmd "\"")
              (set! ezsteps #t) ;; set the needed flag
	      (set! ezstepslst
                    (append (or ezstepslst '())
                            (list (list "subrun" (conc "{subrun=true} " mt-cmd)))))))

	;; process the ezsteps
	(if ezsteps
	    (begin
	      (if (not (common:file-exists? ".ezsteps"))(create-directory ".ezsteps"))
	      ;; if ezsteps was defined then we are sure to have at least one step but check anyway
	      (if (not (> (length ezstepslst) 0))
		  (debug:print-error 0 *default-log-port* "ezsteps defined but ezstepslst is zero length")
		  (let loop ((ezstep (car ezstepslst))
			     (tal    (cdr ezstepslst))
			     (prevstep #f))
                    (debug:print-info 0 *default-log-port* "Processing ezstep \"" (string-intersperse ezstep " ") "\"")
		    ;; check exit-info (vector-ref exit-info 1)
		    (if (launch:einf-exit-status exit-info) ;; (vector-ref exit-info 1)
			(let ((logpro-used
                               (ezsteps:runstep ezstep run-id test-id
                                                exit-info: exit-info mutix: m is-last-step: (null? tal) testconfig: testconfig))
			      (stepname    (car ezstep)))
			  ;; if logpro-used read in the stepname.dat file
			  (if (and logpro-used (common:file-exists? (conc stepname ".dat")))
			      (launch:load-logpro-dat run-id test-id stepname))
			  (if (ezsteps:steprun-good? logpro-used (launch:einf-exit-code exit-info))
			      (if (not (null? tal))
				  (loop (car tal) (cdr tal) stepname))
			      (debug:print 0 *default-log-port* "WARNING: step " (car ezstep) " failed. Stopping")))
			(debug:print 0 *default-log-port* "WARNING: a prior step failed, stopping at " ezstep)))))))))

;;(rmt:get-test-info-by-id run-id test-id) -> testdat

(define (ezsteps:runstep ezstep run-id test-id #!key
                         (exit-info  (make-launch:einf pid: #t exit-status: #t exit-code: #t rollup-status: 0)) ;; pid exit-status exit-code (i.e. process was successfully run) rollup-status
                         (mutex (make-mutex))
                         (testconfig (tests:get-testconfig test-name item-path tconfigreg #t force-create: #t))
                         (ezsteplst      (hash-table-ref/default testconfig "ezsteps" '()))
                         (is-last-step   #f))
  (let* ((stepname       (if (list? ezsteplst) (car ezstep) ezstep))  ;; do stuff to run the step
	 (stepinfo       (if (list? ezsteplst)
                             (cadr ezstep)
                             (let loop ((tocheck ezsteplst))
                               (cond
                                ((null? tocheck) #f)
                                ((equal? (caar tocheck) ezstep)
                                 (cadar tocheck))
                                (else (loop (cdr tocheck))))))))
    (if stepinfo
        (let* (
	       ;; (let ((info (cadr ezstep)))
	       ;; 		   (if (proc? info) "" info)))
	       ;; (stepproc       (let ((info (cadr ezstep)))
	       ;; 		   (if (proc? info) info #f)))
	       (stepparts      (string-match (regexp "^(\\{([^\\}\\{]*)\\}\\s*|)(.*)$") stepinfo))
	       (stepparams     (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each
	       (paramparts     (if (string? stepparams)
			           (map (lambda (x)(string-split x "=")) (string-split-fields "[^;]*=[^;]*" stepparams))
			           '()))
	       (subrun         (alist-ref "subrun" paramparts equal?))
	       (stepcmd        (list-ref stepparts 3))
	       (script         "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!\
	       (logpro-file    (conc stepname ".logpro"))
	       (html-file      (conc stepname ".html"))
	       (dat-file       (conc stepname ".dat"))
	       (tconfig-logpro (configf:lookup testconfig "logpro" stepname))
	       (logpro-used    (common:file-exists? logpro-file)))

          (debug:print 0 *default-log-port* "stepparts: " stepparts ", stepparams: " stepparams
                       ", paramparts: " paramparts ", subrun: " subrun ", stepcmd: " stepcmd)
          
          (if (and tconfig-logpro
	           (not logpro-used)) ;; no logpro file found but have a defn in the testconfig
	      (begin
	        (with-output-to-file logpro-file
	          (lambda ()
	            (print ";; logpro file extracted from testconfig\n"
		           ";;")
	            (print tconfig-logpro)))
	        (set! logpro-used #t)))
          
          ;; NB// can safely assume we are in test-area directory
          (debug:print 4 *default-log-port* "ezsteps:\n stepname: " stepname " stepinfo: " stepinfo " stepparts: " stepparts
		       " stepparams: " stepparams " stepcmd: " stepcmd)
          
          ;; ;; first source the previous environment
          ;; (let ((prev-env (conc ".ezsteps/" prevstep (if (string-search (regexp "csh") 
          ;;      							 (get-environment-variable "SHELL")) ".csh" ".sh"))))
          ;;   (if (and prevstep (common:file-exists? prev-env))
          ;;       (set! script (conc script "source " prev-env))))
          
          ;; call the command using mt_ezstep
          ;; (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "x") " " stepcmd))
          
          (debug:print 4 *default-log-port* "script: " script)
          (rmt:teststep-set-status! run-id test-id stepname "start" "-" #f #f)
          ;; now launch the actual process
          (call-with-environment-variables 
           (list (cons "PATH" (conc (get-environment-variable "PATH") ":.")))
           (lambda () ;; (process-run "/bin/bash" "-c" "exec ls -l /tmp/foobar > /tmp/delme-more.log 2>&1")
             (let* ((cmd (conc stepcmd " > " stepname ".log 2>&1")) ;; >outfile 2>&1 
	            (pid #f))
	       (let ((proc (lambda ()
		             (set! pid (process-run "/bin/bash" (list "-c" cmd))))))
	         (if subrun
                     (begin
                       (debug:print-info 0 *default-log-port* "Running without MT_.* environment variables.")
                       (common:without-vars proc "^MT_.*"))
	             (proc)))
	       
               (with-output-to-file "Makefile.ezsteps"
                 (lambda ()
                   (print stepname ".log :")
                   (print "\t" cmd)
                   (if (common:file-exists? (conc stepname ".logpro"))
                       (print "\tlogpro " stepname ".logpro " stepname ".html < " stepname ".log"))
                   (print)
                   (print stepname " : " stepname ".log")
                   (print))
                 #:append)

	       (rmt:test-set-top-process-pid run-id test-id pid)
	       (let processloop ((i 0))
	         (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
		   (mutex-lock! mutex)
		   (launch:einf-pid-set!         exit-info pid)         ;; (vector-set! exit-info 0 pid)
		   (launch:einf-exit-status-set! exit-info exit-status) ;; (vector-set! exit-info 1 exit-status)
		   (launch:einf-exit-code-set!   exit-info exit-code)   ;; (vector-set! exit-info 2 exit-code)
		   (mutex-unlock! mutex)
		   (if (eq? pid-val 0)
		       (begin
			 (thread-sleep! 2)
			 (processloop (+ i 1))))
		   )))))
          (debug:print-info 0 *default-log-port* "step " stepname " completed with exit code " (launch:einf-exit-code exit-info)) ;; (vector-ref exit-info 2))
          ;; now run logpro if needed
          (if logpro-used
	      (let* ((logpro-exe (or (getenv "LOGPRO_EXE") "logpro"))
                     (pid        (process-run (conc "/bin/sh -c '"logpro-exe" "logpro-file " " (conc stepname ".html") " < " stepname ".log > /dev/null'"))))
	        (let processloop ((i 0))
	          (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
		    (mutex-lock! mutex)
		    ;; (make-launch:einf pid: pid exit-status: exit-status exit-code: exit-code)
		    (launch:einf-pid-set!         exit-info pid)         ;; (vector-set! exit-info 0 pid)
		    (launch:einf-exit-status-set! exit-info exit-status) ;; (vector-set! exit-info 1 exit-status)
		    (launch:einf-exit-code-set!   exit-info exit-code)   ;; (vector-set! exit-info 2 exit-code)
		    (mutex-unlock! mutex)
		    (if (eq? pid-val 0)
			(begin
			  (thread-sleep! 2)
			  (processloop (+ i 1)))))
	          (debug:print-info 0 *default-log-port* "logpro for step " stepname " exited with code " (launch:einf-exit-code exit-info))))) ;; (vector-ref exit-info 2)))))
          
          (let ((exinfo (launch:einf-exit-code exit-info)) ;; (vector-ref exit-info 2))
	        (logfna (if logpro-used (conc stepname ".html") ""))
	        (comment #f))
            (if logpro-used
	        (let ((datfile (conc stepname ".dat")))
	          ;; load the .dat file into the test_data table if it exists
	          (if (common:file-exists? datfile)
		      (set! comment (launch:load-logpro-dat run-id test-id stepname)))
	          (rmt:test-set-log! run-id test-id (conc stepname ".html"))))
            (rmt:teststep-set-status! run-id test-id stepname "end" exinfo comment logfna))
          ;; set the test final status
          (let* ((process-exit-status (launch:einf-exit-code exit-info)) ;; (vector-ref exit-info 2))
	         (this-step-status (cond
			            ((and (eq? process-exit-status 2) logpro-used) 'warn)   ;; logpro 2 = warnings
			            ((and (eq? process-exit-status 3) logpro-used) 'check)  ;; logpro 3 = check
			            ((and (eq? process-exit-status 4) logpro-used) 'waived) ;; logpro 4 = waived
			            ((and (eq? process-exit-status 5) logpro-used) 'abort)  ;; logpro 5 = abort
			            ((and (eq? process-exit-status 6) logpro-used) 'skip)   ;; logpro 6 = skip
			            ((eq? process-exit-status 0)                   'pass)   ;; logpro 0 = pass
			            (else 'fail)))
	         (overall-status   (cond
			            ((eq? (launch:einf-rollup-status exit-info) 2) 'warn) ;; rollup-status (vector-ref exit-info 3)
			            ((eq? (launch:einf-rollup-status exit-info) 0) 'pass) ;; (vector-ref exit-info 3)
			            (else 'fail)))
	         (next-status      (cond 
			            ((eq? overall-status 'pass) this-step-status)
			            ((eq? overall-status 'warn)
			             (if (eq? this-step-status 'fail) 'fail 'warn))
			            ((eq? overall-status 'abort) 'abort)
			            (else 'fail)))
	         (next-state       ;; "RUNNING") ;; WHY WAS THIS CHANGED TO NOT USE (null? tal) ??
	          (cond
	           (is-last-step ;; more to run?
	            "COMPLETED")
	           (else "RUNNING"))))
            (debug:print 4 *default-log-port* "Exit value received: " (launch:einf-exit-code exit-info) " logpro-used: " logpro-used 
		         " this-step-status: " this-step-status " overall-status: " overall-status 
		         " next-status: " next-status " rollup-status: "  (launch:einf-rollup-status exit-info)) ;; (vector-ref exit-info 3))
            (case next-status
	      ((warn)
	       (launch:einf-rollup-status-set! exit-info 2) ;; (vector-set! exit-info 3 2) ;; rollup-status
	       ;; NB// test-set-status! does rdb calls under the hood
	       (tests:test-set-status! run-id test-id next-state "WARN" 
				       (if (eq? this-step-status 'warn) "Logpro warning found" #f)
				       #f))
	      ((check)
	       (launch:einf-rollup-status-set! exit-info 3) ;; (vector-set! exit-info 3 3) ;; rollup-status
	       ;; NB// test-set-status! does rdb calls under the hood
	       (tests:test-set-status! run-id test-id next-state "CHECK" 
				       (if (eq? this-step-status 'check) "Logpro check found" #f)
				       #f))
	      ((waived)
	       (launch:einf-rollup-status-set! exit-info 4) ;; (vector-set! exit-info 3 3) ;; rollup-status
	       ;; NB// test-set-status! does rdb calls under the hood
	       (tests:test-set-status! run-id test-id next-state "WAIVED" 
				       (if (eq? this-step-status 'check) "Logpro waived found" #f)
				       #f))
	      ((abort)
	       (launch:einf-rollup-status-set! exit-info 5) ;; (vector-set! exit-info 3 4) ;; rollup-status
	       ;; NB// test-set-status! does rdb calls under the hood
	       (tests:test-set-status! run-id test-id next-state "ABORT" 
				       (if (eq? this-step-status 'abort) "Logpro abort found" #f)
				       #f))
	      ((skip)
	       (launch:einf-rollup-status-set! exit-info 6) ;; (vector-set! exit-info 3 4) ;; rollup-status
	       ;; NB// test-set-status! does rdb calls under the hood
	       (tests:test-set-status! run-id test-id next-state "SKIP" 
				       (if (eq? this-step-status 'skip) "Logpro skip found" #f)
				       #f))
	      ((pass)
	       (tests:test-set-status! run-id test-id next-state "PASS" #f #f))
	      (else ;; 'fail
	       (launch:einf-rollup-status-set! exit-info 1) ;; (vector-set! exit-info 3 1) ;; force fail, this used to be next-state but that doesn't make sense. should always be "COMPLETED" 
	       (tests:test-set-status! run-id test-id "COMPLETED" "FAIL" (conc "Failed at step " stepname) #f)
	       )))
          logpro-used)
        (begin
          (debug:print-error 0 *default-log-port* "ezstep named "ezstep" does not exist for testid="test-id)
          #f))))

(define (ezsteps:run-from testdat start-step-name-in run-one #!key (rerun-logpro-only #f) )
  ;; TODO: honor rerun-logpro-only
  (if rerun-logpro-only
      (BB> "someday soon...")
  (let* ((test-run-dir  ;; (filedb:get-path *fdb* 
	  (db:test-get-rundir testdat)) ;; )
	 (testconfig    (read-config (conc test-run-dir "/testconfig") #f #t environ-patt: "pre-launch-env-vars"))
	 (ezstepslst    (hash-table-ref/default testconfig "ezsteps" '()))
         (start-step-name (or start-step-name-in (if (null? ezsteplst) #f (car ezsteplst))))
	 (run-mutex     (make-mutex))
	 (rollup-status 0)
	 (exit-info     (vector #t #t #t))
	 (test-id       (db:test-get-id testdat))
	 (run-id        (db:test-get-run_id testdat))
	 (test-name     (db:test-get-testname testdat))
	 (kill-job      #f)) ;; for future use (on re-factoring with launch.scm code
    (let loop ((count 5))
      (if (common:file-exists? test-run-dir)
	  (push-directory test-run-dir)
	  (if (> count 0)
	      (begin
		(debug:print 0 *default-log-port* "WARNING: ezsteps attempting to run but test run directory " test-run-dir " is not there. Waiting and trying again " count " more times")
		(sleep 3)
		(loop (- count 1))))))
    (debug:print-info 0 *default-log-port* "Running in directory " test-run-dir)
    (if (not (common:file-exists? ".ezsteps"))(create-directory ".ezsteps"))
    ;; if ezsteps was defined then we are sure to have at least one step but check anyway
    
     (if (not (> (length ezstepslst) 0))
	(message-window "ERROR: You can only re-run steps defined via ezsteps")
	(begin
	  (let loop ((ezstep   (car ezstepslst))
		     (tal      (cdr ezstepslst))
		     (prevstep #f)
		     (runflag  #f)) ;; flag used to skip steps when not starting at the beginning
	    (if (vector-ref exit-info 1)
		(let* ((stepname  (car ezstep))  ;; do stuff to run the step
		       (stepinfo  (cadr ezstep))
		       (stepparts (string-match (regexp "^(\\{([^\\}]*)\\}\\s*|)(.*)$") stepinfo))
		       (stepparms (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each 
		       (stepcmd   (list-ref stepparts 3))
		       (script    "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!
		       (logpro-used #f))

		  ;; Skip steps until hit start-step-name
		  ;;
		  (if (and start-step-name
			   (not runflag))
		      (if (equal? stepname start-step-name)
			  (set! runflag #t) ;; and continue
			  (if (not (null? tal))
			      (loop (car tal)(cdr tal) stepname #f))))

		  (debug:print 4 *default-log-port* "ezsteps:\n stepname: " stepname " stepinfo: " stepinfo " stepparts: " stepparts
			       " stepparms: " stepparms " stepcmd: " stepcmd)
		  
		  (if (common:file-exists? (conc stepname ".logpro"))(set! logpro-used #t))
		  
		  ;; call the command using mt_ezstep
		  (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd))
		  
		  (debug:print 4 *default-log-port* "script: " script)
		  (rmt:teststep-set-status! run-id test-id stepname "start" "-" #f #f)
		  ;; now launch
		  (let ((pid (process-run script)))
		    (let processloop ((i 0))
		      (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
				  (mutex-lock! run-mutex)
				  (vector-set! exit-info 0 pid)
				  (vector-set! exit-info 1 exit-status)
				  (vector-set! exit-info 2 exit-code)
				  (mutex-unlock! run-mutex)
				  (if (eq? pid-val 0)
				      (begin
					(thread-sleep! 1)
					(processloop (+ i 1))))
				  ))
		    (let ((exinfo (vector-ref exit-info 2))
			  (logfna (if logpro-used (conc stepname ".html") "")))
		      (rmt:teststep-set-status! run-id test-id stepname "end" exinfo #f logfna))
		    (if logpro-used
			(rmt:test-set-log! run-id test-id (conc stepname ".html")))
		    ;; set the test final status
		    (let* ((this-step-status (cond
					      ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)
					      ((eq? (vector-ref exit-info 2) 0)                   'pass)
					      (else 'fail)))
			   (overall-status   (cond
					      ((eq? rollup-status 2) 'warn)
					      ((eq? rollup-status 0) 'pass)
					      (else 'fail)))
			   (next-status      (cond 
					      ((eq? overall-status 'pass) this-step-status)
					      ((eq? overall-status 'warn)
					       (if (eq? this-step-status 'fail) 'fail 'warn))
					      (else 'fail))))
		      (debug:print 4 *default-log-port* "Exit value received: " (vector-ref exit-info 2) " logpro-used: " logpro-used 
				   " this-step-status: " this-step-status " overall-status: " overall-status 
				   " next-status: " next-status " rollup-status: " rollup-status)
		      (case next-status
			((warn)
			 (set! rollup-status 2)
			 ;; NB// test-set-status! does rdb calls under the hood
			 (tests:test-set-status! run-id test-id "RUNNING" "WARN" 
						 (if (eq? this-step-status 'warn) "Logpro warning found" #f)
						 #f))
			((pass)
			 (tests:test-set-status! run-id test-id "RUNNING" "PASS" #f #f))
			(else ;; 'fail
			 (set! rollup-status 1) ;; force fail
			 (tests:test-set-status! run-id test-id "RUNNING" "FAIL" (conc "Failed at step " stepname) #f)
			 ))))
		  (if (and (ezsteps:steprun-good? logpro-used (vector-ref exit-info 2))
			   (not (null? tal)))
		      (if (not run-one) ;; if we got here we completed the step, if run-one is true, stop
			  (loop (car tal) (cdr tal) stepname runflag))))
		(debug:print 4 *default-log-port* "WARNING: a prior step failed, stopping at " ezstep)))
	  
	  ;; Once done with step/steps update the test record
	  ;;
	  (let* ((item-path (db:test-get-item-path testdat)) ;; (item-list->path itemdat))
		 (testinfo  (rmt:get-testinfo-state-status run-id test-id))) ;; refresh the testdat, call it iteminfo in case need prev/curr
	    ;; Am I completed?
	    (if (equal? (db:test-get-state testinfo) "RUNNING") ;; (not (equal? (db:test-get-state testinfo) "COMPLETED"))
		(let ((new-state  (if kill-job "KILLED" "COMPLETED") ;; (if (eq? (vector-ref exit-info 2) 0) ;; exited with "good" status
				  ;; "COMPLETED"
				  ;; (db:test-get-state testinfo)))   ;; else preseve the state as set within the test
				  )
		      (new-status (cond
				   ((not (vector-ref exit-info 1)) "FAIL") ;; job failed to run
				   ((eq? rollup-status 0)
				    ;; if the current status is AUTO the defer to the calculated value (i.e. leave this AUTO)
				    (if (equal? (db:test-get-status testinfo) "AUTO") "AUTO" "PASS"))
				   ((eq? rollup-status 1) "FAIL")
				   ((eq? rollup-status 2)
				    ;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
				    (if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
				   (else "FAIL")))) ;; (db:test-get-status testinfo)))
		  (debug:print-info 2 *default-log-port* "Test NOT logged as COMPLETED, (state=" (db:test-get-state testinfo) "), updating result, rollup-status is " rollup-status)
		  (tests:test-set-status! run-id test-id 
					  new-state
					  new-status
					  (args:get-arg "-m") #f)
		  ;; need to update the top test record if PASS or FAIL and this is a subtest
		  (if (not (equal? item-path ""))
                      (rmt:set-state-status-and-roll-up-items run-id test-name item-path new-state new-status #f))))
	    ;; for automated creation of the rollup html file this is a good place...
	    (if (not (equal? item-path ""))
	      (tests:summarize-items run-id test-id test-name #f)) ;; don't force - just update if no
	    )))
    (pop-directory)
    rollup-status)))

(define (ezsteps:spawn-run-from testdat start-step-name run-one)
  (thread-start! 
   (make-thread
    (lambda ()
      (ezsteps:run-from testdat start-step-name run-one))
    (conc "ezstep run single step " start-step-name " run-one="run-one)))
  )