Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -54,10 +54,18 @@
     ((get-runs)                     (apply db:get-runs db params))
     ((get-runs-by-patt)             (apply db:get-runs-by-patt db params))
     ((lock/unlock-run)              (apply db:lock/unlock-run db params))
     ((update-run-event_time)        (apply db:update-run-event_time db params))
 
+    ;; STEPS
+    ((teststep-set-status!)         (apply db:teststep-set-status! db params))
+
+    ;; TEST DATA
+    ((test-data-rollup)             (apply db:test-data-rollup db params))
+    ((csv->test-data)               (apply db:csv->test-data db params))
+    ((get-steps-data)               (apply db:get-steps-data db params))
+
     ;; MISC
     ((login)                        (apply db:login db params))
     ((general-call)                 (let ((stmtname   (car params))
 					  (realparams (cdr params)))
 				      (db:general-call db stmtname realparams)))

Index: dashboard-tests.scm
==================================================================
--- dashboard-tests.scm
+++ dashboard-tests.scm
@@ -411,39 +411,36 @@
 		      ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
 		       (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
 		      (else #f)))))
       res))
 
-(define (dashboard-tests:get-compressed-steps test-id #!key (work-area #f))
-  (if (or (not work-area)
-	  (file-exists? (conc work-area "/testdat.db")))
-      (let* ((steps-data  (tdb:get-steps-for-test test-id work-area))
-	     (comprsteps  (dashboard-tests:process-steps-table steps-data))) ;; (open-run-close db:get-steps-table #f test-id work-area: work-area)))
-	(map (lambda (x)
-	       ;; take advantage of the \n on time->string
-	       (vector
-		(vector-ref x 0)
-		(let ((s (vector-ref x 1)))
-		  (if (number? s)(seconds->time-string s) s))
-		(let ((s (vector-ref x 2)))
-		  (if (number? s)(seconds->time-string s) s))
-		(vector-ref x 3)    ;; status
-		(vector-ref x 4)
-		(vector-ref x 5)))  ;; time delta
-	     (sort (hash-table-values comprsteps)
-		   (lambda (a b)
-		     (let ((time-a (vector-ref a 1))
-			   (time-b (vector-ref b 1)))
-		       (if (and (number? time-a)(number? time-b))
-			   (if (< time-a time-b)
-			       #t
-			       (if (eq? time-a time-b)
-				   (string<? (conc (vector-ref a 2))
-					     (conc (vector-ref b 2)))
-				   #f))
-			   (string<? (conc time-a)(conc time-b))))))))
-      '()))
+(define (dashboard-tests:get-compressed-steps db test-id)
+  (let* ((steps-data  (db:get-steps-for-test db test-id))
+	 (comprsteps  (dashboard-tests:process-steps-table steps-data))) ;; (open-run-close db:get-steps-table #f test-id work-area: work-area)))
+    (map (lambda (x)
+	   ;; take advantage of the \n on time->string
+	   (vector
+	    (vector-ref x 0)
+	    (let ((s (vector-ref x 1)))
+	      (if (number? s)(seconds->time-string s) s))
+	    (let ((s (vector-ref x 2)))
+	      (if (number? s)(seconds->time-string s) s))
+	    (vector-ref x 3)    ;; status
+	    (vector-ref x 4)
+	    (vector-ref x 5)))  ;; time delta
+	 (sort (hash-table-values comprsteps)
+	       (lambda (a b)
+		 (let ((time-a (vector-ref a 1))
+		       (time-b (vector-ref b 1)))
+		   (if (and (number? time-a)(number? time-b))
+		       (if (< time-a time-b)
+			   #t
+			   (if (eq? time-a time-b)
+			       (string<? (conc (vector-ref a 2))
+					 (conc (vector-ref b 2)))
+			       #f))
+		       (string<? (conc time-a)(conc time-b)))))))))
 
 ;;======================================================================
 ;;
 ;;======================================================================
 (define (examine-test test-id) ;; run-id run-key origtest)
@@ -467,11 +464,11 @@
 	       ;; These next two are intentional bad values to ensure errors if they should not
 	       ;; get filled in properly.
 	       (logfile       "/this/dir/better/not/exist")
 	       (rundir        logfile)
 	       (testdat-path  (conc rundir "/testdat.db")) ;; this gets recalculated until found 
-	       (teststeps     (if testdat (dashboard-tests:get-compressed-steps test-id work-area: rundir) '()))
+	       (teststeps     (if testdat (dashboard-tests:get-compressed-steps db test-id) '()))
 	       (testfullname  (if testdat (db:test-get-fullname testdat) "Gathering data ..."))
 	       (testname      (if testdat (db:test-get-testname testdat) "n/a"))
 	       (testmeta      (if testdat 
 				  (let ((tm (db:testmeta-get-record db testname)))
 				    (if tm tm (make-db:testmeta)))
@@ -524,11 +521,11 @@
 						     (db:get-test-info-by-id db test-id )))))
 			       ;; (debug:print-info 0 "need-update= " need-update " curr-mod-time = " curr-mod-time)
 			       (cond
 				((and need-update newtestdat)
 				 (set! testdat newtestdat)
-				 (set! teststeps    (dashboard-tests:get-compressed-steps test-id work-area: rundir))
+				 (set! teststeps    (dashboard-tests:get-compressed-steps db test-id))
 				 (set! logfile      (conc (db:test-get-rundir testdat) "/" (db:test-get-final_logf testdat)))
 				 (set! rundir       (db:test-get-rundir testdat))
 				 (set! testfullname (db:test-get-fullname testdat))
 				 ;; (debug:print 0 "INFO: teststeps=" (intersperse teststeps "\n    "))
 				 

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -1248,11 +1248,12 @@
      qry
      )
     res))
 
 (define (db:delete-test-records db test-id)
-  (tdb:delete-test-step-records db test-id)
+  (db:general-call 'delete-test-step-records test-id)
+  (db:general-call 'delete-test-data-records test-id)
   (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id))
 
 (define (db:delete-tests-for-run db run-id)
   (sqlite3:execute db "DELETE FROM tests WHERE run_id=?;" run-id))
 
@@ -1438,10 +1439,123 @@
      db 
      "SELECT rundir FROM tests WHERE id=?;"
      test-id)
     ;; (hash-table-set! *test-paths* test-id res)
     res)) ;; ))
+
+;;======================================================================
+;; S T E P S
+;;======================================================================
+
+(define (db:teststep-set-status! db test-id teststep-name state-in status-in comment logfile)
+   (sqlite3:execute 
+    db
+    "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);"
+    test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile "")))
+   
+;; db-get-test-steps-for-run
+(define (db:get-steps-for-test db test-id)
+  (let* ((res '()))
+    (sqlite3:for-each-row 
+     (lambda (id test-id stepname state status event-time logfile)
+       (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
+     db
+     "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
+     test-id)
+    (reverse res)))
+
+(define (db:get-steps-data db test-id)
+  (let ((res '()))
+    (sqlite3:for-each-row 
+     (lambda (id test-id stepname state status event-time logfile)
+       (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
+     db
+     "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
+     test-id)
+    (reverse res)))
+
+;;======================================================================
+;; T E S T  D A T A 
+;;======================================================================
+
+;; WARNING: Do NOT call this for the parent test on an iterated test
+;; Roll up test_data pass/fail results
+;; look at the test_data status field, 
+;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
+;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
+(define (db:test-data-rollup db test-id status)
+  (let ((fail-count 0)
+	(pass-count 0))
+    (sqlite3:for-each-row
+     (lambda (fcount pcount)
+       (set! fail-count fcount)
+       (set! pass-count pcount))
+     db 
+     "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
+                   (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;"
+     test-id test-id)
+    ;; Now rollup the counts to the central megatest.db
+    (db:general-call 'pass-fail-counts fail-count pass-count test-id)
+    ;; if the test is not FAIL then set status based on the fail and pass counts.
+    (db:general-call 'test_data-pf-rollup test-id test-id test-id test-id)))
+
+(define (db:csv->test-data db test-id csvdata)
+  (debug:print 4 "test-id " test-id ", csvdata: " csvdata)
+  (let ((csvlist (csv->list (make-csv-reader
+			     (open-input-string csvdata)
+			     '((strip-leading-whitespace? #t)
+			       (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata)))
+    (for-each 
+     (lambda (csvrow)
+       (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
+	      (category    (list-ref padded-row 0))
+	      (variable    (list-ref padded-row 1))
+	      (value       (any->number-if-possible (list-ref padded-row 2)))
+	      (expected    (any->number-if-possible (list-ref padded-row 3)))
+	      (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
+	      (units       (list-ref padded-row 5))
+	      (comment     (list-ref padded-row 6))
+	      (status      (let ((s (list-ref padded-row 7)))
+			     (if (and (string? s)(or (string-match (regexp "^\\s*$") s)
+						     (string-match (regexp "^n/a$") s)))
+				 #f
+				 s))) ;; if specified on the input then use, else calculate
+	      (type        (list-ref padded-row 8)))
+	 ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
+	 (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value 
+		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
+	 
+	 (if (and (or (not expected)(equal? expected ""))
+		  (or (not tol)     (equal? expected ""))
+		  (or (not units)   (equal? expected "")))
+	     (let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test tdb test-id category variable)))
+			 (set! expected new-expected)
+			 (set! tol      new-tol)
+			 (set! units    new-units)))
+	 
+	 (debug:print 4 "AFTER:  category: " category " variable: " variable " value: " value 
+		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	 ;; calculate status if NOT specified
+	 (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
+	     (if (number? tol) ;; if tol is a number then we do the standard comparison
+		 (let* ((max-val (+ expected tol))
+			(min-val (- expected tol))
+			(result  (and (>=  value min-val)(<= value max-val))))
+		   (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result)
+		   (set! status (if result "pass" "fail")))
+		 (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
+		       (case (string->symbol tol) ;; tol should be >, <, >=, <=
+			 ((>)  (if (>  value expected) "pass" "fail"))
+			 ((<)  (if (<  value expected) "pass" "fail"))
+			 ((>=) (if (>= value expected) "pass" "fail"))
+			 ((<=) (if (<= value expected) "pass" "fail"))
+			 (else (conc "ERROR: bad tol comparator " tol))))))
+	 (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value 
+		      ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
+	 (sqlite3:execute db "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
+			  test-id category variable value expected tol units (if comment comment "") status type)))
+     csvlist)))
 
 ;;======================================================================
 ;; Misc. test related queries
 ;;======================================================================
 
@@ -1607,11 +1721,14 @@
 ;;======================================================================
 ;; A G R E G A T E D   T R A N S A C T I O N   D B   W R I T E S 
 ;;======================================================================
 
 (define db:queries 
-  (list '(register-test          "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');")
+  (list '(update-run-duration     "UPDATE tests SET run_duration=? WHERE id=?;")
+
+	;; TESTS
+	'(register-test          "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');")
 	;; Test state and status
 	'(set-test-state         "UPDATE tests SET state=?   WHERE id=?;")
 	'(set-test-status        "UPDATE tests SET state=?   WHERE id=?;")
 	'(state-status           "UPDATE tests SET state=?,status=? WHERE id=?;")
 	'(state-status-msg       "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;")
@@ -1632,11 +1749,10 @@
 	'(test-set-rundir-by-test-id "UPDATE tests SET rundir=? WHERE id=?")
 	'(test-set-rundir         "UPDATE tests SET rundir=? WHERE run_id=? AND testname=? AND item_path=?;")
 	'(delete-tests-in-state   "DELETE FROM tests WHERE state=? AND run_id=?;")
 	'(tests:test-set-toplog   "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path='';")
 	'(update-cpuload-diskfree "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;")
-	'(update-run-duration     "UPDATE tests SET run_duration=? WHERE id=?;")
 	'(update-uname-host       "UPDATE tests SET uname=?,host=? WHERE id=?;")
 	'(update-test-state       "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
 	'(update-test-status      "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")
 	;; stuff for roll-up-pass-fail-counts
 	'(update-pass-fail-counts "UPDATE tests 
@@ -1658,10 +1774,14 @@
                                          WHERE run_id=? AND testname=?
                                               AND item_path != ''
                                               AND status = 'SKIP') > 0 THEN 'SKIP'
                                   ELSE 'UNKNOWN' END
                        WHERE run_id=? AND testname=? AND item_path='';")
+
+	;; STEPS
+	'(delete-test-step-records "UPDATE test_steps SET state='DELETED' WHERE id=?;")
+	'(delete-test-data-records "UPDATE test_data  SET state='DELETED' WHERE id=?;")
 	))
 
 ;; do not run these as part of the transaction
 (define db:special-queries   '(rollup-tests-pass-fail
 			       ;; db:roll-up-pass-fail-counts  ;; WHY NOT!?

Index: ezsteps.scm
==================================================================
--- ezsteps.scm
+++ ezsteps.scm
@@ -77,12 +77,11 @@
 		  
 		  ;; call the command using mt_ezstep
 		  (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd))
 		  
 		  (debug:print 4 "script: " script)
-		  ;; DO NOT remote
-		  (db:teststep-set-status! #f test-id stepname "start" "-" #f #f work-area: test-run-dir)
+		  (rmt:teststep-set-status! #f test-id stepname "start" "-" #f #f)
 		  ;; now launch
 		  (let ((pid (process-run script)))
 		    (let processloop ((i 0))
 		      (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
 				  (mutex-lock! run-mutex)
@@ -95,12 +94,11 @@
 					(thread-sleep! 1)
 					(processloop (+ i 1))))
 				  ))
 		    (let ((exinfo (vector-ref exit-info 2))
 			  (logfna (if logpro-used (conc stepname ".html") "")))
-		      ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect)
-		      (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: test-run-dir))
+		      (rmt:teststep-set-status! #f test-id stepname "end" exinfo #f logfna))
 		    (if logpro-used
 			(rmt:test-set-log! test-id (conc stepname ".html")))
 		    ;; set the test final status
 		    (let* ((this-step-status (cond
 					      ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -235,12 +235,11 @@
 						   
 						   ;; call the command using mt_ezstep
 						   (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd))
 
 						   (debug:print 4 "script: " script)
-						   ;; DO NOT remote
-						   (tdb:teststep-set-status! test-id stepname "start" "-" #f #f work-area: work-area)
+						   (rmt:teststep-set-status! test-id stepname "start" "-" #f #f)
 						   ;; now launch
 						   (let ((pid (process-run script)))
 						     (let processloop ((i 0))
 						       (let-values (((pid-val exit-status exit-code)(process-wait pid #t)))
 								   (mutex-lock! m)
@@ -253,12 +252,11 @@
 									 (thread-sleep! 2)
 									 (processloop (+ i 1))))
 								   ))
                                                      (let ((exinfo (vector-ref exit-info 2))
                                                            (logfna (if logpro-used (conc stepname ".html") "")))
-						       ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect)
-						       (tdb:teststep-set-status! test-id stepname "end" exinfo #f logfna work-area: work-area))
+						       (rmt:teststep-set-status! test-id stepname "end" exinfo #f logfna))
 						     (if logpro-used
 							 (rmt:test-set-log! test-id (conc stepname ".html")))
 						     ;; set the test final status
 						     (let* ((this-step-status (cond
 									       ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -902,12 +902,11 @@
 	(if (not (setup-for-run))
 	    (begin
 	      (debug:print 0 "Failed to setup, exiting")
 	      (exit 1)))
 	(if (and state status)
-	    ;; DO NOT remote run, makes calls to the testdat.db test db.
-	    (tdb:teststep-set-status! test-id step state status msg logfile work-area: work-area)
+	    (rmt:teststep-set-status! test-id step state status msg logfile)
 	    (begin
 	      (debug:print 0 "ERROR: You must specify :state and :status with every call to -step")
 	      (exit 6))))))
 
 (if (args:get-arg "-step")
@@ -993,16 +992,15 @@
 				       (else ">&")))
 			 (fullcmd    (conc "(" (string-intersperse 
 						(cons cmd params) " ")
 					   ") " redir " " logfile)))
 		    ;; mark the start of the test
-		    ;; DO NOT run remote
-		    (tdb:teststep-set-status! test-id stepname "start" "n/a" (args:get-arg "-m") logfile work-area: work-area)
+		    (rmt:teststep-set-status! test-id stepname "start" "n/a" (args:get-arg "-m") logfile)
 		    ;; run the test step
 		    (debug:print-info 2 "Running \"" fullcmd "\" in directory \"" startingdir)
 		    (change-directory startingdir)
-		    (set! exitstat (system fullcmd)) ;; cmd params))
+		    (set! exitstat (system fullcmd))
 		    (set! *globalexitstatus* exitstat)
 		    ;; (change-directory testpath)
 		    ;; run logpro if applicable ;; (process-run "ls" (list "/foo" "2>&1" "blah.log"))
 		    (if logprofile
 			(let* ((htmllogfile (conc stepname ".html"))
@@ -1013,12 +1011,11 @@
 			  (set! exitstat (system cmd))
 			  (set! *globalexitstatus* exitstat) ;; no necessary
 			  (change-directory testpath)
 			  (rmt:test-set-log! test-id htmllogfile)))
 		    (let ((msg (args:get-arg "-m")))
-		      ;; DO NOT run remote
-		      (tdb:teststep-set-status! test-id stepname "end" exitstat msg logfile work-area: work-area))
+		      (rmt:teststep-set-status! test-id stepname "end" exitstat msg logfile))
 		    )))
 	  (if (or (args:get-arg "-test-status")
 		  (args:get-arg "-set-values"))
 	      (let ((newstatus (cond
 				((number? status)       (if (equal? status 0) "PASS" "FAIL"))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -221,15 +221,23 @@
 ;;  2. Open the testdat.db file and do the query
 ;; If not given the work area
 ;;  1. Do a remote call to get the test path
 ;;  2. Continue as above
 ;; 
-(define (rmt:get-steps-for-test test-id #!key (work-area #f))
-  (let* ((tdb (rmt:open-test-db-by-test-id test-id work-area: work-area)))
-    (if tdb
-	(tdb:get-steps-data tdb test-id)
-	'())))
+(define (rmt:get-steps-for-test test-id)
+  (rmt:send-receive 'get-steps-data (list test-id)))
+
+(define (rmt:teststep-set-status! db test-id teststep-name state-in status-in comment logfile)
+  (let* ((state     (items:check-valid-items "state" state-in))
+	 (status    (items:check-valid-items "status" status-in)))
+    (if (or (not state)(not status))
+	(debug:print 3 "WARNING: Invalid " (if status "status" "state")
+		     " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
+    (rmt:send-receive 'teststep-set-status! (list test-id teststep-name state-in status-in comment logfile))))
+
+(define (rmt:get-steps-for-test test-id)
+  (rmt:send-receive 'get-steps-for-test (list test-id)))
 
 ;;======================================================================
 ;;  T E S T   D A T A 
 ;;======================================================================
 
@@ -245,5 +253,11 @@
 (define (rmt:testmeta-get-record testname)
   (rmt:send-receive 'testmeta-get-record (list testname)))
 
 (define (rmt:testmeta-update-field test-name fld val)
   (rmt:send-receive 'testmeta-update-field (list test-name fld val)))
+
+(define (rmt:test-data-rollup test-id status)
+  (rmt:send-receive 'test-data-rollup (list test-id status)))
+
+(define (db:csv->test-data db test-id csvdata)
+  (rmt:send-receive 'csv->test-data (list test-id csvdata)))

Index: tdb.scm
==================================================================
--- tdb.scm
+++ tdb.scm
@@ -158,21 +158,10 @@
               val TEXT,
               ackstate INTEGER DEFAULT 0,
               CONSTRAINT metadat_constraint UNIQUE (var));"))
   (debug:print 11 "db:testdb-initialize END"))
 
-(define (tdb:get-steps-data tdb test-id)
-  (let ((res '()))
-    (sqlite3:for-each-row 
-     (lambda (id test-id stepname state status event-time logfile)
-       (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
-     tdb
-     "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
-     test-id)
-    (sqlite3:finalize! tdb)
-    (reverse res)))
-
 (define (tdb:read-test-data tdb test-id categorypatt)
   (let ((res '()))
     (sqlite3:for-each-row 
      (lambda (id test_id category variable value expected tol units comment status type)
        (set! res (cons (vector id test_id category variable value expected tol units comment status type) res)))
@@ -183,70 +172,10 @@
 
 ;;======================================================================
 ;; T E S T   D A T A 
 ;;======================================================================
 
-(define (tdb:csv->test-data test-id csvdata #!key (work-area #f))
-  (debug:print 4 "test-id " test-id ", csvdata: " csvdata)
-  (let ((tdb     (tdb:open-test-db-by-test-id-local test-id work-area: work-area)))
-    (if (sqlite3:database? tdb)
-	(let ((csvlist (csv->list (make-csv-reader
-				   (open-input-string csvdata)
-				   '((strip-leading-whitespace? #t)
-				     (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata)))
-	  (for-each 
-	   (lambda (csvrow)
-	     (let* ((padded-row  (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9))
-		    (category    (list-ref padded-row 0))
-		    (variable    (list-ref padded-row 1))
-		    (value       (any->number-if-possible (list-ref padded-row 2)))
-		    (expected    (any->number-if-possible (list-ref padded-row 3)))
-		    (tol         (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number
-		    (units       (list-ref padded-row 5))
-		    (comment     (list-ref padded-row 6))
-		    (status      (let ((s (list-ref padded-row 7)))
-				   (if (and (string? s)(or (string-match (regexp "^\\s*$") s)
-							   (string-match (regexp "^n/a$") s)))
-				       #f
-				       s))) ;; if specified on the input then use, else calculate
-		    (type        (list-ref padded-row 8)))
-	       ;; look up expected,tol,units from previous best fit test if they are all either #f or ''
-	       (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type)
-
-	       (if (and (or (not expected)(equal? expected ""))
-			(or (not tol)     (equal? expected ""))
-			(or (not units)   (equal? expected "")))
-		   (let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test tdb test-id category variable)))
-			       (set! expected new-expected)
-			       (set! tol      new-tol)
-			       (set! units    new-units)))
-
-	       (debug:print 4 "AFTER:  category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	       ;; calculate status if NOT specified
-	       (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers
-		   (if (number? tol) ;; if tol is a number then we do the standard comparison
-		       (let* ((max-val (+ expected tol))
-			      (min-val (- expected tol))
-			      (result  (and (>=  value min-val)(<= value max-val))))
-			 (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result)
-			 (set! status (if result "pass" "fail")))
-		       (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op.
-			     (case (string->symbol tol) ;; tol should be >, <, >=, <=
-			       ((>)  (if (>  value expected) "pass" "fail"))
-			       ((<)  (if (<  value expected) "pass" "fail"))
-			       ((>=) (if (>= value expected) "pass" "fail"))
-			       ((<=) (if (<= value expected) "pass" "fail"))
-			       (else (conc "ERROR: bad tol comparator " tol))))))
-	       (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value 
-			    ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment)
-	       (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);"
-				test-id category variable value expected tol units (if comment comment "") status type)))
-	   csvlist)
-	  (sqlite3:finalize! tdb)))))
-
 ;; ;; get a list of test_data records matching categorypatt
 ;; (define (tdb:read-test-data test-id categorypatt #!key (work-area #f))
 ;;   (let ((tdb  (tdb:open-test-db-by-test-id test-id work-area: work-area)))
 ;;     (if (sqlite3:database? tdb)
 ;; 	(let ((res '()))
@@ -263,62 +192,15 @@
 (define (tdb:load-test-data test-id #!key (work-area #f))
   (let loop ((lin (read-line)))
     (if (not (eof-object? lin))
 	(begin
 	  (debug:print 4 lin)
-	  (tdb:csv->test-data test-id lin work-area: work-area)
+	  (rmt:csv->test-data test-id lin work-area: work-area)
 	  (loop (read-line)))))
   ;; roll up the current results.
-  ;; FIXME: Add the status to 
-  (tdb:test-data-rollup test-id #f work-area: work-area))
-
-;; WARNING: Do NOT call this for the parent test on an iterated test
-;; Roll up test_data pass/fail results
-;; look at the test_data status field, 
-;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
-;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
-(define (tdb:test-data-rollup test-id status #!key (work-area #f))
-  (let ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area))
-	(fail-count 0)
-	(pass-count 0))
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:for-each-row
-	   (lambda (fcount pcount)
-	     (set! fail-count fcount)
-	     (set! pass-count pcount))
-	   tdb 
-	   "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count,
-                   (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;"
-	   test-id test-id)
-
-	  ;; Now rollup the counts to the central megatest.db
-	  (rmt:general-call 'pass-fail-counts fail-count pass-count test-id)
-	  ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" 
-	  ;;                     fail-count pass-count test-id)
-
-	  ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines
-	  ;; next time you read this!
-	  ;;
-	  ;; (cdb:flush-queue *runremote*)
-	  ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set
-	  
-	  ;; if the test is not FAIL then set status based on the fail and pass counts.
-	  (rmt:general-call 'test_data-pf-rollup test-id test-id test-id test-id)
-	  ;; (sqlite3:execute
-	  ;;  db   ;;; NOTE: Should this be WARN,FAIL? A WARN is not a FAIL????? BUG FIXME
-	  ;;  "UPDATE tests
-	  ;;             SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 
-	  ;;                THEN 'FAIL'
-	  ;;             WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND 
-	  ;;                  (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL')
-	  ;;             THEN 'PASS'
-	  ;;             ELSE status
-	  ;;         END WHERE id=?;"
-	  ;;  test-id test-id test-id test-id)
-	  (sqlite3:finalize! tdb)
-	  ))))
+  ;; FIXME: Add the status too 
+  (rmt:test-data-rollup test-id #f))
 
 (define (tdb:get-prev-tol-for-test tdb test-id category variable)
   ;; Finish me?
   (values #f #f #f))
 
@@ -327,207 +209,156 @@
 ;;======================================================================
 
 (define (tdb:step-get-time-as-string vec)
   (seconds->time-string (tdb:step-get-event_time vec)))
 
-;; db-get-test-steps-for-run
-(define (tdb:get-steps-for-test test-id #!key (work-area #f))
-  (let* ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area))
-	 (res '()))
-    (if (sqlite3:database? tdb)
-	(handle-exceptions
-	 exn
-	 (begin
-	   (debug:print 0 "ERROR: error on access to testdat for test with id " test-id)
-	   '())
-	 (begin
-	   (sqlite3:for-each-row 
-	    (lambda (id test-id stepname state status event-time logfile)
-	      (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res)))
-	    tdb
-	    "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
-	    test-id)
-	   (sqlite3:finalize! tdb)
-	   (reverse res)))
-	'())))
-
-;; get a pretty table to summarize steps
-;;
-(define (tdb:get-steps-table test-id #!key (work-area #f))
-  (let ((steps   (tdb:get-steps-for-test test-id work-area: work-area)))
-    ;; organise the steps for better readability
-    (let ((res (make-hash-table)))
-      (for-each 
-       (lambda (step)
-	 (debug:print 6 "step=" step)
-	 (let ((record (hash-table-ref/default 
-			res 
-			(tdb:step-get-stepname step) 
-			;;        stepname                start end status Duration  Logfile 
-			(vector (tdb:step-get-stepname step) ""   "" ""     ""        ""))))
-	   (debug:print 6 "record(before) = " record 
-			"\nid:       " (tdb:step-get-id step)
-			"\nstepname: " (tdb:step-get-stepname step)
-			"\nstate:    " (tdb:step-get-state step)
-			"\nstatus:   " (tdb:step-get-status step)
-			"\ntime:     " (tdb:step-get-event_time step))
-	   (case (string->symbol (tdb:step-get-state step))
-	     ((start)(vector-set! record 1 (tdb:step-get-event_time step))
-	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
-					(tdb:step-get-status step)))
-	      (if (> (string-length (tdb:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (tdb:step-get-logfile step))))
-	     ((end)  
-	      (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
-	      (vector-set! record 3 (tdb:step-get-status step))
-	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
-					  (endt   (any->number (vector-ref record 2))))
-				      (debug:print 4 "record[1]=" (vector-ref record 1) 
-						   ", startt=" startt ", endt=" endt
-						   ", get-status: " (tdb:step-get-status step))
-				      (if (and (number? startt)(number? endt))
-					  (seconds->hr-min-sec (- endt startt)) "-1")))
-	      (if (> (string-length (tdb:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (tdb:step-get-logfile step))))
-	     (else
-	      (vector-set! record 2 (tdb:step-get-state step))
-	      (vector-set! record 3 (tdb:step-get-status step))
-	      (vector-set! record 4 (tdb:step-get-event_time step))))
-	   (hash-table-set! res (tdb:step-get-stepname step) record)
-	   (debug:print 6 "record(after)  = " record 
-			"\nid:       " (tdb:step-get-id step)
-			"\nstepname: " (tdb:step-get-stepname step)
-			"\nstate:    " (tdb:step-get-state step)
-			"\nstatus:   " (tdb:step-get-status step)
-			"\ntime:     " (tdb:step-get-event_time step))))
-       ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
-       (sort steps (lambda (a b)
-		     (cond
-		      ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
-		      ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
-		       (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
-		      (else #f)))))
-      res)))
-
-;; get a pretty table to summarize steps
-;;
-(define (tdb:get-steps-table-list test-id #!key (work-area #f))
-  (let ((steps   (tdb:get-steps-for-test test-id work-area: work-area)))
-    ;; organise the steps for better readability
-    (let ((res (make-hash-table)))
-      (for-each 
-       (lambda (step)
-	 (debug:print 6 "step=" step)
-	 (let ((record (hash-table-ref/default 
-			res 
-			(tdb:step-get-stepname step) 
-			;;        stepname                start end status    
-			(vector (tdb:step-get-stepname step) ""   "" ""     "" ""))))
-	   (debug:print 6 "record(before) = " record 
-			"\nid:       " (tdb:step-get-id step)
-			"\nstepname: " (tdb:step-get-stepname step)
-			"\nstate:    " (tdb:step-get-state step)
-			"\nstatus:   " (tdb:step-get-status step)
-			"\ntime:     " (tdb:step-get-event_time step))
-	   (case (string->symbol (tdb:step-get-state step))
-	     ((start)(vector-set! record 1 (tdb:step-get-event_time step))
-	      (vector-set! record 3 (if (equal? (vector-ref record 3) "")
-					(tdb:step-get-status step)))
-	      (if (> (string-length (tdb:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (tdb:step-get-logfile step))))
-	     ((end)  
-	      (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
-	      (vector-set! record 3 (tdb:step-get-status step))
-	      (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
-					  (endt   (any->number (vector-ref record 2))))
-				      (debug:print 4 "record[1]=" (vector-ref record 1) 
-						   ", startt=" startt ", endt=" endt
-						   ", get-status: " (tdb:step-get-status step))
-				      (if (and (number? startt)(number? endt))
-					  (seconds->hr-min-sec (- endt startt)) "-1")))
-	      (if (> (string-length (tdb:step-get-logfile step))
-		     0)
-		  (vector-set! record 5 (tdb:step-get-logfile step))))
-	     (else
-	      (vector-set! record 2 (tdb:step-get-state step))
-	      (vector-set! record 3 (tdb:step-get-status step))
-	      (vector-set! record 4 (tdb:step-get-event_time step))))
-	   (hash-table-set! res (tdb:step-get-stepname step) record)
-	   (debug:print 6 "record(after)  = " record 
-			"\nid:       " (tdb:step-get-id step)
-			"\nstepname: " (tdb:step-get-stepname step)
-			"\nstate:    " (tdb:step-get-state step)
-			"\nstatus:   " (tdb:step-get-status step)
-			"\ntime:     " (tdb:step-get-event_time step))))
-       ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
-       (sort steps (lambda (a b)
-		     (cond
-		      ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
-		      ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
-		       (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
-		      (else #f)))))
-      res)))
-
-(define (tdb:get-compressed-steps test-id #!key (work-area #f)(tdb #f))
-  (if (or (not work-area)
-	  (file-exists? (conc work-area "/testdat.db")))
-      (let* ((comprsteps (tdb:get-steps-table test-id work-area: work-area)))
-	(map (lambda (x)
-	       ;; take advantage of the \n on time->string
-	       (vector
-		(vector-ref x 0)
-		(let ((s (vector-ref x 1)))
-		  (if (number? s)(seconds->time-string s) s))
-		(let ((s (vector-ref x 2)))
-		  (if (number? s)(seconds->time-string s) s))
-		(vector-ref x 3)    ;; status
-		(vector-ref x 4)
-		(vector-ref x 5)))  ;; time delta
-	     (sort (hash-table-values comprsteps)
-		   (lambda (a b)
-		     (let ((time-a (vector-ref a 1))
-			   (time-b (vector-ref b 1)))
-		       (if (and (number? time-a)(number? time-b))
-			   (if (< time-a time-b)
-			       #t
-			       (if (eq? time-a time-b)
-				   (string<? (conc (vector-ref a 2))
-					     (conc (vector-ref b 2)))
-				   #f))
-			   (string<? (conc time-a)(conc time-b))))))))
-      '()))
-
-(define (tdb:teststep-set-status! test-id teststep-name state-in status-in comment logfile #!key (work-area #f))
-  (let* ((tdb       (tdb:open-test-db-by-test-id-local test-id work-area: work-area))
-	 (state     (items:check-valid-items "state" state-in))
-	 (status    (items:check-valid-items "status" status-in)))
-    (if (or (not state)(not status))
-	(debug:print 3 "WARNING: Invalid " (if status "status" "state")
-		     " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:execute 
-	   tdb
-	   "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);"
-	   test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile ""))
-	  (sqlite3:finalize! tdb)
-	  #t)
-	#f)))
-
-;; this one is a bit broken BUG FIXME
-(define (tdb:delete-test-step-records test-id #!key (work-area #f))
-  ;; Breaking it into two queries for better file access interleaving
-  (let* ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area)))
-    ;; test db's can go away - must check every time
-    (if (sqlite3:database? tdb)
-	(begin
-	  (sqlite3:execute tdb "DELETE FROM test_steps;")
-	  (sqlite3:execute tdb "DELETE FROM test_data;")
-	  (sqlite3:finalize! tdb)))))
+;; get a pretty table to summarize steps
+;;
+(define (tdb:get-steps-table steps);; organise the steps for better readability
+  (let ((res (make-hash-table)))
+    (for-each 
+     (lambda (step)
+       (debug:print 6 "step=" step)
+       (let ((record (hash-table-ref/default 
+		      res 
+		      (tdb:step-get-stepname step) 
+		      ;;        stepname                start end status Duration  Logfile 
+		      (vector (tdb:step-get-stepname step) ""   "" ""     ""        ""))))
+	 (debug:print 6 "record(before) = " record 
+		      "\nid:       " (tdb:step-get-id step)
+		      "\nstepname: " (tdb:step-get-stepname step)
+		      "\nstate:    " (tdb:step-get-state step)
+		      "\nstatus:   " (tdb:step-get-status step)
+		      "\ntime:     " (tdb:step-get-event_time step))
+	 (case (string->symbol (tdb:step-get-state step))
+	   ((start)(vector-set! record 1 (tdb:step-get-event_time step))
+	    (vector-set! record 3 (if (equal? (vector-ref record 3) "")
+				      (tdb:step-get-status step)))
+	    (if (> (string-length (tdb:step-get-logfile step))
+		   0)
+		(vector-set! record 5 (tdb:step-get-logfile step))))
+	   ((end)  
+	    (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
+	    (vector-set! record 3 (tdb:step-get-status step))
+	    (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
+					(endt   (any->number (vector-ref record 2))))
+				    (debug:print 4 "record[1]=" (vector-ref record 1) 
+						 ", startt=" startt ", endt=" endt
+						 ", get-status: " (tdb:step-get-status step))
+				    (if (and (number? startt)(number? endt))
+					(seconds->hr-min-sec (- endt startt)) "-1")))
+	    (if (> (string-length (tdb:step-get-logfile step))
+		   0)
+		(vector-set! record 5 (tdb:step-get-logfile step))))
+	   (else
+	    (vector-set! record 2 (tdb:step-get-state step))
+	    (vector-set! record 3 (tdb:step-get-status step))
+	    (vector-set! record 4 (tdb:step-get-event_time step))))
+	 (hash-table-set! res (tdb:step-get-stepname step) record)
+	 (debug:print 6 "record(after)  = " record 
+		      "\nid:       " (tdb:step-get-id step)
+		      "\nstepname: " (tdb:step-get-stepname step)
+		      "\nstate:    " (tdb:step-get-state step)
+		      "\nstatus:   " (tdb:step-get-status step)
+		      "\ntime:     " (tdb:step-get-event_time step))))
+     ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
+     (sort steps (lambda (a b)
+		   (cond
+		    ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
+		    ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
+		     (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
+		    (else #f)))))
+    res))
+
+;; Move this to steps.scm
+;;
+;; get a pretty table to summarize steps
+;;
+(define (tdb:get-steps-table-list steps)
+  ;; organise the steps for better readability
+  (let ((res (make-hash-table)))
+    (for-each 
+     (lambda (step)
+       (debug:print 6 "step=" step)
+       (let ((record (hash-table-ref/default 
+		      res 
+		      (tdb:step-get-stepname step) 
+		      ;;        stepname                start end status    
+		      (vector (tdb:step-get-stepname step) ""   "" ""     "" ""))))
+	 (debug:print 6 "record(before) = " record 
+		      "\nid:       " (tdb:step-get-id step)
+		      "\nstepname: " (tdb:step-get-stepname step)
+		      "\nstate:    " (tdb:step-get-state step)
+		      "\nstatus:   " (tdb:step-get-status step)
+		      "\ntime:     " (tdb:step-get-event_time step))
+	 (case (string->symbol (tdb:step-get-state step))
+	   ((start)(vector-set! record 1 (tdb:step-get-event_time step))
+	    (vector-set! record 3 (if (equal? (vector-ref record 3) "")
+				      (tdb:step-get-status step)))
+	    (if (> (string-length (tdb:step-get-logfile step))
+		   0)
+		(vector-set! record 5 (tdb:step-get-logfile step))))
+	   ((end)  
+	    (vector-set! record 2 (any->number (tdb:step-get-event_time step)))
+	    (vector-set! record 3 (tdb:step-get-status step))
+	    (vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
+					(endt   (any->number (vector-ref record 2))))
+				    (debug:print 4 "record[1]=" (vector-ref record 1) 
+						 ", startt=" startt ", endt=" endt
+						 ", get-status: " (tdb:step-get-status step))
+				    (if (and (number? startt)(number? endt))
+					(seconds->hr-min-sec (- endt startt)) "-1")))
+	    (if (> (string-length (tdb:step-get-logfile step))
+		   0)
+		(vector-set! record 5 (tdb:step-get-logfile step))))
+	   (else
+	    (vector-set! record 2 (tdb:step-get-state step))
+	    (vector-set! record 3 (tdb:step-get-status step))
+	    (vector-set! record 4 (tdb:step-get-event_time step))))
+	 (hash-table-set! res (tdb:step-get-stepname step) record)
+	 (debug:print 6 "record(after)  = " record 
+		      "\nid:       " (tdb:step-get-id step)
+		      "\nstepname: " (tdb:step-get-stepname step)
+		      "\nstate:    " (tdb:step-get-state step)
+		      "\nstatus:   " (tdb:step-get-status step)
+		      "\ntime:     " (tdb:step-get-event_time step))))
+     ;; (else   (vector-set! record 1 (tdb:step-get-event_time step)))
+     (sort steps (lambda (a b)
+		   (cond
+		    ((<   (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t)
+		    ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) 
+		     (<   (tdb:step-get-id a)        (tdb:step-get-id b)))
+		    (else #f)))))
+    res))
+
+;;
+;; Move to steps.scm
+;;
+(define (tdb:get-compressed-steps comprsteps) ;; from tdb:get-steps-table
+  (map (lambda (x)
+	 ;; take advantage of the \n on time->string
+	 (vector
+	  (vector-ref x 0)
+	  (let ((s (vector-ref x 1)))
+	    (if (number? s)(seconds->time-string s) s))
+	  (let ((s (vector-ref x 2)))
+	    (if (number? s)(seconds->time-string s) s))
+	  (vector-ref x 3)    ;; status
+	  (vector-ref x 4)
+	  (vector-ref x 5)))  ;; time delta
+       (sort (hash-table-values comprsteps)
+	     (lambda (a b)
+	       (let ((time-a (vector-ref a 1))
+		     (time-b (vector-ref b 1)))
+		 (if (and (number? time-a)(number? time-b))
+		     (if (< time-a time-b)
+			 #t
+			 (if (eq? time-a time-b)
+			     (string<? (conc (vector-ref a 2))
+				       (conc (vector-ref b 2)))
+			     #f))
+		     (string<? (conc time-a)(conc time-b))))))))
 
 ;; 
 (define (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes)
   (let ((tdb         (tdb:open-test-db-by-test-id-local test-id work-area: work-area)))
     (if (sqlite3:database? tdb)

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -241,11 +241,11 @@
 	  (mt:process-triggers test-id state real-status)))
     
     ;; if status is "AUTO" then call rollup (note, this one modifies data in test
     ;; run area, it does remote calls under the hood.
     (if (and test-id state status (equal? status "AUTO")) 
-	(tdb:test-data-rollup test-id status work-area: work-area))
+	(rmt:test-data-rollup test-id status))
 
     ;; add metadata (need to do this way to avoid SQL injection issues)
 
     ;; :first_err
     ;; (let ((val (hash-table-ref/default otherdat ":first_err" #f)))