Index: dashboard-tests.scm
==================================================================
--- dashboard-tests.scm
+++ dashboard-tests.scm
@@ -247,11 +247,11 @@
 
 ;;======================================================================
 ;;
 ;;======================================================================
 (define (examine-test db test-id) ;; run-id run-key origtest)
-  (let* ((testdat       (db:get-test-data-by-id db test-id)))
+  (let* ((testdat       (rdb:get-test-data-by-id db test-id)))
     (if (not testdat)
 	(begin
 	  (debug:print 0 "ERROR: No test data found for test " test-id ", exiting")
 	  (exit 1))
 	(let* ((run-id        (if testdat (db:test-get-run_id testdat) #f))

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -76,25 +76,27 @@
     (begin
       (print "Failed to find megatest.config, exiting") 
       (exit 1)))
 
 (define *db* (open-db))
-(server:client-setup *db*)
+;; (server:client-setup *db*)
 
 (define toplevel #f)
 (define dlg      #f)
 (define max-test-num 0)
 (define *keys*   (rdb:get-keys   *db*))
+(define *keys*   (db:get-keys   *db*))
 (define *dbkeys*  (map (lambda (x)(vector-ref x 0))
 		      (append *keys* (list (vector "runname" "blah")))))
 (define *header*       #f)
 (define *allruns*     '())
 (define *buttondat*    (make-hash-table)) ;; <run-id color text test run-key>
 (define *alltestnamelst* '())
 (define *searchpatts*  (make-hash-table))
 (define *num-runs*      8)
 (define *tot-run-count* (rdb:get-num-runs *db* "%"))
+(define *tot-run-count* (db:get-num-runs *db* "%"))
 (define *last-update*   (current-seconds))
 (define *num-tests*     15)
 (define *start-run-offset*  0)
 (define *start-test-offset* 0)
 (define *examine-test-dat* (make-hash-table))

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -27,10 +27,16 @@
 (include "common_records.scm")
 (include "db_records.scm")
 (include "key_records.scm")
 (include "run_records.scm")
 
+;; timestamp type (val1 val2 ...)
+;; type: meta-info, step
+(define *incoming-data*      '())
+(define *incoming-last-time* (current-seconds))
+(define *incoming-mutex*     (make-mutex))
+
 (define (open-db) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
   (let* ((dbpath    (conc *toppath* "/megatest.db")) ;; fname)
 	 (dbexists  (file-exists? dbpath))
 	 (db        (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath))
 	 (handler   (make-busy-timeout 36000)))
@@ -614,23 +620,55 @@
        (set! res (cons p res)))
      db 
      qrystr)
     res))
 
+;;======================================================================
+;; QUEUE UP META, TEST STATUS AND STEPS
+;;======================================================================
+
+(define (db:updater db)
+  (let loop ((start-time (current-time)))
+    (thread-sleep! (+ 2 (random 10))) ;; move save time around to minimize regular collisions
+    (db:write-cached-data db)
+    (loop start-time)))
+    
 (define (db:test-update-meta-info db run-id test-name item-path minutes cpuload diskfree tmpfree)
   (if (not item-path)
       (begin (debug:print 0 "WARNING: ITEMPATH not set.")   
 	     (set! item-path "")))
-  (sqlite3:execute
-   db
-   "UPDATE tests SET cpuload=?,diskfree=?,run_duration=?,state='RUNNING' WHERE run_id=? AND testname=? AND item_path=? AND state NOT IN ('COMPLETED','KILLREQ','KILLED');"
-   cpuload
-   diskfree
-   minutes
-   run-id
-   test-name
-   item-path))
+  (mutex-lock! *incoming-mutex*)
+  (set! *incoming-data* (cons (vector 'meta-info
+				      (current-seconds)
+				      (list cpuload
+					    diskfree
+					    minutes
+					    run-id
+					    test-name
+					    item-path)) ;; run-id test-name item-path minutes cpuload diskfree tmpfree) 
+			      *incoming-data*))
+  (mutex-unlock! *incoming-mutex*))
+
+(define (db:write-cached-data db)
+  (let ((meta-stmt (sqlite3:prepare db "UPDATE tests SET cpuload=?,diskfree=?,run_duration=?,state='RUNNING' WHERE run_id=? AND testname=? AND item_path=? AND state NOT IN ('COMPLETED','KILLREQ','KILLED');"))
+	(step-stmt (sqlite3:prepare db "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);")) ;; strftime('%s','now')#f)
+	(data (sort *incoming-data* (lambda (a b)(< (vector-ref a 1)(vector-ref b 1))))))
+    (debug:print 0 "Writing cached data " data)
+    (mutex-lock! *incoming-mutex*)
+    (for-each (lambda (entry)
+		(case (vector-ref entry 0)
+		  ((meta-info)
+		   (apply sqlite3:execute meta-stmt (vector-ref entry 2)))
+		  ((step-status)
+		   (apply sqlite3:execute step-stmt (vector-ref entry 2)))
+		  (else
+		   (debug:print 0 "ERROR: Queued entry not recognised " entry))))
+	      data)
+    (set! *incoming-data* '())
+    (mutex-unlock! *incoming-mutex*)
+    (sqlite3:finalize! meta-stmt)
+    (sqlite3:finalize! step-stmt)))
 
 (define (db:roll-up-pass-fail-counts db run-id test-name item-path status)
   (if (and (not (equal? item-path ""))
 	   (or (equal? status "PASS")
 	       (equal? status "WARN")
@@ -931,15 +969,17 @@
 	     (or (not state)(not status)))
 	(debug:print 0 "WARNING: Invalid " (if status "status" "state")
 	       " value \"" (if status state-in status-in) "\", update your validvalues section in megatest.config"))
     (if testdat
 	(let ((test-id (test:get-id testdat)))
-	  ;; FIXME - this should not update the logfile unless it is specified.
-	  (sqlite3:execute db 
-			"INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,strftime('%s','now'),?,?);"
-			test-id teststep-name state-in status-in (if comment comment "") (if logfile logfile ""))
-	  #t) ;; fake out a #t - could be execute is returning something complicated
+	  (mutex-lock! *incoming-mutex*)
+	  (set! *incoming-data* (cons (vector 'step-status
+				      (current-seconds)
+				      ;; FIXME - this should not update the logfile unless it is specified.
+				      (list test-id teststep-name state-in status-in (current-seconds) (if comment comment "") (if logfile logfile "")))))
+	  (mutex-unlock! *incoming-mutex*)
+	  #t)
 	(debug:print 0 "ERROR: Can't update " test-name " for run " run-id " -> no such test in db"))))
 
 ;;======================================================================
 ;; Extract ods file from the db
 ;;======================================================================
@@ -1150,10 +1190,18 @@
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:get-tests-for-run host port)
 	  run-id testpatt itempatt states statuses))
       (db:get-tests-for-run db run-id testpatt itempatt states statuses)))
 
+(define (rdb:get-test-data-by-id db test-id)
+  (if *runremote*
+      (let ((host (vector-ref *runremote* 0))
+	    (port (vector-ref *runremote* 1)))
+	((rpc:procedure 'rpc:get-test-data-by-id host port)
+	 test-id))
+      (db:get-test-data-by-id db test-id)))
+      
 (define (rdb:get-keys db)
   (if *runremote*
       (let ((host (vector-ref *runremote* 0))
 	    (port (vector-ref *runremote* 1)))
 	((rpc:procedure 'rdb:get-keys host port)))

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -401,11 +401,12 @@
 	   (new-test-name (if (equal? item-path "") test-name (conc test-name "/" item-path))) ;; just need it to be unique
 	   (testdat       (rdb:get-test-info db run-id test-name item-path)))
       (if (not testdat)
 	  (begin
 	    ;; ensure that the path exists before registering the test
-	    (system (conc "mkdir -p " new-test-path))
+	    ;; NOPE: Cannot! Don't know yet which disk area will be assigned....
+	    ;; (system (conc "mkdir -p " new-test-path))
 	    (rtests:register-test db run-id test-name item-path)
 	    (set! testdat (rdb:get-test-info db run-id test-name item-path))))
       (change-directory test-path)
       (case (if force ;; (args:get-arg "-force")
 		'NOT_STARTED

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -38,10 +38,11 @@
   (debug:print 0 "Attempting to start the server ...")
   (let* ((rpc:listener   (server:find-free-port-and-open (rpc:default-server-port)))
 	 (th1            (make-thread
 			  (cute (rpc:make-server rpc:listener) "rpc:server")
 			  'rpc:server))
+	 (th2            (make-thread (lambda ()(db:updater db))))
 	 (hostname       (if (string=? "-" hostn)
 			     (get-host-name) 
 			     hostn))
 	 (ipaddrstr      (if (string=? "-" hostn)
 			     (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".")
@@ -96,10 +97,15 @@
     
     (rpc:publish-procedure!
      'rdb:test-set-log!
      (lambda (run-id test-name item-path logf)
        (db:test-set-log! db run-id test-name item-path logf)))
+    
+    (rpc:publish-procedure!
+     'rpc:get-test-data-by-id
+     (lambda (test-id)
+       (db:get-test-data-by-id db test-id)))
 
     (rpc:publish-procedure!
      'serve:get-toppath
      (lambda ()
        *toppath*))
@@ -196,11 +202,12 @@
     (set! *rpc:listener* rpc:listener)
     (on-exit (lambda ()
 	       (sqlite3:execute db "DELETE FROM metadat WHERE var='SERVER' and val=?;" host:port)
 	       (sqlite3:finalize! db)))
     (thread-start! th1)
-    (thread-join! th1))) ;; rpc:server)))
+    (thread-start! th2)
+    (thread-join!  th2))) ;; rpc:server)))
 
 (define (server:find-free-port-and-open port)
   (handle-exceptions
    exn
    (begin

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -3,13 +3,14 @@
 BINPATH=$(shell realpath ../bin)
 MEGATEST=$(BINPATH)/megatest
 PATH := $(BINPATH):$(PATH)
 
 runall :
-	cd ../;make install 
+	cd ../;make install
+	mkdir -p /tmp/mt_runs /tmp/mt_links
 	$(BINPATH)/dboard -rows 15 &
-	$(MEGATEST) -keepgoing -runall -target ubuntu/nfs/none :runname `date +w%V.%u.%H` -m "This is a comment specific to a run" -v
+	$(MEGATEST) -runall -target ubuntu/nfs/none :runname `date +w%V.%u.%H` -m "This is a comment specific to a run" -v
 
 test :
 	csi -b -I .. ../megatest.scm -- -runall -target ubuntu/afs/tmp :runname blah
 	cd ../;make test
 	make runall

Index: tests/megatest.config
==================================================================
--- tests/megatest.config
+++ tests/megatest.config
@@ -4,11 +4,11 @@
 datapath TEXT
 
 [setup]
 # exectutable /path/to/megatest
 max_concurrent_jobs 50
-linktree /tmp/runs
+linktree /tmp/mt_links
 
 [jobtools]
 # useshell yes
 # ## launcher launches jobs, the job is managed on the target host
 ## by megatest, comment out launcher to run local
@@ -39,6 +39,6 @@
 ## disks are:
 ## name host:/path/to/area
 ## -or-
 ## name /path/to/area
 [disks]
-1 /tmp
+1 /tmp/mt_runs