Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -5,11 +5,12 @@
 SRCFILES = common.scm items.scm launch.scm \
            ods.scm runconfig.scm server.scm configf.scm \
            db.scm keys.scm margs.scm megatest-version.scm \
            process.scm runs.scm tasks.scm tests.scm genexample.scm \
 	   fs-transport.scm http-transport.scm \
-           client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm
+           client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm \
+	   tree.scm
 
 GUISRCF  = dashboard-tests.scm dashboard-guimonitor.scm 
 
 OFILES   = $(SRCFILES:%.scm=%.o)
 GOFILES  = $(GUISRCF:%.scm=%.o)

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -136,10 +136,11 @@
 
 (define *tests-sort-reverse* #f)
 (define *hide-empty-runs* #f)
 
 (define *current-tab-number* 0)
+(define *updaters* (make-hash-table))
 
 (debug:setup)
 
 (define uidat #f)
 
@@ -633,11 +634,107 @@
 	;; #:title "Disks Areas"
 	(dcommon:section-matrix rawconfig "disks" "Disk area" "Path"))))
      (iup:frame
       #:title "Run statistics"
       (dcommon:run-stats)))))
-   
+
+;;======================================================================
+;; R U N
+;;======================================================================
+;;
+;; display and manage a single run at a time
+
+(define (tree-path->run-id path)
+  (if (not (null? path))
+      (hash-table-ref/default (dboard:data-get-path-run-ids *data*) path #f)
+      #f))
+
+(define dashboard:update-run-summary-tab #f)
+
+;; (define (tests window-id)
+(define (dashboard:one-run)
+  (let* ((tb      (iup:treebox
+		   #:selection-cb
+		   (lambda (obj id state)
+		     ;; (print "obj: " obj ", id: " id ", state: " state)
+		     (let* ((run-path (tree:node->path obj id))
+			    (run-id   (tree-path->run-id (cdr run-path))))
+		       (if run-id
+			   (dboard:data-set-curr-run-id *data*))
+		       (print "path: " (tree:node->path obj id) " run-id: " run-id)))))
+	 (run-matrix (iup:matrix
+		      #:expand "YES"))
+	 (updater  (lambda ()
+		     (let* ((run-id       (dboard:data-get-curr-run-id *data*))
+			    (tests-dat    (mt:get-tests-for-run run-id "%" '() '()
+								qryval: "id,testname,item_path,state,status")) ;; get 'em all
+			    (tests-mindat (dcommon:minimize-test-data tests-dat))
+			    (indices      (common:sparse-list-generate-index tests-mindat)) ;;  proc: set-cell))
+			    (row-indices  (car indices))
+			    (col-indices  (cadr indices))
+			    (max-row      (if (null? row-indices) 1 (apply max (map cadr row-indices))))
+			    (max-col      (if (null? col-indices) 1 (apply max (map cadr col-indices))))
+			    (max-visible  (max (- *num-tests* 15) 3)) ;; *num-tests* is proportional to the size of the window
+			    (numrows      1)
+			    (numcols      1)
+			    (changed      #f))
+		       (iup:attribute-set! run-matrix "CLEARVALUE" "CONTENTS")
+		       (iup:attribute-set! run-matrix "NUMCOL" max-col )
+		       (iup:attribute-set! run-matrix "NUMLIN" (if (< max-row max-visible) max-visible max-row)) ;; min of 20
+		       (iup:attribute-set! run-matrix "NUMCOL_VISIBLE" max-col)
+		       (iup:attribute-set! run-matrix "NUMLIN_VISIBLE" (if (> max-row max-visible) max-visible max-row))
+		       
+		       ;; Row labels
+		       (for-each (lambda (ind)
+				   (let* ((name (car ind))
+					  (num  (cadr ind))
+					  (key  (conc num ":0")))
+				     (if (not (equal? (iup:attribute run-matrix key) name))
+					 (begin
+					   (set! changed #t)
+					   (iup:attribute-set! run-matrix key name)))))
+				 row-indices)
+		       
+		       ;; Col labels
+		       (for-each (lambda (ind)
+				   (let* ((name (car ind))
+					  (num  (cadr ind))
+					  (key  (conc "0:" num)))
+				     (if (not (equal? (iup:attribute run-matrix key) name))
+					 (begin
+					   (set! changed #t)
+					   (iup:attribute-set! run-matrix key name)))))
+				 col-indices)
+		       
+		       ;; Cell contents
+		       (for-each (lambda (entry)
+				   (let* ((row-name  (car entry))
+					  (col-name  (cadr entry))
+					  (valuedat  (caddr entry))
+					  (test-id   (list-ref valuedat 0))
+					  (test-name (list-ref valuedat 1))
+					  (item-path (list-ref valuedat 2))
+					  (state     (list-ref valuedat 3))
+					  (status    (list-ref valuedat 4))
+					  (row-num   (cadr (assoc row-name row-indices)))
+					  (col-num   (cadr (assoc col-name col-indices)))
+					  (key       (conc row-num ":" col-num)))
+				     (if (not (equal? (iup:attribute run-matrix key) value))
+					 (begin
+					   (set! changed #t)
+					   (iup:attribute-set! run-matrix key value)))))
+				 tests-mindat)
+		       (if changed (iup:attribute-set! run-matrix "REDRAW" "ALL"))))))
+    (set! dashboard:update-run-summary-tab updater)
+    (iup:attribute-set! tb "VALUE" "0")
+    (iup:attribute-set! tb "NAME" "Runs")
+    ;;(iup:attribute-set! tb "ADDEXPANDED" "NO")
+    (dboard:data-set-runs-tree! *data* tb)
+    (iup:hbox 
+     tb
+     run-matrix)))
+
 ;;======================================================================
 ;; R U N S 
 ;;======================================================================
 
 (define (make-dashboard-buttons nruns ntests keynames)
@@ -831,16 +928,18 @@
 		    #:tabchangepos-cb (lambda (obj curr prev)
 					(set! *please-update-buttons* #t)
 					(set! *current-tab-number* curr))
 		    (dashboard:summary)
 		    runs-view
+		    (dashboard:one-run)
 		    (dashboard:run-controls)
 		    )))
 	;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c)))
 	(iup:attribute-set! tabs "TABTITLE0" "Summary")
 	(iup:attribute-set! tabs "TABTITLE1" "Runs")
-	(iup:attribute-set! tabs "TABTITLE2" "Run Control")
+	(iup:attribute-set! tabs "TABTITLE2" "Run Summary")
+	(iup:attribute-set! tabs "TABTITLE3" "Run Control")
 	tabs)))
      (vector keycol lftcol header runsvec)))
 
 (if (or (args:get-arg "-rows")
 	(get-environment-variable "DASHBOARDROWS" ))
@@ -889,11 +988,16 @@
 					  (if (not (equal? key "runname"))
 					      (let ((val (hash-table-ref/default *searchpatts* key #f)))
 						(if val (set! res (cons (list key val) res))))))
 					*dbkeys*)
 			      res))
-	     (update-buttons uidat *num-runs* *num-tests*)))
+	     (update-buttons uidat *num-runs* *num-tests*))
+	    ((2)
+	     (dashboard:update-run-summary-tab))
+	    (else
+	     (let ((updater (hash-table-ref/default *updaters* *current-tab-number* #f)))
+	       (if updater (updater)))))
 	  (set! *please-update-buttons* #f)
 	  (set! *last-db-update-time* modtime)
 	  (set! *last-update* run-update-time)))))
 
 ;;======================================================================

Index: dcommon.scm
==================================================================
--- dcommon.scm
+++ dcommon.scm
@@ -25,10 +25,232 @@
 (include "db_records.scm")
 (include "key_records.scm")
 
 ;; yes, this is non-ideal 
 (define dashboard:update-summary-tab #f)
+
+;;======================================================================
+;; C O M M O N   D A T A   S T R U C T U R E
+;;======================================================================
+;; 
+;; A single data structure for all the data used in a dashboard.
+;; Share this structure between newdashboard and dashboard with the 
+;; intent of converging on a single app.
+;;
+(define *data* (make-vector 15 #f))
+(define (dboard:data-get-runs          vec)    (vector-ref  vec 0))
+(define (dboard:data-get-tests         vec)    (vector-ref  vec 1))
+(define (dboard:data-get-runs-matrix   vec)    (vector-ref  vec 2))
+(define (dboard:data-get-tests-tree    vec)    (vector-ref  vec 3))
+(define (dboard:data-get-run-keys      vec)    (vector-ref  vec 4))
+(define (dboard:data-get-curr-test-ids vec)    (vector-ref  vec 5))
+;; (define (dboard:data-get-test-details  vec)    (vector-ref  vec 6))
+(define (dboard:data-get-path-test-ids vec)    (vector-ref  vec 7))
+(define (dboard:data-get-updaters      vec)    (vector-ref  vec 8))
+(define (dboard:data-get-path-run-ids  vec)    (vector-ref  vec 9))
+(define (dboard:data-get-curr-run-id   vec)    (vector-ref  vec 10))
+(define (dboard:data-get-runs-tree     vec)    (vector-ref  vec 11))
+
+(define (dboard:data-set-runs!          vec val)(vector-set! vec 0 val))
+(define (dboard:data-set-tests!         vec val)(vector-set! vec 1 val))
+(define (dboard:data-set-runs-matrix!   vec val)(vector-set! vec 2 val))
+(define (dboard:data-set-tests-tree!    vec val)(vector-set! vec 3 val))
+(define (dboard:data-set-run-keys!      vec val)(vector-set! vec 4 val))
+(define (dboard:data-set-curr-test-ids! vec val)(vector-set! vec 5 val))
+;; (define (dboard:data-set-test-details!  vec val)(vector-set! vec 6 val))
+(define (dboard:data-set-path-test-ids! vec val)(vector-set! vec 7 val))
+(define (dboard:data-set-updaters!      vec val)(vector-set! vec 8 val))
+(define (dboard:data-set-path-run-ids!  vec val)(vector-set! vec 9 val))
+(define (dboard:data-set-curr-run-id!   vec val)(vector-set! vec 10 val))
+(define (dboard:data-set-runs-tree!     vec val)(vector-set! vec 12 val))
+
+(dboard:data-set-run-keys! *data* (make-hash-table))
+
+;; List of test ids being viewed in various panels
+(dboard:data-set-curr-test-ids! *data* (make-hash-table))
+
+;; Look up test-ids by (key1 key2 ... testname [itempath])
+(dboard:data-set-path-test-ids! *data* (make-hash-table))
+
+;; Look up run-ids by ??
+(dboard:data-set-path-run-ids! *data* (make-hash-table))
+
+;;======================================================================
+;; P R O C E S S   R U N S
+;;======================================================================
+
+;; MOVE THIS INTO *data*
+(define *cachedata* (make-hash-table))
+(hash-table-set! *cachedata* "runid-to-col"    (make-hash-table))
+(hash-table-set! *cachedata* "testname-to-row" (make-hash-table))
+
+;; TO-DO
+;;  1. Make "data" hash-table hierarchial store of all displayed data
+;;  2. Update synchash to understand "get-runs", "get-tests" etc.
+;;  3. Add extraction of filters to synchash calls
+;;
+;; Mode is 'full or 'incremental for full refresh or incremental refresh
+(define (run-update keys data runname keypatts testpatt states statuses mode window-id)
+  (let* (;; count and offset => #f so not used
+	 ;; the synchash calls modify the "data" hash
+	 (get-runs-sig    (conc (client:get-signature) " get-runs"))
+	 (get-tests-sig   (conc (client:get-signature) " get-tests"))
+	 (get-details-sig (conc (client:get-signature) " get-test-details"))
+
+	 ;; test-ids to get and display are indexed on window-id in curr-test-ids hash
+	 (test-ids        (hash-table-values (dboard:data-get-curr-test-ids *data*)))
+
+ 	 (run-changes     (synchash:client-get 'db:get-runs get-runs-sig (length keypatts) data runname #f #f keypatts))
+	 (tests-detail-changes (if (not (null? test-ids))
+				   (synchash:client-get 'db:get-test-info-by-ids get-details-sig 0  data test-ids)
+				   '()))
+
+	 ;; Now can calculate the run-ids
+	 (run-hash    (hash-table-ref/default data get-runs-sig #f))
+	 (run-ids     (if run-hash (filter number? (hash-table-keys run-hash)) '()))
+
+	 (test-changes (synchash:client-get 'db:get-tests-for-runs-mindata get-tests-sig 0 data run-ids testpatt states statuses))
+	 (runs-hash    (hash-table-ref/default data get-runs-sig #f))
+	 (header       (hash-table-ref/default runs-hash "header" #f))
+	 (run-ids      (sort (filter number? (hash-table-keys runs-hash))
+			     (lambda (a b)
+			       (let* ((record-a (hash-table-ref runs-hash a))
+				      (record-b (hash-table-ref runs-hash b))
+				      (time-a   (db:get-value-by-header record-a header "event_time"))
+				      (time-b   (db:get-value-by-header record-b header "event_time")))
+				 (> time-a time-b)))
+			     ))
+	 (runid-to-col    (hash-table-ref *cachedata* "runid-to-col"))
+	 (testname-to-row (hash-table-ref *cachedata* "testname-to-row")) 
+	 (colnum       1)
+	 (rownum       0)) ;; rownum = 0 is the header
+;; (debug:print 0 "test-ids " test-ids ", tests-detail-changes " tests-detail-changes)
+    
+	 ;; tests related stuff
+	 ;; (all-testnames (delete-duplicates (map db:test-get-testname test-changes))))
+
+    ;; Given a run-id and testname/item_path calculate a cell R:C
+
+    ;; NOTE: Also build the test tree browser and look up table
+    ;;
+    ;; Each run is unique on its keys and runname or run-id, store in hash on colnum
+    (for-each (lambda (run-id)
+		(let* ((run-record (hash-table-ref/default runs-hash run-id #f))
+		       (key-vals   (map (lambda (key)(db:get-value-by-header run-record header key))
+					keys))
+		       (run-name   (db:get-value-by-header run-record header "runname"))
+		       (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
+		       (run-path   (append key-vals (list run-name))))
+		  (hash-table-set! (dboard:data-get-run-keys *data*) run-id run-path)
+		  (iup:attribute-set! (dboard:data-get-runs-matrix *data*)
+				      (conc rownum ":" colnum) col-name)
+		  (hash-table-set! runid-to-col run-id (list colnum run-record))
+		  ;; Here we update the tests treebox and tree keys
+		  (tree:add-node (dboard:data-get-tests-tree *data*) "Runs" (append key-vals (list run-name))
+				 userdata: (conc "run-id: " run-id))
+		  (set! colnum (+ colnum 1))))
+	      run-ids)
+
+    ;; Scan all tests to be displayed and organise all the test names, respecting what is in the hash table
+    ;; Do this analysis in the order of the run-ids, the most recent run wins
+    (for-each (lambda (run-id)
+		(let* ((run-path       (hash-table-ref (dboard:data-get-run-keys *data*) run-id))
+		       (new-test-dat   (car test-changes))
+		       (removed-tests  (cadr test-changes))
+		       (tests          (sort (map cadr (filter (lambda (testrec)
+								 (eq? run-id (db:mintest-get-run_id (cadr testrec))))
+							       new-test-dat))
+					     (lambda (a b)
+					       (let ((time-a (db:mintest-get-event_time a))
+						     (time-b (db:mintest-get-event_time b)))
+						 (> time-a time-b)))))
+		       ;; test-changes is a list of (( id record ) ... )
+		       ;; Get list of test names sorted by time, remove tests
+		       (test-names (delete-duplicates (map (lambda (t)
+							     (let ((i (db:mintest-get-item_path t))
+								   (n (db:mintest-get-testname  t)))
+							       (if (string=? i "")
+								   (conc "   " i)
+								   n)))
+							   tests)))
+		       (colnum     (car (hash-table-ref runid-to-col run-id))))
+		  ;; for each test name get the slot if it exists and fill in the cell
+		  ;; or take the next slot and fill in the cell, deal with items in the
+		  ;; run view panel? The run view panel can have a tree selector for
+		  ;; browsing the tests/items
+
+		  ;; SWITCH THIS TO USING CHANGED TESTS ONLY
+		  (for-each (lambda (test)
+			      (let* ((test-id   (db:mintest-get-id test))
+				     (state     (db:mintest-get-state test))
+				     (status    (db:mintest-get-status test))
+				     (testname  (db:mintest-get-testname test))
+				     (itempath  (db:mintest-get-item_path test))
+				     (fullname  (conc testname "/" itempath))
+				     (dispname  (if (string=? itempath "") testname (conc "   " itempath)))
+				     (rownum    (hash-table-ref/default testname-to-row fullname #f))
+				     (test-path (append run-path (if (equal? itempath "") 
+								     (list testname)
+								     (list testname itempath)))))
+				(tree:add-node (dboard:data-get-tests-tree *data*) "Runs" 
+					       test-path
+					       userdata: (conc "test-id: " test-id))
+				(hash-table-set! (dboard:data-get-path-test-ids *data*) test-path test-id)
+				(if (not rownum)
+				    (let ((rownums (hash-table-values testname-to-row)))
+				      (set! rownum (if (null? rownums)
+						       1
+						       (+ 1 (apply max rownums))))
+				      (hash-table-set! testname-to-row fullname rownum)
+				      ;; create the label
+				      (iup:attribute-set! (dboard:data-get-runs-matrix *data*)
+							  (conc rownum ":" 0) dispname)
+				      ))
+				;; set the cell text and color
+				;; (debug:print 2 "rownum:colnum=" rownum ":" colnum ", state=" status)
+				(iup:attribute-set! (dboard:data-get-runs-matrix *data*)
+						    (conc rownum ":" colnum)
+						    (if (string=? state "COMPLETED")
+							status
+							state))
+				(iup:attribute-set! (dboard:data-get-runs-matrix *data*)
+						    (conc "BGCOLOR" rownum ":" colnum)
+						    (gutils:get-color-for-state-status state status))
+				))
+			    tests)))
+	      run-ids)
+
+    (let ((updater (hash-table-ref/default  (dboard:data-get-updaters *data*) window-id #f)))
+      (if updater (updater (hash-table-ref/default data get-details-sig #f))))
+
+    (iup:attribute-set! (dboard:data-get-runs-matrix *data*) "REDRAW" "ALL")
+    ;; (debug:print 2 "run-changes: " run-changes)
+    ;; (debug:print 2 "test-changes: " test-changes)
+    (list run-changes test-changes)))
+
+;;======================================================================
+;; TESTS DATA
+;;======================================================================
+
+;; Produce a list of lists ready for common:sparse-list-generate-index
+;;
+(define (dcommon:minimize-test-data tests-dat)
+  (if (null? tests-dat) 
+      '()
+      (let loop ((hed (car tests-dat))
+		 (tal (cdr tests-dat))
+		 (res '()))
+	(let* ((test-id    (vector-ref hed 0)) ;; look at the tests-dat spec for locations
+	       (test-name  (vector-ref hed 1))
+	       (item-path  (vector-ref hed 2))
+	       (state      (vector-ref hed 3))
+	       (status     (vector-ref hed 4))
+	       (newitem    (list test-name item-path (list test-id state status))))
+	  (if (null? tal)
+	      (reverse (cons newitem res))
+	      (loop (car tal)(cdr tal)(cons newitem res)))))))
+	  
 
 ;;======================================================================
 ;; D A T A   T A B L E S
 ;;======================================================================
 
@@ -121,26 +343,10 @@
 				(max-row      (apply max (map cadr (car indices))))
 				(max-col      (apply max (map cadr (cadr indices))))
 				(max-visible  (max (- *num-tests* 15) 3))
 				(numrows      1)
 				(numcols      1)
-				(set-cell     (lambda (rnum cnum rname cname v) ;; rownum colnum value
-						(print "proc called: " rnum " " cnum " " rname " " cname " " v)
-						(if (> rnum numrows)
-						    (begin 
-						      ;; add rows numrows to r
-						      (debug:print 0 "Extending matrix from " numrows " to " rnum)
-						      (iup:attribute-set! stats-matrix  "ADDLIN" (conc numrows "-" (- rnum numrows)))
-						      (set! numrows rnum)))
-						(if (> cnum numcols)
-						    (begin 
-						      ;; add rows numrows to r
-						      (debug:print 0 "Extending matrix from " numcols " to " cnum)
-						      (iup:attribute-set! stats-matrix  "ADDLIN" (conc numcols "-" (- rnum numcols)))
-						      (set! numcols cnum)))
-						(debug:print 0 "Setting row " rnum ", col " cnum " to " v)
-						(iup:attribute-set! stats-matrix (conc rnum ":" cnum) v)))
 				(row-indices  (car indices))
 				(col-indices  (cadr indices)))
 			   (iup:attribute-set! stats-matrix "CLEARVALUE" "CONTENTS")
 			   (iup:attribute-set! stats-matrix "NUMCOL" max-col )
 			   (iup:attribute-set! stats-matrix "NUMLIN" (if (< max-row max-visible) max-visible max-row)) ;; min of 20

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -54,11 +54,11 @@
 	     (limit    500))
     (let* ((full-list (append res testsdat))
 	   (have-more (eq? (length testsdat) limit)))
       (if have-more 
 	  (let ((new-offset (+ offset limit)))
-	    (debug:print-info 0 "More than " limit " tests, have " (length full-list) " tests so far.")
+	    (debug:print-info 4 "More than " limit " tests, have " (length full-list) " tests so far.")
 	    (loop (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status new-offset limit not-in sort-by)
 		  full-list
 		  new-offset
 		  limit))
 	  full-list))))

Index: newdashboard.scm
==================================================================
--- newdashboard.scm
+++ newdashboard.scm
@@ -23,10 +23,11 @@
 (declare (uses gutils))
 (declare (uses db))
 (declare (uses server))
 (declare (uses synchash))
 (declare (uses dcommon))
+(declare (uses tree))
 
 (include "common_records.scm")
 (include "db_records.scm")
 (include "key_records.scm")
 
@@ -83,42 +84,10 @@
 (debug:setup)
 
 (define *tim* (iup:timer))
 (define *ord* #f)
 
-(define *data* (make-vector 10 #f))
-(define-inline (dboard:data-get-runs          vec)    (vector-ref  vec 0))
-(define-inline (dboard:data-get-tests         vec)    (vector-ref  vec 1))
-(define-inline (dboard:data-get-runs-matrix   vec)    (vector-ref  vec 2))
-(define-inline (dboard:data-get-tests-tree    vec)    (vector-ref  vec 3))
-(define-inline (dboard:data-get-run-keys      vec)    (vector-ref  vec 4))
-(define-inline (dboard:data-get-curr-test-ids vec)    (vector-ref  vec 5))
-;; (define-inline (dboard:data-get-test-details  vec)    (vector-ref  vec 6))
-(define-inline (dboard:data-get-path-test-ids vec)    (vector-ref  vec 7))
-(define-inline (dboard:data-get-updaters      vec)    (vector-ref  vec 8))
-
-(define-inline (dboard:data-set-runs!          vec val)(vector-set! vec 0 val))
-(define-inline (dboard:data-set-tests!         vec val)(vector-set! vec 1 val))
-(define-inline (dboard:data-set-runs-matrix!   vec val)(vector-set! vec 2 val))
-(define-inline (dboard:data-set-tests-tree!    vec val)(vector-set! vec 3 val))
-(define-inline (dboard:data-set-run-keys!      vec val)(vector-set! vec 4 val))
-(define-inline (dboard:data-set-curr-test-ids! vec val)(vector-set! vec 5 val))
-;; (define-inline (dboard:data-set-test-details!  vec val)(vector-set! vec 6 val))
-(define-inline (dboard:data-set-path-test-ids! vec val)(vector-set! vec 7 val))
-(define-inline (dboard:data-set-updaters!      vec val)(vector-set! vec 8 val))
-
-(dboard:data-set-run-keys! *data* (make-hash-table))
-
-;; List of test ids being viewed in various panels
-(dboard:data-set-curr-test-ids! *data* (make-hash-table))
-
-;; Look up test-ids by (key1 key2 ... testname [itempath])
-(dboard:data-set-path-test-ids! *data* (make-hash-table))
-
-;; Each test panel has an updater, only call when the tab is exposed
-(dboard:data-set-updaters!      *data* (make-hash-table))
-
 (iup:attribute-set! *tim* "TIME" 300)
 (iup:attribute-set! *tim* "RUN" "YES")
 
 (define (message-window msg)
   (iup:show
@@ -274,108 +243,10 @@
 ;;
 (define (rconfig window-id)
   (iup:vbox
    (iup:frame #:title "Default")))
 
-;;======================================================================
-;; T R E E   S T U F F 
-;;======================================================================
-
-;; path is a list of nodes, each the child of the previous
-;; this routine returns the id so another node can be added
-;; either as a leaf or as a branch
-;;
-;; BUG: This needs a stop sensor for when a branch is exhausted
-;;
-(define (tree-find-node obj path)
-  ;; start at the base of the tree
-  (if (null? path)
-      #f ;; or 0 ????
-      (let loop ((hed      (car path))
-		 (tal      (cdr path))
-		 (depth    0)
-		 (nodenum  0))
-	;; nodes in iup tree are 100% sequential so iterate over nodenum
-	(if (iup:attribute obj (conc "DEPTH" nodenum)) ;; end when no more nodes
-	    (let ((node-depth (string->number (iup:attribute obj (conc "DEPTH" nodenum))))
-		  (node-title (iup:attribute obj (conc "TITLE" nodenum))))
-	      (if (and (equal? depth node-depth)
-		       (equal? hed   node-title)) ;; yep, this is the one!
-		  (if (null? tal) ;; end of the line
-		      nodenum
-		      (loop (car tal)(cdr tal)(+ depth 1)(+ 1 nodenum)))
-		  ;; this is the case where we found part of the hierarchy but not 
-		  ;; all of it, i.e. the node-depth went from deep to less deep
-		  (if (> depth node-depth) ;; (+ 1 node-depth))
-		      #f
-		      (loop hed tal depth (+ nodenum 1)))))
-	    #f))))
-
-;; top is the top node name zeroeth node VALUE=0
-(define (tree-add-node obj top nodelst #!key (userdata #f))
-  (if (not (iup:attribute obj "TITLE0"))
-      (iup:attribute-set! obj "ADDBRANCH0" top))
-  (cond
-   ((not (string=? top (iup:attribute obj "TITLE0")))
-    (print "ERROR: top name " top " doesn't match " (iup:attribute obj "TITLE0")))
-   ((null? nodelst))
-   (else
-    (let loop ((hed      (car nodelst))
-	       (tal      (cdr nodelst))
-	       (depth    1)
-	       (pathl    (list top)))
-      ;; Because the tree dialog changes node numbers when
-      ;; nodes are added or removed we must look up nodes
-      ;; each and every time. 0 is the top node so default
-      ;; to that.
-      (let* ((newpath    (append pathl (list hed)))
-	     (parentnode (tree-find-node obj pathl))
-	     (nodenum    (tree-find-node obj newpath)))
-	;; Add the branch under lastnode if not found
-	(if (not nodenum)
-	    (begin
-	      (iup:attribute-set! obj (conc "ADDBRANCH" parentnode) hed)
-	      (if userdata
-		  (iup:attribute-set! obj (conc "USERDATA"   parentnode) userdata))
-	      (if (null? tal)
-		  #t
-		  ;; reset to top
-		  (loop (car nodelst)(cdr nodelst) 1 (list top)))) 
-	    (if (null? tal) ;; if null here then this path has already been added
-		#t
-		(loop (car tal)(cdr tal)(+ depth 1) newpath))))))))
-
-(define (tree-node->path obj nodenum)
-  ;; (print "\ncurrnode  nodenum  depth  node-depth  node-title   path")
-  (let loop ((currnode 0)
-	     (depth    0)
-	     (path     '()))
-    (let ((node-depth (iup:attribute obj (conc "DEPTH" currnode)))
-	  (node-title (iup:attribute obj (conc "TITLE" currnode))))
-      ;; (display (conc "\n   "currnode "        " nodenum "       " depth "         " node-depth "          " node-title "         " path))
-      (if (> currnode nodenum)
-	  path
-	  (if (not node-depth) ;; #f if we are out of nodes
-	      '()
-	      (let ((ndepth (string->number node-depth)))
-		(if (eq? ndepth depth)
-		    ;; This next is the match condition depth == node-depth
-		    (if (eq? currnode nodenum)
-			(begin
-			  ;; (display " <X>")
-			  (append path (list node-title)))
-			(loop (+ currnode 1)
-			      (+ depth 1)
-			      (append path (list node-title))))
-		    ;; didn't match, reset to base path and keep looking
-		    ;; due to more iup odditys we don't reset to base
-		    (begin 
-		      ;; (display " <L>")
-		      (loop (+ 1 currnode)
-			    2
-			    (append (take path ndepth)(list node-title)))))))))))
-
 ;;======================================================================
 ;; T E S T S
 ;;======================================================================
 
 (define (tree-path->test-id path)
@@ -547,16 +418,16 @@
   (iup:hbox 
    (let* ((tb      (iup:treebox
 		    #:selection-cb
 		    (lambda (obj id state)
 		      ;; (print "obj: " obj ", id: " id ", state: " state)
-		      (let* ((run-path (tree-node->path obj id))
+		      (let* ((run-path (tree:node->path obj id))
 			     (test-id  (tree-path->test-id (cdr run-path))))
 			(if test-id
 			    (hash-table-set! (dboard:data-get-curr-test-ids *data*)
 					     window-id test-id))
-			(print "path: " (tree-node->path obj id) " test-id: " test-id))))))
+			(print "path: " (tree:node->path obj id) " test-id: " test-id))))))
      (iup:attribute-set! tb "VALUE" "0")
      (iup:attribute-set! tb "NAME" "Runs")
      ;;(iup:attribute-set! tb "ADDEXPANDED" "NO")
      (dboard:data-set-tests-tree! *data* tb)
      tb)
@@ -672,163 +543,10 @@
 ;; Browse and control a single run
 ;;
 (define (runcontrol window-id)
   (iup:hbox))
 
-;;======================================================================
-;; P R O C E S S   R U N S
-;;======================================================================
-
-;; MOVE THIS INTO *data*
-(define *cachedata* (make-hash-table))
-(hash-table-set! *cachedata* "runid-to-col"    (make-hash-table))
-(hash-table-set! *cachedata* "testname-to-row" (make-hash-table))
-
-;; TO-DO
-;;  1. Make "data" hash-table hierarchial store of all displayed data
-;;  2. Update synchash to understand "get-runs", "get-tests" etc.
-;;  3. Add extraction of filters to synchash calls
-;;
-;; Mode is 'full or 'incremental for full refresh or incremental refresh
-(define (run-update keys data runname keypatts testpatt states statuses mode window-id)
-  (let* (;; count and offset => #f so not used
-	 ;; the synchash calls modify the "data" hash
-	 (get-runs-sig    (conc (client:get-signature) " get-runs"))
-	 (get-tests-sig   (conc (client:get-signature) " get-tests"))
-	 (get-details-sig (conc (client:get-signature) " get-test-details"))
-
-	 ;; test-ids to get and display are indexed on window-id in curr-test-ids hash
-	 (test-ids        (hash-table-values (dboard:data-get-curr-test-ids *data*)))
-
- 	 (run-changes     (synchash:client-get 'db:get-runs get-runs-sig (length keypatts) data runname #f #f keypatts))
-	 (tests-detail-changes (if (not (null? test-ids))
-				   (synchash:client-get 'db:get-test-info-by-ids get-details-sig 0  data test-ids)
-				   '()))
-
-	 ;; Now can calculate the run-ids
-	 (run-hash    (hash-table-ref/default data get-runs-sig #f))
-	 (run-ids     (if run-hash (filter number? (hash-table-keys run-hash)) '()))
-
-	 (test-changes (synchash:client-get 'db:get-tests-for-runs-mindata get-tests-sig 0 data run-ids testpatt states statuses))
-	 (runs-hash    (hash-table-ref/default data get-runs-sig #f))
-	 (header       (hash-table-ref/default runs-hash "header" #f))
-	 (run-ids      (sort (filter number? (hash-table-keys runs-hash))
-			     (lambda (a b)
-			       (let* ((record-a (hash-table-ref runs-hash a))
-				      (record-b (hash-table-ref runs-hash b))
-				      (time-a   (db:get-value-by-header record-a header "event_time"))
-				      (time-b   (db:get-value-by-header record-b header "event_time")))
-				 (> time-a time-b)))
-			     ))
-	 (runid-to-col    (hash-table-ref *cachedata* "runid-to-col"))
-	 (testname-to-row (hash-table-ref *cachedata* "testname-to-row")) 
-	 (colnum       1)
-	 (rownum       0)) ;; rownum = 0 is the header
-;; (debug:print 0 "test-ids " test-ids ", tests-detail-changes " tests-detail-changes)
-    
-	 ;; tests related stuff
-	 ;; (all-testnames (delete-duplicates (map db:test-get-testname test-changes))))
-
-    ;; Given a run-id and testname/item_path calculate a cell R:C
-
-    ;; NOTE: Also build the test tree browser and look up table
-    ;;
-    ;; Each run is unique on its keys and runname or run-id, store in hash on colnum
-    (for-each (lambda (run-id)
-		(let* ((run-record (hash-table-ref/default runs-hash run-id #f))
-		       (key-vals   (map (lambda (key)(db:get-value-by-header run-record header key))
-					keys))
-		       (run-name   (db:get-value-by-header run-record header "runname"))
-		       (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
-		       (run-path   (append key-vals (list run-name))))
-		  (hash-table-set! (dboard:data-get-run-keys *data*) run-id run-path)
-		  (iup:attribute-set! (dboard:data-get-runs-matrix *data*)
-				      (conc rownum ":" colnum) col-name)
-		  (hash-table-set! runid-to-col run-id (list colnum run-record))
-		  ;; Here we update the tests treebox and tree keys
-		  (tree-add-node (dboard:data-get-tests-tree *data*) "Runs" (append key-vals (list run-name))
-				 userdata: (conc "run-id: " run-id))
-		  (set! colnum (+ colnum 1))))
-	      run-ids)
-
-    ;; Scan all tests to be displayed and organise all the test names, respecting what is in the hash table
-    ;; Do this analysis in the order of the run-ids, the most recent run wins
-    (for-each (lambda (run-id)
-		(let* ((run-path       (hash-table-ref (dboard:data-get-run-keys *data*) run-id))
-		       (new-test-dat   (car test-changes))
-		       (removed-tests  (cadr test-changes))
-		       (tests          (sort (map cadr (filter (lambda (testrec)
-								 (eq? run-id (db:mintest-get-run_id (cadr testrec))))
-							       new-test-dat))
-					     (lambda (a b)
-					       (let ((time-a (db:mintest-get-event_time a))
-						     (time-b (db:mintest-get-event_time b)))
-						 (> time-a time-b)))))
-		       ;; test-changes is a list of (( id record ) ... )
-		       ;; Get list of test names sorted by time, remove tests
-		       (test-names (delete-duplicates (map (lambda (t)
-							     (let ((i (db:mintest-get-item_path t))
-								   (n (db:mintest-get-testname  t)))
-							       (if (string=? i "")
-								   (conc "   " i)
-								   n)))
-							   tests)))
-		       (colnum     (car (hash-table-ref runid-to-col run-id))))
-		  ;; for each test name get the slot if it exists and fill in the cell
-		  ;; or take the next slot and fill in the cell, deal with items in the
-		  ;; run view panel? The run view panel can have a tree selector for
-		  ;; browsing the tests/items
-
-		  ;; SWITCH THIS TO USING CHANGED TESTS ONLY
-		  (for-each (lambda (test)
-			      (let* ((test-id   (db:mintest-get-id test))
-				     (state     (db:mintest-get-state test))
-				     (status    (db:mintest-get-status test))
-				     (testname  (db:mintest-get-testname test))
-				     (itempath  (db:mintest-get-item_path test))
-				     (fullname  (conc testname "/" itempath))
-				     (dispname  (if (string=? itempath "") testname (conc "   " itempath)))
-				     (rownum    (hash-table-ref/default testname-to-row fullname #f))
-				     (test-path (append run-path (if (equal? itempath "") 
-								     (list testname)
-								     (list testname itempath)))))
-				(tree-add-node (dboard:data-get-tests-tree *data*) "Runs" 
-					       test-path
-					       userdata: (conc "test-id: " test-id))
-				(hash-table-set! (dboard:data-get-path-test-ids *data*) test-path test-id)
-				(if (not rownum)
-				    (let ((rownums (hash-table-values testname-to-row)))
-				      (set! rownum (if (null? rownums)
-						       1
-						       (+ 1 (apply max rownums))))
-				      (hash-table-set! testname-to-row fullname rownum)
-				      ;; create the label
-				      (iup:attribute-set! (dboard:data-get-runs-matrix *data*)
-							  (conc rownum ":" 0) dispname)
-				      ))
-				;; set the cell text and color
-				;; (debug:print 2 "rownum:colnum=" rownum ":" colnum ", state=" status)
-				(iup:attribute-set! (dboard:data-get-runs-matrix *data*)
-						    (conc rownum ":" colnum)
-						    (if (string=? state "COMPLETED")
-							status
-							state))
-				(iup:attribute-set! (dboard:data-get-runs-matrix *data*)
-						    (conc "BGCOLOR" rownum ":" colnum)
-						    (gutils:get-color-for-state-status state status))
-				))
-			    tests)))
-	      run-ids)
-
-    (let ((updater (hash-table-ref/default  (dboard:data-get-updaters *data*) window-id #f)))
-      (if updater (updater (hash-table-ref/default data get-details-sig #f))))
-
-    (iup:attribute-set! (dboard:data-get-runs-matrix *data*) "REDRAW" "ALL")
-    ;; (debug:print 2 "run-changes: " run-changes)
-    ;; (debug:print 2 "test-changes: " test-changes)
-    (list run-changes test-changes)))
-
 ;;======================================================================
 ;; D A S H B O A R D
 ;;======================================================================
 
 ;; Main Panel

ADDED   tree.scm
Index: tree.scm
==================================================================
--- /dev/null
+++ tree.scm
@@ -0,0 +1,131 @@
+;;======================================================================
+;; Copyright 2006-2013, Matthew Welland.
+;; 
+;;  This program is made available under the GNU GPL version 2.0 or
+;;  greater. See the accompanying file COPYING for details.
+;; 
+;;  This program is distributed WITHOUT ANY WARRANTY; without even the
+;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+;;  PURPOSE.
+;;======================================================================
+
+(use format)
+(require-library iup)
+(import (prefix iup iup:))
+(use canvas-draw)
+
+(use sqlite3 srfi-1 posix regex regex-case srfi-69)
+(import (prefix sqlite3 sqlite3:))
+
+(declare (unit tree))
+(declare (uses margs))
+(declare (uses launch))
+(declare (uses megatest-version))
+(declare (uses gutils))
+(declare (uses db))
+(declare (uses server))
+(declare (uses synchash))
+(declare (uses dcommon))
+
+(include "common_records.scm")
+(include "db_records.scm")
+(include "key_records.scm")
+
+;;======================================================================
+;; T R E E   S T U F F 
+;;======================================================================
+
+;; path is a list of nodes, each the child of the previous
+;; this routine returns the id so another node can be added
+;; either as a leaf or as a branch
+;;
+;; BUG: This needs a stop sensor for when a branch is exhausted
+;;
+(define (tree:find-node obj path)
+  ;; start at the base of the tree
+  (if (null? path)
+      #f ;; or 0 ????
+      (let loop ((hed      (car path))
+		 (tal      (cdr path))
+		 (depth    0)
+		 (nodenum  0))
+	;; nodes in iup tree are 100% sequential so iterate over nodenum
+	(if (iup:attribute obj (conc "DEPTH" nodenum)) ;; end when no more nodes
+	    (let ((node-depth (string->number (iup:attribute obj (conc "DEPTH" nodenum))))
+		  (node-title (iup:attribute obj (conc "TITLE" nodenum))))
+	      (if (and (equal? depth node-depth)
+		       (equal? hed   node-title)) ;; yep, this is the one!
+		  (if (null? tal) ;; end of the line
+		      nodenum
+		      (loop (car tal)(cdr tal)(+ depth 1)(+ 1 nodenum)))
+		  ;; this is the case where we found part of the hierarchy but not 
+		  ;; all of it, i.e. the node-depth went from deep to less deep
+		  (if (> depth node-depth) ;; (+ 1 node-depth))
+		      #f
+		      (loop hed tal depth (+ nodenum 1)))))
+	    #f))))
+
+;; top is the top node name zeroeth node VALUE=0
+(define (tree:add-node obj top nodelst #!key (userdata #f))
+  (if (not (iup:attribute obj "TITLE0"))
+      (iup:attribute-set! obj "ADDBRANCH0" top))
+  (cond
+   ((not (string=? top (iup:attribute obj "TITLE0")))
+    (print "ERROR: top name " top " doesn't match " (iup:attribute obj "TITLE0")))
+   ((null? nodelst))
+   (else
+    (let loop ((hed      (car nodelst))
+	       (tal      (cdr nodelst))
+	       (depth    1)
+	       (pathl    (list top)))
+      ;; Because the tree dialog changes node numbers when
+      ;; nodes are added or removed we must look up nodes
+      ;; each and every time. 0 is the top node so default
+      ;; to that.
+      (let* ((newpath    (append pathl (list hed)))
+	     (parentnode (tree:find-node obj pathl))
+	     (nodenum    (tree:find-node obj newpath)))
+	;; Add the branch under lastnode if not found
+	(if (not nodenum)
+	    (begin
+	      (iup:attribute-set! obj (conc "ADDBRANCH" parentnode) hed)
+	      (if userdata
+		  (iup:attribute-set! obj (conc "USERDATA"   parentnode) userdata))
+	      (if (null? tal)
+		  #t
+		  ;; reset to top
+		  (loop (car nodelst)(cdr nodelst) 1 (list top)))) 
+	    (if (null? tal) ;; if null here then this path has already been added
+		#t
+		(loop (car tal)(cdr tal)(+ depth 1) newpath))))))))
+
+(define (tree:node->path obj nodenum)
+  ;; (print "\ncurrnode  nodenum  depth  node-depth  node-title   path")
+  (let loop ((currnode 0)
+	     (depth    0)
+	     (path     '()))
+    (let ((node-depth (iup:attribute obj (conc "DEPTH" currnode)))
+	  (node-title (iup:attribute obj (conc "TITLE" currnode))))
+      ;; (display (conc "\n   "currnode "        " nodenum "       " depth "         " node-depth "          " node-title "         " path))
+      (if (> currnode nodenum)
+	  path
+	  (if (not node-depth) ;; #f if we are out of nodes
+	      '()
+	      (let ((ndepth (string->number node-depth)))
+		(if (eq? ndepth depth)
+		    ;; This next is the match condition depth == node-depth
+		    (if (eq? currnode nodenum)
+			(begin
+			  ;; (display " <X>")
+			  (append path (list node-title)))
+			(loop (+ currnode 1)
+			      (+ depth 1)
+			      (append path (list node-title))))
+		    ;; didn't match, reset to base path and keep looking
+		    ;; due to more iup odditys we don't reset to base
+		    (begin 
+		      ;; (display " <L>")
+		      (loop (+ 1 currnode)
+			    2
+			    (append (take path ndepth)(list node-title)))))))))))
+