Index: .mtutil.scm
==================================================================
--- .mtutil.scm
+++ .mtutil.scm
@@ -23,45 +23,46 @@
 (define (str-first-char->number str)
   (char->integer (string-ref str 0)))
  
 ;; example of how to set up and write target mappers
 ;;
-(hash-table-set! *target-mappers*
-		 'prefix-contour
-		 (lambda (target run-name area area-path reason contour mode-patt)
-		   (conc contour "/" target)))
-(hash-table-set! *target-mappers*
-		 'prefix-area-contour
-		 (lambda (target run-name area area-path reason contour mode-patt)
-		   (conc area "/" contour "/" target)))
-  
-(hash-table-set! *runname-mappers*
-		 'corporate-ww
-		 (lambda (target run-name area area-path reason contour mode-patt)
-		   (print "corporate-ww called with: target=" target " run-name=" run-name " area=" area " area-path=" area-path " reason=" reason " contour=" contour " mode-patt=" mode-patt)
-		   (let* ((last-name   (get-last-runname area-path target))
-			  (last-letter (let* ((ch (if (string? last-name)
-						      (let ((len (string-length last-name)))
-							(substring last-name (- len 1) len))
-						      "a"))
-					      (chnum (str-first-char->number ch))
-					      (a     (str-first-char->number "a"))
-					      (z     (str-first-char->number "z")))
-					 (if (and (>= chnum a)(<= chnum z))
-					     chnum
-					     #f)))
-			  (next-letter (if last-letter
-					   (list->string
-					    (list
-					     (integer->char
-					      (+ last-letter 1)))) ;; surely there is an easier way?
-					   "a")))
-		     ;; (print "last-name: " last-name " last-letter: " last-letter " next-letter: " next-letter)
-		     (conc (seconds->wwdate (current-seconds)) next-letter))))
-
-(hash-table-set! *runname-mappers*
-		 'auto
-		 (lambda (target run-name area area-path reason contour mode-patt)
-		   "auto-eh"))
-
-;; (print "Got here!")
+(add-target-mapper 'prefix-contour
+		   (lambda (target run-name area area-path reason contour mode-patt)
+		     (conc contour "/" target)))
+(add-target-mapper 'prefix-area-contour
+		   (lambda (target run-name area area-path reason contour mode-patt)
+		     (conc area "/" contour "/" target)))
+  
+(add-runname-mapper 'corporate-ww
+		    (lambda (target run-name area area-path reason contour mode-patt)
+		      (print "corporate-ww called with: target=" target " run-name=" run-name " area=" area " area-path=" area-path " reason=" reason " contour=" contour " mode-patt=" mode-patt)
+		      (let* ((last-name   (get-last-runname area-path target))
+			     (last-letter (let* ((ch (if (string? last-name)
+							 (let ((len (string-length last-name)))
+							   (substring last-name (- len 1) len))
+							 "a"))
+						 (chnum (str-first-char->number ch))
+						 (a     (str-first-char->number "a"))
+						 (z     (str-first-char->number "z")))
+					    (if (and (>= chnum a)(<= chnum z))
+						chnum
+						#f)))
+			     (next-letter (if last-letter
+					      (list->string
+					       (list
+						(integer->char
+						 (+ last-letter 1)))) ;; surely there is an easier way?
+					      "a")))
+			;; (print "last-name: " last-name " last-letter: " last-letter " next-letter: " next-letter)
+			(conc (seconds->wwdate (current-seconds)) next-letter))))
+
+(add-runname-mapper 'auto
+		    (lambda (target run-name area area-path reason contour mode-patt)
+		      "auto-eh"))
+
+;; run only areas where first letter of area name is "a"
+;;
+(add-area-checker 'first-letter-a
+                  (lambda (area target contour)
+                    (string-match "^a.*$" area)))
+
 

Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -6,14 +6,14 @@
 INSTALL=install
 SRCFILES = common.scm items.scm launch.scm \
    ods.scm runconfig.scm server.scm configf.scm \
    db.scm keys.scm margs.scm megatest-version.scm \
    process.scm runs.scm tasks.scm tests.scm genexample.scm \
-   http-transport.scm filedb.scm \
+   http-transport.scm filedb.scm tdb.scm \
    client.scm synchash.scm daemon.scm mt.scm \
    ezsteps.scm lock-queue.scm sdb.scm \
-   rmt.scm api.scm tdb.scm rpc-transport.scm \
+   rmt.scm api.scm \
    portlogger.scm archive.scm env.scm diff-report.scm cgisetup/models/pgdb.scm
 
 # Eggs to install (straightforward ones)
 EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
 dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \

Index: archive.scm
==================================================================
--- archive.scm
+++ archive.scm
@@ -7,12 +7,11 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 ;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking format md5 message-digest srfi-18)
-(import (prefix sqlite3 sqlite3:))
+(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 format md5 message-digest srfi-18)
 
 (declare (unit archive))
 (declare (uses db))
 (declare (uses common))
 

Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -10,18 +10,13 @@
 
 ;;======================================================================
 ;; C L I E N T S
 ;;======================================================================
 
-(require-extension (srfi 18) extras tcp s11n)
-
-(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest matchable)
-;; (use zmq)
-
-(use (prefix sqlite3 sqlite3:))
-
-(use spiffy uri-common intarweb http-client spiffy-request-vars uri-common intarweb directory-utils)
+(use srfi-18 extras tcp s11n srfi-1 posix regex srfi-69 hostinfo md5
+     message-digest matchable spiffy uri-common intarweb http-client
+     spiffy-request-vars uri-common intarweb directory-utils)
 
 (declare (unit client))
 
 (declare (uses common))
 (declare (uses db))

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -8,14 +8,11 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 ;;======================================================================
 
 (use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 sql-de-lite hostinfo md5 message-digest typed-records directory-utils stack
-     matchable)
-(require-extension regex posix)
-
-(require-extension (srfi 18) extras tcp rpc)
+     matchable regex posix srfi-18 extras)
 
 (import (prefix sqlite3 sqlite3:))
 (import (prefix base64 base64:))
 
 (declare (unit common))
@@ -234,17 +231,18 @@
 ;; RADT => Why do we meed the version check here, this is called only if version misma
 ;;
 (define (common:cleanup-db dbstruct)
   (db:multi-db-sync 
    dbstruct
+   'schema
    ;; 'new2old
    'killservers
    'dejunk
-   ;; 'adj-testids
+   'adj-target
    ;; 'old2new
    'new2old
-   'schema)
+   )
   (if (common:version-changed?)
       (common:set-last-run-version)))
 
 ;; Rotate logs, logic: 
 ;;                 if > 500k and older than 1 week:
@@ -288,11 +286,11 @@
   (if (common:on-homehost?)
       (if (common:version-changed?)
 	  (let* ((mtconf (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.config"))
                 (dbfile (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.db"))
                 (read-only (not (file-write-access? dbfile)))
-                (dbstruct (db:setup)))
+                (dbstruct (db:setup #t)))
 	    (debug:print 0 *default-log-port*
 			 "WARNING: Version mismatch!\n"
 			 "   expected: " (common:version-signature) "\n"
 			 "   got:      " (common:get-last-run-version))
             (cond
@@ -741,11 +739,11 @@
 
 ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage
 (define (common:watchdog)
   (debug:print-info 13 *default-log-port* "common:watchdog entered.")
   (if (common:on-homehost?)
-      (let ((dbstruct (db:setup)))
+      (let ((dbstruct (db:setup #t)))
 	(debug:print-info 13 *default-log-port* "after db:setup with dbstruct="dbstruct)
 	(cond
 	 ((dbr:dbstruct-read-only dbstruct)
 	  (debug:print-info 13 *default-log-port* "loading read-only watchdog")
 	  (common:readonly-watchdog dbstruct))
@@ -2150,93 +2148,81 @@
    ((equal? status "KILLREQ") "purple")
    ((equal? status "RUNNING") "blue")
    ((equal? status "ABORT")   "brown")
    (else "black")))
 
-;;======================================================================
-;; N A N O M S G   C L I E N T
-;;======================================================================
-
-(define (server:get-best-guess-address hostname)
-  (let ((res #f))
-    (for-each 
-     (lambda (adr)
-       (if (not (eq? (u8vector-ref adr 0) 127))
-	   (set! res adr)))
-     ;; NOTE: This can fail when there is no mention of the host in /etc/hosts. FIXME
-     (vector->list (hostinfo-addresses (hostname->hostinfo hostname))))
-    (string-intersperse 
-     (map number->string
-	  (u8vector->list
-	   (if res res (hostname->ip hostname)))) ".")))
-
-
-(define (common:send-dboard-main-changed)
-  (let* ((dashboard-ips (mddb:get-dashboards)))
-    (for-each
-     (lambda (ipadr)
-       (let* ((soc (common:open-nm-req (conc "tcp://" ipadr)))
-	      (msg (conc "main " *toppath*))
-	      (res (common:nm-send-receive-timeout soc msg)))
-	 (if (not res) ;; couldn't reach that dashboard - remove it from db
-	     (print "ERROR: couldn't reach dashboard " ipadr))
-	 res))
-     dashboard-ips)))
-    
-    
-;;======================================================================
-;; D A S H B O A R D   D B 
-;;======================================================================
-
-(define (mddb:open-db)
-  (let* ((db (open-database (conc (get-environment-variable "HOME") "/.dashboard.db"))))
-    (set-busy-handler! db (busy-timeout 10000))
-    (for-each
-     (lambda (qry)
-       (exec (sql db qry)))
-     (list 
-      "CREATE TABLE IF NOT EXISTS vars       (id INTEGER PRIMARY KEY,key TEXT, val TEXT, CONSTRAINT varsconstraint UNIQUE (key));"
-      "CREATE TABLE IF NOT EXISTS dashboards (
-          id         INTEGER PRIMARY KEY,
-          pid        INTEGER,
-          username   TEXT,
-          hostname   TEXT,
-          ipaddr     TEXT,
-          portnum    INTEGER,
-          start_time TIMESTAMP DEFAULT (strftime('%s','now')),
-             CONSTRAINT hostport UNIQUE (hostname,portnum)
-        );"
-      ))
-    db))
-
-;; register a dashboard 
-;;
-(define (mddb:register-dashboard port)
-  (let* ((pid      (current-process-id))
-	 (hostname (get-host-name))
-	 (ipaddr   (server:get-best-guess-address hostname))
-	 (username (current-user-name)) ;; (car userinfo)))
-	 (db      (mddb:open-db)))
-    (print "Register monitor, pid: " pid ", hostname: " hostname ", port: " port ", username: " username)
-    (exec (sql db "INSERT OR REPLACE INTO dashboards (pid,username,hostname,ipaddr,portnum) VALUES (?,?,?,?,?);")
-	   pid username hostname ipaddr port)
-    (close-database db)))
-
-;; unregister a monitor
-;;
-(define (mddb:unregister-dashboard host port)
-  (let* ((db      (mddb:open-db)))
-    (print "Register unregister monitor, host:port=" host ":" port)
-    (exec (sql db "DELETE FROM dashboards WHERE hostname=? AND portnum=?;") host port)
-    (close-database db)))
-
-;; get registered dashboards
-;;
-(define (mddb:get-dashboards)
-  (let ((db (mddb:open-db)))
-    (query fetch-column
-	   (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;"))))
+;; ;;======================================================================
+;; ;; N A N O M S G   C L I E N T
+;; ;;======================================================================
+;; 
+;; 
+;; 
+;; (define (common:send-dboard-main-changed)
+;;   (let* ((dashboard-ips (mddb:get-dashboards)))
+;;     (for-each
+;;      (lambda (ipadr)
+;;        (let* ((soc (common:open-nm-req (conc "tcp://" ipadr)))
+;; 	      (msg (conc "main " *toppath*))
+;; 	      (res (common:nm-send-receive-timeout soc msg)))
+;; 	 (if (not res) ;; couldn't reach that dashboard - remove it from db
+;; 	     (print "ERROR: couldn't reach dashboard " ipadr))
+;; 	 res))
+;;      dashboard-ips)))
+;;     
+;;     
+;; ;;======================================================================
+;; ;; D A S H B O A R D   D B 
+;; ;;======================================================================
+;; 
+;; (define (mddb:open-db)
+;;   (let* ((db (open-database (conc (get-environment-variable "HOME") "/.dashboard.db"))))
+;;     (set-busy-handler! db (busy-timeout 10000))
+;;     (for-each
+;;      (lambda (qry)
+;;        (exec (sql db qry)))
+;;      (list 
+;;       "CREATE TABLE IF NOT EXISTS vars       (id INTEGER PRIMARY KEY,key TEXT, val TEXT, CONSTRAINT varsconstraint UNIQUE (key));"
+;;       "CREATE TABLE IF NOT EXISTS dashboards (
+;;           id         INTEGER PRIMARY KEY,
+;;           pid        INTEGER,
+;;           username   TEXT,
+;;           hostname   TEXT,
+;;           ipaddr     TEXT,
+;;           portnum    INTEGER,
+;;           start_time TIMESTAMP DEFAULT (strftime('%s','now')),
+;;              CONSTRAINT hostport UNIQUE (hostname,portnum)
+;;         );"
+;;       ))
+;;     db))
+;; 
+;; ;; register a dashboard 
+;; ;;
+;; (define (mddb:register-dashboard port)
+;;   (let* ((pid      (current-process-id))
+;; 	 (hostname (get-host-name))
+;; 	 (ipaddr   (server:get-best-guess-address hostname))
+;; 	 (username (current-user-name)) ;; (car userinfo)))
+;; 	 (db      (mddb:open-db)))
+;;     (print "Register monitor, pid: " pid ", hostname: " hostname ", port: " port ", username: " username)
+;;     (exec (sql db "INSERT OR REPLACE INTO dashboards (pid,username,hostname,ipaddr,portnum) VALUES (?,?,?,?,?);")
+;; 	   pid username hostname ipaddr port)
+;;     (close-database db)))
+;; 
+;; ;; unregister a monitor
+;; ;;
+;; (define (mddb:unregister-dashboard host port)
+;;   (let* ((db      (mddb:open-db)))
+;;     (print "Register unregister monitor, host:port=" host ":" port)
+;;     (exec (sql db "DELETE FROM dashboards WHERE hostname=? AND portnum=?;") host port)
+;;     (close-database db)))
+;; 
+;; ;; get registered dashboards
+;; ;;
+;; (define (mddb:get-dashboards)
+;;   (let ((db (mddb:open-db)))
+;;     (query fetch-column
+;; 	   (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;"))))
     
 ;;======================================================================
 ;;  T E S T   L A U N C H I N G   P E R   I T E M   W I T H   H O S T   T Y P E S
 ;;======================================================================
 ;; 

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -85,10 +85,11 @@
 			"-v"
 			"-q"
 			"-use-db-cache"
 			"-skip-version-check"
 			"-repl"
+                        "-rh5.11" ;; fix to allow running on rh5.11
 			)
 		 args:arg-hash
 		 0))
 
 (if (not (null? remargs))
@@ -105,10 +106,17 @@
 ;;
 (if (not (launch:setup))
     (begin
       (print "Failed to find megatest.config, exiting") 
       (exit 1)))
+
+;; deal with RH 5.11 gtk lib or iup lib missing detachbox feature
+;; first check for the switch
+;;
+(if (or (args:get-arg "-rh5.11")
+	(configf:lookup *configdat* "dashboard" "no-detachbox"))
+    (set! iup:detachbox iup:vbox))
 
 (if (not (common:on-homehost?))
     (begin
       (debug:print 0 *default-log-port* "WARNING: Current policy requires running dashboard on homehost: " (common:get-homehost))))
     

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -307,11 +307,11 @@
 ;;     (db:sync-tables db:sync-tests-only *megatest-db* db)
 ;;     db))
 
 ;; This routine creates the db if not already present. It is only called if the db is not already opened
 ;;
-(define (db:open-db dbstruct #!key (areapath #f)) ;; TODO: actually use areapath
+(define (db:open-db dbstruct #!key (areapath #f)(do-sync #t)) ;; TODO: actually use areapath
   (let ((tmpdb-stack (dbr:dbstruct-dbstack dbstruct))) ;; RA => Returns the first reference in dbstruct
     (if (stack? tmpdb-stack)
 	(db:get-db tmpdb-stack) ;; get previously opened db (will create new db handle if all in the stack are already used
         (let* ((dbpath       (db:dbfile-path ))      ;; path to tmp db area
                (dbexists     (file-exists? dbpath))
@@ -337,13 +337,14 @@
           (dbr:dbstruct-tmpdb-set!  dbstruct tmpdb)
           (dbr:dbstruct-dbstack-set! dbstruct (make-stack)) ;; BB: why a stack?  Why would the number of db's be indeterminate?  Is this a legacy of 1.db 2.db .. ?
           (stack-push! (dbr:dbstruct-dbstack dbstruct) tmpdb) ;; olddb is already a (cons db path)
           (dbr:dbstruct-refndb-set! dbstruct refndb)
           ;;	    (mutex-unlock! *rundb-mutex*)
-          (if (or (not dbfexists)
-                  (and modtimedelta
-                       (> modtimedelta 10))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
+          (if (and  (or (not dbfexists)
+			(and modtimedelta
+			     (> modtimedelta 10))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
+		    do-sync)
 	      (begin
 		(debug:print 4 *default-log-port* "filling db " (db:dbdat-get-path tmpdb) " with data \n    from " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta)
 		(db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb refndb tmpdb)
                 (debug:print-info 13 *default-log-port* "db:sync-all-tables-list done.")
                 )
@@ -353,23 +354,22 @@
 
 ;; Make the dbstruct, setup up auxillary db's and call for main db at least once
 ;;
 ;; called in http-transport and replicated in rmt.scm for *local* access. 
 ;;
-(define (db:setup #!key (areapath #f))
+(define (db:setup do-sync #!key (areapath #f))
   ;;
-
   (cond
    (*dbstruct-db* *dbstruct-db*);; TODO: when multiple areas are supported, this optimization will be a hazard
    (else ;;(common:on-homehost?)
     (debug:print-info 13 *default-log-port* "db:setup entered (first time, not cached.)")
     (let* ((dbstruct (make-dbr:dbstruct)))
       (when (not *toppath*)
         (debug:print-info 13 *default-log-port* "in db:setup, *toppath* not set; calling launch:setup")
         (launch:setup areapath: areapath))
       (debug:print-info 13 *default-log-port* "Begin db:open-db")
-      (db:open-db dbstruct areapath: areapath)
+      (db:open-db dbstruct areapath: areapath do-sync: do-sync)
       (debug:print-info 13 *default-log-port* "Done db:open-db")
       (set! *dbstruct-db* dbstruct)
       ;;(debug:print-info 13 *default-log-port* "new dbstruct = "(dbr:dbstruct->alist dbstruct))
       dbstruct))))
    ;; (else
@@ -836,10 +836,40 @@
                                BEGIN 
                                  UPDATE run_stats SET last_update=(strftime('%s','now'))
                                    WHERE id=old.id;
                                END;"))
 
+(define (db:adj-target db)
+  (let ((fields    (configf:get-section *configdat* "fields"))
+	(field-num 0))
+    ;; because we will be refreshing the keys table it is best to clear it here
+    (sqlite3:execute db "DELETE FROM keys;")
+    (for-each
+     (lambda (field)
+       (let ((column (car field))
+	     (spec   (cadr field)))
+	 (handle-exceptions
+	  exn
+	  (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn))
+	      (debug:print 0 *default-log-port* "Target field " column " already exists in the runs table")
+	      (db:general-sqlite-error-dump exn "alter table runs ..." #f "none"))
+	  ;; Add the column if needed
+	  (sqlite3:execute
+	   db
+	   (conc "ALTER TABLE runs ADD COLUMN " column " " spec)))
+	 ;; correct the entry in the keys column
+	 (sqlite3:execute
+	  db
+	  "INSERT INTO keys (id,fieldname,fieldtype) VALUES (?,?,?);"
+	  field-num column spec)
+	 ;; fill in blanks (not allowed as it would be part of the path
+	 (sqlite3:execute
+	  db
+	  (conc "UPDATE runs SET " column "='x' WHERE " column "='';"))
+	 (set! field-num (+ field-num 1))))
+     fields)))
+  
 (define *global-db-store* (make-hash-table))
 
 (define (db:get-access-mode)
   (if (args:get-arg "-use-db-cache") 'cached 'rmt))
 
@@ -923,138 +953,62 @@
 	     (tmpdb    (db:get-db dbstruct))
              (refndb   (dbr:dbstruct-refndb dbstruct))
 	     (allow-cleanup #t) ;; (if run-ids #f #t))
 	     (servers (server:get-list *toppath*)) ;; (tasks:get-all-servers (db:delay-if-busy tdbdat)))
 	     (data-synced 0)) ;; count of changed records (I hope)
-    
-	;; kill servers
-	(if (member 'killservers options)
-	    (for-each
-	     (lambda (server)
-	       (match-let (((mod-time host port start-time pid) server))
-		 (if (and host pid)
-		     (tasks:kill-server host pid))))
-	     servers))
-
-	;; clear out junk records
-	;;
-	(if (member 'dejunk options)
-	    (begin
+
+	(for-each
+	 (lambda (option)
+
+	   (case option
+	     ;; kill servers
+	     ((killservers)
+	      (for-each
+	       (lambda (server)
+		 (match-let (((mod-time host port start-time pid) server))
+			    (if (and host pid)
+				(tasks:kill-server host pid))))
+	       servers))
+
+	     ;; clear out junk records
+	     ;;
+	     ((dejunk)
 	      (db:delay-if-busy mtdb) ;; ok to delay on mtdb
 	      (db:clean-up mtdb)
 	      (db:clean-up tmpdb)
-              (db:clean-up refndb)))
-
-	;; adjust test-ids to fit into proper range
-	;;
-	;; (if (member 'adj-testids options)
-	;;     (begin
-	;;       (db:delay-if-busy mtdb)
-	;;       (db:prep-megatest.db-for-migration mtdb)))
-
-	;; sync runs, test_meta etc.
-	;;
-	(if (member 'old2new options)
-	    ;; (begin
-            (set! data-synced
-                  (+ (db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb tmpdb refndb)
-                     data-synced)))
-			      ;; (db:sync-main-list mtdb) mtdb (db:get-db dbstruct #f))
-;; 	      (for-each 
-;; 	       (lambda (run-id)
-;; 		 (db:delay-if-busy mtdb)
-;; 		 (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id)))
-;; ;;		       (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)))
-;; 		   (debug:print 0 *default-log-port* "INFO: Propagating " (length testrecs) " records for run-id=" run-id " to run specific db")
-;; 		   (db:replace-test-records dbstruct run-id testrecs)
-;; 		   (sqlite3:finalize! (db:dbdat-get-db (dbr:dbstruct-rundb dbstruct)))))
-;; 	       run-ids)))
-
-	;; now ensure all newdb data are synced to megatest.db
-	;; do not use the run-ids list passed in to the function
-	;;
-	(if (member 'new2old options)
-	    (set! data-synced
-		  (+ (db:sync-tables (db:sync-all-tables-list dbstruct) #f tmpdb refndb mtdb)
-		      data-synced)))
-
-
-
-        (if (member 'schema options)
-            (begin
+              (db:clean-up refndb))
+
+	     ;; sync runs, test_meta etc.
+	     ;;
+	     ((old2new)
+	      (set! data-synced
+		    (+ (db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb tmpdb refndb)
+		       data-synced)))
+	     
+	     ;; now ensure all newdb data are synced to megatest.db
+	     ;; do not use the run-ids list passed in to the function
+	     ;;
+	     ((new2old)
+	      (set! data-synced
+		    (+ (db:sync-tables (db:sync-all-tables-list dbstruct) #f tmpdb refndb mtdb)
+		       data-synced)))
+
+	     ((adj-target)
+	      (db:adj-target (db:dbdat-get-db mtdb))
+	      (db:adj-target (db:dbdat-get-db tmpdb))
+	      (db:adj-target (db:dbdat-get-db refndb)))
+	   
+	     ((schema)
               (db:patch-schema-maindb (db:dbdat-get-db mtdb))
               (db:patch-schema-maindb (db:dbdat-get-db tmpdb))
               (db:patch-schema-maindb (db:dbdat-get-db refndb))
               (db:patch-schema-rundb  (db:dbdat-get-db mtdb))
               (db:patch-schema-rundb  (db:dbdat-get-db tmpdb))
               (db:patch-schema-rundb  (db:dbdat-get-db refndb))))
-              
-	;; (let* ((maindb      (make-dbr:dbstruct path: toppath local: #t))
-	;; 	   (src-run-ids (if run-ids run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0)))))
-	;; 	   (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <))
-	;; 	   (count       1)
-	;; 	   (total       (length all-run-ids))
-	;; 	   (dead-runs  '()))
-	;;   ;; first fix schema if needed
-	;;   (map
-	;;    (lambda (th)
-	;; 	 (thread-join! th))
-	;;    (map
-	;; 	(lambda (run-id)
-	;; 	  (thread-start! 
-	;; 	   (make-thread
-	;; 	    (lambda ()
-	;; 	      (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))
-;;                    (if (member 'schema options)
-	;; 		(if (eq? run-id 0)
-	;; 		    (let ((maindb  (db:dbdat-get-db (db:get-db fromdb #f))))
-	;; 		      (db:patch-schema-maindb run-id maindb))
-	;; 		    (db:patch-schema-rundb run-id frundb)))
-	;; 	      (set! count (+ count 1))
-	;; 	      (debug:print 0 *default-log-port* "Finished patching schema for " (if (eq? run-id 0) " main.db " (conc run-id ".db")) ", " count " of " total)))))
-	;; 	all-run-ids))
-	;;   ;; Then sync and fix db's
-	;;   (set! count 0)
-	;;   (process-fork
-	;;    (lambda ()
-	;; 	 (map
-	;; 	  (lambda (th)
-	;; 	    (thread-join! th))
-	;; 	  (map
-	;; 	   (lambda (run-id)
-	;; 	     (thread-start! 
-	;; 	      (make-thread
-	;; 	       (lambda ()
-	;; 		 (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))
-	;; 			(frundb (db:dbdat-get-db (db:get-db fromdb run-id))))
-	;; 		   (if (eq? run-id 0)
-	;; 		       (let ((maindb  (db:dbdat-get-db (db:get-db fromdb #f))))
-;;                             (db:sync-tables (db:sync-main-list dbstruct) #f (db:get-db fromdb #f) mtdb)
-	;; 			 (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f))))
-	;; 		       (begin
-	;; 			 ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db
-;;                             (db:sync-tables db:sync-tests-only #f (db:get-db fromdb run-id) mtdb)
-	;; 			 (db:clean-up-rundb (db:get-db fromdb run-id)))))
-	;; 		 (set! count (+ count 1))
-	;; 		 (debug:print 0 *default-log-port* "Finished clean up of "
-	;; 			      (if (eq? run-id 0)
-	;; 				  " main.db " (conc run-id ".db")) ", " count " of " total)))))
-	;; 	   all-run-ids))))
-
-	;; removed deleted runs
-;; (let ((dbdir (tasks:get-task-db-path)))
-;;   (for-each (lambda (run-id)
-;; 	      (let ((fullname (conc dbdir "/" run-id ".db")))
-;; 		(if (file-exists? fullname)
-;; 		    (begin
-;; 		      (debug:print 0 *default-log-port* "Removing database file for deleted run " fullname)
-;; 		      (delete-file fullname)))))
-;; 	    dead-runs))))
-;; 
-	;; (db:close-all dbstruct)
-	;; (sqlite3:finalize! mdb)
-        (stack-push! (dbr:dbstruct-dbstack dbstruct) tmpdb)
+	
+	   (stack-push! (dbr:dbstruct-dbstack dbstruct) tmpdb))
+	 options)
 	data-synced)))
 
 ;; keeping it around for debugging purposes only
 (define (open-run-close-no-exception-handling  proc idb . params)
   (debug:print-info 11 *default-log-port* "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params)
@@ -1105,17 +1059,17 @@
            (launch:setup)) ;; added because Elena was getting stack dump because *configinfo* below was #f.
   (let* ((configdat (car *configinfo*))  ;; tut tut, global warning...
 	 (keys     (keys:config-get-fields configdat))
 	 (havekeys (> (length keys) 0))
 	 (keystr   (keys->keystr keys))
-	 (fieldstr (keys->key/field keys))
+	 (fieldstr (keys:make-key/field-string configdat))
 	 (db       (db:dbdat-get-db dbdat)))
     (for-each (lambda (key)
 		(let ((keyn key))
 		  (if (member (string-downcase keyn)
 			      (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count"
-				    "pass_count"))
+				    "pass_count" "contour"))
 		      (begin
 			(print "ERROR: your key cannot be named " keyn " as this conflicts with the same named field in the runs table, you must remove your megatest.db and <linktree>/.db before trying again.")
 			(exit 1)))))
 	      keys)
     (sqlite3:with-transaction
@@ -3348,26 +3302,29 @@
                                                     (cons status (map dbr:counts-status state-status-counts)))
                                                    *common:std-statuses* >))
 			    (non-completes     (filter (lambda (x)
 							 (not (equal? x "COMPLETED")))
 						       all-curr-states))
+			    (num-non-completes (length non-completes))
                             (newstate          (cond
 						((> running 0)
 						 "RUNNING") ;; anything running, call the situation running
 						((> bad-not-started 0)  ;; we have an ugly situation, it is completed in the sense we cannot do more.
 						 "COMPLETED") 
-						((> (length non-completes) 0) ;;
+						((> num-non-completes 0) ;;
 						 (car non-completes))  ;;  (remove (lambda (x)(equal? "COMPLETED" x)) all-curr-states)))
 						(else
 						 (car all-curr-states))))
 			                       ;; (if (> running 0)
                                                ;;     "RUNNING"
                                                ;;     (if (> bad-not-started 0)
                                                ;;         "COMPLETED"
                                                ;;         (car all-curr-states))))
-                            (newstatus            (if (> bad-not-started 0)
-                                                      "CHECK"
+                            (newstatus            (if (or (> bad-not-started 0)
+							  (and (equal? newstate "NOT_STARTED")
+							       (> num-non-completes 0)))
+						      "CHECK"
                                                       (car all-curr-statuses))))
                        ;; (print "bad-not-supported: " bad-not-support " all-curr-states: " all-curr-states " all-curr-statuses: " all-curr-states)
                        ;;      " newstate: " newstate " newstatus: " newstatus)
                        ;; NB// Pass the db so it is part of the transaction
                        (db:test-set-state-status db run-id tl-test-id newstate newstatus #f)))))))

Index: docs/Makefile
==================================================================
--- docs/Makefile
+++ docs/Makefile
@@ -13,5 +13,7 @@
 	fossil add html/*
 
 megatest.pdf : megatest.lyx
 	lyx -e pdf2 megatest.lyx
 
+pkts.pdf : pkts.dot
+	dot -Tpdf pkts.dot -o pkts.pdf

ADDED   docs/pkts.dot
Index: docs/pkts.dot
==================================================================
--- /dev/null
+++ docs/pkts.dot
@@ -0,0 +1,59 @@
+digraph megatest_pkts {
+  ranksep=0.05
+  // rankdir=LR
+
+node [shape=box,style=filled];
+  
+  "SENSORS" [ label = "{ Sensor Processing | { file | git | fossil | script }}"
+	      shape = "record"; ];
+  
+  "RUNS"    [ label = "{ Runs Processing | { launch | clean | re-run | archive } | { dispatcher }}";
+	      shape = "record"; ];
+
+  "WORK"    [ label = "{ Work Items | { start task | task competed }}";
+	      shape = "record"; ];
+
+  "USERREQ" [ label = "{ User Requests (Unix and Web) | { launch | clean | re-run | archive }}";
+	      shape = "record"; ];
+
+  "MTAREA1" [ label = "{ Megatest Area 1 | { parallel job\nmanagement | test\nmanagement | data\nrollup }}";
+	      shape = "record"; ];
+  
+  "MTAREA2" [ label = "{ Megatest Area 2 | { parallel job\nmanagement | test\nmanagement | data\nrollup }}";
+	      shape = "record"; ];
+  
+  "MTAREA3" [ label = "More Megatest Areas ... ";
+	      shape = "record"; ];
+
+  "PGDB"    [ label = "postgres database";
+	      shape = "cylinder"; ];
+
+  "WEBAPP"  [ label = "{ Web View | { Runs | Contours | Control | Time View }}";
+	      shape = "record"; ];
+
+  // "WEBCTRL" [ label = "{ Web View \n(control) }";
+  //	      shape = "record"; ];
+  
+  "SENSORS" -> "SPKTS";
+  "RUNS"    -> "run pkts";
+  "run pkts" -> "RUNS";
+  "WORK"    -> "work pkts";
+  "work pkts" -> "RUNS";
+  "USERREQ" -> "user request pkts";
+  "SPKTS"   -> "RUNS";
+  "user request pkts" -> "RUNS";
+  "RUNS"    -> "MTAREA1" -> "PGDB";
+  "RUNS"    -> "MTAREA2" -> "PGDB";
+  "RUNS"    -> "MTAREA3" -> "PGDB";
+  "PGDB"    -> "WEBAPP";
+  // "WEBCTRL" -> "run pkts";
+  
+  subgraph cluster_pkts {
+    label="Packets";
+    "SPKTS" [ label = "Sensor Packets" ];
+    "run pkts";
+    "work pkts";
+    "user request pkts";
+  }
+}
+

ADDED   docs/pkts.pdf
Index: docs/pkts.pdf
==================================================================
--- /dev/null
+++ docs/pkts.pdf
cannot compute difference between binary files

Index: ezsteps.scm
==================================================================
--- ezsteps.scm
+++ ezsteps.scm
@@ -8,12 +8,11 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 ;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking directory-utils)
-(import (prefix sqlite3 sqlite3:))
+(use srfi-1 posix regex srfi-69 directory-utils)
 
 (declare (unit ezsteps))
 (declare (uses db))
 (declare (uses common))
 (declare (uses items))

Index: gutils.scm
==================================================================
--- gutils.scm
+++ gutils.scm
@@ -20,25 +20,34 @@
   (let* ((c1 (map string->number (string-split color1)))
 	 (c2 (map string->number (string-split color2)))
 	 (delta (map (lambda (a b)(abs (- a b))) c1 c2)))
     (null? (filter (lambda (x)(> x 3)) delta))))
 
+(define gutils:colors
+  '((PASS . "70 249 73")
+    (FAIL . "253 33 49")
+    (SKIP . "230 230 0")))
+
+(define (gutils:get-color-spec effective-state)
+  (or (alist-ref effective-state gutils:colors)
+      (alist-ref 'FAIL gutils:colors)))
+ 
 (define (gutils:get-color-for-state-status state status);; #!key (get-label #f))
   ;; ((if get-label cadr car)
   (case (string->symbol state)
     ((COMPLETED) ;; ARCHIVED)
      (case (string->symbol status)
        ((PASS)        (list "70  249 73" status))
        ((WARN WAIVED) (list "255 172 13" status))
-       ((SKIP)        (list "230 230 0" status))
+       ((SKIP)        (list (gutils:get-color-spec 'SKIP) status))
        ((ABORT)       (list "198 36 166" status))
        (else (list "253 33 49" status))))
     ((ARCHIVED)
      (case (string->symbol status)
        ((PASS)        (list "70  170 73" status))
        ((WARN WAIVED) (list "200 130 13" status))
-       ((SKIP)        (list "180 180 0" status))
+       ((SKIP)        (list (gutils:get-color-spec 'SKIP) status))
        (else (list "180 33 49" status))))
     ;;      (if (equal? status "PASS")
     ;;	  '("70 249 73" "PASS")
     ;;	  (if (or (equal? status "WARN")
     ;;		  (equal? status "WAIVED"))
@@ -48,11 +57,13 @@
     ((CHECK)            (list "255 100 50"   state))
     ((REMOTEHOSTSTART)  (list "50 130 195"   state))
     ((RUNNING)          (list "9 131 232"    state))
     ((KILLREQ)          (list "39 82 206"    state))
     ((KILLED)           (list "234 101 17"   state))
-    ((NOT_STARTED)      (list "240 240 240"  state))
+    ((NOT_STARTED)      (case (string->symbol status)
+			  ((CHECK)(list (gutils:get-color-spec 'SKIP) state))
+			  (else   (list "240 240 240"                 state))))
     ;; for xor mode below
     ;;
     ((CLEAN)
      (case (string->symbol status)
        ((CLEAN-FAIL CLEAN-CHECK CLEAN-ABORT)  (list "200 130 13" status)) ;; orange requested for these

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -8,12 +8,11 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 (require-extension (srfi 18) extras tcp s11n)
 
-(use  srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest posix-extras) ;; sqlite3
-;; (import (prefix sqlite3 sqlite3:))
+(use  srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest posix-extras)
 
 (use spiffy uri-common intarweb http-client spiffy-request-vars intarweb spiffy-directory-listing)
 
 ;; Configurations for server
 (tcp-buffer-size 2048)
@@ -109,11 +108,14 @@
     (http-transport:try-start-server ipaddrstr start-port)))
 
 ;; This is recursively run by http-transport:run until sucessful
 ;;
 (define (http-transport:try-start-server ipaddrstr portnum)
-  (let ((config-hostname (configf:lookup *configdat* "server" "hostname")))
+  (let ((config-hostname (configf:lookup *configdat* "server" "hostname"))
+	(config-use-proxy (equal? (configf:lookup *configdat* "client" "use-http_proxy") "yes")))
+    (if (not config-use-proxy)
+	(determine-proxy (constantly #f)))
     (debug:print-info 0 *default-log-port* "http-transport:try-start-server time=" (seconds->time-string (current-seconds)) " ipaddrsstr=" ipaddrstr " portnum=" portnum " config-hostname=" config-hostname)
     (handle-exceptions
 	exn
 	(begin
 	  (print-error-message exn)
@@ -380,11 +382,11 @@
 	       (start-time     (current-milliseconds)))
       ;; Use this opportunity to sync the tmp db to megatest.db
       (if (not server-going) ;; *dbstruct-db* 
 	  (begin
 	    (debug:print 0 *default-log-port* "SERVER: dbprep")
-	    (set! *dbstruct-db*  (db:setup)) ;;  run-id))
+	    (set! *dbstruct-db*  (db:setup #t)) ;;  run-id))
 	    (set! server-going #t)
 	    (debug:print 0 *default-log-port* "SERVER: running, megatest version: " (common:get-full-version)) ;; NOTE: the server is NOT yet marked as running in the log. We do that in the keep-running routine.
 	    (thread-start! *watchdog*)))
       
       ;; when things go wrong we don't want to be doing the various queries too often

Index: key_records.scm
==================================================================
--- key_records.scm
+++ key_records.scm
@@ -10,13 +10,13 @@
 ;;======================================================================
 
 (define-inline (keys->valslots keys) ;; => ?,?,? ....
   (string-intersperse (map (lambda (x) "?") keys) ","))
 
-(define-inline (keys->key/field keys . additional)
-  (string-join (map (lambda (k)(conc k " TEXT"))
-		    (append keys additional)) ","))
+;; (define-inline (keys->key/field keys . additional)
+;;   (string-join (map (lambda (k)(conc k " TEXT"))
+;; 		    (append keys additional)) ","))
 
 (define-inline (item-list->path itemdat)
   (if (list? itemdat)
       (string-intersperse  (map cadr itemdat) "/")
       ""))

Index: keys.scm
==================================================================
--- keys.scm
+++ keys.scm
@@ -65,5 +65,12 @@
 ;;======================================================================
 ;; config file related routines
 ;;======================================================================
 
 (define keys:config-get-fields common:get-fields)
+(define (keys:make-key/field-string confdat)
+  (let ((fields (configf:get-section confdat "fields")))
+    (string-join
+     (map (lambda (field)(conc (car field) " " (cadr field)))
+	  fields)
+     ",")))
+

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -21,13 +21,10 @@
 
 (declare (unit launch))
 (declare (uses common))
 (declare (uses configf))
 (declare (uses db))
-;; (declare (uses sdb))
-(declare (uses tdb))
-;; (declare (uses filedb))
 
 (include "common_records.scm")
 (include "key_records.scm")
 (include "db_records.scm")
 
@@ -1206,11 +1203,11 @@
 (define (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat params)
   (mutex-lock! *launch-setup-mutex*) ;; setting variables and processing the testconfig is NOT thread-safe, reuse the launch-setup mutex
   (let* ((item-path       (item-list->path itemdat))
 	 (contour         #f)) ;; NOT READY FOR THIS (args:get-arg "-contour")))
     (let loop ((delta        (- (current-seconds) *last-launch*))
-	       (launch-delay (string->number (or (configf:lookup *configdat* "setup" "launch-delay") "5"))))
+	       (launch-delay (string->number (or (configf:lookup *configdat* "setup" "launch-delay") "0"))))
       (if (> launch-delay delta)
 	  (begin
 	    (debug:print-info 0 *default-log-port* "Delaying launch of " test-name " for " (- launch-delay delta) " seconds")
 	    (thread-sleep! (- launch-delay delta))
 	    (loop (- (current-seconds) *last-launch*) launch-delay))))

Index: lock-queue.scm
==================================================================
--- lock-queue.scm
+++ lock-queue.scm
@@ -5,12 +5,11 @@
 ;; 
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
-(use sqlite3 srfi-18)
-(import (prefix sqlite3 sqlite3:))
+(use (prefix sqlite3 sqlite3:) srfi-18)
 
 (declare (unit lock-queue))
 (declare (uses common))
 (declare (uses tasks))
 

Index: megatest-version.scm
==================================================================
--- megatest-version.scm
+++ megatest-version.scm
@@ -1,7 +1,7 @@
 ;; Always use two or four digit decimal
 ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..
 
 (declare (unit megatest-version))
 
-(define megatest-version 1.6408)
+(define megatest-version 1.6501)
 

Index: megatest.config
==================================================================
--- megatest.config
+++ megatest.config
@@ -1,23 +1,33 @@
-[fields]
-a text
-b text
-c text
+# [fields]
+# a text
+# b text
+# c text
+
+[defaults]
+usercode    .mtutil.scm
+areafilter  area-to-run
+targtrans   generic-target-translator
+runtrans    generic-runname-translator
 
 [setup]
-pktsdirs /tmp/pkts /some/other/source
+pktsdirs /tmp/mt_pkts /some/other/source
 
 [areas]
 #         path-to-area   map-target-script(future, optional)
-fullrun   path=tests/fullrun
+# someqa     path=../megatestqa/someqa; targtrans=somefunc; areafilter=area-to-run
+fullrun   path=tests/fullrun; 
 # targtrans is name of scheme proc stored in .mtutil.scm, which lives in PWD where mtutil is run
+#           the target translator can return: a/target OR (list/of targets/to apply/run) OR #f i.e. run nothing
 # ext-tests path=ext-tests; targtrans=prefix-contour;
 ext-tests path=ext-tests
 
 [contours]
 #     mode-patt/tag-expr
-quick selector=QUICKPATT/quick
-full  areas=fullrun,ext-tests; selector=MAXPATT/
-all   areas=fullrun,ext-tests
-snazy areas=%; selector=QUICKPATT/
+quick areas=ext-tests;   selector=/QUICKPATT
+# quick areas=fullrun,ext-tests;   selector=QUICKPATT/quick
+# full  areas=fullrun,ext-tests; selector=MAXPATT/
+# short areas=fullrun,ext-tests; selector=MAXPATT/
+# all   areas=fullrun,ext-tests
+# snazy selector=QUICKPATT/
 
 [nopurpose]

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -11,20 +11,18 @@
 ;; (include "megatest-version.scm")
 
 ;; fake out readline usage of toplevel-command
 (define (toplevel-command . a) #f)
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 readline apropos json http-client directory-utils rpc typed-records;; (srfi 18) extras)
-     http-client srfi-18 extras format) ;;  zmq extras)
+(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (prefix base64 base64:)
+     readline apropos json http-client directory-utils typed-records
+     http-client srfi-18 extras format)
 
 ;; Added for csv stuff - will be removed
 ;;
 (use sparse-vectors)
 
-(import (prefix sqlite3 sqlite3:))
-(import (prefix base64 base64:))
-(import (prefix rpc rpc:))
 (require-library mutils)
 
 ;; (use zmq)
 
 (declare (uses common))
@@ -403,11 +401,12 @@
          "-list-targets"
          "-show-runconfig"
          ;;"-list-db-targets"
          "-show-runconfig"
          "-show-config"
-         "-show-cmdinfo"))
+         "-show-cmdinfo"
+	 "-cleanup-db"))
        (no-watchdog-args-vals (filter (lambda (x) x)
                                       (map args:get-arg no-watchdog-args)))
        (start-watchdog (null? no-watchdog-args-vals)))
   ;;(BB> "no-watchdog-args="no-watchdog-args "no-watchdog-args-vals="no-watchdog-args-vals) 
   (if start-watchdog
@@ -432,11 +431,11 @@
 ;;
 (if (or (args:get-arg "-log")(args:get-arg "-server")) ;; redirect the log always when a server
     (handle-exceptions
 	exn
 	(begin
-	  (print "ERROR: Failed to switch to log output. " ((conition-property-accessor 'exn 'message) exn))
+	  (print "ERROR: Failed to switch to log output. " ((condition-property-accessor 'exn 'message) exn))
 	  )
       (let* ((tl   (or (args:get-arg "-log")(launch:setup)))   ;; run launch:setup if -server, ensure we do NOT run launch:setup if -log specified
 	     (logf (or (args:get-arg "-log") ;; use -log unless we are a server, then craft a logfile name
 		       (conc tl "/logs/server-" (current-process-id) "-" (get-host-name) ".log")))
 	     (oup  (open-logfile logf)))
@@ -497,10 +496,24 @@
 
 ;; for some switches always print the command to stderr
 ;;
 (if (args:any? "-run" "-runall" "-remove-runs" "-set-state-status")
     (debug:print 0 *default-log-port* (string-intersperse (argv) " ")))
+
+;; some switches imply homehost. Exit here if not on homehost
+;;
+(let ((homehost-required  (list "-cleanup-db" "-server")))
+  (if (apply args:any? homehost-required)
+      (if (not (common:on-homehost?))
+	  (for-each
+	   (lambda (switch)
+	     (if (args:get-arg switch)
+		 (begin
+		   (debug:print 0 *default-log-port* "ERROR: you must be on the homehost to run with " switch
+				", you can move homehost by removing the .homehost file but this will disrupt any runs in progress.")
+		   (exit 1))))
+	   homehost-required))))
 
 ;;======================================================================
 ;; Misc setup stuff
 ;;======================================================================
 
@@ -1863,11 +1876,11 @@
     (begin
       (if (not (launch:setup))
 	  (begin
 	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
 	    (exit 1)))
-      (let ((dbstruct (db:setup *toppath*)))
+      (let ((dbstruct (db:setup #f areapath: *toppath*)))
         (common:cleanup-db dbstruct))
       (set! *didsomething* #t)))
 
 (if (args:get-arg "-mark-incompletes")
     (begin
@@ -1922,11 +1935,11 @@
 	(args:get-arg "-repl")
 	(args:get-arg "-load"))
     (let* ((toppath (launch:setup))
 	   (dbstruct (if (and toppath
                               (common:on-homehost?))
-                         (db:setup)
+                         (db:setup #t)
                          #f))) ;; make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f)))
       (if *toppath*
 	  (cond
 	   ((getenv "MT_RUNSCRIPT")
 	    ;; How to run megatest scripts
@@ -2011,11 +2024,11 @@
 ;; ;; ;; redo me       (set! *didsomething* #t)))
 
 (if (args:get-arg "-import-megatest.db")
     (begin
       (db:multi-db-sync 
-       (db:setup)
+       (db:setup #f)
        'killservers
        'dejunk
        'adj-testids
        'old2new
        ;; 'new2old
@@ -2023,11 +2036,11 @@
       (set! *didsomething* #t)))
 
 (if (args:get-arg "-sync-to-megatest.db")
     (begin
       (db:multi-db-sync 
-       (db:setup)
+       (db:setup #f)
        'new2old
        )
       (set! *didsomething* #t)))
 
 (if (args:get-arg "-sync-to")

Index: mlaunch.scm
==================================================================
--- mlaunch.scm
+++ mlaunch.scm
@@ -15,12 +15,11 @@
 ;;   take jobs from the given queue and keep launching them keeping
 ;;   the cpu load at the targeted level
 ;;
 ;;======================================================================
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking format)
-(import (prefix sqlite3 sqlite3:))
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 format)
 
 (declare (unit mlaunch))
 (declare (uses db))
 (declare (uses common))
 

Index: mtut.scm
==================================================================
--- mtut.scm
+++ mtut.scm
@@ -25,13 +25,58 @@
 
 (include "megatest-fossil-hash.scm")
 
 (require-library stml)
 
-(define *target-mappers*  (make-hash-table)) ;; '())
-(define *runname-mappers* (make-hash-table)) ;; '())
+;; stuff for the mapper and checker functions
+;;
+(define *target-mappers*  (make-hash-table)) 
+(define *runname-mappers* (make-hash-table)) 
+(define *area-checkers*   (make-hash-table)) 
+
+;; helpers for mappers/checkers
+(define (add-target-mapper name proc)
+  (hash-table-set! *target-mappers* name proc))
+(define (add-runname-mapper name proc)
+  (hash-table-set! *runname-mappers* name proc))
+(define (add-area-checker name proc)
+  (hash-table-set! *area-checkers* name proc))
+
+;; given a runkey, xlatr-key and other info return one of the following:
+;;   list of targets, null list to skip processing
+;;   
+(define (map-targets mtconf aval-alist runkey area contour #!key (xlatr-key-in #f))
+  (let* ((xlatr-key (or xlatr-key-in
+                        (conf-get/default mtconf aval-alist 'targtrans)))
+         (proc      (hash-table-ref/default *target-mappers* xlatr-key #f)))
+    (if proc
+        (begin
+          (print "Using target mapper: " area-xlatr)
+          (handle-exceptions
+           exn
+           (begin
+             (print "FAILED TO RUN TARGET MAPPER FOR " area ", called " area-xlatr)
+             (print "   function is: " (hash-table-ref/default *target-mappers* xlatr-key #f ) )
+             (print " message: " ((condition-property-accessor 'exn 'message) exn))
+             runkey)
+           (proc runkey area contour)))
+        (begin
+          (if xlatr-key 
+              (print "ERROR: Failed to find named target translator " xlatr-key ", using original target."))
+          `(,runkey))))) ;; no proc then use runkey
 
+;; given mtconf and areaconf extract a translator/filter, first look at areaconf
+;; then if not found look at default
+;;
+(define (conf-get/default mtconf areaconf keyname #!key (default #f))
+  (let ((res (or (alist-ref keyname areaconf)
+                 (configf:lookup mtconf "default" (conc keyname))
+                 default)))
+    (if res
+        (string->symbol res)
+        res)))
+  
 ;; this needs some thought regarding security implications.
 ;;
 ;;   i. Check that owner of the file and calling user are same?
 ;;  ii. Check that we are in a legal megatest area?
 ;; iii. Have some form of authentication or record of the md5sum or similar of the file?
@@ -40,11 +85,11 @@
 ;;
 (if (file-exists? "megatest.config")
     (if (file-exists? ".mtutil.so")
 	(load ".mtutil.so")
 	(if (file-exists? ".mtutil.scm")
-	(load ".mtutil.scm"))))
+            (load ".mtutil.scm"))))
 
 ;; Disabled help items
 ;;  -rollup                 : (currently disabled) fill run (set by :runname)  with latest test(s)
 ;;                            from prior runs with same keys
 ;; Contour actions
@@ -154,10 +199,32 @@
 (define *action-keys*
   '((run         . "-run")
     (sync        . "")
     (archive     . "-archive")
     (set-ss      . "-set-state-status")))
+
+;; Card types:
+;;
+;; A action
+;; U username (Unix)
+;; D timestamp
+;; T card type
+
+;; utilitarian alist for standard cards
+;;
+(define *additional-cards*
+  '(
+    ;; Standard Cards
+    (A  . action    )
+    (D  . timestamp )
+    (T  . cardtype  )
+    (U  . user      ) ;; username
+    (Z  . shar1sum  )
+
+    ;; Extras
+    (a  . runkey    ) ;; needed for matching up pkts with target derived from runkey
+    ))
 
 ;; inlst is an alternative input
 ;;
 (define (lookup-param-by-key key #!key (inlst #f))
   (fold (lambda (a res)
@@ -260,22 +327,14 @@
 	   (loop (get-line) date node time))))
        (else ;; no more datat and last node on branch not found
 	(close-input-port timeline-port)
 	(values  (common:date-time->seconds (conc date " " time)) node))))))
 
-
 ;;======================================================================
 ;; GLOBALS
 ;;======================================================================
 
-;; Card types:
-;;
-;; a action
-;; u username (Unix)
-;; D timestamp
-;; T card type
-
 ;; process args
 (define *action* (if (> (length (argv)) 1)
 		     (cadr (argv))
 		     #f))
 (define remargs (args:get-args 
@@ -402,11 +461,11 @@
 			(if (hash-table? args-alist) ;; seriously?
 			    (hash-table->alist args-alist)
 			    args-alist)
 			(hash-table->alist args:arg-hash))) ;; if no args-alist then we assume this is a call driven directly by commandline
 	 (alldat    (apply append (list 'T "cmd"
-					'a action
+					'A action
 					'U (current-user-name)
 					'D sched)
                            extra-dat
 			   (map (lambda (x)
 				  (let* ((param (car x))
@@ -418,12 +477,12 @@
 						    #f)))
 				    (if (or pmeta smeta)                     ;; construct the switch/param pair.
 					(list meta value)
 					'())))
 				(filter cdr args-data)))))
-;; (print  "Alldat: " alldat
-;;         " args-data: " args-data)
+    (print  "Alldat: " alldat
+	    " args-data: " args-data)
     (add-z-card
      (apply construct-sdat alldat))))
 
 (define (simple-setup start-dir-in)
   (let* ((start-dir (or start-dir-in "."))
@@ -452,16 +511,19 @@
 ;;  ii. Pass the pkt keys and values to this proc and go from there.
 ;; iii. Maybe have an abstraction alist with meaningful names for the pkt keys
 ;;
 ;; Override the run start time record with sched. Usually #f is fine.
 ;; 
-(define (create-run-pkt mtconf action area runkey runname mode-patt tag-expr pktsdir reason contour sched dbdest append-conf runtrans)
+(define (create-run-pkt mtconf action area runkey target runname mode-patt 
+                        tag-expr pktsdir reason contour sched dbdest append-conf
+                        runtrans)
   (let* ((good-val   (lambda (inval)(and inval (string? inval)(not (string-null? inval)))))
 	 (area-dat   (val->alist (or (configf:lookup mtconf "areas" area) "")))
 	 (area-path  (alist-ref 'path      area-dat))
-	 (area-xlatr (alist-ref 'targtrans area-dat))
-	 (new-runname (let* ((callname (if (string? runtrans)(string->symbol runtrans) #f))
+	 ;; (area-xlatr (alist-ref 'targtrans area-dat))
+         ;; (xlatr-key  (if area-xlatr (string->symbol area-xlatr) #f))
+         (new-runname (let* ((callname (if (string? runtrans)(string->symbol runtrans) #f))
 			     (mapper   (if callname (hash-table-ref/default *runname-mappers* callname #f) #f)))
 			;; (print "callname=" callname " runtrans=" runtrans " mapper=" mapper)
 			(if (and callname
 				 (not (equal? callname "auto"))
 				 (not mapper))
@@ -477,28 +539,11 @@
 			      (print "(mapper " (string-intersperse (list runkey runname area area-path reason contour mode-patt) ", ") ")")
 			      (mapper runkey runname area area-path reason contour mode-patt))
 			    (case callname
 			      ((auto) runname)
 			      (else   runtrans)))))
-	 (new-target (if area-xlatr 
-			 (let ((xlatr-key (string->symbol area-xlatr)))
-			   (if (hash-table-exists? *target-mappers* xlatr-key)
-			       (begin
-				 (print "Using target mapper: " area-xlatr)
-				 (handle-exceptions
-				     exn
-				     (begin
-				       (print "FAILED TO RUN TARGET MAPPER FOR " area ", called " area-xlatr)
-				       (print "   function is: " (hash-table-ref/default *target-mappers* xlatr-key #f ) )
-				       (print " message: " ((condition-property-accessor 'exn 'message) exn))
-				       runkey)
-				   ((hash-table-ref *target-mappers* xlatr-key)
-				    runkey new-runname area area-path reason contour mode-patt)))
-			       (begin
-				 (print "ERROR: Failed to find named target translator " xlatr-key ", using original target.")
-				 runkey)))
-			 runkey))
+	 (new-target     target) ;; I believe we will want target manipulation here .. (map-targets xlatr-key runkey area contour))
 	 (actual-action  (if action
 			     (if (equal? action "sync-prepend")
 				 "sync"
 				 action)
 			     "run"))) ;; this has gotten a bit ugly. Need a function to handle actions processing.
@@ -505,11 +550,11 @@
     ;; some hacks to remove switches not needed in certain cases
     (case (string->symbol (or action "run"))
       ((sync sync-prepend)
        (set! new-target #f)
        (set! runame     #f)))
-    (print "area-path: " area-path " area-xlatr: " area-xlatr " orig-target: " runkey " new-target: " new-target)
+    ;; (print "area-path: " area-path " orig-target: " runkey " new-target: " new-target)
     (let-values (((uuid pkt)
 		  (command-line->pkt
 		   actual-action
 		   (append 
 		    `(("-start-dir"  . ,area-path)
@@ -530,25 +575,36 @@
 			`(("-preclean"  . " ")
 			  ("-rerun-all" . " "))      ;; if run we *always* want preclean set, use single space as placeholder
 			'())
 		    )
 		   sched
-                   extra-dat: `((a . ,runkey))  ;; we need the run key for marking the run as launched
+                   extra-dat: `(a ,runkey)  ;; we need the run key for marking the run as launched
                    )))
       (with-output-to-file
 	  (conc pktsdir "/" uuid ".pkt")
 	(lambda ()
 	  (print pkt))))))
 
-
+;; look for areas=a1,a2,a3 OR areafn=somefuncname
+;;
 (define (val-alist->areas val-alist)
-  (string-split (or (alist-ref 'areas val-alist) "") ","))
+  (let ((areas-string   (alist-ref 'areas  val-alist))
+	(areas-procname (alist-ref 'areafn val-alist)))
+    (if areas-procname ;; areas-procname take precedence
+	areas-procname
+	(string-split (or areas-string "") ","))))
 
-(define (area-allowed? area areas)
-  (or (not areas)
-      (null? areas)
-      (member area areas)))
+(define (area-allowed? area areas runkey contour)
+  (cond
+   ((not areas) #t) ;; no spec
+   ((string? areas) ;; 
+    (let ((check-fn (hash-table-ref/default *area-checkers* areas #f)))
+      (if check-fn
+	  (check-fn area runkey contour)
+	  #f)))
+   ((list? areas)(member area areas))
+   (else #f))) ;; shouldn't get here 
 
 ;; (use trace)(trace create-run-pkt)
 
 ;; collect all needed data and create run pkts for contours with changed inputs
 ;;
@@ -581,26 +637,35 @@
 			(optional   (if (> len-key 3)(cadddr keyparts) #f))
 			;; (val-list   (string-split-fields ";\\s*" val #:infix)) ;; (string-split val)) ;; runname-rule params
 			(val-alist  (val->alist val))
 			(runname    (make-runname "" ""))
 			(runtrans   (alist-ref 'runtrans val-alist))
+
+			;; these may or may not be defined and not all are used in each handler type in the case below
+			(run-name   (alist-ref 'run-name val-alist))
+			(target     (alist-ref 'target   val-alist))
+			(crontab    (alist-ref 'cron     val-alist))
+			(areas      (val-alist->areas    val-alist)) ;; areas can be a single string (a reference to call an areas function), or a list of area names.
+			(dbdest     (alist-ref 'dbdest   val-alist))
+			(appendconf (alist-ref 'appendconf val-alist))
+			(file-globs (alist-ref 'glob val-alist))
 			
 			(runstarts  (find-pkts pdb '(runstart) `((o . ,contour)
 								 (t . ,runkey))))
 			(rspkts     (get-pkt-alists runstarts))
 			;; starttimes is for run start times and is used to know when the last run was launched
 			(starttimes (get-pkt-times rspkts)) ;; sort by age (youngest first) and delete duplicates by target
-			(last-run (if (null? starttimes) ;; if '() then it has never been run, else get the max
-				      0
-				      (apply max (map cdr starttimes))))
+			(last-run   (if (null? starttimes) ;; if '() then it has never been run, else get the max
+					0
+					(apply max (map cdr starttimes))))
 			;; synctimes is for figuring out the last time a sync was done
-			(syncstarts   (find-pkts pdb '(syncstart) '())) ;; no qualifiers, a sync does all tarets etc.
-			(sspkts       (get-pkt-alists syncstarts))
-			(synctimes    (get-pkt-times  sspkts))
-			(last-sync (if (null? synctimes) ;; if '() then it has never been run, else get the max
-				      0
-				      (apply max (map cdr synctimes))))
+			(syncstarts (find-pkts pdb '(syncstart) '())) ;; no qualifiers, a sync does all tarets etc.
+			(sspkts     (get-pkt-alists syncstarts))
+			(synctimes  (get-pkt-times  sspkts))
+			(last-sync  (if (null? synctimes) ;; if '() then it has never been run, else get the max
+					0
+					(apply max (map cdr synctimes))))
 			)
 
 		   (let ((delta (lambda (x)
 				  (round (/ (- (current-seconds) x) 60)))))
 		     (print "runkey: " runkey ", ruletype: " ruletype ", action: " action ", last-run: " last-run " time since; last-run: " (delta last-run) ", last-sync: " (delta last-sync)))
@@ -614,49 +679,53 @@
 		   
 		   (case (string->symbol (or ruletype "no-such-rule"))
 
 		     ((no-such-rule) (print "ERROR: no such rule for " sense))
 
+		     ;; Handle crontab like rules
+		     ;;
 		     ((scheduled)
 		      (if (not (alist-ref 'cron val-alist)) ;; gotta have cron spec
 			  (print "ERROR: bad sense spec \"" (string-intersperse sense " ") "\" params: " val-alist)
-			  (let* ((run-name (alist-ref 'run-name val-alist))
-				 (target   (alist-ref 'target   val-alist))
-				 (crontab  (alist-ref 'cron     val-alist))
-                                 (areas    (val-alist->areas val-alist))
+			  (let* (
 				 ;; (action   (alist-ref 'action   val-alist))
-				 (cron-safe-string (string-translate (string-intersperse (string-split (alist-ref 'cron val-alist)) "-") "*" "X"))
+				 (cron-safe-string (string-translate (string-intersperse (string-split crontab) "-") "*" "X"))
 				 (runname  std-runname)) ;; (conc "sched" (time->string (seconds->local-time (current-seconds)) "%M%H%d")))))
 			    ;; (print "last-run: " last-run " need-run: " need-run)
 			    ;; (if need-run
 			    (case (string->symbol action)
 			      ((sync sync-prepend)
 			       (if (common:extended-cron crontab #f last-sync)
 				   (push-run-spec torun contour runkey
 						  `((message . ,(conc ruletype ":sync-" cron-safe-string))
 						    (action  . ,action)
-						    (dbdest  . ,(alist-ref 'dbdest val-alist))
-						    (append  . ,(alist-ref 'appendconf val-alist))))))
+						    (dbdest  . ,dbdest)
+						    (append  . ,appendconf)
+						    (areas   . ,areas)))))
 			      ((run)
 			       (if (common:extended-cron crontab #f last-run)
 				   (push-run-spec torun contour runkey
-						  `((message . ,(conc ruletype ":" cron-safe-string))
-						    (runname . ,runname)
+						  `((message  . ,(conc ruletype ":" cron-safe-string))
+						    (runname  . ,runname)
 						    (runtrans . ,runtrans)
-						    (action  . ,action)
-						    (target  . ,target)))))
+						    (action   . ,action)
+						    (areas    . ,areas)
+						    (target   . ,target)))))
                               ((remove)
                                (push-run-spec torun contour runkey
-						  `((message . ,(conc ruletype ":" cron-safe-string))
-						    (runname . ,runname)
+						  `((message  . ,(conc ruletype ":" cron-safe-string))
+						    (runname  . ,runname)
 						    (runtrans . ,runtrans)
-						    (action  . ,action)
-						    (target  . ,target))))
+						    (action   . ,action)
+						    (areas    . ,areas)
+						    (target   . ,target))))
 			      (else
 			       (print "ERROR: action \"" action "\" has no scheduled handler")
 			       )))))
 
+		     ;; script based sensors
+		     ;;
 		     ((script)
 		      ;; syntax is a little different here. It is a list of commands to run, "scriptname = extra_parameters;scriptname = ..."
 		      ;; where scriptname may be repeated multiple times. The script must return unix-epoch of last change, new-target-name and new-run-name
 		      ;; the script is called like this:  scriptname contour runkey std-runname action extra_param1 extra_param2 ...
 		      (for-each
@@ -689,19 +758,23 @@
 				 (if need-run
 				     (let* ((key-msg    `((message  . ,(conc ruletype ":" message))
 							  (runname  . ,runname)
 							  (runtrans . ,runtrans)
 							  (action   . ,action)
-							  (target   . ,new-target))))
+							  (areas    . ,areas)
+							  (target   . ,new-target) ;; overriding with result from runing the script
+                                                          )))
 				       (print "key-msg: " key-msg)
 				       (push-run-spec torun contour
 						      (if optional  ;; we need to be able to differentiate same contour, different behavior. 
 							  (conc runkey ":" optional)  ;; NOTE: NOT COMPLETELY IMPLEMENTED. DO NOT USE
 							  runkey)
 						      key-msg)))))))
 		       val-alist)) ;; iterate over the param split by ;\s*
 
+		     ;; fossil scm based triggers
+		     ;;
 		     ((fossil)
 		      (for-each
 		       (lambda (fspec)
 			 (print "fspec: " fspec)
 			 (let* ((url         (symbol->string (car fspec))) ;; THIS COULD BE TROUBLE. Add option to reading line to return as string.
@@ -713,66 +786,73 @@
 			   (fossil:clone-or-sync url fname fdir) ;; )
 			   (let-values (((datetime node)
 					 (fossil:last-change-node-and-time fdir fname branch)))
 			     (if (null? starttimes)
 				 (push-run-spec torun contour runkey
-						`((message . ,(conc "fossil:" branch "-neverrun"))
-						  (runname . ,(conc runname "-" node))
+						`((message  . ,(conc "fossil:" branch "-neverrun"))
+						  (runname  . ,(conc runname "-" node))
 						  (runtrans . ,runtrans)
-						  (target  . ,runkey)))
+						  (areas    . ,areas)
+						  ;; (target   . ,runkey)
+                                                  ))
 				 (if (> datetime last-run) ;; change time is greater than last-run time
 				     (push-run-spec torun contour runkey
-						    `((message . ,(conc "fossil:" branch "-" node))
-						      (runname . ,(conc runname "-" node))
+						    `((message  . ,(conc "fossil:" branch "-" node))
+						      (runname  . ,(conc runname "-" node))
 						      (runtrans . ,runtrans)
-						      (target  . ,runkey)))))
+						      (areas    . ,areas)
+						      ;; (target   . ,runkey)
+                                                      ))))
 			     (print "Got datetime=" datetime " node=" node))))
 		       val-alist))
-		     
+
+		     ;; sensor looking for one or more files newer than reference
+		     ;;
 		     ((file file-or) ;; one or more files must be newer than the reference
-		      (let* ((file-globs  (alist-ref 'glob val-alist))
-                             (areas       (val-alist->areas val-alist))
-			     (youngestdat (common:get-youngest (common:bash-glob file-globs)))
+		      (let* ((youngestdat (common:get-youngest (common:bash-glob file-globs)))
 			     (youngestmod (car youngestdat)))
 			;; (print "youngestmod: " youngestmod " starttimes: " starttimes)
 			(if (null? starttimes) ;; this target has never been run
 			    (push-run-spec torun contour runkey
-					   `((message . "file:neverrun")
-					     (action  . ,action)
+					   `((message  . "file:neverrun")
+					     (action   . ,action)
 					     (runtrans . ,runtrans)
-					     (target  . ,runkey)
-					     (runname . ,runname)))
+					     ;; (target   . ,runkey)
+					     (areas    . ,areas)
+					     (runname  . ,runname)))
 			;; (for-each
 			;;  (lambda (starttime) ;; look at the time the last run was kicked off for this contour
 			;;    (if (> youngestmod (cdr starttime))
 			;; 	   (begin
 			;; 	     (print "starttime younger than youngestmod: " starttime " Youngestmod: " youngestmod)
 			    (if (> youngestmod last-run)
 				(push-run-spec torun contour runkey
-					       `((message . ,(conc ruletype ":" (cadr youngestdat)))
-						 (action  . ,action)
-						 (target  . ,runkey)
+					       `((message  . ,(conc ruletype ":" (cadr youngestdat)))
+						 (action   . ,action)
+						 ;; (target   . ,runkey)
 						 (runtrans . ,runtrans)
-						 (runname . ,runname)
+						 (areas    . ,areas)
+						 (runname  . ,runname)
 						 ))))))
-		      ;; starttimes))
 
+		     ;; all globbed files must be newer than the reference
+		     ;;
 		     ((file-and) ;; all files must be newer than the reference
-		      (let* ((file-globs  (alist-ref 'glob val-alist))
-			     (youngestdat (common:get-youngest file-globs))
+		      (let* ((youngestdat (common:get-youngest file-globs))
 			     (youngestmod (car youngestdat))
 			     (success     #t)) ;; any cases of not true, set flag to #f for AND
 			;; (print "youngestmod: " youngestmod " starttimes: " starttimes)
 			(if (null? starttimes) ;; this target has never been run
 			    (push-run-spec torun contour runkey
-					   `((message . "file:neverrun")
-					     (runname . ,runname)
+					   `((message  . "file:neverrun")
+					     (runname  . ,runname)
 					     (runtrans . ,runtrans)
-					     (target  . ,runkey)
-					     (action  . ,action)))
+					     (areas    . ,areas)
+					     ;; (target   . ,runkey)
+					     (action   . ,action)))
 			    ;; NB// I think this is wrong. It should be looking at last-run only.
-			    (if (> youngestmod last-run)
+			    (if (> youngestmod last-run) ;; WAIT!! Shouldn't file-and be looking at the *oldest* file (thus all are younger than ...)
 				
 				;; 			    (for-each
 				;; 			     (lambda (starttime) ;; look at the time the last run was kicked off for this contour
 				;; 			       (if (< youngestmod (cdr starttime))
 				;; 				   (set! success #f)))
@@ -779,78 +859,89 @@
 				;; 			     starttimes))
 				;; 			(if success
 				;; 			    (begin
 				;; 			      (print "starttime younger than youngestmod: " starttime " Youngestmod: " youngestmod)
 				(push-run-spec torun contour runkey
-					       `((message . ,(conc ruletype ":" (cadr youngestdat)))
-						 (runname . ,runname)
+					       `((message  . ,(conc ruletype ":" (cadr youngestdat)))
+						 (runname  . ,runname)
 						 (runtrans . ,runtrans)
-						 (target  . ,runkey)
-						 (action  . ,action)
+						 ;; (target   . ,runkey)
+						 (areas    . ,areas)
+						 (action   . ,action)
 						 ))))))
 		     (else (print "ERROR: unrecognised rule \"" ruletype)))))
 	       keydats))) ;; sense rules
 	  (hash-table-keys rgconf))
 	 
 	 ;; now have to run populated
 	 (for-each
 	  (lambda (contour)
-	    (print "contour: " contour)
-	    (let* ((val       (or (configf:lookup mtconf "contours" contour) ""))
-		   (val-alist (val->alist val))
-		   (areas     (val-alist->areas val-alist))
-		   (selector  (alist-ref 'selector val-alist))
-		   (mode-tag  (and selector (string-split-fields "/" selector #:infix)))
-		   (mode-patt (and mode-tag (if (eq? (length mode-tag) 2)(cadr mode-tag) #f)))
-		   (tag-expr  (and mode-tag (if (null? mode-tag) #f (car mode-tag)))))
+	    (let* ((cval       (or (configf:lookup mtconf "contours" contour) ""))
+		   (cval-alist (val->alist cval))                     ;; BEWARE ... NOT the same val-alist as above!
+		   (areas      (val-alist->areas cval-alist))
+		   (selector   (alist-ref 'selector cval-alist))
+		   (mode-tag   (and selector (string-split-fields "/" selector #:infix)))
+		   (mode-patt  (and mode-tag (if (eq? (length mode-tag) 2)(cadr mode-tag) #f)))
+		   (tag-expr   (and mode-tag (if (null? mode-tag) #f (car mode-tag)))))
+	      (print "contour: " contour " areas=" areas " cval=" cval)
 	      (for-each
 	       (lambda (runkeydatset)
 		 ;; (print "runkeydatset: ")(pp runkeydatset)
 		 (let ((runkey     (car runkeydatset))
 		       (runkeydats (cadr runkeydatset)))
 		   (for-each
 		    (lambda (runkeydat)
 		      (for-each
 		       (lambda (area)
-                         (if (area-allowed? area areas) ;; is this area to be handled (from areas=a,b,c ...)
-                             (let ((runname (alist-ref 'runname runkeydat))
-                                   (runtrans (alist-ref 'runtrans runkeydat))
-                                   (reason  (alist-ref 'message runkeydat))
-                                   (sched   (alist-ref 'sched   runkeydat))
-                                   (action  (alist-ref 'action  runkeydat))
-                                   (dbdest  (alist-ref 'dbdest  runkeydat))
-                                   (append  (alist-ref 'append  runkeydat))
-                                   (target  (or (alist-ref 'target  runkeydat) runkey))) ;; override with target if forced
-                               (print "Have: runkey=" runkey " contour=" contour " area=" area " action=" action " tag-expr=" tag-expr " mode-patt=" mode-patt " target=" target)
-                               (if (case (or (and action (string->symbol action)) 'noaction)  ;; ensure we have the needed data to run this action
-                                     ((noaction) #f)
-                                     ((run)      (and runname reason))
-				 ((sync sync-prepend)     (and reason dbdest))
-                                     (else       #f))
-                                   ;; instead of unwrapping the runkeydat alist, pass it directly to create-run-pkt
-                                   (create-run-pkt mtconf action area runkey runname mode-patt tag-expr pktsdir reason contour sched dbdest append runtrans) 
-                                   (print "ERROR: Missing info to make a " action " call: runkey=" runkey " contour=" contour " area=" area  " tag-expr=" tag-expr " mode-patt=" mode-patt " dbdest=" dbdest)
-                                   ))
-                             (print "NOTE: skipping " runkeydat " for area, not in " areas)))
-		       all-areas))
+			 (if (area-allowed? area areas runkey contour) ;; is this area to be handled (from areas=a,b,c OR using areafn=abcfn and *area-checks* ...)
+                             (let* ((aval       (or (configf:lookup mtconf "areas" area) ""))
+                                    (aval-alist (val->alist aval))
+                                    (runname    (alist-ref 'runname runkeydat))
+                                    (runtrans   (alist-ref 'runtrans runkeydat))
+                                    
+                                    (reason     (alist-ref 'message runkeydat))
+                                    (sched      (alist-ref 'sched   runkeydat))
+                                    (action     (alist-ref 'action  runkeydat))
+                                    (dbdest     (alist-ref 'dbdest  runkeydat))
+                                    (append     (alist-ref 'append  runkeydat))
+                                    (targets    (or (alist-ref 'target  runkeydat)
+                                                    (map-targets mtconf aval-alist runkey area contour)))) ;; override with target if forced
+                               ;; NEED TO EXPAND RUNKEY => ALL TARGETS MAPPED AND THEN FOREACH .... 
+                               (for-each
+                                (lambda (target)
+                                  (print "Creating pkt for runkey=" runkey " target=" target " contour=" contour " area=" area " action=" action " tag-expr=" tag-expr " mode-patt=" mode-patt)
+                                  (if (case (or (and action (string->symbol action)) 'noaction)  ;; ensure we have the needed data to run this action
+                                        ((noaction)           #f)
+                                        ((run)                (and runname reason))
+                                        ((sync sync-prepend)  (and reason dbdest))
+                                        (else                 #f))
+                                      ;; instead of unwrapping the runkeydat alist, pass it directly to create-run-pkt
+                                      (create-run-pkt mtconf action area runkey target runname mode-patt
+                                                      tag-expr pktsdir reason contour sched dbdest append 
+                                                      runtrans) 
+                                      (print "ERROR: Missing info to make a " action " call: runkey=" runkey " contour=" contour " area=" area  " tag-expr=" tag-expr " mode-patt=" mode-patt " dbdest=" dbdest)
+                                      ))
+                                targets))
+                             (print "NOTE: skipping " runkeydat " for area \"" area "\", not in " areas)))
+                       all-areas))
 		    runkeydats)))
 	       (let ((res (configf:get-section torun contour))) ;; each contour / target
 		 ;; (print "res=" res)
 		 res))))
 	  (hash-table-keys torun)))))))
 
 (define (pkt->cmdline pkta)
-  (let ((action (or (lookup-action-by-key (alist-ref 'a pkta)) "noaction")))
+  (let ((action (or (lookup-action-by-key (alist-ref 'A pkta)) "noaction")))
     (fold (lambda (a res)
 	    (let* ((key (car a)) ;; get the key name
 		   (val (cdr a))
 		   (par (or (lookup-param-by-key key)  ;; need to check also if it is a switch
 			    (lookup-param-by-key key inlst: *switch-keys*))))
 	      ;; (print "key: " key " val: " val " par: " par)
 	      (if par
 		  (conc res " " (param-translate par) " " val)
-		  (if (member key '(a Z U D T)) ;; a is the action
+		  (if (alist-ref key *additional-cards*) ;; these cards do not translate to parameters or switches
 		      res
 		      (begin
 			(print "ERROR: Unknown key in packet \"" key "\" with value \"" val "\"")
 			res)))))
 	  (conc "megatest " (if (not (member action '("sync")))
@@ -893,11 +984,11 @@
 	      (torun     (make-hash-table)) ;; target => ( ... info ... )
 	      (rgentargs (hash-table-keys rgconf))) ;; these are the targets registered for automatically triggering
 	 (for-each
 	  (lambda (pktdat)
 	    (let* ((pkta    (alist-ref 'apkt pktdat))
-		   (action  (alist-ref 'a pkta))
+		   (action  (alist-ref 'A pkta))
 		   (cmdline (pkt->cmdline pkta))
 		   (uuid    (alist-ref 'Z pkta))
 		   (logf    (conc logdir "/" uuid "-run.log"))
 		   (fullcmd (conc "NBFAKE_LOG=" logf " nbfake " cmdline)))
 	      (print "RUNNING: " fullcmd)

Index: process.scm
==================================================================
--- process.scm
+++ process.scm
@@ -13,11 +13,10 @@
 ;; Process convience utils
 ;;======================================================================
 
 (use regex)
 (declare (unit process))
-;;(declare (uses common))
 
 (define (process:conservative-read port)
   (let loop ((res ""))
     (if (not (eof-object? (peek-char port)))
 	(loop (conc res (read-char port)))

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -11,13 +11,11 @@
 
 (use format typed-records) ;; RADT => purpose of json format??
 
 (declare (unit rmt))
 (declare (uses api))
-(declare (uses tdb))
 (declare (uses http-transport))
-;;(declare (uses nmsg-transport))
 (include "common_records.scm")
 
 ;;
 ;; THESE ARE ALL CALLED ON THE CLIENT SIDE!!!
 ;;
@@ -275,11 +273,11 @@
     res))
 
 (define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
   (let* ((qry-is-write   (not (member cmd api:read-only-queries)))
 	 (db-file-path   (db:dbfile-path)) ;;  0))
-	 (dbstruct-local (db:setup))  ;; make-dbr:dbstruct path:  dbdir local: #t)))
+	 (dbstruct-local (db:setup #t))  ;; make-dbr:dbstruct path:  dbdir local: #t)))
 	 (read-only      (not (file-write-access? db-file-path)))
 	 (start          (current-milliseconds))
 	 (resdat         (if (not (and read-only qry-is-write))
 			     (let ((v (api:execute-requests dbstruct-local (vector (symbol->string cmd) params))))
 			       (handle-exceptions ;; there has been a long history of receiving strange errors from values returned by the client when things go wrong..
@@ -755,14 +753,10 @@
 ;;  T E S T   D A T A 
 ;;======================================================================
 
 (define (rmt:read-test-data run-id test-id categorypatt #!key (work-area #f)) 
   (rmt:send-receive 'read-test-data run-id (list run-id test-id categorypatt)))
-;;   (let ((tdb  (rmt:open-test-db-by-test-id run-id test-id work-area: work-area)))
-;;     (if tdb
-;; 	(tdb:read-test-data tdb test-id categorypatt)
-;; 	'())))
 
 (define (rmt:testmeta-add-record testname)
   (rmt:send-receive 'testmeta-add-record #f (list testname)))
 
 (define (rmt:testmeta-get-record testname)

Index: runconfigs.config
==================================================================
--- runconfigs.config
+++ runconfigs.config
@@ -26,13 +26,13 @@
 # file:   files changes since last run trigger new run
 # script: script is called with unix seconds as last parameter (other parameters are preserved)
 #
 # contour:sensetype:action params            data
 # commented out for debug
-# quick:file:run             runtrans=auto;         glob=/home/matt/data/megatest/*.scm
+quick:file:run             runtrans=auto;         glob=/home/matt/data/megatest/*.scm foo.touchme
 # snazy:file:run             runtrans=corporate-ww; glob=/home/matt/data/megatest/*.scm
-short:file:run             runtrans=short;        glob=/home/matt/data/megatest/*.scm
+# short:file:run       runtrans=short;        glob=/home/matt/data/megatest/*.scm
 
 # script returns change-time (unix epoch), new-target-name, run-name
 #
 # quick:script:run           checkfossil = http://www.kiatoa.com/fossils/megatest v1.63;\
 #                            checkfossil = http://www.kiatoa.com/fossils/megatest_qa trunk

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -8,13 +8,12 @@
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 ;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) 
+(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (srfi 18) 
      posix-extras directory-utils pathname-expand typed-records format)
-(import (prefix sqlite3 sqlite3:))
 
 (declare (unit runs))
 (declare (uses db))
 (declare (uses common))
 (declare (uses items))

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -1,7 +1,7 @@
 
-;; Copyright 2006-2012, Matthew Welland.
+;; Copyright 2006-2017, Matthew Welland.
 ;; 
 ;;  This program is made available under the GNU GPL version 2.0 or
 ;;  greater. See the accompanying file COPYING for details.
 ;; 
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
@@ -9,11 +9,10 @@
 ;;  PURPOSE.
 
 (require-extension (srfi 18) extras tcp s11n)
 
 (use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest directory-utils posix-extras matchable)
-;; (use zmq)
 
 (use spiffy uri-common intarweb http-client spiffy-request-vars)
 
 (declare (unit server))
 
@@ -20,12 +19,10 @@
 (declare (uses common))
 (declare (uses db))
 (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
 (declare (uses synchash))
 (declare (uses http-transport))
-(declare (uses rpc-transport))
-;;(declare (uses nmsg-transport))
 (declare (uses launch))
 (declare (uses daemon))
 
 (include "common_records.scm")
 (include "db_records.scm")
@@ -435,5 +432,18 @@
 	;; (* 3 24 60 60) ;; default to three days
 	;;(* 60 60 1)     ;; default to one hour
 	(* 60 5)          ;; default to five minutes
 	)))
 
+(define (server:get-best-guess-address hostname)
+  (let ((res #f))
+    (for-each 
+     (lambda (adr)
+       (if (not (eq? (u8vector-ref adr 0) 127))
+	   (set! res adr)))
+     ;; NOTE: This can fail when there is no mention of the host in /etc/hosts. FIXME
+     (vector->list (hostinfo-addresses (hostname->hostinfo hostname))))
+    (string-intersperse 
+     (map number->string
+	  (u8vector->list
+	   (if res res (hostname->ip hostname)))) ".")))
+

Index: tests/fullrun/runconfigs.config
==================================================================
--- tests/fullrun/runconfigs.config
+++ tests/fullrun/runconfigs.config
@@ -1,10 +1,10 @@
 [default]
 SOMEVAR This should show up in SOMEVAR3
 VARNOVAL
 VARNOVAL_WITHSPACE
-QUICK %
+QUICKPATT test_mt_vars,test2,priority_9
 
 # target based getting of config file, look at afs.config and nfs.config
 [include #{getenv fsname}.config]
 
 [include #{getenv MT_RUN_AREA_HOME}/common_runconfigs.config]

Index: utils/mk_wrapper
==================================================================
--- utils/mk_wrapper
+++ utils/mk_wrapper
@@ -1,13 +1,13 @@
 #!/bin/bash
 
 prefix=$1
 cmd=$2
 target=$3
+cfgfile="$prefix/bin/.$(lsb_release -sr)/cfg.sh"
 
 if [ "$LD_LIBRARY_PATH" != "" ];then
-  cfgfile="$prefix/bin/.$(lsb_release -sr)/cfg.sh"
   echo "INFO: Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >&2
 ( cat << __EOF
 if [ "\$LD_LIBRARY_PATH" != "" ];then
   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\$LD_LIBRARY_PATH
 else
@@ -18,15 +18,10 @@
   echo 
 else
   echo "INFO: LD_LIBRARY_PATH not set" >&2
 fi
 
-# echo "#!/bin/bash" > $target
-# if [ "$LD_LIBRARY_PATH" != "" ];then
-#   echo "source $prefix/bin/.\$(lsb_release -sr)/cfg.sh" >> $target
-# fi
-# echo "exec $prefix/bin/.\$(lsb_release -sr)/$cmd \"\$@\"" >> $target
 echo "#!/bin/bash" > $target
 
 if [[ $cmd =~ dboard ]]; then
     cat >> $target <<'EOF'
 
@@ -57,8 +52,15 @@
 fi
 EOF
 
 fi
 
+# echo "#!/bin/bash" > $target
+# echo "exec $prefix/bin/.\$(lsb_release -sr)/$cmd \"\$@\"" >> $target
+
 echo "lsbr=\$(lsb_release -sr)" >> $target
-echo "if [[ -e \$lsbr ]];then source \$lsbr;fi" >> $target
+if [ "$LD_LIBRARY_PATH" != "" ];then
+  echo "source $prefix/bin/.\$lsbr/cfg.sh" >> $target
+fi
+
+# echo "if [[ -e \$lsbr ]];then source \$lsbr;fi" >> $target
 echo "exec $prefix/bin/.\$lsbr/$cmd \"\$@\"" >> $target