Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -48,21 +48,14 @@
   (let ((ok (and (socket? serverdat)
 		 (cdb:logout serverdat *toppath* (client:get-signature)))))
     ok))
 
 (define (client:connect iface port)
-  (case (server:get-transport)
-    ((rpc)  (rpc:client-connect  iface port))
-    ((http) (http:client-connect iface port))
-    ((zmq)  (zmq:client-connect  iface port))
-    (else   (rpc:client-connect  iface port))))
+  (http:client-connect iface port))
 
 (define (client:setup areapath #!key (remaining-tries 100) (failed-connects 0))
-  (case (server:get-transport)
-    ((rpc) (rpc-transport:client-setup remaining-tries: remaining-tries failed-connects: failed-connects)) ;;(client:setup-rpc run-id))
-    ((http)(client:setup-http *alldat* areapath remaining-tries: remaining-tries failed-connects: failed-connects))
-    (else  (rpc-transport:client-setup remaining-tries: remaining-tries failed-connects: failed-connects)))) ;; (client:setup-rpc run-id))))
+  (client:setup-http *alldat* areapath remaining-tries: remaining-tries failed-connects: failed-connects))
 
 (set-fn 'client:setup client:setup)
 
 
 ;; Do all the connection work, look up the transport type and set up the

Index: common_records.scm
==================================================================
--- common_records.scm
+++ common_records.scm
@@ -205,13 +205,16 @@
   (mtconfig          #f)
   (log-port          #f)
   (areadat           #f) ;; i.e. runremote
   (rmt-mutex         (make-mutex))
   (db-sync-mutex     (make-mutex))
+  (db-with-db-mutex  (make-mutex))
   (read-only-queries api:read-only-queries)
   (write-queries     api:write-queries)
-
+  (max-api-process-requests 0)
+  (api-process-request-count 0)
+  
   ;; database related
   (tmppath           #f) ;; tmp path for dbs
 
   ;; runremote fields
   (hh-dat            #f) ;; (exec-fn 'common:get-homehost)) ;; homehost record ( addr . hhflag )
@@ -222,10 +225,20 @@
   (server-timeout    #f) ;; (exec-fn 'server:expiration-timeout))
   (force-server      #f)
   (ro-mode           #f)  
   (ro-mode-checked   #f) ;; flag that indicates we have checked for ro-mode
   (ulex:conn         #f) ;; ulex db conn is not exactly a db connector, more like a network connector 
+
+  ;; dbstruct
+  (tmpdb       #f)
+  (dbstack     #f) ;; stack for tmp db handles, do not initialize with a stack
+  (mtdb        #f)
+  (refndb      #f)
+  (homehost    #f) ;; not used yet
+  (on-homehost #f) ;; not used yet
+  (read-only   #f)
+
   )
 
 (define *alldat* (make-alldat))
 
 ;; Some of these routines use:

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -57,19 +57,21 @@
 ;;======================================================================
 
 ;; each db entry is a pair ( db . dbfilepath )
 ;; I propose this record evolves into the area record
 ;;
-(defstruct dbr:dbstruct 
-  (tmpdb       #f)
-  (dbstack     #f) ;; stack for tmp db handles, do not initialize with a stack
-  (mtdb        #f)
-  (refndb      #f)
-  (homehost    #f) ;; not used yet
-  (on-homehost #f) ;; not used yet
-  (read-only   #f)
-  )                ;; goal is to converge on one struct for an area but for now it is too confusing
+;; MERGED INTO *alldat*
+;;
+;; (defstruct dbr:dbstruct 
+;;   (tmpdb       #f)
+;;   (dbstack     #f) ;; stack for tmp db handles, do not initialize with a stack
+;;   (mtdb        #f)
+;;   (refndb      #f)
+;;   (homehost    #f) ;; not used yet
+;;   (on-homehost #f) ;; not used yet
+;;   (read-only   #f)
+;;   )                ;; goal is to converge on one struct for an area but for now it is too confusing
   
 
 ;; record for keeping state,status and count for doing roll-ups in
 ;; iterated tests
 ;;
@@ -102,26 +104,10 @@
 	   (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
 	   (print-call-chain (current-error-port))
 	   default)))
    (apply sqlite3:first-result db stmt params)))
 
-;; Get/open a database
-;;    if run-id => get run specific db
-;;    if #f     => get main db
-;;    if db already open - return inmem
-;;    if db not open, open inmem, rundb and sync then return inmem
-;;    inuse gets set automatically for rundb's
-;;
-(define (db:get-db dbstruct) ;;  run-id) 
-  (if (stack? (dbr:dbstruct-dbstack dbstruct))
-      (if (stack-empty? (dbr:dbstruct-dbstack dbstruct))
-          (let ((newdb (db:open-megatest-db path: (common:get-db-tmp-area *alldat*))))
-            ;; (stack-push! (dbr:dbstruct-dbstack dbstruct) newdb)
-            newdb)
-          (stack-pop! (dbr:dbstruct-dbstack dbstruct)))
-      (db:open-db dbstruct)))
-
 ;; mod-read:
 ;;     'mod   modified data
 ;;     'read  read data
 ;; Locks the mutex and depending on 'mod or 'read passed, sets the last timestamp in dbstruct
 ;;
@@ -133,41 +119,10 @@
 ;; 	    (dbr:dbstruct-mtime-set! dbstruct (current-milliseconds))
 ;; 	    (dbr:dbstruct-rtime-set! dbstruct (current-milliseconds)))
 ;; 	(dbr:dbstruct-inuse-set! dbstruct #f)
 ;; 	(mutex-unlock! *rundb-mutex*))))
 
-;; (db:with-db dbstruct run-id sqlite3:exec "select blah fgrom blaz;")
-;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
-;;
-(define (db:with-db dbstruct run-id r/w proc . params)
-  (let* ((have-struct (dbr:dbstruct? dbstruct))
-         (dbdat (if have-struct 
-                    (db:get-db dbstruct)
-                    #f))
-	 (db    (if have-struct
-		    (db:dbdat-get-db dbdat)
-		    dbstruct))
-	 (use-mutex (> *api-process-request-count* 25)))
-    (if (and use-mutex
-	     (common:low-noise-print 120 "over-50-parallel-api-requests"))
-	(debug:print-info 0 *default-log-port* *api-process-request-count* " parallel api requests being processed in process " (current-process-id) ", throttling access"))
-    (if (common:low-noise-print 600 (conc "parallel-api-requests" *max-api-process-requests*))
-	(debug:print-info 2 *default-log-port* "Parallel api request count: " *api-process-request-count* " max parallel requests: " *max-api-process-requests*))
-    (handle-exceptions
-     exn
-     (begin
-       (print-call-chain (current-error-port))
-       (debug:print-error 0 *default-log-port* "sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn))
-       ;; there is no recovering at this time. exit
-       (exit 50))
-     (if use-mutex (mutex-lock! *db-with-db-mutex*))
-     (let ((res (apply proc db params)))
-       (if use-mutex (mutex-unlock! *db-with-db-mutex*))
-       ;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
-       (if dbdat (stack-push! (dbr:dbstruct-dbstack dbstruct) dbdat))
-       res))))
-
 ;;======================================================================
 ;; K E E P   F I L E D B   I N   dbstruct
 ;;======================================================================
 
 ;; (define (db:get-filedb dbstruct run-id)
@@ -273,89 +228,10 @@
          (exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
          (exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
 	)))
 
 
-;; This routine creates the db if not already present. It is only called if the db is not already opened
-;;
-(define (db:open-db dbstruct #!key (areapath #f)(do-sync #t)) ;; TODO: actually use areapath
-  (let ((tmpdb-stack (dbr:dbstruct-dbstack dbstruct))) ;; RA => Returns the first reference in dbstruct
-    (if (stack? tmpdb-stack)
-	(db:get-db tmpdb-stack) ;; get previously opened db (will create new db handle if all in the stack are already used
-        (let* ((max-stale-tmp (configf:lookup-number *configdat* "server" "filling-db-max-stale-seconds" default: 10))
-               (dbpath       (common:get-db-tmp-area *alldat* ))      ;; path to tmp db area
-               (dbexists     (common:file-exists? dbpath))
-	       (tmpdbfname   (conc dbpath "/megatest.db"))
-	       (dbfexists    (common:file-exists? tmpdbfname))  ;; (conc dbpath "/megatest.db")))
-               (mtdbexists   (common:file-exists? (conc *toppath* "/megatest.db")))
-							 
-               (mtdbmodtime  (if mtdbexists (common:lazy-sqlite-db-modification-time (conc *toppath* "/megatest.db"))   #f))
-	        		 (tmpdbmodtime (if dbfexists  (common:lazy-sqlite-db-modification-time tmpdbfname) #f)) 
-               (mtdb         (db:open-megatest-db))
-               (mtdbpath     (db:dbdat-get-path mtdb))
-               (tmpdb        (db:open-megatest-db path: dbpath)) ;; lock-create-open dbpath db:initialize-main-db))
-               (refndb       (db:open-megatest-db path: dbpath name: "megatest_ref.db"))
-               (write-access (file-write-access? mtdbpath))
-	       
-	       ;;(mtdbmodtime (if mtdbexists
-	       ;;(common:lazy-sqlite-db-modification-time mtdbpath)
-	       ;;#f)) ; moving this before db:open-megatest-db is
-	       ;;called. if wal mode is on -WAL and -shm file get
-	       ;;created with causing the tmpdbmodtime timestamp
-	       ;;always greater than mtdbmodtime (tmpdbmodtime (if
-	       ;;dbfexists (common:lazy-sqlite-db-modification-time
-	       ;;tmpdbfname) #f))
-
-	       ;;if wal mode is on -WAL and -shm file get created when
-	       ;;db:open-megatest-db is called. modtimedelta will
-	       ;;always be < 10 so db in tmp not get synced
-	       ;;(tmpdbmodtime (if dbfexists (db:get-last-update-time
-	       ;;(car tmpdb)) #f)) (fmt (file-modification-time
-	       ;;tmpdbfname))
-	       
-	       (modtimedelta (and mtdbmodtime tmpdbmodtime (- mtdbmodtime tmpdbmodtime))))
-
-	  (handle-exceptions
-	   exn
-	   (let ((call-chain (get-call-chain))
-		 (msg        ((condition-property-accessor 'exn 'message) exn)))
-	     (debug:print 0 *default-log-port* "ERROR: attempted to drop triggers on MTRA/megatest.db but failed. Error is " msg)
-	     (set! write-access #f)) ;; if we failed to drop the triggers then we probably don't have write access
-	   (when write-access
-		 (sqlite3:execute (car mtdb) "drop trigger if exists update_tests_trigger")
-		 (sqlite3:execute (car mtdb) "drop trigger if exists update_runs_trigger")))
-          
-          ;;(print "mtdbmodtime " mtdbmodtime " tmpdbmodtime "
-          ;;tmpdbmodtime " mtdbpath " mtdbpath " " (conc *toppath*
-          ;;"/megatest.db")) (debug:print-info 13 *default-log-port*
-          ;;"db:open-db>> mtdbpath="mtdbpath" mtdbexists="mtdbexists"
-          ;;and write-access="write-access)
-          (if (and dbexists (not write-access))
-              (begin
-                (set! *db-write-access* #f)
-                (dbr:dbstruct-read-only-set! dbstruct #t)))
-          (dbr:dbstruct-mtdb-set!   dbstruct mtdb)
-          (dbr:dbstruct-tmpdb-set!  dbstruct tmpdb)
-          (dbr:dbstruct-dbstack-set! dbstruct (make-stack))   ;; why a stack?
-          (stack-push! (dbr:dbstruct-dbstack dbstruct) tmpdb) ;; olddb is already a (cons db path)
-          (dbr:dbstruct-refndb-set! dbstruct refndb)
-          ;;	    (mutex-unlock! *rundb-mutex*)
-          (if (and  (or (not dbfexists)
-			(and modtimedelta
-			     (> modtimedelta max-stale-tmp))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
-		    do-sync)
-	      (begin
-		(debug:print 1 *default-log-port* "filling db " (db:dbdat-get-path tmpdb) " with data \n    from " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta)
-		(db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb refndb tmpdb)
-					;touch tmp db to avoid wal mode wierdness  
-		(set! (file-modification-time tmpdbfname) (current-seconds))  
-                (debug:print-info 13 *default-log-port* "db:sync-all-tables-list done.")
-                )
-	      (debug:print 4 *default-log-port* " db, " (db:dbdat-get-path tmpdb) " already exists or fresh enough, not propogating data from\n     " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta) )
-	  ;; (db:multi-db-sync dbstruct 'old2new))  ;; migrate data from megatest.db automatically
-          tmpdb))))
-
 
 (define (db:get-last-update-time db)
   (let ((last-update-time #f))
     (sqlite3:for-each-row 
      (lambda (lup) 
@@ -362,34 +238,10 @@
        (set! last-update-time lup))     
      db    
      "select max(lup) from ( select max(last_update) as lup  from tests union select max(last_update) as lup from runs);")
     last-update-time))
 
-;; Make the dbstruct, setup up auxillary db's and call for main db at least once
-;;
-;; called in http-transport and replicated in rmt.scm for *local* access. 
-;;
-(define (db:setup do-sync #!key (areapath #f))
-  ;;
-  (cond
-   (*dbstruct-db* *dbstruct-db*);; TODO: when multiple areas are supported, this optimization will be a hazard
-   (else ;;(common:on-homehost?)
-    (debug:print-info 13 *default-log-port* "db:setup entered (first time, not cached.)")
-    (let* ((dbstruct (make-dbr:dbstruct)))
-      (when (not *toppath*)
-        (debug:print-info 13 *default-log-port* "in db:setup, *toppath* not set; calling launch:setup")
-        (launch:setup areapath: areapath))
-      (debug:print-info 13 *default-log-port* "Begin db:open-db")
-      (db:open-db dbstruct areapath: areapath do-sync: do-sync)
-      (debug:print-info 13 *default-log-port* "Done db:open-db")
-      (set! *dbstruct-db* dbstruct)
-      ;;(debug:print-info 13 *default-log-port* "new dbstruct = "(dbr:dbstruct->alist dbstruct))
-      dbstruct))))
-   ;; (else
-   ;;  (debug:print 0 *default-log-port* "ERROR: attempt to open database when not on homehost. Exiting. Homehost: " (common:get-homehost))
-   ;;  (exit 1))))
-
 ;; Open the classic megatest.db file (defaults to open in toppath)
 ;;
 ;;   NOTE: returns a dbdat not a dbstruct!
 ;;
 
@@ -475,93 +327,10 @@
 ;; 	 (handler (make-busy-timeout 3600)))
 ;;     (sqlite3:set-busy-handler! db handler)
 ;;     (db:initialize-run-id-db db)
 ;;     (cons db #f)))
 
-;; just tests, test_steps and test_data tables
-(define db:sync-tests-only
-  (list
-   ;; (list "strs"
-   ;;       '("id"             #f)
-   ;;       '("str"            #f))
-   (list "tests" 
-	 '("id"             #f)
-	 '("run_id"         #f)
-	 '("testname"       #f)
-	 '("host"           #f)
-	 '("cpuload"        #f)
-	 '("diskfree"       #f)
-	 '("uname"          #f)
-	 '("rundir"         #f)
-	 '("shortdir"       #f)
-	 '("item_path"      #f)
-	 '("state"          #f)
-	 '("status"         #f)
-	 '("attemptnum"     #f)
-	 '("final_logf"     #f)
-	 '("logdat"         #f)
-	 '("run_duration"   #f)
-	 '("comment"        #f)
-	 '("event_time"     #f)
-	 '("fail_count"     #f)
-	 '("pass_count"     #f)
-	 '("archived"       #f)
-         '("last_update"    #f))
-  (list "test_steps"
-	 '("id"             #f)
-	 '("test_id"        #f)
-	 '("stepname"       #f)
-	 '("state"          #f)
-	 '("status"         #f)
-	 '("event_time"     #f)
-	 '("comment"        #f)
-	 '("logfile"        #f)
-         '("last_update"    #f))
-   (list "test_data"
-	 '("id"             #f)
-	 '("test_id"        #f)
-	 '("category"       #f)
-	 '("variable"       #f)
-	 '("value"          #f)
-	 '("expected"       #f)
-	 '("tol"            #f)
-	 '("units"          #f)
-	 '("comment"        #f)
-	 '("status"         #f)
-	 '("type"           #f)
-         '("last_update"    #f))))
-
-;; needs db to get keys, this is for syncing all tables
-;;
-(define (db:sync-main-list dbstruct)
-  (let ((keys  (db:get-keys dbstruct)))
-    (list
-     (list "keys"
-	   '("id"        #f)
-	   '("fieldname" #f)
-	   '("fieldtype" #f))
-     (list "metadat" '("var" #f) '("val" #f))
-     (append (list "runs" 
-		   '("id"  #f))
-	     (map (lambda (k)(list k #f))
-		  (append keys
-			  (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count" "contour" "last_update"))))
-     (list "test_meta"
-	   '("id"             #f)
-	   '("testname"       #f)
-	   '("owner"          #f)
-	   '("description"    #f)
-	   '("reviewed"       #f)
-	   '("iterated"       #f)
-	   '("avg_runtime"    #f)
-	   '("avg_disk"       #f)
-	   '("tags"           #f)
-	   '("jobgroup"       #f)))))
-
-(define (db:sync-all-tables-list dbstruct)
-  (append (db:sync-main-list dbstruct)
-	  db:sync-tests-only))
 
 ;; use bunch of Unix commands to try to break the lock and recreate the db
 ;;
 (define (db:move-and-recreate-db dbdat)
   (let* ((dbpath   (db:dbdat-get-path        dbdat))
@@ -2019,26 +1788,10 @@
 
 ;; use a global for some primitive caching, it is just silly to
 ;; re-read the db over and over again for the keys since they never
 ;; change
 
-;; why get the keys from the db? why not get from the *configdat*
-;; using keys:config-get-fields?
-
-(define (db:get-keys dbstruct)
-  (if *db-keys* *db-keys* 
-      (let ((res '()))
-	(db:with-db dbstruct #f #f
-		    (lambda (db)
-		      (sqlite3:for-each-row 
-		       (lambda (key)
-			 (set! res (cons key res)))
-		       db
-		       "SELECT fieldname FROM keys ORDER BY id DESC;")))
-	(set! *db-keys* res)
-	res)))
-
 ;; look up values in a header/data structure
 (define (db:get-value-by-header row header field)
   (if (or (null? header) (not row))
       #f
       (let loop ((hed (car header))
@@ -4759,6 +4512,6 @@
 
 ;; (db:extract-ods-file db "outputfile.ods" '(("sysname" "%")("fsname" "%")("datapath" "%")) "%")
 
 ;; tiresome setup for rmtmod (and other mods) goes here
 ;; (set-fn 'db:dbfile-path common:get-db-tmp-area)
-(set-fn 'db:setup       db:setup)
+(set-fn 'db:setup       dbmod#db:setup)

Index: dbmod.scm
==================================================================
--- dbmod.scm
+++ dbmod.scm
@@ -23,11 +23,11 @@
 
 (module dbmod
 	*
 	
 (import scheme chicken data-structures extras)
-(import (prefix sqlite3 sqlite3:) posix typed-records srfi-18 srfi-69 format ports srfi-1 matchable)
+(import (prefix sqlite3 sqlite3:) posix typed-records srfi-18 srfi-69 format ports srfi-1 matchable stack)
 (import commonmod)
 ;; (use (prefix ulex ulex:))
 
 (include "common_records.scm")
 
@@ -34,8 +34,255 @@
 ;; ;; legacy handling of structure for managing db's. Refactor this into dbr:?
 (define (db:dbdat-get-db dbdat)
   (if (pair? dbdat)
       (car dbdat)
       dbdat))
+
+;; Make the dbstruct, setup up auxillary db's and call for main db at least once
+;;
+;; called in http-transport and replicated in rmt.scm for *local* access. 
+;;
+(define (db:setup do-sync alldat #!key (areapath #f))
+  (let* ((log-port (alldat-log-port alldat)))
+    (cond
+     ((alldat-dbstack alldat) alldat) ;; already initialized
+     ((not (alldat-areapath alldat))  ;; no top path yet? Just exit
+      (debug:print-info 13 log-port "in db:setup, area-path not set; give up and exit.")
+      (exit 1))
+     (else ;;(common:on-homehost?)
+      (debug:print-info 13 log-port "db:setup entered (first time, not cached.)")
+      (debug:print-info 13 log-port "Begin db:open-db")
+      (db:open-db alldat areapath: areapath do-sync: do-sync)
+      (debug:print-info 13 log-port "Done db:open-db")
+      ;; (set! *dbstruct-db* dbstruct)
+      alldat))))
+
+;; This routine creates the db if not already present. It is only called if the db is not already opened
+;;
+(define (db:open-db alldat #!key (areapath #f)(do-sync #t)) ;; TODO: actually use areapath
+  (let ((tmpdb-stack (alldat-dbstack alldat))) ;; RA => Returns the first reference in alldat
+    (if (stack? tmpdb-stack)
+	(db:get-db tmpdb-stack) ;; get previously opened db (will create new db handle if all in the stack are already used
+        (let* ((max-stale-tmp (configf:lookup-number *configdat* "server" "filling-db-max-stale-seconds" default: 10))
+               (dbpath       (common:get-db-tmp-area alldat))      ;; path to tmp db area
+               (dbexists     (common:file-exists? dbpath))
+	       (tmpdbfname   (conc dbpath "/megatest.db"))
+	       (dbfexists    (common:file-exists? tmpdbfname))  ;; (conc dbpath "/megatest.db")))
+               (mtdbexists   (common:file-exists? (conc *toppath* "/megatest.db")))
+							 
+               (mtdbmodtime  (if mtdbexists (common:lazy-sqlite-db-modification-time (conc *toppath* "/megatest.db"))   #f))
+	        		 (tmpdbmodtime (if dbfexists  (common:lazy-sqlite-db-modification-time tmpdbfname) #f)) 
+               (mtdb         (db:open-megatest-db))
+               (mtdbpath     (db:dbdat-get-path mtdb))
+               (tmpdb        (db:open-megatest-db path: dbpath)) ;; lock-create-open dbpath db:initialize-main-db))
+               (refndb       (db:open-megatest-db path: dbpath name: "megatest_ref.db"))
+               (write-access (file-write-access? mtdbpath))
+	       
+	       ;;(mtdbmodtime (if mtdbexists
+	       ;;(common:lazy-sqlite-db-modification-time mtdbpath)
+	       ;;#f)) ; moving this before db:open-megatest-db is
+	       ;;called. if wal mode is on -WAL and -shm file get
+	       ;;created with causing the tmpdbmodtime timestamp
+	       ;;always greater than mtdbmodtime (tmpdbmodtime (if
+	       ;;dbfexists (common:lazy-sqlite-db-modification-time
+	       ;;tmpdbfname) #f))
+
+	       ;;if wal mode is on -WAL and -shm file get created when
+	       ;;db:open-megatest-db is called. modtimedelta will
+	       ;;always be < 10 so db in tmp not get synced
+	       ;;(tmpdbmodtime (if dbfexists (db:get-last-update-time
+	       ;;(car tmpdb)) #f)) (fmt (file-modification-time
+	       ;;tmpdbfname))
+	       
+	       (modtimedelta (and mtdbmodtime tmpdbmodtime (- mtdbmodtime tmpdbmodtime))))
+
+	  (handle-exceptions
+	   exn
+	   (let ((call-chain (get-call-chain))
+		 (msg        ((condition-property-accessor 'exn 'message) exn)))
+	     (debug:print 0 log-port "ERROR: attempted to drop triggers on MTRA/megatest.db but failed. Error is " msg)
+	     (set! write-access #f)) ;; if we failed to drop the triggers then we probably don't have write access
+	   (when write-access
+		 (sqlite3:execute (car mtdb) "drop trigger if exists update_tests_trigger")
+		 (sqlite3:execute (car mtdb) "drop trigger if exists update_runs_trigger")))
+          
+          ;;(print "mtdbmodtime " mtdbmodtime " tmpdbmodtime "
+          ;;tmpdbmodtime " mtdbpath " mtdbpath " " (conc *toppath*
+          ;;"/megatest.db")) (debug:print-info 13 log-port
+          ;;"db:open-db>> mtdbpath="mtdbpath" mtdbexists="mtdbexists"
+          ;;and write-access="write-access)
+          (if (and dbexists (not write-access))
+              (begin
+                (set! *db-write-access* #f)
+                (alldat-read-only-set! alldat #t)))
+          (alldat-mtdb-set!   alldat mtdb)
+          (alldat-tmpdb-set!  alldat tmpdb)
+          (alldat-dbstack-set! alldat (make-stack))   ;; why a stack?
+          (stack-push! (alldat-dbstack alldat) tmpdb) ;; olddb is already a (cons db path)
+          (alldat-refndb-set! alldat refndb)
+          ;;	    (mutex-unlock! *rundb-mutex*)
+          (if (and  (or (not dbfexists)
+			(and modtimedelta
+			     (> modtimedelta max-stale-tmp))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
+		    do-sync)
+	      (begin
+		(debug:print 1 log-port "filling db " (db:dbdat-get-path tmpdb) " with data \n    from " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta)
+		(db:sync-tables (db:sync-all-tables-list alldat) #f mtdb refndb tmpdb)
+					;touch tmp db to avoid wal mode wierdness  
+		(set! (file-modification-time tmpdbfname) (current-seconds))  
+                (debug:print-info 13 log-port "db:sync-all-tables-list done.")
+                )
+	      (debug:print 4 log-port " db, " (db:dbdat-get-path tmpdb) " already exists or fresh enough, not propogating data from\n     " (db:dbdat-get-path mtdb) " mod time delta: " modtimedelta) )
+	  ;; (db:multi-db-sync alldat 'old2new))  ;; migrate data from megatest.db automatically
+          tmpdb))))
+
+;; Get/open a database
+;;    if run-id => get run specific db
+;;    if #f     => get main db
+;;    if db already open - return inmem
+;;    if db not open, open inmem, rundb and sync then return inmem
+;;    inuse gets set automatically for rundb's
+;;
+(define (db:get-db alldat) ;;  run-id) 
+  (if (stack? (alldat-dbstack alldat))
+      (if (stack-empty? (alldat-dbstack alldat))
+          (let ((newdb (db:open-megatest-db path: (common:get-db-tmp-area alldat))))
+            ;; (stack-push! (alldat-dbstack alldat) newdb)
+            newdb)
+          (stack-pop! (alldat-dbstack alldat)))
+      (db:open-db alldat)))
+
+(define (db:sync-all-tables-list alldat)
+  (append (db:sync-main-list alldat)
+	  db:sync-tests-only))
+
+;; just tests, test_steps and test_data tables
+(define db:sync-tests-only
+  (list
+   ;; (list "strs"
+   ;;       '("id"             #f)
+   ;;       '("str"            #f))
+   (list "tests" 
+	 '("id"             #f)
+	 '("run_id"         #f)
+	 '("testname"       #f)
+	 '("host"           #f)
+	 '("cpuload"        #f)
+	 '("diskfree"       #f)
+	 '("uname"          #f)
+	 '("rundir"         #f)
+	 '("shortdir"       #f)
+	 '("item_path"      #f)
+	 '("state"          #f)
+	 '("status"         #f)
+	 '("attemptnum"     #f)
+	 '("final_logf"     #f)
+	 '("logdat"         #f)
+	 '("run_duration"   #f)
+	 '("comment"        #f)
+	 '("event_time"     #f)
+	 '("fail_count"     #f)
+	 '("pass_count"     #f)
+	 '("archived"       #f)
+         '("last_update"    #f))
+  (list "test_steps"
+	 '("id"             #f)
+	 '("test_id"        #f)
+	 '("stepname"       #f)
+	 '("state"          #f)
+	 '("status"         #f)
+	 '("event_time"     #f)
+	 '("comment"        #f)
+	 '("logfile"        #f)
+         '("last_update"    #f))
+   (list "test_data"
+	 '("id"             #f)
+	 '("test_id"        #f)
+	 '("category"       #f)
+	 '("variable"       #f)
+	 '("value"          #f)
+	 '("expected"       #f)
+	 '("tol"            #f)
+	 '("units"          #f)
+	 '("comment"        #f)
+	 '("status"         #f)
+	 '("type"           #f)
+         '("last_update"    #f))))
+
+;; needs db to get keys, this is for syncing all tables
+;;
+(define (db:sync-main-list alldat)
+  (let ((keys  (db:get-keys alldat)))
+    (list
+     (list "keys"
+	   '("id"        #f)
+	   '("fieldname" #f)
+	   '("fieldtype" #f))
+     (list "metadat" '("var" #f) '("val" #f))
+     (append (list "runs" 
+		   '("id"  #f))
+	     (map (lambda (k)(list k #f))
+		  (append keys
+			  (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count" "contour" "last_update"))))
+     (list "test_meta"
+	   '("id"             #f)
+	   '("testname"       #f)
+	   '("owner"          #f)
+	   '("description"    #f)
+	   '("reviewed"       #f)
+	   '("iterated"       #f)
+	   '("avg_runtime"    #f)
+	   '("avg_disk"       #f)
+	   '("tags"           #f)
+	   '("jobgroup"       #f)))))
+
+;; why get the keys from the db? why not get from the *configdat*
+;; using keys:config-get-fields?
+
+(define (db:get-keys alldat)
+  (if *db-keys* *db-keys* 
+      (let ((res '()))
+	(db:with-db alldat #f #f
+		    (lambda (db)
+		      (sqlite3:for-each-row 
+		       (lambda (key)
+			 (set! res (cons key res)))
+		       db
+		       "SELECT fieldname FROM keys ORDER BY id DESC;")))
+	(set! *db-keys* res)
+	res)))
+
+;; (db:with-db alldat run-id sqlite3:exec "select blah fgrom blaz;")
+;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
+;;
+(define (db:with-db alldat run-id r/w proc . params)
+  (let* ((have-struct      (alldat? alldat))
+         (dbdat            (if have-struct 
+		           	  (db:get-db alldat)
+		           	  #f))
+	 (db               (if have-struct
+		           	  (db:dbdat-get-db dbdat)
+		           	  alldat))
+	 (use-mutex        (> (alldat-api-process-request-count alldat) 25))
+	 (db-with-db-mutex (alldat-db-with-db-mutex alldat))
+	 (log-port         (alldat-log-port alldat)))
+    (if (and use-mutex
+	     (common:low-noise-print 120 "over-50-parallel-api-requests"))
+	(debug:print-info 0 log-port (alldat-api-process-request-count alldat) " parallel api requests being processed in process " (current-process-id) ", throttling access"))
+    (if (common:low-noise-print 600 (conc "parallel-api-requests" (alldat-max-api-process-requests alldat)))
+	(debug:print-info 2 log-port "Parallel api request count: " (alldat-api-process-request-count alldat) " max parallel requests: " (alldat-max-api-process-requests alldat)))
+    (handle-exceptions
+     exn
+     (begin
+       (print-call-chain (current-error-port))
+       (debug:print-error 0 log-port "sqlite3 issue in db:with-db, alldat=" alldat ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn))
+       ;; there is no recovering at this time. exit
+       (exit 50))
+     (if use-mutex (mutex-lock! db-with-db-mutex))
+     (let ((res (apply proc db params)))
+       (if use-mutex (mutex-unlock! db-with-db-mutex))
+       (if dbdat (stack-push! (alldat-dbstack alldat) dbdat))
+       res))))
+
 
 
 )

ADDED   fixpath.sh
Index: fixpath.sh
==================================================================
--- /dev/null
+++ fixpath.sh
@@ -0,0 +1,1 @@
+export PATH=$(readlink -f ./bin):$PATH

DELETED fs-transport.scm
Index: fs-transport.scm
==================================================================
--- fs-transport.scm
+++ /dev/null
@@ -1,52 +0,0 @@
-
-;; Copyright 2006-2012, Matthew Welland.
-;; 
-;; This file is part of Megatest.
-;; 
-;;     Megatest is free software: you can redistribute it and/or modify
-;;     it under the terms of the GNU General Public License as published by
-;;     the Free Software Foundation, either version 3 of the License, or
-;;     (at your option) any later version.
-;; 
-;;     Megatest is distributed in the hope that it will be useful,
-;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;;     GNU General Public License for more details.
-;; 
-;;     You should have received a copy of the GNU General Public License
-;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
-
-(require-extension (srfi 18) extras tcp s11n)
-
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
-(import (prefix sqlite3 sqlite3:))
-
-(use spiffy uri-common intarweb http-client spiffy-request-vars)
-
-(tcp-buffer-size 2048)
-
-(declare (unit fs-transport))
-
-(declare (uses common))
-(declare (uses db))
-(declare (uses tests))
-(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
-
-(include "common_records.scm")
-(include "db_records.scm")
-
-
-;;======================================================================
-;; F S   T R A N S P O R T   S E R V E R
-;;======================================================================
-
-;; There is no "server" per se but a convience routine to make it non
-;; necessary to be reopening the db over and over again.
-;;
-
-(define (fs:process-queue-item packet)
-  (if (not *dbstruct-db*) ;; we will require that (setup-for-run) has already been called
-      (set! *dbstruct-db* (db:setup-db)))
-  (debug:print-info 11 *default-log-port* "fs:process-queue-item called with packet=" packet)
-  (db:process-queue-item *dbstruct-db* packet))
-      

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -2239,10 +2239,13 @@
 	      (set! *db* dbstruct)
 	      (import extras) ;; might not be needed
 	      ;; (import csi)
 	      (import readline)
 	      (import apropos)
+	      (import dbmod)
+	      (import rmtmod)
+	      (import commonmod)
 	      ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ...
 
 	      (if *use-new-readline*
 		  (begin
 		    (install-history-file (get-environment-variable "HOME") ".megatest_history") ;;  [homedir] [filename] [nlines])

Index: migrate-fix.scm
==================================================================
--- migrate-fix.scm
+++ migrate-fix.scm
@@ -2,11 +2,11 @@
 ;; functions needed during the transition to modules
 ;;
 ;; NOTE: the definition in dbmod seems to "win" - make it available everywhere
 ;;
 (set-fn 'client:setup client:setup)
-(set-fn 'db:setup       db:setup)
+;; (set-fn 'db:setup       db:setup)
 (set-fn 'server:expiration-timeout server:expiration-timeout)
 (set-fn 'common:get-homehost       common:get-homehost)
 (set-fn 'server:check-if-running   server:check-if-running)
 (set-fn 'api:execute-requests      api:execute-requests)
 (set-fn 'http-transport:close-connections  http-transport:close-connections )

ADDED   oldsrc/rpc-transport.scm
Index: oldsrc/rpc-transport.scm
==================================================================
--- /dev/null
+++ oldsrc/rpc-transport.scm
@@ -0,0 +1,237 @@
+
+;; Copyright 2006-2012, Matthew Welland.
+;; 
+;; This file is part of Megatest.
+;; 
+;;     Megatest is free software: you can redistribute it and/or modify
+;;     it under the terms of the GNU General Public License as published by
+;;     the Free Software Foundation, either version 3 of the License, or
+;;     (at your option) any later version.
+;; 
+;;     Megatest is distributed in the hope that it will be useful,
+;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;     GNU General Public License for more details.
+;; 
+;;     You should have received a copy of the GNU General Public License
+;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
+;;
+
+(require-extension (srfi 18) extras tcp s11n rpc)
+(import (prefix rpc rpc:))
+
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
+(import (prefix sqlite3 sqlite3:))
+
+(declare (unit rpc-transport))
+
+(declare (uses common))
+(declare (uses db))
+(declare (uses tests))
+(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
+
+(include "common_records.scm")
+(include "db_records.scm")
+
+;; procstr is the name of the procedure to be called as a string
+(define (rpc-transport:autoremote procstr params)
+  (handle-exceptions
+   exn
+   (begin
+     (debug:print 1 *default-log-port* "Remote failed for " proc " " params)
+     (apply (eval (string->symbol procstr)) params))
+   ;; (if *runremote*
+   ;;    (apply (eval (string->symbol (conc "remote:" procstr))) params)
+   (apply (eval (string->symbol procstr)) params)))
+
+;; all routes though here end in exit ...
+;;
+;; start_server? 
+;;
+(define (rpc-transport:launch run-id)
+  (let* ((tdbdat (tasks:open-db)))
+    (BB> "rpc-transport:launch fired for run-id="run-id)
+    (set! *run-id*   run-id)
+    (if (args:get-arg "-daemonize")
+        (daemon:ize))
+    (if (server:check-if-running run-id)
+        (begin
+          (debug:print 0 *default-log-port* "INFO: Server for run-id " run-id " already running")
+          (exit 0)))
+    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
+               (remtries  4))
+      (if (not server-id)
+          (if (> remtries 0)
+              (begin
+                (thread-sleep! 2)
+                (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
+                      (- remtries 1)))
+              (begin
+                ;; since we didn't get the server lock we are going to clean up and bail out
+                (debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
+                (tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " rpc-transport:launch")))
+          (begin
+            (rpc-transport:run (if (args:get-arg "-server")(args:get-arg "-server") "-") run-id server-id)
+            (exit))))))
+
+(define (rpc-transport:run hostn run-id server-id)
+  (debug:print 2 *default-log-port* "Attempting to start the rpc server ...")
+   ;; (trace rpc:publish-procedure!)
+
+  (rpc:publish-procedure! 'server:login server:login)
+  (rpc:publish-procedure! 'testing (lambda () "Just testing"))
+
+  (let* ((db              #f)
+	 (hostname        (get-host-name))
+	 (ipaddrstr       (let ((ipstr (if (string=? "-" hostn)
+					   ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".")
+					   (server:get-best-guess-address hostname)
+					   #f)))
+			    (if ipstr ipstr hostn))) ;; hostname))) 
+	 (start-port      (open-run-close tasks:server-get-next-port tasks:open-db))
+	 (link-tree-path  (configf:lookup *configdat* "setup" "linktree"))
+	 (rpc:listener    (rpc-transport:find-free-port-and-open (rpc:default-server-port)))
+	 (th1             (make-thread
+			   (lambda ()
+			     ((rpc:make-server rpc:listener) #t))
+			   "rpc:server"))
+			   ;; (cute (rpc:make-server rpc:listener) "rpc:server")
+			   ;; 'rpc:server))
+	 (hostname        (if (string=? "-" hostn)
+			      (get-host-name) 
+			      hostn))
+	 (ipaddrstr       (if (string=? "-" hostn)
+			      (server:get-best-guess-address hostname) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".")
+			      #f))
+	 (portnum         (rpc:default-server-port))
+	 (host:port       (conc (if ipaddrstr ipaddrstr hostname) ":" portnum))
+	 (tdb             (tasks:open-db)))
+    (thread-start! th1)
+    (set! db *dbstruct-db*)
+    (open-run-close tasks:server-set-interface-port 
+		    tasks:open-db 
+		    server-id 
+		    ipaddrstr portnum)
+    (debug:print 0 *default-log-port* "Server started on " host:port)
+    
+    ;; (trace rpc:publish-procedure!)
+    ;; (rpc:publish-procedure! 'server:login server:login)
+    ;; (rpc:publish-procedure! 'testing (lambda () "Just testing"))
+
+    ;;======================================================================
+    ;;	  ;; end of publish-procedure section
+    ;;======================================================================
+    ;;
+    (on-exit (lambda ()
+	       (open-run-close tasks:server-set-state! tasks:open-db server-id "stopped")))
+
+    (set! *rpc:listener* rpc:listener)
+    (tasks:server-set-state! tdb server-id "running")
+    (set! *dbstruct-db*  (db:setup run-id))
+    ;; if none running or if > 20 seconds since 
+    ;; server last used then start shutdown
+    (let loop ((count 0))
+      (thread-sleep! 5) ;; no need to do this very often
+      (let ((numrunning -1)) ;; (db:get-count-tests-running db)))
+	(if (or (> numrunning 0)
+		(> (+ *db-last-access* 60)(current-seconds)))
+	    (begin
+	      (debug:print-info 0 *default-log-port* "Server continuing, tests running: " numrunning ", seconds since last db access: " (- (current-seconds) *db-last-access*))
+	      (loop (+ 1 count)))
+	    (begin
+	      (debug:print-info 0 *default-log-port* "Starting to shutdown the server side")
+	      (open-run-close tasks:server-delete-record tasks:open-db server-id " rpc-transport:try-start-server stop")
+	      (thread-sleep! 10)
+	      (debug:print-info 0 *default-log-port* "Max cached queries was " *max-cache-size*)
+	      (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting")
+	      ))))))
+
+(define (rpc-transport:find-free-port-and-open port)
+  (handle-exceptions
+   exn
+	  (begin
+     (print "Failed to bind to port " (rpc:default-server-port) ", trying next port")
+     (rpc-transport:find-free-port-and-open (+ port 1)))
+   (rpc:default-server-port port)
+   (tcp-read-timeout 240000)
+   (tcp-listen (rpc:default-server-port) 10000)))
+
+(define (rpc-transport:ping run-id host port)
+  (handle-exceptions
+   exn
+   (begin
+     (print "SERVER_NOT_FOUND")
+     (exit 1))
+   (let ((login-res ((rpc:procedure 'server:login host port) *toppath*)))
+     (if (and (list? login-res)
+	      (car login-res))
+	 (begin
+	   (print "LOGIN_OK")
+	   (exit 0))
+	 (begin
+	   (print "LOGIN_FAILED")
+	   (exit 1))))))
+
+(define (rpc-transport:client-setup run-id #!key (remtries 10))
+  (if *runremote*
+      (begin
+	(debug:print-error 0 *default-log-port* "Attempt to connect to server but already connected")
+	#f)
+      (let* ((host-info (hash-table-ref/default *runremote* run-id #f))) ;; (open-run-close db:get-var #f "SERVER"))
+	(if host-info
+	    (let ((iface    (car host-info))
+		  (port     (cadr host-info))
+		  (ping-res ((rpc:procedure 'server:login host port) *toppath*)))
+	      (if ping-res
+		  (let ((server-dat (list iface port #f #f #f)))
+		    (hash-table-set! *runremote* run-id server-dat)
+		    server-dat)
+		  (begin
+		    (server:try-running *toppath*)
+		    (thread-sleep! 2)
+		    (rpc-transport:client-setup run-id (- remtries 1)))))
+ 	    (let* ((server-db-info (open-run-close tasks:get-server tasks:open-db run-id)))
+ 	      (debug:print-info 0 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
+	      (if server-db-info
+ 		  (let* ((iface     (tasks:hostinfo-get-interface server-db-info))
+ 			 (port      (tasks:hostinfo-get-port      server-db-info))
+			 (server-dat (list iface port #f #f #f))
+ 			 (ping-res  ((rpc:procedure 'server:login host port) *toppath*)))
+ 		    (if start-res
+ 			(begin
+ 			  (hash-table-set! *runremote* run-id server-dat)
+			  server-dat)
+			(begin
+			  (server:try-running *toppath*)
+			  (thread-sleep! 2)
+			  (rpc-transport:client-setup run-id (- remtries 1)))))
+		  (begin
+		    (server:try-running *toppath*)
+		    (thread-sleep! 2)
+		    (rpc-transport:client-setup run-id (- remtries 1)))))))))
+;; 
+;; 	     (port     (if (and hostinfo (> (length hostdat) 1))(cadr hostdat) #f)))
+;; 	(if (and port
+;; 		 (string->number port))
+;; 	    (let ((portn (string->number port)))
+;; 	      (debug:print-info 2 *default-log-port* "Setting up to connect to host " host ":" port)
+;; 	      (handle-exceptions
+;; 	       exn
+;; 	       (begin
+;; 		 (debug:print-error 0 *default-log-port* "Failed to open a connection to the server at host: " host " port: " port)
+;; 		 (debug:print 0 *default-log-port* "   EXCEPTION: " ((condition-property-accessor 'exn 'message) exn))
+;; 		 ;; (open-run-close 
+;; 		 ;;  (lambda (db . param) 
+;; 		 ;;    (sqlite3:execute db "DELETE FROM metadat WHERE var='SERVER'"))
+;; 		 ;;  #f)
+;; 		 (set! *runremote* #f))
+;; 	       (if (and (not (args:get-arg "-server")) ;; no point in the server using the server using the server
+;; 			((rpc:procedure 'server:login host portn) *toppath*))
+;; 		   (begin
+;; 		     (debug:print-info 2 *default-log-port* "Logged in and connected to " host ":" port)
+;; 		     (set! *runremote* (vector host portn)))
+;; 		   (begin
+;; 		     (debug:print-info 2 *default-log-port* "Failed to login or connect to " host ":" port)
+;; 		     (set! *runremote* #f)))))
+;; 	    (debug:print-info 2 *default-log-port* "no server available")))))
+

DELETED rpc-transport.scm
Index: rpc-transport.scm
==================================================================
--- rpc-transport.scm
+++ /dev/null
@@ -1,237 +0,0 @@
-
-;; Copyright 2006-2012, Matthew Welland.
-;; 
-;; This file is part of Megatest.
-;; 
-;;     Megatest is free software: you can redistribute it and/or modify
-;;     it under the terms of the GNU General Public License as published by
-;;     the Free Software Foundation, either version 3 of the License, or
-;;     (at your option) any later version.
-;; 
-;;     Megatest is distributed in the hope that it will be useful,
-;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;;     GNU General Public License for more details.
-;; 
-;;     You should have received a copy of the GNU General Public License
-;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
-;;
-
-(require-extension (srfi 18) extras tcp s11n rpc)
-(import (prefix rpc rpc:))
-
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
-(import (prefix sqlite3 sqlite3:))
-
-(declare (unit rpc-transport))
-
-(declare (uses common))
-(declare (uses db))
-(declare (uses tests))
-(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
-
-(include "common_records.scm")
-(include "db_records.scm")
-
-;; procstr is the name of the procedure to be called as a string
-(define (rpc-transport:autoremote procstr params)
-  (handle-exceptions
-   exn
-   (begin
-     (debug:print 1 *default-log-port* "Remote failed for " proc " " params)
-     (apply (eval (string->symbol procstr)) params))
-   ;; (if *runremote*
-   ;;    (apply (eval (string->symbol (conc "remote:" procstr))) params)
-   (apply (eval (string->symbol procstr)) params)))
-
-;; all routes though here end in exit ...
-;;
-;; start_server? 
-;;
-(define (rpc-transport:launch run-id)
-  (let* ((tdbdat (tasks:open-db)))
-    (BB> "rpc-transport:launch fired for run-id="run-id)
-    (set! *run-id*   run-id)
-    (if (args:get-arg "-daemonize")
-        (daemon:ize))
-    (if (server:check-if-running run-id)
-        (begin
-          (debug:print 0 *default-log-port* "INFO: Server for run-id " run-id " already running")
-          (exit 0)))
-    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
-               (remtries  4))
-      (if (not server-id)
-          (if (> remtries 0)
-              (begin
-                (thread-sleep! 2)
-                (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
-                      (- remtries 1)))
-              (begin
-                ;; since we didn't get the server lock we are going to clean up and bail out
-                (debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
-                (tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " rpc-transport:launch")))
-          (begin
-            (rpc-transport:run (if (args:get-arg "-server")(args:get-arg "-server") "-") run-id server-id)
-            (exit))))))
-
-(define (rpc-transport:run hostn run-id server-id)
-  (debug:print 2 *default-log-port* "Attempting to start the rpc server ...")
-   ;; (trace rpc:publish-procedure!)
-
-  (rpc:publish-procedure! 'server:login server:login)
-  (rpc:publish-procedure! 'testing (lambda () "Just testing"))
-
-  (let* ((db              #f)
-	 (hostname        (get-host-name))
-	 (ipaddrstr       (let ((ipstr (if (string=? "-" hostn)
-					   ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".")
-					   (server:get-best-guess-address hostname)
-					   #f)))
-			    (if ipstr ipstr hostn))) ;; hostname))) 
-	 (start-port      (open-run-close tasks:server-get-next-port tasks:open-db))
-	 (link-tree-path  (configf:lookup *configdat* "setup" "linktree"))
-	 (rpc:listener    (rpc-transport:find-free-port-and-open (rpc:default-server-port)))
-	 (th1             (make-thread
-			   (lambda ()
-			     ((rpc:make-server rpc:listener) #t))
-			   "rpc:server"))
-			   ;; (cute (rpc:make-server rpc:listener) "rpc:server")
-			   ;; 'rpc:server))
-	 (hostname        (if (string=? "-" hostn)
-			      (get-host-name) 
-			      hostn))
-	 (ipaddrstr       (if (string=? "-" hostn)
-			      (server:get-best-guess-address hostname) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".")
-			      #f))
-	 (portnum         (rpc:default-server-port))
-	 (host:port       (conc (if ipaddrstr ipaddrstr hostname) ":" portnum))
-	 (tdb             (tasks:open-db)))
-    (thread-start! th1)
-    (set! db *dbstruct-db*)
-    (open-run-close tasks:server-set-interface-port 
-		    tasks:open-db 
-		    server-id 
-		    ipaddrstr portnum)
-    (debug:print 0 *default-log-port* "Server started on " host:port)
-    
-    ;; (trace rpc:publish-procedure!)
-    ;; (rpc:publish-procedure! 'server:login server:login)
-    ;; (rpc:publish-procedure! 'testing (lambda () "Just testing"))
-
-    ;;======================================================================
-    ;;	  ;; end of publish-procedure section
-    ;;======================================================================
-    ;;
-    (on-exit (lambda ()
-	       (open-run-close tasks:server-set-state! tasks:open-db server-id "stopped")))
-
-    (set! *rpc:listener* rpc:listener)
-    (tasks:server-set-state! tdb server-id "running")
-    (set! *dbstruct-db*  (db:setup run-id))
-    ;; if none running or if > 20 seconds since 
-    ;; server last used then start shutdown
-    (let loop ((count 0))
-      (thread-sleep! 5) ;; no need to do this very often
-      (let ((numrunning -1)) ;; (db:get-count-tests-running db)))
-	(if (or (> numrunning 0)
-		(> (+ *db-last-access* 60)(current-seconds)))
-	    (begin
-	      (debug:print-info 0 *default-log-port* "Server continuing, tests running: " numrunning ", seconds since last db access: " (- (current-seconds) *db-last-access*))
-	      (loop (+ 1 count)))
-	    (begin
-	      (debug:print-info 0 *default-log-port* "Starting to shutdown the server side")
-	      (open-run-close tasks:server-delete-record tasks:open-db server-id " rpc-transport:try-start-server stop")
-	      (thread-sleep! 10)
-	      (debug:print-info 0 *default-log-port* "Max cached queries was " *max-cache-size*)
-	      (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting")
-	      ))))))
-
-(define (rpc-transport:find-free-port-and-open port)
-  (handle-exceptions
-   exn
-	  (begin
-     (print "Failed to bind to port " (rpc:default-server-port) ", trying next port")
-     (rpc-transport:find-free-port-and-open (+ port 1)))
-   (rpc:default-server-port port)
-   (tcp-read-timeout 240000)
-   (tcp-listen (rpc:default-server-port) 10000)))
-
-(define (rpc-transport:ping run-id host port)
-  (handle-exceptions
-   exn
-   (begin
-     (print "SERVER_NOT_FOUND")
-     (exit 1))
-   (let ((login-res ((rpc:procedure 'server:login host port) *toppath*)))
-     (if (and (list? login-res)
-	      (car login-res))
-	 (begin
-	   (print "LOGIN_OK")
-	   (exit 0))
-	 (begin
-	   (print "LOGIN_FAILED")
-	   (exit 1))))))
-
-(define (rpc-transport:client-setup run-id #!key (remtries 10))
-  (if *runremote*
-      (begin
-	(debug:print-error 0 *default-log-port* "Attempt to connect to server but already connected")
-	#f)
-      (let* ((host-info (hash-table-ref/default *runremote* run-id #f))) ;; (open-run-close db:get-var #f "SERVER"))
-	(if host-info
-	    (let ((iface    (car host-info))
-		  (port     (cadr host-info))
-		  (ping-res ((rpc:procedure 'server:login host port) *toppath*)))
-	      (if ping-res
-		  (let ((server-dat (list iface port #f #f #f)))
-		    (hash-table-set! *runremote* run-id server-dat)
-		    server-dat)
-		  (begin
-		    (server:try-running *toppath*)
-		    (thread-sleep! 2)
-		    (rpc-transport:client-setup run-id (- remtries 1)))))
- 	    (let* ((server-db-info (open-run-close tasks:get-server tasks:open-db run-id)))
- 	      (debug:print-info 0 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
-	      (if server-db-info
- 		  (let* ((iface     (tasks:hostinfo-get-interface server-db-info))
- 			 (port      (tasks:hostinfo-get-port      server-db-info))
-			 (server-dat (list iface port #f #f #f))
- 			 (ping-res  ((rpc:procedure 'server:login host port) *toppath*)))
- 		    (if start-res
- 			(begin
- 			  (hash-table-set! *runremote* run-id server-dat)
-			  server-dat)
-			(begin
-			  (server:try-running *toppath*)
-			  (thread-sleep! 2)
-			  (rpc-transport:client-setup run-id (- remtries 1)))))
-		  (begin
-		    (server:try-running *toppath*)
-		    (thread-sleep! 2)
-		    (rpc-transport:client-setup run-id (- remtries 1)))))))))
-;; 
-;; 	     (port     (if (and hostinfo (> (length hostdat) 1))(cadr hostdat) #f)))
-;; 	(if (and port
-;; 		 (string->number port))
-;; 	    (let ((portn (string->number port)))
-;; 	      (debug:print-info 2 *default-log-port* "Setting up to connect to host " host ":" port)
-;; 	      (handle-exceptions
-;; 	       exn
-;; 	       (begin
-;; 		 (debug:print-error 0 *default-log-port* "Failed to open a connection to the server at host: " host " port: " port)
-;; 		 (debug:print 0 *default-log-port* "   EXCEPTION: " ((condition-property-accessor 'exn 'message) exn))
-;; 		 ;; (open-run-close 
-;; 		 ;;  (lambda (db . param) 
-;; 		 ;;    (sqlite3:execute db "DELETE FROM metadat WHERE var='SERVER'"))
-;; 		 ;;  #f)
-;; 		 (set! *runremote* #f))
-;; 	       (if (and (not (args:get-arg "-server")) ;; no point in the server using the server using the server
-;; 			((rpc:procedure 'server:login host portn) *toppath*))
-;; 		   (begin
-;; 		     (debug:print-info 2 *default-log-port* "Logged in and connected to " host ":" port)
-;; 		     (set! *runremote* (vector host portn)))
-;; 		   (begin
-;; 		     (debug:print-info 2 *default-log-port* "Failed to login or connect to " host ":" port)
-;; 		     (set! *runremote* #f)))))
-;; 	    (debug:print-info 2 *default-log-port* "no server available")))))
-

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -67,30 +67,18 @@
 ;; all routes though here end in exit ...
 ;;
 ;; start_server
 ;;
 (define (server:launch run-id transport-type)
-  (case transport-type
-    ((http)(http-transport:launch))
-    ;;((nmsg)(nmsg-transport:launch run-id))
-    ;;((rpc)  (rpc-transport:launch run-id))
-    (else (debug:print-error 0 *default-log-port* "unknown server type " transport-type))))
+  (http-transport:launch))
 
 ;;======================================================================
 ;; S E R V E R   U T I L I T I E S 
 ;;======================================================================
 
 ;; Get the transport
-(define (server:get-transport)
-  (if *transport-type*
-      *transport-type*
-      (let ((ttype (string->symbol
-		    (or (args:get-arg "-transport")
-			(configf:lookup *configdat* "server" "transport")
-			"rpc"))))
-	(set! *transport-type* ttype)
-	ttype)))
+(define (server:get-transport) 'http)
 	    
 ;; Generate a unique signature for this server
 (define (server:mk-signature)
   (message-digest-string (md5-primitive) 
 			 (with-output-to-string
@@ -101,19 +89,11 @@
 ;; When using zmq this would send the message back (two step process)
 ;; with spiffy or rpc this simply returns the return data to be returned
 ;; 
 (define (server:reply return-addr query-sig success/fail result)
   (debug:print-info 11 *default-log-port* "server:reply return-addr=" return-addr ", result=" result)
-  ;; (send-message pubsock target send-more: #t)
-  ;; (send-message pubsock 
-  (case (server:get-transport)
-    ((rpc)  (db:obj->string (vector success/fail query-sig result)))
-    ((http) (db:obj->string (vector success/fail query-sig result)))
-    ((fs)   result)
-    (else 
-     (debug:print-error 0 *default-log-port* "unrecognised transport type: " *transport-type*)
-     result)))
+  (db:obj->string (vector success/fail query-sig result)))  ;; (send-message pubsock target send-more: #t)
 
 ;; Given a run id start a server process    ### NOTE ### > file 2>&1 
 ;; if the run-id is zero and the target-host is set 
 ;; try running on that host
 ;;   incidental: rotate logs in logs/ dir.
@@ -451,11 +431,12 @@
 	     ((NOREPLY)  #f)
 	     ((LOGIN_OK) #t)
 	     (else       #f))
 	   (loop (read-line) inl))))))
 
-;; NOT USED (well, ok, reference in rpc-transport but otherwise not used).
+;; NOT USED (well, ok, was referenced in rpc-transport but otherwise
+;; not used).
 ;;
 (define (server:login toppath)
   (lambda (toppath)
     (set! *db-last-access* (current-seconds)) ;; might not be needed.
     (if (equal? *toppath* toppath)

ADDED   utils/fslrept
Index: utils/fslrept
==================================================================
--- /dev/null
+++ utils/fslrept
cannot compute difference between binary files