Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -54,11 +54,11 @@
     (else   (rpc:client-connect  iface port))))
 
 (define (client:setup areapath #!key (remaining-tries 100) (failed-connects 0))
   (case (server:get-transport)
     ((rpc) (rpc-transport:client-setup remaining-tries: remaining-tries failed-connects: failed-connects)) ;;(client:setup-rpc run-id))
-    ((http)(client:setup-http areapath remaining-tries: remaining-tries failed-connects: failed-connects))
+    ((http)(client:setup-http *runremote* areapath remaining-tries: remaining-tries failed-connects: failed-connects))
     (else  (rpc-transport:client-setup remaining-tries: remaining-tries failed-connects: failed-connects)))) ;; (client:setup-rpc run-id))))
 
 ;; Do all the connection work, look up the transport type and set up the
 ;; connection if required.
 ;;
@@ -70,11 +70,11 @@
 ;; client:setup
 ;;
 ;; lookup_server, need to remove *runremote* stuff
 ;;
 
-(define (client:setup-http areapath #!key (remaining-tries 100) (failed-connects 0)(area-dat #f))
+(define (client:setup-http runremote areapath #!key (remaining-tries 100) (failed-connects 0)) ;; (area-dat #f))
   (debug:print-info 2 *default-log-port* "client:setup remaining-tries=" remaining-tries)
   (server:start-and-wait areapath)
   (if (<= remaining-tries 0)
       (begin
 	(debug:print-error 0 *default-log-port* "failed to start or connect to server")
@@ -81,41 +81,37 @@
 	(exit 1))
       ;;
       ;; Alternatively here, we can get the list of candidate servers and work our way
       ;; through them searching for a good one.
       ;;
-      (let* ((server-dat (server:get-rand-best areapath)) ;; (server:get-first-best areapath))
-	     (runremote  (or area-dat *runremote*)))
+      (let* ((server-dat (server:get-rand-best areapath))) ;; (server:get-first-best areapath))
 	(if (not server-dat) ;; no server found
-	    (client:setup-http areapath remaining-tries: (- remaining-tries 1))
+	    (client:setup-http runremote areapath remaining-tries: (- remaining-tries 1))
 	    (let ((host  (cadr  server-dat))
 		  (port  (caddr server-dat)))
 	      (debug:print-info 4 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
-	      (if (and (not area-dat)
-		       (not *runremote*))
-		  (set! *runremote* (make-remote)))
 	      (if (and host port)
 		  (let* ((start-res (case *transport-type*
 				      ((http)(http-transport:client-connect host port))))
 			 (ping-res  (case *transport-type* 
 				      ((http)(rmt:login-no-auto-client-setup start-res)))))
 		    (if (and start-res
 			     ping-res)
-			(let ((runremote (or area-dat *runremote*))) ;; it might have been generated only a few statements ago
-			  (remote-conndat-set! runremote start-res) ;; (hash-table-set! runremote run-id start-res)
+			(begin
+			  (remote-conndat-set! runremote start-res)
 			  (debug:print-info 2 *default-log-port* "connected to " (http-transport:server-dat-make-url start-res))
 			  start-res)
 			(begin    ;; login failed but have a server record, clean out the record and try again
 			  (debug:print-info 0 *default-log-port* "client:setup, login unsuccessful, will attempt to start server ... start-res=" start-res ", server-dat=" server-dat) ;; had runid.  Fixes part of Randy;s ticket 1405717332
 			  (case *transport-type* 
 			    ((http)(http-transport:close-connections)))
 			  (remote-conndat-set! runremote #f)  ;; (hash-table-delete! runremote run-id)
 			  (thread-sleep! 1)
-			  (client:setup-http areapath remaining-tries: (- remaining-tries 1))
+			  (client:setup-http runremote areapath remaining-tries: (- remaining-tries 1))
 			  )))
 		  (begin    ;; no server registered
 		    ;; (server:kind-run areapath)
 		    (server:start-and-wait areapath)
 		    (debug:print-info 0 *default-log-port* "client:setup, no server registered, remaining-tries=" remaining-tries)
 		    (thread-sleep! 1) ;; (+ 5 (random (- 20 remaining-tries))))  ;; give server a little time to start up, randomize a little to avoid start storms.
-		    (client:setup-http areapath remaining-tries: (- remaining-tries 1)))))))))
+		    (client:setup-http runremote areapath remaining-tries: (- remaining-tries 1)))))))))
 

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -158,11 +158,10 @@
 (define *no-sync-db*          #f)
 
 ;; SERVER
 (define *my-client-signature* #f)
 (define *transport-type*    'http)             ;; override with [server] transport http|rpc|nmsg
-(define *runremote*         #f)                ;; if set up for server communication this will hold <host port>
 ;; (define *max-cache-size*    0)
 (define *logged-in-clients* (make-hash-table))
 (define *server-id*         #f)
 (define *server-info*       #f)  ;; good candidate for easily convert to non-global
 (define *time-to-exit*      #f)
@@ -985,14 +984,10 @@
 					  (sqlite3:interrupt! db)
 					  (sqlite3:finalize! db #t)
 					  ;; (vector-set! *task-db* 0 #f)
 					  (set! *task-db* #f)))))
                               (http-client#close-all-connections!)
-                              ;; (if (and *runremote*
-                              ;;          (remote-conndat *runremote*))
-                              ;;     (begin
-                              ;;       (http-client#close-all-connections!))) ;; for http-client
                               (if (not (eq? *default-log-port* (current-error-port)))
                                   (close-output-port *default-log-port*))
 			      (set! *default-log-port* (current-error-port))) "Cleanup db exit thread"))
 	  (th2 (make-thread (lambda ()
 			      (debug:print 4 *default-log-port* "Attempting clean exit. Please be patient and wait a few seconds...")

Index: common_records.scm
==================================================================
--- common_records.scm
+++ common_records.scm
@@ -52,10 +52,37 @@
   (ro-mode           #f)  
   (ro-mode-checked   #f) ;; flag that indicates we have checked for ro-mode
   (ulex:conn         #f) ;; ulex db conn is not exactly a db connector, more like a network connector 
   )
 
+;; Pulled from http-transport.scm
+
+(define (make-http-transport:server-dat)(make-vector 6))
+(define (http-transport:server-dat-get-iface         vec)    (vector-ref  vec 0))
+(define (http-transport:server-dat-get-port          vec)    (vector-ref  vec 1))
+(define (http-transport:server-dat-get-api-uri       vec)    (vector-ref  vec 2))
+(define (http-transport:server-dat-get-api-url       vec)    (vector-ref  vec 3))
+(define (http-transport:server-dat-get-api-req       vec)    (vector-ref  vec 4))
+(define (http-transport:server-dat-get-last-access   vec)    (vector-ref  vec 5))
+(define (http-transport:server-dat-get-socket        vec)    (vector-ref  vec 6))
+
+(define (http-transport:server-dat-make-url vec)
+  (if (and (http-transport:server-dat-get-iface vec)
+	   (http-transport:server-dat-get-port  vec))
+      (conc "http://" 
+	    (http-transport:server-dat-get-iface vec)
+	    ":"
+	    (http-transport:server-dat-get-port  vec))
+      #f))
+
+(define (http-transport:server-dat-update-last-access vec)
+  (if (vector? vec)
+      (vector-set! vec 5 (current-seconds))
+      (begin
+	(print-call-chain (current-error-port))
+	(debug:print-error 0 *default-log-port* "call to http-transport:server-dat-update-last-access with non-vector!!"))))
+
 
 ;; Some of these routines use:
 ;;
 ;;     http://www.cs.toronto.edu/~gfb/scheme/simple-macros.html
 ;;

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -335,35 +335,11 @@
 	    (close-connection! api-dat)
             ;;(close-idle-connections!)
 	    #t))
 	#f)))
 
-
-(define (make-http-transport:server-dat)(make-vector 6))
-(define (http-transport:server-dat-get-iface         vec)    (vector-ref  vec 0))
-(define (http-transport:server-dat-get-port          vec)    (vector-ref  vec 1))
-(define (http-transport:server-dat-get-api-uri       vec)    (vector-ref  vec 2))
-(define (http-transport:server-dat-get-api-url       vec)    (vector-ref  vec 3))
-(define (http-transport:server-dat-get-api-req       vec)    (vector-ref  vec 4))
-(define (http-transport:server-dat-get-last-access   vec)    (vector-ref  vec 5))
-(define (http-transport:server-dat-get-socket        vec)    (vector-ref  vec 6))
-
-(define (http-transport:server-dat-make-url vec)
-  (if (and (http-transport:server-dat-get-iface vec)
-	   (http-transport:server-dat-get-port  vec))
-      (conc "http://" 
-	    (http-transport:server-dat-get-iface vec)
-	    ":"
-	    (http-transport:server-dat-get-port  vec))
-      #f))
-
-(define (http-transport:server-dat-update-last-access vec)
-  (if (vector? vec)
-      (vector-set! vec 5 (current-seconds))
-      (begin
-	(print-call-chain (current-error-port))
-	(debug:print-error 0 *default-log-port* "call to http-transport:server-dat-update-last-access with non-vector!!"))))
+;; http-transport:server-dat definition moved to common_records.scm
 
 ;;
 ;; connect
 ;;
 (define (http-transport:client-connect iface port)

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -44,10 +44,13 @@
 (declare (uses tests))
 (declare (uses genexample))
 ;; (declare (uses daemon))
 (declare (uses db))
 ;; (declare (uses dcommon))
+
+(declare (uses commonmod))
+(declare (uses rmtmod))
 
 (declare (uses tdb))
 (declare (uses mt))
 (declare (uses api))
 (declare (uses tasks)) ;; only used for debugging.

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -41,220 +41,43 @@
 
 ;;======================================================================
 ;;  S U P P O R T   F U N C T I O N S
 ;;======================================================================
 
-;; if a server is either running or in the process of starting call client:setup
-;; else return #f to let the calling proc know that there is no server available
-;;
-(define (rmt:get-connection-info areapath #!key (area-dat #f)) ;; TODO: push areapath down.
-  (let* ((runremote (or area-dat *runremote*))
-	 (cinfo     (if (remote? runremote)
-			(remote-conndat runremote)
-			#f)))
-	  (if cinfo
-	      cinfo
-	      (if (server:check-if-running areapath)
-		  (client:setup areapath)
-		  #f))))
-
 (define *send-receive-mutex* (make-mutex)) ;; should have separate mutex per run-id
 
-
+(define *runremote* (make-remote))
 
 ;; this entry point can decide based on cmd whether to dispatch to old api calls via remote or via ulex
 ;;
 (define (rmt:send-receive cmd rid params #!key (attemptnum 1)(area-dat #f))
   (let* ((areapath      *toppath*);; TODO - resolve from dbstruct to be compatible with multiple areas
 	 (runremote     (or area-dat
 			    *runremote*)))
     ;; ensure we have a record for our connection for given area
-    (if (not runremote)                   ;; can remove this one. should never get here.         
+    (if (not (remote-hh-dat runremote))
 	(begin
-	  (set! *runremote* (make-remote))
-	  (set! runremote   *runremote*)
 	  (remote-server-timeout-set! runremote (server:expiration-timeout))
-	  (remote-hh-dat-set!         runremote (common:get-homehost ))
+	  (remote-hh-dat-set!         runremote (common:get-homehost))
 	  )) ;; new runremote will come from this on next iteration
-    
-    (if (member cmd '(blah))
-      (begin
-	(mutex-lock! *send-receive-mutex*)
-	(if (not *runremote*)(set! *runremote* (make-remote)))
-	(let ((ulex:conn (remote-ulex:conn *runremote*)))
-	  (if (not ulex:conn)(remote-ulex:conn-set! *runremote* (rmtmod:setup-ulex areapath)))
-	  (rmtmod:send-receive-ulex ulex:conn cmd rid params attemptnum area-dat)))
-      (rmt:send-receive-orig *default-log-port* runremote *rmt-mutex* areapath *db-multi-sync-mutex* cmd rid params attemptnum: attemptnum area-dat: area-dat ro-queries: api:read-only-queries))))
-
-;; RA => e.g. usage (rmt:send-receive 'get-var #f (list varname))
-;;
-;;  add multi-sync-mutex 
-;;
-(define (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected
-
-  #;(common:telemetry-log (conc "rmt:"(->string cmd))
-                        payload: `((rid . ,rid)
-                                   (params . ,params)))
-                          
-  
-  ;; do all the prep locked under the rmt-mutex
-  ;;(mutex-lock! rmt-mutex)
-  
-  ;; 1. check if server is started IFF cmd is a write OR if we are not on the homehost, store in runremote
-  ;; 2. check the age of the connections. refresh the connection if it is older than timeout-20 seconds.
-  ;; 3. do the query, if on homehost use local access
-  ;;
-  (let* ((start-time    (current-seconds)) ;; snapshot time so all use cases get same value
-
-	 (readonly-mode (rmtmod:calc-ro-mode runremote toppath)))
-
-    ;; ensure we have a homehost record
-    (if (not (pair? (remote-hh-dat runremote)))  ;; not on homehost
+
+    ;; ensure we have a homehost record, do this here instead of in -orig
+    (if (or (not (remote-hh-dat runremote))
+	    (not (pair? (remote-hh-dat runremote))))  ;; not on homehost
 	(remote-hh-dat-set! runremote (common:get-homehost)))
-    
-    ;;(print "BB> readonly-mode is "readonly-mode" dbfile is "dbfile)
-    (cond
-     ;; give up if more than 15 attempts
-     ((> attemptnum 15)
-      (debug:print 0 log-port "ERROR: 15 tries to start/connect to server. Giving up.")
-      (exit 1))
-
-     ;; readonly mode, read request-  handle it - case 2
-     ((and readonly-mode
-           (member cmd api:read-only-queries)) 
-      ;; (mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case 2")
-      (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries)
-      )
-
-     ;; readonly mode, write request.  Do nothing, return #f
-     (readonly-mode (extras-readonly-mode rmt-mutex log-port cmd params))
-
-     ;; This block was for pre-emptively resetting the connection if there had been no communication for some time.
-     ;; I don't think it adds any value. If the server is not there, just fail and start a new connection.
-     ;; also, the expire-time calculation might not be correct. We want, time-since-last-server-access > (server:get-timeout)
-     ;;
-     ;; reset the connection if it has been unused too long
-     ((and runremote
-           (remote-conndat runremote)
-	   (> (current-seconds) ;; if it has been more than server-timeout seconds since last contact, close this connection and start a new on
-	      (+ (http-transport:server-dat-get-last-access (remote-conndat runremote))
-		 (remote-server-timeout runremote))))
-      (debug:print-info 0 log-port "Connection to " (remote-server-url runremote) " expired due to no accesses, forcing new connection.")
-      (http-transport:close-connections area-dat: runremote)
-      (remote-conndat-set! runremote #f) ;; invalidate the connection, thus forcing a new connection.
-      ;; (mutex-unlock! rmt-mutex)
-      (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum))
-     
-
-     ;; on homehost and this is a read
-     ((and (not (remote-force-server runremote)) ;; honor forced use of server, i.e. server NOT required
-	   (cdr (remote-hh-dat runremote))       ;; on homehost
-           (member cmd api:read-only-queries))   ;; this is a read
-      ;; (mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case  5")
-      (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
-
-     ;; on homehost and this is a write, we already have a server, but server has died
-     ((and (cdr (remote-hh-dat runremote))           ;; on homehost
-           (not (member cmd api:read-only-queries))  ;; this is a write
-           (remote-server-url runremote)             ;; have a server
-           (not (server:ping (remote-server-url runremote))))  ;; server has died. NOTE: this is not a cheap call! Need better approach.
-      (set! *runremote* (make-remote))
-      (remote-force-server-set! runremote (common:force-server?))
-      ;; (mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case  6")
-      (rmt:send-receive-orig log-port runremote rmt-mutex toppath  multi-sync-mutex cmd rid params attemptnum: attemptnum))
-
-     ;; on homehost and this is a write, we already have a server
-     ((and (not (remote-force-server runremote))     ;; honor forced use of server, i.e. server NOT required
-	   (cdr (remote-hh-dat runremote))           ;; on homehost
-           (not (member cmd api:read-only-queries))  ;; this is a write
-           (remote-server-url runremote))            ;; have a server
-      ;;(mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case  4.1")
-      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
-
-     ;;  on homehost, no server contact made and this is a write, passively start a server 
-     ((and (not (remote-force-server runremote))     ;; honor forced use of server, i.e. server NOT required
-	   (cdr (remote-hh-dat runremote))           ;; have homehost
-           (not (remote-server-url runremote))       ;; no connection yet
-	   (not (member cmd api:read-only-queries))) ;; not a read-only query
-      (debug:print-info 12 log-port "rmt:send-receive, case  8")
-      (let ((server-url  (server:check-if-running toppath))) ;; (server:read-dotserver->url toppath))) ;; (server:check-if-running toppath))) ;; Do NOT want to run server:check-if-running - very expensive to do for every write call
-	(if server-url
-	    (remote-server-url-set! runremote server-url) ;; the string can be consumed by the client setup if needed
-	    (if (common:force-server?)
-		(server:start-and-wait toppath)
-		(server:kind-run toppath))))
-      (remote-force-server-set! runremote (common:force-server?))
-      ;; (mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case  8.1")
-      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
-
-     ((or (and (remote-force-server runremote)              ;; we are forcing a server and don't yet have a connection to one
-	       (not (remote-conndat runremote)))
-	  (and (not (cdr (remote-hh-dat runremote)))        ;; not on a homehost 
-	       (not (remote-conndat runremote))))           ;; and no connection
-      (debug:print-info 12 log-port "rmt:send-receive, case 9, hh-dat: " (remote-hh-dat runremote) " conndat: " (remote-conndat runremote))
-      ;;(mutex-unlock! rmt-mutex)
-      (if (not (server:check-if-running toppath)) ;; who knows, maybe one has started up?
-	  (server:start-and-wait toppath))
-      (remote-conndat-set! runremote (rmt:get-connection-info toppath)) ;; calls client:setup which calls client:setup-http
-      (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum)) ;; TODO: add back-off timeout as
-
-     ;; all set up if get this far, dispatch the query
-     ((and (not (remote-force-server runremote))
-	   (cdr (remote-hh-dat runremote))) ;; we are on homehost
-      ;;(mutex-unlock! rmt-mutex)
-      (debug:print-info 12 log-port "rmt:send-receive, case 10")
-      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd (if rid rid 0) params ro-queries: api:read-only-queries))
-
-     ;; not on homehost, do server query
-     (else (extras-case-11 log-port runremote cmd params attemptnum rid)))))
+
+    (if (member cmd '(blah))
+	(begin
+	  (mutex-lock! *send-receive-mutex*)
+	  (let ((ulex:conn (remote-ulex:conn runremote)))
+	    (if (not ulex:conn)(remote-ulex:conn-set! runremote (rmtmod:setup-ulex areapath)))
+	    (rmtmod:send-receive-ulex ulex:conn cmd rid params attemptnum area-dat)))
+	(rmt:send-receive-orig *default-log-port* runremote *rmt-mutex* areapath *db-multi-sync-mutex* cmd rid params attemptnum: attemptnum area-dat: area-dat ro-queries: api:read-only-queries))))
 
 ;; bunch of small functions factored out of send-receive to make debug easier
 ;;
 
-(define (extras-case-11 *default-log-port* runremote cmd params attemptnum rid)
-  ;; (mutex-unlock! *rmt-mutex*)
-  (debug:print-info 12 *default-log-port* "rmt:send-receive, case  9")
-  ;; (mutex-lock! *rmt-mutex*)
-  (let* ((conninfo (remote-conndat runremote))
-	 (dat      (case (remote-transport runremote)
-		     ((http) (condition-case ;; handling here has
-					     ;; caused a lot of
-					     ;; problems. However it
-					     ;; is needed to deal with
-					     ;; attemtped
-					     ;; communication to
-					     ;; servers that have gone
-					     ;; away
-			      (http-transport:client-api-send-receive 0 conninfo cmd params)
-			      ((commfail)(vector #f "communications fail"))
-			      ((exn)(vector #f "other fail" (print-call-chain)))))
-		     (else
-		      (debug:print 0 *default-log-port* "ERROR: transport " (remote-transport runremote) " not supported")
-		      (exit))))
-	 (success  (if (vector? dat) (vector-ref dat 0) #f))
-	 (res      (if (vector? dat) (vector-ref dat 1) #f)))
-    (if (and (vector? conninfo) (< 5 (vector-length conninfo)))
-	(http-transport:server-dat-update-last-access conninfo) ;; refresh access time
-	(begin
-	  (debug:print 0 *default-log-port* "INFO: Should not get here! conninfo=" conninfo)
-	  (set! conninfo #f)
-	  (remote-conndat-set! *runremote* #f) ;; NOTE: *runremote* is global copy of runremote. Purpose: factor out global.
-	  (http-transport:close-connections  area-dat: runremote)))
-    (debug:print-info 13 *default-log-port* "rmt:send-receive, case  9. conninfo=" conninfo " dat=" dat " runremote = " runremote)
-    (mutex-unlock! *rmt-mutex*)
-    (if success ;; success only tells us that the transport was
-	;; successful, have to examine the data to see if
-	;; there was a detected issue at the other end
-	(extras-transport-succeded *default-log-port* *rmt-mutex* attemptnum runremote res params rid cmd)
-	(extras-transport-failed *default-log-port* *rmt-mutex* attemptnum runremote cmd rid params)
-	)))
-
 ;; (define (rmt:update-db-stats run-id rawcmd params duration)
 ;;   (mutex-lock! *db-stats-mutex*)
 ;;   (handle-exceptions
 ;;    exn
 ;;    (begin
@@ -864,13 +687,16 @@
   (rmt:send-receive 'test-set-archive-block-id run-id (list run-id test-id archive-block-id)))
 
 (define (rmt:test-get-archive-block-info archive-block-id)
   (rmt:send-receive 'test-get-archive-block-info #f (list archive-block-id)))
 
-(set-functions rmt:send-receive                       remote-server-url-set!
-	       http-transport:close-connections	      remote-conndat-set!
-	       debug:print                            debug:print-info
-	       debug:print-error 
-	       remote-ro-mode                         remote-ro-mode-set!
-	       remote-ro-mode-checked-set!            remote-ro-mode-checked
-	       #f                                     #f
-	       api:execute-requests                   api:read-only-queries)
+(set-functions http-transport:client-api-send-receive  ;; a
+	       http-transport:close-connections	       ;; b
+	       api:execute-requests                    ;; c
+	       api:read-only-queries                   ;; d
+	       client:setup                            ;; e
+	       server:kind-run                         ;; f
+	       server:start-and-wait                   ;; g
+	       server:check-if-running                 ;; h
+	       server:ping                             ;; i
+	       common:force-server?                    ;; j
+	       )

Index: rmtmod.scm
==================================================================
--- rmtmod.scm
+++ rmtmod.scm
@@ -23,44 +23,44 @@
 
 (module rmtmod
 	*
 	
 (import scheme chicken data-structures extras)
-(import (prefix sqlite3 sqlite3:) posix typed-records srfi-18 srfi-69 format ports srfi-1)
+(import (prefix sqlite3 sqlite3:) posix typed-records srfi-18 srfi-69 format ports srfi-1 matchable)
 (import commonmod)
 (use (prefix ulex ulex:))
 
 (include "common_records.scm")
 
 ;; Hack to make these functions visible to the refactored code, goal is to eliminate these over time.
-(define (rmt:send-receive . params) #f)
+;; (define (rmt:send-receive . params) #f)
 (define (http-transport:close-connections . params) #f)
 ;; from remote defstruct in common.scm
 (define (api:execute-requests . params) #f)
-
-(define (set-functions send-receive        rsus
-		       close-connections   rcs
-		       dbgp                dbgpinfo
-		       dbgperr
-		       ro-mode             ro-mode-set
-		       ro-mode-checked-set ro-mode-checked
-
-		       dbfile-path         dbsetup
-		       exec-req            read-only-queries
-		       ) 
-  (set! rmt:send-receive            send-receive)
-  (set! remote-server-url-set!           rsus)
-  (set! http-transport:close-connections close-connections)
-  (set! remote-conndat-set!              rcs)
-  ;; print stuff
-  (set! debug:print                      dbgp)
-  (set! debug:print-info                 dbgpinfo)
-  (set! debug:print-error                dbgperr)
-  ;;
-  ;; db stuff for local db access
-  (set! apt:execute-requests             exec-req)
-  )
+(define (api:read-only-queries . params) #f)
+(define (http-transport:client-api-send-receive . params) #f)
+(define (client:setup . params) #f)
+(define (server:kind-run . params) #f)
+(define (server:start-and-wait . params) #f)
+(define (server:check-if-running . params) #f)
+(define (server:ping . params) #f)
+(define (common:force-server? . params) #f)
+;; 'send-receive rmt:send-receive ...
+(define (set-functions . alldata)
+  (match
+   alldata
+   ((a b c d e f g h i j) ;; e f g h i j k l)
+    (set! http-transport:client-api-send-receive a)
+    (set! http-transport:close-connections       b)
+    (set! apt:execute-requests                   d)
+    (set! client:setup                           e)
+    (set! server:kind-run                        f)
+    (set! server:start-and-wait                  g)
+    (set! server:check-if-running                h)
+    (set! server:ping                            i)
+    (set! common:force-server?                   j)
+    )))
 
 (define (rmt:open-qry-close-locally log-port multi-sync-mutex cmd run-id params #!key (ro-queries '())(remretries 5))
   (let* ((qry-is-write   (not (member cmd ro-queries)))
 	 (db-file-path   (exec-fn 'db:dbfile-path)) ;;  0))
 	 (dbstruct-local (exec-fn 'db:setup #t))  ;; make-dbr:dbstruct path:  dbdir local: #t)))
@@ -122,21 +122,21 @@
   ;;(mutex-unlock! rmt-mutex)
   (debug:print-info 12 log-port "rmt:send-receive, case 3")
   (debug:print 0 log-port "WARNING: write transaction requested on a readonly area.  cmd="cmd" params="params)
   #f)
 
-(define (extras-transport-failed log-port rmt-mutex attemptnum runremote cmd rid params)
+(define (extras-transport-failed log-port rmt-mutex attemptnum runremote areapath cmd rid params)
   (debug:print 0 log-port "WARNING: communication failed. Trying again, try num: " attemptnum)
   ;;(mutex-lock! rmt-mutex)
   (remote-conndat-set!    runremote #f)
   (http-transport:close-connections area-dat: runremote)
   (remote-server-url-set! runremote #f)
   ;;(mutex-unlock! rmt-mutex)
   (debug:print-info 12 log-port "rmt:send-receive, case  9.1")
-  (rmt:send-receive cmd rid params attemptnum: (+ attemptnum 1)))
+  (rmt:send-receive-orig log-port runremote rmt-mutex areapath cmd rid params attemptnum: (+ attemptnum 1)))
   
-(define (extras-transport-succeded log-port rmt-mutex attemptnum runremote res params rid cmd)
+(define (extras-transport-succeded log-port rmt-mutex attemptnum runremote areapath res params rid cmd)
   (if (and (vector? res)
 	   (eq? (vector-length res) 2)
 	   (eq? (vector-ref res 1) 'overloaded)) ;; since we are
 						 ;; looking at the
 						 ;; data to carry the
@@ -159,12 +159,192 @@
 	(http-transport:close-connections area-dat: runremote)
 	;; (set! *runremote* #f) ;; force starting over
 	(remote-server-url-set! runremote #f) ;; I am hoping this will force a redo on server connection. NOT TESTED
 	;;(mutex-unlock! rmt-mutex)
 	(thread-sleep! wait-delay)
-	(rmt:send-receive cmd rid params attemptnum: (+ attemptnum 1)))
+	(rmt:send-receive-orig log-port runremote rmt-mutex areapath cmd rid params attemptnum: (+ attemptnum 1)))
       res)) ;; All good, return res
+
+;; RA => e.g. usage (rmt:send-receive 'get-var #f (list varname))
+;;
+;;  add multi-sync-mutex 
+;;
+(define (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected
+
+  #;(common:telemetry-log (conc "rmt:"(->string cmd))
+                        payload: `((rid . ,rid)
+                                   (params . ,params)))
+                          
+  
+  ;; do all the prep locked under the rmt-mutex
+  ;;(mutex-lock! rmt-mutex)
+  
+  ;; 1. check if server is started IFF cmd is a write OR if we are not on the homehost, store in runremote
+  ;; 2. check the age of the connections. refresh the connection if it is older than timeout-20 seconds.
+  ;; 3. do the query, if on homehost use local access
+  ;;
+  (let* ((start-time    (current-seconds)) ;; snapshot time so all use cases get same value
+
+	 (readonly-mode (rmtmod:calc-ro-mode runremote toppath)))
+
+    ;; (assert (not (pair? (remote-hh-dat runremote))))
+    
+    ;;(print "BB> readonly-mode is "readonly-mode" dbfile is "dbfile)
+    (cond
+     ;; give up if more than 15 attempts
+     ((> attemptnum 15)
+      (debug:print 0 log-port "ERROR: 15 tries to start/connect to server. Giving up.")
+      (exit 1))
+
+     ;; readonly mode, read request-  handle it - case 2
+     ((and readonly-mode
+           (member cmd api:read-only-queries)) 
+      ;; (mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case 2")
+      (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries)
+      )
+
+     ;; readonly mode, write request.  Do nothing, return #f
+     (readonly-mode (extras-readonly-mode rmt-mutex log-port cmd params))
+
+     ;; This block was for pre-emptively resetting the connection if there had been no communication for some time.
+     ;; I don't think it adds any value. If the server is not there, just fail and start a new connection.
+     ;; also, the expire-time calculation might not be correct. We want, time-since-last-server-access > (server:get-timeout)
+     ;;
+     ;; reset the connection if it has been unused too long
+     ((and runremote
+           (remote-conndat runremote)
+	   (> (current-seconds) ;; if it has been more than server-timeout seconds since last contact, close this connection and start a new on
+	      (+ (http-transport:server-dat-get-last-access (remote-conndat runremote))
+		 (remote-server-timeout runremote))))
+      (debug:print-info 0 log-port "Connection to " (remote-server-url runremote) " expired due to no accesses, forcing new connection.")
+      (http-transport:close-connections area-dat: runremote)
+      (remote-conndat-set! runremote #f) ;; invalidate the connection, thus forcing a new connection.
+      ;; (mutex-unlock! rmt-mutex)
+      (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum))
+     
+
+     ;; on homehost and this is a read
+     ((and (not (remote-force-server runremote)) ;; honor forced use of server, i.e. server NOT required
+	   (pair? (remote-hh-dat runremote))
+	   (cdr (remote-hh-dat runremote))       ;; on homehost
+           (member cmd api:read-only-queries))   ;; this is a read
+      ;; (mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case  5")
+      (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
+
+     ;; on homehost and this is a write, we already have a server, but server has died
+     ((and (cdr (remote-hh-dat runremote))           ;; on homehost
+           (not (member cmd api:read-only-queries))  ;; this is a write
+           (remote-server-url runremote)             ;; have a server
+           (not (server:ping (remote-server-url runremote))))  ;; server has died. NOTE: this is not a cheap call! Need better approach.
+      ;; (set! *runremote* (make-remote)) ;; WARNING - broken this.
+      (remote-force-server-set! runremote (common:force-server?))
+      ;; (mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case  6")
+      (rmt:send-receive-orig log-port runremote rmt-mutex toppath  multi-sync-mutex cmd rid params attemptnum: attemptnum))
+
+     ;; on homehost and this is a write, we already have a server
+     ((and (not (remote-force-server runremote))     ;; honor forced use of server, i.e. server NOT required
+	   (cdr (remote-hh-dat runremote))           ;; on homehost
+           (not (member cmd api:read-only-queries))  ;; this is a write
+           (remote-server-url runremote))            ;; have a server
+      ;;(mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case  4.1")
+      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
+
+     ;;  on homehost, no server contact made and this is a write, passively start a server 
+     ((and (not (remote-force-server runremote))     ;; honor forced use of server, i.e. server NOT required
+	   (cdr (remote-hh-dat runremote))           ;; have homehost
+           (not (remote-server-url runremote))       ;; no connection yet
+	   (not (member cmd api:read-only-queries))) ;; not a read-only query
+      (debug:print-info 12 log-port "rmt:send-receive, case  8")
+      (let ((server-url  (server:check-if-running toppath))) ;; (server:read-dotserver->url toppath))) ;; (server:check-if-running toppath))) ;; Do NOT want to run server:check-if-running - very expensive to do for every write call
+	(if server-url
+	    (remote-server-url-set! runremote server-url) ;; the string can be consumed by the client setup if needed
+	    (if (common:force-server?)
+		(server:start-and-wait toppath)
+		(server:kind-run toppath))))
+      (remote-force-server-set! runremote (common:force-server?))
+      ;; (mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case  8.1")
+      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd 0 params ro-queries: api:read-only-queries))
+
+     ((or (and (remote-force-server runremote)              ;; we are forcing a server and don't yet have a connection to one
+	       (not (remote-conndat runremote)))
+	  (and (not (cdr (remote-hh-dat runremote)))        ;; not on a homehost 
+	       (not (remote-conndat runremote))))           ;; and no connection
+      (debug:print-info 12 log-port "rmt:send-receive, case 9, hh-dat: " (remote-hh-dat runremote) " conndat: " (remote-conndat runremote))
+      ;;(mutex-unlock! rmt-mutex)
+      (if (not (server:check-if-running toppath)) ;; who knows, maybe one has started up?
+	  (server:start-and-wait toppath))
+      (remote-conndat-set! runremote (rmt:get-connection-info runremote toppath)) ;; calls client:setup which calls client:setup-http
+      (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum)) ;; TODO: add back-off timeout as
+
+     ;; all set up if get this far, dispatch the query
+     ((and (not (remote-force-server runremote))
+	   (cdr (remote-hh-dat runremote))) ;; we are on homehost
+      ;;(mutex-unlock! rmt-mutex)
+      (debug:print-info 12 log-port "rmt:send-receive, case 10")
+      (rmt:open-qry-close-locally  log-port multi-sync-mutex cmd (if rid rid 0) params ro-queries: api:read-only-queries))
+
+     ;; not on homehost, do server query
+     (else (extras-case-11 log-port rmt-mutex runremote toppath cmd params attemptnum rid)))))
+
+(define (extras-case-11 log-port rmt-mutex runremote areapath cmd params attemptnum rid)
+  ;; (mutex-unlock! rmt-mutex)
+  (debug:print-info 12 log-port "rmt:send-receive, case  9")
+  ;; (mutex-lock! rmt-mutex)
+  (let* ((conninfo (remote-conndat runremote))
+	 (dat      (case (remote-transport runremote)
+		     ((http) (condition-case ;; handling here has
+					     ;; caused a lot of
+					     ;; problems. However it
+					     ;; is needed to deal with
+					     ;; attemtped
+					     ;; communication to
+					     ;; servers that have gone
+					     ;; away
+			      (http-transport:client-api-send-receive 0 conninfo cmd params)
+			      ((commfail)(vector #f "communications fail"))
+			      ((exn)(vector #f "other fail" (print-call-chain)))))
+		     (else
+		      (debug:print 0 log-port "ERROR: transport " (remote-transport runremote) " not supported")
+		      (exit))))
+	 (success  (if (vector? dat) (vector-ref dat 0) #f))
+	 (res      (if (vector? dat) (vector-ref dat 1) #f)))
+    (if (and (vector? conninfo) (< 5 (vector-length conninfo)))
+	(http-transport:server-dat-update-last-access conninfo) ;; refresh access time
+	(begin
+	  (debug:print 0 log-port "INFO: Should not get here! conninfo=" conninfo)
+	  (set! conninfo #f)
+	  (remote-conndat-set! runremote #f) ;; NOTE: *runremote* is global copy of runremote. Purpose: factor out global.
+	  (http-transport:close-connections  area-dat: runremote)))
+    (debug:print-info 13 log-port "rmt:send-receive, case  9. conninfo=" conninfo " dat=" dat " runremote = " runremote)
+    ;; (mutex-unlock! rmt-mutex)
+    (if success ;; success only tells us that the transport was
+	;; successful, have to examine the data to see if
+	;; there was a detected issue at the other end
+	(extras-transport-succeded log-port rmt-mutex attemptnum runremote areapath res params rid cmd)
+	(extras-transport-failed log-port rmt-mutex attemptnum runremote areapath cmd rid params)
+	)))
+
+;; if a server is either running or in the process of starting call client:setup
+;; else return #f to let the calling proc know that there is no server available
+;;
+(define (rmt:get-connection-info runremote areapath #!key (area-dat #f)) ;; TODO: push areapath down.
+  (let* (;; (runremote (or area-dat runremote))
+	 (cinfo     (if (remote? runremote)
+			(remote-conndat runremote)
+			#f)))
+	  (if cinfo
+	      cinfo
+	      (if (server:check-if-running areapath)
+		  (client:setup runremote areapath)
+		  #f))))
+
+
 
 ;;======================================================================
 ;; ulex and steps stuff
 ;;======================================================================