Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -76,15 +76,17 @@ (mutex-lock! *send-receive-mutex*) (if (not *runremote*)(set! *runremote* (make-remote))) (let ((ulex:conn (remote-ulex:conn *runremote*))) (if (not ulex:conn)(remote-ulex:conn-set! *runremote* (rmtmod:setup-ulex *toppath*))) (rmtmod:send-receive-ulex ulex:conn cmd rid params attemptnum area-dat))) - (rmt:send-receive-orig *default-log-port* runremote *rmt-mutex* areapath cmd rid params attemptnum: attemptnum area-dat: area-dat)))) + (rmt:send-receive-orig *default-log-port* runremote *rmt-mutex* areapath *db-multi-sync-mutex* cmd rid params attemptnum: attemptnum area-dat: area-dat)))) ;; RA => e.g. usage (rmt:send-receive 'get-var #f (list varname)) ;; -(define (rmt:send-receive-orig log-port runremote rmt-mutex toppath cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected +;; add multi-sync-mutex +;; +(define (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected #;(common:telemetry-log (conc "rmt:"(->string cmd)) payload: `((rid . ,rid) (params . ,params))) @@ -115,11 +117,11 @@ ;; readonly mode, read request- handle it - case 2 ((and readonly-mode (member cmd api:read-only-queries)) (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 2") - (rmt:open-qry-close-locally cmd 0 params) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params) ) ;; readonly mode, write request. Do nothing, return #f (readonly-mode (extras-readonly-mode rmt-mutex log-port cmd params)) @@ -135,20 +137,20 @@ (remote-server-timeout runremote)))) (debug:print-info 0 log-port "Connection to " (remote-server-url runremote) " expired due to no accesses, forcing new connection.") (http-transport:close-connections area-dat: runremote) (remote-conndat-set! runremote #f) ;; invalidate the connection, thus forcing a new connection. (mutex-unlock! rmt-mutex) - (rmt:send-receive-orig log-port runremote rmt-mutex toppath cmd rid params attemptnum: attemptnum)) + (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum)) ;; on homehost and this is a read ((and (not (remote-force-server runremote)) ;; honor forced use of server, i.e. server NOT required (cdr (remote-hh-dat runremote)) ;; on homehost (member cmd api:read-only-queries)) ;; this is a read (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 5") - (rmt:open-qry-close-locally cmd 0 params)) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params)) ;; on homehost and this is a write, we already have a server, but server has died ((and (cdr (remote-hh-dat runremote)) ;; on homehost (not (member cmd api:read-only-queries)) ;; this is a write (remote-server-url runremote) ;; have a server @@ -155,20 +157,20 @@ (not (server:ping (remote-server-url runremote)))) ;; server has died. NOTE: this is not a cheap call! Need better approach. (set! *runremote* (make-remote)) (remote-force-server-set! runremote (common:force-server?)) (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 6") - (rmt:send-receive-orig log-port runremote rmt-mutex toppath cmd rid params attemptnum: attemptnum)) + (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum)) ;; on homehost and this is a write, we already have a server ((and (not (remote-force-server runremote)) ;; honor forced use of server, i.e. server NOT required (cdr (remote-hh-dat runremote)) ;; on homehost (not (member cmd api:read-only-queries)) ;; this is a write (remote-server-url runremote)) ;; have a server (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 4.1") - (rmt:open-qry-close-locally cmd 0 params)) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params)) ;; on homehost, no server contact made and this is a write, passively start a server ((and (not (remote-force-server runremote)) ;; honor forced use of server, i.e. server NOT required (cdr (remote-hh-dat runremote)) ;; have homehost (not (remote-server-url runremote)) ;; no connection yet @@ -181,11 +183,11 @@ (server:start-and-wait toppath) (server:kind-run toppath)))) (remote-force-server-set! runremote (common:force-server?)) (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 8.1") - (rmt:open-qry-close-locally cmd 0 params)) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd 0 params)) ((or (and (remote-force-server runremote) ;; we are forcing a server and don't yet have a connection to one (not (remote-conndat runremote))) (and (not (cdr (remote-hh-dat runremote))) ;; not on a homehost (not (remote-conndat runremote)))) ;; and no connection @@ -192,18 +194,18 @@ (debug:print-info 12 log-port "rmt:send-receive, case 9, hh-dat: " (remote-hh-dat runremote) " conndat: " (remote-conndat runremote)) (mutex-unlock! rmt-mutex) (if (not (server:check-if-running toppath)) ;; who knows, maybe one has started up? (server:start-and-wait toppath)) (remote-conndat-set! runremote (rmt:get-connection-info toppath)) ;; calls client:setup which calls client:setup-http - (rmt:send-receive-orig log-port runremote rmt-mutex toppath cmd rid params attemptnum: attemptnum)) ;; TODO: add back-off timeout as + (rmt:send-receive-orig log-port runremote rmt-mutex toppath multi-sync-mutex cmd rid params attemptnum: attemptnum)) ;; TODO: add back-off timeout as ;; all set up if get this far, dispatch the query ((and (not (remote-force-server runremote)) (cdr (remote-hh-dat runremote))) ;; we are on homehost (mutex-unlock! rmt-mutex) (debug:print-info 12 log-port "rmt:send-receive, case 10") - (rmt:open-qry-close-locally cmd (if rid rid 0) params)) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd (if rid rid 0) params)) ;; not on homehost, do server query (else (extras-case-11 log-port runremote cmd params attemptnum rid))))) ;; bunch of small functions factored out of send-receive to make debug easier @@ -301,11 +303,11 @@ (cons 'none 0)) (loop (car tal)(cdr tal) newmax-cmd currmax))))))) (mutex-unlock! *db-stats-mutex*) res)) -(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5)) +(define (rmt:open-qry-close-locally log-port multi-sync-mutex cmd run-id params #!key (remretries 5)) (let* ((qry-is-write (not (member cmd api:read-only-queries))) (db-file-path (db:dbfile-path)) ;; 0)) (dbstruct-local (db:setup #t)) ;; make-dbr:dbstruct path: dbdir local: #t))) (read-only (not (file-write-access? db-file-path))) (start (current-milliseconds)) @@ -312,11 +314,11 @@ (resdat (if (not (and read-only qry-is-write)) (let ((v (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)))) (handle-exceptions ;; there has been a long history of receiving strange errors from values returned by the client when things go wrong.. exn ;; This is an attempt to detect that situation and recover gracefully (begin - (debug:print0 *default-log-port* "ERROR: bad data from server " v " message: " ((condition-property-accessor 'exn 'message) exn)) + (debug:print0 log-port "ERROR: bad data from server " v " message: " ((condition-property-accessor 'exn 'message) exn)) (vector #t '())) ;; should always get a vector but if something goes wrong return a dummy (if (and (vector? v) (> (vector-length v) 1)) (let ((newvec (vector (vector-ref v 0)(vector-ref v 1)))) newvec) ;; by copying the vector while inside the error handler we should force the detection of a corrupted record @@ -324,28 +326,28 @@ (vector #t '()))) (success (vector-ref resdat 0)) (res (vector-ref resdat 1)) (duration (- (current-milliseconds) start))) (if (and read-only qry-is-write) - (debug:print 0 *default-log-port* "ERROR: attempt to write to read-only database ignored. cmd=" cmd)) + (debug:print 0 log-port "ERROR: attempt to write to read-only database ignored. cmd=" cmd)) (if (not success) (if (> remretries 0) (begin - (debug:print-error 0 *default-log-port* "local query failed. Trying again.") + (debug:print-error 0 log-port "local query failed. Trying again.") (thread-sleep! (/ (random 5000) 1000)) ;; some random delay - (rmt:open-qry-close-locally cmd run-id params remretries: (- remretries 1))) + (rmt:open-qry-close-locally log-port multi-sync-mutex cmd run-id params remretries: (- remretries 1))) (begin - (debug:print-error 0 *default-log-port* "too many retries in rmt:open-qry-close-locally, giving up") + (debug:print-error 0 log-port "too many retries in rmt:open-qry-close-locally, giving up") #f)) (begin ;; (rmt:update-db-stats run-id cmd params duration) ;; mark this run as dirty if this was a write, the watchdog is responsible for syncing it (if qry-is-write (let ((start-time (current-seconds))) - (mutex-lock! *db-multi-sync-mutex*) -/ (set! *db-last-access* start-time) ;; THIS IS PROBABLY USELESS? (we are on a client) - (mutex-unlock! *db-multi-sync-mutex*))))) + (mutex-lock! multi-sync-mutex) + ;; (set! *db-last-access* start-time) ;; THIS IS PROBABLY USELESS? (we are on a client) + (mutex-unlock! multi-sync-mutex))))) res)) (define (rmt:send-receive-no-auto-client-setup connection-info cmd run-id params) (let* ((run-id (if run-id run-id 0)) (res (handle-exceptions