Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -77,27 +77,25 @@ (define *waiting-queue* (make-hash-table)) (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar -(define *write-frequency* (make-hash-table)) ;; run-id => (vector (current-seconds) 0)) (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) (define *time-zero* (current-seconds)) ;; for the watchdog ;; DATABASE -(define *dbstruct-db* #f) +(define *dbstruct-db* #f) ;; used when local access is triggered in rmt.scm + (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) (define *db-sync-mutex* (make-mutex)) (define *db-multi-sync-mutex* (make-mutex)) (define *db-local-sync* (make-hash-table)) ;; used to record last touch of db -(define *megatest-db* #f) (define *last-db-access* (current-seconds)) ;; update when db is accessed via server (define *db-write-access* #t) -(define *inmemdb* #f) (define *task-db* #f) ;; (vector db path-to-db) (define *db-access-allowed* #t) ;; flag to allow access (define *db-access-mutex* (make-mutex)) (define *db-cache-path* #f) @@ -106,11 +104,10 @@ (define *transport-type* 'http) (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg (define *runremote* (make-hash-table)) ;; if set up for server communication this will hold (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) -(define *client-non-blocking-mode* #f) (define *server-id* #f) (define *server-info* #f) (define *time-to-exit* #f) (define *received-response* #f) (define *default-numtries* 10) @@ -569,25 +566,11 @@ #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") (if (and no-hurry (debug:debug-mode 18)) (rmt:print-db-stats)) (let ((th1 (make-thread (lambda () ;; thread for cleaning up, give it five seconds - ;; (let ((run-ids (hash-table-keys *db-local-sync*))) - ;; (if (and (not (null? run-ids)) - ;; (or (common:legacy-sync-recommended) - ;; (configf:lookup *configdat* "setup" "megatest-db"))) - ;; (if no-hurry - ;; (db:multi-db-sync run-ids 'new2old)) - ;; )) - (if *dbstruct-db* (db:close-all *dbstruct-db*)) - (if *inmemdb* (db:close-all *inmemdb*)) - (if (and *megatest-db* - (sqlite3:database? *megatest-db*)) - (begin - (sqlite3:interrupt! *megatest-db*) - (sqlite3:finalize! *megatest-db* #t) - (set! *megatest-db* #f))) + (if *dbstruct-db* (db:close-all *dbstruct-db*)) ;; one second allocated (if *task-db* (let ((db (cdr *task-db*))) (if (sqlite3:database? db) (begin (sqlite3:interrupt! db) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -275,23 +275,17 @@ ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; ;; called in http-transport and replicated in rmt.scm for *local* access. ;; (define (db:setup #!key (areapath #f)) ;; . junk) ;; #!key (run-id #f) (local #f)) - (let* (;; (dbdir (db:dbfile-path)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) - (dbstruct (make-dbr:dbstruct))) ;; ) ;; path: dbdir local: local))) - (db:open-db dbstruct areapath: #f) - dbstruct)) - -;; open the local db for direct access (no server) -;; -(define (db:open-local-db-handle) (or *dbstruct-db* - (let ((dbstruct (db:setup))) ;; #f local: #t))) + (let* (;; (dbdir (db:dbfile-path)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) + (dbstruct (make-dbr:dbstruct))) ;; ) ;; path: dbdir local: local))) + (db:open-db dbstruct areapath: #f) (set! *dbstruct-db* dbstruct) dbstruct))) - + ;; Open the classic megatest.db file (defaults to open in toppath) ;; ;; NOTE: returns a dbdat not a dbstruct! ;; (define (db:open-megatest-db #!key (path #f)(name #f)) @@ -320,11 +314,11 @@ (refndb (dbr:dbstruct-refndb dbstruct)) ;; (runid (dbr:dbstruct-run-id dbstruct)) ) (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id) ;; (mutex-lock! *http-mutex*) - (db:sync-tables (db:sync-all-tables-list tmpdb) #f tmpdb refndb mtdb))) + (db:sync-tables (db:sync-all-tables-list dbstruct) #f tmpdb refndb mtdb))) ;; (if (eq? run-id 0) ;; ;; runid equal to 0 is main.db ;; (if maindb ;; (if (or (not (number? mtime)) ;; (not (number? stime)) @@ -3211,11 +3205,11 @@ statuses) statuses))) *common:std-statuses* >)) (newstate (if (null? all-curr-states) "NOT_STARTED" (car all-curr-states))) (newstatus (if (null? all-curr-statuses) "n/a" (car all-curr-statuses)))) - (print "Setting toplevel to: " newstate "/" newstatus) + ;; (print "Setting toplevel to: " newstate "/" newstatus) (db:test-set-state-status-by-id dbstruct run-id tl-test-id newstate newstatus #f))))))) (define db:roll-up-pass-fail-counts db:set-state-status-and-roll-up-items) ;; call with state = #f to roll up with out accounting for state/status of this item @@ -3288,11 +3282,11 @@ (if (directory? path) (debug:print 2 *default-log-port* "Found path: " path) (debug:print 2 *default-log-port* "No such path: " path))) ;; ) db "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='' AND run_id=?;" - test-name) + test-name run-id) res)))) ;;====================================================================== ;; A G R E G A T E D T R A N S A C T I O N D B W R I T E S ;;====================================================================== @@ -3517,11 +3511,11 @@ ;; Run this remotely!! ;; (define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path) (let* ((dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat)) - (keys (db:get-keys db)) + (keys (db:get-keys dbstruct)) (selstr (string-intersperse keys ",")) (qrystr (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")) (keyvals #f) (tests-hash (make-hash-table))) ;; first look up the key values from the run selected by run-id Index: fs-transport.scm ================================================================== --- fs-transport.scm +++ fs-transport.scm @@ -35,10 +35,10 @@ ;; There is no "server" per se but a convience routine to make it non ;; necessary to be reopening the db over and over again. ;; (define (fs:process-queue-item packet) - (if (not *megatest-db*) ;; we will require that (setup-for-run) has already been called - (set! *megatest-db* (open-db))) + (if (not *dbstruct-db*) ;; we will require that (setup-for-run) has already been called + (set! *dbstruct-db* (db:setup-db))) (debug:print-info 11 *default-log-port* "fs:process-queue-item called with packet=" packet) - (db:process-queue-item *megatest-db* packet)) + (db:process-queue-item *dbstruct-db* packet)) Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -58,11 +58,10 @@ (server:get-best-guess-address hostname) #f))) (if ipstr ipstr hostn))) ;; hostname))) (start-port (portlogger:open-run-close portlogger:find-port)) (link-tree-path (configf:lookup *configdat* "setup" "linktree"))) - ;; (set! db *inmemdb*) (debug:print-info 0 *default-log-port* "portlogger recommended port: " start-port) (root-path (if link-tree-path link-tree-path (current-directory))) ;; WARNING: SECURITY HOLE. FIX ASAP! (handle-directory spiffy-directory-listing) @@ -82,11 +81,11 @@ (dat ($ 'dat)) (res #f)) (cond ((equal? (uri-path (request-uri (current-request))) '(/ "api")) - (send-response body: (api:process-request *inmemdb* $) ;; the $ is the request vars proc + (send-response body: (api:process-request *dbstruct-db* $) ;; the $ is the request vars proc headers: '((content-type text/plain))) (mutex-lock! *heartbeat-mutex*) (set! *last-db-access* (current-seconds)) (mutex-unlock! *heartbeat-mutex*)) ((equal? (uri-path (request-uri (current-request))) @@ -393,18 +392,17 @@ (server-timeout (server:get-timeout))) (let loop ((count 0) (server-state 'available) (bad-sync-count 0)) - ;; Use this opportunity to sync the inmemdb to db - (if *inmemdb* + ;; Use this opportunity to sync the tmp db to megatest.db + (if *dbstruct-db* (let ((start-time (current-milliseconds)) (sync-time #f) (rem-time #f)) - ;; inmemdb is a dbstruct (condition-case - (db:sync-touched *inmemdb* *run-id* force-sync: #t) + (db:sync-touched *dbstruct-db* *run-id* force-sync: #t) ((sync-failed)(cond ((> bad-sync-count 10) ;; time to give up (http-transport:server-shutdown server-id port)) (else ;; (> bad-sync-count 0) ;; we've had a fail or two, delay and loop (thread-sleep! 5) @@ -421,22 +419,19 @@ (> rem-time 0)) (thread-sleep! rem-time) (thread-sleep! 4))) ;; fallback for if the math is changed ... ;; - ;; no *inmemdb* yet, set running after our first pass through and start the db + ;; no *dbstruct-db* yet, set running after our first pass through and start the db ;; (if (eq? server-state 'available) (let ((new-server-id (tasks:server-am-i-the-server? (db:delay-if-busy tdbdat) run-id))) ;; try to ensure no double registering of servers (if (equal? new-server-id server-id) (begin (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep") (thread-sleep! 0.5) ;; give some margin for queries to complete before switching from file based access to server based access - (set! *inmemdb* (db:setup)) ;; run-id)) - ;; force initialization - ;; (db:get-db *inmemdb* #t) - ;; (db:get-db *inmemdb* run-id) + (set! *dbstruct-db* (db:setup)) ;; run-id)) (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running")) (begin ;; gotta exit nicely (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "collision") (http-transport:server-shutdown server-id port)))))) @@ -490,12 +485,12 @@ (define (http-transport:server-shutdown server-id port) (let ((tdbdat (tasks:open-db))) (debug:print-info 0 *default-log-port* "Starting to shutdown the server.") ;; need to delete only *my* server entry (future use) - (set! *time-to-exit* #t) - (if *inmemdb* (db:sync-touched *inmemdb* *run-id* force-sync: #t)) + (if *dbstruct-db* (db:sync-touched *dbstruct-db* *run-id* force-sync: #t)) + (set! *time-to-exit* #t) ;; tell on-exit to be fast as we've already cleaned up ;; ;; start_shutdown ;; (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "shutting-down") (portlogger:open-run-close portlogger:set-port port "released") Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -764,13 +764,10 @@ ;; (client:launch run-id) ;; (client:launch 0) ;; without run-id we'll start a server for "0" #t )))))) -;; MAY STILL NEED THIS -;; (set! *megatest-db* (make-dbr:dbstruct path: *toppath* local: #t)))))))))) - (if (or (args:get-arg "-list-servers") (args:get-arg "-stop-server") (args:get-arg "-kill-server")) (let ((tl (launch:setup))) (if tl @@ -1897,11 +1894,11 @@ (if (or (getenv "MT_RUNSCRIPT") (args:get-arg "-repl") (args:get-arg "-load")) (let* ((toppath (launch:setup)) - (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f))) + (dbstruct (if toppath (db:setup)))) ;; make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f))) (if dbstruct (cond ((getenv "MT_RUNSCRIPT") ;; How to run megatest scripts ;; @@ -1915,11 +1912,10 @@ (repl)) (else (begin (set! *db* dbstruct) - (set! *client-non-blocking-mode* #t) (import extras) ;; might not be needed ;; (import csi) (import readline) (import apropos) ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ... @@ -1934,11 +1930,12 @@ (or (get-environment-variable "HOME") ".") "/.megatest_history")) (current-input-port (make-gnu-readline-port "megatest> ")))) (if (args:get-arg "-repl") (repl) (load (args:get-arg "-load"))) - (db:close-all dbstruct)) + ;; (db:close-all dbstruct) <= taken care of by on-exit call + ) (exit))) (set! *didsomething* #t)))) ;;====================================================================== ;; Wait on a run to complete Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -219,13 +219,13 @@ (define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5)) (let* ((qry-is-write (not (member cmd api:read-only-queries))) (db-file-path (db:dbfile-path)) ;; 0)) (dbstruct-local (if *dbstruct-db* *dbstruct-db* - (let* ((db (db:setup))) ;; make-dbr:dbstruct path: dbdir local: #t))) - (set! *dbstruct-db* db) - db))) + (let* ((dbstruct (db:setup))) ;; make-dbr:dbstruct path: dbdir local: #t))) + (set! *dbstruct-db* dbstruct) + dbstruct))) (read-only (not (file-write-access? db-file-path))) (start (current-milliseconds)) (resdat (if (not (and read-only qry-is-write)) (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)) (vector #t '()))) Index: rpc-transport.scm ================================================================== --- rpc-transport.scm +++ rpc-transport.scm @@ -96,11 +96,11 @@ #f)) (portnum (rpc:default-server-port)) (host:port (conc (if ipaddrstr ipaddrstr hostname) ":" portnum)) (tdb (tasks:open-db))) (thread-start! th1) - (set! db *inmemdb*) + (set! db *dbstruct-db*) (open-run-close tasks:server-set-interface-port tasks:open-db server-id ipaddrstr portnum) (debug:print 0 *default-log-port* "Server started on " host:port) @@ -116,11 +116,11 @@ (on-exit (lambda () (open-run-close tasks:server-set-state! tasks:open-db server-id "stopped"))) (set! *rpc:listener* rpc:listener) (tasks:server-set-state! tdb server-id "running") - (set! *inmemdb* (db:setup run-id)) + (set! *dbstruct-db* (db:setup run-id)) ;; if none running or if > 20 seconds since ;; server last used then start shutdown (let loop ((count 0)) (thread-sleep! 5) ;; no need to do this very often (let ((numrunning -1)) ;; (db:get-count-tests-running db)))