Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -133,11 +133,11 @@ ;; TEST DATA test-data-rollup csv->test-data ;; MISC - sync-inmem->db + sync-cachedb->db drop-all-triggers create-all-triggers update-tesdata-on-repilcate-db ;; TESTMETA @@ -344,11 +344,11 @@ ;; TEST DATA ((test-data-rollup) (apply db:test-data-rollup dbstruct params)) ((csv->test-data) (apply db:csv->test-data dbstruct params)) ;; MISC - ((sync-inmem->db) (let ((run-id (car params))) + ((sync-cachedb->db) (let ((run-id (car params))) (db:sync-touched dbstruct run-id db:initialize-main-db force-sync: #t))) ((get-toplevels-and-incompletes) (apply db:get-toplevels-and-incompletes dbstruct params)) ((mark-incomplete) #f);;(thread-start! (make-thread (lambda () ;; no need to block on this one ;; (apply db:find-and-mark-incomplete dbstruct params) ;; #t)))) Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -98,11 +98,11 @@ (include "megatest-version.scm") (include "megatest-fossil-hash.scm") ;; http - use the old http + in /tmp db -;; tcp - use tcp transport with inmem db +;; tcp - use tcp transport with cachedb db ;; nfs - use direct to disk access (read-only) ;; (define rmt:transport-mode (make-parameter 'tcp)) (define (get-full-version) Index: dashboard-transport-mode.scm.template ================================================================== --- dashboard-transport-mode.scm.template +++ dashboard-transport-mode.scm.template @@ -1,22 +1,22 @@ ;;====================================================================== ;; set up transport, db cache and sync methods ;; ;; sync-method: 'original, 'attach or 'none -;; cache-method: 'tmp, 'inmem or 'none +;; cache-method: 'tmp or 'none ;; rmt:transport-mode: 'http, 'tcp, 'nfs ;; ;; NOTE: NOT ALL COMBINATIONS WORK ;; ;;====================================================================== -;; uncomment this block to test without tcp or inmem +;; uncomment this block to test without tcp or cachedb ;; (dbfile:sync-method 'none) ;; (dbfile:cache-method 'none) ;; (rmt:transport-mode 'nfs) -;; uncomment this block to test with tcp and inmem +;; uncomment this block to test with tcp and cachedb (dbfile:sync-method 'original) (dbfile:cache-method 'none) (rmt:transport-mode 'nfs) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -227,12 +227,12 @@ ;; Get/open a database ;; if run-id => get run specific db ;; if #f => get main db ;; if run-id is a string treat it as a filename -;; if db already open - return inmem -;; if db not open, open inmem, rundb and sync then return inmem +;; if db already open - return cachedb +;; if db not open, open cachedb, rundb and sync then return cachedb ;; inuse gets set automatically for rundb's ;; ;; (define db:get-db db:get-subdb) ;; (define (db:get-db subdb #;dbstruct run-id) ;; RENAME TO db:get-dbh @@ -1574,11 +1574,11 @@ qrystr ))) (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " target: " target " offset: " offset " limit: " count) res)) -;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db) +;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using cachedb db) ??? ;; ;; NOTE: This DOESN'T (necessarily) get the real run ids, but the number of the .db!! (define (db:get-changed-run-ids since-time) (let* ((dbdir (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir")) Index: dbfile.scm ================================================================== --- dbfile.scm +++ dbfile.scm @@ -52,12 +52,12 @@ (define dbfile:testsuite-name (make-parameter #f)) (define keep-age-param (make-parameter 10)) ;; qif file age, if over move to attic (define num-run-dbs (make-parameter 10)) ;; number of db's in .mtdb (define dbfile:sync-method (make-parameter 'attach)) ;; 'attach or 'original -(define dbfile:cache-method (make-parameter 'inmem)) ;; 'direct -(define dbcache-mode (make-parameter 'tmp)) ;; 'inmem, 'tmp (changes what open inmem routine does) +(define dbfile:cache-method (make-parameter 'cachedb)) ;; 'direct +(define dbcache-mode (make-parameter 'tmp)) ;; 'cachedb, 'tmp (changes what open cachedb routine does) ;; 'original - use old condition code ;; 'suicide-mode - create mtrah/stop-the-train with info on what went wrong ;; else use no condition code (should be production mode) @@ -76,13 +76,13 @@ (homehost #f) (tmppath #f) (read-only #f) (subdbs (make-hash-table)) ;; - ;; for the inmem approach (see dbmod.scm) + ;; for the cachedb approach (see dbmod.scm) ;; this is one db per server - (inmem #f) ;; handle for the in memory copy + (cachedb #f) ;; handle for the in memory copy (dbfile #f) ;; path to the db file on disk (dbfname #f) ;; short name of db file on disk (used to validate accessing correct db) (ondiskdb #f) ;; handle for the on-disk file (dbtmpname #f) ;; path to db file in /tmp (non-imem method) (dbdat #f) ;; create a dbdat for the downstream calls such as db:with-db @@ -252,12 +252,12 @@ ;; NOTE: most usage should call dbfile:open-db to get a dbdat ;; ;; if run-id => get run specific db ;; if #f => get main db ;; if run-id is a string treat it as a filename - DON'T use this - we'll get rid of it. -;; if db already open - return inmem -;; if db not open, open inmem, rundb and sync then return inmem +;; if db already open - return cachedb +;; if db not open, open cachedb, rundb and sync then return cachedb ;; inuse gets set automatically for rundb's ;; (define (dbfile:get-dbdat dbstruct run-id) (let* ((subdb (dbfile:get-subdb dbstruct run-id))) (if (stack-empty? (dbr:subdb-dbstack subdb)) Index: dbmod.scm ================================================================== --- dbmod.scm +++ dbmod.scm @@ -65,11 +65,11 @@ run-id )"/"(dbmod:run-id->dbfname run-id))) ;;====================================================================== -;; Read-only inmem cached direct from disk method +;; Read-only cachedb cached direct from disk method ;;====================================================================== (define *dbmod:nfs-db-handles* (make-hash-table)) ;; dbfname -> dbstruct ;; called in rmt.scm nfs-transport-handler @@ -82,20 +82,20 @@ (let* ((newdbstruct (dbmod:open-dbmoddb areapath run-id dbfname init-proc keys syncdir: 'fromdisk))) (hash-table-set! *dbmod:nfs-db-handles* dbfname newdbstruct) newdbstruct)))) ;;====================================================================== -;; The inmem one-db file per server method goes in here +;; The cachedb one-db file per server method goes in here ;;====================================================================== ;; NOTE: the r/w is now w/r, #t=db modified by query, #f=db NOT modified by query (define (dbmod:with-db dbstruct run-id w/r proc params) (let* ((use-mutex (or (and w/r ;; use the mutex on queries that modify the db and for sync to disk (> *api-process-request-count* 5)) ;; when writes are happening throttle more (> *api-process-request-count* 50))) (dbdat (dbmod:open-db dbstruct run-id (dbfile:db-init-proc))) - (dbh (dbr:dbdat-dbh dbdat)) ;; this will be the inmem handle + (dbh (dbr:dbdat-dbh dbdat)) ;; this will be the cachedb handle (dbfile (dbr:dbdat-dbfile dbdat))) ;; if nfs mode do a sync if delta > 2 (let* ((last-update (dbr:dbstruct-last-update dbstruct)) (sync-proc (dbr:dbstruct-sync-proc dbstruct)) (curr-secs (current-seconds))) @@ -114,11 +114,12 @@ res))) (define (db:with-db dbstruct run-id w/r proc . params) (dbmod:with-db dbstruct run-id w/r proc params)) -(define (dbmod:open-inmem-db init-proc #!optional (dbfullname #f)) +;; +(define (dbmod:open-cachedb-db init-proc dbfullname) (let* ((db (if dbfullname (dbmod:safely-open-db dbfullname init-proc #t) (sqlite3:open-database ":memory:"))) (handler (sqlite3:make-busy-timeout 3600))) (sqlite3:set-busy-handler! db handler) @@ -127,20 +128,20 @@ (define (dbmod:open-db dbstruct run-id dbinit) (or (dbr:dbstruct-dbdat dbstruct) (let* ((dbdat (make-dbr:dbdat dbfile: (dbr:dbstruct-dbfile dbstruct) - dbh: (dbr:dbstruct-inmem dbstruct) + dbh: (dbr:dbstruct-cachedb dbstruct) ))) (dbr:dbstruct-dbdat-set! dbstruct dbdat) dbdat))) ;; NOT USED? (define (dbmod:need-on-disk-db-handle) (case (dbfile:cache-method) ((none tmp) #t) - ((inmem) + ((cachedb) (case (dbfile:sync-method) ((original) #t) ((attach) #t) ;; we need it to force creation of the on-disk file - FIXME (else (debug:print 0 *default-log-port* "Unknown dbfile:sync-method setting: " @@ -161,12 +162,12 @@ (init-proc db)) db)))) (define *sync-in-progress* #f) -;; Open the inmem db and the on-disk db -;; populate the inmem db with data +;; Open the cachedb db and the on-disk db +;; populate the cachedb db with data ;; ;; Updates fields in dbstruct ;; Returns dbstruct ;; ;; * This routine creates the db if not found @@ -185,26 +186,26 @@ "-"dbfname))) (if (not (file-exists? tmpdir))(create-directory tmpdir)) ;; check if tmpdb already exists, either delete it or ;; add something to the name fname)) - (inmem (dbmod:open-inmem-db init-proc - ;; (if (eq? (dbfile:cache-method) 'inmem) + (cachedb (dbmod:open-cachedb-db init-proc + ;; (if (eq? (dbfile:cache-method) 'cachedb) ;; #f tmpdb ;; ) )) (write-access (file-write-access? dbpath)) (db (dbmod:safely-open-db dbfullname init-proc write-access)) (tables (db:sync-all-tables-list keys))) - (if (not (and (sqlite3:database? inmem) + (if (not (and (sqlite3:database? cachedb) (sqlite3:database? db))) (begin (debug:print 0 *default-log-port* "ERROR: Failed to properly open "dbfname-in", exiting immediately.") - (exit))) ;; (assert (sqlite3:database? inmem) "FATAL: open-dbmoddb: inmem is not a db") + (exit))) ;; (assert (sqlite3:database? cachedb) "FATAL: open-dbmoddb: cachedb is not a db") ;; (assert (sqlite3:database? db) "FATAL: open-dbmoddb: db is not a db") - (dbr:dbstruct-inmem-set! dbstruct inmem) + (dbr:dbstruct-cachedb-set! dbstruct cachedb) (dbr:dbstruct-ondiskdb-set! dbstruct db) (dbr:dbstruct-dbfile-set! dbstruct dbfullname) (dbr:dbstruct-dbtmpname-set! dbstruct tmpdb) (dbr:dbstruct-dbfname-set! dbstruct dbfname) (dbr:dbstruct-sync-proc-set! dbstruct @@ -212,12 +213,12 @@ (if *sync-in-progress* (debug:print 3 *default-log-port* "WARNING: overlapping calls to sync to disk") (let* ((sync-cmd (conc "megatest -db2db -from "tmpdb" -to "dbfullname"&")) (synclock-file (conc dbfullname".sync-lock"))) ;; (mutex-lock! *db-with-db-mutex*) ;; this mutex is used when overloaded or during a query that modifies the db - ;; (if (eq? (dbfile:cache-method) 'inmem) - ;; (dbmod:sync-gasket tables last-update inmem db + ;; (if (eq? (dbfile:cache-method) 'cachedb) + ;; (dbmod:sync-gasket tables last-update cachedb db ;; dbfullname syncdir keys) (if (< (file-modification-time tmpdb) (file-modification-time dbfullname)) (debug:print 0 *default-log-port* "Skipping sync, "tmpdb" older than "dbfullname) (if (file-exists? synclock-file) @@ -229,38 +230,38 @@ (system sync-cmd) ;; (mutex-unlock! *db-with-db-mutex*) ;; (thread-sleep! 0.5) ;; ensure at least 1/2 second down time between sync calls (set! *sync-in-progress* #f)))))) )))) - ;; (dbmod:sync-tables tables #f db inmem) + ;; (dbmod:sync-tables tables #f db cachedb) ;; (if db - (dbmod:sync-gasket tables #f inmem db dbfullname 'fromdest keys) ;; ) ;; load into inmem + (dbmod:sync-gasket tables #f cachedb db dbfullname 'fromdest keys) ;; ) ;; load into cachedb (dbr:dbstruct-last-update-set! dbstruct (current-seconds)) ;; should this be offset back in time by one second? dbstruct)) ;; (if (eq? syncdir 'todisk) ;; sync to disk normally, sync from in dashboard -;; (dbmod:sync-tables tables last-update inmem db) -;; (dbmod:sync-tables tables last-update db inmem)))) +;; (dbmod:sync-tables tables last-update cachedb db) +;; (dbmod:sync-tables tables last-update db cachedb)))) ;; ;; direction: 'fromdest 'todest ;; -(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction keys) - (assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db") - (assert (sqlite3:database? inmem) "FATAL: sync-gasket: dbh is not a db") +(define (dbmod:sync-gasket tables last-update cachedb dbh dbfname direction keys) + (assert (sqlite3:database? cachedb) "FATAL: sync-gasket: cachedb is not a db") + (assert (sqlite3:database? cachedb) "FATAL: sync-gasket: dbh is not a db") (debug:print-info 0 *default-log-port* "dbmod:sync-gasket called with sync-method="(dbfile:sync-method)) (case (dbfile:sync-method) ((none) #f) ((attach) - (dbmod:attach-sync tables inmem dbfname direction)) + (dbmod:attach-sync tables cachedb dbfname direction)) ((newsync) - (dbmod:new-sync tables inmem dbh dbfname direction)) + (dbmod:new-sync tables cachedb dbh dbfname direction)) (else (case direction ((todisk) ;; i.e. from the cache db to the mtrah db - (dbmod:sync-tables tables last-update keys inmem dbh)) + (dbmod:sync-tables tables last-update keys cachedb dbh)) (else - (dbmod:sync-tables tables last-update keys dbh inmem)))))) + (dbmod:sync-tables tables last-update keys dbh cachedb)))))) (define (dbmod:close-db dbstruct) ;; do final sync to disk file ;; (do-sync ...) (sqlite3:finalize! (dbr:dbstruct-ondiskdb dbstruct))) Index: tcp-transportmod.scm ================================================================== --- tcp-transportmod.scm +++ tcp-transportmod.scm @@ -577,11 +577,11 @@ (exit))) (let* ((last-update (dbr:dbstruct-last-update dbstruct)) (curr-secs (current-seconds))) (if (and (eq? (tt-state ttdat) 'running) - (> (- curr-secs last-update) 3)) ;; every 3-4 seconds update the db? maybe this should be refresh the inmem? + (> (- curr-secs last-update) 3)) ;; every 3-4 seconds update the db? (begin (set! (file-modification-time (tt-servinf-file ttdat)) (current-seconds)) ((dbr:dbstruct-sync-proc dbstruct) last-update) (dbr:dbstruct-last-update-set! dbstruct curr-secs)))) Index: transport-mode.scm.template ================================================================== --- transport-mode.scm.template +++ transport-mode.scm.template @@ -1,22 +1,22 @@ ;;====================================================================== ;; set up transport, db cache and sync methods ;; ;; sync-method: 'original, 'attach or 'none -;; cache-method: 'tmp, 'inmem or 'none +;; cache-method: 'tmp 'none ;; rmt:transport-mode: 'http, 'tcp, 'nfs ;; ;; NOTE: NOT ALL COMBINATIONS WORK ;; ;;====================================================================== -;; uncomment this block to test without tcp or inmem +;; uncomment this block to test without tcp ;; (dbfile:sync-method 'none) ;; (dbfile:cache-method 'none) ;; (rmt:transport-mode 'nfs) -;; uncomment this block to test with tcp and inmem -(dbfile:sync-method 'original) ;; attach) +;; uncomment this block to test with tcp +(dbfile:sync-method 'original) ;; attach) ;; original (dbfile:cache-method 'tmp) (rmt:transport-mode 'tcp)