Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -64,11 +64,11 @@ dbstruct (begin (mutex-lock! *rundb-mutex*) (let ((dbdat (if (or (not run-id) (eq? run-id 0)) - (db:open-main dbstruct) + (db:open-main dbstruct area-dat) (db:open-rundb dbstruct area-dat run-id) ))) ;; db prunning would go here (mutex-unlock! *rundb-mutex*) dbdat)))) @@ -85,11 +85,11 @@ ;; mod-read: ;; 'mod modified data ;; 'read read data ;; -(define (db:done-with dbstruct area-dat run-id mod-read) +(define (db:done-with dbstruct run-id mod-read) (if (not (sqlite3:database? dbstruct)) (begin (mutex-lock! *rundb-mutex*) (if (eq? mod-read 'mod) (dbr:dbstruct-set-mtime! dbstruct (current-milliseconds)) @@ -98,15 +98,15 @@ (mutex-unlock! *rundb-mutex*)))) ;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;") ;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no ;; -(define (db:with-db dbstruct area-dat area-dat run-id r/w proc . params) +(define (db:with-db dbstruct area-dat run-id r/w proc . params) (let* ((dbdat (if (vector? dbstruct) - (db:get-db dbstruct run-id) + (db:get-db dbstruct area-dat run-id) dbstruct)) ;; cheat, allow for passing in a dbdat - (db (db:dbdat-get-db dbdat area-dat))) + (db (db:dbdat-get-db dbdat))) (db:delay-if-busy dbdat area-dat) (handle-exceptions exn (begin (debug:print 0 "ERROR: sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn)) @@ -170,24 +170,24 @@ ;; open an sql database inside a file lock ;; ;; returns: db existed-prior-to-opening ;; -(define (db:lock-create-open fname initproc) +(define (db:lock-create-open fname initproc area-dat) (if (file-exists? fname) (let ((db (sqlite3:open-database fname))) (sqlite3:set-busy-handler! db (make-busy-timeout 136000)) - (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") + (db:set-sync db area-dat) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") db) (let* ((parent-dir (pathname-directory fname)) (dir-writable (file-write-access? parent-dir))) (if dir-writable (let ((exists (file-exists? fname)) (lock (obtain-dot-lock fname 1 5 10)) (db (sqlite3:open-database fname))) (sqlite3:set-busy-handler! db (make-busy-timeout 136000)) - (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") + (db:set-sync db area-dat) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") (if (not exists)(initproc db)) (release-dot-lock fname) db) (begin (debug:print 0 "ERROR: no such db in non-writable dir " fname) @@ -201,11 +201,11 @@ (dbr:dbstruct-get-localdb dbstruct run-id) (dbr:dbstruct-get-inmem dbstruct)))) ;; (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) (if (or rdb do-not-open) rdb - (let* ((dbpath (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db")) + (let* ((dbpath (db:dbfile-path run-id area-dat)) ;; (conc toppath "/db/" run-id ".db")) (dbexists (file-exists? dbpath)) (inmem (if local #f (db:open-inmem-db))) (refdb (if local #f (db:open-inmem-db))) (db (db:lock-create-open dbpath ;; this is the database physically on disk (lambda (db) @@ -222,15 +222,16 @@ "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');" (* run-id 30000) ;; allow for up to 30k tests per run run-id) ;; do a dummy query to test that the table exists and the db is truly readable (sqlite3:execute db "SELECT * FROM tests WHERE id=?;" (* run-id 30000)) - )))) ;; add strings db to rundb, not in use yet + )) + area-dat)) ;; add strings db to rundb, not in use yet ;; )) ;; (sqlite3:open-database dbpath)) (olddb (if *megatest-db* *megatest-db* - (let ((db (db:open-megatest-db))) + (let ((db (db:open-megatest-db area-dat))) (set! *megatest-db* db) db))) (write-access (file-write-access? dbpath)) ;; (handler (make-busy-timeout 136000)) ) @@ -261,14 +262,17 @@ ;; (define (db:open-main dbstruct area-dat) ;; (conc toppath "/megatest.db") (car configinfo))) (let ((mdb (dbr:dbstruct-get-main dbstruct))) (if mdb mdb - (let* ((dbpath (db:dbfile-path 0)) + (let* ((dbpath (db:dbfile-path 0 area-dat)) (dbexists (file-exists? dbpath)) - (db (db:lock-create-open dbpath db:initialize-main-db)) - (olddb (db:open-megatest-db)) + (db (db:lock-create-open dbpath + (lambda (db) + (db:initialize-main-db db area-dat)) + area-dat)) + (olddb (db:open-megatest-db area-dat)) (write-access (file-write-access? dbpath)) (dbdat (cons db dbpath))) (if (and dbexists (not write-access)) (set! *db-write-access* #f)) (dbr:dbstruct-set-main! dbstruct dbdat) @@ -276,11 +280,11 @@ dbdat)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; (define (db:setup run-id #!key (local #f)) - (let* ((dbdir (db:dbfile-path #f)) ;; (conc (configf:lookup configdat "setup" "linktree") "/.db")) + (let* ((dbdir (db:dbfile-path #f area-dat)) ;; (conc (configf:lookup configdat "setup" "linktree") "/.db")) (dbstruct (make-dbr:dbstruct path: dbdir local: local))) dbstruct)) ;; Open the classic megatest.db file in toppath ;; @@ -288,12 +292,13 @@ (let* ((toppath (megatest:area-path area-dat)) (dbpath (conc toppath "/megatest.db")) (dbexists (file-exists? dbpath)) (db (db:lock-create-open dbpath (lambda (db) - (db:initialize-main-db db) - (db:initialize-run-id-db db)))) + (db:initialize-main-db db area-dat) + (db:initialize-run-id-db db)) + area-dat)) (write-access (file-write-access? dbpath))) (if (and dbexists (not write-access)) (set! *db-write-access* #f)) (cons db dbpath))) @@ -319,11 +324,11 @@ (> mtime stime) force-sync) (begin (db:delay-if-busy maindb area-dat) (db:delay-if-busy olddb area-dat) - (let ((num-synced (db:sync-tables area-dat (db:sync-main-list maindb) maindb olddb))) + (let ((num-synced (db:sync-tables area-dat (db:sync-main-list maindb area-dat) maindb olddb))) (dbr:dbstruct-set-stime! dbstruct (current-milliseconds)) num-synced) 0)) (begin ;; this can occur when using local access (i.e. not in a server) @@ -463,12 +468,12 @@ '("status" #f) '("type" #f)))) ;; needs db to get keys, this is for syncing all tables ;; -(define (db:sync-main-list db) - (let ((keys (db:get-keys db))) +(define (db:sync-main-list db area-dat) + (let ((keys (db:get-keys db area-dat))) (list (list "keys" '("id" #f) '("fieldname" #f) '("fieldtype" #f)) @@ -540,11 +545,11 @@ (full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) " " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );")) (fromdat '()) (fromdats '()) (totrecords 0) - (batch-len (string->number (or (configf:lookup configdat "sync" "batchsize") "10"))) + (batch-len (string->number (or (configf:lookup (megatest:area-configdat area-dat) "sync" "batchsize") "10"))) (todat (make-hash-table)) (count 0)) ;; set up the field->num table (for-each @@ -634,14 +639,14 @@ ;; 'new2old - sync .db/{main,1,2,3 ...}.db to megatest.db ;; 'closeall - close all opened dbs ;; ;; run-ids: '(1 2 3 ...) or #f (for all) ;; -(define (db:multi-db-sync run-ids . options) +(define (db:multi-db-sync run-ids area-dat . options) (let* ((toppath (launch:setup-for-run)) (dbstruct (if toppath (make-dbr:dbstruct path: toppath) #f)) - (mtdb (if toppath (db:open-megatest-db))) + (mtdb (if toppath (db:open-megatest-db area-dat))) (allow-cleanup (if run-ids #f #t)) (run-ids (if run-ids run-ids (if toppath (begin (db:delay-if-busy mtdb area-dat) @@ -673,11 +678,11 @@ ;; sync runs, test_meta etc. ;; (if (member 'old2new options) (begin - (db:sync-tables area-dat (db:sync-main-list mtdb) mtdb (db:get-db dbstruct area-dat #f)) + (db:sync-tables area-dat (db:sync-main-list mtdb area-dat) mtdb (db:get-db dbstruct area-dat #f)) (for-each (lambda (run-id) (db:delay-if-busy mtdb area-dat) (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id)) (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))) @@ -689,32 +694,32 @@ ;; now ensure all newdb data are synced to megatest.db ;; do not use the run-ids list passed in to the function ;; (if (member 'new2old options) (let* ((maindb (make-dbr:dbstruct path: toppath local: #t)) - (src-run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0)))) + (src-run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb area-dat 0)))) (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <)) (count 1) (total (length all-run-ids)) (dead-runs '())) (for-each (lambda (run-id) (debug:print 0 "Processing run " (if (eq? run-id 0) " main.db " run-id) ", " count " of " total) (set! count (+ count 1)) (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)) - (frundb (db:dbdat-get-db (db:get-db fromdb run-id)))) + (frundb (db:dbdat-get-db (db:get-db fromdb area-dat run-id)))) ;; (db:delay-if-busy frundb) ;; (db:delay-if-busy mtdb) ;; (db:clean-up frundb) (if (eq? run-id 0) (begin - (db:sync-tables area-dat (db:sync-main-list dbstruct area-dat) (db:get-db fromdb #f) mtdb) - (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f)))) + (db:sync-tables area-dat (db:sync-main-list dbstruct area-dat) (db:get-db fromdb area-dat #f) mtdb) + (set! dead-runs (db:clean-up-maindb (db:get-db fromdb area-dat #f)))) (begin ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db - (db:sync-tables area-dat db:sync-tests-only (db:get-db fromdb run-id) mtdb) - (db:clean-up-rundb (db:get-db fromdb run-id)) + (db:sync-tables area-dat db:sync-tests-only (db:get-db fromdb area-dat run-id) mtdb) + (db:clean-up-rundb (db:get-db fromdb area-dat run-id)) )))) all-run-ids) ;; removed deleted runs (let ((dbdir (tasks:get-task-db-path))) (for-each (lambda (run-id) @@ -1093,11 +1098,11 @@ 136000)))) ;; 136000))) (sqlite3:set-busy-handler! db handler) (if (not dbexists) (begin (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log (id INTEGER PRIMARY KEY,event_time TIMESTAMP DEFAULT (strftime('%s','now')),logline TEXT,pwd TEXT,cmdline TEXT,pid INTEGER);") - (db:set-sync db) ;; (sqlite3:execute db (conc "PRAGMA synchronous = 0;")) + (db:set-sync db area-dat) ;; (sqlite3:execute db (conc "PRAGMA synchronous = 0;")) )) db)) (define (db:log-local-event . loglst) (let ((logline (apply conc loglst))) @@ -2864,11 +2869,11 @@ ;; Run this remotely!! ;; (define (db:get-matching-previous-test-run-records dbstruct area-dat run-id test-name item-path) (let* ((dbdat (db:get-db dbstruct area-dat #f)) (db (db:dbdat-get-db dbdat)) - (keys (db:get-keys db)) + (keys (db:get-keys db area-dat)) (selstr (string-intersperse (map (lambda (x)(vector-ref x 0)) keys) ",")) (qrystr (string-intersperse (map (lambda (x)(conc (vector-ref x 0) "=?")) keys) " AND ")) (keyvals #f) (tests-hash (make-hash-table))) ;; first look up the key values from the run selected by run-id Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -333,11 +333,11 @@ (mutex-lock! *db-multi-sync-mutex*) (if (and (not (equal? legacy-sync "no")) (hash-table-ref/default *db-local-sync* run-id #f)) ;; (if (> (- start-time last-write) 5) ;; every five seconds (begin ;; let ((sync-time (- (current-seconds) start-time))) - (db:multi-db-sync (list run-id) 'new2old) + (db:multi-db-sync (list run-id) *area-dat* 'new2old) (if (common:low-noise-print 30 "sync new to old") (let ((sync-time (- (current-seconds) start-time))) (debug:print-info 0 "Sync of newdb to olddb for run-id " run-id " completed in " sync-time " seconds"))) ;; (if (> sync-time 10) ;; took more than ten seconds, start a server for this run ;; (begin @@ -1417,10 +1417,11 @@ (exit 1))) ;; keep this one local ;; (open-run-close db:clean-up #f) (db:multi-db-sync #f ;; do all run-ids + *area-dat* ;; 'new2old 'killservers 'dejunk ;; 'adj-testids ;; 'old2new @@ -1529,10 +1530,11 @@ (if (args:get-arg "-import-megatest.db") (begin (db:multi-db-sync #f ;; do all run-ids + *area-dat* 'killservers 'dejunk 'adj-testids 'old2new ;; 'new2old @@ -1541,10 +1543,11 @@ (if (args:get-arg "-sync-to-megatest.db") (begin (db:multi-db-sync #f ;; do all run-ids + *area-dat* 'new2old ) (set! *didsomething* #t))) ;;====================================================================== Index: tests/unittests/basicserver.scm ================================================================== --- tests/unittests/basicserver.scm +++ tests/unittests/basicserver.scm @@ -12,12 +12,12 @@ (test "setup for run" #t (begin (launch:setup-for-run *area-dat*) (string? (getenv "MT_RUN_AREA_HOME")))) ;; NON Server tests go here -(test #f #f (db:dbdat-get-path *db* *area-dat*)) -(test #f #f (db:get-run-name-from-id *db* run-id)) +(test #f #f (db:dbdat-get-path *db*)) +(test #f #f (db:get-run-name-from-id *db* *area-dat* run-id)) ;; (test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) ;; (exit) ;; Server tests go here