Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -446,11 +446,11 @@ ;;====================================================================== ;; ;;====================================================================== (define (examine-test test-id) ;; run-id run-key origtest) (let* ((db-path (conc *toppath* "db/main.db")) - (db (make-dbr:dbstruct path: *toppath*)) + (db (make-dbr:dbstruct path: *toppath* local: #t)) (tdb (tdb:open-test-db-by-test-id-local test-id)) (testdat (db:get-test-info-by-id db test-id)) (db-mod-time 0) ;; (file-modification-time db-path)) (last-update 0) ;; (current-seconds)) (request-update #t)) Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -86,11 +86,11 @@ (if (not (setup-for-run)) (begin (print "Failed to find megatest.config, exiting") (exit 1))) -(define *db* (make-dbr:dbstruct path: *toppath*)) +(define *db* (make-dbr:dbstruct path: *toppath* local: #t)) ;; (define sdb:qry (make-sdb:qry)) ;; 'init #f) ;; (if (args:get-arg "-host") ;; (begin Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -98,14 +98,15 @@ ;; (define (db:open-rundb dbstruct run-id) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) (let ((rdb (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) (if rdb rdb - (let* ((toppath (dbr:dbstruct-get-path dbstruct)) + (let* ((local (dbr:dbstruct-get-local dbstruct)) + (toppath (dbr:dbstruct-get-path dbstruct)) (dbpath (conc toppath "/db/" run-id ".db")) (dbexists (file-exists? dbpath)) - (inmem (open-inmem-db)) + (inmem (if local #f (open-inmem-db))) (db (sqlite3:open-database dbpath)) (write-access (file-write-access? dbpath)) (handler (make-busy-timeout 136000))) (if (and dbexists (not write-access)) (set! *db-write-access* #f)) ;; only unset so other db's also can use this control @@ -113,14 +114,17 @@ (begin (sqlite3:set-busy-handler! db handler) (sqlite3:execute db "PRAGMA synchronous = 0;"))) (if (not dbexists)(db:initialize-run-id-db db run-id)) (dbr:dbstruct-set-runvec! dbstruct run-id 'rundb db) - (dbr:dbstruct-set-runvec! dbstruct run-id 'inmem inmem) (dbr:dbstruct-set-runvec! dbstruct run-id 'inuse #t) - (db:sync-tables db:sync-tests-only db inmem) - inmem)))) + (if local + db + (begin + (dbr:dbstruct-set-runvec! dbstruct run-id 'inmem inmem) + (db:sync-tables db:sync-tests-only db inmem) + inmem)))))) ;; This routine creates the db. It is only called if the db is not already opened ;; (define (db:open-main dbstruct) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) (let ((mdb (dbr:dbstruct-get-main dbstruct))) @@ -159,16 +163,18 @@ (db:sync-tables db:sync-tests-only inmem rundb) (vector-set! runvec (dbr:dbstruct-field-name->run 'stime (current-milliseconds))))))) (hash-table-values (vector-ref dbstruct 1)))) ;; close all opened run-id dbs -(define (db:close-all-db) +(define (db:close-all dbstruct) + ;; finalize main.db + (sqlite3:finalize! (db:get-db dbstruct #f)) (for-each - (lambda (db) - (finalize! db)) - (hash-table-values (vector-ref *open-dbs* 1))) - (finalize! (vector-ref *open-dbs* 0))) + (lambda (runvec) + (let ((rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))) + (sqlite3:finalize! rundb))) + (hash-table-values (vector-ref dbstruct 1)))) (define (open-inmem-db) (let* ((db (sqlite3:open-database ":memory:")) (handler (make-busy-timeout 3600))) (db:initialize db) @@ -364,18 +370,18 @@ ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) (debug:print-info 11 "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params) (if (or *db-write-access* (not (member proc *db:all-write-procs*))) - (let* ((db (cond - ((sqlite3:database? idb) idb) - ((not idb) (make-dbr:dbstruct path: *toppath*)) - ((procedure? idb) (idb)) - (else (make-dbr:dbstruct path: *toppath*)))) + (let* ((db (cond + ((sqlite3:database? idb) idb) + ((not idb) (debug:print 0 "ERROR: cannot open-run-close with #f anymore")) + ((procedure? idb) (idb)) + (else (debug:print 0 "ERROR: cannot open-run-close with #f anymore")))) (res #f)) (set! res (apply proc db params)) - (if (not idb)(sqlite3:finalize! db)) + (if (not idb)(sqlite3:finalize! dbstruct)) (debug:print-info 11 "open-run-close-no-exception-handling END" ) res) #f)) (define (open-run-close-exception-handling proc idb . params) @@ -389,13 +395,13 @@ (debug:print-info 0 "trying db call one more time....") (apply open-run-close-no-exception-handling proc idb params)) (apply open-run-close-no-exception-handling proc idb params))) ;; (define open-run-close -(define open-run-close (if (debug:debug-mode 2) - open-run-close-no-exception-handling - open-run-close-exception-handling)) +(define open-run-close ;; (if (debug:debug-mode 2) + open-run-close-no-exception-handling) + ;; open-run-close-exception-handling)) (define (db:initialize-megatest-db db) (let* ((configdat (car *configinfo*)) ;; tut tut, global warning... (keys (keys:config-get-fields configdat)) (havekeys (> (length keys) 0)) @@ -819,13 +825,14 @@ ;; replace header and keystr with a call to runs:get-std-run-fields ;; ;; keypatts: ( (KEY1 "abc%def")(KEY2 "%") ) ;; runpatts: patt1,patt2 ... ;; -(define (db:get-runs db runpatt count offset keypatts) - (let* ((res '()) - (keys (db:get-keys db)) +(define (db:get-runs dbstruct runpatt count offset keypatts) + (let* ((db (db:get-db dbstruct #f)) + (res '()) + (keys (db:get-keys dbstruct)) (runpattstr (db:patt->like "runname" runpatt)) (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (header (append keys remfields)) (keystr (conc (keys->keystr keys) "," (string-intersperse remfields ","))) Index: db_records.scm ================================================================== --- db_records.scm +++ db_records.scm @@ -8,16 +8,17 @@ ;; |-monitor.db ;; |-sdb.db ;; |-fdb.db ;; |-1.db ;; |-.db -(define (make-dbr:dbstruct #!key (path #f)) +(define (make-dbr:dbstruct #!key (path #f)(local #f)) (vector #f ;; the main db (contains runs, test_meta etc.) NOT CACHED IN MEM (make-hash-table) ;; run-id => [ rundb inmemdb last-mod last-read last-sync ] #f ;; the global string db (use for state, status etc.) - path)) ;; path to database files/megatest area + path ;; path to database files/megatest area + local)) ;; read-only local access ;; get and set main db (define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) (define-inline (dbr:dbstruct-set-main! vec db)(vector-set! vec 0 db)) @@ -65,12 +66,16 @@ ;; the string db (define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 2)) (define-inline (dbr:dbstruct-set-strdb! vec db)(vector-set! vec 2 db)) ;; path -(define-inline (dbr:dbstruct-set-path! vec path)(vector-set! vec 3)) (define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 3)) +(define-inline (dbr:dbstruct-set-path! vec path)(vector-set! vec 3)) + +;; local +(define-inline (dbr:dbstruct-get-local vec) (vector-ref vec 4)) +(define-inline (dbr:dbstruct-set-local! vec val)(vector-set! vec 4 val)) (define (make-db:test)(make-vector 20)) (define-inline (db:test-get-id vec) (vector-ref vec 0)) (define-inline (db:test-get-run_id vec) (vector-ref vec 1)) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -561,19 +561,19 @@ ;; NOTE: list-runs and list-db-targets operate on local db!!! ;; (if (or (args:get-arg "-list-runs") (args:get-arg "-list-db-targets")) (if (setup-for-run) - (let* ((db (make-dbr:dbstruct path: *toppath* local: #t)) + (let* ((dbstruct (make-dbr:dbstruct path: *toppath* local: #t)) (runpatt (args:get-arg "-list-runs")) (testpatt (if (args:get-arg "-testpatt") (args:get-arg "-testpatt") "%")) - (runsdat (db:get-runs db runpatt #f #f '())) + (runsdat (db:get-runs dbstruct runpatt #f #f '())) (runs (db:get-rows runsdat)) (header (db:get-header runsdat)) - (keys (db:get-keys db)) + (keys (db:get-keys dbstruct)) (db-targets (args:get-arg "-list-db-targets")) (seen (make-hash-table))) ;; Each run (for-each (lambda (run) @@ -586,11 +586,11 @@ (hash-table-set! seen targetstr #t) ;; (print "[" targetstr "]")))) (print targetstr)))) (if (not db-targets) (let* ((run-id (db:get-value-by-header run header "id")) - (tests (db:get-tests-for-run db run-id testpatt '() '() #f #f #f 'testname 'asc #f))) + (tests (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc #f))) (print "Run: " targetstr "/" (db:get-value-by-header run header "runname") " status: " (db:get-value-by-header run header "state") " run-id: " run-id ", number tests: " (length tests)) (for-each (lambda (test) @@ -614,11 +614,11 @@ "\n uname: " (sdb:qry 'getstr (db:test-get-uname test)) "\n rundir: " (filedb:get-path *fdb* (db:test-get-rundir test)) ) ;; Each test ;; DO NOT remote run - (let ((steps (db:get-steps-for-test db (db:test-get-id test)))) + (let ((steps (db:get-steps-for-test dbstruct run-id (db:test-get-id test)))) (for-each (lambda (step) (format #t " Step: ~20a State: ~10a Status: ~10a Time ~22a\n" (tdb:step-get-stepname step) @@ -626,11 +626,12 @@ (tdb:step-get-status step) (tdb:step-get-event_time step))) steps))))) tests))))) runs) - (set! *didsomething* #t)))) + (db:close-all dbstruct) + (set! *didsomething* #t)))) ;;====================================================================== ;; full run ;;====================================================================== @@ -815,17 +816,17 @@ ;; else do a general-run-call (general-run-call "-test-paths" "Get paths to tests" (lambda (target runname keys keyvals) - (let* ((db (make-dbr:dbstruct path: *toppath* local: #t)) + (let* ((dbstruct (make-dbr:dbstruct path: *toppath* local: #t)) ;; DO NOT run remote - (paths (db:test-get-paths-matching db keys target))) + (paths (db:test-get-paths-matching dbstruct keys target))) (for-each (lambda (path) (print path)) paths) - (sqlite3:finalize! db)))))) + (db:close-all dbstruct)))))) ;;====================================================================== ;; Extract a spreadsheet from the runs database ;;====================================================================== @@ -832,18 +833,18 @@ (if (args:get-arg "-extract-ods") (general-run-call "-extract-ods" "Make ods spreadsheet" (lambda (target runname keys keyvals) - (let ((db (make-dbr:dbstruct path: *toppath* local: #t)) + (let ((dbstruct (make-dbr:dbstruct path: *toppath* local: #t)) (outputfile (args:get-arg "-extract-ods")) (runspatt (args:get-arg ":runname")) (pathmod (args:get-arg "-pathmod"))) ;; (keyvalalist (keys->alist keys "%"))) (debug:print 2 "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvals: " keyvals) - (db:extract-ods-file db outputfile keyvals (if runspatt runspatt "%") pathmod) - (sqlite3:finalize! db) + (db:extract-ods-file dbstruct outputfile keyvals (if runspatt runspatt "%") pathmod) + (db:close-all dbstruct) (set! *didsomething* #t))))) ;;====================================================================== ;; execute the test ;; - gets called on remote host @@ -1128,14 +1129,14 @@ ;;====================================================================== (if (or (args:get-arg "-repl") (args:get-arg "-load")) (let* ((toppath (setup-for-run)) - (db (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))) - (if db + (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))) + (if dbstruct (begin - (set! *db* db) + (set! *db* dbstruct) (set! *client-non-blocking-mode* #t) (import readline) (import apropos) ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ... (gnu-history-install-file-manager @@ -1142,25 +1143,28 @@ (string-append (or (get-environment-variable "HOME") ".") "/.megatest_history")) (current-input-port (make-gnu-readline-port "megatest> ")) (if (args:get-arg "-repl") (repl) - (load (args:get-arg "-load")))) + (load (args:get-arg "-load"))) + (db:close-all dbstruct)) (exit)) (set! *didsomething* #t))) +;; Not converted to use dbstruct yet +;; (if (args:get-arg "-convert-to-norm") (let* ((toppath (setup-for-run)) - (db (if toppath (make-dbr:dbstruct path: toppath local: #t)))) + (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t)))) (for-each (lambda (field) (let ((dat '())) (debug:print-info 0 "Getting data for field " field) (sqlite3:for-each-row (lambda (id val) (set! dat (cons (list id val) dat))) - db + (get-db db run-id) (conc "SELECT id," field " FROM tests;")) (debug:print-info 0 "found " (length dat) " items for field " field) (let ((qry (sqlite3:prepare db (conc "UPDATE tests SET " field "=? WHERE id=?;")))) (for-each (lambda (item) @@ -1168,10 +1172,11 @@ (if (not (equal? newval (cadr item))) (debug:print-info 0 "Converting " (cadr item) " to " newval " for test #" (car item))) (sqlite3:execute qry newval (car item)))) dat) (sqlite3:finalize! qry)))) + (db:close-all dbstruct) (list "uname" "rundir" "final_logf" "comment")) (set! *didsomething* #t))) ;;====================================================================== ;; Exit and clean up