Index: configf.scm ================================================================== --- configf.scm +++ configf.scm @@ -217,16 +217,17 @@ (envar (and environ-patt (string-search (regexp environ-patt) curr-section-name))) (realval (if envar (config:eval-string-in-environment val) val))) (debug:print-info 6 "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name) - (if (and envar - (string? realval) - (not (string-search (integer->char 0) realval))) - ;; (debug:print-info 4 "read-config key=" key ", val=" val ", realval=" realval) - (setenv key realval) - (debug:print 0 "ERROR: bad value for setenv, key=" key ", value=" realval)) + (if envar + (if (and (string? realval)(string? key)) + (handle-exceptions + exn + (debug:print 0 "ERROR: bad value for setenv, key=" key ", value=" realval) + (setenv key realval)) + (debug:print 0 "ERROR: bad value for setenv, key=" key ", value=" realval))) (hash-table-set! res curr-section-name (config:assoc-safe-add alist key realval)) (loop (configf:read-line inp res allow-system) curr-section-name key #f))) (configf:key-no-val ( x key val) (let* ((alist (hash-table-ref/default res curr-section-name '()))) (hash-table-set! res curr-section-name Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -44,21 +44,20 @@ (define *completed-writes* (make-hash-table)) (define *incoming-last-time* (current-seconds)) (define *incoming-mutex* (make-mutex)) (define *completed-mutex* (make-mutex)) +;; Get/open a database +;; if run-id => get run specific db +;; if #f => get main db +;; if db already open - return inmem +;; if db not open, open inmem, rundb and sync then return inmem +;; (define (db:get-db dbstruct run-id) - (let ((db (if run-id - (hash-table-ref/default (vector-ref dbstruct 1) run-id #f) - (vector-ref dbstruct 0)))) - (if db - db - (let ((db (open-db run-id))) - (if run-id - (hash-table-set! (vector-ref dbstruct 1) run-id db) - (vector-set! dbstruct 0 db)) - db)))) + (if run-id + (db:open-rundb dbstruct run-id) + (db:open-main dbstruct))) (define (db:set-sync db) (let* ((syncval (config-lookup *configdat* "setup" "synchronous")) (val (cond ;; 0 | OFF | 1 | NORMAL | 2 | FULL; ((not syncval) #f) @@ -73,11 +72,10 @@ #f)))) (if val (begin (debug:print-info 9 "db:set-sync, setting pragma synchronous to " val) (sqlite3:execute db (conc "PRAGMA synchronous = '" val "';")))))) -;; (sqlite3:execute db "PRAGMA synchronous = normal;")))) ;; need a default? ;;====================================================================== ;; K E E P F I L E D B I N dbstruct ;;====================================================================== @@ -99,84 +97,72 @@ ;; (define (db:get-path dbstruct id) (let ((fdb (db:get-filedb dbstruct))) (filedb:get-path db id))) -;;====================================================================== -;; U S E F I L E D B T O S T O R E S T R I N G S -;; -;; N O T E ! ! T H I S C L O B B E R S M U L T I P L E //// T O / -;; -;; Replace with something proper! -;; -;;====================================================================== - -;; Use to save a stored string, pad with _ to deal with trimming the prepending of / -;; -(define (db:save-string dbstruct str) - (let ((fdb (db:get-filedb dbstruct))) - (filedb:register-path fdb (conc "_" str)))) - -;; Use to get a stored string -;; -(define (db:get-string dbstruct id) - (let ((fdb (db:get-filedb dbstruct))) - (string-drop (filedb:get-path fdb id) 2))) +;; This routine creates the db. It is only called if the db is not already opened +;; +(define (db:open-rundb dbstruct run-id) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) + (let ((rdb (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) + (if rdb + rdb + (let* ((toppath (dbr:dbstruct-get-path dbstruct)) + (dbpath (conc toppath "/db/" run-id ".db")) + (dbexists (file-exists? dbpath)) + (inmem (open-inmem-db)) + (db (sqlite3:open-database dbpath)) + (write-access (file-write-access? dbpath)) + (handler (make-busy-timeout 136000))) + (if (and dbexists (not write-access)) + (set! *db-write-access* #f)) ;; only unset so other db's also can use this control + (if write-access + (begin + (sqlite3:set-busy-handler! db handler) + (sqlite3:execute db "PRAGMA synchronous = 0;"))) + (if (not dbexists)(db:initialize-run-id-db db run-id)) + (dbr:dbstruct-set-runvec! dbstruct run-id 'rundb db) + (dbr:dbstruct-set-runvec! dbstruct run-id 'inmem inmem) + inmem)))) ;; This routine creates the db. It is only called if the db is not already opened ;; -(define (open-db dbstruct run-id) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) - (if (not *toppath*) - (if (not (setup-for-run)) - (begin - (debug:print 0 "ERROR: Attempted to open db when not in megatest area. Exiting.") - (exit)))) - (let* ((dbpath (if run-id - (conc *toppath* "/db/" run-id ".db") - (let ((dbdir (conc *toppath* "/db"))) ;; use this opportunity to create our db dir - (if (not (directory-exists? dbdir)) - (create-direcory dbdir)) - (conc *toppath* "/megatest.db")))) - (dbexists (file-exists? dbpath)) - (db (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) - (write-access (file-write-access? dbpath)) - (handler (make-busy-timeout (if (args:get-arg "-override-timeout") - (string->number (args:get-arg "-override-timeout")) - 136000)))) ;; 136000))) ;; 136000 = 2.2 minutes - (if (and dbexists - (not write-access)) - (set! *db-write-access* write-access)) ;; only unset so other db's also can use this control - (debug:print-info 11 "open-db, dbpath=" dbpath " argv=" (argv)) - (if write-access (sqlite3:set-busy-handler! db handler)) - (if (not dbexists) - (if (not run-id) ;; do the megatest.db - (db:initialize-megatest-db db) - (db:initialize-run-id-db db run-id))) - (sqlite3:execute db "PRAGMA synchronous = 0;") - db)) +(define (db:open-main dbstruct) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) + (let ((mdb (dbr:dbstruct-get-main dbstruct))) + (if mdb + mdb + (let* ((toppath (dbr:dbstruct-get-path dbstruct)) + (dbpath (let ((dbdir (conc *toppath* "/db"))) ;; use this opportunity to create our db dir + (if (not (directory-exists? dbdir)) + (create-direcory dbdir)) + (conc *toppath* "/db/main.db"))) + (dbexists (file-exists? dbpath)) + (db (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) + (write-access (file-write-access? dbpath)) + (handler (make-busy-timeout 136000))) + (if (and dbexists (not write-access)) + (set! *db-write-access* #f)) + (if write-access + (begin + (sqlite3:set-busy-handler! db handler) + (sqlite3:execute db "PRAGMA synchronous = 0;"))) + (if (not dbexists) + (db:initialize-megatest-db db)) + (dbr:dbstruct-set-main! dbstruct db) + db)))) ;; close all opened run-id dbs (define (db:close-all-db) (for-each (lambda (db) (finalize! db)) (hash-table-values (vector-ref *open-dbs* 1))) (finalize! (vector-ref *open-dbs* 0))) -(define (open-in-mem-db) - (let* ((path (configf:lookup *configdat* "setup" "tmpdb")) - (fname (if path (conc path "/temp-megatest.db") #f)) - (exists (and path (file-exists? fname))) - (db (if path - (begin - (create-directory path #t) - (sqlite3:open-database fname)) - (sqlite3:open-database ":memory:"))) +(define (open-inmem-db) + (let* ((db (sqlite3:open-database ":memory:")) (handler (make-busy-timeout 3600))) - (if (or (not path) - (not exists)) - (db:initialize db)) + (db:initialize db) (sqlite3:set-busy-handler! db handler) (set! sdb:qry (make-sdb:qry)) ;; we open the normalization helpers here (set! *fdb* (filedb:open-db (conc *toppath* "/db/paths.db"))) db)) @@ -312,124 +298,10 @@ (count (cdr dat))) (if (> count 0) (debug:print 0 (format #f " ~10a ~5a" tblname count))))) (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b)))))))) -;; (define (db:sync-to fromdb todb) -;; ;; strategy -;; ;; 1. Get all run-ids -;; ;; 2. For each run-id -;; ;; a. Sync that run in a transaction -;; (let ((trecchgd 0) -;; (rrecchgd 0) -;; (tmrecchgd 0)) -;; -;; ;; First sync test_meta data -;; (let ((tmgetstmt (sqlite3:prepare todb "SELECT id,testname,author,owner,description,reviewed,iterated,avg_runtime,avg_disk,tags,jobgroup FROM test_meta WHERE id=?;")) -;; (tmputstmt (sqlite3:prepare todb "INSERT OR REPLACE INTO test_meta (id,testname,author,owner,description,reviewed,iterated,avg_runtime,avg_disk,tags,jobgroup) -;; VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")) -;; (tmdats (db:testmeta-get-all fromdb))) -;; ;; (debug:print 7 "Updating as many as " (length tdats) " records for run " run-id) -;; (for-each -;; (lambda (tmdat) ;; iterate over tests -;; (let ((testm-id (vector-ref tmdat 0))) -;; (sqlite3:with-transaction -;; todb -;; (lambda () -;; (let ((curr-tmdat #f)) -;; (sqlite3:for-each-row -;; (lambda (a . b) -;; (set! curr-tmdat (apply vector a b))) -;; tmgetstmt testm-id) -;; (if (not (equal? curr-tmdat tmdat)) ;; something changed -;; (begin -;; (debug:print 0 " test-id: " testm-id -;; "\ncurr-tdat: " curr-tmdat -;; "\n tdat: " tmdat) -;; (apply sqlite3:execute tmputstmt (vector->list tmdat)) -;; (set! tmrecchgd (+ tmrecchgd 1))))))))) -;; tmdats) -;; (sqlite3:finalize! tmgetstmt) -;; (sqlite3:finalize! tmputstmt)) -;; -;; ;; First sync tests data -;; (let ((run-ids (db:get-all-run-ids fromdb)) -;; (tgetstmt (sqlite3:prepare todb "SELECT id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment FROM tests WHERE id=?;")) -;; (tputstmt (sqlite3:prepare todb "INSERT OR REPLACE INTO tests (id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment) -;; VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );"))) -;; (for-each -;; (lambda (run-id) -;; (let ((tdats (db:get-all-tests-info-by-run-id fromdb run-id))) -;; ;; (debug:print 7 "Updating as many as " (length tdats) " records for run " run-id) -;; (for-each -;; (lambda (tdat) ;; iterate over tests -;; (let ((test-id (vector-ref tdat 0))) -;; (sqlite3:with-transaction -;; todb -;; (lambda () -;; (let ((curr-tdat #f)) -;; (sqlite3:for-each-row -;; (lambda (a . b) -;; (set! curr-tdat (apply vector a b))) -;; tgetstmt -;; test-id) -;; (if (not (equal? curr-tdat tdat)) ;; something changed -;; (begin -;; (debug:print 0 " test-id: " test-id -;; "\ncurr-tdat: " curr-tdat -;; "\n tdat: " tdat) -;; (apply sqlite3:execute tputstmt (vector->list tdat)) -;; (set! trecchgd (+ trecchgd 1))))))))) -;; tdats))) -;; run-ids) -;; (sqlite3:finalize! tgetstmt) -;; (sqlite3:finalize! tputstmt)) -;; -;; ;; Next sync runs table -;; (let* ((rdats '()) -;; (keys (db:get-keys fromdb)) -;; (rstdfields (conc "id," (string-intersperse keys ",") ",runname,state,status,owner,event_time,comment,fail_count,pass_count")) -;; (rnumfields (length (string-split rstdfields ","))) -;; (runslots (string-intersperse (make-list rnumfields "?") ",")) -;; (rgetstmt (sqlite3:prepare todb (conc "SELECT " rstdfields " FROM runs WHERE id=?;"))) -;; (rputstmt (sqlite3:prepare todb (conc "INSERT OR REPLACE INTO runs (" rstdfields ") VALUES ( " runslots " );")))) -;; ;; first collect all the source run data -;; (sqlite3:for-each-row -;; (lambda (a . b) -;; (set! rdats (cons (apply vector a b) rdats))) -;; fromdb -;; (conc "SELECT " rstdfields " FROM runs;")) -;; (sqlite3:with-transaction -;; todb -;; (lambda () -;; (for-each -;; (lambda (rdat) -;; (let ((run-id (vector-ref rdat 0)) -;; (curr-rdat #f)) -;; ;; first get the current value of the equivalent row from the target -;; ;; read, then insert/overwrite if different -;; (sqlite3:for-each-row -;; (lambda (a . b) -;; (set! curr-rdat (apply vector a b))) -;; rgetstmt -;; run-id) -;; (if (not (equal? curr-rdat rdat)) -;; (begin -;; (debug:print 0 " run-id: " run-id -;; "\ncurr-rdat: " curr-rdat -;; "\n rdat: " rdat) -;; (set! rrecchgd (+ rrecchgd 1)) -;; (apply sqlite3:execute rputstmt (vector->list rdat)))))) -;; rdats))) -;; (sqlite3:finalize! rgetstmt) -;; (sqlite3:finalize! rputstmt)) -;; -;; (if (> rrecchgd 0) (debug:print 0 "synced " rrecchgd " changed records in runs table")) -;; (if (> trecchgd 0) (debug:print 0 "synced " trecchgd " changed records in tests table")) -;; (if (> tmrecchgd 0) (debug:print 0 "sync'd " tmrecchgd " changed records in test_meta table")) -;; (+ rrecchgd trecchgd tmrecchgd))) - (define (db:sync-back) (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*)) ;; (db:sync-to *inmemdb* *db*)) ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) @@ -1055,11 +927,11 @@ (let ((finalres (vector header res))) ;; (hash-table-set! *run-info-cache* run-id finalres) finalres))) (define (db:set-comment-for-run dbstruct run-id comment) - (sqlite3:execute db "UPDATE runs SET comment=? WHERE id=?;" (sdb:qry 'getid comment) run-id)) + (sqlite3:execute (db:get-db dbstruct #f) "UPDATE runs SET comment=? WHERE id=?;" (sdb:qry 'getid comment) run-id)) ;; does not (obviously!) removed dependent data. But why not!!? (define (db:delete-run dbstruct run-id) ;; First set any related tests to DELETED (let ((db (db:get-db dbstruct run-id))) Index: db_records.scm ================================================================== --- db_records.scm +++ db_records.scm @@ -1,5 +1,75 @@ +;;====================================================================== +;; dbstruct +;;====================================================================== + +;; +;; -path-|-megatest.db +;; |-db-|-main.db +;; |-monitor.db +;; |-sdb.db +;; |-fdb.db +;; |-1.db +;; |-.db +(define (make-dbr:dbstruct #!key (path #f)) + (make-vector + #f ;; the main db (contains runs, test_meta etc.) NOT CACHED IN MEM + (make-hash-table) ;; run-id => [ rundb inmemdb last-mod last-read last-sync ] + #f ;; the global string db (use for state, status etc.) + path)) ;; path to database files/megatest area + +;; get and set main db +(define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) +(define-inline (dbr:dbstruct-set-main! vec db)(vector-set! vec 0 db)) + +;; get a rundb vector +(define (dbr:dbstruct-get-rundb-rec vec run-id) + (let* ((dbhash (vector-ref vec 1)) + (runvec (hash-table-ref/default dbhash run-id))) + (if runvec + runvec + (begin + (hash-table-set! dbhash run-id (vector #f #f -1 -1 -1)) + (dbr:dbstruct-get-rundb-rec vec run-id))))) + +;; [ rundb inmemdb last-mod last-read last-sync ] +(define-inline (dbr:dbstruct-field-name->num field-name) + (case field-name + ((rundb) 0) ;; the on-disk db + ((inmem) 1) ;; the in-memory db + ((mtime) 2) ;; last modification time + ((rtime) 3) ;; last read time + ((stime) 4) ;; last sync time + (else -1))) + +;; get/set rundb fields +(define (dbr:dbstruct-get-runrec vec run-id field-name) + (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) + (vector-ref runvec (dbr:dbstruct-field-name->num field-name)))) + +(define (dbr:dbstruct-set-runvec! vec run-id field-name val) + (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) + (vector-set! runvec (dbr:dbstruct-field-name->num field-name) rundb))) + +;; get/set inmemdb +(define (dbr:dbstruct-get-inmemdb vec run-id) + (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) + (vector-ref runvec 1))) + +(define (dbr:dbstruct-set-inmemdb! vec run-id inmemdb) + (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) + (vector-set! runvec 1 inmemdb))) + +;; the string db +(define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 2)) +(define-inline (dbr:dbstruct-set-strdb! vec db)(vector-set! vec 2 db)) + +;; path +(define-inline (dbr:dbstruct-set-path! vec path)(vector-set! vec 3)) +(define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 3)) + + (define (make-db:test)(make-vector 20)) (define-inline (db:test-get-id vec) (vector-ref vec 0)) (define-inline (db:test-get-run_id vec) (vector-ref vec 1)) (define-inline (db:test-get-testname vec) (vector-ref vec 2)) (define-inline (db:test-get-state vec) (vector-ref vec 3)) Index: tests/rununittest.sh ================================================================== --- tests/rununittest.sh +++ tests/rununittest.sh @@ -1,10 +1,13 @@ #!/bin/bash # Usage: rununittest.sh testname debuglevel # +# Ensure all is made +(cd ..;make && make install) + # Clean setup # rm -f simplerun/megatest.db simplerun/monitor.db simplerun/db/monitor.db rm -rf simplelinks/ simpleruns/ mkdir -p simplelinks simpleruns ADDED tests/unittests/basicserver.scm Index: tests/unittests/basicserver.scm ================================================================== --- /dev/null +++ tests/unittests/basicserver.scm @@ -0,0 +1,114 @@ +;;====================================================================== +;; S E R V E R +;;====================================================================== + +;; Run like this: +;; +;; (cd ..;make && make install) && ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) + +(set! *transport-type* 'http) + +(test "setup for run" #t (begin (setup-for-run) + (string? (getenv "MT_RUN_AREA_HOME")))) + +(test "server-register, get-best-server" #t (let ((res #f)) + (open-run-close tasks:server-register tasks:open-db 1 "bob" 1234 100 'live 'http) + (set! res (open-run-close tasks:get-best-server tasks:open-db)) + (number? (vector-ref res 3)))) + +(test "de-register server" #f (let ((res #f)) + (open-run-close tasks:server-deregister tasks:open-db "bob" port: 1234) + (vector? (open-run-close tasks:get-best-server tasks:open-db)))) + +(define server-pid #f) + +;; Not sure how the following should work, replacing it with system of megatest -server +;; (test "launch server" #t (let ((pid (process-fork (lambda () +;; ;; (daemon:ize) +;; (server:launch 'http))))) +;; (set! server-pid pid) +;; (number? pid))) +(system "../../bin/megatest -server - -debug 22 > server.log 2> server.log &") + +(let loop ((n 10)) + (thread-sleep! 1) ;; need to wait for server to start. + (let ((res (open-run-close tasks:get-best-server tasks:open-db))) + (print "tasks:get-best-server returned " res) + (if (and (not res) + (> n 0)) + (loop (- n 1))))) + +(test "get-best-server" #t (begin + (client:launch) + (let ((dat (open-run-close tasks:get-best-server tasks:open-db))) + (vector? dat)))) + +(define *keys* (keys:config-get-fields *configdat*)) +(define *keyvals* (keys:target->keyval *keys* "a/b/c")) + +(test #f #t (string? (car *runremote*))) +(test #f '(#t "successful login") (rmt:login)) ;; *runremote* *toppath* *my-client-signature*))) + +(test #f #f (rmt:get-test-info-by-id 99)) ;; get non-existant test + +;; RUNS +(test #f 1 (rmt:register-run *keyvals* "firstrun" "new" "n/a" (current-user-name))) +(test "get run info" "firstrun" (let ((rinfo (rmt:get-run-info 1))) + (vector-ref (vector-ref rinfo 1) 3))) +(test "get runname from id" "firstrun" (rmt:get-run-name-from-id 1)) + +;; TESTS +(test "get tests (no data)" '() (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f)) +(test "register test" #t (rmt:general-call 'register-test 1 "test1" "")) +(test "get tests (some data)" 1 (length (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f))) +(test "get test id" 1 (rmt:get-test-id 1 "test1" "")) +(test "sync back" #t (> (rmt:sync-inmem->db) 0)) +(test "get test id from main" 1 (db:get-test-id *db* 1 "test1" "")) +(test "get keys" #t (list? (rmt:get-keys))) +(test "set comment" #t (begin (rmt:general-call 'set-test-comment "this is a comment" 1) #t)) +(test "get comment" "this is a comment" (let ((trec (rmt:get-test-info-by-id 1))) + (db:test-get-comment trec))) + +;; MORE RUNS +(test "get runs" #t (let* ((runs (rmt:get-runs "%" #f #f '())) + (header (vector-ref runs 0)) + (data (vector-ref runs 1))) + (and (list? header) + (list? data) + (vector? (car data))))) + +(test "get local testinfo" "test1" (vector-ref (db:get-testinfo-state-status *db* 1) 2)) +(test "get testinfo" "test1" (vector-ref (rmt:get-testinfo-state-status 1) 2)) + +;;====================================================================== +;; D B +;;====================================================================== + +(test "pass fail counts" #t (rmt:general-call 'pass-fail-counts 10 9 1)) +(test "get pass fail counts" 19 (let ((dat (rmt:get-test-info-by-id 1))) + (+ (db:test-get-pass_count dat) + (db:test-get-fail_count dat)))) + +(define testregistry (make-hash-table)) +(for-each + (lambda (tname) + (for-each + (lambda (itempath) + (let ((tkey (conc tname "/" itempath)) + (rpass (random 10)) + (rfail (random 10))) + (hash-table-set! testregistry tkey (list tname itempath)) + (rmt:general-call 'register-test 1 tname itempath) + (let* ((tid (rmt:get-test-id 1 tname itempath)) + (tdat (rmt:get-test-info-by-id tid))) + (rmt:general-call 'pass-fail-counts rpass rfail (db:test-get-id tdat)) + (let* ((resdat (rmt:get-test-info-by-id tid))) + (test "set/get pass fail counts" (list rpass rfail) + (list (db:test-get-pass_count resdat) + (db:test-get-fail_count resdat))))))) + (list "" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j"))) + (list "test1" "test2" "test3" "test4" "test5")) + + +(test #f '(#t "exit process started") (rmt:kill-server)) ;; *toppath* *my-client-signature* #f))) +