Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -70,11 +70,11 @@ (run-id (cadr params)) (realparams (cddr params))) (db:with-db dbstruct run-id #t ;; these are all for modifying the db (lambda (db) (db:general-call db stmtname realparams))))) - ((sync-inmem->db) (db:sync-back)) + ((sync-inmem->db) (db:sync-touched dbstruct)) ((kill-server) (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*) ;; (db:sync-to *inmemdb* *db*) (let ((hostname (car *runremote*)) (port (cadr *runremote*)) (pid (if (null? params) #f (car params))) Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -58,14 +58,14 @@ (begin (debug:print 0 "ERROR: failed to find megatest.config, exiting") (exit)))) (push-directory *toppath*) ;; This is probably NOT needed ;; clients get the sdb:qry proc created here - (if (not sdb:qry) - (begin - (set! sdb:qry (make-sdb:qry (conc *toppath* "/db/strings.db"))) ;; we open the normalization helpers here - (sdb:qry 'setup #f))) + ;; (if (not sdb:qry) + ;; (begin + ;; (set! sdb:qry (make-sdb:qry (conc *toppath* "/db/strings.db"))) ;; we open the normalization helpers here + ;; (sdb:qry 'setup #f))) (debug:print-info 11 "*transport-type* is " *transport-type* ", *runremote* is " *runremote*) (let* ((hostinfo (open-run-close tasks:get-best-server tasks:open-db))) (debug:print-info 11 "CLIENT SETUP, hostinfo=" hostinfo) (set! *transport-type* (if hostinfo Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -26,12 +26,12 @@ (declare (uses common)) (declare (uses db)) (declare (uses gutils)) (declare (uses rmt)) (declare (uses ezsteps)) -(declare (uses sdb)) -(declare (uses filedb)) +;; (declare (uses sdb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -76,11 +76,12 @@ lbl) (store-label "testcomment" (iup:label "TestComment " #:expand "HORIZONTAL") (lambda (testdat) - (sdb:qry 'getstr (db:test-get-comment testdat)))) + ;; (sdb:qry 'getstr + (db:test-get-comment testdat))) ;; ) (store-label "testid" (iup:label "TestId " #:expand "HORIZONTAL") (lambda (testdat) (db:test-get-id testdat))) @@ -183,15 +184,18 @@ (iup:label "" #:expand "VERTICAL"))) (apply iup:vbox ; #:expand "YES" (list ;; NOTE: Yes, the host can change! (store-label "HostName" - (iup:label (sdb:qry 'getstr (db:test-get-host testdat)) #:expand "HORIZONTAL") + (iup:label ;; (sdb:qry 'getstr + (db:test-get-host testdat) ;; ) + #:expand "HORIZONTAL") (lambda (testdat)(db:test-get-host testdat))) (store-label "Uname" (iup:label " " #:expand "HORIZONTAL") - (lambda (testdat)(sdb:qry 'getstr (db:test-get-uname testdat)))) + (lambda (testdat) ;; (sdb:qry 'getstr + (db:test-get-uname testdat))) ;; ) (store-label "DiskFree" (iup:label (conc (db:test-get-diskfree testdat)) #:expand "HORIZONTAL") (lambda (testdat)(conc (db:test-get-diskfree testdat)))) (store-label "CPULoad" (iup:label (conc (db:test-get-cpuload testdat)) #:expand "HORIZONTAL") @@ -526,11 +530,12 @@ (cond ((and need-update newtestdat) (set! testdat newtestdat) (set! teststeps (dashboard-tests:get-compressed-steps db test-id)) (set! logfile (conc (db:test-get-rundir testdat) "/" (db:test-get-final_logf testdat))) - (set! rundir (filedb:get-path *fdb* (db:test-get-rundir testdat))) + (set! rundir ;; (filedb:get-path *fdb* + (db:test-get-rundir testdat)) ;; ) (set! testfullname (db:test-get-fullname testdat)) ;; (debug:print 0 "INFO: teststeps=" (intersperse teststeps "\n ")) ;; I don't see why this was implemented this way. Please comment it ... ;; (if (eq? curr-mod-time db-mod-time) ;; do only once if same Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -28,12 +28,12 @@ (declare (uses keys)) (declare (uses ods)) (declare (uses fs-transport)) (declare (uses client)) (declare (uses mt)) -(declare (uses sdb)) -(declare (uses filedb)) +;; (declare (uses sdb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "db_records.scm") (include "key_records.scm") (include "run_records.scm") @@ -48,17 +48,20 @@ ;; if db already open - return inmem ;; if db not open, open inmem, rundb and sync then return inmem ;; inuse gets set automatically for rundb's ;; (define (db:get-db dbstruct run-id) - (mutex-lock! *rundb-mutex*) - (let ((db (if run-id - (db:open-rundb dbstruct run-id) - (db:open-main dbstruct)))) - ;; db prunning would go here - (mutex-unlock! *rundb-mutex*) - db)) + (if (sqlite3:database? dbstruct) ;; pass sqlite3 databases on through + dbstruct + (begin + (mutex-lock! *rundb-mutex*) + (let ((db (if run-id + (db:open-rundb dbstruct run-id) + (db:open-main dbstruct)))) + ;; db prunning would go here + (mutex-unlock! *rundb-mutex*) + db)))) ;; mod-read: ;; 'mod modified data ;; 'read read data ;; @@ -81,29 +84,29 @@ ;;====================================================================== ;; K E E P F I L E D B I N dbstruct ;;====================================================================== -(define (db:get-filedb dbstruct run-id) - (let ((db (vector-ref dbstruct 2))) - (if db - db - (let ((fdb (filedb:open-db (conc *toplevel* "/db/files.db")))) - (vector-set! dbstruct 2 fdb) - fdb)))) - -;; Can also be used to save arbitrary strings -;; -(define (db:save-path dbstruct path) - (let ((fdb (db:get-filedb dbstruct))) - (filedb:register-path fdb path))) - -;; Use to get a path. To get an arbitrary string see next define -;; -(define (db:get-path dbstruct id) - (let ((fdb (db:get-filedb dbstruct))) - (filedb:get-path db id))) +;; (define (db:get-filedb dbstruct run-id) +;; (let ((db (vector-ref dbstruct 2))) +;; (if db +;; db +;; (let ((fdb (filedb:open-db (conc *toplevel* "/db/files.db")))) +;; (vector-set! dbstruct 2 fdb) +;; fdb)))) +;; +;; ;; Can also be used to save arbitrary strings +;; ;; +;; (define (db:save-path dbstruct path) +;; (let ((fdb (db:get-filedb dbstruct))) +;; (filedb:register-path fdb path))) +;; +;; ;; Use to get a path. To get an arbitrary string see next define +;; ;; +;; (define (db:get-path dbstruct id) +;; (let ((fdb (db:get-filedb dbstruct))) +;; (filedb:get-path db id))) ;; This routine creates the db. It is only called if the db is not already opened ;; (define (db:open-rundb dbstruct run-id) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) (let ((rdb (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) @@ -111,11 +114,11 @@ rdb (let* ((local (dbr:dbstruct-get-local dbstruct)) (toppath (dbr:dbstruct-get-path dbstruct)) (dbpath (conc toppath "/db/" run-id ".db")) (dbexists (file-exists? dbpath)) - (inmem (if local #f (open-inmem-db))) + (inmem (if local #f (db:open-inmem-db))) (db (sqlite3:open-database dbpath)) (write-access (file-write-access? dbpath)) (handler (make-busy-timeout 136000))) (if (and dbexists (not write-access)) (set! *db-write-access* #f)) ;; only unset so other db's also can use this control @@ -122,11 +125,11 @@ (if write-access (begin (if (not dbexists) (begin (db:initialize-run-id-db db) - (sdb:initialize db) + ;; (sdb:initialize db) )) ;; add strings db to rundb, not in use yet (sqlite3:set-busy-handler! db handler) (sqlite3:execute db "PRAGMA synchronous = 0;"))) (dbr:dbstruct-set-runvec-val! dbstruct run-id 'rundb db) (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inuse #t) @@ -157,45 +160,65 @@ (if write-access (begin (sqlite3:set-busy-handler! db handler) (sqlite3:execute db "PRAGMA synchronous = 0;"))) (if (not dbexists) - (db:initialize-megatest-db db)) + (db:initialize-main-db db)) (dbr:dbstruct-set-main! dbstruct db) db)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; (define (db:setup #!key (local #f)) (let ((dbstruct (make-dbr:dbstruct path: *toppath* local: local))) (db:get-db dbstruct #f) ;; force one call to main - (if (not sdb:qry) - (begin - (set! sdb:qry (make-sdb:qry (conc *toppath* "/db/strings.db"))) ;; we open the normalization helpers here - (sdb:qry 'setup #f) - ;; Initialize with some known needed strings, NOTE: set this up to only execute on first db initialization - (for-each - (lambda (str) - (sdb:qry 'get-id str)) - (list "" "logs/final.log")))) - ;; (sdb:qry 'setdb ( + ;; (if (not sdb:qry) + ;; (begin + ;; (set! sdb:qry (make-sdb:qry (conc *toppath* "/db/strings.db"))) ;; we open the normalization helpers here + ;; (sdb:qry 'setup #f) + ;; ;; Initialize with some known needed strings, NOTE: set this up to only execute on first db initialization + ;; (for-each + ;; (lambda (str) + ;; (sdb:qry 'get-id str)) + ;; (list "" "logs/final.log")))) ;; (set! *fdb* (filedb:open-db (conc *toppath* "/db/paths.db"))) dbstruct)) + +;; Open the classic megatest.db file in toppath +;; +(define (db:open-megatest-db) + (let* ((dbpath (conc *toppath* "/megatest.db")) + (dbexists (file-exists? dbpath)) + (db (sqlite3:open-database dbpath)) + (write-access (file-write-access? dbpath)) + (handler (make-busy-timeout 136000))) + (if (and dbexists (not write-access)) + (set! *db-write-access* #f)) + (if write-access + (begin + (sqlite3:set-busy-handler! db handler) + (sqlite3:execute db "PRAGMA synchronous = 0;"))) + (if (not dbexists) + (begin + (db:initialize-main-db db) + (db:initialize-run-id-db db))) + db)) ;; sync all touched runs to disk (define (db:sync-touched dbstruct) - (for-each - (lambda (runvec) - (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime))) - (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime))) - (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb))) - (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))) - (if (> mtime stime) - (begin - (db:sync-tables db:sync-tests-only inmem rundb) - (vector-set! runvec (dbr:dbstruct-field-name->run 'stime (current-milliseconds))))))) - (hash-table-values (vector-ref dbstruct 1)))) + (let ((tot-synced 0)) + (for-each + (lambda (runvec) + (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime))) + (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime))) + (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb))) + (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))) + (if (> mtime stime) + (let ((num-sunced (db:sync-tables db:sync-tests-only inmem rundb))) + (set! tot-synced (+ tot-synced num-synced)) + (vector-set! runvec (dbr:dbstruct-field-name->run 'stime (current-milliseconds))))))) + (hash-table-values (vector-ref dbstruct 1))))) ;; close all opened run-id dbs (define (db:close-all dbstruct) ;; finalize main.db (sqlite3:finalize! (db:get-db dbstruct #f)) @@ -204,42 +227,43 @@ (let ((rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))) (if (sqlite3:database? rundb) (sqlite3:finalize! rundb) (debug:print 0 "WARNING: attempting to close databases but got " rundb " instead of a database")))) (hash-table-values (vector-ref dbstruct 1))) - (sdb:qry 'finalize! #f)) + ;; (sdb:qry 'finalize! #f) + ) ;; (filedb:finalize-db! *fdb*)) -(define (open-inmem-db) +(define (db:open-inmem-db) (let* ((db (sqlite3:open-database ":memory:")) (handler (make-busy-timeout 3600))) (db:initialize-run-id-db db) - (sdb:initialize db) ;; for future use + ;; (sdb:initialize db) ;; for future use (sqlite3:set-busy-handler! db handler) db)) ;; just tests, test_steps and test_data tables (define db:sync-tests-only (list - (list "strs" - '("id" #f) - '("str" #f)) + ;; (list "strs" + ;; '("id" #f) + ;; '("str" #f)) (list "tests" '("id" #f) '("run_id" #f) '("testname" #f) '("host" #f) '("cpuload" #f) '("diskfree" #f) '("uname" #f) - '("rundir_id" #f) - '("shortdir_id" #f) + '("rundir" #f) + '("shortdir" #f) '("item_path" #f) '("state" #f) '("status" #f) '("attemptnum" #f) - '("final_logf_id" #f) + '("final_logf" #f) '("logdat" #f) '("run_duration" #f) '("comment" #f) '("event_time" #f) '("fail_count" #f) @@ -288,17 +312,17 @@ '("testname" #f) '("host" #f) '("cpuload" #f) '("diskfree" #f) '("uname" #f) - '("rundir_id" #f) - '("shortdir_id" #f) + '("rundir" #f) + '("shortdir" #f) '("item_path" #f) '("state" #f) '("status" #f) '("attemptnum" #f) - '("final_logf_id" #f) + '("final_logf" #f) '("logdat" #f) '("run_duration" #f) '("comment" #f) '("event_time" #f) '("fail_count" #f) @@ -328,11 +352,12 @@ ;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) ) (define (db:sync-tables tbls fromdb todb) (let ((stmts (make-hash-table)) ;; table-field => stmt (all-stmts '()) ;; ( ( stmt1 value1 ) ( stml2 value2 )) (numrecs (make-hash-table)) - (start-time (current-milliseconds))) + (start-time (current-milliseconds)) + (tot-count 0)) (for-each ;; table (lambda (tabledat) (let* ((tablename (car tabledat)) (fields (cdr tabledat)) (num-fields (length fields)) @@ -357,10 +382,12 @@ (sqlite3:for-each-row (lambda (a . b) (set! fromdat (cons (apply vector a b) fromdat))) fromdb full-sel) + + (debug:print 0 "INFO: found " (length fromdat) " records to sync") ;; read the target table (sqlite3:for-each-row (lambda (a . b) (hash-table-set! todat a (apply vector a b))) @@ -395,13 +422,15 @@ (debug:print 0 "INFO: db sync, total run time " runtime " ms") (for-each (lambda (dat) (let ((tblname (car dat)) (count (cdr dat))) + (set! tot-count (+ tot-count count)) (if (> count 0) (debug:print 0 (format #f " ~10a ~5a" tblname count))))) - (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b)))))))) + (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b)))))) + tot-count)) (define (db:sync-back) (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*)) ;; (db:sync-to *inmemdb* *db*)) ;; keeping it around for debugging purposes only @@ -436,11 +465,11 @@ ;; (define open-run-close (define open-run-close ;; (if (debug:debug-mode 2) open-run-close-no-exception-handling) ;; open-run-close-exception-handling)) -(define (db:initialize-megatest-db db) +(define (db:initialize-main-db db) (let* ((configdat (car *configinfo*)) ;; tut tut, global warning... (keys (keys:config-get-fields configdat)) (havekeys (> (length keys) 0)) (keystr (keys->keystr keys)) (fieldstr (keys->key/field keys))) @@ -505,17 +534,17 @@ testname TEXT DEFAULT 'noname', host TEXT DEFAULT 'n/a', cpuload REAL DEFAULT -1, diskfree INTEGER DEFAULT -1, uname TEXT DEFAULT 'n/a', - rundir_id INTEGER DEFAULT -1, - shortdir_id INTEGER DEFAULT -1, + rundir TEXT DEFAULT '/tmp/badname', + shortdir TEXT DEFAULT '/tmp/badname', item_path TEXT DEFAULT '', state TEXT DEFAULT 'NOT_STARTED', status TEXT DEFAULT 'FAIL', attemptnum INTEGER DEFAULT 0, - final_logf_id INTEGER DEFAULT 1, -- 'logs/final.log', + final_logf TEXT DEFAULT 'logs/final.log', logdat TEXT DEFAULT '', run_duration INTEGER DEFAULT 0, comment TEXT DEFAULT '', event_time TIMESTAMP DEFAULT (strftime('%s','now')), fail_count INTEGER DEFAULT 0, @@ -956,16 +985,17 @@ (lambda (run-info) (let ((run-name (cadr run-info)) (run-id (car run-info))) (sqlite3:for-each-row (lambda (state count) - (let* ((stateparts (string-split state "|")) - (newstate (conc (car stateparts) "\n" (cadr stateparts)))) - (hash-table-set! totals newstate (+ (hash-table-ref/default totals newstate 0) count)) - (set! res (cons (list runname newstate count) res)))) + (if (string? state) + (let* ((stateparts (string-split state "|")) + (newstate (conc (car stateparts) "\n" (cadr stateparts)))) + (hash-table-set! totals newstate (+ (hash-table-ref/default totals newstate 0) count)) + (set! res (cons (list runname newstate count) res))))) (db:get-db dbstruct run-id) - "SELECT state||'|'||status AS s,count(id) FROM tests AS t ON ORDER BY s DESC;" ) + "SELECT state||'|'||status AS s,count(id) FROM tests AS t ORDER BY s DESC;" ) ;; (set! res (reverse res)) (for-each (lambda (state) (set! res (cons (list "Totals" state (hash-table-ref totals state)) res))) (sort (hash-table-keys totals) string>=)))) runs-info) @@ -1034,11 +1064,12 @@ (let ((finalres (vector header res))) ;; (hash-table-set! *run-info-cache* run-id finalres) finalres))) (define (db:set-comment-for-run dbstruct run-id comment) - (sqlite3:execute (db:get-db dbstruct #f) "UPDATE runs SET comment=? WHERE id=?;" (sdb:qry 'getid comment) run-id)) + (sqlite3:execute (db:get-db dbstruct #f) "UPDATE runs SET comment=? WHERE id=?;" comment ;; (sdb:qry 'getid comment) + run-id)) ;; does not (obviously!) removed dependent data. But why not!!? (define (db:delete-run dbstruct run-id) ;; First set any related tests to DELETED (let ((db (db:get-db dbstruct run-id))) @@ -1135,11 +1166,11 @@ ;; states and statuses are required to be lists, empty is ok ;; not-in #t = above behaviour, #f = must match (define (db:get-tests-for-run dbstruct run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals) (let* ((qryvalstr (case qryvals ((shortlist) "id,run_id,testname,item_path,state,status") - ((#f) "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir_id,item_path,run_duration,final_logf_id,comment") + ((#f) db:test-record-qry-selector) ;; "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment") (else qryvals))) (res '()) ;; if states or statuses are null then assume match all when not-in is false (states-qry (if (null? states) #f @@ -1218,20 +1249,20 @@ (debug:print-info 8 "db:get-tests-for-run qry=" qry) (db:with-db dbstruct run-id #f (lambda (db) (sqlite3:for-each-row (lambda (id testname item-path state status) - ;; id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment - (set! res (cons (vector id run-id testname state status -1 "" -1 -1 "" "-" item-path -1 "-" "-") res))) + ;; id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment + (set! res (cons (vector id run-id testname state status -1 "" -1 -1 "" "-" item-path -1 "-" "-") res))) db qry run-id))) res)) (define (db:get-testinfo-state-status dbstruct run-id test-id) (let ((res #f)) - (db:with-db dbstruct #f + (db:with-db dbstruct run-id #f (lambda (db) (sqlite3:for-each-row (lambda (run-id testname item-path state status) ;; id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment (set! res (vector test-id run-id testname state status -1 "" -1 -1 "" "-" item-path -1 "-" "-"))) @@ -1297,17 +1328,19 @@ ;; (define (db:test-set-state-status-by-id dbstruct run-id test-id newstate newstatus newcomment) (let ((db (db:get-db dbstruct run-id))) (cond ((and newstate newstatus newcomment) - (sqlite3:execute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus (sdb:qry 'getid newcomment) test-id)) + (sqlite3:execute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus newcomment ;; (sdb:qry 'getid newcomment) + test-id)) ((and newstate newstatus) (sqlite3:execute db "UPDATE tests SET state=?,status=? WHERE id=?;" newstate newstatus test-id)) (else (if newstate (sqlite3:execute db "UPDATE tests SET state=? WHERE id=?;" newstate test-id)) (if newstatus (sqlite3:execute db "UPDATE tests SET status=? WHERE id=?;" newstatus test-id)) - (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" (sdb:qry 'getid newcomment) test-id)))) + (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment ;; (sdb:qry 'getid newcomment) + test-id)))) (mt:process-triggers test-id newstate newstatus))) ;; Never used, but should be? (define (db:test-set-state-status-by-run-id-testname db run-id test-name item-path status state) (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" @@ -1370,26 +1403,43 @@ (db:get-db dbstruct run-id) "SELECT id FROM tests WHERE testname=? AND item_path=?;" testname item-path) res)) -(define db:test-record-qry-selector "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir_id,item_path,run_duration,final_logf_id,comment,shortdir_id") +(define db:test-record-fields '("id" "run_id" "testname" "state" "status" "event_time" + "host" "cpuload" "diskfree" "uname" "rundir" "item_path" + "run_duration" "final_logf" "comment" "shortdir")) + +(define db:test-record-qry-selector (string-intersperse db:test-record-fields ",")) ;; NOTE: Use db:test-get* to access records -;; NOTE: This needs rundir_id decoding? Decide, decode here or where used? For the moment decode where used. +;; NOTE: This needs rundir decoding? Decide, decode here or where used? For the moment decode where used. (define (db:get-all-tests-info-by-run-id dbstruct run-id) (let ((db (db:get-db dbstruct run-id)) (res '())) (sqlite3:for-each-row - (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run-duration final-logf-id comment short-dir-id) + (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir) ;; 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 - (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run-duration final-logf-id comment short-dir-id) + (set! res (cons (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment shortdir) res))) (db:get-db dbstruct run-id) (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE run_id=?;") run-id) res)) + +(define (db:replace-test-records dbstruct run-id testrecs) + (let* ((qmarks (string-intersperse (make-list (length db:test-record-fields) "?") ",")) + (qrystr (conc "INSERT OR REPLACE INTO tests (" db:test-record-qry-selector ") VALUES (" qmarks ");")) + (qry (sqlite3:prepare (db:get-db dbstruct run-id) qrystr))) + (debug:print 8 "INFO: replace-test-records, qrystr=" qrystr) + (for-each + (lambda (rec) + ;; (debug:print 0 "INFO: Inserting values: " (string-intersperse (map conc (vector->list rec)) ", ")) + (apply sqlite3:execute qry (vector->list rec))) + testrecs) + (sqlite3:finalize! qry))) + ;; Get test data using test_id (define (db:get-test-info-by-id dbstruct run-id test-id) (let ((db (db:get-db dbstruct run-id)) (res #f)) @@ -1448,12 +1498,14 @@ (let ((db (db:get-db dbstruct run-id))) (sqlite3:execute db "INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment,logfile) VALUES(?,?,?,?,?,?,?);" test-id teststep-name state-in status-in (current-seconds) - (sdb:qry 'getid (if comment comment "")) - (sdb:qry 'getid (if logfile logfile ""))))) + ;; (sdb:qry 'getid + (if comment comment "") ;; ) + ;; (sdb:qry 'getid + (if logfile logfile "")))) ;; ) ;; db-get-test-steps-for-run (define (db:get-steps-for-test db run-id test-id) (let* ((db (db:get-db dbstruct run-id)) (res '())) @@ -1594,11 +1646,11 @@ keynames (string-split target "/")) " AND ")) (testqry (tests:match->sqlqry testpatt)) (runsqry (sqlite3:prepare (db:get-db dbstruct #f)(conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';"))) - (tstsqry (conc "SELECT rundir_id FROM tests WHERE " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;"))) + (tstsqry (conc "SELECT rundir FROM tests WHERE " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;"))) (debug:print 8 "db:test-get-paths-matching-keynames-target-new\n runsqry=" runsqry "\n tstqry=" tstqry) (sqlite3:for-each-row (lambda (rid) (set! row-ids (cons rid row-ids))) runsqry) @@ -1665,20 +1717,20 @@ (sqlite3:execute (db:get-db dbstruct run-id) 'register-test run-id test-name item-path)) (define (db:test-get-logfile-info dbstruct run-id test-name) (let ((res #f)) (sqlite3:for-each-row - (lambda (path-id final_logf-id) - (let ((path (sdb:qry 'getstr path-id)) - (final_logf (sdb:qry 'getstr final_logf-id))) - (set! logf final_logf) - (set! res (list path final_logf)) - (if (directory? path) - (debug:print 2 "Found path: " path) - (debug:print 2 "No such path: " path)))) - (db:get-db dbstruct run-id) - "SELECT rundir_id,final_logf_id FROM tests WHERE testname=? AND item_path='';" + (lambda (path final_logf) + ;; (let ((path (sdb:qry 'getstr path-id)) + ;; (final_logf (sdb:qry 'getstr final_logf-id))) + (set! logf final_logf) + (set! res (list path final_logf)) + (if (directory? path) + (debug:print 2 "Found path: " path) + (debug:print 2 "No such path: " path))) ;; ) + (db:get-db dbstruct run-id) + "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='';" test-name) res)) ;;====================================================================== ;; A G R E G A T E D T R A N S A C T I O N D B W R I T E S @@ -1691,13 +1743,13 @@ '(register-test "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');") ;; Test state and status '(set-test-state "UPDATE tests SET state=? WHERE id=?;") '(set-test-status "UPDATE tests SET state=? WHERE id=?;") '(state-status "UPDATE tests SET state=?,status=? WHERE id=?;") ;; DONE - '(state-status-msg "UPDATE tests SET state=?,status=?,comment_id=? WHERE id=?;") ;; DONE + '(state-status-msg "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;") ;; DONE ;; Test comment - '(set-test-comment "UPDATE tests SET comment_id=? WHERE id=?;") + '(set-test-comment "UPDATE tests SET comment=? WHERE id=?;") '(set-test-start-time "UPDATE tests SET event_time=strftime('%s','now') WHERE id=?;") ;; DONE '(pass-fail-counts "UPDATE tests SET pass_count=?,fail_count=? WHERE id=?;") ;; test_data-pf-rollup is used to set a tests PASS/FAIL based on the pass/fail info from the steps '(test_data-pf-rollup "UPDATE tests SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 @@ -1705,15 +1757,15 @@ WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL') THEN 'PASS' ELSE status END WHERE id=?;") ;; DONE - '(test-set-log "UPDATE tests SET final_logf_id=? WHERE id=?;") ;; DONE - '(test-set-rundir-by-test-id "UPDATE tests SET rundir_id=? WHERE id=?") ;; DONE - '(test-set-rundir "UPDATE tests SET rundir_id=? AND testname=? AND item_path=?;") ;; DONE + '(test-set-log "UPDATE tests SET final_logf=? WHERE id=?;") ;; DONE + '(test-set-rundir-by-test-id "UPDATE tests SET rundir=? WHERE id=?") ;; DONE + '(test-set-rundir "UPDATE tests SET rundir=? AND testname=? AND item_path=?;") ;; DONE '(delete-tests-in-state "DELETE FROM tests WHERE state=?;") ;; DONE - '(tests:test-set-toplog "UPDATE tests SET final_logf_id=? WHERE run_id=? AND testname=? AND item_path='';") + '(tests:test-set-toplog "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path='';") '(update-cpuload-diskfree "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;") ;; DONE '(update-uname-host "UPDATE tests SET uname=?,host=? WHERE id=?;") ;; DONE '(update-test-state "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") '(update-test-status "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") ;; stuff for roll-up-pass-fail-counts @@ -1886,11 +1938,11 @@ (lambda (id itempath state status run_duration logf-id comment-id) (let ((logf (db:get-string dbstruct logf-id)) (comment (db:get-string dbstruct comment-id))) (set! res (cons (vector id itempath state status run_duration logf comment) res))) (db:get-db dbstruct run-id) - "SELECT id,item_path,state,status,run_duration,final_logf_id,comment_id FROM tests WHERE testname=? AND item_path != '';" + "SELECT id,item_path,state,status,run_duration,final_logf,comment FROM tests WHERE testname=? AND item_path != '';" test-name) res))) ;;====================================================================== ;; Tests meta data @@ -2042,11 +2094,11 @@ (results (list runsheader)) (testdata-header (list "Run Id" "Testname" "Item Path" "Category" "Variable" "Value" "Expected" "Tol" "Units" "Status" "Comment")) (mainqry (conc "SELECT t.testname,r.id,runname," keysstr ",t.testname, t.item_path,tm.description,t.state,t.status, - final_logf_id,run_duration, + final_logf,run_duration, strftime('%m/%d/%Y %H:%M:%S',datetime(t.event_time,'unixepoch'),'localtime'), tm.tags,r.owner,t.comment, author, tm.owner,reviewed, diskfree,uname,rundir, Index: ezsteps.scm ================================================================== --- ezsteps.scm +++ ezsteps.scm @@ -16,20 +16,21 @@ (declare (unit ezsteps)) (declare (uses db)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) -(declare (uses sdb)) -(declare (uses filedb)) +;; (declare (uses sdb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") (define (ezsteps:run-from testdat start-step-name run-one) - (let* ((test-run-dir (filedb:get-path *fdb* (db:test-get-rundir testdat))) + (let* ((test-run-dir ;; (filedb:get-path *fdb* + (db:test-get-rundir testdat)) ;; ) (testconfig (read-config (conc test-run-dir "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) (ezstepslst (hash-table-ref/default testconfig "ezsteps" '())) (run-mutex (make-mutex)) (rollup-status 0) (exit-info (vector #t #t #t)) Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -528,11 +528,11 @@ (thread-start! th2) (thread-start! th3) (set! *didsomething* #t) (thread-join! th2)) (debug:print 0 "ERROR: Failed to setup for megatest"))) - (sdb:qry 'finalize) + ;; (sdb:qry 'finalize) (exit))) (define (http-transport:server-signal-handler signum) (handle-exceptions exn Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -19,13 +19,13 @@ (declare (unit launch)) (declare (uses common)) (declare (uses configf)) (declare (uses db)) -(declare (uses sdb)) +;; (declare (uses sdb)) (declare (uses tdb)) -(declare (uses filedb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -25,12 +25,12 @@ (declare (uses client)) (declare (uses tests)) (declare (uses genexample)) (declare (uses daemon)) (declare (uses db)) -(declare (uses sdb)) -(declare (uses filedb)) +;; (declare (uses sdb)) +;; (declare (uses filedb)) (declare (uses tdb)) (declare (uses mt)) (declare (uses api)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! @@ -243,10 +243,11 @@ "-gen-megatest-area" "-mark-incompletes" "-convert-to-norm" "-convert-to-old" + "-import-megatest.db" "-logging" "-v" ;; verbose 2, more than normal (normal is 1) "-q" ;; quiet 0, errors/warnings only ) @@ -358,11 +359,11 @@ "-stop-server" "-show-cmdinfo" "-list-runs"))) (if (setup-for-run) (begin - (set! *fdb* (filedb:open-db (conc *toppath* "/db/paths.db"))) + ;; (set! *fdb* (filedb:open-db (conc *toppath* "/db/paths.db"))) ;; if not list or kill then start a client (if appropriate) (if (or (args-defined? "-h" "-version" "-gen-megatest-area" "-gen-megatest-test") (eq? (length (hash-table-keys args:arg-hash)) 0)) (debug:print-info 1 "Server connection not needed") ;; ok, so lets connect to the server @@ -615,13 +616,14 @@ (equal? (db:test-get-status test) "WARN") (equal? (db:test-get-state test) "NOT_STARTED"))) (begin (print " cpuload: " (db:test-get-cpuload test) "\n diskfree: " (db:test-get-diskfree test) - "\n uname: " (sdb:qry 'getstr (db:test-get-uname test)) - "\n rundir: " (sdb:qry 'getstr ;; (filedb:get-path *fdb* - (db:test-get-rundir test)) + "\n uname: " ;; (sdb:qry 'getstr + (db:test-get-uname test) ;; ) + "\n rundir: " ;; (sdb:qry 'getstr ;; (filedb:get-path *fdb* + (db:test-get-rundir test) ;; ) ) ;; Each test ;; DO NOT remote run (let ((steps (db:get-steps-for-test dbstruct run-id (db:test-get-id test)))) (for-each @@ -1173,19 +1175,36 @@ (conc "SELECT id," field " FROM tests;")) (debug:print-info 0 "found " (length dat) " items for field " field) (let ((qry (sqlite3:prepare db (conc "UPDATE tests SET " field "=? WHERE id=?;")))) (for-each (lambda (item) - (let ((newval (sdb:qry 'getid (cadr item)))) + (let ((newval ;; (sdb:qry 'getid + (cadr item))) ;; ) (if (not (equal? newval (cadr item))) (debug:print-info 0 "Converting " (cadr item) " to " newval " for test #" (car item))) (sqlite3:execute qry newval (car item)))) dat) (sqlite3:finalize! qry)))) (db:close-all dbstruct) (list "uname" "rundir" "final_logf" "comment")) (set! *didsomething* #t))) + +(if (args:get-arg "-import-megatest.db") + (let* ((toppath (setup-for-run)) + (dbstruct (if toppath (make-dbr:dbstruct path: toppath) #f)) + (mtdb (if toppath (db:open-megatest-db))) + (run-ids (if toppath (db:get-run-ids mtdb)))) + (for-each + (lambda (run-id) + (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id))) + (debug:print 0 "INFO: Updating " (length testrecs) " records for run-id=" run-id) + (db:replace-test-records dbstruct run-id testrecs))) + run-ids) + (set! *didsomething* #t) + (db:close-all dbstruct))) + + ;;====================================================================== ;; Exit and clean up ;;====================================================================== @@ -1194,11 +1213,11 @@ ;; this is the socket if we are a client ;; (if (and *runremote* ;; (socket? *runremote*)) ;; (close-socket *runremote*)) -(if sdb:qry (sdb:qry 'finalize #f)) +;; (if sdb:qry (sdb:qry 'finalize #f)) ;; (if *fdb* (filedb:finalize-db! *fdb*)) (if (not *didsomething*) (debug:print 0 help)) Index: mt.scm ================================================================== --- mt.scm +++ mt.scm @@ -18,11 +18,11 @@ (declare (uses runconfig)) (declare (uses tests)) (declare (uses server)) (declare (uses runs)) (declare (uses rmt)) -(declare (uses filedb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -127,12 +127,12 @@ ;; T R I G G E R S ;;====================================================================== (define (mt:process-triggers run-id test-id newstate newstatus) (let* ((test-dat (rmt:get-test-info-by-id run-id test-id)) - (test-rundir (rmt:sdb-qry 'getstr ;; (filedb:get-path *fdb* - (db:test-get-rundir test-dat))) ;; ) + (test-rundir ;; (rmt:sdb-qry 'getstr ;; (filedb:get-path *fdb* + (db:test-get-rundir test-dat)) ;; ) ;; ) (test-name (db:test-get-testname test-dat)) (tconfig #f) (state (if newstate newstate (db:test-get-state test-dat))) (status (if newstatus newstatus (db:test-get-status test-dat)))) (if (and test-rundir ;; #f means no dir set yet Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -19,11 +19,11 @@ (declare (uses items)) (declare (uses runconfig)) (declare (uses tests)) (declare (uses server)) (declare (uses mt)) -(declare (uses filedb)) +;; (declare (uses filedb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -1251,12 +1251,14 @@ action) ((run-wait) (debug:print 1 "Waiting for run " runkey ", run=" runnamepatt " to complete")) (else (debug:print-info 0 "action not recognised " action))) - (let ((sorted-tests (sort tests (lambda (a b)(let ((dira (rmt:sdb-qry 'getstr (db:test-get-rundir a))) ;; (filedb:get-path *fdb* (db:test-get-rundir a))) - (dirb (rmt:sdb-qry 'getstr (db:test-get-rundir b)))) ;; ((filedb:get-path *fdb* (db:test-get-rundir b)))) + (let ((sorted-tests (sort tests (lambda (a b)(let ((dira ;; (rmt:sdb-qry 'getstr + (db:test-get-rundir a)) ;; ) ;; (filedb:get-path *fdb* (db:test-get-rundir a))) + (dirb ;; (rmt:sdb-qry 'getstr + (db:test-get-rundir b))) ;; ) ;; ((filedb:get-path *fdb* (db:test-get-rundir b)))) (if (and (string? dira)(string? dirb)) (> (string-length dira)(string-length dirb)) #f))))) (test-retry-time (make-hash-table)) (allow-run-time 10)) ;; seconds to allow for killing tests before just brutally killing 'em @@ -1270,11 +1272,12 @@ (if (not (null? tal)) (loop (car tal)(cdr tal)))) (let* ((item-path (db:test-get-item-path new-test-dat)) (test-name (db:test-get-testname new-test-dat)) (run-dir ;;(filedb:get-path *fdb* - (rmt:sdb-qry 'getid (db:test-get-rundir new-test-dat))) ;; run dir is from the link tree + ;; (rmt:sdb-qry 'getid + (db:test-get-rundir new-test-dat)) ;; ) ;; run dir is from the link tree (real-dir (if (file-exists? run-dir) (resolve-pathname run-dir) #f)) (test-state (db:test-get-state new-test-dat)) (test-fulln (db:test-get-fullname new-test-dat))) Index: tests.scm ================================================================== --- tests.scm +++ tests.scm @@ -21,11 +21,11 @@ (declare (uses db)) (declare (uses tdb)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) -(declare (uses sdb)) +;; (declare (uses sdb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -134,12 +134,14 @@ ;; Check for waiver eligibility ;; (define (tests:check-waiver-eligibility testdat prev-testdat) (let* ((test-registry (make-hash-table)) (testconfig (tests:get-testconfig (db:test-get-testname testdat) test-registry #f)) - (test-rundir (sdb:qry 'passstr (db:test-get-rundir testdat))) - (prev-rundir (sdb:qry 'passstr (db:test-get-rundir prev-testdat))) + (test-rundir ;; (sdb:qry 'passstr + (db:test-get-rundir testdat)) ;; ) + (prev-rundir ;; (sdb:qry 'passstr + (db:test-get-rundir prev-testdat)) ;; ) (waivers (configf:section-vars testconfig "waivers")) (waiver-rx (regexp "^(\\S+)\\s+(.*)$")) (diff-rule "diff %file1% %file2%") (logpro-rule "diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html")) (if (not (file-exists? test-rundir)) Index: tests/Makefile ================================================================== --- tests/Makefile +++ tests/Makefile @@ -168,11 +168,11 @@ clean : rm cleanprep kill : killall -v mtest main.sh dboard || true - rm -rf */megatest.db */logging.db */monitor.db fullrun/tmp/mt_*/* || true + rm -rf *run/db/* */megatest.db */logging.db */monitor.db fullrun/tmp/mt_*/* || true killall -v mtest dboard || true hardkill : kill sleep 2;killall -v mtest main.sh dboard -9 Index: tests/fullrun/tests/test_mt_vars/currentisblah.sh ================================================================== --- tests/fullrun/tests/test_mt_vars/currentisblah.sh +++ tests/fullrun/tests/test_mt_vars/currentisblah.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -grep CURRENT megatest.sh | grep /tmp/nada +grep -e '^CURRENT' megatest.sh | grep /tmp/nada Index: tests/rununittest.sh ================================================================== --- tests/rununittest.sh +++ tests/rununittest.sh @@ -7,11 +7,11 @@ (cd ..;make && make install) # Clean setup # rm -f simplerun/megatest.db simplerun/monitor.db simplerun/db/monitor.db -rm -rf simplelinks/ simpleruns/ +rm -rf simplelinks/ simpleruns/ simplerun/db/ mkdir -p simplelinks simpleruns (cd simplerun;cp ../../*_records.scm .;perl -pi.bak -e 's/define-inline/define/' *_records.scm) # Run the test $1 is the unit test to run cd simplerun;echo '(load "../tests.scm")' | ../../bin/megatest -repl -debug $2 $1 Index: tests/unittests/inmemdb.scm ================================================================== --- tests/unittests/inmemdb.scm +++ tests/unittests/inmemdb.scm @@ -23,11 +23,11 @@ (define *keyvals* (keys:target->keyval *keys* "a/b/c")) (test #f #t (string? (car *runremote*))) (test #f '(#t "successful login") (rmt:login)) ;; *runremote* *toppath* *my-client-signature*))) -(define inmem (open-in-mem-db)) +(define inmem (db:open-inmem-db)) (define (inmem-test t b) (test "inmem sync to" t (db:sync-to *db* inmem)) (test "inmem sync back" b (db:sync-to inmem *db*))) Index: tests/unittests/server.scm ================================================================== --- tests/unittests/server.scm +++ tests/unittests/server.scm @@ -61,13 +61,12 @@ (test "get tests (no data)" '() (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f)) (test "register test" #t (rmt:general-call 'register-test 1 1 "test1" "")) (test "get tests (some data)" 1 (length (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f))) (test "get test id" 1 (rmt:get-test-id 1 "test1" "")) -(print "SKIPPING sync back for now") -;; (test "sync back" #t (> (rmt:sync-inmem->db) 0)) -;; (test "get test id from main" 1 (db:get-test-id *db* 1 "test1" "")) +(test "sync back" #t (> (rmt:sync-inmem->db) 0)) +(test "get test id from main" 1 (db:get-test-id *db* 1 "test1" "")) (test "get keys" #t (list? (rmt:get-keys))) (test "set comment" #t (begin (rmt:general-call 'set-test-comment 1 "this is a comment" 1) #t)) (test "get comment" "this is a comment" (let ((trec (rmt:get-test-info-by-id 1 1))) (db:test-get-comment trec))) @@ -78,12 +77,12 @@ (data (vector-ref runs 1))) (and (list? header) (list? data) (vector? (car data))))) -(test "get local testinfo" "test1" (vector-ref (db:get-testinfo-state-status *db* 1) 2)) -(test "get testinfo" "test1" (vector-ref (rmt:get-testinfo-state-status 1) 2)) +(test "get local testinfo" "test1" (vector-ref (db:get-testinfo-state-status *db* 1 1) 2)) +(test "get testinfo" "test1" (vector-ref (rmt:get-testinfo-state-status 1 1) 2)) ;;====================================================================== ;; D B ;;======================================================================