Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -71,11 +71,11 @@ (run-id (cadr params)) (realparams (cddr params))) (db:with-db dbstruct run-id #t ;; these are all for modifying the db (lambda (db) (db:general-call db stmtname realparams))))) - ((sync-inmem->db) (db:sync-touched dbstruct force-sync: #t)) + ((sync-inmem->db) (db:sync-touched dbstruct run-id force-sync: #t)) ;; ((kill-server) ;; (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*) ;; (db:sync-to *inmemdb* *db*) ;; (let ((hostname (car *runremote*)) ;; (port (cadr *runremote*)) ;; (pid (if (null? params) #f (car params))) Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -237,13 +237,13 @@ (iup:frame #:title "Set fields" (iup:vbox (iup:hbox (iup:label "Comment:") (let ((txtbox (iup:textbox #:action (lambda (val a b) - (rmt:test-set-state-status-by-id run-id test-id #f #f b) + (rmt:test-set-state-status-by-id run-id test-id #f #f b) ;; IDEA: Just set a variable with the proc to call? - (open-run-close db:test-set-state-status-by-id db test-id #f #f b) + (rmt:test-set-state-status-by-id run-id test-id #f #f b) (set! newcomment b)) #:value (db:test-get-comment testdat) #:expand "HORIZONTAL"))) (set! wtxtbox txtbox) txtbox)) @@ -285,11 +285,11 @@ (iup:attribute-set! wtxtbox "VALUE" c) (if (not *dashboard-comment-share-slot*) (set! *dashboard-comment-share-slot* wtxtbox))) )))) (begin - (open-run-close db:test-set-state-status-by-id db test-id #f status #f) + (rmt:test-set-state-status-by-id run-id test-id #f status #f) (db:test-set-status! testdat status)))))))) btn)) (map cadr *common:std-statuses*)))) ;; (list "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP")))) (vector-set! *state-status* 1 (lambda (status color) @@ -366,11 +366,11 @@ (let ((comment (iup:attribute comnt "VALUE")) (test-id (db:test-get-id testdat))) (if (or (not wpatt) (string-match wregx comment)) (begin - (open-run-close db:test-set-state-status-by-id #f test-id #f "WAIVED" comment) + (rmt:test-set-state-status-by-id run-id test-id #f "WAIVED" comment) (db:test-set-status! testdat "WAIVED") (cmtcmd comment) (iup:destroy! dlog)))))) (iup:button "Cancel" #:expand "HORIZONTAL" Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -68,13 +68,13 @@ (define (db:done-with dbstruct run-id mod-read) (if (not (sqlite3:database? dbstruct)) (begin (mutex-lock! *rundb-mutex*) (if (eq? mod-read 'mod) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'mtime (current-milliseconds)) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'rtime (current-milliseconds))) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inuse #f) + (dbr:dbstruct-set-mtime! dbstruct (current-milliseconds)) + (dbr:dbstruct-set-rtime! dbstruct (current-milliseconds))) + (dbr:dbstruct-set-inuse! dbstruct #f) (mutex-unlock! *rundb-mutex*)))) ;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;") ;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no ;; @@ -109,15 +109,15 @@ ;; (filedb:get-path db id))) ;; This routine creates the db. It is only called if the db is not already opened ;; (define (db:open-rundb dbstruct run-id) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) - (let ((rdb (dbr:dbstruct-get-runvec-val dbstruct run-id 'inmem))) ;; (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) + (let ((rdb (dbr:dbstruct-get-inmem dbstruct))) ;; (dbr:dbstruct-get-runrec dbstruct run-id 'inmem))) (if rdb rdb (let* ((local (dbr:dbstruct-get-local dbstruct)) - (toppath (dbr:dbstruct-get-path dbstruct)) + (toppath (dbr:dbstruct-get-path dbstruct)) (dbpath (conc toppath "/db/" run-id ".db")) (dbexists (file-exists? dbpath)) (inmem (if local #f (db:open-inmem-db))) (refdb (if local #f (db:open-inmem-db))) (db (sqlite3:open-database dbpath)) @@ -132,20 +132,20 @@ (db:initialize-run-id-db db) ;; (sdb:initialize db) )) ;; add strings db to rundb, not in use yet (sqlite3:set-busy-handler! db handler) (sqlite3:execute db "PRAGMA synchronous = 1;"))) ;; was 0 but 0 is a gamble - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'rundb db) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inuse #t) + (dbr:dbstruct-set-rundb! dbstruct db) + (dbr:dbstruct-set-inuse! dbstruct #t) (if local (begin - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inmem db) ;; direct access ... + (dbr:dbstruct-set-inmem! dbstruct db) ;; direct access ... db) (begin - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inmem inmem) + (dbr:dbstruct-set-inmem! dbstruct inmem) (db:sync-tables db:sync-tests-only db inmem) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'refdb refdb) + (dbr:dbstruct-set-refdb! dbstruct refdb) (db:sync-tables db:sync-tests-only db refdb) inmem)))))) ;; This routine creates the db. It is only called if the db is not already opened ;; @@ -199,40 +199,53 @@ (db:initialize-main-db db) (db:initialize-run-id-db db))) db)) ;; sync all touched runs to disk +;; (define (db:sync-touched dbstruct #!key (force-sync #f)) (let ((tot-synced 0)) (for-each (lambda (runvec) (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime))) (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime))) (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb))) - (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))) + (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem))) + (refdb (vector-ref runvec (dbr:dbstruct-field-name->num 'refdb)))) (if (or (> mtime stime) force-sync) - (let ((num-synced (db:sync-tables db:sync-tests-only inmem rundb))) + (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb))) (set! tot-synced (+ tot-synced num-synced)) (vector-set! runvec (dbr:dbstruct-field-name->num 'stime) (current-milliseconds)))))) (hash-table-values (vector-ref dbstruct 1))) tot-synced)) + +;; sync run to disk if touched +;; +(define (db:sync-touched dbstruct #!key (force-sync #f)) + (let ((mtime (dbr:dbstruct-get-mtime dbstruct)) + (stime (dbr:dbstruct-get-stime dbstruct)) + (rundb (dbr:dbstruct-get-rundb dbstruct)) + (inmem (dbr:dbstruct-get-inmem dbstruct)) + (refdb (dbr:dbstruct-get-refdb dbstruct))) + (if (or (not (number? mtime)) + (not (number? stime)) + (> mtime stime) + force-sync) + (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb))) + (dbr:dbstruct-set-stime! dbstruct (current-milliseconds)) + num-synced) + 0))) ;; close all opened run-id dbs (define (db:close-all dbstruct) ;; finalize main.db (db:sync-touched dbstruct force-sync: #t) (sqlite3:finalize! (db:get-db dbstruct #f)) - (for-each - (lambda (runvec) - (let ((rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))) - (if (sqlite3:database? rundb) - (sqlite3:finalize! rundb) - (debug:print 0 "WARNING: attempting to close databases but got " rundb " instead of a database")))) - (hash-table-values (vector-ref dbstruct 1))) - ;; (sdb:qry 'finalize! #f) - ) - ;; (filedb:finalize-db! *fdb*)) + (let ((rundb (dbr:dbstruct-get-rundb dbstruct))) + (if (sqlite3:database? rundb) + (sqlite3:finalize! rundb) + (debug:print 0 "WARNING: attempting to close databases but got " rundb " instead of a database")))) (define (db:open-inmem-db) (let* ((db (sqlite3:open-database ":memory:")) (handler (make-busy-timeout 3600))) (db:initialize-run-id-db db) @@ -316,11 +329,11 @@ '("avg_disk" #f) '("tags" #f) '("jobgroup" #f))))) ;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) ) -(define (db:sync-tables tbls fromdb todb) +(define (db:sync-tables tbls fromdb todb . slave-dbs) (cond ((not fromdb) (debug:print 0 "ERROR: db:sync-tables called with fromdb missing") -1) ((not todb) (debug:print 0 "ERROR: db:sync-tables called with todb missing") -2) ((not (sqlite3:database? fromdb)) (debug:print 0 "ERROR: db:sync-tables called with fromdb not a database " fromdb) -3) @@ -369,32 +382,35 @@ (hash-table-set! todat a (apply vector a b))) todb full-sel) ;; first pass implementation, just insert all changed rows - (let ((stmth (sqlite3:prepare todb full-ins))) - (sqlite3:with-transaction - todb - (lambda () - (for-each ;; - (lambda (fromrow) - (let* ((a (vector-ref fromrow 0)) - (curr (hash-table-ref/default todat a #f)) - (same #t)) - (let loop ((i 0)) - (if (or (not curr) - (not (equal? (vector-ref fromrow i)(vector-ref curr i)))) - (set! same #f)) - (if (and same - (< i (- num-fields 1))) - (loop (+ i 1)))) - (if (not same) - (begin - (apply sqlite3:execute stmth (vector->list fromrow)) - (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0))))))) - fromdat))) - (sqlite3:finalize! stmth)))) + (for-each + (lambda (targdb) + (let ((stmth (sqlite3:prepare targdb full-ins))) + (sqlite3:with-transaction + targdb + (lambda () + (for-each ;; + (lambda (fromrow) + (let* ((a (vector-ref fromrow 0)) + (curr (hash-table-ref/default todat a #f)) + (same #t)) + (let loop ((i 0)) + (if (or (not curr) + (not (equal? (vector-ref fromrow i)(vector-ref curr i)))) + (set! same #f)) + (if (and same + (< i (- num-fields 1))) + (loop (+ i 1)))) + (if (not same) + (begin + (apply sqlite3:execute stmth (vector->list fromrow)) + (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0))))))) + fromdat))) + (sqlite3:finalize! stmth))) + (append (list todb) slave-dbs)))) tbls) (let ((runtime (- (current-milliseconds) start-time))) (debug:print 0 "INFO: db sync, total run time " runtime " ms") (for-each (lambda (dat) @@ -1323,11 +1339,11 @@ (else (if newstate (sqlite3:execute db "UPDATE tests SET state=? WHERE id=?;" newstate test-id)) (if newstatus (sqlite3:execute db "UPDATE tests SET status=? WHERE id=?;" newstatus test-id)) (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment ;; (sdb:qry 'getid newcomment) test-id)))) - (mt:process-triggers test-id newstate newstatus))) + (mt:process-triggers run-id test-id newstate newstatus))) ;; Never used, but should be? (define (db:test-set-state-status-by-run-id-testname db run-id test-name item-path status state) (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" state status run-id test-name item-path)) Index: db_records.scm ================================================================== --- db_records.scm +++ db_records.scm @@ -9,77 +9,102 @@ ;; |-sdb.db ;; |-fdb.db ;; |-1.db ;; |-.db ;; -(define (make-dbr:dbstruct #!key (path #f)(local #f)) - (vector - #f ;; the main db (contains runs, test_meta etc.) NOT CACHED IN MEM - (make-hash-table) ;; run-id => [ rundb inmemdb last-mod last-read last-sync refdb ] - #f ;; the global string db (use for state, status etc.) - path ;; path to database files/megatest area - local)) ;; read-only local access - ;; ;; Accessors for a dbstruct ;; -;; get and set main db -(define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) -(define-inline (dbr:dbstruct-set-main! vec db)(vector-set! vec 0 db)) -;; get the runs hash -(define-inline (dbr:dbstruct-get-dbhash vec) (vector-ref vec 1)) -;; the string db -(define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 2)) -(define-inline (dbr:dbstruct-set-strdb! vec db)(vector-set! vec 2 db)) -;; path -(define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 3)) -(define-inline (dbr:dbstruct-set-path! vec path)(vector-set! vec 3)) -;; local -(define-inline (dbr:dbstruct-get-local vec) (vector-ref vec 4)) -(define-inline (dbr:dbstruct-set-local! vec val)(vector-set! vec 4 val)) - -;; get a rundb vector, create it if not already existing -(define (dbr:dbstruct-get-rundb-rec vec run-id) - (let* ((dbhash (dbr:dbstruct-get-dbhash vec)) ;; get the runs hash - (runvec (hash-table-ref/default dbhash run-id #f))) ;; get the vector for run-id - (if (vector? runvec) - runvec ;; rundb inmemdb last-mod last-read last-sync in-use refdb - (let ((nvec (vector #f #f -1 -1 -1 #f #f))) - (hash-table-set! dbhash run-id nvec) - nvec)))) - -;; [ rundb inmemdb last-mod last-read last-sync ] -(define-inline (dbr:dbstruct-field-name->num field-name) - (case field-name - ((rundb) 0) ;; the on-disk db - ((inmem) 1) ;; the in-memory db - ((mtime) 2) ;; last modification time - ((rtime) 3) ;; last read time - ((stime) 4) ;; last sync time - ((inuse) 5) ;; is the db currently in use, #t yes, #f no. - ((refdb) 6) ;; the db used for reference (can be on disk or inmem) - (else -1))) - -;; get/set rundb fields -(define (dbr:dbstruct-get-runvec-val vec run-id field-name) - (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id)) - (fieldnum (dbr:dbstruct-field-name->num field-name))) - ;; (vector-set! runvec (dbr:dbstruct-field-name->num 'inuse) #t) - (vector-ref runvec fieldnum))) - -(define (dbr:dbstruct-set-runvec-val! vec run-id field-name val) - (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) - (vector-set! runvec (dbr:dbstruct-field-name->num field-name) val))) - -;; get/set inmemdb -(define (dbr:dbstruct-get-inmemdb vec run-id) - (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) - (vector-ref runvec 1))) - -(define (dbr:dbstruct-set-inmemdb! vec run-id inmemdb) - (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) - (vector-set! runvec 1 inmemdb))) + +(define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) +(define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 1)) +(define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 2)) +(define-inline (dbr:dbstruct-get-local vec) (vector-ref vec 3)) +(define-inline (dbr:dbstruct-get-rundb vec) (vector-ref vec 4)) +(define-inline (dbr:dbstruct-get-inmem vec) (vector-ref vec 5)) +(define-inline (dbr:dbstruct-get-mtime vec) (vector-ref vec 6)) +(define-inline (dbr:dbstruct-get-rtime vec) (vector-ref vec 7)) +(define-inline (dbr:dbstruct-get-stime vec) (vector-ref vec 8)) +(define-inline (dbr:dbstruct-get-inuse vec) (vector-ref vec 9)) +(define-inline (dbr:dbstruct-get-refdb vec) (vector-ref vec 10)) + +(define-inline (dbr:dbstruct-set-main! vec val)(vector-set! vec 0 val)) +(define-inline (dbr:dbstruct-set-strdb! vec val)(vector-set! vec 1 val)) +(define-inline (dbr:dbstruct-set-path! vec val)(vector-set! vec 2 val)) +(define-inline (dbr:dbstruct-set-local! vec val)(vector-set! vec 3 val)) +(define-inline (dbr:dbstruct-set-rundb! vec val)(vector-set! vec 4 val)) +(define-inline (dbr:dbstruct-set-inmem! vec val)(vector-set! vec 5 val)) +(define-inline (dbr:dbstruct-set-mtime! vec val)(vector-set! vec 6 val)) +(define-inline (dbr:dbstruct-set-rtime! vec val)(vector-set! vec 7 val)) +(define-inline (dbr:dbstruct-set-stime! vec val)(vector-set! vec 8 val)) +(define-inline (dbr:dbstruct-set-inuse! vec val)(vector-set! vec 9 val)) +(define-inline (dbr:dbstruct-set-refdb! vec val)(vector-set! vec 10 val)) + +;; constructor for dbstruct +;; +(define (make-dbr:dbstruct #!key (path #f)(local #f)) + (let ((v (make-vector 11 #f))) + (dbr:dbstruct-set-path! v path) + (dbr:dbstruct-set-local! v local) + v)) + +;; ;; get and set main db +;; (define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) +;; (define-inline (dbr:dbstruct-set-main! vec db)(vector-set! vec 0 db)) +;; ;; get the runs hash +;; (define-inline (dbr:dbstruct-get-dbhash vec) (vector-ref vec 1)) +;; ;; the string db +;; (define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 2)) +;; (define-inline (dbr:dbstruct-set-strdb! vec db)(vector-set! vec 2 db)) +;; ;; path +;; (define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 3)) +;; (define-inline (dbr:dbstruct-set-path! vec path)(vector-set! vec 3)) +;; ;; local +;; (define-inline (dbr:dbstruct-get-local vec) (vector-ref vec 4)) +;; (define-inline (dbr:dbstruct-set-local! vec val)(vector-set! vec 4 val)) +;; +;; ;; get a rundb vector, create it if not already existing +;; (define (dbr:dbstruct-get-rundb-rec vec run-id) +;; (let* ((dbhash (dbr:dbstruct-get-dbhash vec)) ;; get the runs hash +;; (runvec (hash-table-ref/default dbhash run-id #f))) ;; get the vector for run-id +;; (if (vector? runvec) +;; runvec ;; rundb inmemdb last-mod last-read last-sync in-use refdb +;; (let ((nvec (vector #f #f -1 -1 -1 #f #f))) +;; (hash-table-set! dbhash run-id nvec) +;; nvec)))) +;; +;; ;; [ rundb inmemdb last-mod last-read last-sync ] +;; (define-inline (dbr:dbstruct-field-name->num field-name) +;; (case field-name +;; ((rundb) 0) ;; the on-disk db +;; ((inmem) 1) ;; the in-memory db +;; ((mtime) 2) ;; last modification time +;; ((rtime) 3) ;; last read time +;; ((stime) 4) ;; last sync time +;; ((inuse) 5) ;; is the db currently in use, #t yes, #f no. +;; ((refdb) 6) ;; the db used for reference (can be on disk or inmem) +;; (else -1))) +;; +;; ;; get/set rundb fields +;; (define (dbr:dbstruct-get-runvec-val vec run-id field-name) +;; (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id)) +;; (fieldnum (dbr:dbstruct-field-name->num field-name))) +;; ;; (vector-set! runvec (dbr:dbstruct-field-name->num 'inuse) #t) +;; (vector-ref runvec fieldnum))) +;; +;; (define (dbr:dbstruct-set-runvec-val! vec run-id field-name val) +;; (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) +;; (vector-set! runvec (dbr:dbstruct-field-name->num field-name) val))) +;; +;; ;; get/set inmemdb +;; (define (dbr:dbstruct-get-inmemdb vec run-id) +;; (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) +;; (vector-ref runvec 1))) +;; +;; (define (dbr:dbstruct-set-inmemdb! vec run-id inmemdb) +;; (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id))) +;; (vector-set! runvec 1 inmemdb))) (define (make-db:test)(make-vector 20)) (define-inline (db:test-get-id vec) (vector-ref vec 0)) (define-inline (db:test-get-run_id vec) (vector-ref vec 1)) Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -157,11 +157,11 @@ (debug:print 0 "WARNING: failed to start on portnum: " portnum ", trying next port") (thread-sleep! 0.1) (http-transport:try-start-server ipaddrstr (+ portnum 1) server-id)) (print "ERROR: Tried and tried but could not start the server"))) ;; any error in following steps will result in a retry - (set! *runremote* (list ipaddrstr portnum)) + (set! *server-info* (list ipaddrstr portnum)) (open-run-close tasks:server-set-interface-port tasks:open-db server-id ipaddrstr portnum) (debug:print 1 "INFO: Trying to start server on " ipaddrstr ":" portnum) @@ -399,11 +399,11 @@ (let* ((server-info (let loop ((start-time (current-seconds)) (changed #t) (last-sdat "not this")) (let ((sdat #f)) (mutex-lock! *heartbeat-mutex*) - (set! sdat *runremote*) + (set! sdat *server-info*) (mutex-unlock! *heartbeat-mutex*) (if (and sdat (not changed) (> (- (current-seconds) start-time) 2)) sdat @@ -442,11 +442,11 @@ (if (< count 1) ;; 3x3 = 9 secs aprox (loop (+ count 1))) ;; Check that iface and port have not changed (can happen if server port collides) (mutex-lock! *heartbeat-mutex*) - (set! sdat *runremote*) + (set! sdat *server-info*) (mutex-unlock! *heartbeat-mutex*) (if (or (not (equal? sdat (list iface port))) (not server-id)) (begin