Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -70,11 +70,11 @@ (run-id (cadr params)) (realparams (cddr params))) (db:with-db dbstruct run-id #t ;; these are all for modifying the db (lambda (db) (db:general-call db stmtname realparams))))) - ((sync-inmem->db) (db:sync-touched dbstruct)) + ((sync-inmem->db) (db:sync-touched dbstruct force-sync: #t)) ((kill-server) (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*) ;; (db:sync-to *inmemdb* *db*) (let ((hostname (car *runremote*)) (port (cadr *runremote*)) (pid (if (null? params) #f (car params))) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -64,16 +64,18 @@ ;; mod-read: ;; 'mod modified data ;; 'read read data ;; (define (db:done-with dbstruct run-id mod-read) - (mutex-lock! *rundb-mutex*) - (if (eq? mod-read 'mod) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'mtime (current-milliseconds)) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'rtime (current-milliseconds))) - (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inuse #f) - (mutex-unlock! *rundb-mutex*)) + (if (not (sqlite3:database? dbstruct)) + (begin + (mutex-lock! *rundb-mutex*) + (if (eq? mod-read 'mod) + (dbr:dbstruct-set-runvec-val! dbstruct run-id 'mtime (current-milliseconds)) + (dbr:dbstruct-set-runvec-val! dbstruct run-id 'rtime (current-milliseconds))) + (dbr:dbstruct-set-runvec-val! dbstruct run-id 'inuse #f) + (mutex-unlock! *rundb-mutex*)))) ;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;") ;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no ;; (define (db:with-db dbstruct run-id r/w proc . params) @@ -202,27 +204,29 @@ (db:initialize-main-db db) (db:initialize-run-id-db db))) db)) ;; sync all touched runs to disk -(define (db:sync-touched dbstruct) +(define (db:sync-touched dbstruct #!key (force-sync #f)) (let ((tot-synced 0)) (for-each (lambda (runvec) (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime))) (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime))) (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb))) (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))) - (if (> mtime stime) - (let ((num-sunced (db:sync-tables db:sync-tests-only inmem rundb))) + (if (or (> mtime stime) force-sync) + (let ((num-synced (db:sync-tables db:sync-tests-only inmem rundb))) (set! tot-synced (+ tot-synced num-synced)) - (vector-set! runvec (dbr:dbstruct-field-name->run 'stime (current-milliseconds))))))) - (hash-table-values (vector-ref dbstruct 1))))) + (vector-set! runvec (dbr:dbstruct-field-name->num 'stime) (current-milliseconds)))))) + (hash-table-values (vector-ref dbstruct 1))) + tot-synced)) ;; close all opened run-id dbs (define (db:close-all dbstruct) ;; finalize main.db + (db:sync-touched dbstruct force-sync: #t) (sqlite3:finalize! (db:get-db dbstruct #f)) (for-each (lambda (runvec) (let ((rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))) (if (sqlite3:database? rundb) @@ -253,25 +257,25 @@ '("testname" #f) '("host" #f) '("cpuload" #f) '("diskfree" #f) '("uname" #f) - '("rundir" #f) - '("shortdir" #f) + '("rundir" #f) + '("shortdir" #f) '("item_path" #f) '("state" #f) '("status" #f) '("attemptnum" #f) - '("final_logf" #f) + '("final_logf" #f) '("logdat" #f) '("run_duration" #f) '("comment" #f) '("event_time" #f) '("fail_count" #f) '("pass_count" #f) '("archived" #f)) - (list "test_steps" + (list "test_steps" '("id" #f) '("test_id" #f) '("stepname" #f) '("state" #f) '("status" #f) @@ -291,11 +295,11 @@ '("status" #f) '("type" #f)))) ;; needs db to get keys, this is for syncing all tables ;; -(define (db:tbls db) +(define (db:sync-main-list db) (let ((keys (db:get-keys db))) (list (list "keys" '("id" #f) '("fieldname" #f) @@ -304,41 +308,10 @@ (append (list "runs" '("id" #f)) (map (lambda (k)(list k #f)) (append keys (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count")))) - (list "tests" - '("id" #f) - '("run_id" #f) - '("testname" #f) - '("host" #f) - '("cpuload" #f) - '("diskfree" #f) - '("uname" #f) - '("rundir" #f) - '("shortdir" #f) - '("item_path" #f) - '("state" #f) - '("status" #f) - '("attemptnum" #f) - '("final_logf" #f) - '("logdat" #f) - '("run_duration" #f) - '("comment" #f) - '("event_time" #f) - '("fail_count" #f) - '("pass_count" #f) - '("archived" #f)) - (list "test_steps" - '("id" #f) - '("test_id" #f) - '("stepname" #f) - '("state" #f) - '("status" #f) - '("event_time" #f) - '("comment" #f) - '("logfile" #f)) (list "test_meta" '("id" #f) '("testname" #f) '("owner" #f) '("description" #f) @@ -428,13 +401,10 @@ (if (> count 0) (debug:print 0 (format #f " ~10a ~5a" tblname count))))) (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b)))))) tot-count)) -(define (db:sync-back) - (db:sync-tables (db:tbls *inmemdb*) *inmemdb* *db*)) ;; (db:sync-to *inmemdb* *db*)) - ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) (debug:print-info 11 "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params) (if (or *db-write-access* (not (member proc *db:all-write-procs*))) @@ -1425,20 +1395,22 @@ (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE run_id=?;") run-id) res)) (define (db:replace-test-records dbstruct run-id testrecs) - (let* ((qmarks (string-intersperse (make-list (length db:test-record-fields) "?") ",")) - (qrystr (conc "INSERT OR REPLACE INTO tests (" db:test-record-qry-selector ") VALUES (" qmarks ");")) - (qry (sqlite3:prepare (db:get-db dbstruct run-id) qrystr))) - (debug:print 8 "INFO: replace-test-records, qrystr=" qrystr) - (for-each - (lambda (rec) - ;; (debug:print 0 "INFO: Inserting values: " (string-intersperse (map conc (vector->list rec)) ", ")) - (apply sqlite3:execute qry (vector->list rec))) - testrecs) - (sqlite3:finalize! qry))) + (db:with-db dbstruct run-id #t + (lambda (db) + (let* ((qmarks (string-intersperse (make-list (length db:test-record-fields) "?") ",")) + (qrystr (conc "INSERT OR REPLACE INTO tests (" db:test-record-qry-selector ") VALUES (" qmarks ");")) + (qry (sqlite3:prepare db qrystr))) + ;; (debug:print 8 "INFO: replace-test-records, qrystr=" qrystr) + (for-each + (lambda (rec) + (debug:print 0 "INFO: Inserting values: " (string-intersperse (map conc (vector->list rec)) ", ")) + (apply sqlite3:execute qry (vector->list rec))) + testrecs) + (sqlite3:finalize! qry))))) ;; Get test data using test_id (define (db:get-test-info-by-id dbstruct run-id test-id) (let ((db (db:get-db dbstruct run-id)) Index: db_records.scm ================================================================== --- db_records.scm +++ db_records.scm @@ -39,12 +39,12 @@ ;; get a rundb vector, create it if not already existing (define (dbr:dbstruct-get-rundb-rec vec run-id) (let* ((dbhash (dbr:dbstruct-get-dbhash vec)) ;; get the runs hash (runvec (hash-table-ref/default dbhash run-id #f))) ;; get the vector for run-id (if (vector? runvec) - runvec - (let ((nvec (vector #f #f -1 -1 -1 #f))) + runvec ;; rundb inmemdb last-mod last-read last-sync in-use + (let ((nvec (vector #f #f -1 -1 -1 #f))) (hash-table-set! dbhash run-id nvec) nvec)))) ;; [ rundb inmemdb last-mod last-read last-sync ] (define-inline (dbr:dbstruct-field-name->num field-name) @@ -52,11 +52,11 @@ ((rundb) 0) ;; the on-disk db ((inmem) 1) ;; the in-memory db ((mtime) 2) ;; last modification time ((rtime) 3) ;; last read time ((stime) 4) ;; last sync time - ((inuse) 5) ;; is the db currently in use + ((inuse) 5) ;; is the db currently in use, #t yes, #f no. (else -1))) ;; get/set rundb fields (define (dbr:dbstruct-get-runvec-val vec run-id field-name) (let ((runvec (dbr:dbstruct-get-rundb-rec vec run-id)) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -1191,10 +1191,12 @@ (if (args:get-arg "-import-megatest.db") (let* ((toppath (setup-for-run)) (dbstruct (if toppath (make-dbr:dbstruct path: toppath) #f)) (mtdb (if toppath (db:open-megatest-db))) (run-ids (if toppath (db:get-run-ids mtdb)))) + ;; sync runs, test_meta etc. + (db:sync-tables (db:sync-main-list mtdb) mtdb (db:get-db dbstruct #f)) (for-each (lambda (run-id) (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id))) (debug:print 0 "INFO: Updating " (length testrecs) " records for run-id=" run-id) (db:replace-test-records dbstruct run-id testrecs)))