Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -458,10 +458,13 @@ (full-sel (conc "SELECT " (string-intersperse (map car fields) ",") " FROM " tablename ";")) (full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) " " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );")) (fromdat '()) + (fromdats '()) + (totrecords 0) + (batch-len (string->number (or (configf:lookup *configdat* "sync" "batchsize") "10"))) (todat (make-hash-table)) (count 0)) ;; set up the field->num table (for-each @@ -471,15 +474,20 @@ fields) ;; read the source table (sqlite3:for-each-row (lambda (a . b) - (set! fromdat (cons (apply vector a b) fromdat))) + (set! fromdat (cons (apply vector a b) fromdat)) + (if (> (length fromdat) batch-len) + (begin + (set! fromdats (cons fromdat fromdats)) + (set! fromdat '()) + (set! totrecords (+ totrecords 1))))) (db:dbdat-get-db fromdb) full-sel) - (debug:print-info 2 "found " (length fromdat) " records to sync") + (debug:print-info 2 "found " totrecords " records to sync") ;; read the target table (sqlite3:for-each-row (lambda (a . b) (hash-table-set! todat a (apply vector a b))) @@ -490,30 +498,33 @@ (for-each (lambda (targdb) (let* ((db (db:dbdat-get-db targdb)) (stmth (sqlite3:prepare db full-ins))) ;; (db:delay-if-busy targdb) ;; NO WAITING - (sqlite3:with-transaction - db - (lambda () - (for-each ;; - (lambda (fromrow) - (let* ((a (vector-ref fromrow 0)) - (curr (hash-table-ref/default todat a #f)) - (same #t)) - (let loop ((i 0)) - (if (or (not curr) - (not (equal? (vector-ref fromrow i)(vector-ref curr i)))) - (set! same #f)) - (if (and same - (< i (- num-fields 1))) - (loop (+ i 1)))) - (if (not same) - (begin - (apply sqlite3:execute stmth (vector->list fromrow)) - (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0))))))) - fromdat))) + (for-each + (lambda (fromdat-lst) + (sqlite3:with-transaction + db + (lambda () + (for-each ;; + (lambda (fromrow) + (let* ((a (vector-ref fromrow 0)) + (curr (hash-table-ref/default todat a #f)) + (same #t)) + (let loop ((i 0)) + (if (or (not curr) + (not (equal? (vector-ref fromrow i)(vector-ref curr i)))) + (set! same #f)) + (if (and same + (< i (- num-fields 1))) + (loop (+ i 1)))) + (if (not same) + (begin + (apply sqlite3:execute stmth (vector->list fromrow)) + (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0))))))) + fromdat-lst)))) + fromdats) (sqlite3:finalize! stmth))) (append (list todb) slave-dbs)))) tbls) (let* ((runtime (- (current-milliseconds) start-time)) (should-print (common:low-noise-print 30 "db sync" (> runtime 500)))) ;; low and high sync times treated as separate. Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -295,11 +295,11 @@ (let ((start-time (current-seconds)) (servers-started (make-hash-table))) (for-each (lambda (run-id) (mutex-lock! *db-multi-sync-mutex*) - (if (hash-table-ref/default *db-local-sync* run-id 0) + (if (hash-table-ref/default *db-local-sync* run-id #f) ;; (if (> (- start-time last-write) 5) ;; every five seconds (let ((sync-time (- (current-seconds) start-time))) (db:multi-db-sync (list run-id) 'new2old) (if (common:low-noise-print 30 "sync new to old") (begin Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -103,17 +103,18 @@ (rmt:send-receive cmd run-id params attemptnum: (+ attemptnum 1))))) (let ((max-avg-qry (string->number (or (configf:lookup *configdat* "server" "server-query-threshold") "10")))) (debug:print-info 4 "no server and read-only query, bypassing normal channel") ;; (if (rmt:write-frequency-over-limit? cmd run-id)(server:kind-run run-id)) - (let ((curr-max (rmt:get-max-query-average run-id))) - (if (> (cdr curr-max) max-avg-qry) + (let* ((curr-max (rmt:get-max-query-average run-id)) + (curr-max-val (cdr curr-max))) + (if (> curr-max-val max-avg-qry) (if (common:low-noise-print 10 "start server due to max average query too long") - (begin - (debug:print-info 0 "Max average query, " (inexact->exact (round (cdr curr-max))) "ms (" (car curr-max) ") exceeds " max-avg-qry ", try starting server ...") - (server:kind-run run-id) - (debug:print-info 3 "Max average query, " (inexact->exact (round (cdr curr-max))) "ms (" (car curr-max) ") below " max-avg-qry ", not starting server..."))))) + (begin + (debug:print-info 0 "Max average query, " (inexact->exact (round curr-max-val)) "ms (" (car curr-max) ") exceeds " max-avg-qry "ms, try starting server ...") + (server:kind-run run-id)) + (debug:print-info 3 "Max average query, " (inexact->exact (round curr-max-val)) "ms (" (car curr-max) ") below " max-avg-qry "ms, not starting server...")))) (rmt:open-qry-close-locally cmd run-id params))))) (define (rmt:update-db-stats run-id rawcmd params duration) (mutex-lock! *db-stats-mutex*) (handle-exceptions Index: synchash.scm ================================================================== --- synchash.scm +++ synchash.scm @@ -117,11 +117,11 @@ (map make-indexed data)))) ;; add each element keyed by the keynum'th val (else ;; (debug:print-info 2 "Non-get runs call") (map make-indexed newdat)))) ;; (debug:print-info 2 "postdat: " postdat) - (if (not indb)(sqlite3:finalize! db)) + ;; (if (not indb)(sqlite3:finalize! db)) (if (not synchash) (begin (set! synchash (make-hash-table)) (hash-table-set! *synchashes* synckey synchash))) (synchash:get-delta postdat synchash))) Index: tests/fdktestqa/testqa/Makefile ================================================================== --- tests/fdktestqa/testqa/Makefile +++ tests/fdktestqa/testqa/Makefile @@ -1,9 +1,10 @@ BINDIR = $(PWD)/../../../bin PATH := $(BINDIR):$(PATH) MEGATEST = $(BINDIR)/megatest DASHBOARD = $(BINDIR)/dashboard +NEWDASHBOARD = $(BINDIR)/newdashboard RUNNAME = a all : $(MEGATEST) -remove-runs -target a/b :runname c -testpatt %/% @@ -24,11 +25,14 @@ $(MEGATEST) -runtests bigrun3 -target a/bigrun3 :runname $(RUNNAME) dashboard : $(DASHBOARD) -rows 20 & +newdashboard : + $(NEWDASHBOARD) & + compile : (cd ../../..;make -j && make install) clean : rm -rf ../simple*/*/* megatest.db db/* ../simple*/.db/* logs/* monitor.db