Changes In Branch v1.64-external-sync Excluding Merge-Ins
This is equivalent to a diff from 85fa0e2f14 to 22e44afa46
|
2017-07-05
| ||
| 12:33 | merged v1.64-external-sync to address syncing problems on high server load check-in: 1907a04c96 user: bjbarcla tags: v1.64 | |
| 12:32 | added speculative fix to send-receive to address issues seen by Eric Brown in ticket 220402979 check-in: 6890b4c9d0 user: bjbarcla tags: v1.64 | |
|
2017-07-04
| ||
| 22:59 | Merged from v1.64 check-in: c269abcad7 user: matt tags: v1.65 | |
|
2017-06-30
| ||
| 09:37 | Run sync-to-megatest.db instead of in process sync when sync time exceeds 300 ms. Closed-Leaf check-in: 22e44afa46 user: mrwellan tags: v1.64-external-sync | |
|
2017-06-29
| ||
| 23:17 | queued coalesced queries (trying again) check-in: 0a1b205bcf user: matt tags: v1.64-coalesced-queries | |
| 17:50 | Partial implementation of -keep-records, still some work to do... check-in: c10b954bde user: mrwellan tags: v1.64-keep-records | |
|
2017-06-28
| ||
| 14:01 | Added missing schema patch for test_rundat check-in: 85fa0e2f14 user: mrwellan tags: v1.64 | |
|
2017-06-27
| ||
| 22:23 | Switched back to doing cleaning out of old records in -cleanup-db and added cleanup of test_rundat and test_steps tables. check-in: 299fe5e984 user: matt tags: v1.64 | |
Modified megatest.scm from [d534113939] to [d13232bf26].
| ︙ | ︙ | |||
2157 2158 2159 2160 2161 2162 2163 |
'adj-testids
'old2new
;; 'new2old
)
(set! *didsomething* #t)))
(if (args:get-arg "-sync-to-megatest.db")
| < | | | < > | 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 |
'adj-testids
'old2new
;; 'new2old
)
(set! *didsomething* #t)))
(if (args:get-arg "-sync-to-megatest.db")
(let ((res (db:multi-db-sync
(db:setup #f)
'new2old)))
(print "Synced " res " records to megatest.db")
(set! *didsomething* #t)))
(if (args:get-arg "-sync-to")
(let ((toppath (launch:setup)))
(tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
(set! *didsomething* #t)))
|
| ︙ | ︙ |
Modified server.scm from [afd86af346] to [8046a654d7].
| ︙ | ︙ | |||
455 456 457 458 459 460 461 462 463 464 465 466 467 468 |
;;
(define (server:writable-watchdog dbstruct)
(thread-sleep! 0.05) ;; delay for startup
(let ((legacy-sync (common:run-sync?))
(debug-mode (debug:debug-mode 1))
(last-time (current-seconds))
(no-sync-db (db:open-no-sync-db))
(this-wd-num (begin (mutex-lock! *wdnum*mutex) (let ((x *wdnum*)) (set! *wdnum* (add1 *wdnum*)) (mutex-unlock! *wdnum*mutex) x))))
(set! *no-sync-db* no-sync-db) ;; make the no sync db available to api calls
(debug:print-info 2 *default-log-port* "Periodic sync thread started.")
(debug:print-info 3 *default-log-port* "watchdog starting. legacy-sync is " legacy-sync" pid="(current-process-id)" this-wd-num="this-wd-num)
(if (and legacy-sync (not *time-to-exit*))
(let* (;;(dbstruct (db:setup))
(mtdb (dbr:dbstruct-mtdb dbstruct))
| > | 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 |
;;
(define (server:writable-watchdog dbstruct)
(thread-sleep! 0.05) ;; delay for startup
(let ((legacy-sync (common:run-sync?))
(debug-mode (debug:debug-mode 1))
(last-time (current-seconds))
(no-sync-db (db:open-no-sync-db))
(sync-duration 0) ;; run time of the sync in milliseconds
(this-wd-num (begin (mutex-lock! *wdnum*mutex) (let ((x *wdnum*)) (set! *wdnum* (add1 *wdnum*)) (mutex-unlock! *wdnum*mutex) x))))
(set! *no-sync-db* no-sync-db) ;; make the no sync db available to api calls
(debug:print-info 2 *default-log-port* "Periodic sync thread started.")
(debug:print-info 3 *default-log-port* "watchdog starting. legacy-sync is " legacy-sync" pid="(current-process-id)" this-wd-num="this-wd-num)
(if (and legacy-sync (not *time-to-exit*))
(let* (;;(dbstruct (db:setup))
(mtdb (dbr:dbstruct-mtdb dbstruct))
|
| ︙ | ︙ | |||
484 485 486 487 488 489 490 |
(not recently-synced))))
(debug:print-info 13 *default-log-port* "WD writable-watchdog top of loop. need-sync="need-sync" sync-in-progress="sync-in-progress" should-sync="should-sync" start-time="start-time" mt-mod-time="mt-mod-time" recently-synced="recently-synced" will-sync="will-sync)
;; (if recently-synced (debug:print-info 0 *default-log-port* "Skipping sync due to recently-synced flag=" recently-synced))
;; (debug:print-info 0 *default-log-port* "need-sync: " need-sync " sync-in-progress: " sync-in-progress " should-sync: " should-sync " will-sync: " will-sync)
(if will-sync (set! *db-sync-in-progress* #t))
(mutex-unlock! *db-multi-sync-mutex*)
(if will-sync
| > > | > | | | | | | | > > > > > > > > > > > > > > > > > > > > > > | 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 |
(not recently-synced))))
(debug:print-info 13 *default-log-port* "WD writable-watchdog top of loop. need-sync="need-sync" sync-in-progress="sync-in-progress" should-sync="should-sync" start-time="start-time" mt-mod-time="mt-mod-time" recently-synced="recently-synced" will-sync="will-sync)
;; (if recently-synced (debug:print-info 0 *default-log-port* "Skipping sync due to recently-synced flag=" recently-synced))
;; (debug:print-info 0 *default-log-port* "need-sync: " need-sync " sync-in-progress: " sync-in-progress " should-sync: " should-sync " will-sync: " will-sync)
(if will-sync (set! *db-sync-in-progress* #t))
(mutex-unlock! *db-multi-sync-mutex*)
(if will-sync
(let ((sync-start (current-milliseconds)))
(if (< sync-duration 300)
(let ((res (db:sync-to-megatest.db dbstruct no-sync-db: no-sync-db))) ;; did we sync any data? If so need to set the db touched flag to keep the server alive
(set! sync-duration (- (current-milliseconds) sync-start))
(if (> res 0) ;; some records were transferred, keep the db alive
(begin
(mutex-lock! *heartbeat-mutex*)
(set! *db-last-access* (current-seconds))
(mutex-unlock! *heartbeat-mutex*)
(debug:print-info 0 *default-log-port* "sync called, " res " records transferred."))
(debug:print-info 2 *default-log-port* "sync called but zero records transferred")))
;; TODO: factor this next routine out into a function
(with-input-from-pipe ;; this should not block other threads but need to verify this
"megatest -sync-to-megatest.db"
(lambda ()
(let loop ((inl (read-line))
(res #f))
(if (eof-object? inl)
(begin
(set! sync-duration (- (current-milliseconds) sync-start))
(cond
((not res)
(debug:print 0 *default-log-port* "ERROR: sync from /tmp db to megatest.db appears to have failed. Recommended that you stop your runs and run \"megatest -cleanup-db\""))
((> res 0)
(mutex-lock! *heartbeat-mutex*)
(set! *db-last-access* (current-seconds))
(mutex-unlock! *heartbeat-mutex*))))
(let ((num-synced (let ((matches (string-match "^Synced (\\d+).*$" inl)))
(if matches
(string->number (cadr matches))
#f))))
(loop (read-line)
(or num-synced res))))))))))
(if will-sync
(begin
(mutex-lock! *db-multi-sync-mutex*)
(set! *db-sync-in-progress* #f)
(set! *db-last-sync* start-time)
(mutex-unlock! *db-multi-sync-mutex*)))
(if (and debug-mode
|
| ︙ | ︙ |