Overview
| Comment: | Merged in cached writes |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.80-matt-fixme |
| Files: | files | file ages | folders |
| SHA1: |
cdc7397963fdb3b84f71b25b4d231aac |
| User & Date: | matt on 2023-10-13 20:42:13 |
| Other Links: | branch diff | manifest | tags |
Context
|
2023-10-14
| ||
| 20:19 | removed a bit of not-needed junk from rmt.scm Leaf check-in: ffe3df4e65 user: matt tags: v1.80-matt-fixme | |
|
2023-10-13
| ||
| 20:42 | Merged in cached writes check-in: cdc7397963 user: matt tags: v1.80-matt-fixme | |
| 20:41 | Oops. Forgot to clean out the cache after processing the cached writes Leaf check-in: b564f3f99c user: matt tags: v1.80-cached-writes | |
|
2023-10-12
| ||
| 21:04 | Cached writes for steps working correctly check-in: eee1dce5a3 user: matt tags: v1.80-cached-writes | |
|
2023-10-09
| ||
| 19:51 | Merged v1.80 in check-in: 38506ffe03 user: matt tags: v1.80 | |
Changes
Modified api.scm from [5fa313076b] to [13a08c65d1].
| ︙ | ︙ | |||
369 370 371 372 373 374 375 |
((dec-var) (apply db:dec-var dbstruct params))
((del-var) (apply db:del-var dbstruct params))
((add-var) (apply db:add-var dbstruct params))
((insert-run) (apply db:insert-run dbstruct params))
;; STEPS
| > | > | 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 |
((dec-var) (apply db:dec-var dbstruct params))
((del-var) (apply db:del-var dbstruct params))
((add-var) (apply db:add-var dbstruct params))
((insert-run) (apply db:insert-run dbstruct params))
;; STEPS
((teststep-set-status!)
;; (apply db:teststep-set-status! dbstruct params))
(db:add-cached-write dbstruct db:teststep-set-status! run-id params))
((delete-steps-for-test!) (apply db:delete-steps-for-test! dbstruct params))
;; TEST DATA
((test-data-rollup) (apply db:test-data-rollup dbstruct params))
((csv->test-data) (apply db:csv->test-data dbstruct params))
;; MISC
|
| ︙ | ︙ |
Modified db.scm from [a33d322bf7] to [eca878b6ec].
| ︙ | ︙ | |||
4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 |
(begin
(debug:print 0 *default-log-port* "WARNING: path given, " outputfile " is relative, prefixing with current directory")
(conc (current-directory) "/" outputfile)))
results)
;; brutal clean up
(dbfile:add-dbdat dbstruct #f dbdat)
(system "rm -rf tempdir")))
;; (db:extract-ods-file db "outputfile.ods" '(("sysname" "%")("fsname" "%")("datapath" "%")) "%")
;;======================================================================
;; moving watch dogs here due to dependencies
;;======================================================================
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 |
(begin
(debug:print 0 *default-log-port* "WARNING: path given, " outputfile " is relative, prefixing with current directory")
(conc (current-directory) "/" outputfile)))
results)
;; brutal clean up
(dbfile:add-dbdat dbstruct #f dbdat)
(system "rm -rf tempdir")))
;;======================================================================
;; cached writes stuff
;;======================================================================
(define (db:add-cached-write dbstruct proc run-id params)
(debug:print 0 *default-log-port* "Adding cached write for run-id "run-id" params " params)
(mutex-lock! *cached-writes-mutex*)
(let* ((hkey (cons dbstruct run-id))
(cached-writes-queue (hash-table-ref/default *cached-writes-queues* hkey '())))
(hash-table-set! *cached-writes-queues* hkey (cons (list proc params) cached-writes-queue)))
(if (not *cached-writes-flag*)
(begin
(set! *cached-writes-flag* #t)
(thread-start! (make-thread
(lambda ()
(debug:print 0 *default-log-port* "process cached writes thread started.")
(thread-sleep! 1)
(db:process-cached-writes-queue))))))
(mutex-unlock! *cached-writes-mutex*))
(define (db:process-cached-writes-queue)
(mutex-lock! *cached-writes-mutex*)
(hash-table-for-each
*cached-writes-queues*
(lambda (hkey writes-list)
(let* ((dbstruct (car hkey))
(run-id (cdr hkey)))
(debug:print 0 *default-log-port* "Processing "(length writes-list)" cached writes for run "run-id)
(db:with-db
dbstruct
run-id
#t
(lambda (dbdat db)
(sqlite3:with-transaction
db
(lambda ()
(for-each
(lambda (queued-write)
(match queued-write
((proc params)(apply proc dbstruct params))
(else (assert #f "BAD queued-write"))))
writes-list)))
(hash-table-delete! *cached-writes-queues* hkey))))))
(set! *cached-writes-flag* #f)
(mutex-unlock! *cached-writes-mutex*))
;; (db:extract-ods-file db "outputfile.ods" '(("sysname" "%")("fsname" "%")("datapath" "%")) "%")
;;======================================================================
;; moving watch dogs here due to dependencies
;;======================================================================
|
| ︙ | ︙ |
Modified dbfile.scm from [4b315f3788] to [172c69b638].
| ︙ | ︙ | |||
1570 1571 1572 1573 1574 1575 1576 1577 1578 |
(result (or stmth
(let* ((newstmth (sqlite3:prepare db stmt)))
;; (db:hoh-set! stmt-cache db stmt newstmth)
(hash-table-set! stmt-cache stmt newstmth)
newstmth))))
(mutex-unlock! *get-cache-stmth-mutex*)
result))
)
| > > > > > > > > > > | 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 |
(result (or stmth
(let* ((newstmth (sqlite3:prepare db stmt)))
;; (db:hoh-set! stmt-cache db stmt newstmth)
(hash-table-set! stmt-cache stmt newstmth)
newstmth))))
(mutex-unlock! *get-cache-stmth-mutex*)
result))
;;======================================================================
;; cached writes - run list of procs inside transaction
;; NOTE: this only works because we have once database per process
;;======================================================================
(define *cached-writes-mutex* (make-mutex))
(define *cached-writes-flag* #f)
(define *cached-writes-queues* (make-hash-table)) ;; dbstruct->list of writes
)
|
Modified dbmod.scm from [1d31a00395] to [7ac026a501].
| ︙ | ︙ | |||
117 118 119 120 121 122 123 | (debug:print-info 0 *default-log-port* "dbmod:with-db, database is busy, will try "count" more times.") (thread-sleep! 1) (loop (- count 1))) (begin (debug:print-info 0 *default-log-port* "dbmod:with-db, database is busy, giving up.") (exit 1)))) (exn () | | > | > | 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 |
(debug:print-info 0 *default-log-port* "dbmod:with-db, database is busy, will try "count" more times.")
(thread-sleep! 1)
(loop (- count 1)))
(begin
(debug:print-info 0 *default-log-port* "dbmod:with-db, database is busy, giving up.")
(exit 1))))
(exn ()
(dbfile:print-err exn "ERROR: Unknown error with db for run-id "
run-id", message: "
((condition-property-accessor 'exn 'message) exn)
", details: "(condition->list exn))
(exit 2))))))
(if use-mutex (mutex-unlock! *db-with-db-mutex*))
res)))
(define (db:with-db dbstruct run-id w/r proc . params)
(dbmod:with-db dbstruct run-id w/r proc params))
|
| ︙ | ︙ |