︙ | | |
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
|
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
|
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
|
(lambda ()
;; handle-exceptions
;; exn
;; (begin
;; (debug:print 0 "ERROR: Failed to create tables. Look at your [fields] section, should be: fieldname TEXT DEFAULT 'yourdefault'")
;; (exit))
(sqlite3:execute db "CREATE TABLE IF NOT EXISTS keys (id INTEGER PRIMARY KEY, fieldname TEXT, fieldtype TEXT, CONSTRAINT keyconstraint UNIQUE (fieldname));")
(for-each (lambda (key)
(sqlite3:execute db "INSERT OR REPLACE INTO keys (fieldname,fieldtype) VALUES (?,?);" key "TEXT"))
keys)
(for-each
(lambda (key)
(let* ((fieldname #f)
(fieldtype #f))
(sqlite3:for-each-row
(lambda (fn ft)
(set! fieldname fn)
(set! fieldtype ft))
db
"SELECT fieldname,fieldtype FROM keys WHERE fieldname=?" key)
(if (not fieldname)
(sqlite3:execute db "INSERT INTO keys (fieldname,fieldtype) VALUES (?,?);" key "TEXT"))))
keys)
(sqlite3:execute db (conc
"CREATE TABLE IF NOT EXISTS runs (id INTEGER PRIMARY KEY, \n "
fieldstr (if havekeys "," "") "
runname TEXT DEFAULT 'norun',
contour TEXT DEFAULT '',
state TEXT DEFAULT '',
status TEXT DEFAULT '',
|
︙ | | |
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
|
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
|
+
+
+
+
+
+
+
+
+
+
-
+
|
;; (sqlite3:execute db "CREATE VIEW runs_tests AS SELECT * FROM runs INNER JOIN tests ON runs.id=tests.run_id;")
(sqlite3:execute db "CREATE TABLE IF NOT EXISTS extradat (id INTEGER PRIMARY KEY, run_id INTEGER, key TEXT, val TEXT);")
(sqlite3:execute db "CREATE TABLE IF NOT EXISTS metadat (id INTEGER PRIMARY KEY, var TEXT, val TEXT,
CONSTRAINT metadat_constraint UNIQUE (var));")
(sqlite3:execute db "CREATE TABLE IF NOT EXISTS access_log (id INTEGER PRIMARY KEY, user TEXT, accessed TIMESTAMP, args TEXT);")
;; Must do this *after* running patch db !! No more.
;; cannot use db:set-var since it will deadlock, hardwire the code here
(let* ((prev-version #f)
(curr-version (common:version-signature)))
(sqlite3:for-each-row
(lambda (ver)
(set! prev-version ver))
db
"SELECT val FROM metadat WHERE var='MEGATEST_VERSION';")
(if prev-version
(if (not (equal? prev-version curr-version))
(sqlite3:execute db "UPDATE metadat SET val=? WHERE var=?;" curr-version "MEGATEST_VERSION"))
(sqlite3:execute db "INSERT OR REPLACE INTO metadat (var,val) VALUES (?,?);" "MEGATEST_VERSION" (common:version-signature))
(sqlite3:execute db "INSERT OR REPLACE INTO metadat (var,val) VALUES (?,?);" "MEGATEST_VERSION" curr-version) ))
(debug:print-info 11 *default-log-port* "db:initialize END") ;; ))))
;;======================================================================
;; R U N S P E C I F I C D B
;;======================================================================
;; (define (db:initialize-run-id-db db)
|
︙ | | |
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
|
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
|
(sqlite3:with-transaction
db
(lambda ()
(sqlite3:execute db "DELETE FROM test_steps WHERE test_id IN (SELECT id FROM tests WHERE state='DELETED' AND event_time<?);" targtime)
(sqlite3:execute db "DELETE FROM test_data WHERE test_id IN (SELECT id FROM tests WHERE state='DELETED' AND event_time<?);" targtime)
(sqlite3:execute db "DELETE FROM tests WHERE state='DELETED' AND event_time<?;" targtime)))))))
;; set tests with state currstate and status currstatus to newstate and newstatus
;; use currstate = #f and or currstatus = #f to apply to any state or status respectively
;; WARNING: SQL injection risk. NB// See new but not yet used "faster" version below
;;
;; AND NOT (item_path='' AND testname in (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));")))
;; (debug:print 0 *default-log-port* "QRY: " qry)
;; (db:delay-if-busy)
;;
;; NB// This call only operates on toplevel tests. Consider replacing it with more general call
;;
(define (db:set-tests-state-status dbstruct run-id testnames currstate currstatus newstate newstatus)
(let ((test-ids '()))
(for-each
(lambda (testname)
(let ((qry (conc "UPDATE tests SET state=?,status=? WHERE "
(if currstate (conc "state='" currstate "' AND ") "")
(if currstatus (conc "status='" currstatus "' AND ") "")
" run_id=? AND testname LIKE ?;"))
(test-id (db:get-test-id dbstruct run-id testname "")))
(db:with-db
dbstruct
run-id
#t
(lambda (dbdat db)
(sqlite3:execute db qry
(or newstate currstate "NOT_STARTED")
(or newstatus currstate "UNKNOWN")
run-id testname)))
(if test-id
(begin
(set! test-ids (cons test-id test-ids))
(mt:process-triggers dbstruct run-id test-id newstate newstatus)))))
testnames)
test-ids))
;; ;; speed up for common cases with a little logic
;; ;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id
;;
;; NOTE: run-id is not used
;; ;;
(define (db:test-set-state-status dbstruct run-id test-id newstate newstatus newcomment)
(db:with-db
dbstruct
run-id #t
(lambda (dbdat db)
(db:test-set-state-status-db db run-id test-id newstate newstatus newcomment))))
(define (db:test-set-state-status-db db run-id test-id newstate newstatus newcomment)
;; clear cache after this, I think that makes sense
(cond
((and newstate newstatus newcomment)
(sqlite3:execute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus newcomment ;; (sdb:qry 'getid newcomment)
test-id))
((and newstate newstatus)
(sqlite3:execute db "UPDATE tests SET state=?,status=? WHERE id=?;" newstate newstatus test-id))
(else
(if newstate (sqlite3:execute db "UPDATE tests SET state=? WHERE id=?;" newstate test-id))
(if newstatus (sqlite3:execute db "UPDATE tests SET status=? WHERE id=?;" newstatus test-id))
(if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment ;; (sdb:qry 'getid newcomment)
test-id))))
;; (mt:process-triggers dbstruct run-id test-id newstate newstatus)) ;; NOTE: Moved into calling function
(let* ((hash-key (cons run-id test-id)))
(hash-table-delete! *db:get-test-info-by-id-cache* hash-key)
(hash-table-delete! *db:get-test-state-status-by-id-cache* hash-key))
)
;; NEW BEHAVIOR: Count tests running in all runs!
;;
(define (db:get-count-tests-running dbstruct run-id) ;; fastmode)
(let* ((qry ;; (if fastmode
;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND NOT (uname = 'n/a' AND item_path = '') LIMIT 1;"
|
︙ | | |
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
|
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
|
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
|
#f
(lambda (dbdat db)
(sqlite3:first-result
db
"SELECT count(id) FROM tests WHERE state in ('LAUNCHED','NOT_STARTED','REMOTEHOSTSTART','RUNNING','KILLREQ') AND run_id=?;")
run-id)))
(define *db:get-test-id-cache* (make-hash-table))
;; map run-id, testname item-path to test-id
(define (db:get-test-id dbstruct run-id testname item-path)
(let* ((hash-key (list run-id testname item-path))
(cache-result (hash-table-ref/default *db:get-test-id-cache* hash-key #f)))
(if cache-result
(cdr cache-result)
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(db:first-result-default
db
"SELECT id FROM tests WHERE testname=? AND item_path=? AND run_id=?;"
#f ;; the default
testname item-path run-id))))
(let* ((res (db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(db:first-result-default
db
"SELECT id FROM tests WHERE testname=? AND item_path=? AND run_id=?;"
#f ;; the default
testname item-path run-id)))))
(if res (hash-table-set! *db:get-test-id-cache* hash-key (cons (current-seconds) res)))
res))))
;; overload the unused attemptnum field for the process id of the runscript or
;; ezsteps step script in progress
;;
(define (db:test-set-top-process-pid dbstruct run-id test-id pid)
(db:with-db
dbstruct
|
︙ | | |
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
|
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
|
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-
-
-
-
-
+
+
+
+
+
+
|
(let* ((run-ids (db:get-all-run-ids mtdb)))
(for-each
(lambda (run-id)
(let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id)))
(db:prep-megatest.db-adj-test-ids (dbr:dbdat-dbh mtdb) run-id testrecs)))
run-ids)))
(define *db:get-test-info-by-id-cache* (make-hash-table))
;; Get test data using test_id
;;
(define (db:get-test-info-by-id dbstruct run-id test-id)
(let* ((hash-key (cons run-id test-id))
(cache-result (hash-table-ref/default *db:get-test-info-by-id-cache* hash-key #f)))
(if cache-result
(cdr cache-result)
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let ((res #f))
(sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test
(lambda (id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum archived last-update)
;; 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
(set! res (vector id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum archived last-update)))
db
;; (db:get-cache-stmth dbdat db
;; (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id=? AND run_id=?;"))
(conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id=? AND run_id=?;")
test-id run-id)
res))))
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let ((res #f))
(sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test
(lambda (id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum archived last-update)
;; 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
(set! res (vector id run-id testname state status event-time host cpuload diskfree uname rundir-id item-path run_duration final-logf-id comment short-dir-id attemptnum archived last-update)))
db
;; (db:get-cache-stmth dbdat db
;; (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id=? AND run_id=?;"))
(conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id=? AND run_id=?;")
test-id run-id)
(hash-table-set! *db:get-test-info-by-id-cache* hash-key (cons (current-seconds) res))
res))))))
(define *db:get-test-state-status-by-id-cache* (make-hash-table))
;; Get test state, status using test_id
;;
(define (db:get-test-state-status-by-id dbstruct run-id test-id)
(let* ((hash-key (cons run-id test-id))
(cache-result (hash-table-ref/default *db:get-test-state-status-by-id-cache* hash-key #f)))
(if cache-result
(cdr cache-result)
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let ((res (cons #f #f)))
;; (stmth (db:get-cache-stmth dbdat db "SELECT state,status FROM tests WHERE id=?;")))
(sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test
(lambda (state status)
(cons state status))
db
"SELECT state,status FROM tests WHERE id=? AND run_id=?;" ;; stmth try not compiling this one - yes, this fixed the bind issue
test-id run-id)
res))))
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let ((res (cons #f #f)))
;; (stmth (db:get-cache-stmth dbdat db "SELECT state,status FROM tests WHERE id=?;")))
(sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test
(lambda (state status)
(cons state status))
db
"SELECT state,status FROM tests WHERE id=? AND run_id=?;" ;; stmth try not compiling this one - yes, this fixed the bind issue
test-id run-id)
(hash-table-set! *db:get-test-state-status-by-id-cache* hash-key (cons (current-seconds) res))
res))))))
;; Use db:test-get* to access
;; Get test data using test_ids. NB// Only works within a single run!!
;;
(define (db:get-test-info-by-ids dbstruct run-id test-ids)
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let ((res '()))
(sqlite3:for-each-row
(lambda (a . b)
;; 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
(set! res (cons (apply vector a b) res)))
db
(conc "SELECT " db:test-record-qry-selector " FROM tests WHERE id in ("
(string-intersperse (map conc test-ids) ",") ");"))
res))))
;; try every second until tries times proc
;;
(define (db:keep-trying-until-true proc params tries)
(let* ((res (apply proc params)))
(if res
res
(if (> tries 0)
(begin
(thread-sleep! 1)
(db:keep-trying-until-true proc params (- tries 1)))
(begin
;; (debug:print-info 0 *default-log-port* "proc never returned true, params="params)
(print"db:keep-trying-until-true proc never returned true, proc = " proc " params =" params " tries = " tries)
#f)))))
(define (db:get-test-info dbstruct run-id test-name item-path)
(let* ((test-id (db:get-test-id dbstruct run-id test-name item-path)))
(db:get-test-info-by-id dbstruct run-id test-id)))
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(db:get-test-info-db db run-id test-name item-path))))
;; (db:with-db
;; dbstruct
;; run-id
;; #f
;; (lambda (dbdat db)
;; (db:get-test-info-db db run-id test-name item-path))))
(define (db:get-test-info-db db run-id test-name item-path)
(let ((res #f))
(sqlite3:for-each-row
(lambda (a . b)
(set! res (apply vector a b)))
db
|
︙ | | |
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
|
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
|
(begin
(debug:print-error 0 *default-log-port* "reception failed. Received \"" msg "\" but cannot translate it.")
(print-call-chain (current-error-port))
msg))) ;; crude reply for when things go awry
((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize))))
(else msg))) ;; rpc
;; ; This is to be the big daddy call NOPE: Replaced by db:set-state-status-and-roll-up-items
;; ;
;; define (db:test-set-state-status dbstruct run-id test-id state status msg)
;; (let ((dbdat (db:get-subdb dbstruct run-id)))
;; (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
;; (db:general-call dbdat 'set-test-start-time (list test-id)))
;; ;; (if msg
;; ;; (db:general-call dbdat 'state-status-msg (list state status msg test-id))
;; ;; (db:general-call dbdat 'state-status (list state status test-id)))
;; (db:set-state-status-and-roll-up-items dbstruct run-id test-id #f state status msg)
;; ;; process the test_data table
;; (if (and test-id state status (equal? status "AUTO"))
;; (db:test-data-rollup dbstruct run-id test-id status))
;; (mt:process-triggers dbstruct run-id test-id state status)))
;; state is the priority rollup of all states
;; status is the priority rollup of all completed statesfu
;;
;; if test-name is an integer work off that as test-id instead of test-name test-path
;;
(define (db:set-state-status-and-roll-up-items dbstruct run-id test-name item-path state status comment)
;; establish info on incoming test followed by info on top level test
|
︙ | | |
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
|
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
|
-
+
-
+
+
|
(let ((tr-res
(sqlite3:with-transaction
db
(lambda ()
;; NB// Pass the db so it is part fo the transaction
(db:test-set-state-status-db db run-id test-id state status comment) ;; this call sets the item state/status
(if (not (equal? item-path "")) ;; only roll up IF incoming test is an item
(let* ((state-status-counts (db:get-all-state-status-counts-for-test db run-id test-name item-path state status)) ;; item-path is used to exclude current state/status of THIS test
(let* ((state-status-counts (db:get-all-state-status-counts-for-test-db db run-id test-name item-path state status)) ;; item-path is used to exclude current state/status of THIS test
(state-statuses (db:roll-up-rules state-status-counts state status))
(newstate (car state-statuses))
(newstatus (cadr state-statuses)))
(set! new-state-eh newstate)
(set! new-status-eh newstatus)
(debug:print 4 *default-log-port* "BB> tl-test-id="tl-test-id" ; "test-name":"item-path" newstate="newstate" newstatus="newstatus" len(sscs)="(length state-status-counts) " state-status-counts: "
(debug:print 4 *default-log-port* "BB> tl-test-id="tl-test-id" ; "test-name":"item-path
" newstate="newstate" newstatus="newstatus" len(sscs)="(length state-status-counts) " state-status-counts: "
(apply conc
(map (lambda (x)
(conc
(with-output-to-string (lambda () (pp (dbr:counts->alist x)))) " | "))
state-status-counts))); end debug:print
(if tl-test-id
(db:test-set-state-status-db db run-id tl-test-id newstate newstatus #f)) ;; we are still in the transaction - must access the db and not the dbstruct
|
︙ | | |
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
|
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
|
+
+
+
+
+
-
+
-
+
|
(define (db:get-all-state-status-counts-for-run dbstruct run-id)
(db:with-db
dbstruct #f #f
(lambda (dbdat db)
(db:get-all-state-status-counts-for-run-db dbdat db run-id))))
(define (db:get-all-state-status-counts-for-test dbstruct run-id test-name item-path item-state-in item-status-in)
(db:with-db
dbstruct run-id #f
(lambda (dbdat db)
(db:get-all-state-status-counts-for-test-db db run-id test-name item-path item-state-in item-status-in))))
;; BBnote: db:get-all-state-status-counts-for-test returns dbr:counts object aggregating state and status of items of a given test, *not including rollup state/status*
;; BBnote: db:get-all-state-status-counts-for-test returns dbr:counts object aggregating state and status of items of a given test, *not including rollup state/status*
;;
;; NOTE: This is called within a transaction
;;
(define (db:get-all-state-status-counts-for-test db run-id test-name item-path item-state-in item-status-in)
(define (db:get-all-state-status-counts-for-test-db db run-id test-name item-path item-state-in item-status-in)
(let* ((test-info (db:get-test-info-db db run-id test-name item-path))
(item-state (or item-state-in (db:test-get-state test-info)))
(item-status (or item-status-in (db:test-get-status test-info)))
(other-items-count-recs (sqlite3:map-row
(lambda (state status count)
(make-dbr:counts state: state status: status count: count))
db
|
︙ | | |
︙ | | |
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
|
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
+
+
|
(module dbmod
*
(import scheme
chicken
data-structures
extras
files
(prefix sqlite3 sqlite3:)
matchable
posix
typed-records
srfi-1
srfi-18
srfi-69
commonmod
|
︙ | | |
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
-
|
(curr-secs (current-seconds)))
(if (> (- curr-secs last-update) 3)
(begin
(sync-proc last-update)
;; MOVE THIS CALL TO INSIDE THE sync-proc CALL
(dbr:dbstruct-last-update-set! dbstruct curr-secs)
)))
(assert (sqlite3:database? dbh) "FATAL: bad db handle in dbmod:with-db")
(if use-mutex (mutex-lock! *db-with-db-mutex*))
(let* ((res (apply proc dbdat dbh params)))
(if use-mutex (mutex-unlock! *db-with-db-mutex*))
res)))
|
︙ | | |
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
(lambda ()
(let* ((db (sqlite3:open-database dbfullname))
(handler (sqlite3:make-busy-timeout 136000)))
(sqlite3:set-busy-handler! db handler)
(if write-access
(init-proc db))
db))))
;; try every second until tries times proc
;;
(define (db:keep-trying-until-true proc params tries)
(let* ((res (apply proc params)))
(if res
res
(if (> tries 0)
(begin
(thread-sleep! 1)
(db:keep-trying-until-true proc params (- tries 1)))
(begin
;; (debug:print-info 0 *default-log-port* "proc never returned true, params="params)
(print"db:keep-trying-until-true proc never returned true, proc = " proc " params =" params " tries = " tries)
#f)))))
(define *sync-in-progress* #f)
;; Open the inmem db and the on-disk db
;; populate the inmem db with data
;;
;; Updates fields in dbstruct
|
︙ | | |
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
|
-
+
-
-
+
-
-
-
+
+
+
+
-
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-
-
-
-
+
-
+
+
-
+
-
+
-
+
|
(dbexists (file-exists? dbfullname))
(tmpdir (conc "/tmp/"(current-user-name)))
(tmpdb (let* ((fname (conc tmpdir"/" (string-translate areapath "/" ".")"-"(current-process-id)"-"dbfname)))
(if (not (file-exists? tmpdir))(create-directory tmpdir))
;; check if tmpdb already exists, either delete it or
;; add something to the name
fname))
(inmem (dbmod:open-inmem-db init-proc
#;(inmem (dbmod:open-inmem-db init-proc
(if (eq? (dbcache-mode) 'inmem)
#f
tmpdb)
))
(write-access (file-write-access? dbpath))
(db (dbmod:safely-open-db dbfullname init-proc write-access))
(tables (db:sync-all-tables-list keys)))
(if (not (and (sqlite3:database? inmem)
(sqlite3:database? db)))
(if (not (sqlite3:database? db)) ;; db is our master database in the .mtdb dir
(begin
(debug:print 0 *default-log-port* "ERROR: Failed to properly open "dbfname-in", exiting immediately.")
(exit)))
;; (assert (sqlite3:database? inmem) "FATAL: open-dbmoddb: inmem is not a db")
;; (assert (sqlite3:database? db) "FATAL: open-dbmoddb: db is not a db")
(dbr:dbstruct-inmem-set! dbstruct inmem)
;; we sync to tmpdb here so that we use file-copy to get intial database
(dbmod:db-to-db-sync dbfullname tmpdb 0 init-proc keys)
(let* ((inmem (dbmod:open-inmem-db init-proc tmpdb)))
(dbr:dbstruct-inmem-set! dbstruct inmem))
(dbr:dbstruct-ondiskdb-set! dbstruct db)
(dbr:dbstruct-dbfile-set! dbstruct dbfullname)
(dbr:dbstruct-dbfname-set! dbstruct dbfname)
; (dbr:dbstruct-dbfname-set! dbstruct dbfname)
(dbr:dbstruct-sync-proc-set! dbstruct
(lambda (last-update)
(if *sync-in-progress*
(debug:print 3 *default-log-port* "WARNING: overlapping calls to sync to disk")
(thread-start!
(make-thread
(begin
(mutex-lock! *db-with-db-mutex*) ;; this mutex is used when overloaded or during a query that modifies the db
(set! *sync-in-progress* #t)
(dbmod:sync-gasket tables last-update inmem db
dbfullname syncdir)
(mutex-unlock! *db-with-db-mutex*)
(thread-sleep! 0.5) ;; ensure at least 1/2 second down time between sync calls
(set! *sync-in-progress* #f)))))
(lambda ()
(mutex-lock! *db-with-db-mutex*) ;; this mutex is used when overloaded or during a query that modifies the db
(set! *sync-in-progress* #t)
#;(dbmod:sync-gasket tables last-update inmem db
dbfullname syncdir)
(system (conc "megatest -db2db -from "tmpdb" -to "dbfullname))
(mutex-unlock! *db-with-db-mutex*)
(thread-sleep! 0.5) ;; ensure at least 1/2 second down time between sync calls
(set! *sync-in-progress* #f)))))))
;; (dbmod:sync-tables tables #f db inmem)
;; (if db
(dbmod:sync-gasket tables #f inmem db dbfullname 'fromdest) ;; ) ;; load into inmem
(dbr:dbstruct-last-update-set! dbstruct (current-seconds)) ;; should this be offset back in time by one second?
dbstruct))
;; (if (eq? syncdir 'todisk) ;; sync to disk normally, sync from in dashboard
;; (dbmod:sync-tables tables last-update inmem db)
;; (dbmod:sync-tables tables last-update db inmem))))
;; direction: 'fromdest 'todest
;;
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction)
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction keys)
(assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db")
(assert (sqlite3:database? inmem) "FATAL: sync-gasket: dbh is not a db")
(assert (sqlite3:database? dbh) "FATAL: sync-gasket: dbh is not a db")
(debug:print-info 0 *default-log-port* "Db sync using "(dbfile:sync-method)" method")
(case (dbfile:sync-method)
((none) #f)
((attach)
(dbmod:attach-sync tables inmem dbfname direction))
((newsync)
((newsync) ;; DON'T USE THIS ONE. IT IS BORKED
(dbmod:new-sync tables inmem dbh dbfname direction))
(else
(case direction
((todisk)
(dbmod:sync-tables tables last-update inmem dbh)
(dbmod:sync-tables tables last-update keys inmem dbh)
)
(else
(dbmod:sync-tables tables last-update dbh inmem))))))
(dbmod:sync-tables tables last-update keys dbh inmem))))))
(define (dbmod:close-db dbstruct)
;; do final sync to disk file
;; (do-sync ...)
(sqlite3:finalize! (dbr:dbstruct-ondiskdb dbstruct)))
;;======================================================================
|
︙ | | |
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
|
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
|
-
+
+
+
+
+
+
+
+
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
+
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
;;
;; if last-update specified ("field-name" . time-in-seconds)
;; then sync only records where field-name >= time-in-seconds
;; IFF field-name exists
;;
;; Use (db:sync-all-tables-list keys) to get the tbls input
;;
(define (dbmod:sync-tables tbls last-update fromdb todb)
(define (dbmod:sync-tables tbls last-update keys fromdb todb)
(debug:print-info 0 *default-log-port* "dbmod:sync-tables called, from: "fromdb", to: "todb)
(assert (sqlite3:database? fromdb) "FATAL: dbmod:sync-tables called with fromdb not a database" fromdb)
(assert (sqlite3:database? todb) "FATAL: dbmod:sync-tables called with fromdb not a database" todb)
(let ((specials `(("keys" "fieldname")
("metadat" "var")
,(cons "runs" (cons "runname" keys))
("tests" "run_id" "testname" "item_path")
("test_meta" "testname")
("test_steps" "test_id" "stepname" "state")
("test_data" "test_id" "category" "variable")))
(let ((stmts (make-hash-table)) ;; table-field => stmt
(stmts (make-hash-table)) ;; table-field => stmt
(all-stmts '()) ;; ( ( stmt1 value1 ) ( stml2 value2 ))
(numrecs (make-hash-table))
(start-time (current-milliseconds))
(tot-count 0))
(for-each ;; table
(lambda (tabledat)
(let* ((tablename (car tabledat))
(fields (cdr tabledat))
(has-last-update (member "last_update" fields))
(use-last-update (dbmod:calc-use-last-update has-last-update fields last-update))
(last-update-value (if use-last-update ;; no need to check for has-last-update - it is already accounted for
(if (number? last-update)
last-update
(cdr last-update))
#f))
(last-update-field (if use-last-update
(if (number? last-update)
"last_update"
(car last-update))
#f))
(num-fields (length fields))
(field->num (make-hash-table))
(num->field (apply vector (map car fields))) ;; BBHERE
(full-sel (conc "SELECT " (string-intersperse (map car fields) ",")
" FROM " tablename (if use-last-update ;; apply last-update criteria
(conc " WHERE " last-update-field " >= " last-update-value)
"")
";"))
(full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) "
" VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );"))
(fromdat '())
(fromdats '())
(totrecords 0)
(batch-len 100) ;; (string->number (or (configf:lookup *configdat* "sync" "batchsize") "100")))
(todat (make-hash-table))
(count 0)
(let* ((count (match tabledat
(field-names (map car fields)))
;; set up the field->num table
(for-each
(lambda (field)
(hash-table-set! field->num field count)
(set! count (+ count 1)))
fields)
((tablename . fields)
(debug:print-info 0 *default-log-port* "Syncing table "tablename)
;; read the source table
;; store a list of all rows in the table in fromdat, up to batch-len.
;; Then add fromdat to the fromdats list, clear fromdat and repeat.
(sqlite3:for-each-row
(lambda (a . b)
(set! fromdat (cons (apply vector a b) fromdat))
(if (> (length fromdat) batch-len)
(begin
(set! fromdats (cons fromdat fromdats))
(set! fromdat '())
(set! totrecords (+ totrecords 1)))))
fromdb
full-sel)
(dbmod:sync-table tablename fields fromdb todb specials))
;; Count less than batch-len as a record
(if (> (length fromdat) 0)
(set! totrecords (+ totrecords 1)))
(else
;; tack on remaining records in fromdat
(if (not (null? fromdat))
(set! fromdats (cons fromdat fromdats)))
(debug:print-warn 0 *default-log-port* "Bad tabledat entry: "tabledat)
(sqlite3:for-each-row
(lambda (a . b)
(hash-table-set! todat a (apply vector a b)))
todb
full-sel)
;; first pass implementation, just insert all changed rows
(let* ((db todb)
(drp-trigger (if (member "last_update" field-names)
(db:drop-trigger db tablename)
#f))
0))))
(has-last-update (member "last_update" field-names))
(is-trigger-dropped (if has-last-update
(db:is-trigger-dropped db tablename)
#f))
(stmth (sqlite3:prepare db full-ins))
(changed-rows 0))
(for-each
(lambda (fromdat-lst)
(mutex-lock! *db-transaction-mutex*)
(sqlite3:with-transaction
db
(lambda ()
(for-each ;;
(lambda (fromrow)
(let* ((a (vector-ref fromrow 0))
(curr (hash-table-ref/default todat a #f))
(same #t))
(let loop ((i 0))
(if (or (not curr)
(not (equal? (vector-ref fromrow i)(vector-ref curr i))))
(set! same #f))
(if (and same
(< i (- num-fields 1)))
(loop (+ i 1))))
(if (not same)
(begin
(apply sqlite3:execute stmth (vector->list fromrow))
(hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0)))
(set! changed-rows (+ changed-rows 1))))))
fromdat-lst)))
(mutex-unlock! *db-transaction-mutex*))
fromdats)
(set! tot-count (+ tot-count count))))
(sqlite3:finalize! stmth)
(if (member "last_update" field-names)
(db:create-trigger db tablename)))
))
tbls)
(let* ((runtime (- (current-milliseconds) start-time))
(debug:print-info 0 *default-log-port* "dbmod:sync-tables completed in "(- (current-milliseconds) start-time)"ms")
(should-print (or ;; (debug:debug-mode 12)
(common:low-noise-print 120 "db sync")
(> runtime 500)))) ;; low and high sync times treated as separate.
(for-each
(lambda (dat)
(let ((tblname (car dat))
(count (cdr dat)))
(set! tot-count (+ tot-count count))))
(sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
tot-count))
(define (dbmod:sync-table tablename fields from-db to-db specials)
(let* ((key-fields (alist-ref tablename specials equal?))
(field-names (map car fields))
(has-last-update (member "last_update" field-names))
(fields-sans-lu (filter (lambda (x)
(not (member x '("id" "last_update"))))
field-names))
(get-vals (lambda (db id fields)
(debug:print-info 0 *default-log-port* "get-vals: fields="fields", id="id)
(let* ((qry (conc "SELECT "(string-intersperse fields ",")" FROM "tablename" WHERE id=?;"))
(res #f))
(sqlite3:for-each-row
(lambda tuple
(set! res tuple))
db qry id)
res)))
(clean-up-qry (lambda (from-id)
(debug:print-info 0 *default-log-port* "key-fields="key-fields", from-id="from-id)
(let* ((vals (get-vals from-db from-id key-fields))
(qry (conc "DELETE FROM "tablename" WHERE "(string-intersperse key-fields "=? AND ")"=?;")))
(debug:print-info 0 *default-log-port* "qry: "qry", vals="vals)
(apply sqlite3:execute to-db qry vals))))
(get-ids (lambda (db)
(sqlite3:fold-row (lambda (res id)
(cons id res))
'()
db
(conc "SELECT id FROM "tablename";"))))
(get-val (lambda (db fieldname id)
(let* ((res #f)
(sql (conc "SELECT "fieldname" FROM "tablename" WHERE id=?;")))
(sqlite3:for-each-row
(lambda (val)
(set! res val))
db
sql
id)
;; (debug:print-info 0 *default-log-port* "get-val "db" "fieldname" "id", sql="sql", res="res)
res)))
(get-row (lambda (db id)
(let* ((res #f))
(sqlite3:for-each-row
(lambda tuple
(set! res tuple))
db
(conc "SELECT " (string-intersperse fields-sans-lu ",")
" FROM "tablename" WHERE id=?;")
id)
res)))
(ins-row (lambda (db id row)
(let* ((qry (conc "INSERT INTO "tablename" (id,"
(string-intersperse fields-sans-lu ",")
") VALUES ("id","
(string-intersperse
(make-list (length fields-sans-lu) "?")
",")
");"))
(proc (lambda ()
(apply sqlite3:execute db qry row))))
;; (debug:print-info 0 *default-log-port* "qry="qry)
(handle-exceptions ;; on exception do the cleanup qry then try one more time
exn
(begin
;; (clean-up-qry id)
(proc))
(proc)))))
(num-inserts 0)
(num-updates 0)
)
;; (debug:print-info 0 *default-log-port* "field-names: "field-names", fields-sans-lu: "fields-sans-lu)
;; (sqlite3:with-transaction
;; from-db
;; (lambda ()
(let* ((from-ids (get-ids from-db)))
;; (debug:print-info 0 *default-log-port* "Table "tablename", has "(length from-ids)" records.")
(sqlite3:with-transaction
to-db
(lambda ()
(let* ((to-ids (get-ids to-db)))
;; (debug:print 0 *default-log-port* "to-ids="to-ids)
(for-each ;; from-id
(lambda (from-id)
(if (member from-id to-ids)
(for-each ;; case where record exists, do one by one the fields if different
(lambda (fieldname)
(let* ((from-val (get-val from-db fieldname from-id))
(dest-val (get-val to-db fieldname from-id)))
#;(debug:print 0 *default-log-port*
"fieldname="fieldname
", from-id="from-id
", from-val="from-val
", dest-val="dest-val
)
(if (not (equal? from-val dest-val))
(let* ((qry-proc (lambda ()
(sqlite3:execute to-db (conc "UPDATE "tablename" SET "fieldname"=? WHERE id=?;")
from-val from-id))))
(handle-exceptions ;; try to remove the offending record and re-try once the update
exn
(begin
;; (clean-up-qry from-id)
(qry-proc))
(qry-proc))
(set! num-updates (+ num-updates 1))))))
fields-sans-lu)
(let ((row (get-row from-db from-id))) ;; need to insert the row
(debug:print 0 *default-log-port* "from-id="from-id", to-ids="to-ids", row="row)
(set! num-inserts (+ num-inserts 1))
(ins-row to-db from-id row))))
from-ids)))))
(+ num-inserts num-updates)))
;; (for-each ;; table
;; (lambda (tabledat)
;; (let* ((tablename (car tabledat))
;; (fields (cdr tabledat))
;; (has-last-update (member "last_update" fields))
;; (use-last-update (dbmod:calc-use-last-update has-last-update fields last-update))
;; (last-update-value (if use-last-update ;; no need to check for has-last-update - it is already accounted for
;; (if (number? last-update)
;; last-update
;; (cdr last-update))
;; #f))
;; (last-update-field (if use-last-update
;; (if (number? last-update)
;; "last_update"
;; (car last-update))
;; #f))
;; (num-fields (length fields))
;; (field->num (make-hash-table))
;; (num->field (apply vector (map car fields))) ;; BBHERE
;; (full-sel (conc "SELECT " (string-intersperse (map car fields) ",")
;; " FROM " tablename (if use-last-update ;; apply last-update criteria
;; (conc " WHERE " last-update-field " >= " last-update-value)
;; "")
;; ";"))
;; (full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) "
;; " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );"))
;; (fromdat '())
;; (fromdats '())
;; (totrecords 0)
;; (batch-len 10000000) ;; (string->number (or (configf:lookup *configdat* "sync" "batchsize") "100")))
;; (todat (make-hash-table))
;; (count 0)
;; (field-names (map car fields)))
;;
;; (debug:print-info 0 *default-log-port* "Syncing table "tablename)
;;
;; ;; set up the field->num table
;; (for-each
;; (lambda (field)
;; (hash-table-set! field->num field count)
;; (set! count (+ count 1)))
;; fields)
;;
;; ;; read the source table
;; ;; store a list of all rows in the table in fromdat, up to batch-len.
;; ;; Then add fromdat to the fromdats list, clear fromdat and repeat.
;; (sqlite3:for-each-row
;; (lambda (a . b)
;; (set! fromdat (cons (apply vector a b) fromdat))
;; (if (> (length fromdat) batch-len)
;; (begin
;; (set! fromdats (cons fromdat fromdats))
;; (set! fromdat '())
;; (set! totrecords (+ totrecords 1)))))
;; fromdb
;; full-sel)
;;
;; (debug:print-info 0 *default-log-port* "Have "totrecords" records to update.")
;; ;; Count less than batch-len as a record
;; (if (> (length fromdat) 0)
;; (set! totrecords (+ totrecords 1)))
;;
;; ;; tack on remaining records in fromdat
;; (if (not (null? fromdat))
;; (set! fromdats (cons fromdat fromdats)))
;;
;; (sqlite3:for-each-row
;; (lambda (a . b)
;; (hash-table-set! todat a (apply vector a b)))
;; todb
;; full-sel)
;;
;; ;; first pass implementation, just insert all changed rows
;;
;; (let* ((db todb)
;; (has-last-update (member "last_update" field-names))
;; (drp-trigger (if has-last-update
;; (db:drop-trigger db tablename)
;; #f))
;; (is-trigger-dropped (if has-last-update
;; (db:is-trigger-dropped db tablename)
;; #f))
;; (stmth (sqlite3:prepare db full-ins))
;; (changed-rows 0))
;; (for-each
;; (lambda (fromdat-lst)
;; (mutex-lock! *db-transaction-mutex*)
;; (sqlite3:with-transaction
;; db
;; (lambda ()
;; (for-each ;;
;; (lambda (fromrow)
;; (let* ((a (vector-ref fromrow 0))
;; (curr (hash-table-ref/default todat a #f))
;; (same #t))
;; (let loop ((i 0))
;; (if (or (not curr)
;; (not (equal? (vector-ref fromrow i)(vector-ref curr i))))
;; (set! same #f))
;; (if (and same
;; (< i (- num-fields 1)))
;; (loop (+ i 1))))
;; (if (not same)
;; (begin
;; (apply sqlite3:execute stmth (vector->list fromrow))
;; (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0)))
;; (set! changed-rows (+ changed-rows 1))))))
;; fromdat-lst)))
;; (mutex-unlock! *db-transaction-mutex*))
;; fromdats)
;;
;; (sqlite3:finalize! stmth)
;; (if (member "last_update" field-names)
;; (db:create-trigger db tablename)))
;; ))
;; tbls)
;; (let* ((runtime (- (current-milliseconds) start-time))
;; (should-print (or ;; (debug:debug-mode 12)
;; (common:low-noise-print 120 "db sync")
;; (> runtime 500)))) ;; low and high sync times treated as separate.
;; (for-each
;; (lambda (dat)
;; (let ((tblname (car dat))
;; (count (cdr dat)))
;; (set! tot-count (+ tot-count count))))
;; (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
(define (has-last-update dbh tablename)
(let* ((has-last #f))
(sqlite3:for-each-row
(lambda (name)
(if (equal? name "last_update")
(set! has-last #t)))
dbh
|
︙ | | |
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
|
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
|
-
+
+
|
(mode 'full)
(no-update '("keys")) ;; do
)
(debug:print 0 *default-log-port* "Doing sync "direction" "destdbfile)
(if (not (sqlite3:auto-committing? dbh))
(debug:print 0 *default-log-port* "Skipping sync due to transaction in flight.")
(let* ((table-names (map car tables))
(dest-exists (file-exists? destdbfile)))
(dest-exists (file-exists? destdbfile))
(start-time (current-milliseconds)))
(assert dest-exists "FATAL: sync called with non-existant file, "destdbfile)
;; attach the destdbfile
;; for each table
;; insert into dest.<table> select * from src.<table> where last_update>last_update
;; done
(debug:print 0 *default-log-port* "Attaching "destdbfile" as auxdb")
(sqlite3:execute dbh (conc "ATTACH '"destdbfile"' AS auxdb;"))
|
︙ | | |
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
|
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
|
-
+
-
-
+
+
-
+
+
+
|
;; (debug:print 0 *default-log-port* "stmt8="stmt8)
;; (if (sqlite3:auto-committing? dbh)
;; (begin
(mutex-lock! *db-transaction-mutex*)
(sqlite3:with-transaction
dbh
(lambda ()
(debug:print-info 0 *default-log-port* "Sync from "fromdb table" to "todb table" using "stmt1)
(debug:print-info 0 *default-log-port* "Sync from "fromdb table" to "todb table" using INSERT OR UPDATE")
(sqlite3:execute dbh stmt1) ;; get all new rows
#;(if (member "last_update" fields)
(sqlite3:execute dbh stmt8)) ;; get all updated rows
(if (member "last_update" fields)
(sqlite3:execute dbh stmt8)) ;; get all updated rows
;; (sqlite3:execute dbh stmt5)
;; (sqlite3:execute dbh stmt4) ;; if it worked this would be better for incremental up
;; (sqlite3:execute dbh stmt6)
))
(debug:print 0 *default-log-port* "Synced table "table
" in "(- (current-milliseconds) start-ms)"ms") ;; )
(mutex-unlock! *db-transaction-mutex*)))
;; (debug:print 0 *default-log-port* "Skipping sync of table "table" due to transaction in flight."))))
table-names)
(sqlite3:execute dbh "DETACH auxdb;"))))
(sqlite3:execute dbh "DETACH auxdb;")
(debug:print-info 0 *default-log-port* "Total sync time: "(- (current-milliseconds) start-time)"ms")
-1)))
;; FAILED ATTEMPTS
;; (if (not (has-last-update dbh table))
;; (sqlite3:execute dbh (conc "ALTER TABLE "table" ADD COLUMN last_update INTEGER;")))
;; (if (not (has-last-update dbh (conc "auxdb."table)))
;; (sqlite3:execute dbh (conc "ALTER TABLE auxdb."table" ADD COLUMN last_update INTEGER;")))
|
︙ | | |
747
748
749
750
751
752
753
754
755
|
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
|
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
(begin
(debug:print-info 0 *default-log-port* "Found old test in LAUNCHED state, test-id=" test-id
" 1 day since event_time marked")
(set! oldlaunched (cons (list test-id run-dir uname testname item-path run-id) oldlaunched)))))
stmth3
run-id))))
(list incompleted oldlaunched toplevels)))
(define (db:set-state-status-by-state-status dbstruct run-id testname currstate currstatus newstate newstatus)
)
;; clear caches needed
(let* ((qry (conc "UPDATE tests SET state=?,status=? WHERE "
(if currstate (conc "state='" currstate "' AND ") "")
(if currstatus (conc "status='" currstatus "' AND ") "")
" run_id=? AND testname LIKE ?;")))
(db:with-db
dbstruct
run-id
#t
(lambda (dbdat db)
(sqlite3:execute db qry
(or newstate currstate "NOT_STARTED")
(or newstatus currstate "UNKNOWN")
run-id testname)))))
;;======================================================================
;; db to db sync
;;======================================================================
(define (dbmod:db-to-db-sync src-db dest-db last-update init-proc keys)
(if (and (file-exists? src-db) ;; can't proceed without a source
(file-read-access? src-db))
(let* ((have-dest (file-exists? dest-db))
(dest-file-wr (and have-dest
(file-write-access? dest-db))) ;; exists and writable
(dest-dir (or (pathname-directory dest-db)
"."))
(dest-dir-wr (and (file-exists? dest-dir)
(file-write-access? dest-dir)))
(d-wr (or (and have-dest
dest-file-wr)
dest-dir-wr))
(copied (if (and (not have-dest)
dest-dir-wr)
(begin
(file-copy src-db dest-db)
#t)
#f)))
(if copied
(begin
(debug:print-info 0 *default-log-port* "db-to-db-sync done with file-copy")
#t)
(let* ((tables (db:sync-all-tables-list keys))
(sdb (dbmod:safely-open-db src-db init-proc #t))
(ddb (dbmod:safely-open-db dest-db init-proc d-wr))
(res (dbmod:sync-gasket tables last-update sdb ddb dest-db 'todest keys)))
(sqlite3:finalize! sdb)
(sqlite3:finalize! ddb)
res)))
#f))
)
|
︙ | | |
282
283
284
285
286
287
288
289
290
291
292
293
294
295
|
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
|
+
|
-list-run-time : list time requered to complete runs. It supports following switches
-run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
-list-test-time : list time requered to complete each test in a run. It following following arguments
-runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
-syscheck : do some very basic checks; write access and space in tmp, home, runs, links and
is $DISPLAY valid
-list-waivers : dump waivers for specified target, runname, testpatt to stdout
-db2db : sync db to db, use -from, -to for dbs, -period and -timeout for continuous sync
Diff report
-diff-rep : generate diff report (must include -src-target, -src-runname, -target, -runname
and either -diff-email or -diff-html)
-src-target <target>
-src-runname <target>
-diff-email <emails> : comma separated list of email addresses to send diff report
|
︙ | | |
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
|
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
|
-
+
+
|
":state"
"-state"
":status"
"-status"
"-list-runs"
"-testdata-csv"
"-testpatt"
"--modepatt"
;; "--modepatt"
"-modepatt"
"-tagexpr"
"-itempatt"
"-setlog"
"-set-toplog"
"-runstep"
"-logpro"
"-m"
"-rerun"
"-days"
"-rename-run"
"-from"
"-to"
"-dest"
"-source"
"-time-stamp"
;; values and messages
":category"
":variable"
|
︙ | | |
376
377
378
379
380
381
382
383
384
385
386
387
388
389
|
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
|
+
+
|
"-pathmod"
"-env2file"
"-envcap"
"-envdelta"
"-setvars"
"-set-state-status"
"-import-sexpr"
"-period" ;; sync period in seconds
"-timeout" ;; exit sync if timeout in seconds exceeded since last change
;; move runs stuff here
"-remove-keep"
"-set-run-status"
"-age"
;; archive
|
︙ | | |
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
|
452
453
454
455
456
457
458
459
460
461
462
463
464
465
|
-
|
"-clean-cache"
"-no-cache"
"-cache-db"
"-cp-eventtime-to-publishtime"
"-use-db-cache"
"-prepend-contour"
;; misc
"-repl"
"-lock"
"-unlock"
"-list-servers"
"-kill-servers"
"-run-wait" ;; wait on a run to complete (i.e. no RUNNING)
|
︙ | | |
493
494
495
496
497
498
499
500
501
502
503
504
505
506
|
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
|
+
|
"-create-megatest-area"
"-mark-incompletes"
"-convert-to-norm"
"-convert-to-old"
"-import-megatest.db"
"-sync-to-megatest.db"
"-db2db"
"-sync-brute-force"
"-logging"
"-v" ;; verbose 2, more than normal (normal is 1)
"-q" ;; quiet 0, errors/warnings only
"-diff-rep"
|
︙ | | |
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
|
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
(set! *didsomething* #t)))
(if (args:get-arg "-sync-to")
(let ((toppath (launch:setup)))
(tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
(set! *didsomething* #t)))
;; use with -from and -to
;;
(if (args:get-arg "-db2db")
(let* ((duh (launch:setup))
(src-db (args:get-arg "-from"))
(dest-db (args:get-arg "-to"))
(sync-period (args:get-arg "-period")) ;; NOT IMPLEMENTED YET
(sync-timeout (args:get-arg "-timeout")) ;; NOT IMPLEMENTED YET
(lockfile (conc dest-db".lock"))
(keys (db:get-keys #f))
)
(if (and src-db dest-db)
(begin
(debug:print-info 0 *default-log-port* "Attempting to sync data from "src-db" to "dest-db"...")
(let ((res (dbmod:db-to-db-sync src-db dest-db 0 (dbfile:db-init-proc) keys)))
(if res
(debug:print 0 *default-log-port* "Synced " res " records from "src-db" to "dest-db)
(debug:print 0 *default-log-port* "No sync due to permissions or non-existant source db."))))
(debug:print 0 *default-log-port* "Skipping sync, there is a sync in progress."))
(set! *didsomething* #t))
(debug:print 0 *default-log-port* "Usage for -db2db; -to and -from must be specified"))
(if (args:get-arg "-list-test-time")
(let* ((toppath (launch:setup)))
(task:get-test-times)
(set! *didsomething* #t)))
(if (args:get-arg "-list-run-time")
(let* ((toppath (launch:setup)))
|
︙ | | |
︙ | | |
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
|
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
|
+
-
+
+
|
(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) posix-extras directory-utils call-with-environment-variables)
(import (prefix sqlite3 sqlite3:))
(declare (unit mt))
(declare (uses debugprint))
(declare (uses db))
(declare (uses dbmod))
(declare (uses common))
(declare (uses items))
(declare (uses runconfig))
(declare (uses tests))
(declare (uses server))
(declare (uses runs))
(declare (uses rmt))
(declare (uses rmtmod))
(import debugprint
rmtmod)
rmtmod
dbmod)
(include "common_records.scm")
(include "key_records.scm")
(include "db_records.scm")
(include "run_records.scm")
(include "test_records.scm")
|
︙ | | |
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
-
+
-
+
|
(debug:print-info 0 *default-log-port* "TRIGGERED on " trigger ", running command " fullcmd " output at " (get-environment-variable "NBFAKE_LOG"))
(process-run fullcmd)
(if prev-nbfake-log
(setenv "NBFAKE_LOG" prev-nbfake-log)
(unsetenv "NBFAKE_LOG"))
))
(define (mt:process-triggers dbstruct run-id test-id newstate newstatus)
(define (mt:process-triggers run-id test-id newstate newstatus)
(if test-id
(let* ((test-dat (db:get-test-info-by-id dbstruct run-id test-id)))
(let* ((test-dat (rmt:get-test-info-by-id run-id test-id)))
(if test-dat
(let* ((test-rundir (db:test-get-rundir test-dat)) ;; ) ;; )
(test-name (db:test-get-testname test-dat))
(item-path (db:test-get-item-path test-dat))
(duration (db:test-get-run_duration test-dat))
(comment (db:test-get-comment test-dat))
(event-time (db:test-get-event_time test-dat))
|
︙ | | |
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
|
-
-
-
-
-
-
-
-
-
-
|
(define (mt:test-set-state-status-by-id run-id test-id newstate newstatus newcomment)
(if (not (and run-id test-id))
(begin
(debug:print-error 0 *default-log-port* "bad data handed to mt:test-set-state-status-by-id, run-id=" run-id ", test-id=" test-id ", newstate=" newstate)
(print-call-chain (current-error-port))
#f)
(begin
;; cond
;; ((and newstate newstatus newcomment)
;; (rmt:general-call 'state-status-msg run-id newstate newstatus newcomment test-id))
;; ((and newstate newstatus)
;; (rmt:general-call 'state-status run-id newstate newstatus test-id))
;; (else
;; (if newstate (rmt:general-call 'set-test-state run-id newstate test-id))
;; (if newstatus (rmt:general-call 'set-test-status run-id newstatus test-id))
;; (if newcomment (rmt:general-call 'set-test-comment run-id newcomment test-id))))
(rmt:set-state-status-and-roll-up-items run-id test-id #f newstate newstatus newcomment)
;; (mt:process-triggers run-id test-id newstate newstatus)
#t)))
(define (mt:test-set-state-status-by-id-unless-completed run-id test-id newstate newstatus newcomment)
(let* ((test-vec (rmt:get-testinfo-state-status run-id test-id))
(state (vector-ref test-vec 3)))
(if (equal? state "COMPLETED")
|
︙ | | |
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
|
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
|
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
;; (mt:process-triggers run-id test-id new-state new-status)
#t);)
;;(mt:test-set-state-status-by-id run-id test-id new-state new-status new-comment)))
(define (mt:test-set-state-status-by-testname-unless-completed run-id test-name item-path new-state new-status new-comment)
(let ((test-id (rmt:get-test-id run-id test-name item-path)))
(mt:test-set-state-status-by-id-unless-completed run-id test-id new-state new-status new-comment)))
;; state and status are extra hints not usually used in the calculation
;;
(define (rmt:set-state-status-and-roll-up-items run-id test-name item-path state status comment)
(assert (number? run-id) "FATAL: Run id required.")
(rmt:client-side-set-state-status-and-roll-up run-id test-name item-path state status comment)
;; (rmtmod:send-receive 'set-state-status-and-roll-up-items run-id (list run-id test-name item-path state status comment))
)
(define (rmt:client-side-set-state-status-and-roll-up run-id test-name item-path state status comment)
;; establish info on incoming test followed by info on top level test
;; BBnote - for mode itemwait, linkage between upstream test & matching item status is propagated to run queue in db:prereqs-not-met
(let* ((test-id (if (number? test-name)
test-name
(db:keep-trying-until-true
rmt:get-test-id
(list run-id test-name item-path)
10)))
;; (rmt:get-test-id run-id test-name item-path)))
(testdat (rmt:get-test-info-by-id run-id test-id))
;; (test-id (db:test-get-id testdat))
(test-name (if (number? test-name)
(db:test-get-testname testdat)
test-name))
(item-path (db:test-get-item-path testdat))
(tl-test-id (rmt:get-test-id run-id test-name ""))
(tl-testdat (rmt:get-test-info-by-id run-id test-id))
(new-state-eh #f)
(new-status-eh #f))
(if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
(rmt:general-call 'set-test-start-time run-id test-id))
(let* ((res (begin
(rmt:test-set-state-status run-id test-id state status comment) ;; this call sets the item state/status
(if (not (equal? item-path "")) ;; only roll up IF incoming test is an item
(let* ((state-status-counts (rmt:get-all-state-status-counts-for-test run-id test-name item-path state status)) ;; item-path is used to exclude current state/status of THIS test
(state-statuses (db:roll-up-rules state-status-counts state status))
(newstate (car state-statuses))
(newstatus (cadr state-statuses)))
(set! new-state-eh newstate)
(set! new-status-eh newstatus)
(debug:print 4 *default-log-port* "BB> tl-test-id="tl-test-id" ; "test-name":"item-path
" newstate="newstate" newstatus="newstatus" len(sscs)="(length state-status-counts) " state-status-counts: "
(apply conc
(map (lambda (x)
(conc
(with-output-to-string (lambda () (pp (dbr:counts->alist x)))) " | "))
state-status-counts))); end debug:print
(if tl-test-id
(rmt:test-set-state-status run-id tl-test-id newstate newstatus #f)) ;; we are still in the transaction - must access the db and not the dbstruct
)))))
(if (and test-id state status (equal? status "AUTO"))
(rmt:test-data-rollup run-id test-id status))
(if new-state-eh ;; moved from db:test-set-state-status
(mt:process-triggers run-id test-id new-state-eh new-status-eh))
res)))
;; select end_time-now from
;; (select testname,item_path,event_time+run_duration as
;; end_time,strftime('%s','now') as now from tests where state in
;; ('RUNNING','REMOTEHOSTSTART','LAUNCHED'));
;;
;; NOT EASY TO MIGRATE TO db{file,mod}
;;
(define (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period)
(let* ((incompleted '())
(oldlaunched '())
(toplevels '())
;; The default running-deadtime is 720 seconds = 12 minutes.
;; "(running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))" = 200 + (2 * (200 + 30 + 30))
(deadtime-trim (or ovr-deadtime cfg-deadtime))
(server-start-allowance 200)
(server-overloaded-budget 200)
(launch-monitor-off-time (or test-stats-update-period 30))
(launch-monitor-on-time-budget 30)
(launch-monitor-period (+ launch-monitor-off-time launch-monitor-on-time-budget server-overloaded-budget))
(remotehoststart-deadtime-default (+ server-start-allowance server-overloaded-budget 30))
(remotehoststart-deadtime (or deadtime-trim remotehoststart-deadtime-default))
(running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))
(running-deadtime (or deadtime-trim running-deadtime-default))) ;; two minutes (30 seconds between updates, this leaves 3x grace period)
(debug:print-info 4 *default-log-port* "running-deadtime = " running-deadtime)
(debug:print-info 4 *default-log-port* "deadtime-trim = " deadtime-trim)
(let* ((dat (rmt:get-toplevels-and-incompletes run-id running-deadtime remotehoststart-deadtime)))
(set! oldlaunched (list-ref dat 1))
(set! toplevels (list-ref dat 2))
(set! incompleted (list-ref dat 0)))
(debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, "
(length toplevels) " old LAUNCHED toplevel tests and "
(length incompleted) " tests marked RUNNING but apparently dead.")
;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE.
;;
;; (db:delay-if-busy dbdat)
(let* ((min-incompleted-ids (map car incompleted)) ;; do 'em all
(all-ids (append min-incompleted-ids (map car oldlaunched))))
(if (> (length all-ids) 0)
(begin
;; (launch:is-test-alive "localhost" 435)
(debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ")
" as DEAD")
(for-each
(lambda (test-id)
(let* ((tinfo (rmt:get-test-info-by-id run-id test-id))
(run-dir (db:test-get-rundir tinfo))
(host (db:test-get-host tinfo))
(pid (db:test-get-process_id tinfo))
(result (rmt:get-status-from-final-status-file run-dir)))
(if (and (list? result) (> (length result) 1) (equal? "PASS" (cadr result)) (equal? "COMPLETED" (car result)))
(begin
(debug:print 0 *default-log-port* "INFO: test " test-id " actually passed, so marking PASS not DEAD")
(rmt:set-state-status-and-roll-up-items
run-id test-id 'foo "COMPLETED" "PASS"
"Test stopped responding but it has PASSED; marking it PASS in the DB."))
(let ((is-alive (and (not (eq? pid 0)) ;; 0 is default in re-used field "attemptnum" where pid stored.
(commonmod:is-test-alive host pid))))
(if is-alive
(debug:print 0 *default-log-port* "INFO: test " test-id " on host " host
" has a process on pid " pid ", NOT setting to DEAD.")
(begin
(debug:print 0 *default-log-port* "INFO: test " test-id
" final state/status is not COMPLETED/PASS. It is " result)
(rmt:set-state-status-and-roll-up-items
run-id test-id 'foo "COMPLETED" "DEAD"
"Test stopped responding while in RUNNING or REMOTEHOSTSTART; presumed dead.")))))))
;; call end of eud of run detection for posthook - from merge, is it needed?
;; (launch:end-of-run-check run-id)
all-ids)
)))))
(define (mt:lazy-read-test-config test-name)
(let ((tconf (hash-table-ref/default *testconfigs* test-name #f)))
(if tconf
tconf
(let ((test-dirs (tests:get-tests-search-path *configdat*)))
(let loop ((hed (car test-dirs))
(tal (cdr test-dirs)))
|
︙ | | |