| ︙ | | |
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
-
+
-
-
+
|
(if (runs:lownoise "runners-softlock-wait" 360)
(debug:print-info 0 *default-log-port* "Other runners in flight, giving up some time..."))
(thread-sleep! 2))
(begin
(if (runs:lownoise "runners-softlock-nowait" 360)
(debug:print-info 0 *default-log-port* "No runners in flight, updating softlock"))
(let* ((ouf (open-output-file my-lock-file)))
(with-output-to-port ouf
(with-output-to-port ouf (lambda ()(print (current-seconds))))
(lambda ()(print (current-seconds))))
;; Fourth try, do accounting through time
;; Fourth try, do accounting through time....
;;
(define (runs:parallel-runners-mgmt rdat)
(let ((time-to-check (configf:lookup-number *configdat* "runners" "time-to-check" default: 10)) ;; 28
(time-to-wait (configf:lookup-number *configdat* "runners" "time-to-wait" default: 30))
(now-time (current-seconds)))
(if (> (- now-time (runs:dat-last-fuel-check rdat)) time-to-check) ;; time to check
(runs:wait-on-softlock rdat "runners"))))
|
| ︙ | | |
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
|
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
-
-
-
+
+
|
(launch:setup force-reread: #t)
(fatal-loop (+ count 1)))
(begin
(debug:print 0 *default-log-port* "FATAL: *configdat* was inaccessible! This should never happen. Retried " count
" times. Message: " msg)
(debug:print 0 *default-log-port* "Call chain:")
(with-output-to-port *default-log-port*
(lambda ()
(print "*configdat* is >>"*configdat*"<<")
|
| ︙ | | |
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
|
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
|
-
+
-
+
|
;(set! run-count config-reruns))
;; (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
(let ((sighand (lambda (signum)
;; (signal-mask! signum) ;; to mask or not? seems to cause issues in exiting
(set! *time-to-exit* #t)
(print "Received signal " signum ", cleaning up before exit. Please wait...")
(debug:print 0 *default-log-port* "Received signal " signum ", cleaning up before exit. Please wait...")
(let ((th1 (make-thread (lambda ()
;; (let ((tdbdat (tasks:open-db)))
(rmt:tasks-set-state-given-param-key task-key "killed") ;; )
(print "Killed by signal " signum ". Exiting")
(debug:print 0 *default-log-port* "Killed by signal " signum ". Exiting")
(thread-sleep! 3)
(exit))))
(th2 (make-thread (lambda ()
(thread-sleep! 5)
(debug:print 0 *default-log-port* "Done")
(exit 4)))))
(thread-start! th2)
|
| ︙ | | |
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
|
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
|
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
|
(debug:print-info 1 *default-log-port* "Adding \"" (string-intersperse required-tests " ") "\" to the run queue"))
;; NOTE: these are all parent tests, items are not expanded yet.
(debug:print-info 4 *default-log-port* "test-records=" (hash-table->alist test-records))
(let ((reglen (configf:lookup *configdat* "setup" "runqueue")))
(if (> (length (hash-table-keys test-records)) 0)
(let* ((keep-going #t)
(run-queue-retries 5)
#;(th1 (make-thread (lambda ()
(handle-exceptions
exn
(begin
(print-call-chain)
(print " message: " ((condition-property-accessor 'exn 'message) exn)))
(runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests
(any->number reglen) all-tests-registry)))
"runs:run-tests-queue"))
;; (th1 (make-thread (lambda ()
;; (handle-exceptions
;; exn
;; (begin
;; (print-call-chain)
;; (print " message: " ((condition-property-accessor 'exn 'message) exn)))
;; (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests
;; (any->number reglen) all-tests-registry)))
;; "runs:run-tests-queue"))
(th2 (make-thread (lambda () ;; BBQ: why are we visiting ALL runs here?
;; (rmt:find-and-mark-incomplete-all-runs))))) CAN'T INTERRUPT IT ...
(let ((run-ids (rmt:get-all-run-ids)))
(for-each (lambda (run-id)
(if keep-going
(handle-exceptions
exn
|
| ︙ | | |
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
|
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
|
-
+
|
(begin
;; wait for less than max jobs here
(if (runs:dat-wait-for-jobs-function runsdat)
((runs:dat-wait-for-jobs-function runsdat) testdat-rec))
(if (not (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat flags))
(begin
(print "ERROR: Failed to launch the test. Exiting as soon as possible")
(debug:print 0 *default-log-port* "ERROR: Failed to launch the test. Exiting as soon as possible")
(set! *globalexitstatus* 1) ;;
(process-signal (current-process-id) signal/kill))
)
;; wait again here?
))))))
((KILLED)
(debug:print 1 *default-log-port* "NOTE: " full-test-name " is already running or was explictly killed, use -force to launch it.")
|
| ︙ | | |
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
|
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
|
-
+
-
+
|
))
runs)
;; special case - archive get
(if (equal? (args:get-arg "-archive") "get")
(archive:bup-get-data "get" #f #f test-records rp-mutex bup-mutex))
(if (or (equal? (args:get-arg "-archive") "save") (equal? (args:get-arg "-archive") "save-remove"))
(begin
(print "db archive started")
(debug:print 0 *default-log-port* "db archive started")
(archive:megatest-db target runnamepatt)
(print "db archived")))
(debug:print 0 *default-log-port* "db archived")))
)
#t
)
(define (runs:remove-test-directory test mode) ;; remove-data-only)
(let* ((run-dir (db:test-get-rundir test)) ;; run dir is from the link tree
(real-dir (if (common:file-exists? run-dir)
|
| ︙ | | |
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
|
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
|
-
+
|
(lambda (key)
(let* ((idx (cadr key))
(fld (car key))
(val (configf:lookup test-conf "test_meta" fld)))
;; (debug:print 5 *default-log-port* "idx: " idx " fld: " fld " val: " val)
(if (and val (not (equal? (vector-ref currrecord idx) val)))
(begin
(print "Updating " test-name " " fld " to " val)
(debug:print 0 *default-log-port* "Updating " test-name " " fld " to " val)
(rmt:testmeta-update-field test-name fld val)))))
'(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9)("jobgroup" 10)))))
;; find tests with matching tags, tagpatt is a string "tagpatt1,tagpatt2%, ..."
;;
(define (runs:get-tests-matching-tags tagpatt)
(let* ((tagdata (rmt:get-tests-tags))
|
| ︙ | | |