| ︙ | | |
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
-
-
+
+
-
+
-
+
|
(keyvallst (keys->vallist keys)) ;; extracts the values from remainder of (argv)
(runname (get-with-default ":runname" #f))
(state (get-with-default ":state" "no"))
(status (get-with-default ":status" "n/a"))
(allvals (append (list runname state status user) keyvallst))
(qryvals (append (list runname) keyvallst))
(key=?str (string-intersperse (map (lambda (k)(conc (key:get-fieldname k) "=?")) keys) " AND ")))
;; (print "keys: " keys " allvals: " allvals " keyvallst: " keyvallst)
(print "NOTE: using key " (string-intersperse keyvallst "/") " for this run")
(debug:print 3 "keys: " keys " allvals: " allvals " keyvallst: " keyvallst)
(debug:print 2 "NOTE: using key " (string-intersperse keyvallst "/") " for this run")
(if (and runname (null? (filter (lambda (x)(not x)) keyvallst))) ;; there must be a better way to "apply and"
(let ((res #f))
(apply sqlite3:execute db (conc "INSERT OR IGNORE INTO runs (runname,state,status,owner,event_time" comma keystr ") VALUES (?,?,?,?,strftime('%s','now')" comma valslots ");")
allvals)
(apply sqlite3:for-each-row
(lambda (id)
(set! res id))
db
(let ((qry (conc "SELECT id FROM runs WHERE (runname=? " andstr key=?str ");")))
;; (print "qry: " qry)
;(debug:print 4 "qry: " qry)
qry)
qryvals)
(sqlite3:execute db "UPDATE runs SET state=?,status=? WHERE id=?;" state status res)
res)
(begin
(print "ERROR: Called without all necessary keys")
(debug:print 0 "ERROR: Called without all necessary keys")
#f))))
;; runs:get-runs-by-patt
;; get runs by list of criteria
;; register a test run with the db
;;
;; Use: (db-get-value-by-header (db:get-header runinfo)(db:get-row runinfo))
|
| ︙ | | |
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
-
+
|
(for-each (lambda (keyval)
(let* ((key (vector-ref keyval 0))
(fulkey (conc ":" key))
(patt (args:get-arg fulkey)))
(if patt
(set! key-patt (conc key-patt " AND " key " like '" patt "'"))
(begin
(print "ERROR: searching for runs with no pattern set for " fulkey)
(debug:print 0 "ERROR: searching for runs with no pattern set for " fulkey)
(exit 6)))))
keys)
(sqlite3:for-each-row
(lambda (a . r)
(set! res (cons (list->vector (cons a r)) res)))
db
(conc "SELECT " keystr " FROM runs WHERE runname like ? " key-patt ";")
|
| ︙ | | |
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
-
+
-
+
-
+
-
+
|
(if s (string-split s) #f))))
(if valid-values
(if (member item valid-values)
item #f)
item)))
(define (teststep-set-status! db run-id test-name teststep-name state-in status-in itemdat comment)
;; (print "run-id: " run-id " test-name: " test-name)
(debug:print 4 "run-id: " run-id " test-name: " test-name)
(let* ((state (check-valid-items "state" state-in))
(status (check-valid-items "status" status-in))
(item-path (item-list->path itemdat))
(testdat (db:get-test-info db run-id test-name item-path)))
;; (print "testdat: " testdat)
(debug:print 5 "testdat: " testdat)
(if (and testdat ;; if the section exists then force specification BUG, I don't like how this works.
(or (not state)(not status)))
(print "WARNING: Invalid " (if status "status" "state")
(debug:print 0 "WARNING: Invalid " (if status "status" "state")
" value \"" (if status status-in state-in) "\", update your validstates section in megatest.config"))
(if testdat
(let ((test-id (test:get-id testdat)))
(sqlite3:execute db
"INSERT OR REPLACE into test_steps (test_id,stepname,state,status,event_time,comment) VALUES(?,?,?,?,strftime('%s','now'),?);"
test-id teststep-name state status (if comment comment "")))
(print "ERROR: Can't update " test-name " for run " run-id " -> no such test in db"))))
(debug:print 0 "ERROR: Can't update " test-name " for run " run-id " -> no such test in db"))))
(define (test-get-kill-request db run-id test-name itemdat)
(let* ((item-path (item-list->path itemdat))
(testdat (db:get-test-info db run-id test-name item-path)))
(equal? (test:get-state testdat) "KILLREQ")))
(define (test-set-meta-info db run-id testname itemdat)
|
| ︙ | | |
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
-
-
-
+
+
+
-
+
-
+
-
+
-
+
-
+
|
testname
item-path)))
(define (test-update-meta-info db run-id testname itemdat minutes)
(let ((item-path (item-list->path itemdat))
(cpuload (get-cpu-load))
(diskfree (get-df (current-directory))))
(if (not cpuload) (begin (print "WARNING: CPULOAD not found.") (set! cpuload "n/a")))
(if (not diskfree) (begin (print "WARNING: DISKFREE not found.") (set! diskfree "n/a")))
(if (not item-path)(begin (print "WARNING: ITEMPATH not set.") (set! item-path "")))
(if (not cpuload) (begin (debug:print 0 "WARNING: CPULOAD not found.") (set! cpuload "n/a")))
(if (not diskfree) (begin (debug:print 0 "WARNING: DISKFREE not found.") (set! diskfree "n/a")))
(if (not item-path)(begin (debug:print 0 "WARNING: ITEMPATH not set.") (set! item-path "")))
;; (let ((testinfo (db:get-test-info db run-id testname item-path)))
;; (if (and (not (equal? (db:test-get-status testinfo) "COMPLETED"))
;; (not (equal? (db:test-get-status testinfo) "KILLREQ"))
(sqlite3:execute
db
"UPDATE tests SET cpuload=?,diskfree=?,run_duration=?,state='RUNNING' WHERE run_id=? AND testname=? AND item_path=? AND state NOT IN ('COMPLETED','KILLREQ','KILLED');"
cpuload
diskfree
minutes
run-id
testname
item-path)))
(define (set-megatest-env-vars db run-id)
(let ((keys (db-get-keys db)))
(for-each (lambda (key)
(sqlite3:for-each-row
(lambda (val)
(print "setenv " (key:get-fieldname key) " " val)
(debug:print 2 "setenv " (key:get-fieldname key) " " val)
(setenv (key:get-fieldname key) val))
db
(conc "SELECT " (key:get-fieldname key) " FROM runs WHERE id=?;")
run-id))
keys)))
(define (set-item-env-vars itemdat)
(for-each (lambda (item)
(print "setenv " (car item) " " (cadr item))
(debug:print 2 "setenv " (car item) " " (cadr item))
(setenv (car item) (cadr item)))
itemdat))
(define (get-all-legal-tests)
(let* ((tests (glob (conc *toppath* "/tests/*")))
(res '()))
;; (print "INFO: Looking at tests " (string-intersperse tests ","))
(debug:print 4 "INFO: Looking at tests " (string-intersperse tests ","))
(for-each (lambda (testpath)
(if (file-exists? (conc testpath "/testconfig"))
(set! res (cons (last (string-split testpath "/")) res))))
tests)
res))
(define (runs:can-run-more-tests db)
(let ((num-running (db:get-count-tests-running db))
(max-concurrent-jobs (config-lookup *configdat* "setup" "max_concurrent_jobs")))
;; (print "max-concurrent-jobs: " max-concurrent-jobs ", num-running: " num-running)
(debug:print 2 "max-concurrent-jobs: " max-concurrent-jobs ", num-running: " num-running)
(if (or (not max-concurrent-jobs)
(and max-concurrent-jobs
(string->number max-concurrent-jobs)
(not (>= num-running (string->number max-concurrent-jobs)))))
#t
(begin
(print "WARNING: Max running jobs exceeded, current number running: " num-running
(debug:print 0 "WARNING: Max running jobs exceeded, current number running: " num-running
", max_concurrent_jobs: " max-concurrent-jobs)
#f))))
(define (run-tests db test-names)
(let* ((keys (db-get-keys db))
(keyvallst (keys->vallist keys #t))
(run-id (register-run db keys))) ;; test-name)))
|
| ︙ | | |
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
|
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
|
-
-
+
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
|
(sleep 2)))
test-names)
;; (run-waiting-tests db)
(if (args:get-arg "-keepgoing")
(let ((estrem (db:estimated-tests-remaining db run-id)))
(if (> estrem 0)
(begin
(print "Keep going, estimated " estrem " tests remaining to run, will continue in 10 seconds ...")
(sleep 10)
(debug:print 1 "Keep going, estimated " estrem " tests remaining to run, will continue in 3 seconds ...")
(sleep 3)
(run-waiting-tests db)
(loop (+ numtimes 1)))))))))
;; VERY INEFFICIENT! Move stuff that should be done once up to calling proc
(define (run-one-test db run-id test-name keyvallst)
(print "Launching test " test-name)
(debug:print 1 "Launching test " test-name)
;; All these vars might be referenced by the testconfig file reader
(setenv "MT_TEST_NAME" test-name) ;;
(setenv "MT_RUNNAME" (args:get-arg ":runname"))
(set-megatest-env-vars db run-id) ;; these may be needed by the launching process
(change-directory *toppath*)
(let* ((test-path (conc *toppath* "/tests/" test-name))
(test-configf (conc test-path "/testconfig"))
(testexists (and (file-exists? test-configf)(file-read-access? test-configf)))
(test-conf (if testexists (read-config test-configf) (make-hash-table)))
(waiton (let ((w (config-lookup test-conf "requirements" "waiton")))
(if (string? w)(string-split w)'()))))
(if (not testexists)
(begin
(print "ERROR: Can't find config file " test-configf)
(debug:print 0 "ERROR: Can't find config file " test-configf)
(exit 2))
;; put top vars into convenient variables and open the db
(let* (;; db is always at *toppath*/db/megatest.db
(items (hash-table-ref/default test-conf "items" '()))
(itemstable (hash-table-ref/default test-conf "itemstable" '()))
(allitems (if (or (not (null? items))(not (null? itemstable)))
(append (item-assoc->item-list items)
(item-table->item-list itemstable))
'(()))) ;; a list with one null list is a test with no items
(runconfigf (conc *toppath* "/runconfigs.config")))
(print "items: ")(pp allitems)
(debug:print 1 "items: ")(pp allitems)
(if (args:get-arg "-m")
(db:set-comment-for-run db run-id (args:get-arg "-m")))
(let loop ((itemdat (car allitems))
(tal (cdr allitems)))
;; (lambda (itemdat) ;;; ((ripeness "overripe") (temperature "cool") (season "summer"))
;; Handle lists of items
(let* ((item-path (item-list->path itemdat)) ;; (string-intersperse (map cadr itemdat) "/"))
(new-test-path (string-intersperse (cons test-path (map cadr itemdat)) "/"))
(new-test-name (if (equal? item-path "") test-name (conc test-name "/" item-path))) ;; just need it to be unique
(testdat #f)
(num-running (db:get-count-tests-running db))
(max-concurrent-jobs (config-lookup *configdat* "setup" "max_concurrent_jobs"))
(parent-test (and (not (null? items))(equal? item-path "")))
(single-test (and (null? items) (equal? item-path "")))
(item-test (not (equal? item-path ""))))
;; (print "max-concurrent-jobs: " max-concurrent-jobs ", num-running: " num-running)
(debug:print 3 "max-concurrent-jobs: " max-concurrent-jobs ", num-running: " num-running)
(if (runs:can-run-more-tests db)
(begin
(let loop2 ((ts (db:get-test-info db run-id test-name item-path)) ;; #f)
(ct 0))
(if (and (not ts)
(< ct 10))
(begin
(register-test db run-id test-name item-path)
(db:test-set-comment db run-id test-name item-path "")
(loop2 (db:get-test-info db run-id test-name item-path)
(+ ct 1)))
(if ts
(set! testdat ts)
(begin
(print "WARNING: Couldn't register test " test-name " with item path " item-path ", skipping")
(debug:print 0 "WARNING: Couldn't register test " test-name " with item path " item-path ", skipping")
(if (not (null? tal))
(loop (car tal)(cdr tal)))))))
(change-directory test-path)
;; this block is here only to inform the user early on
(if (file-exists? runconfigf)
(setup-env-defaults db runconfigf run-id *already-seen-runconfig-info*)
(print "WARNING: You do not have a run config file: " runconfigf))
;; (print "run-id: " run-id " test-name: " test-name " item-path: " item-path " testdat: " (test:get-status testdat) " test-state: " (test:get-state testdat))
(debug:print 0 "WARNING: You do not have a run config file: " runconfigf))
(debug:print 4 "run-id: " run-id " test-name: " test-name " item-path: " item-path " testdat: " (test:get-status testdat) " test-state: " (test:get-state testdat))
(case (if (args:get-arg "-force")
'NOT_STARTED
(if testdat
(string->symbol (test:get-state testdat))
'failed-to-insert))
((failed-to-insert)
(print "ERROR: Failed to insert the record into the db"))
(debug:print 0 "ERROR: Failed to insert the record into the db"))
((NOT_STARTED COMPLETED)
;; (print "Got here, " (test:get-state testdat))
(debug:print 6 "Got here, " (test:get-state testdat))
(let ((runflag #f))
(cond
;; i.e. this is the parent test to a suite of items, never "run" it
(parent-test
(set! runflag #f))
;; -force, run no matter what
((args:get-arg "-force")(set! runflag #t))
|
| ︙ | | |
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
|
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
|
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
|
((and (args:get-arg "-keepgoing")
(member (test:get-status testdat) '("FAIL")))
(set! runflag #f))
((and (not (args:get-arg "-rerun"))
(member (test:get-status testdat) '("FAIL" "n/a")))
(set! runflag #t))
(else (set! runflag #f)))
;; (print "RUNNING => runflag: " runflag " STATE: " (test:get-state testdat) " STATUS: " (test:get-status testdat))
(debug:print 6 "RUNNING => runflag: " runflag " STATE: " (test:get-state testdat) " STATUS: " (test:get-status testdat))
(if (not runflag)
(if (not parent-test)
(print "NOTE: Not starting test " new-test-name " as it is state \"COMPLETED\" and status \"" (test:get-status testdat) "\", use -force to override"))
(debug:print 1 "NOTE: Not starting test " new-test-name " as it is state \"COMPLETED\" and status \"" (test:get-status testdat) "\", use -force to override"))
(let* ((get-prereqs-cmd (lambda ()
(db-get-prereqs-not-met db run-id waiton))) ;; check before running ....
(launch-cmd (lambda ()
(launch-test db run-id test-conf keyvallst test-name test-path itemdat)))
(testrundat (list get-prereqs-cmd launch-cmd)))
(if (or (args:get-arg "-force")
(null? ((car testrundat)))) ;; are there any tests that must be run before this one...
((cadr testrundat)) ;; this is the line that launches the test to the remote host
(if (not (args:get-arg "-keepgoing"))
(hash-table-set! *waiting-queue* new-test-name testrundat)))))))
((KILLED)
(print "NOTE: " new-test-name " is already running or was explictly killed, use -force to launch it."))
(debug:print 1 "NOTE: " new-test-name " is already running or was explictly killed, use -force to launch it."))
((LAUNCHED REMOTEHOSTSTART RUNNING)
(if (> (- (current-seconds)(+ (db:test-get-event_time testdat)
(db:test-get-run_duration testdat)))
100) ;; i.e. no update for more than 100 seconds
(begin
(print "WARNING: Test " test-name " appears to be dead. Forcing it to state INCOMPLETE and status STUCK/DEAD")
(debug:print 0 "WARNING: Test " test-name " appears to be dead. Forcing it to state INCOMPLETE and status STUCK/DEAD")
(test-set-status! db run-id test-name "INCOMPLETE" "STUCK/DEAD" itemdat "Test is stuck or dead"))
(print "NOTE: " test-name " is already running")))
(else (print "ERROR: Failed to launch test " new-test-name ". Unrecognised state " (test:get-state testdat))))))
(debug:print 2 "NOTE: " test-name " is already running")))
(else (debug:print 0 "ERROR: Failed to launch test " new-test-name ". Unrecognised state " (test:get-state testdat))))))
(if (not (null? tal))
(loop (car tal)(cdr tal)))))))))
(define (run-waiting-tests db)
(let ((numtries 0)
(last-try-time (current-seconds))
(times (list 1))) ;; minutes to wait before trying again to kick off runs
;; BUG this hack of brute force retrying works quite well for many cases but
;; what is needed is to check the db for tests that have failed less than
;; N times or never been started and kick them off again
(let loop ((waiting-test-names (hash-table-keys *waiting-queue*)))
(cond
((not (runs:can-run-more-tests db))
(sleep 2)
(loop waiting-test-names))
((null? waiting-test-names)
(print "All tests launched"))
(debug:print 1 "All tests launched"))
(else
(set! numtries (+ numtries 1))
(for-each (lambda (testname)
(if (runs:can-run-more-tests db)
(let* ((testdat (hash-table-ref *waiting-queue* testname))
(prereqs ((car testdat)))
(ldb (if db db (open-db))))
;; (print "prereqs remaining: " prereqs)
(debug:print 2 "prereqs remaining: " prereqs)
(if (null? prereqs)
(begin
(print "Prerequisites met, launching " testname)
(debug:print 2 "Prerequisites met, launching " testname)
((cadr testdat))
(hash-table-delete! *waiting-queue* testname)))
(if (not db)
(sqlite3:finalize! ldb)))))
waiting-test-names)
;; (sleep 10) ;; no point in rushing things at this stage?
(loop (hash-table-keys *waiting-queue*)))))))
(define (get-dir-up-one dir)
(let ((dparts (string-split dir "/")))
(conc "/" (string-intersperse
(take dparts (- (length dparts) 1))
"/"))))
;; Remove runs
;; fields are passing in through
(define (runs:remove-runs db runnamepatt testpatt itempatt)
(let* ((keys (db-get-keys db))
(rundat (runs:get-runs-by-patt db keys runnamepatt))
(header (vector-ref rundat 0))
(runs (vector-ref rundat 1)))
(print "Header: " header)
(debug:print 1 "Header: " header)
(for-each
(lambda (run)
(let ((runkey (string-intersperse (map (lambda (k)
(db:get-value-by-header run header (vector-ref k 0))) keys) "/")))
(let* ((run-id (db:get-value-by-header run header "id") )
(tests (db-get-tests-for-run db (db:get-value-by-header run header "id") testpatt itempatt))
(lasttpath "/does/not/exist/I/hope"))
(if (not (null? tests))
(begin
(print "Removing tests for run: " runkey " " (db:get-value-by-header run header "runname"))
(debug:print 1 "Removing tests for run: " runkey " " (db:get-value-by-header run header "runname"))
(for-each
(lambda (test)
(print " " (db:test-get-testname test) " id: " (db:test-get-id test) " " (db:test-get-item-path test))
(debug:print 1 " " (db:test-get-testname test) " id: " (db:test-get-id test) " " (db:test-get-item-path test))
(db:delete-test-records db (db:test-get-id test))
(if (> (string-length (db:test-get-rundir test)) 5) ;; bad heuristic but should prevent /tmp /home etc.
(let ((fullpath (db:test-get-rundir test))) ;; "/" (db:test-get-item-path test))))
(set! lasttpath fullpath)
(print "rm -rf " fullpath)
(debug:print 1 "rm -rf " fullpath)
(system (conc "rm -rf " fullpath))
(let ((cmd (conc "rmdir -p " (get-dir-up-one fullpath))))
(print cmd)
(debug:print 1 cmd)
(system cmd))
)))
tests)))
(let ((remtests (db-get-tests-for-run db (db:get-value-by-header run header "id"))))
(if (null? remtests) ;; no more tests remaining
(let* ((dparts (string-split lasttpath "/"))
(runpath (conc "/" (string-intersperse
(take dparts (- (length dparts) 1))
"/"))))
(print "Removing run: " runkey " " (db:get-value-by-header run header "runname"))
(debug:print 1 "Removing run: " runkey " " (db:get-value-by-header run header "runname"))
(db:delete-run db run-id)
;; need to figure out the path to the run dir and remove it if empty
;; (if (null? (glob (conc runpath "/*")))
;; (begin
;; (print "Removing run dir " runpath)
;; (debug:print 1 "Removing run dir " runpath)
;; (system (conc "rmdir -p " runpath))))
)))
)))
runs)))
|