Megatest

Diff
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Annotate Patch Side-by-side Diff

Differences From Artifact [3ee9ed6771]:

To Artifact [f47e8f14e8]:

︙
224
225
226
227
228
229
230
231

232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264


265
266
267
268
269
270
271


272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290


291
292
293
294
295
296
297
298
299
300
301
302


303
304
305
306
307
308
309

224
225
226
227
228
229
230

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

248

249
250
251
252
253
254

255
256
257
258
259
260


261
262
263
264
265
266
267


268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300


301
302
303
304
305
306
307
308
309








-
+
















-
+
-






-






-
-
+
+





-
-
+
+



















+
+










-
-
+
+









;;======================================================================
;; Remove old run(s)
;;======================================================================

;; since several actions can be specified on the command line the removal
;; is done first
(define (operate-on db action)
(define (operate-on action)
  (cond
   ((not (args:get-arg ":runname"))
    (debug:print 0 "ERROR: Missing required parameter for " action ", you must specify the run name pattern with :runname patt")
    (exit 2))
   ((not (args:get-arg "-testpatt"))
    (debug:print 0 "ERROR: Missing required parameter for " action ", you must specify the test pattern with -testpatt")
    (exit 3))
   ((not (args:get-arg "-itempatt"))
    (print "ERROR: Missing required parameter for " action ", you must specify the items with -itempatt")
    (exit 4))
   (else
    (if (not (car *configinfo*))
	(begin
	  (debug:print 0 "ERROR: Attempted " action "on test(s) but run area config file not found")
	  (exit 1))
	;; put test parameters into convenient variables
	(runs:operate-on  db
	(runs:operate-on  action
			  action
			  (args:get-arg ":runname")
			  (args:get-arg "-testpatt")
			  (args:get-arg "-itempatt")
			  state: (args:get-arg ":state") 
			  status: (args:get-arg ":status")
			  new-state-status: (args:get-arg "-set-state-status")))
    (sqlite3:finalize! db)
    (set! *didsomething* #t))))
	  
(if (args:get-arg "-remove-runs")
    (general-run-call 
     "-remove-runs"
     "remove runs"
     (lambda (db target runname keys keynames keyvallst)
       (operate-on db 'remove-runs))))
     (lambda (target runname keys keynames keyvallst)
       (operate-on 'remove-runs))))

(if (args:get-arg "-set-state-status")
    (general-run-call 
     "-set-state-status"
     "set state and status"
     (lambda (db target runname keys keynames keyvallst)
       (operate-on db 'set-state-status))))
     (lambda (target runname keys keynames keyvallst)
       (operate-on 'set-state-status))))

;;======================================================================
;; Query runs
;;======================================================================

(if (args:get-arg "-list-runs")
    (let* ((db       (begin
		       (setup-for-run)
		       (open-db)))
	   (runpatt  (args:get-arg "-list-runs"))
	   (testpatt (args:get-arg "-testpatt"))
	   (itempatt (args:get-arg "-itempatt"))
	   (runsdat  (db:get-runs db runpatt #f #f '()))
	   (runs     (db:get-rows runsdat))
	   (header   (db:get-header runsdat))
	   (keys     (db:get-keys db))
	   (keynames (map key:get-fieldname keys)))
      (if (not (args:get-arg "-server"))
	  (server:client-setup db))
      (sqlite3:finalize! db)
      (set! db #f)
      ;; Each run
      (for-each 
       (lambda (run)
	 (debug:print 1 "Run: "
		(string-intersperse (map (lambda (x)
					   (db:get-value-by-header run header x))
					 keynames) "/")
		"/"
		(db:get-value-by-header run header "runname")
		" status: " (db:get-value-by-header run header "state"))
	 (let ((run-id (db:get-value-by-header run header "id")))
	   (let ((tests (db:get-tests-for-run db run-id testpatt itempatt '() '())))
	 (let ((run-id (open-run-close db:get-value-by-header run header "id")))
	   (let ((tests (open-run-close db:get-tests-for-run db run-id testpatt itempatt '() '())))
	     ;; Each test
	     (for-each 
	      (lambda (test)
		(format #t
			"  Test: ~25a State: ~15a Status: ~15a Runtime: ~5@as Time: ~22a Host: ~10a\n"
			(conc (db:test-get-testname test)
			      (if (equal? (db:test-get-item-path test) "")
︙
320
321
322
323
324
325
326
327

328
329
330
331
332
333
334

320
321
322
323
324
325
326

327
328
329
330
331
332
333
334








-
+








		    (begin
		      (print "         cpuload:  " (db:test-get-cpuload test)
			     "\n         diskfree: " (db:test-get-diskfree test)
			     "\n         uname:    " (db:test-get-uname test)
			     "\n         rundir:   " (db:test-get-rundir test)
			     )
		      ;; Each test
		      (let ((steps (db:get-steps-for-test db (db:test-get-id test))))
		      (let ((steps (open-run-close db:get-steps-for-test db (db:test-get-id test))))
			(for-each 
			 (lambda (step)
			   (format #t 
				   "    Step: ~20a State: ~10a Status: ~10a Time ~22a\n"
				   (db:step-get-stepname step)
				   (db:step-get-state step)
				   (db:step-get-status step)
︙
376
377
378
379
380
381
382
383

384
385
386
387
388

389
390
391
392
393
394
395
396

376
377
378
379
380
381
382

383
384
385
386
387

388

389
390
391
392
393
394
395








-
+




-
+
-








;;   process deferred tasks per above steps

;; run all tests are are Not COMPLETED and PASS or CHECK
(if (args:get-arg "-runall")
    (general-run-call 
     "-runall"
     "run all tests"
     (lambda (db target runname keys keynames keyvallst)
     (lambda (target runname keys keynames keyvallst)
;;       (let ((flags (make-hash-table)))
;;	 (for-each (lambda (parm)
;;		     (hash-table-set! flags parm (args:get-arg parm)))
;;		   (list "-rerun" "-force" "-itempatt"))
	 (runs:run-tests db
	 (runs:run-tests target
			 target
			 runname
			 (args:get-arg "-runtests")
			 user
			 args:arg-hash)))) ;; )

;;======================================================================
;; run one test
︙
409
410
411
412
413
414
415
416
417


418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433


434
435
436
437
438
439
440
441
442
443
444
445
446
447
448


449
450
451
452
453
454
455

408
409
410
411
412
413
414


415
416

417
418
419
420
421
422
423
424
425
426
427
428
429


430
431

432
433
434
435
436
437
438
439
440
441
442
443


444
445
446
447
448
449
450
451
452








-
-
+
+
-













-
-
+
+
-












-
-
+
+








;;    - if test run time > allowed run time then kill job
;;    - if cannot access db > allowed disconnect time then kill job

(if (args:get-arg "-runtests")
  (general-run-call 
   "-runtests" 
   "run a test" 
   (lambda (db target runname keys keynames keyvallst)
     (runs:run-tests db
   (lambda (target runname keys keynames keyvallst)
     (runs:run-tests target
		     target
		     runname
		     (args:get-arg "-runtests")
		     user
		     args:arg-hash))))

;;======================================================================
;; Rollup into a run
;;======================================================================

(if (args:get-arg "-rollup")
    (general-run-call 
     "-rollup" 
     "rollup tests" 
     (lambda (db target runname keys keynames keyvallst)
       (runs:rollup-run db
     (lambda (target runname keys keynames keyvallst)
       (runs:rollup-run keys
			keys
			(keys->alist keys "na")
			(args:get-arg ":runname") 
			user))))

;;======================================================================
;; Lock or unlock a run
;;======================================================================

(if (or (args:get-arg "-lock")(args:get-arg "-unlock"))
    (general-run-call 
     (if (args:get-arg "-lock") "-lock" "-unlock")
     "lock/unlock tests" 
     (lambda (db target runname keys keynames keyvallst)
       (runs:handle-locking db
     (lambda (target runname keys keynames keyvallst)
       (runs:handle-locking 
		  target
		  keys
		  (args:get-arg ":runname") 
		  (args:get-arg "-lock")
		  (args:get-arg "-unlock")
		  user))))
︙
480
481
482
483
484
485
486
487




488
489

490
491

492
493
494
495
496
497
498
499
500
501
502




503
504
505
506
507
508
509

477
478
479
480
481
482
483

484
485
486
487
488

489
490

491
492
493
494
495
496
497
498
499



500
501
502
503
504
505
506
507
508
509
510








-
+
+
+
+

-
+

-
+








-
-
-
+
+
+
+








		(exit 1)))
	  (if (not (setup-for-run))
	      (begin
		(debug:print 0 "Failed to setup, giving up on -test-paths or -test-files, exiting")
		(exit 1)))
	  (set! db (open-db))    
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	      (server:client-setup db)
	      (begin
		(sqlite3:finalize! db)
		(set! db #f)))
	  (let* ((itempatt (args:get-arg "-itempatt"))
		 (keys     (db:get-keys db))
		 (keys     (open-run-close db:get-keys db))
		 (keynames (map key:get-fieldname keys))
		 (paths    (db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
		 (paths    (open-run-close db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
	    (set! *didsomething* #t)
	    (for-each (lambda (path)
			(print path))
		      paths)))
	;; else do a general-run-call
	(general-run-call 
	 "-test-files"
	 "Get paths to test"
	 (lambda (db target runname keys keynames keyvallst)
	   (let* ((itempatt (args:get-arg "-itempatt"))
		  (paths    (db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
	 (lambda (target runname keys keynames keyvallst)
	   (let* ((db       #f)
		  (itempatt (args:get-arg "-itempatt"))
		  (paths    (open-run-close db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
	     (for-each (lambda (path)
			 (print path))
		       paths))))))

;;======================================================================
;; Archive tests
;;======================================================================
︙
528
529
530
531
532
533
534
535
536
537
538
539

540
541

542
543
544
545
546
547
548
549
550
551
552




553
554
555
556
557
558
559
560
561
562
563
564
565
566



567
568
569
570
571

572
573
574
575
576
577
578

529
530
531
532
533
534
535



536

537
538

539
540
541
542
543
544
545
546
547



548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563


564
565
566
567
568
569
570

571
572
573
574
575
576
577
578








-
-
-

-
+

-
+








-
-
-
+
+
+
+












-
-
+
+
+




-
+








	      (begin
		(debug:print 0 "ERROR: -target is required.")
		(exit 1)))
	  (if (not (setup-for-run))
	      (begin
		(debug:print 0 "Failed to setup, giving up on -archive, exiting")
		(exit 1)))
	  (set! db (open-db))   
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	  (let* ((itempatt (args:get-arg "-itempatt"))
		 (keys     (db:get-keys db))
		 (keys     (open-run-close db:get-keys db))
		 (keynames (map key:get-fieldname keys))
		 (paths    (db:test-get-paths-matching db keynames target)))
		 (paths    (open-run-close db:test-get-paths-matching db keynames target)))
	    (set! *didsomething* #t)
	    (for-each (lambda (path)
			(print path))
		      paths)))
	;; else do a general-run-call
	(general-run-call 
	 "-test-paths"
	 "Get paths to tests"
	 (lambda (db target runname keys keynames keyvallst)
	   (let* ((itempatt (args:get-arg "-itempatt"))
		  (paths    (db:test-get-paths-matching db keynames target)))
	 (lambda (target runname keys keynames keyvallst)
	   (let* ((db       #f)
		  (itempatt (args:get-arg "-itempatt"))
		  (paths    (open-run-close db:test-get-paths-matching db keynames target)))
	     (for-each (lambda (path)
			 (print path))
		       paths))))))

;;======================================================================
;; Extract a spreadsheet from the runs database
;;======================================================================

(if (args:get-arg "-extract-ods")
    (general-run-call
     "-extract-ods"
     "Make ods spreadsheet"
     (lambda (db target runname keys keynames keyvallst)
       (let ((outputfile (args:get-arg "-extract-ods"))
     (lambda (target runname keys keynames keyvallst)
       (let ((db         #f)
	     (outputfile (args:get-arg "-extract-ods"))
	     (runspatt   (args:get-arg ":runname"))
	     (pathmod    (args:get-arg "-pathmod"))
	     (keyvalalist (keys->alist keys "%")))
	 (debug:print 2 "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvalalist: " keyvalalist)
	 (db:extract-ods-file db outputfile keyvalalist (if runspatt runspatt "%") pathmod)))))
	 (open-run-close db:extract-ods-file db outputfile keyvalalist (if runspatt runspatt "%") pathmod)))))

;;======================================================================
;; execute the test
;;    - gets called on remote host
;;    - receives info from the -execute param
;;    - passes info to steps via MT_CMDINFO env var (future is to use a dot file)
;;    - gathers host info and
︙
608
609
610
611
612
613
614
615




616
617

618
619
620
621

622
623
624
625
626
627
628

608
609
610
611
612
613
614

615
616
617
618
619

620
621
622
623

624
625
626
627
628
629
630
631








-
+
+
+
+

-
+



-
+








	  (change-directory testpath)
	  (if (not (setup-for-run))
	      (begin
		(debug:print 0 "Failed to setup, exiting")
		(exit 1)))
	  (set! db (open-db))
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	      (server:client-setup db)
	      (begin
		(sqlite3:finalize! db)
		(set! db #f)))
	  (if (and state status)
	      (db:teststep-set-status! db test-id step state status itemdat (args:get-arg "-m") logfile)
	      (open-run-close db:teststep-set-status! db test-id step state status itemdat (args:get-arg "-m") logfile)
	      (begin
		(debug:print 0 "ERROR: You must specify :state and :status with every call to -step")
		(exit 6)))
	  (sqlite3:finalize! db)
	  (if db (sqlite3:finalize! db))
	  (set! *didsomething* #t))))

(if (or (args:get-arg "-setlog")       ;; since setting up is so costly lets piggyback on -test-status
	(args:get-arg "-set-toplog")
	(args:get-arg "-test-status")
	(args:get-arg "-set-values")
	(args:get-arg "-load-test-data")
︙
647
648
649
650
651
652
653
654




655
656
657

658
659
660

661
662

663
664

665
666
667
668
669

670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686

687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709

710
711

712
713
714
715
716
717

718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738

739
740
741
742


743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759





760
761

762
763
764
765
766
767
768

650
651
652
653
654
655
656

657
658
659
660
661
662

663
664
665

666
667

668
669

670
671
672
673
674

675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691

692


693
694
695
696
697
698




699
700
701
702
703
704
705
706
707
708

709
710

711






712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732

733
734
735


736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752


753
754
755
756
757
758

759
760
761
762
763
764
765
766








-
+
+
+
+


-
+


-
+

-
+

-
+




-
+
















-
+
-
-






-
-
-
-










-
+

-
+
-
-
-
-
-
-
+




















-
+


-
-
+
+















-
-
+
+
+
+
+

-
+








	  (change-directory testpath)
	  (if (not (setup-for-run))
	      (begin
		(debug:print 0 "Failed to setup, exiting")
		(exit 1)))
	  (set! db (open-db))
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	      (server:client-setup db)
	      (begin
		(sqlite3:finalize! db)
		(set! db #f)))
	  (if (args:get-arg "-load-test-data")
	      ;; has sub commands that are rdb:
	      (db:load-test-data db test-id))
	      (open-run-close db:load-test-data db test-id))
	  (if (args:get-arg "-setlog")
	      (let ((logfname (args:get-arg "-setlog")))
		(db:test-set-log! db test-id logfname)))
		(open-run-close db:test-set-log! db test-id logfname)))
	  (if (args:get-arg "-set-toplog")
	      (rtests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
	      (open-run-close tests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
	  (if (args:get-arg "-summarize-items")
	      (tests:summarize-items db run-id test-name #t)) ;; do force here
	      (open-run-close tests:summarize-items db run-id test-name #t)) ;; do force here
	  (if (args:get-arg "-runstep")
	      (if (null? remargs)
		  (begin
		    (debug:print 0 "ERROR: nothing specified to run!")
		    (sqlite3:finalize! db)
		    (if db (sqlite3:finalize! db))
		    (exit 6))
		  (let* ((stepname   (args:get-arg "-runstep"))
			 (logprofile (args:get-arg "-logpro"))
			 (logfile    (conc stepname ".log"))
			 (cmd        (if (null? remargs) #f (car remargs)))
			 (params     (if cmd (cdr remargs) '()))
			 (exitstat   #f)
			 (shell      (last (string-split (get-environment-variable "SHELL") "/")))
			 (redir      (case (string->symbol shell)
				       ((tcsh csh ksh)    ">&")
				       ((zsh bash sh ash) "2>&1 >")
				       (else ">&")))
			 (fullcmd    (conc "(" (string-intersperse 
						(cons cmd params) " ")
					   ") " redir " " logfile)))
		    ;; mark the start of the test
		    (db:teststep-set-status! db test-id stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
		    (open-run-close db:teststep-set-status! db test-id stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
		    ;; close the db
		    ;; (sqlite3:finalize! db)
		    ;; run the test step
		    (debug:print 2 "INFO: Running \"" fullcmd "\"")
		    (change-directory startingdir)
		    (set! exitstat (system fullcmd)) ;; cmd params))
		    (set! *globalexitstatus* exitstat)
		    (change-directory testpath)
		    ;; re-open the db
		    ;; (set! db (open-db))
		    ;; (if (not (args:get-arg "-server"))
		    ;;     (server:client-setup db))
		    ;; run logpro if applicable ;; (process-run "ls" (list "/foo" "2>&1" "blah.log"))
		    (if logprofile
			(let* ((htmllogfile (conc stepname ".html"))
			       (oldexitstat exitstat)
			       (cmd         (string-intersperse (list "logpro" logprofile htmllogfile "<" logfile ">" (conc stepname "_logpro.log")) " ")))
			  (debug:print 2 "INFO: running \"" cmd "\"")
			  (change-directory startingdir)
			  (set! exitstat (system cmd))
			  (set! *globalexitstatus* exitstat) ;; no necessary
			  (change-directory testpath)
			  (db:test-set-log! db test-id htmllogfile)))
			  (open-run-close db:test-set-log! db test-id htmllogfile)))
		    (let ((msg (args:get-arg "-m")))
		      (db:teststep-set-status! db test-id stepname "end" exitstat itemdat msg logfile))
		      (open-run-close db:teststep-set-status! db test-id stepname "end" exitstat itemdat msg logfile))
		    ;; (sqlite3:finalize! db)
		    ;;(if (not (eq? exitstat 0))
		    ;;	(exit 254)) ;; (exit exitstat) doesn't work?!?
		  ;; open the db
		  ;; mark the end of the test
		  )))
		    )))
	  (if (or (args:get-arg "-test-status")
		  (args:get-arg "-set-values"))
	      (let ((newstatus (cond
				((number? status)       (if (equal? status 0) "PASS" "FAIL"))
				((and (string? status)
				      (string->number status))(if (equal? (string->number status) 0) "PASS" "FAIL"))
				(else status)))
		    ;; transfer relevant keys into a hash to be passed to test-set-status!
		    ;; could use an assoc list I guess. 
		    (otherdata (let ((res (make-hash-table)))
				 (for-each (lambda (key)
					     (if (args:get-arg key)
						 (hash-table-set! res key (args:get-arg key))))
					   (list ":value" ":tol" ":expected" ":first_err" ":first_warn" ":units" ":category" ":variable"))
				 res)))
		(if (and (args:get-arg "-test-status")
			 (or (not state)
			     (not status)))
		    (begin
		      (debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help)
		      (sqlite3:finalize! db)
		      ;; (sqlite3:finalize! db)
		      (exit 6)))
		(let ((msg (args:get-arg "-m")))
		  (rtests:test-set-status! db test-id state newstatus msg otherdata))))
	  (sqlite3:finalize! db)
		  (open-run-close tests:test-set-status! db test-id state newstatus msg otherdata))))
	  (if db (sqlite3:finalize! db))
	  (set! *didsomething* #t))))

;;======================================================================
;; Various helper commands can go below here
;;======================================================================

(if (args:get-arg "-showkeys")
    (let ((db #f)
	  (keys #f))
      (if (not (setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting")
	    (exit 1)))
      (set! db (open-db))
      (if (not (args:get-arg "-server"))
	  (server:client-setup db))
      (set! keys (db:get-keys db))
	  (server:client-setup db)
	  (begin
	    (sqlite3:finalize! db)
	    (set! db #f)))
      (set! keys (open-run-close db:get-keys db))
      (debug:print 1 "Keys: " (string-intersperse (map key:get-fieldname keys) ", "))
      (sqlite3:finalize! db)
      (if db (sqlite3:finalize! db))
      (set! *didsomething* #t)))

(if (args:get-arg "-gui")
    (begin
      (debug:print 0 "Look at the dashboard for now")
      ;; (megatest-gui)
      (set! *didsomething* #t)))
︙
783
784
785
786
787
788
789
790
791
792

793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811






812
813
814
815
816
817
818

781
782
783
784
785
786
787



788

789
790
791
792
793
794
795
796
797
798
799
800
801
802
803



804
805
806
807
808
809
810
811
812
813
814
815
816








-
-
-
+
-















-
-
-
+
+
+
+
+
+









(if (args:get-arg "-rebuild-db")
    (begin
      (if (not (setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting") 
	    (exit 1)))
      ;; now can find our db
      (set! db (open-db))
      (patch-db db)
      (open-run-close patch-db #f)
      (sqlite3:finalize! db)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the tests meta data from the testconfig files
;;======================================================================

(if (args:get-arg "-update-meta")
    (begin
      (if (not (setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting") 
	    (exit 1)))
      ;; now can find our db
      (set! db (open-db))
      (if (not (args:get-arg "-server"))
	  (server:client-setup db))
      (runs:update-all-test_meta db)
      (sqlite3:finalize! db)
	  (server:client-setup db)
	  (begin
	    (sqlite3:finalize! db)
	    (set! db #f)))
      (open-run-close runs:update-all-test_meta db)
      (if db (sqlite3:finalize! db))
      (set! *didsomething* #t)))

;;======================================================================
;; Start a repl
;;======================================================================

(if (args:get-arg "-repl")
︙
This page was generated in about 0.102s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00