Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -38,11 +38,11 @@ ARCHSTR=$(shell lsb_release -sr) # ARCHSTR=$(shell bash -c "echo \$$MACHTYPE") PNGFILES = $(shell cd docs/manual;ls *png) -all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard +all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut mtest: $(OFILES) readline-fix.scm megatest.o csc $(CSCOPTS) $(OFILES) megatest.o -o mtest dboard : $(OFILES) $(GOFILES) dashboard.scm @@ -49,10 +49,13 @@ csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o dboard ndboard : newdashboard.scm $(OFILES) $(GOFILES) csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard +mtut: $(OFILES) mtut.scm + csc $(CSCOPTS) $(OFILES) mtut.scm -o mtut + # install documentation to $(PREFIX)/docs # DOES NOT REBUILD DOCS # $(PREFIX)/share/docs/megatest_manual.html : docs/manual/megatest_manual.html mkdir -p $(PREFIX)/share/docs @@ -100,10 +103,17 @@ $(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard $(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard chmod a+x $(PREFIX)/bin/newdashboard + +$(PREFIX)/bin/.$(ARCHSTR)/mtut : mtut + $(INSTALL) mtut $(PREFIX)/bin/.$(ARCHSTR)/mtut + +$(PREFIX)/bin/mtutil : $(PREFIX)/bin/.$(ARCHSTR)/mtut utils/mk_wrapper + utils/mk_wrapper $(PREFIX) mtut $(PREFIX)/bin/mtutil + chmod a+x $(PREFIX)/bin/mtutil #$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard # $(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard # $(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard utils/mk_wrapper @@ -178,11 +188,11 @@ $(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \ $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ - $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun + $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun $(PREFIX)/bin/mtutil $(PREFIX)/bin/.$(ARCHSTR) : mkdir -p $(PREFIX)/bin/.$(ARCHSTR) mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -7,11 +7,12 @@ ;; This program is distributed WITHOUT ANY WARRANTY; without even the ;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ;; PURPOSE. ;;====================================================================== -(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 sql-de-lite hostinfo md5 message-digest typed-records directory-utils stack) +(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 sql-de-lite hostinfo md5 message-digest typed-records directory-utils stack + matchable) (require-extension regex posix) (require-extension (srfi 18) extras tcp rpc) (import (prefix sqlite3 sqlite3:)) @@ -712,19 +713,10 @@ ;;====================================================================== ;; M I S C U T I L S ;;====================================================================== -;; one-of args defined -(define (args-defined? . param) - (let ((res #f)) - (for-each - (lambda (arg) - (if (args:get-arg arg)(set! res #t))) - param) - res)) - ;; convert stuff to a number if possible (define (any->number val) (cond ((number? val) val) ((string? val) (string->number val)) @@ -747,25 +739,10 @@ (set! res #t)))) (string-split patts ",")) res) #t)) -;; (map print (map car (hash-table->alist (read-config "runconfigs.config" #f #t)))) -(define (common:get-runconfig-targets #!key (configf #f)) - (let ((targs (sort (map car (hash-table->alist - (or configf ;; NOTE: There is no value in using runconfig:read here. - (read-config (conc *toppath* "/runconfigs.config") - #f #t) - (make-hash-table)))) - string curmod last-mod) + (list curmod fname) + res))) + '(0 "n/a") + all-files))) + ;;====================================================================== ;; T A R G E T S , S T A T E , S T A T U S , ;; R U N N A M E A N D T E S T P A T T ;;====================================================================== + +;; (map print (map car (hash-table->alist (read-config "runconfigs.config" #f #t)))) +;; +(define (common:get-runconfig-targets #!key (configf #f)) + (let ((targs (sort (map car (hash-table->alist + (or configf ;; NOTE: There is no value in using runconfig:read here. + (read-config (conc *toppath* "/runconfigs.config") + #f #t) + (make-hash-table)))) + string in runconfigs instead of default TESTPATT + if -testpatt and -tagexpr are not specified + -new state/status : specify new state/status for set-ss + +Misc + -start-dir path : switch to this directory before running mtutil + -set-vars V1=1,V2=2 : Add environment variables to a run NB// these are + overwritten by values set in config files. + -log logfile : send stdout and stderr to logfile + -repl : start a repl (useful for extending megatest) + -load file.scm : load and run file.scm + -debug N|N,M,O... : enable debug messages 0-N or N and M and O ... + +Examples: + +# Start a megatest run in the area \"mytests\" +mtutil -area mytests -action run -target v1.63/aa3e -mode-patt MYPATT -tag-expr quick + +# Start a contour +mtutil run -contour quick -target v1.63/aa3e + +Called as " (string-intersperse (argv) " ") " +Version " megatest-version ", built from " megatest-fossil-hash )) + +;; args and pkt key specs +;; +(define *arg-keys* + '(("-run" . r) + ("-area" . G) ;; maps to group + ("-target" . t) + ("-run-name" . n) + ("-state" . e) + ("-status" . s) + ("-contour" . c) + ("-test-patt" . p) ;; idea, enhance margs ("-test-patt" "-testpatt") => yields one value in "-test-patt" + ("-mode-patt" . o) + ("-tag-expr" . x) + ("-item-patt" . i) + ;; misc + ("-start-dir" . S) + ("-msg" . M) + ("-set-vars" . v) + ("-debug" . #f) ;; for *verbosity* > 2 + ("-load" . #f) ;; load and exectute a scheme file + ("-log" . #f) + )) +(define *switch-keys* + '(("-h" . #f) + ("-help" . #f) + ("--help" . #f) + ("-manual" . #f) + ("-version" . #f) + ;; misc + ("-repl" . #f) + ("-immediate" . I) + )) + +(define (lookup-param-by-key key #!key (inlst #f)) + (fold (lambda (a res) + (if (eq? (cdr a) key) + (car a) + res)) + #f + (or inlst *arg-keys*))) + +;; given a mtutil param, return the old megatest equivalent +;; +(define (param-translate param) + (or (alist-ref (string->symbol param) + '((-tag-expr . "-tagexpr") + (-mode-patt . "--modepatt") + (-run-name . "-runname") + (-test-patt . "-testpatt") + (-msg . "-m"))) + param)) + +;; Card types: +;; +;; a action +;; u username (Unix) +;; D timestamp +;; T card type + +;; process args +(define *action* (if (> (length (argv)) 1) + (cadr (argv)) + #f)) +(define remargs (args:get-args + (if *action* (cdr (argv)) (argv)) ;; args:get-args dumps first in argv list (the program name) + (map car *arg-keys*) + (map car *switch-keys*) + args:arg-hash + 0)) + +(if (or (member *action* '("-h" "-help" "help" "--help")) + (args:any-defined? "-h" "-help" "--help")) + (begin + (print help) + (exit 1))) + +;; (print "*action*: " *action*) +;; (let-values (((uuid pkt) +;; (command-line->pkt #f args:arg-hash))) +;; (print pkt)) + +;; Add args that use remargs here +;; +(if (and (not (null? remargs)) + (not (or + (args:get-arg "-runstep") + (args:get-arg "-envcap") + (args:get-arg "-envdelta") + ))) + (debug:print-error 0 *default-log-port* "Unrecognised arguments: " (string-intersperse (if (list? remargs) remargs (argv)) " "))) + +;;====================================================================== +;; pkts +;;====================================================================== + +(define (with-queue-db mtconf proc) + (let* ((pktsdirs (configf:lookup mtconf "setup" "pktsdirs")) + (pktsdir (if pktsdirs (car (string-split pktsdirs " ")) #f)) + (toppath (configf:lookup mtconf "dyndat" "toppath")) + (pdbpath (or (configf:lookup mtconf "setup" "pdbpath") pktsdir))) + (if (not (and pktsdir toppath pdbpath)) + (begin + (print "ERROR: settings are missing in your megatest.config for area management.") + (print " you need to have pktsdir in the [setup] section.")) + (let* ((pdb (open-queue-db pdbpath "pkts.db" + schema: '("CREATE TABLE groups (id INTEGER PRIMARY KEY,groupname TEXT, CONSTRAINT group_constraint UNIQUE (groupname));")))) + (proc pktsdirs pktsdir pdb) + (dbi:close pdb))))) + +(define (load-pkts-to-db mtconf) + (with-queue-db + mtconf + (lambda (pktsdirs pktsdir pdb) + (for-each + (lambda (pktsdir) ;; look at all + (if (and (file-exists? pktsdir) + (directory? pktsdir) + (file-read-access? pktsdir)) + (let ((pkts (glob (conc pktsdir "/*.pkt")))) + (for-each + (lambda (pkt) + (let* ((uuid (cadr (string-match ".*/([0-9a-f]+).pkt" pkt))) + (exists (lookup-by-uuid pdb uuid #f))) + (if (not exists) + (let* ((pktdat (string-intersperse + (with-input-from-file pkt read-lines) + "\n")) + (apkt (convert-pkt->alist pktdat)) + (ptype (alist-ref 'T apkt))) + (add-to-queue pdb pktdat uuid (or ptype 'cmd) #f 0) + (print "Added " uuid " of type " ptype " to queue")) + (print "pkt: " uuid " exists, skipping...") + ))) + pkts)))) + (string-split pktsdirs))))) + +;;====================================================================== +;; Runs +;;====================================================================== + +;; make a runname +;; +(define (make-runname valparts) + "ww07.1a") + +;; collect, translate, collate and assemble a pkt from the command-line +;; +(define (command-line->pkt action args-alist) + (let* ((args-data (if args-alist + args-alist + (hash-table->alist args:arg-hash))) + (alldat (apply append (list 'a action + 'U (current-user-name)) + (map (lambda (x) + (let* ((param (car x)) + (value (cdr x)) + (pmeta (assoc param *arg-keys*)) + (smeta (assoc param *switch-keys*)) + (meta (if (or pmeta smeta) + (cdr (or pmeta smeta)) + #f))) + (if (or pmeta smeta) + (list meta value) + '()))) + (filter cdr args-data))))) + (print "Alldat: " alldat + " args-data: " args-data) + (add-z-card + (apply construct-sdat alldat)))) + +(define (simple-setup start-dir-in) + (let* ((start-dir (or start-dir-in ".")) + (mtconfig (or (args:get-arg "-config") "megatest.config")) + (mtconfdat (find-and-read-config ;; NB// sets MT_RUN_AREA_HOME as side effect + mtconfig + ;; environ-patt: "env-override" + given-toppath: start-dir + ;; pathenvvar: "MT_RUN_AREA_HOME" + )) + (mtconf (if mtconfdat (car mtconfdat) #f))) + ;; we set some dynamic data in a section called "dyndata" + (if mtconf + (begin + (configf:section-var-set! mtconf "dyndat" "toppath" start-dir))) + (print "TOPPATH: " (configf:lookup mtconf "dyndat" "toppath")) + mtconfdat)) + +;; make a run request pkt from basic data +;; +(define (create-run-pkt mtconf area runkey runname mode-patt tag-expr pktsdir reason) + (let ((area-path (configf:lookup mtconf "areas" area))) + (let-values (((uuid pkt) + (command-line->pkt + "run" + (append + `(("-target" . ,runkey) + ("-run-name" . ,runname) + ("-start-dir" . ,area-path) + ("-msg" . ,reason)) + (if mode-patt + `(("-mode-patt" . ,mode-patt)) + '()) + (if tag-expr + `(("-tag-expr" . ,tag-expr)) + '()) + (if (not (or mode-patt tag-expr)) + `(("-item-patt" . "%")) + '()))))) + (with-output-to-file + (conc pktsdir "/" uuid ".pkt") + (lambda () + (print pkt)))))) + +;; collect all needed data and create run pkts for contours with changed inputs +;; +(define (generate-run-pkts mtconf toppath) + (with-queue-db + mtconf + (lambda (pktsdirs pktsdir pdb) + (let* ((rgconfdat (find-and-read-config (conc toppath "/rungen.config"))) + (rgconf (car rgconfdat)) + (areas (map car (configf:get-section mtconf "areas"))) + (contours (configf:get-section mtconf "contours")) + (torun (make-hash-table)) ;; target => ( ... info ... ) + (rgentargs (hash-table-keys rgconf))) ;; these are the targets registered for automatically triggering + + (for-each + (lambda (runkey) + (let* ((keydats (configf:get-section rgconf runkey))) + (for-each + (lambda (sense) ;; these are the sense rules + (let* ((key (car sense)) + (val (cadr sense)) + (keyparts (string-split key ":")) + (contour (car keyparts)) + (ruletype (let ((res (cdr keyparts))) + (if (null? res) #f (cadr keyparts)))) + (valparts (string-split val)) ;; runname-rule params + (runname (make-runname #f)) + (runstarts (find-pkts pdb '(runstart) `((o . ,contour) + (t . ,runkey)))) + (rspkts (map (lambda (x) + (alist-ref 'pkta x)) + runstarts)) + (starttimes ;; sort by age (youngest first) and delete duplicates by target + (delete-duplicates + (sort + (map (lambda (x) + `(,(alist-ref 't x) . ,(string->number (alist-ref 'D x)))) + rspkts) + (lambda (a b)(> (cdr a)(cdr b)))) ;; sort descending + (lambda (a b)(equal? (car a)(car b))))) ;; remove duplicates by target + ) + ;; look in runstarts for matching runs by target and contour + ;; get the timestamp for when that run started and pass it + ;; to the rule logic here where "ruletype" will be applied + ;; if it comes back "changed" then proceed to register the runs + + (case (string->symbol ruletype) + ((file) + (let* ((file-globs (cdr valparts)) + (youngestdat (common:get-youngest file-globs)) + (youngestmod (car youngestdat))) + ;; (print "youngestmod: " youngestmod " starttimes: " starttimes) + (if (null? starttimes) ;; this target has never been run + (configf:section-var-set! torun contour runkey `("file:neverrun" ,runname)) + (for-each + (lambda (starttime) ;; look at the time the last run was kicked off for this contour + (if (> youngestmod (cdr starttime)) + (begin + (print "starttime younger than youngestmod: " starttime " Youngestmod: " youngestmod) + (configf:section-var-set! torun contour runkey `(,(conc ruletype ":" (cadr youngestdat)) ,runname))))) + starttimes)) + ))))) + keydats))) + (hash-table-keys rgconf)) + + ;; now have torun populated + (for-each + (lambda (contour) + (let* ((mode-tag (string-split (or (configf:lookup mtconf "contours" contour) "") "/")) + (mode-patt (if (eq? (length mode-tag) 2)(cadr mode-tag) #f)) + (tag-expr (if (null? mode-tag) #f (car mode-tag)))) + (for-each + (lambda (runkeydat) + (let* ((runkey (car runkeydat)) + (info (cadr runkeydat))) + (for-each + (lambda (area) + (let ((runname (cadr info)) + (reason (car info))) + (print "runkey: " runkey " contour: " contour " info: " info " area: " area " tag-expr: " tag-expr " mode-patt: " mode-patt) + (create-run-pkt mtconf area runkey runname mode-patt tag-expr pktsdir reason))) + areas))) + (configf:get-section torun contour)))) + (hash-table-keys torun)))))) + + +(define (pkt->cmdline pkta) + (fold (lambda (a res) + (let* ((key (car a)) ;; get the key name + (val (cdr a)) + (par (lookup-param-by-key key))) + ;; (print "key: " key " val: " val " par: " par) + (if par + (conc res " " (param-translate par) " " val) + res))) + "megatest -run" + pkta)) + +(define (write-pkt pktsdir uuid pkt) + (if pktsdir + (with-output-to-file + (conc pktsdir "/" uuid ".pkt") + (lambda () + (print pkt))) + (print "ERROR: cannot process commands without a pkts directory"))) + +;; collect all needed data and create run pkts for contours with changed inputs +;; +(define (dispatch-commands mtconf toppath) + (with-queue-db + mtconf + (lambda (pktsdirs pktsdir pdb) + (let* ((rgconfdat (find-and-read-config (conc toppath "/rungen.config"))) + (rgconf (car rgconfdat)) + (areas (configf:get-section mtconf "areas")) + (contours (configf:get-section mtconf "contours")) + (pkts (find-pkts pdb '(cmd) '())) + (torun (make-hash-table)) ;; target => ( ... info ... ) + (rgentargs (hash-table-keys rgconf))) ;; these are the targets registered for automatically triggering + (for-each + (lambda (pktdat) + (let* ((pkta (alist-ref 'pkta pktdat)) + (cmdline (pkt->cmdline pkta)) + (uuid (alist-ref 'Z pkta)) + (logf (conc "logs/" uuid "-run.log"))) + (system (conc "NBFAKE_LOG=" logf " nbfake " cmdline)) + (mark-processed pdb (list (alist-ref 'id pktdat))) + (let-values (((ack-uuid ack-pkt) + (add-z-card + (construct-sdat 'P uuid + 'T "runstart" + 'c (alist-ref 'o pkta) ;; THIS IS WRONG! SHOULD BE 'c + 't (alist-ref 't pkta))))) + (write-pkt pktsdir ack-uuid ack-pkt)))) + pkts))))) + +(define (get-pkts-dir mtconf) + (let ((pktsdirs (configf:lookup mtconf "setup" "pktsdirs")) + (pktsdir (if pktsdirs (car (string-split pktsdirs " ")) #f))) + pktsdir)) + +(if *action* + (case (string->symbol *action*) + ((run remove rerun set-ss archive kill) + (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir"))) + (mtconf (car mtconfdat)) + (pktsdirs (configf:lookup mtconf "setup" "pktsdirs")) + (pktsdir (if pktsdirs (car (string-split pktsdirs " ")) #f)) + (adjargs (hash-table-copy args:arg-hash))) + ;; (for-each + ;; (lambda (key) + ;; (if (not (member key *legal-params*)) + ;; (hash-table-delete! adjargs key))) ;; we need to delete any params intended for mtutil + ;; (hash-table-keys adjargs)) + (let-values (((uuid pkt) + (command-line->pkt *action* adjargs))) + (write-pkt pktsdir uuid pkt)))) + ((dispatch import rungen) + (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir"))) + (mtconf (car mtconfdat)) + (toppath (configf:lookup mtconf "dyndat" "toppath"))) + (case (string->symbol *action*) + ((import) (load-pkts-to-db mtconf)) ;; import pkts + ((rungen) (generate-run-pkts mtconf toppath)) + ((dispatch) (dispatch-commands mtconf toppath))))))) + +(if (or (args:get-arg "-repl") + (args:get-arg "-load")) + (begin + (import extras) ;; might not be needed + ;; (import csi) + (import readline) + (import apropos) + ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ... + + (install-history-file (get-environment-variable "HOME") ".mtutil_history") ;; [homedir] [filename] [nlines]) + (current-input-port (make-readline-port "mtutil> ")) + (if (args:get-arg "-repl") + (repl) + (load (args:get-arg "-load"))))) Index: runconfig.scm ================================================================== --- runconfig.scm +++ runconfig.scm @@ -80,6 +80,68 @@ environ-patt: (conc "(default" (if targ (conc "|" targ ")") ")"))) (debug:print 0 *default-log-port* "WARNING: You do not have a run config file: " runconfigf)))) - + +;; given (a (b c) d) return ((a b d)(a c d)) +;; NOTE: this feels like it has been done before - perhaps with items handling? +;; +(define (runconfig:combinations inlst) + (let loop ((hed (car inlst)) + (tal (cdr inlst)) + (res '())) + ;; (print "res: " res " hed: " hed) + (if (list? hed) + (let ((newres (if (null? res) ;; first time through convert incoming items to list of items + (map list hed) + (apply append + (map (lambda (r) ;; iterate over items in res + (map (lambda (h) ;; iterate over items in hed + (append r (list h))) + hed)) + res))))) + ;; (print "newres1: " newres) + (if (null? tal) + newres + (loop (car tal)(cdr tal) newres))) + (let ((newres (if (null? res) + (list (list hed)) + (map (lambda (r) + (append r (list hed))) + res)))) + ;; (print "newres2: " newres) + (if (null? tal) + newres + (loop (car tal)(cdr tal) newres)))))) + +;; multi-part expand +;; Given a/b,c,d/e,f return a/b/e a/b/f a/c/e a/c/f a/d/e a/d/f +;; +(define (runconfig:expand target) + (let* ((parts (map (lambda (x) + (string-split x ",")) + (string-split target "/")))) + (map (lambda (x) + (string-intersperse x "/")) + (runconfig:combinations parts)))) + +;; multi-target expansion +;; a/b/c/x,y,z a/b/d/x,y => a/b/c/x a/b/c/y a/b/c/z a/b/d/x a/b/d/y +;; +(define (runconfig:expand-target target-strs) + (delete-duplicates + (apply append (map runconfig:expand (string-split target-strs " "))))) + +#| + (if (null? target-strs) + '() + (let loop ((hed (car target-strs)) + (tal (cdr target-strs)) + (res '())) + ;; first break all parts into individual target patterns + (if (string-index hed " ") ;; this is a multi-target target + (let ((newres (append (string-split hed " ") res))) + (runconfig:expand-target newres)) + (if (string-index hed ",") ;; this is a multi-target where one or more parts are comma separated + +|# ADDED rungen.config Index: rungen.config ================================================================== --- /dev/null +++ rungen.config @@ -0,0 +1,7 @@ +[v1.63/tip/dev] +# file: files changes since last run trigger new run +# script: script is called with unix seconds as last parameter (other parameters are preserved) +# +# contour:sensetype runname params +quick:file auto *.scm +quick:script auto checkfossil.sh v1.63