Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -44,10 +44,11 @@ megatest.scm : transport-mode.scm dashboard.scm : dashboard-transport-mode.scm # dbmod.import.o is just a hack here mofiles/dbfile.o : mofiles/debugprint.o mofiles/commonmod.o dbmod.import.o +configf.o : commonmod.import.o db.o : dbmod.import.o mofiles/debugprint.o : mofiles/mtargs.o # ftail.scm rmtmod.scm commonmod.scm removed # MSRCFILES = ducttape-lib.scm pkts.scm stml2.scm cookie.scm mutils.scm \ @@ -106,12 +107,12 @@ @echo $(MTESTHASH) dboard : $(OFILES) $(GOFILES) dashboard.scm $(MOFILES) $(MOIMPFILES) megatest-version.scm megatest-fossil-hash.scm csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) $(MOFILES) $(MOIMPFILES) -o dboard -mtut: $(OFILES) $(MOFILES) megatest-fossil-hash.scm mtut.scm megatest-version.scm - csc $(CSCOPTS) $(OFILES) $(MOFILES) mtut.scm -o mtut +mtut: $(OFILES) $(MOFILES) $(MOIMPFILES) megatest-fossil-hash.scm mtut.scm megatest-version.scm + csc $(CSCOPTS) $(OFILES) $(MOFILES) $(MOIMPFILES) mtut.scm -o mtut # include makefile.inc TCMTOBJS = \ api.o \ @@ -369,22 +370,22 @@ $(INSTALL) lib/libxcb-xlib.so.0 $(PREFIX)/bin/.$(ARCHSTR)/lib/libxcb-xlib.so.0; \ fi install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \ $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ - $(PREFIX)/bin/.$(ARCHSTR)/mtexec $(PREFIX)/bin/mtexec $(PREFIX)/bin/serialize-env \ $(PREFIX)/bin/nbfind $(PREFIX)/bin/mtrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ $(PREFIX)/bin/mt-old-to-new.sh $(PREFIX)/bin/mt-new-to-old.sh \ $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun \ $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun $(PREFIX)/bin/mtutil \ $(PREFIX)/bin/tcmt $(PREFIX)/share/db/mt-pg.sql \ $(PREFIX)/share/js/jquery-3.1.0.slim.min.js \ $(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so \ $(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so.0 \ $(PREFIX)/bin/.$(ARCHSTR)/lib/libxcb-xlib.so.0 + # $(PREFIX)/bin/.$(ARCHSTR)/ndboard - +# $(PREFIX)/bin/.$(ARCHSTR)/mtexec $(PREFIX)/bin/mtexec $(PREFIX)/bin/serialize-env \ # $(PREFIX)/bin/newdashboard $(PREFIX)/bin/.$(ARCHSTR) : mkdir -p $(PREFIX)/bin/.$(ARCHSTR) mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib Index: archive.scm ================================================================== --- archive.scm +++ archive.scm @@ -21,10 +21,12 @@ (use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 format md5 message-digest srfi-18) (declare (unit archive)) (declare (uses db)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "db_records.scm") ;;====================================================================== Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -27,10 +27,12 @@ (declare (unit client)) (declare (uses common)) (declare (uses db)) (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. +(declare (uses commonmod)) +(import commonmod) (module client * ) Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -24,10 +24,12 @@ matchable regex posix (srfi 18) extras ;; tcp (prefix nanomsg nmsg:) (prefix sqlite3 sqlite3:) pkts (prefix dbi dbi:) ) +(use posix-extras pathname-expand files) + (declare (unit common)) (declare (uses commonmod)) (import commonmod) @@ -210,12 +212,10 @@ ;; Miscellaneous (define *triggers-mutex* (make-mutex)) ;; block overlapping processing of triggers (define *numcpus-cache* (make-hash-table)) -(use posix-extras pathname-expand files) - ;; this plugs a hole in posix-extras in recent chicken versions > 4.9) (let-values (( (chicken-release-number chicken-major-version) (apply values (map string->number (take @@ -722,13 +722,10 @@ ;; (begin ;; (hash-table-set! *common:denoise* key currtime) ;; #t) ;; #f))) -(define (common:get-megatest-exe) - (or (getenv "MT_MEGATEST") "megatest")) - (define (common:read-encoded-string instr) (handle-exceptions exn (handle-exceptions exn @@ -1046,26 +1043,10 @@ (define (common:get-disks #!key (configf #f)) (hash-table-ref/default (or configf (read-config "megatest.config" #f #t)) "disks" '("none" ""))) -;;====================================================================== -;; return first command that exists, else #f -;; -(define (common:which cmds) - (if (null? cmds) - #f - (let loop ((hed (car cmds)) - (tal (cdr cmds))) - (let ((res (with-input-from-pipe (conc "which " hed) read-line))) - (if (and (string? res) - (common:file-exists? res)) - res - (if (null? tal) - #f - (loop (car tal)(cdr tal)))))))) - (define (common:get-install-area) (let ((exe-path (car (argv)))) (if (common:file-exists? exe-path) (handle-exceptions exn Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -34,11 +34,13 @@ extras files matchable md5 message-digest + pathname-expand posix + posix-extras regex regex-case srfi-1 srfi-18 srfi-69 @@ -152,10 +154,54 @@ ;;====================================================================== ;; misc conversion, data manipulation functions ;;====================================================================== +;;====================================================================== +;; return first command that exists, else #f +;; +(define (common:which cmds) + (if (null? cmds) + #f + (let loop ((hed (car cmds)) + (tal (cdr cmds))) + (let ((res (with-input-from-pipe (conc "which " hed) read-line))) + (if (and (string? res) + (file-exists? res)) + res + (if (null? tal) + #f + (loop (car tal)(cdr tal)))))))) + +(define (common:get-megatest-exe) + (let* ((mtexe (or (get-environment-variable "MT_MEGATEST") + (common:which '("megatest")) + "megatest"))) + (if (file-exists? mtexe) + (realpath mtexe) + mtexe))) + +(define (common:get-megatest-exe-dir) + (let* ((mtexe (common:get-megatest-exe))) + (pathname-directory mtexe))) + +;; more generic and comprehensive version of get-megatest-exe +;; +(define (common:get-mtexe) + (let* ((mtpathdir (common:get-megatest-exe-dir))) + (or (common:get-megatest-exe) + (if mtpathdir + (conc mtpathdir"/megatest") + #f) + "megatest"))) + +(define (common:get-megatest-exe-path) + (let* ((mtpathdir (common:get-megatest-exe-dir))) + (conc mtpathdir":"(get-environment-variable "PATH") ":."))) + +(define (realpath x) (resolve-pathname (pathname-expand (or x "/dev/null")) )) + ;; if it looks like a number -> convert it to a number, else return it ;; (define (lazy-convert inval) (let* ((as-num (if (string? inval)(string->number inval) #f))) (or as-num inval))) Index: configf.scm ================================================================== --- configf.scm +++ configf.scm @@ -25,10 +25,13 @@ (use regex regex-case matchable) ;; directory-utils) (declare (unit configf)) (declare (uses process)) (declare (uses env)) (declare (uses keys)) +(declare (uses commonmod)) +(declare (uses commonmod.import)) +(import commonmod) (include "common_records.scm") ;; return list (path fullpath configname) (define (find-config configname #!key (toppath #f)) @@ -97,10 +100,12 @@ (define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget|scm|sh|rp|gv|g|mtrah)\\s+([^\\}\\{]*)\\}(.*)")) (define (configf:system ht cmd) (system cmd) ) + +(define configf:imports "(import commonmod)") (define (configf:process-line l ht allow-system #!key (linenum #f)) (let loop ((res l)) (if (string? res) (let ((matchdat (string-search configf:var-expand-regex res))) @@ -111,11 +116,11 @@ (poststr (list-ref matchdat 4)) (result #f) (start-time (current-seconds)) (cmdsym (string->symbol cmdtype)) (fullcmd (case cmdsym - ((scheme scm) (conc "(lambda (ht)" cmd ")")) + ((scheme scm) (conc "(lambda (ht)(begin " configf:imports cmd "))")) ((system) (conc "(lambda (ht)(configf:system ht \"" cmd "\"))")) ((shell sh) (conc "(lambda (ht)(string-translate (shell \"" cmd "\") \"\n\" \" \"))")) ((realpath rp)(conc "(lambda (ht)(common:nice-path \"" cmd "\"))")) ((getenv gv) (conc "(lambda (ht)(get-environment-variable \"" cmd "\"))")) ((mtrah) (conc "(lambda (ht)" Index: dashboard-guimonitor.scm ================================================================== --- dashboard-guimonitor.scm +++ dashboard-guimonitor.scm @@ -34,10 +34,12 @@ (declare (unit dashboard-guimonitor)) (declare (uses common)) (declare (uses keys)) (declare (uses db)) (declare (uses tasks)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "db_records.scm") (include "run_records.scm") (include "task_records.scm") Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -47,10 +47,13 @@ (declare (uses subrun)) (declare (uses mt)) (declare (uses dbmod)) ;; (declare (uses dbmemmod)) (declare (uses dbfile)) +(declare (uses commonmod)) +(declare (uses commonmod.import)) +(import commonmod) (import dbmod dbfile) (include "common_records.scm") (include "db_records.scm") Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -31,10 +31,12 @@ (declare (uses dbfile)) (declare (uses keys)) (declare (uses ods)) (declare (uses client)) (declare (uses mt)) +(declare (uses commonmod)) +(import commonmod) (use (srfi 18) extras tcp stack Index: diff-report.scm ================================================================== --- diff-report.scm +++ diff-report.scm @@ -17,10 +17,12 @@ ;; (declare (unit diff-report)) (declare (uses common)) (declare (uses rmt)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (use matchable) (use fmt) (use ducttape-lib) Index: ezsteps.scm ================================================================== --- ezsteps.scm +++ ezsteps.scm @@ -27,10 +27,12 @@ (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) ;; (declare (uses sdb)) ;; (declare (uses filedb)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -38,11 +40,11 @@ ;;(rmt:get-test-info-by-id run-id test-id) -> testdat ;; TODO: deprecate me in favor of ezsteps.scm ;; -(define (launch:runstep ezstep run-id test-id exit-info m tal testconfig all-steps-dat) +(define (ezsteps:runstep ezstep run-id test-id exit-info m tal testconfig all-steps-dat) (let* ((stepname (car ezstep)) ;; do stuff to run the step (stepinfo (cadr ezstep)) ;; (let ((info (cadr ezstep))) ;; (if (proc? info) "" info))) ;; (stepproc (let ((info (cadr ezstep))) @@ -63,11 +65,12 @@ (script "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!\ (logpro-file (conc stepname ".logpro")) (html-file (conc stepname ".html")) (dat-file (conc stepname ".dat")) (tconfig-logpro (configf:lookup testconfig "logpro" stepname)) - (logpro-used (common:file-exists? logpro-file))) + (logpro-used (common:file-exists? logpro-file)) + (mtexepath (common:get-megatest-exe-path))) (setenv "MT_STEP_NAME" stepname) (hash-table-set! all-steps-dat stepname `((params . ,paramparts))) (debug:print 0 *default-log-port* "stepparts: " stepparts ", stepparams: " stepparams ", paramparts: " paramparts ", subrun: " subrun ", stepcmd: " stepcmd) @@ -96,11 +99,11 @@ (debug:print 4 *default-log-port* "script: " script) (rmt:teststep-set-status! run-id test-id stepname "start" "-" #f #f) ;; now launch the actual process (call-with-environment-variables - (list (cons "PATH" (conc (get-environment-variable "PATH") ":."))) + (list (cons "PATH" mtexepath)) (lambda () ;; (process-run "/bin/bash" "-c" "exec ls -l /tmp/foobar > /tmp/delme-more.log 2>&1") (let* ((cmd (conc stepcmd " > " stepname ".log 2>&1")) ;; >outfile 2>&1 (pid #f)) (let ((proc (lambda () (set! pid (process-run "/bin/bash" (list "-c" cmd)))))) Index: index-tree.scm ================================================================== --- index-tree.scm +++ index-tree.scm @@ -29,10 +29,12 @@ (declare (uses lock-queue)) (declare (uses db)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") Index: items.scm ================================================================== --- items.scm +++ items.scm @@ -21,10 +21,12 @@ ;; (temperature "cool medium hot") ;; (season "summer winter fall spring"))) (declare (unit items)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") ;; Puts out all combinations (define (process-itemlist hierdepth curritemkey itemlist) Index: keys.scm ================================================================== --- keys.scm +++ keys.scm @@ -24,10 +24,12 @@ (use sqlite3 srfi-1 posix regex regex-case srfi-69) (import (prefix sqlite3 sqlite3:)) (declare (unit keys)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) (include "key_records.scm") (include "common_records.scm") (define (keys->keystr keys) ;; => key1,key2,key3,additiona1, ... Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -188,11 +188,11 @@ (tal (cdr ezstepslst)) (prevstep #f)) (debug:print-info 0 *default-log-port* "Processing ezstep \"" (string-intersperse ezstep " ") "\"") ;; check exit-info (vector-ref exit-info 1) (if (launch:einf-exit-status exit-info) ;; (vector-ref exit-info 1) - (let* ((logpro-used (launch:runstep ezstep run-id test-id exit-info m tal testconfig all-steps-dat)) + (let* ((logpro-used (ezsteps:runstep ezstep run-id test-id exit-info m tal testconfig all-steps-dat)) (stepname (car ezstep)) (stepparms (hash-table-ref all-steps-dat stepname))) (setenv "MT_STEP_NAME" stepname) (pp (hash-table->alist all-steps-dat)) ;; if logpro-used read in the stepname.dat file Index: lock-queue.scm ================================================================== --- lock-queue.scm +++ lock-queue.scm @@ -19,10 +19,12 @@ (use (prefix sqlite3 sqlite3:) srfi-18) (declare (unit lock-queue)) (declare (uses common)) (declare (uses tasks)) +(declare (uses commonmod)) +(import commonmod) ;;====================================================================== ;; attempt to prevent overlapping updates of rollup files by queueing ;; update requests in an sqlite db ;;====================================================================== Index: mlaunch.scm ================================================================== --- mlaunch.scm +++ mlaunch.scm @@ -28,6 +28,8 @@ (use sqlite3 srfi-1 posix regex regex-case srfi-69 format) (declare (unit mlaunch)) (declare (uses db)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) Index: monitor.scm ================================================================== --- monitor.scm +++ monitor.scm @@ -23,10 +23,12 @@ (declare (unit runs)) (declare (uses db)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") Index: mtexec.scm ================================================================== --- mtexec.scm +++ mtexec.scm @@ -29,10 +29,12 @@ ;; (declare (uses common)) (declare (uses margs)) (declare (uses configf)) ;; (declare (uses rmt)) +(declare (uses commonmod)) +(import commonmod) ;; (use ducttape-lib) (include "megatest-version.scm") (include "megatest-fossil-hash.scm") Index: mtut.scm ================================================================== --- mtut.scm +++ mtut.scm @@ -30,10 +30,13 @@ (declare (uses common)) (declare (uses margs)) (declare (uses configf)) ;; (declare (uses rmt)) +(declare (uses commonmod)) +(declare (uses commonmod.import)) +(import commonmod) (use ducttape-lib) (include "megatest-fossil-hash.scm") Index: newdashboard.scm ================================================================== --- newdashboard.scm +++ newdashboard.scm @@ -29,10 +29,12 @@ (prefix dbi dbi:)) (declare (uses common)) (declare (uses megatest-version)) (declare (uses margs)) +(declare (uses commonmod)) +(import commonmod) ;; (declare (uses launch)) ;; (declare (uses gutils)) ;; (declare (uses db)) ;; (declare (uses server)) Index: ods.scm ================================================================== --- ods.scm +++ ods.scm @@ -17,10 +17,12 @@ ;; (use csv-xml regex) (declare (unit ods)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) (define ods:dirs '("Configurations2" "Configurations2/toolpanel" "Configurations2/menubar" Index: runconfig.scm ================================================================== --- runconfig.scm +++ runconfig.scm @@ -22,10 +22,12 @@ (use format directory-utils) (declare (unit runconfig)) (declare (uses common)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (define (runconfig:read fname target environ-patt) (let ((ht (make-hash-table))) Index: subrun.scm ================================================================== --- subrun.scm +++ subrun.scm @@ -23,17 +23,20 @@ call-with-environment-variables) (declare (unit subrun)) ;;(declare (uses runs)) (declare (uses db)) (declare (uses common)) +(declare (uses commonmod)) ;;(declare (uses items)) ;;(declare (uses runconfig)) ;;(declare (uses tests)) ;;(declare (uses server)) (declare (uses mt)) ;;(declare (uses archive)) ;; (declare (uses filedb)) + +(import commonmod) ;(include "common_records.scm") ;;(include "key_records.scm") (include "db_records.scm") ;; provides db:test-get-id ;;(include "run_records.scm") @@ -135,11 +138,11 @@ (subrun:unset-subrun-removed test-run-dir)) (let* ((log-prefix "run") (switches (subrun:selector+log-switches test-run-dir log-prefix)) (run-wait (equal? run-mode "yes")) - (cmd (conc "megatest " sub-cmd " " switches" " + (cmd (conc (common:get-mtexe)" "sub-cmd" "switches" " (if run-wait "-run-wait " "")))) cmd)) (define (subrun:sanitize-path inpath) @@ -232,20 +235,24 @@ (list (car x) (cdr x))) switch-alist)) " "))) res)) +;; NOTE: Here we run sub megatest but this is not intended for one version +;; of megatest to test another version. Thus we propagate the (define (subrun:exec-sub-megatest test-run-dir action-switches-str log-prefix) - (let* ((selector-switches (subrun:selector+log-switches test-run-dir log-prefix)) - (cmd (conc "megatest " selector-switches " " action-switches-str )) + (let* ((mtpathdir (common:get-megatest-exe-dir)) + (mtexe (common:get-mtexe)) + (selector-switches (subrun:selector+log-switches test-run-dir log-prefix)) + (cmd (conc mtexe" "selector-switches" "action-switches-str )) (pid #f) (proc (lambda () (debug:print-info 0 *default-log-port* "Running sub megatest command: "cmd) ;;(set! pid (process-run "/usr/bin/xterm" (list )))))) (set! pid (process-run "/bin/bash" (list "-c" cmd)))))) (call-with-environment-variables - (list (cons "PATH" (conc (get-environment-variable "PATH") ":."))) + (list (cons "PATH" (common:get-megatest-exe-path))) (lambda () (common:without-vars proc "^MT_.*"))) (let processloop ((i 0)) (let-values (((pid-val exit-status exit-code)(process-wait pid #t))) (if (eq? pid-val 0) Index: tasks.scm ================================================================== --- tasks.scm +++ tasks.scm @@ -25,10 +25,12 @@ (declare (uses dbfile)) (declare (uses db)) (declare (uses rmt)) (declare (uses common)) (declare (uses pgdb)) +(declare (uses commonmod)) +(import commonmod) (import dbfile) ;; (import pgdb) ;; pgdb is a module (include "task_records.scm") Index: tcmt.scm ================================================================== --- tcmt.scm +++ tcmt.scm @@ -30,10 +30,12 @@ (declare (uses margs)) (declare (uses rmt)) (declare (uses common)) ;; (declare (uses megatest-version)) +(declare (uses commonmod)) +(import commonmod) (include "megatest-version.scm") (include "megatest-fossil-hash.scm") (include "db_records.scm") Index: tcp-transportmod.scm ================================================================== --- tcp-transportmod.scm +++ tcp-transportmod.scm @@ -167,11 +167,11 @@ ;; (match res ((status errmsg result meta) (if (equal? result server-id) (begin - (debug:print 0 *default-log-port* "Ping to "host":"port" successful.") + ;; (debug:print 0 *default-log-port* "Ping to "host":"port" successful.") #t) ;; then we are good (begin (debug:print 0 *default-log-port* "WARNING: server-id does not match, expected: "server-id", got: "result) #f))) (else Index: tdb.scm ================================================================== --- tdb.scm +++ tdb.scm @@ -32,10 +32,12 @@ (declare (uses keys)) (declare (uses ods)) (declare (uses client)) (declare (uses mt)) (declare (uses db)) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "db_records.scm") (include "key_records.scm") (include "run_records.scm")