Index: configf.scm ================================================================== --- configf.scm +++ configf.scm @@ -19,21 +19,26 @@ (declare (uses process)) (include "common_records.scm") ;; return list (path fullpath configname) -(define (find-config configname) - (let* ((cwd (string-split (current-directory) "/"))) - (let loop ((dir cwd)) - (let* ((path (conc "/" (string-intersperse dir "/"))) - (fullpath (conc path "/" configname))) - (if (file-exists? fullpath) - (list path fullpath configname) - (let ((remcwd (take dir (- (length dir) 1)))) - (if (null? remcwd) - (list #f #f #f) ;; #f #f) - (loop remcwd)))))))) +(define (find-config configname #!key (toppath #f)) + (if toppath + (let ((cfname (conc toppath "/" configname))) + (if (file-exists? cfname) + (list toppath cfname configname) + (list #f #f #f))) + (let* ((cwd (string-split (current-directory) "/"))) + (let loop ((dir cwd)) + (let* ((path (conc "/" (string-intersperse dir "/"))) + (fullpath (conc path "/" configname))) + (if (file-exists? fullpath) + (list path fullpath configname) + (let ((remcwd (take dir (- (length dir) 1)))) + (if (null? remcwd) + (list #f #f #f) ;; #f #f) + (loop remcwd))))))))) (define (config:assoc-safe-add alist key val) (let ((newalist (filter (lambda (x)(not (equal? key (car x)))) alist))) (append newalist (list (list key val))))) @@ -208,29 +213,31 @@ (loop (configf:read-line inp res) curr-section-name #f #f)))) (else (debug:print 0 "ERROR: problem parsing " path ",\n \"" inl "\"") (set! var-flag #f) (loop (configf:read-line inp res) curr-section-name #f #f)))))))) -(define (find-and-read-config fname #!key (environ-patt #f)) +(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)) (let* ((curr-dir (current-directory)) - (configinfo (find-config fname)) + (configinfo (find-config fname toppath: given-toppath)) (toppath (car configinfo)) (configfile (cadr configinfo))) (if toppath (change-directory toppath)) (let ((configdat (if configfile (read-config configfile #f #t environ-patt: environ-patt) #f))) ;; (make-hash-table)))) (if toppath (change-directory curr-dir)) (list configdat toppath configfile fname)))) (define (config-lookup cfgdat section var) - (let ((sectdat (hash-table-ref/default cfgdat section '()))) - (if (null? sectdat) - #f - (let ((match (assoc var sectdat))) - (if match ;; (and match (list? match)(> (length match) 1)) - (cadr match) - #f)) - ))) + (if (hash-table? cfgdat) + (let ((sectdat (hash-table-ref/default cfgdat section '()))) + (if (null? sectdat) + #f + (let ((match (assoc var sectdat))) + (if match ;; (and match (list? match)(> (length match) 1)) + (cadr match) + #f)) + )) + #f)) (define (configf:section-vars cfgdat section) (let ((sectdat (hash-table-ref/default cfgdat section '()))) (if (null? sectdat) '() Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -41,10 +41,11 @@ (dbexists (file-exists? dbpath)) (db (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) (handler (make-busy-timeout (if (args:get-arg "-override-timeout") (string->number (args:get-arg "-override-timeout")) 36000)))) ;; 136000))) + (debug:print 4 "INFO: dbpath=" dbpath) (sqlite3:set-busy-handler! db handler) (if (not dbexists) (db:initialize db)) db)) Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -47,10 +47,11 @@ (let* ((cmdinfo (read (open-input-string (base64:base64-decode encoded-cmd))))) (setenv "MT_CMDINFO" encoded-cmd) (if (list? cmdinfo) ;; ((testpath /tmp/mrwellan/jazzmind/src/example_run/tests/sqlitespeed) ;; (test-name sqlitespeed) (runscript runscript.rb) (db-host localhost) (run-id 1)) (let* ((testpath (assoc/default 'testpath cmdinfo)) + (top-path (assoc/default 'toppath cmdinfo)) (work-area (assoc/default 'work-area cmdinfo)) (test-name (assoc/default 'test-name cmdinfo)) (runscript (assoc/default 'runscript cmdinfo)) (ezsteps (assoc/default 'ezsteps cmdinfo)) (db-host (assoc/default 'db-host cmdinfo)) @@ -66,11 +67,10 @@ (fullrunscript (if runscript (conc testpath "/" runscript) #f)) (db #f) (rollup-status 0)) (debug:print 2 "Exectuing " test-name " (id: " test-id ") on " (get-host-name)) - (change-directory testpath) ;; apply pre-overrides before other variables. The pre-override vars must not ;; clobbers things from the official sources such as megatest.config and runconfigs.config (if (string? set-vars) (let ((varpairs (string-split set-vars ","))) (debug:print 4 "varpairs: " varpairs) @@ -87,15 +87,16 @@ (setenv "MT_ITEM_INFO" (conc itemdat)) (setenv "MT_RUNNAME" runname) (setenv "MT_MEGATEST" megatest) (setenv "MT_TARGET" target) (if mt-bindir-path (setenv "PATH" (conc (getenv "PATH") ":" mt-bindir-path))) - + (change-directory top-path) (if (not (setup-for-run)) (begin (debug:print 0 "Failed to setup, exiting") (exit 1))) + (change-directory *toppath*) ;; now can find our db (set! db (open-db)) (if (not (args:get-arg "-server")) (server:client-setup db)) ;; (set! *cache-on* #t) @@ -347,12 +348,16 @@ ;; set up the very basics needed for doing anything here. (define (setup-for-run) ;; would set values for KEYS in the environment here for better support of env-override but ;; have chicken/egg scenario. need to read megatest.config then read it again. Going to - ;; pass on that idea for now. - (set! *configinfo* (find-and-read-config (if (args:get-arg "-config")(args:get-arg "-config") "megatest.config") environ-patt: "env-override")) + ;; pass on that idea for now + ;; special case + (set! *configinfo* (find-and-read-config + (if (args:get-arg "-config")(args:get-arg "-config") "megatest.config") + environ-patt: "env-override" + given-toppath: (get-environment-variable "MT_RUN_AREA_HOME"))) (set! *configdat* (if (car *configinfo*)(car *configinfo*) #f)) (set! *toppath* (if (car *configinfo*)(cadr *configinfo*) #f)) (if *toppath* (setenv "MT_RUN_AREA_HOME" *toppath*) ;; to be deprecated (debug:print 0 "ERROR: failed to find the top path to your run setup.")) @@ -507,10 +512,18 @@ ;; - could be ssh to host from hosts table (update regularly with load) ;; - could be netbatch ;; (launch-test db (cadr status) test-conf)) (define (launch-test db run-id runname test-conf keyvallst test-name test-path itemdat params) (change-directory *toppath*) + (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute" + (list ;; (list "MT_TEST_RUN_DIR" work-area) + (list "MT_RUN_AREA_HOME" *toppath*) + (list "MT_TEST_NAME" test-name) + ;; (list "MT_ITEM_INFO" (conc itemdat)) + (list "MT_RUNNAME" runname) + ;; (list "MT_TARGET" mt_target) + )) (let* ((useshell (config-lookup *configdat* "jobtools" "useshell")) (launcher (config-lookup *configdat* "jobtools" "launcher")) (runscript (config-lookup test-conf "setup" "runscript")) (ezsteps (> (length (hash-table-ref/default test-conf "ezsteps" '())) 0)) ;; don't send all the steps, could be big (diskspace (config-lookup test-conf "requirements" "diskspace")) @@ -537,10 +550,11 @@ (fullcmd #f) ;; (define a (with-output-to-string (lambda ()(write x)))) (mt-bindir-path #f) (item-path (item-list->path itemdat)) (testinfo (rdb:get-test-info db run-id test-name item-path)) (test-id (db:test-get-id testinfo)) + (mt_target (string-intersperse (map cadr keyvallst) "/")) (debug-param (if (args:get-arg "-debug")(list "-debug" (args:get-arg "-debug")) '()))) (if hosts (set! hosts (string-split hosts))) ;; set the megatest to be called on the remote host (if (not remote-megatest)(set! remote-megatest local-megatest)) ;; "megatest")) (set! mt-bindir-path (pathname-directory remote-megatest)) @@ -557,19 +571,20 @@ (create-directory work-area #t) (debug:print 0 "WARNING: No disk work area specified - running in the test directory under tmp_run"))) (set! cmdparms (base64:base64-encode (with-output-to-string (lambda () ;; (list 'hosts hosts) (write (list (list 'testpath test-path) + (list 'toppath *toppath*) (list 'work-area work-area) (list 'test-name test-name) (list 'runscript runscript) (list 'run-id run-id ) (list 'test-id test-id ) (list 'itemdat itemdat ) (list 'megatest remote-megatest) (list 'ezsteps ezsteps) - (list 'target (string-intersperse (map cadr keyvallst) "/")) + (list 'target mt_target) (list 'env-ovrd (hash-table-ref/default *configdat* "env-override" '())) (list 'set-vars (if params (hash-table-ref/default params "-setvars" #f))) (list 'runname runname) (list 'mt-bindir-path mt-bindir-path))))))) ;; (string-intersperse keyvallst " ")))) ;; clean out step records from previous run if they exist @@ -594,22 +609,28 @@ (let* ((commonprevvals (alist->env-vars (hash-table-ref/default *configdat* "env-override" '()))) (testprevvals (alist->env-vars (hash-table-ref/default test-conf "pre-launch-env-overrides" '()))) (miscprevvals (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute" - (append (list (list "MT_TEST_NAME" test-name) + (append (list (list "MT_TEST_RUN_DIR" work-area) + (list "MT_TEST_NAME" test-name) (list "MT_ITEM_INFO" (conc itemdat)) - (list "MT_RUNNAME" runname)) + (list "MT_RUNNAME" runname) + (list "MT_TARGET" mt_target) + ) itemdat))) (launch-results (apply cmd-run-proc-each-line (if useshell (string-intersperse fullcmd " ") (car fullcmd)) print (if useshell '() (cdr fullcmd))))) ;; launcher fullcmd)));; (apply cmd-run-proc-each-line launcher print fullcmd))) ;; (cmd-run->list fullcmd)) + (with-output-to-file "mt_launch.log" + (lambda () + (apply print launch-results))) (debug:print 2 "Launching completed, updating db") (debug:print 2 "Launch results: " launch-results) (if (not launch-results) (begin (print "ERROR: Failed to run " (string-intersperse fullcmd " ") ", exiting now") @@ -621,7 +642,8 @@ (process-signal (current-process-id) signal/kill) )) (alist->env-vars miscprevvals) (alist->env-vars testprevvals) (alist->env-vars commonprevvals) - launch-results))) + launch-results)) + (change-directory *toppath*)) Index: megatest-version.scm ================================================================== --- megatest-version.scm +++ megatest-version.scm @@ -1,7 +1,7 @@ ;; Always use two digit decimal ;; 1.01, 1.02...1.10,1.11 ... 1.99,2.00.. (declare (unit megatest-version)) -(define megatest-version 1.43) +(define megatest-version 1.4402) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -81,12 +81,14 @@ fields category,variable,value,comment Queries -list-runs patt : list runs matching pattern \"patt\", % is the wildcard -showkeys : show the keys used in this megatest setup - -test-path targpatt : get the most recent test path(s) matching targpatt e.g. %/%... + -test-files targpatt : get the most recent test path/file matching targpatt e.g. %/%... returns list sorted by age ascending, see examples below + -test-paths : get the test paths matching target, runname, item and test + patterns. Misc -rebuild-db : bring the database schema up to date -update-meta : update the tests metadata for all tests -env2file fname : write the environment to fname.csh and fname.sh @@ -106,12 +108,12 @@ -gen-megatest-area : create a skeleton megatest area. You will be prompted for paths -gen-megatest-test : create a skeleton megatest test. You will be prompted for info Examples -# Get test path, the '.' is required, could use '*' or a specific path/file -megatest -test-path . -target ubuntu/n%/no% :runname w49% -testpatt test_mt% +# Get test path, use '.' to get a single path or a specific path/file pattern +megatest -test-files 'logs/*.log' -target ubuntu/n%/no% :runname w49% -testpatt test_mt% Called as " (string-intersperse (argv) " "))) ;; -gui : start a gui interface ;; -config fname : override the runconfig file with fname @@ -157,10 +159,11 @@ "-setvars" "-set-state-status" "-debug" ;; for *verbosity* > 2 "-gen-megatest-test" "-override-timeout" + "-test-files" ;; -test-paths is for listing all ) (list "-h" "-force" "-xterm" "-showkeys" @@ -174,11 +177,10 @@ "-repl" "-lock" "-unlock" ;; queries "-test-paths" ;; get path(s) to a test, ordered by youngest first - "-test-path" ;; -test-paths is deprecated "-runall" ;; run all tests "-remove-runs" "-usequeue" "-rebuild-db" @@ -451,11 +453,11 @@ ;;====================================================================== ;; Get paths to tests ;;====================================================================== ;; Get test paths matching target, runname, testpatt, and itempatt -(if (or (args:get-arg "-test-path")(args:get-arg "-test-paths")) +(if (or (args:get-arg "-test-files")(args:get-arg "-test-paths")) ;; if we are in a test use the MT_CMDINFO data (if (getenv "MT_CMDINFO") (let* ((startingdir (current-directory)) (cmdinfo (read (open-input-string (base64:base64-decode (getenv "MT_CMDINFO"))))) (testpath (assoc/default 'testpath cmdinfo)) @@ -465,38 +467,39 @@ (run-id (assoc/default 'run-id cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) (db #f) (state (args:get-arg ":state")) (status (args:get-arg ":status")) - (target (args:get-arg "-target"))) - (change-directory testpath) + (target (args:get-arg "-target")) + (toppath (assoc/default 'toppath cmdinfo))) + (change-directory toppath) (if (not target) (begin (debug:print 0 "ERROR: -target is required.") (exit 1))) (if (not (setup-for-run)) (begin - (debug:print 0 "Failed to setup, giving up on -test-path, exiting") + (debug:print 0 "Failed to setup, giving up on -test-paths or -test-files, exiting") (exit 1))) (set! db (open-db)) (if (not (args:get-arg "-server")) (server:client-setup db)) (let* ((itempatt (args:get-arg "-itempatt")) (keys (rdb:get-keys db)) (keynames (map key:get-fieldname keys)) - (paths (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-path")))) + (paths (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-files")))) (set! *didsomething* #t) (for-each (lambda (path) (print path)) paths))) ;; else do a general-run-call (general-run-call - "-test-path" + "-test-files" "Get paths to test" (lambda (db target runname keys keynames keyvallst) (let* ((itempatt (args:get-arg "-itempatt")) - (paths (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-path")))) + (paths (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-files")))) (for-each (lambda (path) (print path)) paths)))))) ;;====================================================================== Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -209,12 +209,16 @@ ;; now add non-directly referenced dependencies (i.e. waiton) (if (not (null? test-names)) (let loop ((hed (car test-names)) (tal (cdr test-names))) ;; 'return-procs tells the config reader to prep running system but return a proc (let* ((config (tests:get-testconfig hed 'return-procs)) - (waitons (string-split (let ((w (config-lookup config "requirements" "waiton"))) - (if w w ""))))) + (waitons (if config (string-split (let ((w (config-lookup config "requirements" "waiton"))) + (if w w ""))) + (begin + (debug:print 0 "ERROR: non-existent required test \"" hed "\"") + (sqlite3:finalize! db) + (exit 1))))) ;; check for hed in waitons => this would be circular, remove it and issue an ;; error (if (member hed waitons) (begin (debug:print 0 "ERROR: test " hed " has listed itself as a waiton, please correct this!") @@ -368,11 +372,11 @@ ;; we made new tal by sticking hed at the back of the list (loop (car newtal)(cdr newtal))) ;; the waiton is FAIL so no point in trying to run hed ever again (if (not (null? tal)) (begin - (debug:print 1 "WARN: Dropping test " (db:test-get-test-name hed) "/" (db:test-get-item-path hed) + (debug:print 1 "WARN: Dropping test " (db:test-get-testname hed) "/" (db:test-get-item-path hed) " from the launch list as it has prerequistes that are FAIL") (loop (car tal)(cdr tal))))))))) ;; case where an items came in as a list been processed ((and (list? items) ;; thus we know our items are already calculated ADDED tests/fullrun/ez_pass_linked/testconfig Index: tests/fullrun/ez_pass_linked/testconfig ================================================================== --- /dev/null +++ tests/fullrun/ez_pass_linked/testconfig @@ -0,0 +1,13 @@ +[setup] + +[ezsteps] +lookittmp ls /tmp +lookithome ls /home + +[test_meta] +author matt +owner bob +description This test runs a single ezstep which is expected to pass, no logpro file. + +tags first,single +reviewed 09/10/2011, by Matt Index: tests/fullrun/runconfigs.config ================================================================== --- tests/fullrun/runconfigs.config +++ tests/fullrun/runconfigs.config @@ -1,6 +1,6 @@ -[include common_runconfigs.config] +[include #{getenv MT_RUN_AREA_HOME}/common_runconfigs.config] WACKYVAR0 #{get ubuntu/nfs/none CURRENT} WACKYVAR1 #{scheme (args:get-arg "-target")} [default/ubuntu/nfs] ADDED tests/fullrun/tests/ez_pass_linked Index: tests/fullrun/tests/ez_pass_linked ================================================================== --- /dev/null +++ tests/fullrun/tests/ez_pass_linked @@ -0,0 +1,1 @@ +../ez_pass_linked/