Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -70,13 +70,11 @@ Usage: megatest [options] -h : this help -version : print megatest version (currently " megatest-version ") Launching and managing runs - -runall : run all tests that are not state COMPLETED and status PASS, - CHECK or KILLED - -runtests tst1,tst2 ... : run tests + -runall : run all tests or as specified by -testpatt -remove-runs : remove the data for a run, requires -runname and -testpatt Optionally use :state and :status -set-state-status X,Y : set state to X and status to Y, requires controls per -remove-runs -rerun FAIL,WARN... : force re-run for tests with specificed status(s) -lock : lock run specified by target and runname @@ -282,11 +280,12 @@ "-get-run-status" ;; queries "-test-paths" ;; get path(s) to a test, ordered by youngest first - "-runall" ;; run all tests + "-runall" ;; run all tests, respects -testpatt + "-run" ;; alias for -runall "-remove-runs" "-rebuild-db" "-cleanup-db" "-rollup" "-update-meta" @@ -1127,11 +1126,11 @@ ;; put task in deferred queue ;; if still ok to run tasks ;; process deferred tasks per above steps ;; run all tests are are Not COMPLETED and PASS or CHECK -(if (args:get-arg "-runall") +(if (or (args:get-arg "-runall")(args:get-arg "-run")) (general-run-call "-runall" "run all tests" (lambda (target runname keys keyvals) (runs:run-tests target Index: tests/fdktestqa/fdk.config ================================================================== --- tests/fdktestqa/fdk.config +++ tests/fdktestqa/fdk.config @@ -27,10 +27,10 @@ server-query-threshold 0 [jobtools] # launcher nbq -P ch_vp -C SLES11_EM64T_4G -Q /ciaf/fdk -# launcher nbfake -# maxload 4 +launcher nbfake +maxload 4 -launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log +# launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log ADDED tests/release/Makefile Index: tests/release/Makefile ================================================================== --- /dev/null +++ tests/release/Makefile @@ -0,0 +1,10 @@ + + +dashboard : compile + dashboard -rows 24 & + +compile : runs + cd ../..;make -j install + +runs : + mkdir -p runs ADDED tests/release/tests/fullrun/results.logpro Index: tests/release/tests/fullrun/results.logpro ================================================================== --- /dev/null +++ tests/release/tests/fullrun/results.logpro @@ -0,0 +1,140 @@ +;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com +;; +;; License GPL. + +(define logbody "LogFileBody") + +(define pass-specs '( ;; testname num-expected max-runtime + ("exit_0" 1 20) + ("ezlog_fail_then_pass" 1 20) + ("ezlog_pass" 1 20) + ("ez_pass" 1 20) + ("lineitem_pass" 1 20) + ("priority_1" 1 20) + ("priority_10" 1 20) + ("priority_10_waiton_1" 1 20) + ("priority_3" 1 20) + ("priority_4" 1 20) + ;; ("priority_5" 1 20) + ("priority_6" 1 20) +;; ("priority_7" 1 20) + ("priority_8" 1 20) + ("priority_9" 1 20) + ("runfirst" 7 20) + ("singletest" 1 20) + ("singletest2" 1 20) + ("special" 1 20) + ("sqlitespeed" 10 20) + ("test1" 1 20) + ("test2" 6 20) + ("test_mt_vars" 6 20) + )) + +(define fail-specs '( ;; testname num-expected max-runtime + ("exit_1" 1 20) + ("ez_exit2_fail" 1 20) + ("ez_fail" 1 20) + ("ez_fail_quick" 1 20) + ("ezlog_fail" 1 20) + ("lineitem_fail" 1 20) + ("logpro_required_fail" 1 20) + ("manual_example" 1 20) + ("neverrun" 1 20))) + +(define warn-specs '(("ezlog_warn" 1 20))) + +(define nost-specs '(("wait_no_items1" 1 20) + ("wait_no_items2" 1 20) + ("wait_no_items3" 1 20) + ("wait_no_items4" 1 20) + ;; ("no_items" 1 20) + )) + +(define (check-one-test estate estatus testname count runtime) + (let* ((rxe (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s"))) + (msg1 (conc testname " expecting count of " count)) + (msg2 (conc testname " expecting runtime less than " runtime))) + (expect:required in logbody = count msg1 rxe) + ;;(expect:value in logbody count < msg2 rxe) + )) + +;; Special cases +;; +(expect:ignore in logbody >= 0 "db_sync test might not have run" #/Test: db_sync/) +(expect:ignore in logbody >= 0 "all_toplevel may not yet be done" #/Test: all_toplevel/) +(expect:error in logbody = 0 "tests left in RUNNING state" #/State: RUNNING/) +(expect:required in logbody = 1 "priority_2 is KILLED" #/Test: priority_2\s+State: KILLED\s+Status: KILLED/) +(expect:required in logbody = 1 "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +(expect:required in logbody = 1 "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +(expect:required in logbody = 1 "testxz has 1 NOT_STARTED test" #/Test: testxz\s+State: NOT_STARTED/) +(expect:required in logbody = 1 "no items" #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/) +(expect:warning in logbody = 1 "dynamic waiton" #/Test: dynamic_waiton/) +(expect:required in logbody = 29 "blocktestxz has 29 tests" #/Test: blocktestxz/) + +;; General cases +;; +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "PASS" testdat)) + pass-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "FAIL" testdat)) + fail-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "WARN" testdat)) + warn-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat)) + nost-specs) + +;; Catch all. +;; +(expect:error in logbody = 0 "Tests not accounted for" #/Test: /) + + +;; ;; define your hooks +;; (hook:first-error "echo \"Error hook activated: #{escaped errmsg}\"") +;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"") +;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"") +;; +;; ;; first ensure your run at least started +;; ;; +;; (trigger "Init" #/This is a header/) +;; (trigger "InitEnd" #/^\s*$/) +;; (section "Init" "Init" "InitEnd") +;; +;; (trigger "Body" #/^.*$/) ;; anything starts the body +;; ;; (trigger "EndBody" #/This had better never match/) +;; +;; (section "Body" "Body" "EndBody") +;; +;; (trigger "Blah2" #/^begin Blah2/) +;; (trigger "Blah2End" #/^end Blah2/) +;; (section "Blah2" "Blah2" "Blah2End") +;; +;; (expect:required in "Init" = 1 "Header" #/This is a header/) +;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/) +;; (expect:value in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/) +;; (expect:value in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/) +;; (expect:value in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/) +;; +;; ;; Using match number +;; (expect:value in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; ;; Comparison instead of tolerance +;; (expect:value in "LogFileBody" 1.9 > "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; (expect:ignore in "Blah2" < 99 "FALSE ERROR" #/ERROR/) +;; (expect:ignore in "Body" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +;; (expect:warning in "Body" = 0 "Any warning" #/WARNING/) +;; (expect:error in "Body" = 0 "ERROR BLAH" (list #/ERROR/ #/error/)) ;; but disallow any other errors +;; +;; ;(expect in "Init" < 1 "Junk" #/This is bogus/) ADDED tests/release/tests/itempatt/results.logpro Index: tests/release/tests/itempatt/results.logpro ================================================================== --- /dev/null +++ tests/release/tests/itempatt/results.logpro @@ -0,0 +1,144 @@ +;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com +;; +;; License GPL. + +(define logbody "LogFileBody") + +(define pass-specs '( ;; testname num-expected max-runtime + ;; ("exit_0" 1 20) + ;; ("ezlog_fail_then_pass" 1 20) + ;; ("ezlog_pass" 1 20) + ;; ("ez_pass" 1 20) + ;; ("lineitem_pass" 1 20) + ;; ("priority_1" 1 20) + ;; ("priority_10" 1 20) + ;; ("priority_10_waiton_1" 1 20) + ;; ("priority_3" 1 20) + ;; ("priority_4" 1 20) + ;; ;; ("priority_5" 1 20) + ;; ("priority_6" 1 20) +;; ;; ("priority_7" 1 20) + ;; ("priority_8" 1 20) + ;; ("priority_9" 1 20) + ("runfirst" 2 20) + ;; ("singletest" 1 20) + ;; ("singletest2" 1 20) + ;; ("special" 1 20) + ;; ("sqlitespeed" 10 20) + ;; ("test1" 1 20) + ;; ("test2" 6 20) + ;; ("test_mt_vars" 6 20) + )) + +(define fail-specs '( ;; testname num-expected max-runtime + ;; ("exit_1" 1 20) + ;; ("ez_exit2_fail" 1 20) + ;; ("ez_fail" 1 20) + ;; ("ez_fail_quick" 1 20) + ;; ("ezlog_fail" 1 20) + ;; ("lineitem_fail" 1 20) + ;; ("logpro_required_fail" 1 20) + ;; ("manual_example" 1 20) + ;; ("neverrun" 1 20) + )) + +(define warn-specs '( + ;; ("ezlog_warn" 1 20) + )) + +(define nost-specs '( + ;; ("wait_no_items1" 1 20) + ;; ("wait_no_items2" 1 20) + ;; ("wait_no_items3" 1 20) + ;; ("wait_no_items4" 1 20) + ;; ("no_items" 1 20) + )) + +(define (check-one-test estate estatus testname count runtime) + (let* ((rxe (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s"))) + (msg1 (conc testname " expecting count of " count)) + (msg2 (conc testname " expecting runtime less than " runtime))) + (expect:required in logbody = count msg1 rxe) + ;;(expect:value in logbody count < msg2 rxe) + )) + +;; Special cases +;; +;; (expect:ignore in logbody >= 0 "db_sync test might not have run" #/Test: db_sync/) +;; (expect:ignore in logbody >= 0 "all_toplevel may not yet be done" #/Test: all_toplevel/) +(expect:error in logbody = 0 "tests left in RUNNING state" #/State: RUNNING/) +;; (expect:required in logbody = 1 "priority_2 is KILLED" #/Test: priority_2\s+State: KILLED\s+Status: KILLED/) +;; (expect:required in logbody = 1 "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "testxz has 1 NOT_STARTED test" #/Test: testxz\s+State: NOT_STARTED/) +;; (expect:required in logbody = 1 "no items" #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/) +;; (expect:warning in logbody = 1 "dynamic waiton" #/Test: dynamic_waiton/) +;; (expect:required in logbody = 29 "blocktestxz has 29 tests" #/Test: blocktestxz/) + +;; General cases +;; +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "PASS" testdat)) + pass-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "FAIL" testdat)) + fail-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "WARN" testdat)) + warn-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat)) + nost-specs) + +;; Catch all. +;; +(expect:error in logbody = 0 "Tests not accounted for" #/Test: /) + + +;; ;; define your hooks +;; (hook:first-error "echo \"Error hook activated: #{escaped errmsg}\"") +;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"") +;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"") +;; +;; ;; first ensure your run at least started +;; ;; +;; (trigger "Init" #/This is a header/) +;; (trigger "InitEnd" #/^\s*$/) +;; (section "Init" "Init" "InitEnd") +;; +;; (trigger "Body" #/^.*$/) ;; anything starts the body +;; ;; (trigger "EndBody" #/This had better never match/) +;; +;; (section "Body" "Body" "EndBody") +;; +;; (trigger "Blah2" #/^begin Blah2/) +;; (trigger "Blah2End" #/^end Blah2/) +;; (section "Blah2" "Blah2" "Blah2End") +;; +;; (expect:required in "Init" = 1 "Header" #/This is a header/) +;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/) +;; (expect:value in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/) +;; (expect:value in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/) +;; (expect:value in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/) +;; +;; ;; Using match number +;; (expect:value in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; ;; Comparison instead of tolerance +;; (expect:value in "LogFileBody" 1.9 > "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; (expect:ignore in "Blah2" < 99 "FALSE ERROR" #/ERROR/) +;; (expect:ignore in "Body" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +;; (expect:warning in "Body" = 0 "Any warning" #/WARNING/) +;; (expect:error in "Body" = 0 "ERROR BLAH" (list #/ERROR/ #/error/)) ;; but disallow any other errors +;; +;; ;(expect in "Init" < 1 "Junk" #/This is bogus/) Index: tests/release/tests/itempatt/testconfig ================================================================== --- tests/release/tests/itempatt/testconfig +++ tests/release/tests/itempatt/testconfig @@ -1,8 +1,8 @@ [ezsteps] runtem $MTRUNNER $MTTESTDIR/fullrun megatest -runtests runfirst/%2 -target ubuntu/nfs/none -runname release_itempatt -results $MTRUNNER $MTTESTDIR/fullrun megatest -list-runs release_toplevel -target ubuntu/nfs/none -runname release_itempatt +results $MTRUNNER $MTTESTDIR/fullrun megatest -list-runs release_itempatt -target ubuntu/nfs/none [requirements] # waiton #{getenv ALL_TOPLEVEL_TESTS} # This is a "toplevel" test, it does not require waitons to be non-FAIL to run ADDED tests/release/tests/test2/results.logpro Index: tests/release/tests/test2/results.logpro ================================================================== --- /dev/null +++ tests/release/tests/test2/results.logpro @@ -0,0 +1,144 @@ +;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com +;; +;; License GPL. + +(define logbody "LogFileBody") + +(define pass-specs '( ;; testname num-expected max-runtime + ;; ("exit_0" 1 20) + ;; ("ezlog_fail_then_pass" 1 20) + ;; ("ezlog_pass" 1 20) + ;; ("ez_pass" 1 20) + ;; ("lineitem_pass" 1 20) + ;; ("priority_1" 1 20) + ;; ("priority_10" 1 20) + ;; ("priority_10_waiton_1" 1 20) + ;; ("priority_3" 1 20) + ;; ("priority_4" 1 20) + ;; ;; ("priority_5" 1 20) + ;; ("priority_6" 1 20) +;; ;; ("priority_7" 1 20) + ;; ("priority_8" 1 20) + ;; ("priority_9" 1 20) + ("runfirst" 2 20) + ;; ("singletest" 1 20) + ;; ("singletest2" 1 20) + ;; ("special" 1 20) + ;; ("sqlitespeed" 10 20) + ;; ("test1" 1 20) + ;; ("test2" 6 20) + ;; ("test_mt_vars" 6 20) + )) + +(define fail-specs '( ;; testname num-expected max-runtime + ;; ("exit_1" 1 20) + ;; ("ez_exit2_fail" 1 20) + ;; ("ez_fail" 1 20) + ;; ("ez_fail_quick" 1 20) + ;; ("ezlog_fail" 1 20) + ;; ("lineitem_fail" 1 20) + ;; ("logpro_required_fail" 1 20) + ;; ("manual_example" 1 20) + ;; ("neverrun" 1 20) + )) + +(define warn-specs '( + ;; ("ezlog_warn" 1 20) + )) + +(define nost-specs '( + ;; ("wait_no_items1" 1 20) + ;; ("wait_no_items2" 1 20) + ;; ("wait_no_items3" 1 20) + ;; ("wait_no_items4" 1 20) + ;; ("no_items" 1 20) + )) + +(define (check-one-test estate estatus testname count runtime) + (let* ((rxe (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s"))) + (msg1 (conc testname " expecting count of " count)) + (msg2 (conc testname " expecting runtime less than " runtime))) + (expect:required in logbody = count msg1 rxe) + ;;(expect:value in logbody count < msg2 rxe) + )) + +;; Special cases +;; +;; (expect:ignore in logbody >= 0 "db_sync test might not have run" #/Test: db_sync/) +;; (expect:ignore in logbody >= 0 "all_toplevel may not yet be done" #/Test: all_toplevel/) +(expect:error in logbody = 0 "tests left in RUNNING state" #/State: RUNNING/) +;; (expect:required in logbody = 1 "priority_2 is KILLED" #/Test: priority_2\s+State: KILLED\s+Status: KILLED/) +;; (expect:required in logbody = 1 "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "testxz has 1 NOT_STARTED test" #/Test: testxz\s+State: NOT_STARTED/) +;; (expect:required in logbody = 1 "no items" #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/) +;; (expect:warning in logbody = 1 "dynamic waiton" #/Test: dynamic_waiton/) +;; (expect:required in logbody = 29 "blocktestxz has 29 tests" #/Test: blocktestxz/) + +;; General cases +;; +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "PASS" testdat)) + pass-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "FAIL" testdat)) + fail-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "WARN" testdat)) + warn-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat)) + nost-specs) + +;; Catch all. +;; +(expect:error in logbody = 0 "Tests not accounted for" #/Test: /) + + +;; ;; define your hooks +;; (hook:first-error "echo \"Error hook activated: #{escaped errmsg}\"") +;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"") +;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"") +;; +;; ;; first ensure your run at least started +;; ;; +;; (trigger "Init" #/This is a header/) +;; (trigger "InitEnd" #/^\s*$/) +;; (section "Init" "Init" "InitEnd") +;; +;; (trigger "Body" #/^.*$/) ;; anything starts the body +;; ;; (trigger "EndBody" #/This had better never match/) +;; +;; (section "Body" "Body" "EndBody") +;; +;; (trigger "Blah2" #/^begin Blah2/) +;; (trigger "Blah2End" #/^end Blah2/) +;; (section "Blah2" "Blah2" "Blah2End") +;; +;; (expect:required in "Init" = 1 "Header" #/This is a header/) +;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/) +;; (expect:value in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/) +;; (expect:value in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/) +;; (expect:value in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/) +;; +;; ;; Using match number +;; (expect:value in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; ;; Comparison instead of tolerance +;; (expect:value in "LogFileBody" 1.9 > "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; (expect:ignore in "Blah2" < 99 "FALSE ERROR" #/ERROR/) +;; (expect:ignore in "Body" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +;; (expect:warning in "Body" = 0 "Any warning" #/WARNING/) +;; (expect:error in "Body" = 0 "ERROR BLAH" (list #/ERROR/ #/error/)) ;; but disallow any other errors +;; +;; ;(expect in "Init" < 1 "Junk" #/This is bogus/) ADDED tests/release/tests/test2/results_a.logpro Index: tests/release/tests/test2/results_a.logpro ================================================================== --- /dev/null +++ tests/release/tests/test2/results_a.logpro @@ -0,0 +1,144 @@ +;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com +;; +;; License GPL. + +(define logbody "LogFileBody") + +(define pass-specs '( ;; testname num-expected max-runtime + ;; ("exit_0" 1 20) + ;; ("ezlog_fail_then_pass" 1 20) + ;; ("ezlog_pass" 1 20) + ;; ("ez_pass" 1 20) + ;; ("lineitem_pass" 1 20) + ;; ("priority_1" 1 20) + ;; ("priority_10" 1 20) + ;; ("priority_10_waiton_1" 1 20) + ;; ("priority_3" 1 20) + ;; ("priority_4" 1 20) + ;; ;; ("priority_5" 1 20) + ;; ("priority_6" 1 20) +;; ;; ("priority_7" 1 20) + ;; ("priority_8" 1 20) + ;; ("priority_9" 1 20) + ("runfirst" 2 20) + ;; ("singletest" 1 20) + ;; ("singletest2" 1 20) + ;; ("special" 1 20) + ;; ("sqlitespeed" 10 20) + ;; ("test1" 1 20) + ;; ("test2" 6 20) + ;; ("test_mt_vars" 6 20) + )) + +(define fail-specs '( ;; testname num-expected max-runtime + ;; ("exit_1" 1 20) + ;; ("ez_exit2_fail" 1 20) + ;; ("ez_fail" 1 20) + ;; ("ez_fail_quick" 1 20) + ;; ("ezlog_fail" 1 20) + ;; ("lineitem_fail" 1 20) + ;; ("logpro_required_fail" 1 20) + ;; ("manual_example" 1 20) + ;; ("neverrun" 1 20) + )) + +(define warn-specs '( + ;; ("ezlog_warn" 1 20) + )) + +(define nost-specs '( + ;; ("wait_no_items1" 1 20) + ;; ("wait_no_items2" 1 20) + ;; ("wait_no_items3" 1 20) + ;; ("wait_no_items4" 1 20) + ;; ("no_items" 1 20) + )) + +(define (check-one-test estate estatus testname count runtime) + (let* ((rxe (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s"))) + (msg1 (conc testname " expecting count of " count)) + (msg2 (conc testname " expecting runtime less than " runtime))) + (expect:required in logbody = count msg1 rxe) + ;;(expect:value in logbody count < msg2 rxe) + )) + +;; Special cases +;; +;; (expect:ignore in logbody >= 0 "db_sync test might not have run" #/Test: db_sync/) +;; (expect:ignore in logbody >= 0 "all_toplevel may not yet be done" #/Test: all_toplevel/) +(expect:error in logbody = 0 "tests left in RUNNING state" #/State: RUNNING/) +;; (expect:required in logbody = 1 "priority_2 is KILLED" #/Test: priority_2\s+State: KILLED\s+Status: KILLED/) +;; (expect:required in logbody = 1 "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "testxz has 1 NOT_STARTED test" #/Test: testxz\s+State: NOT_STARTED/) +;; (expect:required in logbody = 1 "no items" #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/) +;; (expect:warning in logbody = 1 "dynamic waiton" #/Test: dynamic_waiton/) +;; (expect:required in logbody = 29 "blocktestxz has 29 tests" #/Test: blocktestxz/) + +;; General cases +;; +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "PASS" testdat)) + pass-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "FAIL" testdat)) + fail-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "WARN" testdat)) + warn-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat)) + nost-specs) + +;; Catch all. +;; +(expect:error in logbody = 0 "Tests not accounted for" #/Test: /) + + +;; ;; define your hooks +;; (hook:first-error "echo \"Error hook activated: #{escaped errmsg}\"") +;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"") +;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"") +;; +;; ;; first ensure your run at least started +;; ;; +;; (trigger "Init" #/This is a header/) +;; (trigger "InitEnd" #/^\s*$/) +;; (section "Init" "Init" "InitEnd") +;; +;; (trigger "Body" #/^.*$/) ;; anything starts the body +;; ;; (trigger "EndBody" #/This had better never match/) +;; +;; (section "Body" "Body" "EndBody") +;; +;; (trigger "Blah2" #/^begin Blah2/) +;; (trigger "Blah2End" #/^end Blah2/) +;; (section "Blah2" "Blah2" "Blah2End") +;; +;; (expect:required in "Init" = 1 "Header" #/This is a header/) +;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/) +;; (expect:value in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/) +;; (expect:value in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/) +;; (expect:value in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/) +;; +;; ;; Using match number +;; (expect:value in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; ;; Comparison instead of tolerance +;; (expect:value in "LogFileBody" 1.9 > "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; (expect:ignore in "Blah2" < 99 "FALSE ERROR" #/ERROR/) +;; (expect:ignore in "Body" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +;; (expect:warning in "Body" = 0 "Any warning" #/WARNING/) +;; (expect:error in "Body" = 0 "ERROR BLAH" (list #/ERROR/ #/error/)) ;; but disallow any other errors +;; +;; ;(expect in "Init" < 1 "Junk" #/This is bogus/) ADDED tests/release/tests/test2/results_b.logpro Index: tests/release/tests/test2/results_b.logpro ================================================================== --- /dev/null +++ tests/release/tests/test2/results_b.logpro @@ -0,0 +1,144 @@ +;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com +;; +;; License GPL. + +(define logbody "LogFileBody") + +(define pass-specs '( ;; testname num-expected max-runtime + ;; ("exit_0" 1 20) + ;; ("ezlog_fail_then_pass" 1 20) + ;; ("ezlog_pass" 1 20) + ;; ("ez_pass" 1 20) + ;; ("lineitem_pass" 1 20) + ;; ("priority_1" 1 20) + ;; ("priority_10" 1 20) + ;; ("priority_10_waiton_1" 1 20) + ;; ("priority_3" 1 20) + ;; ("priority_4" 1 20) + ;; ;; ("priority_5" 1 20) + ;; ("priority_6" 1 20) +;; ;; ("priority_7" 1 20) + ;; ("priority_8" 1 20) + ;; ("priority_9" 1 20) + ("runfirst" 2 20) + ;; ("singletest" 1 20) + ;; ("singletest2" 1 20) + ;; ("special" 1 20) + ;; ("sqlitespeed" 10 20) + ;; ("test1" 1 20) + ;; ("test2" 6 20) + ;; ("test_mt_vars" 6 20) + )) + +(define fail-specs '( ;; testname num-expected max-runtime + ;; ("exit_1" 1 20) + ;; ("ez_exit2_fail" 1 20) + ;; ("ez_fail" 1 20) + ;; ("ez_fail_quick" 1 20) + ;; ("ezlog_fail" 1 20) + ;; ("lineitem_fail" 1 20) + ;; ("logpro_required_fail" 1 20) + ;; ("manual_example" 1 20) + ;; ("neverrun" 1 20) + )) + +(define warn-specs '( + ;; ("ezlog_warn" 1 20) + )) + +(define nost-specs '( + ;; ("wait_no_items1" 1 20) + ;; ("wait_no_items2" 1 20) + ;; ("wait_no_items3" 1 20) + ;; ("wait_no_items4" 1 20) + ;; ("no_items" 1 20) + )) + +(define (check-one-test estate estatus testname count runtime) + (let* ((rxe (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s"))) + (msg1 (conc testname " expecting count of " count)) + (msg2 (conc testname " expecting runtime less than " runtime))) + (expect:required in logbody = count msg1 rxe) + ;;(expect:value in logbody count < msg2 rxe) + )) + +;; Special cases +;; +;; (expect:ignore in logbody >= 0 "db_sync test might not have run" #/Test: db_sync/) +;; (expect:ignore in logbody >= 0 "all_toplevel may not yet be done" #/Test: all_toplevel/) +(expect:error in logbody = 0 "tests left in RUNNING state" #/State: RUNNING/) +;; (expect:required in logbody = 1 "priority_2 is KILLED" #/Test: priority_2\s+State: KILLED\s+Status: KILLED/) +;; (expect:required in logbody = 1 "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/) +;; (expect:required in logbody = 1 "testxz has 1 NOT_STARTED test" #/Test: testxz\s+State: NOT_STARTED/) +;; (expect:required in logbody = 1 "no items" #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/) +;; (expect:warning in logbody = 1 "dynamic waiton" #/Test: dynamic_waiton/) +;; (expect:required in logbody = 29 "blocktestxz has 29 tests" #/Test: blocktestxz/) + +;; General cases +;; +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "PASS" testdat)) + pass-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "FAIL" testdat)) + fail-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "COMPLETED" "WARN" testdat)) + warn-specs) + +(for-each + (lambda (testdat) + (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat)) + nost-specs) + +;; Catch all. +;; +(expect:error in logbody = 0 "Tests not accounted for" #/Test: /) + + +;; ;; define your hooks +;; (hook:first-error "echo \"Error hook activated: #{escaped errmsg}\"") +;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"") +;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"") +;; +;; ;; first ensure your run at least started +;; ;; +;; (trigger "Init" #/This is a header/) +;; (trigger "InitEnd" #/^\s*$/) +;; (section "Init" "Init" "InitEnd") +;; +;; (trigger "Body" #/^.*$/) ;; anything starts the body +;; ;; (trigger "EndBody" #/This had better never match/) +;; +;; (section "Body" "Body" "EndBody") +;; +;; (trigger "Blah2" #/^begin Blah2/) +;; (trigger "Blah2End" #/^end Blah2/) +;; (section "Blah2" "Blah2" "Blah2End") +;; +;; (expect:required in "Init" = 1 "Header" #/This is a header/) +;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/) +;; (expect:value in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/) +;; (expect:value in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/) +;; (expect:value in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/) +;; (expect:value in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/) +;; +;; ;; Using match number +;; (expect:value in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; ;; Comparison instead of tolerance +;; (expect:value in "LogFileBody" 1.9 > "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2) +;; +;; (expect:ignore in "Blah2" < 99 "FALSE ERROR" #/ERROR/) +;; (expect:ignore in "Body" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +;; (expect:warning in "Body" = 0 "Any warning" #/WARNING/) +;; (expect:error in "Body" = 0 "ERROR BLAH" (list #/ERROR/ #/error/)) ;; but disallow any other errors +;; +;; ;(expect in "Init" < 1 "Junk" #/This is bogus/) ADDED tests/release/tests/test2/testconfig Index: tests/release/tests/test2/testconfig ================================================================== --- /dev/null +++ tests/release/tests/test2/testconfig @@ -0,0 +1,26 @@ +# test2 from the tests/Makefile + +[vars] +tname test2 + +[ezsteps] +clean $MTRUNNER $MTTESTDIR/fullrun megatest -remove-runs -testpatt % -target ubuntu/nfs/none -runname #{get var tname}% +part1 $MTRUNNER $MTTESTDIR/fullrun megatest -run -testpatt ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none -runname #{get var tname} -preclean +part2 $MTRUNNER $MTTESTDIR/fullrun megatest -run -testpatt %/,%/ai -reqtarg ubuntu/nfs/none -runname #{get var tname}_a -preclean +part3 $MTRUNNER $MTTESTDIR/fullrun megatest -runtests %/,%/ai -reqtarg ubuntu/nfs/none -runname #{get var tname}_b -preclean +part4 $MTRUNNER $MTTESTDIR/fullrun megatest -run -testpatt runfirst/%,%/ai -reqtarg ubuntu/nfs/none -runname #{get var tname}_a -preclean +part5 $MTRUNNER $MTTESTDIR/fullrun megatest -run -testpatt %/,%/winter -reqtarg ubuntu/nfs/none -runname #{get var tname}_a -preclean +part6 $MTRUNNER $MTTESTDIR/fullrun megatest -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -target ubuntu/nfs/none -runname #{get var tname} + +results $MTRUNNER $MTTESTDIR/fullrun megatest -list-runs #{get var tname} -target ubuntu/nfs/none +results_a $MTRUNNER $MTTESTDIR/fullrun megatest -list-runs #{get var tname}_a -target ubuntu/nfs/none +results_b $MTRUNNER $MTTESTDIR/fullrun megatest -list-runs #{get var tname}_b -target ubuntu/nfs/none + +[requirements] +# waiton #{getenv ALL_TOPLEVEL_TESTS} + +# This is a "toplevel" test, it does not require waitons to be non-FAIL to run +# mode toplevel + + +# test2 : fullprep Index: utils/Makefile.installall ================================================================== --- utils/Makefile.installall +++ utils/Makefile.installall @@ -34,12 +34,13 @@ endif # Set this on the command line of your make call if needed: make PROXY=host.com:1234 PROXY= +# http://code.call-cc.org/dev-snapshots/2015/06/07/chicken-4.10.0rc1.tar.gz # Select version of chicken, sqlite3 etc -CHICKEN_VERSION=4.9.0.1 +CHICKEN_VERSION=4.10.0rc1 SQLITE3_VERSION=3080500 # http://www.sqlite.org/2014/sqlite-autoconf-3080500.tar.gz # Override IUPBRANCH to use other than trunk IUPBRANCH=iup-3.10.1 @@ -136,10 +137,13 @@ wget http://code.call-cc.org/dev-snapshots/2014/04/17/chicken-4.9.0rc1.tar.gz chicken-4.9.0.1.tar.gz : wget http://code.call-cc.org/releases/4.9.0/chicken-4.9.0.1.tar.gz +chicken-4.10.0rc1.tar.gz : + wget http://code.call-cc.org/dev-snapshots/2015/06/07/chicken-4.10.0rc1.tar.gz + # git clone git://code.call-cc.org/chicken-core # git clone http://code.call-cc.org/git/chicken-core.git $(CHICKEN_INSTALL) : chicken-core/chicken.scm $(PREFIX)/setup-chicken4x.sh $(PREFIX)/setup-chicken4x.csh cd chicken-core;make PLATFORM=linux PREFIX=$(PREFIX)