Index: archive.scm ================================================================== --- archive.scm +++ archive.scm @@ -231,16 +231,23 @@ (archive-path (if (vector? archive-block-info) (vector-ref archive-block-info 2) ;; look in db.scm for test-get-archive-block-info for the vector record info #f)) ;; no archive found? (archive-internal-path (conc (common:get-testsuite-name) "-" run-id "/latest/" test-partial-path))) - ;; some sanity checks - (if (and prev-test-physical-path + ;; some sanity checks, move an existing path out of the way - iif it is not a toplevel with children + ;; + (if (and (not toplevel/children) ;; special handling needed for toplevel with children + prev-test-physical-path (file-exists? prev-test-physical-path)) ;; what to do? abort or clean up or link it in? - (debug:print 0 "ERROR: the old directory " prev-test-physical-path ", still exists! This should not be.")) + (let* ((base (pathname-directory prev-test-physical-path)) + (dirn (pathname-file prev-test-physical-path)) + (newn (conc base "/." dirn))) + (debug:print 0 "ERROR: the old directory " prev-test-physical-path ", still exists! Moving it to " newn) + (rename-file prev-test-physical-path newn))) - (if archive-path ;; no point in proceeding if there is no actual archive + (if (and archive-path ;; no point in proceeding if there is no actual archive + (not toplevel/children)) (begin ;; CREATE WORK AREA ;; test-src-path == #f ==> don't copy in data from tests directory ;; itemdat == string ==> use directly (create-work-area run-id run-name keyvals test-id #f best-disk test-name item-path) ;; #!key (remtries 2)) @@ -259,9 +266,10 @@ (debug:print 0 "ERROR: unable to get data for run-id=" run-id ", test-id=" test-id) (exit 1)))) ;; new-test-path won't work - must use best-disk instead? Nope, new-test-path but tack on /.. (bup-restore-params (list "-d" archive-path "restore" "-C" (conc new-test-path "/..") archive-internal-path))) (debug:print-info 0 "Restoring archived data to " new-test-physical-path " from archive in " archive-path " ... " archive-internal-path) - (run-n-wait bup-exe params: bup-restore-params print-cmd: #f))) + (run-n-wait bup-exe params: bup-restore-params print-cmd: #f) + (mt:test-set-state-status-by-id run-id test-id "COMPLETED" #f #f))) (debug:print 0 "ERROR: No archive path in the record for run-id=" run-id " test-id=" test-id)))) (filter vector? tests)))) Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -557,11 +557,11 @@ " -state RUNNING")))) (run-test (lambda (x) (iup:attribute-set! command-text-box "VALUE" (conc "megatest -target " keystring " -runname " runname - " -runtests " (conc testname "/" (if (equal? item-path "") + " -run -testpatt " (conc testname "/" (if (equal? item-path "") "%" item-path)) )))) (remove-test (lambda (x) (iup:attribute-set! @@ -589,12 +589,19 @@ command-text-box "VALUE" (conc "megatest -remove-runs -target " keystring " -runname " runname " -testpatt " (conc testname "/" (if (equal? item-path "") "%" item-path)) - " -v")) - ))) + " -v")))) + (archive-test (lambda (x) + (iup:attribute-set! + command-text-box "VALUE" + (conc "megatest -target " keystring " -runname " runname + " -archive save-remove -testpatt " (conc testname "/" (if (equal? item-path "") + "%" + item-path)) + ))))) (cond ((not testdat)(begin (print "ERROR: bad test info for " test-id)(exit 1))) ((not rundat)(begin (print "ERROR: found test info but there is a problem with the run info for " run-id)(exit 1))) (else ;; (test-set-status! db run-id test-name state status itemdat) @@ -610,16 +617,17 @@ (host-info-panel testdat store-label) ;; The controls (iup:frame #:title "Actions" (iup:vbox (iup:hbox - (iup:button "View Log" #:action viewlog #:size "80x") - (iup:button "Start Xterm" #:action xterm #:size "80x") - (iup:button "Run Test" #:action run-test #:size "80x") - (iup:button "Clean Test" #:action remove-test #:size "80x") + (iup:button "View Log" #:action viewlog #:size "80x") + (iup:button "Start Xterm" #:action xterm #:size "80x") + (iup:button "Run Test" #:action run-test #:size "80x") + (iup:button "Clean Test" #:action remove-test #:size "80x") (iup:button "CleanRunExecute!" #:action clean-run-execute #:size "80x") - (iup:button "Kill All Jobs" #:action kill-jobs #:size "80x") + (iup:button "Kill All Jobs" #:action kill-jobs #:size "80x") + (iup:button "Archive Test" #:action archive-test #:size "80x") (iup:button "Close" #:action (lambda (x)(exit)) #:size "80x")) (apply iup:hbox (list command-text-box command-launch-button)))) (set-fields-panel dbstruct run-id test-id testdat) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -2258,11 +2258,13 @@ (db:with-db dbstruct run-id #t (lambda (db) - (sqlite3:execute db qry newstate newstatus run-id testname))))) + (sqlite3:execute db qry newstate newstatus run-id testname) + (mt:process-triggers run-id test-id newstate newstatus) + )))) testnames)) ;; speed up for common cases with a little logic ;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id ;; @@ -2829,11 +2831,12 @@ (let ((dbdat (db:get-db dbstruct run-id))) (if (member state '("LAUNCHED" "REMOTEHOSTSTART")) (db:general-call dbdat 'set-test-start-time (list test-id))) (if msg (db:general-call dbdat 'state-status-msg (list state status msg test-id)) - (db:general-call dbdat 'state-status (list state status test-id))))) + (db:general-call dbdat 'state-status (list state status test-id))) + (mt:process-triggers run-id test-id state status))) ;; call with state = #f to roll up with out accounting for state/status of this item ;; (define (db:roll-up-pass-fail-counts dbstruct run-id test-name item-path state status) (if (not (equal? item-path "")) Index: docs/manual/getting_started.txt ================================================================== --- docs/manual/getting_started.txt +++ docs/manual/getting_started.txt @@ -20,62 +20,62 @@ for building Megatest on Linux. footnote:[An example footnote.] indexterm:[Example index entry] - - -And now for something completely different: ((monkeys)), lions and -tigers (Bengal and Siberian) using the alternative syntax index -entries. -(((Big cats,Lions))) -(((Big cats,Tigers,Bengal Tiger))) -(((Big cats,Tigers,Siberian Tiger))) -Note that multi-entry terms generate separate index entries. - -Here are a couple of image examples: an image:images/smallnew.png[] -example inline image followed by an example block image: - -.Tiger block image -image::images/tiger.png[Tiger image] - -Followed by an example table: - -.An example table -[width="60%",options="header"] -|============================================== -| Option | Description -| -a 'USER GROUP' | Add 'USER' to 'GROUP'. -| -R 'GROUP' | Disables access to 'GROUP'. -|============================================== - -.An example example -=============================================== -Lorum ipum... -=============================================== - -[[X1]] -Sub-section with Anchor -~~~~~~~~~~~~~~~~~~~~~~~ -Sub-section at level 2. - -Chapter Sub-section -^^^^^^^^^^^^^^^^^^^ -Sub-section at level 3. - -Chapter Sub-section -+++++++++++++++++++ -Sub-section at level 4. - -This is the maximum sub-section depth supported by the distributed -AsciiDoc configuration. -footnote:[A second example footnote.] - - -The Second Chapter ------------------- -An example link to anchor at start of the <>. -indexterm:[Second example index entry] - -An example link to a bibliography entry <>. - - +// +// +// And now for something completely different: ((monkeys)), lions and +// tigers (Bengal and Siberian) using the alternative syntax index +// entries. +// (((Big cats,Lions))) +// (((Big cats,Tigers,Bengal Tiger))) +// (((Big cats,Tigers,Siberian Tiger))) +// Note that multi-entry terms generate separate index entries. +// +// Here are a couple of image examples: an image:images/smallnew.png[] +// example inline image followed by an example block image: +// +// .Tiger block image +// image::images/tiger.png[Tiger image] +// +// Followed by an example table: +// +// .An example table +// [width="60%",options="header"] +// |============================================== +// | Option | Description +// | -a 'USER GROUP' | Add 'USER' to 'GROUP'. +// | -R 'GROUP' | Disables access to 'GROUP'. +// |============================================== +// +// .An example example +// =============================================== +// Lorum ipum... +// =============================================== +// +// [[X1]] +// Sub-section with Anchor +// ~~~~~~~~~~~~~~~~~~~~~~~ +// Sub-section at level 2. +// +// Chapter Sub-section +// ^^^^^^^^^^^^^^^^^^^ +// Sub-section at level 3. +// +// Chapter Sub-section +// +++++++++++++++++++ +// Sub-section at level 4. +// +// This is the maximum sub-section depth supported by the distributed +// AsciiDoc configuration. +// footnote:[A second example footnote.] +// +// +// The Second Chapter +// ------------------ +// An example link to anchor at start of the <>. +// indexterm:[Second example index entry] +// +// An example link to a bibliography entry <>. +// +// Index: docs/manual/howto.txt ================================================================== --- docs/manual/howto.txt +++ docs/manual/howto.txt @@ -37,32 +37,73 @@ Examining The Environment ~~~~~~~~~~~~~~~~~~~~~~~~~ During Config File Processing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is often helpful to know the content of variables in various +contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined. + +For example, if an item list is not being generated as expected you +can inject the startup of an xterm as if it were an item: + +.Original items table +----------------- +[items] +CELLNAME [system getcellname.sh] +----------------- + +.Items table modified for debug +----------------- +[items] +DEBUG [system xterm] +CELLNAME [system getcellnames.sh] +----------------- + +When this test is run an xterm will pop up. In that xterm the +environment is exactly that in which the script "getcellnames.sh" +would run. You can now debug the script to find out why it isn't +working as expected. Organising Your Tests and Tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The default location "tests" for storing tests can be extended by +adding to your tests-paths section. + ---------------------------- +[misc] +parent #{shell dirname $(readlink -f .)} + [tests-paths] 1 #{get misc parent}/simplerun/tests ---------------------------- +The above example shows how you can use addition sections in your +config file to do complex processing. By putting results of relatively +slow operations into variables the processing of your configs can be +kept fast. + +Alternative Method for Running your Job Script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.Directly running job in testconfig ------------------- [setup] -------------------- - -The runscript method is a brute force way to run scripts where the -user is responsible for setting STATE and STATUS - -------------------- runscript main.csh ------------------- +The runscript method is essentially a brute force way to run scripts where the +user is responsible for setting STATE and STATUS and managing the details of running a test. + Debugging Server Problems ~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some handy Unix commands to track down issues with servers not +communicating with your test manager processes. Please put in tickets +at https://www.kiatoa.com/fossils/megatest if you have problems with +servers getting stuck. ---------------- sudo lsof -i sudo netstat -lptu sudo netstat -tulpn Index: docs/manual/megatest_manual.html ================================================================== --- docs/manual/megatest_manual.html +++ docs/manual/megatest_manual.html @@ -825,69 +825,37 @@

Current Items

ww05 - migrate to inmem-db

-

Keep as much the same as possible. Add internal reference to almost -eliminate contention on db(s).

-
    -
  1. -

    -Add internal reference db -

    -
    2. -
  2. -

    -Verify that actions are accessing correct db -

    -
      -
    1. -

      --runtests - inmem -

      -
      2. -
    2. -

      --list-runs - local (but not megatest.db) -

      -
      3. -
    3. -

      -dashboard - local (but not megatest.db) -

      -
      4. -
    -
    3. -
  3. -

    -Mirror db to /var/tmp… -

    -
    4. -
  4. -

    -Dashboard read db from per-run db. -

    -
    5. -
  5. -

    -Dashboard read db from /var/tmp -

    -
    6. -
  6. -

    -Runs register in tasks table in monitor.db -

    -
    7. -
  7. -

    -Server polls tasks table for next action (in addition?) -

    -
    8. -
  8. -

    -Change run loop to execute in server, triggered by call to polling of tasks table -

    +
      +
    1. +

      +Switch to inmem db with fast sync to on disk db’s [DONE] +

      +
      2. +
    2. +

      +Server polls tasks table for next action +

      +
        +
      1. +

        +Task table used for tracking runner process [DONE] +

        +
        2. +
      2. +

        +Task table used for jobs to run +

        +
        3. +
      3. +

        +Task table used for queueing runner actions (remove runs, cleanRunExecute, etc) +

        +
        4. +
@@ -905,88 +873,48 @@

Chicken scheme and a number of "eggs" are required for building Megatest. See the script installall.sch in the utils directory of the distribution for a mostly automated way to install everything needed for building Megatest on Linux.


[An example footnote.]

-

And now for something completely different: monkeys, lions and -tigers (Bengal and Siberian) using the alternative syntax index -entries. - - - -Note that multi-entry terms generate separate index entries.

-

Here are a couple of image examples: an -images/smallnew.png - -example inline image followed by an example block image:

-
-
-Tiger image -
-
Figure 1. Tiger block image
-
-

Followed by an example table:

- - --- - - - - - - - - - - - - - - - -
Table 1. An example table
Option Description

-a USER GROUP

Add USER to GROUP.

-R GROUP

Disables access to GROUP.

-
-
Example 1. An example example
-
-

Lorum ipum…

-
-
-
-

Sub-section with Anchor

-

Sub-section at level 2.

-
-

Chapter Sub-section

-

Sub-section at level 3.

-
-
Chapter Sub-section
-

Sub-section at level 4.

-

This is the maximum sub-section depth supported by the distributed -AsciiDoc configuration. -
[A second example footnote.]

-
-
-
- - -
-

The Second Chapter

-
-

An example link to anchor at start of the first sub-section.

-

An example link to a bibliography entry [taoup].

+

Writing Tests

-

The First Chapter of the Second Part

+

Creating a new Test

-

Chapters grouped into book parts are at level 1 and can contain -sub-sections.

+

The following steps will add a test "yourtestname" to your testsuite. This assumes +starting from a directory where you already have a megatest.config and +runconfigs.config.

+
    +
  1. +

    +Create a directory tests/yourtestname +

    +
    2. +
  2. +

    +Create a file tests/yourtestname/testconfig +

    +
    3. +
+
+
Contents of minimal testconfig
+
+
[ezsteps]
+stepname1 stepname.sh
+
+# test_meta is a section for storing additional data on your test
+[test_meta]
+author myname
+owner  myname
+description An example test
+reviewed never
+
+

This test runs a single step called "stepname1" which runs a script +"stepname.sh". Note that although it is common to put the actions +needed for a test step into a script it is not necessary.

How To Do Things

Tricks

@@ -1019,32 +947,67 @@

Examining The Environment

During Config File Processing

+

It is often helpful to know the content of variables in various +contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.

+

For example, if an item list is not being generated as expected you +can inject the startup of an xterm as if it were an item:

+
+
Original items table
+
+
[items]
+CELLNAME [system getcellname.sh]
+
+
+
Items table modified for debug
+
+
[items]
+DEBUG [system xterm]
+CELLNAME [system getcellnames.sh]
+
+

When this test is run an xterm will pop up. In that xterm the +environment is exactly that in which the script "getcellnames.sh" +would run. You can now debug the script to find out why it isn’t +working as expected.

Organising Your Tests and Tasks

+

The default location "tests" for storing tests can be extended by +adding to your tests-paths section.

-
[tests-paths]
+
[misc]
+parent #{shell dirname $(readlink -f .)}
+
+[tests-paths]
 1 #{get misc parent}/simplerun/tests
-
-
-
[setup]
-
-

The runscript method is a brute force way to run scripts where the -user is responsible for setting STATE and STATUS

-
-
-
runscript main.csh
-
+

The above example shows how you can use addition sections in your +config file to do complex processing. By putting results of relatively +slow operations into variables the processing of your configs can be +kept fast.

+
+
+

Alternative Method for Running your Job Script

+
+
Directly running job in testconfig
+
+
[setup]
+runscript main.csh
+
+

The runscript method is essentially a brute force way to run scripts where the +user is responsible for setting STATE and STATUS and managing the details of running a test.

Debugging Server Problems

+

Some handy Unix commands to track down issues with servers not +communicating with your test manager processes. Please put in tickets +at https://www.kiatoa.com/fossils/megatest if you have problems with +servers getting stuck.

sudo lsof -i
 sudo netstat -lptu
 sudo netstat -tulpn
@@ -1106,27 +1069,55 @@

Mode

The default (i.e. if mode is not specified) is normal. All pre-dependent tests must be COMPLETED and PASS, CHECK or WAIVED before the test will start

-
mode   normal
+
[requirements]
+mode   normal

The toplevel mode requires only that the prior tests are COMPLETED.

-
mode toplevel
+
[requirements]
+mode toplevel
+
+

A item based waiton will start items in a test when the same-named +item is COMPLETED and PASS, CHECK or WAIVED in the prior test. This +was historically called "itemwait" mode. The terms "itemwait" and +"itemmatch" are synonyms.

+
+
+
[requirements]
+mode itemmatch
-

A item based waiton will start items in a test when the -same-named item is COMPLETED and PASS, CHECK or WAIVED -in the prior test

+
+
+

Itemmap

+

For cases were the dependent test has a similar but not identical +itempath to the downstream test an itemmap can allow for itemmatch +mode

-
mode itemmatch
+
[requirements]
+mode itemmatch
+itemmap .*x/ y/
+
+# ## pattern replacement notes
+#
+# ## Example
+# ## Remove everything up to the last /
+itemmap .*/
+#
+# ## Example
+# ## Replace foo/ with bar/
+itemmap foo/ bar/
+
Autogeneration waiton list for dynamic flow dependency trees
-
# With a toplevel test you may wish to generate your list
+
[requirements]
+# With a toplevel test you may wish to generate your list
 # of tests to run dynamically
 #
 # waiton #{shell get-valid-tests-to-run.sh}
@@ -1137,33 +1128,47 @@ 
runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s

Skip

-
-
-

Header

+

A test with a skip section will conditional skip running.

+
Skip section example
-
[skip]
+
[skip]
+prevrunning x
+# rundelay 30m 15s

Skip on Still-running Tests

# NB// If the prevrunning line exists with *any* value the test will
-# automatically SKIP if the same-named test is currently RUNNING
+# automatically SKIP if the same-named test is currently RUNNING. The
+# "x" can be any string. Comment out the prevrunning line to turn off
+# skip.
 
+[skip]
 prevrunning x

Skip if a File Exists

-
fileexists /path/to/a/file # skip if /path/to/a/file exists
+
[skip]
+fileexists /path/to/a/file # skip if /path/to/a/file exists
+
+
+
+

Skip if test ran more recently than specified time

+
+
Skip if this test has been run in the past fifteen minutes and 15 seconds.
+
+
[skip]
+rundelay 15m 15s

Controlled waiver propagation

If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig: @@ -1231,11 +1236,12 @@

Override the Toplevel HTML File

Megatest generates a simple html file summary for top level tests of -iterated tests. The generation can be overridden.

+iterated tests. The generation can be overridden. NOTE: the output of +the script is captured from stdout to create the html.

For test "runfirst" override the toplevel generation with a script "mysummary.sh"
# Override the rollup for specific tests
 [testrollup]
@@ -1250,11 +1256,11 @@
 
These routines can be called from the megatest repl.

 
-
+
@@ -1290,11 +1296,11 @@
 
Table 2. API Server Management CallsTable 1. API Server Management Calls
 
 
 
 
 

 
-
+

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -56,36 +56,62 @@
 
 The default (i.e. if mode is not specified) is normal. All pre-dependent tests
 must be COMPLETED and PASS, CHECK or WAIVED before the test will start
 
 -------------------
+[requirements]
 mode   normal
 -------------------
 
 The toplevel mode requires only that the prior tests are COMPLETED.
 
 -------------------
+[requirements]
 mode toplevel
 -------------------
 
-A item based waiton will start items in a test when the
-same-named item is COMPLETED and PASS, CHECK or WAIVED
-in the prior test
+A item based waiton will start items in a test when the same-named
+item is COMPLETED and PASS, CHECK or WAIVED in the prior test. This
+was historically called "itemwait" mode. The terms "itemwait" and
+"itemmatch" are synonyms.
+
+-------------------
+[requirements]
+mode itemmatch
+-------------------
+
+Itemmap
+^^^^^^^
+
+For cases were the dependent test has a similar but not identical
+itempath to the downstream test an itemmap can allow for itemmatch
+mode
 
 -------------------
+[requirements]
 mode itemmatch
+itemmap .*x/ y/
+
+# ## pattern replacement notes
+#
+# ## Example
+# ## Remove everything up to the last /
+itemmap .*/
+#
+# ## Example
+# ## Replace foo/ with bar/
+itemmap foo/ bar/
 -------------------
 
+.Autogeneration waiton list for dynamic flow dependency trees
 -------------------
-
+[requirements]
 # With a toplevel test you may wish to generate your list 
 # of tests to run dynamically
 #
 # waiton #{shell get-valid-tests-to-run.sh}
-------------------
-
-
+-------------------
 
 Run time limit
 ^^^^^^^^^^^^^^
 
 -----------------
@@ -93,33 +119,48 @@
 -----------------
 
 Skip
 ^^^^
 
-Header
-^^^^^^
+A test with a skip section will conditional skip running.
 
+.Skip section example
 -----------------
 [skip]
+prevrunning x
+# rundelay 30m 15s
 -----------------
 
 Skip on Still-running Tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 -----------------
-# NB// If the prevrunning line exists with *any* value the test will 
-# automatically SKIP if the same-named test is currently RUNNING
+# NB// If the prevrunning line exists with *any* value the test will
+# automatically SKIP if the same-named test is currently RUNNING. The
+# "x" can be any string. Comment out the prevrunning line to turn off
+# skip.
 
+[skip]
 prevrunning x
 -----------------
 
 Skip if a File Exists
 ^^^^^^^^^^^^^^^^^^^^^
 
 -----------------
+[skip]
 fileexists /path/to/a/file # skip if /path/to/a/file exists
 -----------------
+
+Skip if test ran more recently than specified time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.Skip if this test has been run in the past fifteen minutes and 15 seconds.
+-----------------
+[skip]
+rundelay 15m 15s
+-----------------
 
 Controlled waiver propagation
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
@@ -188,11 +229,13 @@
 
 Override the Toplevel HTML File
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Megatest generates a simple html file summary for top level tests of
-iterated tests. The generation can be overridden.
+iterated tests. The generation can be overridden. NOTE: the output of
+the script is captured from stdout to create the html.
+
 
 .For test "runfirst" override the toplevel generation with a script "mysummary.sh"
 -----------------
 # Override the rollup for specific tests
 [testrollup]

Index: docs/manual/writing_tests.txt
==================================================================
--- docs/manual/writing_tests.txt
+++ docs/manual/writing_tests.txt
@@ -1,13 +1,33 @@
 
 Writing Tests
 =============
 
-The First Chapter of the Second Part
-------------------------------------
-Chapters grouped into book parts are at level 1 and can contain
-sub-sections.
+Creating a new Test
+-------------------
+
+The following steps will add a test "yourtestname" to your testsuite. This assumes
+starting from a directory where you already have a megatest.config and
+runconfigs.config.
+
+. Create a directory tests/yourtestname
+. Create a file tests/yourtestname/testconfig
+
+.Contents of minimal testconfig
+--------------------
+[ezsteps]
+stepname1 stepname.sh
+
+# test_meta is a section for storing additional data on your test
+[test_meta]
+author myname
+owner  myname
+description An example test
+reviewed never
+--------------------
 
-
+This test runs a single step called "stepname1" which runs a script
+"stepname.sh". Note that although it is common to put the actions
+needed for a test step into a script it is not necessary.
 
 :numbered!:
 

Index: docs/plan.txt
==================================================================
--- docs/plan.txt
+++ docs/plan.txt
@@ -9,24 +9,15 @@
 -------------
 
 ww05 - migrate to inmem-db
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Keep as much the same as possible. Add internal reference to almost
-eliminate contention on db(s). 
-
-. Add internal reference db
-. Verify that actions are accessing correct db
-.. -runtests  - inmem
-.. -list-runs - local (but not megatest.db)
-.. dashboard  - local (but not megatest.db)
-. Mirror db to /var/tmp...
-. Dashboard read db from per-run db.
-. Dashboard read db from /var/tmp
-. Runs register in tasks table in monitor.db
-. Server polls tasks table for next action (in addition?)
-. Change run loop to execute in server, triggered by call to polling of tasks table
+. Switch to inmem db with fast sync to on disk db's [DONE]
+. Server polls tasks table for next action
+.. Task table used for tracking runner process [DONE]
+.. Task table used for jobs to run
+.. Task table used for queueing runner actions (remove runs, cleanRunExecute, etc)
 
 
 // ww32
 // ~~~~
 // 

Index: gutils.scm
==================================================================
--- gutils.scm
+++ gutils.scm
@@ -22,27 +22,33 @@
 	 (delta (map (lambda (a b)(abs (- a b))) c1 c2)))
     (null? (filter (lambda (x)(> x 3)) delta))))
 
 (define (gutils:get-color-for-state-status state status);; #!key (get-label #f))
   ;; ((if get-label cadr car)
-   (case (string->symbol state)
-     ((COMPLETED ARCHIVED)
-      (case (string->symbol status)
-	((PASS)        (list "70  249 73" status))
-	((WARN WAIVED) (list "255 172 13" status))
-	((SKIP)        (list "230 230 0" status))
-	(else (list "223 33 49" status))))
-     ;;      (if (equal? status "PASS")
-     ;;	  '("70 249 73" "PASS")
-     ;;	  (if (or (equal? status "WARN")
-     ;;		  (equal? status "WAIVED"))
-     ;;	      (list "255 172 13" status)
-     ;;	      (list "223 33 49"  status)))) ;; greenish orangeish redish
-     ((LAUNCHED)         (list "101 123 142"  state))
-     ((CHECK)            (list "255 100 50"   state))
-     ((REMOTEHOSTSTART)  (list "50 130 195"   state))
-     ((RUNNING)          (list "9 131 232"    state))
-     ((KILLREQ)          (list "39 82 206"    state))
-     ((KILLED)           (list "234 101 17"   state))
-     ((NOT_STARTED)      (list "240 240 240"  state))
-     (else               (list "192 192 192"  state))))
+  (case (string->symbol state)
+    ((COMPLETED) ;; ARCHIVED)
+     (case (string->symbol status)
+       ((PASS)        (list "70  249 73" status))
+       ((WARN WAIVED) (list "255 172 13" status))
+       ((SKIP)        (list "230 230 0" status))
+       (else (list "253 33 49" status))))
+    ((ARCHIVED)
+     (case (string->symbol status)
+       ((PASS)        (list "70  170 73" status))
+       ((WARN WAIVED) (list "200 130 13" status))
+       ((SKIP)        (list "180 180 0" status))
+       (else (list "180 33 49" status))))
+    ;;      (if (equal? status "PASS")
+    ;;	  '("70 249 73" "PASS")
+    ;;	  (if (or (equal? status "WARN")
+    ;;		  (equal? status "WAIVED"))
+    ;;	      (list "255 172 13" status)
+    ;;	      (list "223 33 49"  status)))) ;; greenish orangeish redish
+    ((LAUNCHED)         (list "101 123 142"  state))
+    ((CHECK)            (list "255 100 50"   state))
+    ((REMOTEHOSTSTART)  (list "50 130 195"   state))
+    ((RUNNING)          (list "9 131 232"    state))
+    ((KILLREQ)          (list "39 82 206"    state))
+    ((KILLED)           (list "234 101 17"   state))
+    ((NOT_STARTED)      (list "240 240 240"  state))
+    (else               (list "192 192 192"  state))))
 

Index: megatest-version.scm
==================================================================
--- megatest-version.scm
+++ megatest-version.scm
@@ -1,7 +1,7 @@
 ;; Always use two or four digit decimal
 ;; 1.01, 1.02...1.10,1.11,1,1101 ... 1.99,2.00..
 
 (declare (unit megatest-version))
 
-(define megatest-version 1.6018)
+(define megatest-version 1.6019)
 

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -65,11 +65,11 @@
 ;;                            from prior runs with same keys
 
 (define help (conc "
 Megatest, documentation at http://www.kiatoa.com/fossils/megatest
   version " megatest-version "
-  license GPL, Copyright Matt Welland 2006-2012
+  license GPL, Copyright Matt Welland 2006-2015
 
 Usage: megatest [options]
   -h                      : this help
   -version                : print megatest version (currently " megatest-version ")
 
@@ -282,11 +282,11 @@
 			"-get-run-status"
 
 			;; queries
 			"-test-paths" ;; get path(s) to a test, ordered by youngest first
 
-			"-runall"    ;; run all tests, respects -testpatt
+			"-runall"    ;; run all tests, respects -testpatt, defaults to %
 			"-run"       ;; alias for -runall
 			"-remove-runs"
 			"-rebuild-db"
 			"-cleanup-db"
 			"-rollup"
@@ -304,10 +304,17 @@
 			"-q" ;; quiet 0, errors/warnings only
 		       )
 		 args:arg-hash
 		 0))
 
+(if (and (not (null? remargs))
+	 (not (or
+	       (args:get-arg "-runstep"))
+	      ;; add more args that use remargs here
+	      ))
+    (debug:print 0 "ERROR: Unrecognised arguments: " (string-intersperse (if (list? remargs) remargs (argv))  " ")))
+
 ;; The watchdog is to keep an eye on things like db sync etc.
 ;;
 (define *time-zero* (current-seconds))
 (define *watchdog*
   (make-thread 
@@ -1138,11 +1145,12 @@
      "-runall"
      "run all tests"
      (lambda (target runname keys keyvals)
        (runs:run-tests target
 		       runname
-		       (args:get-arg "-testpatt")
+		       (or (args:get-arg "-testpatt")
+			   "%")
 		       user
 		       args:arg-hash))))
 
 ;;======================================================================
 ;; run one test

Index: mt.scm
==================================================================
--- mt.scm
+++ mt.scm
@@ -6,11 +6,11 @@
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
 ;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 ;;  PURPOSE.
 
 
-(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) posix-extras directory-utils)
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) posix-extras directory-utils call-with-environment-variables)
 (import (prefix sqlite3 sqlite3:))
 
 (declare (unit mt))
 (declare (uses db))
 (declare (uses common))
@@ -140,28 +140,33 @@
 	 (state         (if newstate  newstate  (db:test-get-state  test-dat)))
 	 (status        (if newstatus newstatus (db:test-get-status test-dat))))
     (if (and test-rundir   ;; #f means no dir set yet
 	     (file-exists? test-rundir)
 	     (directory? test-rundir))
-	(begin
-	  (push-directory test-rundir)
-	  (set! tconfig (mt:lazy-read-test-config test-name))
-	  (pop-directory)
-	  (for-each (lambda (trigger)
-		      (let ((cmd  (configf:lookup tconfig "triggers" trigger))
-			    (logf (conc  test-rundir "/last-trigger.log")))
-			(if cmd
-			    ;; Putting the commandline into ( )'s means no control over the shell. 
-			    ;; stdout and stderr will be caught in the NBFAKE or mt_launch.log files
-			    ;; or equivalent. No need to do this. Just run it?
-			    (let ((fullcmd (conc cmd " " test-id " " test-rundir " " trigger "&")))
-			      (debug:print-info 0 "TRIGGERED on " trigger ", running command " fullcmd)
-			      (process-run fullcmd)))))
-		    (list
-		     (conc state "/" status)
-		     (conc state "/")
-		     (conc "/" status)))))))
+	(call-with-environment-variables
+	 (list (cons "MT_TEST_NAME" test-name)
+	       (cons "MT_TEST_RUN_DIR" test-rundir)
+	       (cons "MT_ITEMPATH"     (db:test-get-item-path test-dat)))
+	 (lambda ()
+	   (push-directory test-rundir)
+	   (set! tconfig (mt:lazy-read-test-config test-name))
+	   (for-each (lambda (trigger)
+		       (let ((cmd  (configf:lookup tconfig "triggers" trigger))
+			     (logf (conc  test-rundir "/last-trigger.log")))
+			 (if cmd
+			     ;; Putting the commandline into ( )'s means no control over the shell. 
+			     ;; stdout and stderr will be caught in the NBFAKE or mt_launch.log files
+			     ;; or equivalent. No need to do this. Just run it?
+			     (let ((fullcmd (conc cmd " " test-id " " test-rundir " " trigger "&")))
+			       (debug:print-info 0 "TRIGGERED on " trigger ", running command " fullcmd)
+			       (process-run fullcmd)))))
+		     (list
+		      (conc state "/" status)
+		      (conc state "/")
+		      (conc "/" status)))
+	   (pop-directory))
+	  ))))
 
 ;;======================================================================
 ;;  S T A T E   A N D   S T A T U S   F O R   T E S T S 
 ;;======================================================================
 

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -34,10 +34,12 @@
   (let* ((testname (db:test-get-testname   test))
 	 (itempath (db:test-get-item-path test)))
     (conc testname (if (equal? itempath "") "" (conc "(" itempath ")")))))
 
 ;; This is the *new* methodology. One record to inform them and in the chaos, organise them.
+;;
+;; NOT YET UTILIZED
 ;;
 (define (runs:create-run-record)
   (let* ((mconfig      (if *configdat*
 		           *configdat*
 		           (if (launch:setup-for-run)
@@ -216,11 +218,11 @@
 	 (test-records       (make-hash-table))
 	 ;; need to process runconfigs before generating these lists
 	 (all-tests-registry #f)  ;; (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names
 	 (all-test-names     #f)  ;; (hash-table-keys all-tests-registry))
 	 (test-names         #f)  ;; (tests:filter-test-names all-test-names test-patts))
-	 (required-tests     #f)  ;;(lset-intersection equal? (string-split test-patts ",") test-names))) ;; test-names)) ;; Added test-names as initial for required-tests but that failed to work
+	 (required-tests     #f)  ;; Put fully qualified test/testpath names in this list to be done
 	 (task-key           (conc (hash-table->alist flags) " " (get-host-name) " " (current-process-id)))
 	 (tdbdat             (tasks:open-db)))
 
     (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
 
@@ -259,10 +261,17 @@
     (set! all-tests-registry (tests:get-all))
     (set! all-test-names     (hash-table-keys all-tests-registry))
     (set! test-names         (tests:filter-test-names all-test-names test-patts))
 
     ;; I think seeding required-tests with all test-names makes sense but lack analysis to back that up.
+
+    ;; NEW STRATEGY HERE:
+    ;; 1. fill required tests with test-patts
+    ;; 2. scan testconfigs and if waitons, itemwait, itempatt calc prior test test-patt
+    ;; 3. repeat until all deps propagated
+    
+    ;; any tests with direct mention in test-patts can be added to required
     ;;
     (set! required-tests     (lset-intersection equal? (string-split test-patts ",") all-test-names))
     ;; (set! required-tests     (lset-intersection equal? test-names all-test-names))
     
     ;; look up all tests matching the comma separated list of globs in
@@ -302,10 +311,11 @@
     ;; refactoring this block into tests:get-full-data
     ;;
     ;; What happended, this code is now duplicated in tests!?
     ;;
     ;;======================================================================
+    
     (if (not (null? test-names))
 	(let loop ((hed (car test-names))
 		   (tal (cdr test-names)))         ;; 'return-procs tells the config reader to prep running system but return a proc
 	  (change-directory *toppath*) ;; PLEASE OPTIMIZE ME!!! I think this should be a no-op but there are several places where change-directories could be happening.
 	  (setenv "MT_TEST_NAME" hed) ;; 
@@ -376,20 +386,58 @@
 					     #f      ;; spare - used for item-path
 					     )))
 	    (for-each 
 	     (lambda (waiton)
 	       (if (and waiton (not (member waiton test-names)))
-		   (begin
-		     (set! required-tests (cons waiton required-tests))
-		     (set! test-names (cons waiton test-names))))) ;; was an append, now a cons
+		   (let* ((new-test-patts  (tests:extend-test-patts test-patts hed waiton #f))
+			  (waiton-record   (hash-table-ref/default test-records waiton #f))
+			  (waiton-tconfig  (if waiton-record (vector-ref waiton-record 1) #f))
+			  (waiton-itemized (and waiton-tconfig
+						(or (hash-table-ref/default waiton-tconfig "items" #f)
+						    (hash-table-ref/default waiton-tconfig "itemstable" #f)))))
+		     (debug:print-info 0 "Test " waiton " has " (if waiton-record "a" "no") " waiton-record and" (if waiton-itemized " " " no ") "items")
+		     ;; need to account for test-patt here, if I am test "a", selected with a test-patt of "hed/b%"
+		     ;; and we are waiting on "waiton" we need to add "waiton/,waiton/b%" to test-patt
+		     ;; is this satisfied by merely appending "/" to the waiton name added to the list?
+		     ;;
+		     ;; This approach causes all of the items in an upstream test to be run 
+
+		     ;; if we have this waiton already processed once we can analzye it for extending
+		     ;; tests to be run, since we can't properly process waitons unless they have been
+		     ;; initially added we add them again to be processed on second round AND add the hed
+		     ;; back in to also be processed on second round
+		     ;;
+		     (if waiton-tconfig
+			 (begin
+			   (set! test-names (cons waiton test-names)) ;; need to process this one, only add once the waiton tconfig read
+			   (if waiton-itemized
+			       (begin
+				 (debug:print-info 0 "New test patts: " new-test-patts ", prev test patts: " test-patts)
+				 (set! required-tests (cons (conc waiton "/") required-tests))
+				 (set! test-patts new-test-patts))
+			       (begin
+				 (debug:print-info 0 "Adding non-itemized test " waiton " to required-tests")
+				 (set! required-tests (cons waiton required-tests)))))
+			 (begin
+			   (debug:print-info 0 "No testconfig info yet for " waiton ", setting up to re-process it")
+			   (set! tal (append (cons waiton tal)(list hed))))) ;; (cons (conc waiton "/") required-tests))
+			 
+		     ;; NOPE: didn't work. required needs to be plain test names. Try tacking on to test-patts
+		     ;;  - doesn't work
+		     ;; (set! test-patts (conc test-patts "," waiton "/"))
+		     
+		     ;; (set! test-names (cons waiton test-names))))) ;; was an append, now a cons
+		     )))
 	     waitons)
 	    (let ((remtests (delete-duplicates (append waitons tal))))
 	      (if (not (null? remtests))
-		  (loop (car remtests)(cdr remtests)))))))
+		  (begin
+		    (debug:print-info 0 "Preprocessing continues for " (string-intersperse remtests ", "))
+		    (loop (car remtests)(cdr remtests))))))))
 
     (if (not (null? required-tests))
-	(debug:print-info 1 "Adding " required-tests " to the run queue"))
+	(debug:print-info 1 "Adding \"" (string-intersperse required-tests " ") "\" to the run queue"))
     ;; NOTE: these are all parent tests, items are not expanded yet.
     (debug:print-info 4 "test-records=" (hash-table->alist test-records))
     (let ((reglen (configf:lookup *configdat* "setup" "runqueue")))
       (if (> (length (hash-table-keys test-records)) 0)
 	  (let* ((keep-going        #t)
@@ -427,10 +475,11 @@
 		(begin
 		  (if (not (hash-table-ref/default flags "-preclean" #f))
 		      (hash-table-set! flags "-preclean" #t))
 		  (if (not (hash-table-ref/default flags "-rerun" #f))
 		      (hash-table-set! flags "-rerun" "STUCK/DEAD,n/a,ZERO_ITEMS"))
+		  ;; recursive call to self
 		  (runs:run-tests target runname test-patts user flags run-count: (- run-count 1)))))
 	  (debug:print-info 0 "No tests to run")))
     (debug:print-info 4 "All done by here")
     (rmt:tasks-set-state-given-param-key task-key "done")
     ;; (sqlite3:finalize! tasks-db)

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -72,11 +72,28 @@
   (delete-duplicates
    (filter (lambda (testname)
 	     (tests:match test-patts testname #f))
 	   test-names)))
 
-;; tests:glob-like-match
+;; given test-b that is waiting on test-a extend test-patt appropriately
+;;
+(define (tests:extend-test-patts test-patt test-b test-a itemmap)
+  (let* ((patts      (string-split test-patt ","))
+	 (test-b-len (+ (string-length test-b) 1))
+	 (patts-b    (map (lambda (x)
+			    (let ((newpatt (conc test-a "/" (substring x test-b-len (string-length x)))))
+			      ;; (print "in map, x=" x ", newpatt=" newpatt)
+			      newpatt))
+			  (filter (lambda (x)
+				    (eq? (substring-index (conc test-b "/") x) 0))
+				  patts))))
+    (string-intersperse (delete-duplicates (append patts (if (null? patts-b)
+							     (list (conc test-a "/%"))
+							     patts-b)))
+			",")))
+  
+;; tests:glob-like-match 
 (define (tests:glob-like-match patt str) 
   (let ((like (substring-index "%" patt)))
     (let* ((notpatt  (equal? (substring-index "~" patt) 0))
 	   (newpatt  (if notpatt (substring patt 1) patt))
 	   (finpatt  (if like

Index: tests/fdktestqa/testqa/Makefile
==================================================================
--- tests/fdktestqa/testqa/Makefile
+++ tests/fdktestqa/testqa/Makefile
@@ -13,10 +13,18 @@
 bigbig :
 	for tn in a b c d;do \
 	   ($(MEGATEST) -runtests % -target a/b :runname $tn & ) ; \
 	done
 
+waitonpatt :
+	megatest -remove-runs -runname waitonpatt -target a/b -testpatt %
+	NUMTESTS=15 megatest -run -target a/b -runname waitonpatt -testpatt bigrun3/%8
+
+waitonall :
+	megatest -remove-runs -runname waitonall -target a/b -testpatt %
+	NUMTESTS=20 megatest -run -target a/b -runname waitonall -testpatt alltop
+
 bigrun :
 	NUMTESTS=$(NUMTESTS) $(MEGATEST) -runtests bigrun -target a/bigrun :runname a$(shell date +%V)
 
 bigrun2 :
 	NUMTESTS=$(NUMTESTS) $(MEGATEST) -runtests bigrun2 -target a/bigrun2 :runname a$(shell date +%V)
@@ -34,6 +42,5 @@
 compile :
 	(cd ../../..;make -j && make install)
 
 clean :
 	rm -rf ../simple*/*/* megatest.db db/* ../simple*/.db/* logs/* monitor.db
-

ADDED   tests/fdktestqa/testqa/tests/alltop/testconfig
Index: tests/fdktestqa/testqa/tests/alltop/testconfig
==================================================================
--- /dev/null
+++ tests/fdktestqa/testqa/tests/alltop/testconfig
@@ -0,0 +1,19 @@
+# Add additional steps here. Format is "stepname script"
+[vars]
+step1var step1.sh
+
+[ezsteps]
+step1 megatest -list-runs $MT_RUNNAME -target $MT_TARGET -itempatt %
+
+# Test requirements are specified here
+[requirements]
+waiton setup bigrun bigrun3 bigrun2
+priority 0
+
+# test_meta is a section for storing additional data on your test
+[test_meta]
+author matt
+owner  matt
+description An example test
+tags tagone,tagtwo
+reviewed never

Index: tests/fdktestqa/testqa/tests/bigrun/testconfig
==================================================================
--- tests/fdktestqa/testqa/tests/bigrun/testconfig
+++ tests/fdktestqa/testqa/tests/bigrun/testconfig
@@ -5,11 +5,11 @@
 [ezsteps]
 step1 #{get vars step1var}
 
 # Test requirements are specified here
 [requirements]
-# waiton setup
+waiton setup
 priority 0
 
 # Iteration for your tests are controlled by the items section
 [items]
 NUMBER #{scheme (string-intersperse (map number->string (sort (let loop ((a 0)(res '()))(if (<= a (or (any->number (get-environment-variable "NUMTESTS")) 2500))(loop (+ a 1)(cons a res)) res)) <)) " ")}

Index: tests/fdktestqa/testqa/tests/bigrun3/testconfig
==================================================================
--- tests/fdktestqa/testqa/tests/bigrun3/testconfig
+++ tests/fdktestqa/testqa/tests/bigrun3/testconfig
@@ -5,10 +5,18 @@
 # Test requirements are specified here
 [requirements]
 waiton bigrun2
 priority 0
 mode itemwait
+# pattern replacement
+#
+# Remove everything up to the last /
+# itemmap .*/
+#
+# Replace foo/ with bar/
+# itemmap foo/ bar/
+# 
 itemmap .*/
 
 # Iteration for your tests are controlled by the items section
 [items]
 NUMBER #{scheme (string-intersperse (map (lambda (x)(conc "blah/" x)) \

Index: tests/fullrun/megatest.config
==================================================================
--- tests/fullrun/megatest.config
+++ tests/fullrun/megatest.config
@@ -211,10 +211,13 @@
 plain hosts: xena, phoebe
 strong command: NBFAKE_HOST=zeus nbfake
 arm hosts: cubian
 
 [archive]
+
+# where to get bup executable
+# bup /path/to/bup
 
 # use machines of these flavor
 useflavors plain
 targsize 2G
 

ADDED   tests/release/tests/toprun/results.logpro
Index: tests/release/tests/toprun/results.logpro
==================================================================
--- /dev/null
+++ tests/release/tests/toprun/results.logpro
@@ -0,0 +1,140 @@
+;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com
+;;  
+;;   License GPL.
+
+(define logbody "LogFileBody")
+
+(define pass-specs '( ;; testname num-expected max-runtime
+		     ("exit_0"		        1  20)
+		     ("ezlog_fail_then_pass"    1  20)
+		     ("ezlog_pass"	        1  20)
+		     ("ez_pass"		        1  20)
+		     ("lineitem_pass"	        1  20)
+		     ("priority_1"	        1  20)
+		     ("priority_10"	        1  20)
+		     ("priority_10_waiton_1"    1  20)
+		     ("priority_3"	        1  20)
+		     ("priority_4"	        1  20)
+		     ;; ("priority_5"	        1  20)
+		     ("priority_6"	        1  20)
+;;		     ("priority_7"	        1  20)
+		     ("priority_8"	        1  20)
+		     ("priority_9"	        1  20)
+		     ("runfirst"	        7  20)
+		     ("singletest"	        1  20)
+		     ("singletest2"	        1  20)
+		     ("special"		        1  20)
+		     ("sqlitespeed"	       10  20)
+		     ("test1"		        1  20)
+		     ("test2"		        6  20)
+		     ("test_mt_vars"	        6  20)
+		     ))
+
+(define fail-specs '( ;; testname num-expected max-runtime
+		     ("exit_1"		        1  20)
+		     ("ez_exit2_fail"           1  20)
+		     ("ez_fail"		        1  20)
+		     ("ez_fail_quick"	        1  20)
+		     ("ezlog_fail"	        1  20)
+		     ("lineitem_fail"	        1  20)
+		     ("logpro_required_fail"    1  20)
+		     ("manual_example"	        1  20)
+		     ("neverrun"	        1  20)))
+		     
+(define warn-specs   '(("ezlog_warn"	        1  20)))
+
+(define nost-specs   '(("wait_no_items1"        1  20)
+		       ("wait_no_items2"        1  20)
+		       ("wait_no_items3"        1  20)
+		       ("wait_no_items4"        1  20)
+		       ;; ("no_items"              1  20)
+		       ))
+
+(define (check-one-test estate estatus testname count runtime)
+   (let* ((rxe      (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s")))
+	  (msg1     (conc testname " expecting count of " count))
+	  (msg2     (conc testname " expecting runtime less than " runtime)))
+     (expect:required in logbody = count msg1 rxe)
+     ;;(expect:value    in logbody count < msg2 rxe)
+     ))
+
+;; Special cases
+;;
+(expect:ignore   in logbody >= 0  "db_sync test might not have run"  #/Test: db_sync/)
+(expect:ignore   in logbody >= 0  "all_toplevel may not yet be done" #/Test: all_toplevel/)
+(expect:error    in logbody =  0  "tests left in RUNNING state"      #/State: RUNNING/)
+(expect:required in logbody =  1  "priority_2 is KILLED"             #/Test: priority_2\s+State: KILLED\s+Status: KILLED/)
+(expect:required in logbody =  1  "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
+(expect:required in logbody =  1  "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
+(expect:required in logbody =  1  "testxz has 1 NOT_STARTED test"    #/Test: testxz\s+State: NOT_STARTED/)
+(expect:required in logbody =  1  "no items"                         #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/)
+(expect:warning  in logbody =  1  "dynamic waiton"                   #/Test: dynamic_waiton/)
+(expect:required in logbody = 29  "blocktestxz has 29 tests"         #/Test: blocktestxz/)
+
+;; General cases
+;;
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "PASS" testdat))
+ pass-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "FAIL" testdat))
+ fail-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "WARN" testdat))
+ warn-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat))
+ nost-specs)
+
+;; Catch all.
+;;
+(expect:error    in logbody = 0   "Tests not accounted for"     #/Test: /)
+
+
+;; ;; define your hooks
+;; (hook:first-error   "echo \"Error hook activated: #{escaped errmsg}\"")
+;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"")
+;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"")
+;; 
+;; ;; first ensure your run at least started
+;; ;;
+;; (trigger "Init"     #/This is a header/)
+;; (trigger "InitEnd"  #/^\s*$/)
+;; (section "Init" "Init" "InitEnd")
+;; 
+;; (trigger "Body"     #/^.*$/) ;; anything starts the body
+;; ;; (trigger "EndBody"  #/This had better never match/)
+;; 
+;; (section "Body"     "Body" "EndBody")
+;; 
+;; (trigger "Blah2"    #/^begin Blah2/)
+;; (trigger "Blah2End" #/^end Blah2/)
+;; (section "Blah2"    "Blah2" "Blah2End")
+;; 
+;; (expect:required in "Init"  = 1 "Header"      #/This is a header/)
+;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/)
+;; (expect:value    in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/)
+;; (expect:value    in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/)
+;; (expect:value    in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/)
+;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/)
+;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/)
+;; 
+;; ;; Using match number
+;; (expect:value    in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2)
+;; 
+;; ;; Comparison instead of tolerance
+;; (expect:value    in "LogFileBody" 1.9 >   "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2)
+;; 
+;; (expect:ignore   in "Blah2" < 99 "FALSE ERROR" #/ERROR/)
+;; (expect:ignore   in "Body"  < 99 "Ignore the word error in comments" #/^\/\/.*error/)
+;; (expect:warning  in "Body"  = 0 "Any warning" #/WARNING/)
+;; (expect:error    in "Body"  = 0 "ERROR BLAH"  (list #/ERROR/ #/error/)) ;; but disallow any other errors
+;; 
+;; ;(expect in "Init"  < 1 "Junk"        #/This is bogus/)

ADDED   tests/release/tests/toprun/testconfig
Index: tests/release/tests/toprun/testconfig
==================================================================
--- /dev/null
+++ tests/release/tests/toprun/testconfig
@@ -0,0 +1,15 @@
+[misc]
+rname release_toprun
+rdir  $MTTESTDIR/fullrun
+
+[ezsteps]
+cleantop $MTRUNNER #{get misc rdir} $MTPATH megatest -remove-runs -testpatt %     -target ubuntu/nfs/none -runname #{get misc rname} -testpatt %
+runall   $MTRUNNER #{get misc rdir} $MTPATH megatest -run -testpatt %             -target ubuntu/nfs/none -runname #{get misc rname} -runwait
+runtop   $MTRUNNER #{get misc rdir} $MTPATH megatest -runtests all_toplevel       -target ubuntu/nfs/none -runname #{get misc rname} -runwait
+results  $MTRUNNER #{get misc rdir} $MTPATH megatest -list-runs #{get misc rname} -target ubuntu/nfs/none
+
+[requirements]
+# waiton  #{getenv ALL_TOPLEVEL_TESTS}
+
+# This is a "toplevel" test, it does not require waitons to be non-FAIL to run
+# mode toplevel


Table 3. API Keys Related CallsTable 2. API Keys Related Calls