Index: docs/manual/Makefile
==================================================================
--- docs/manual/Makefile
+++ docs/manual/Makefile
@@ -28,17 +28,17 @@
 
 # design_spec.html : $(SRCFILES) $(CSVFILES)
 #         asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt
 #
 
-all : server.ps megatest_manual.html client.ps complex-itemmap.png
+all : server.ps megatest_manual.html client.ps complex-itemmap.png megatest_manual.pdf
 
 megatest_manual.html : megatest_manual.txt  *.txt installation.txt *png
 	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
 #	dos2unix megatest_manual.html
 
-megatest.pdf : megatest_manual.txt *.txt *png
+megatest_manual.pdf : megatest_manual.txt *.txt *png
 	a2x -a toc -f pdf megatest_manual.txt
 
 server.ps : server.dot
 	dot -Tps server.dot > server.ps
 

Index: docs/manual/complex-itemmap.png
==================================================================
--- docs/manual/complex-itemmap.png
+++ docs/manual/complex-itemmap.png
cannot compute difference between binary files

Index: docs/manual/getting_started.txt
==================================================================
--- docs/manual/getting_started.txt
+++ docs/manual/getting_started.txt
@@ -16,11 +16,11 @@
 // Copyright 2006-2012, Matthew Welland.
 
 Getting Started
 ---------------
 
-[partintro]
+// [partintro]
 .Getting started with Megatest
 -------------------
 Creating a testsuite or flow and your first test or task.
 -------------------
 

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="generator" content="AsciiDoc 8.6.7">
+<meta name="generator" content="AsciiDoc 8.6.9">
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -84,14 +84,20 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-pre {
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
   padding: 0;
   margin: 0;
 }
+pre {
+  white-space: pre-wrap;
+}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -216,11 +222,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; }
+span.image img { border-style: none; vertical-align: text-bottom; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -412,16 +418,10 @@
 /*
  * xhtml11 specific
  *
  * */
 
-tt {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 div.tableblock > table {
@@ -451,16 +451,10 @@
 /*
  * html5 specific
  *
  * */
 
-.monospaced {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 thead, p.tableblock.header {
@@ -536,10 +530,12 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
+
+
 @media screen {
   body {
     max-width: 50em; /* approximately 80 characters wide */
     margin-left: 16em;
   }
@@ -911,16 +907,1593 @@
 </div>
 </div>
 <div class="sect1">
 <h2 id="_getting_started">Getting Started</h2>
 <div class="sectionbody">
-<div class="openblock">
+<div class="listingblock">
 <div class="title">Getting started with Megatest</div>
+<div class="content monospaced">
+<pre>Creating a testsuite or flow and your first test or task.</pre>
+</div></div>
+<div class="paragraph"><p>After installing Megatest you can create a flow or testsuite and add some
+tests using the helpers. Here is a quickstart sequence to get you up and
+running your first automated testsuite.</p></div>
+<div class="sect2">
+<h3 id="_creating_a_megatest_area">Creating a Megatest Area</h3>
+<div class="sect3">
+<h4 id="_choose_target_keys">Choose Target Keys</h4>
+<div class="paragraph"><p>First choose your "target" keys. These are used to organise your runs in a
+way that is meaningful to your project. If you are unsure about what to use
+for keys just use a single generic key such as "RUNTYPE". These keys will be
+used to hand values to your tests via environment variables so ensure they
+are unique. Prefixing them with something such as PROJKEYS_ is a good
+strategy.</p></div>
+<div class="paragraph"><p>Examples of keys:</p></div>
+<table class="tableblock frame-all grid-all"
+style="
+width:60%;
+">
+<caption class="title">Table 1. Example keys</caption>
+<col style="width:50%;">
+<col style="width:50%;">
+<thead>
+<tr>
+<th class="tableblock halign-left valign-top" > Option            </th>
+<th class="tableblock halign-left valign-top" > Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">RELEASE/ITERATION</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">This example is used by Megatest for its internal QA.</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">ARCH/OS/RELEASE</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">For a software project targeting multiple platforms</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">UCTRLR/NODETYPE</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">Microcontroller project with different controllers
+running same software</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect3">
+<h4 id="_create_area_config_files">Create Area Config Files</h4>
+<div class="paragraph"><p>You will need to choose locations for your runs (the data generated every
+time you run the testsuite) and link tree. For getting started answer the
+prompts with "runs" and "links". We use the Unix editor "vi" in the examples
+below but you can use any plain text editor.</p></div>
+<div class="listingblock">
+<div class="title">Using the helper to create a Megatest area</div>
+<div class="content monospaced">
+<pre>megatest -create-megatest-area
+
+# optional: verify that the settings are ok
+vi megatest.config
+vi runconfigs.config</pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_creating_a_test">Creating a Test</h3>
+<div class="paragraph"><p>Choose the test name for your first test and run the helper. You can edit
+the files after the initial creation. You will need to enter names and
+scripts for the steps to be run and then edit the
+tests/&lt;testname&gt;/testconfig file and modify the logpro rules to properly
+process the log output from your steps. For your first test just hit enter
+for the "waiton", "priority" and iteration variable prompts.</p></div>
+<div class="paragraph"><p>Hint: for geting started make your logpro rules very liberal. expect:error
+patterns should match nothing and comment out expect:required rules.</p></div>
+<div class="listingblock">
+<div class="title">Using the helper to create a Megatest test</div>
+<div class="content monospaced">
+<pre>megatest -create-test myfirsttest
+
+# then edit the generated config
+vi tests/myfirsttest/testconfig</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_running_your_test">Running your test</h3>
+<div class="paragraph"><p>First choose a target and runname. If you have a two-place target such as
+RELEASE/ITERATION a target would look like v1.0/aff3 where v1.0 is the
+RELEASE and aff3 is the ITERATION. For a run name just use something like
+run1.</p></div>
+<div class="listingblock">
+<div class="title">Running all tests (testpatt of "%" matches all tests)</div>
+<div class="content monospaced">
+<pre>megatest -run -target v1.0/aff3 -runname run1 -testpatt % -log run1.log</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_viewing_the_results">Viewing the results</h3>
+<div class="paragraph"><p>Start the dashboard and browse your run in the "Runs" tab.</p></div>
+<div class="listingblock">
+<div class="title">Starting dashboard</div>
+<div class="content monospaced">
+<pre>dashboard -rows 24</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_study_plan">Study Plan</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Megatest is an extensive program with a lot to learn. Following are some paths through the material to smooth the learning path.</p></div>
+<div class="sect2">
+<h3 id="_basic_concepts_suggest_you_pick_these_up_on_the_way">Basic Concepts (suggest you pick these up on the way)</h3>
+<div class="ulist"><ul>
+<li>
+<p>
+Components of automation; run, test, iteration
+</p>
+</li>
+<li>
+<p>
+Selectors; target, runname, and testpatt
+</p>
+</li>
+</ul></div>
+</div>
+<div class="sect2">
+<h3 id="_running_testsuites_or_automation">Running Testsuites or Automation</h3>
+<div class="ulist"><ul>
+<li>
+<p>
+Using the dashboard gui (recommended)
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+Using the "Runs" panel.
+</p>
+</li>
+<li>
+<p>
+Using the "Run Control" panel.
+</p>
+</li>
+<li>
+<p>
+Using a test control panel
+</p>
+</li>
+<li>
+<p>
+The Right Mouse Button menu
+</p>
+</li>
+<li>
+<p>
+Debug features
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+xterm
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+pstree
+</p>
+</li>
+<li>
+<p>
+log files; mt_copy.log, mt_launch.log
+</p>
+</li>
+<li>
+<p>
+variables; megatest.csh, megatest.sh
+</p>
+</li>
+<li>
+<p>
+testconfig dump, *testconfig
+</p>
+</li>
+</ul></div>
+</li>
+<li>
+<p>
+State/status buttons
+</p>
+</li>
+<li>
+<p>
+Run, Clean, KillReq
+</p>
+</li>
+<li>
+<p>
+ReRunClean
+</p>
+</li>
+</ul></div>
+</li>
+</ul></div>
+</li>
+<li>
+<p>
+Using the command line
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+Getting help; megatest -h, megatest -manual
+</p>
+</li>
+<li>
+<p>
+Starting runs; megatest -run
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+Selection controls; -target, -runname and -testpatt
+</p>
+</li>
+</ul></div>
+</li>
+</ul></div>
+</li>
+</ul></div>
+</div>
+<div class="sect2">
+<h3 id="_writing_tests_and_flows">Writing Tests and Flows</h3>
+<div class="ulist"><ul>
+<li>
+<p>
+tests/<em>testname</em>/testconfig <a href="megatest_manual.html#_the_testconfig_file">testconfig details</a>
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+iteration (one test applied to many inputs) <a href="megatest_manual.html#_iteration">test iteration</a>
+</p>
+</li>
+<li>
+<p>
+dependencies <a href="megatest_manual.html#_requirements_section">test requirements</a>
+</p>
+</li>
+</ul></div>
+</li>
+<li>
+<p>
+megatest areas
+</p>
+<div class="ulist"><ul>
+<li>
+<p>
+megatest.config
+</p>
+</li>
+</ul></div>
+</li>
+</ul></div>
+</div>
+<div class="sect2">
+<h3 id="_advanced_topics">Advanced Topics</h3>
+<div class="ulist"><ul>
+<li>
+<p>
+Removing and keeping runs selectively <a href="megatest_manual.html#_managing_old_runs">managing runs</a>
+</p>
+</li>
+<li>
+<p>
+Subruns <a href="megatest_manual.html#_nested_runs">nested runs</a>
+</p>
+</li>
+<li>
+<p>
+Config file features <a href="megatest_manual.html#_config_file_helpers">config file features</a>
+</p>
+</li>
+</ul></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_writing_tests">Writing Tests</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_creating_a_new_test">Creating a new Test</h3>
+<div class="paragraph"><p>The following steps will add a test "yourtestname" to your testsuite. This assumes
+starting from a directory where you already have a megatest.config and
+runconfigs.config.</p></div>
+<div class="olist arabic"><ol class="arabic">
+<li>
+<p>
+Create a directory tests/yourtestname
+</p>
+</li>
+<li>
+<p>
+Create a file tests/yourtestname/testconfig
+</p>
+</li>
+</ol></div>
+<div class="listingblock">
+<div class="title">Contents of minimal testconfig</div>
+<div class="content monospaced">
+<pre>[ezsteps]
+stepname1 stepname.sh
+
+# test_meta is a section for storing additional data on your test
+[test_meta]
+author myname
+owner  myname
+description An example test
+reviewed never</pre>
+</div></div>
+<div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script
+"stepname.sh". Note that although it is common to put the actions
+needed for a test step into a script it is not necessary.</p></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_how_to_do_things">How To Do Things</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_process_runs">Process Runs</h3>
+<div class="sect3">
+<h4 id="_remove_runs">Remove Runs</h4>
+<div class="paragraph"><p>From the dashboard click on the button (PASS/FAIL&#8230;) for one of the tests. From the test control panel that
+comes up push the clean test button. The command field will be prefilled with a template command for removing
+that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.</p></div>
+<div class="listingblock">
+<div class="title">Remove the test diskperf and all it&#8217;s items</div>
+<div class="content monospaced">
+<pre>megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Remove all tests for all runs and all targets</div>
+<div class="content monospaced">
+<pre>megatest -remove-runs -target %/%/% -runname % -testpatt % -v</pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_archive_runs">Archive Runs</h4>
+<div class="paragraph"><p>Megatest supports using the bup backup tool (<a href="https://bup.github.io/">https://bup.github.io/</a>) to archive your tests for efficient storage
+and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run
+durations, time stamps etc.) are all preserved in the megatest database.</p></div>
+<div class="paragraph"><p>For setup information see the Archiving topic in the reference section of this manual.</p></div>
+<div class="sect4">
+<h5 id="_to_archive">To Archive</h5>
+<div class="paragraph"><p>Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.</p></div>
+<div class="listingblock">
+<div class="title">Archive a full run</div>
+<div class="content monospaced">
+<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %</pre>
+</div></div>
+</div>
+<div class="sect4">
+<h5 id="_to_restore">To Restore</h5>
+<div class="listingblock">
+<div class="title">Retrieve a single test</div>
+<div class="content monospaced">
+<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%</pre>
+</div></div>
+<div class="paragraph"><p>Hint: You can browse the archive using bup commands directly.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>bup -d /path/to/bup/archive ftp</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h3>
+<div class="listingblock">
+<div class="title">In megatest.config</div>
+<div class="content monospaced">
+<pre>[host-types]
+general ssh #{getbgesthost general}
+nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+
+[hosts]
+general cubian xena
+
+[launchers]
+envsetup general
+xor/%/n 4C16G
+% nbgeneral
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_tricks">Tricks</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
+configuring and generally getting the most out of Megatest.</p></div>
+<div class="sect2">
+<h3 id="_limiting_your_running_jobs">Limiting your running jobs</h3>
+<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
+<div class="paragraph"><p>In your testconfig:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[test_meta]
+jobgroup group1</pre>
+</div></div>
+<div class="paragraph"><p>In your megatest.config:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[jobgroups]
+group1 10
+custdes 4</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_debugging_tricks">Debugging Tricks</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_examining_the_environment">Examining The Environment</h3>
+<div class="sect3">
+<h4 id="_test_control_panel_xterm">Test Control Panel - xterm</h4>
+<div class="paragraph"><p>From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the
+window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run
+scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way
+to debug your tests.</p></div>
+</div>
+<div class="sect3">
+<h4 id="_during_config_file_processing">During Config File Processing</h4>
+<div class="paragraph"><p>It is often helpful to know the content of variables in various
+contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.</p></div>
+<div class="paragraph"><p>For example, if an item list is not being generated as expected you
+can inject the startup of an xterm as if it were an item:</p></div>
+<div class="listingblock">
+<div class="title">Original items table</div>
+<div class="content monospaced">
+<pre>[items]
+CELLNAME [system getcellname.sh]</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Items table modified for debug</div>
+<div class="content monospaced">
+<pre>[items]
+DEBUG [system xterm]
+CELLNAME [system getcellnames.sh]</pre>
+</div></div>
+<div class="paragraph"><p>When this test is run an xterm will pop up. In that xterm the
+environment is exactly that in which the script "getcellnames.sh"
+would run. You can now debug the script to find out why it isn&#8217;t
+working as expected.</p></div>
+</div>
+<div class="sect3">
+<h4 id="_organising_your_tests_and_tasks">Organising Your Tests and Tasks</h4>
+<div class="paragraph"><p>The default location "tests" for storing tests can be extended by
+adding to your tests-paths section.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[misc]
+parent #{shell dirname $(readlink -f .)}
+
+[tests-paths]
+1 #{get misc parent}/simplerun/tests</pre>
+</div></div>
+<div class="paragraph"><p>The above example shows how you can use addition sections in your
+config file to do complex processing. By putting results of relatively
+slow operations into variables the processing of your configs can be
+kept fast.</p></div>
+</div>
+<div class="sect3">
+<h4 id="_alternative_method_for_running_your_job_script">Alternative Method for Running your Job Script</h4>
+<div class="listingblock">
+<div class="title">Directly running job in testconfig</div>
+<div class="content monospaced">
+<pre>[setup]
+runscript main.csh</pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is essentially a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS and managing the details of running a test.</p></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_debugging_server_problems">Debugging Server Problems</h3>
+<div class="paragraph"><p>Some handy Unix commands to track down issues with servers not
+communicating with your test manager processes. Please put in tickets
+at <a href="https://www.kiatoa.com/fossils/megatest">https://www.kiatoa.com/fossils/megatest</a> if you have problems with
+servers getting stuck.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>sudo lsof -i
+sudo netstat -lptu
+sudo netstat -tulpn</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_reference">Reference</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_megatest_use_modes">Megatest Use Modes</h3>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:80%;
+">
+<caption class="title">Table 2. Base commands</caption>
+<col style="width:20%;">
+<col style="width:40%;">
+<col style="width:40%;">
+<thead>
+<tr>
+<th class="tableblock halign-center valign-top" >Use case                </th>
+<th class="tableblock halign-left valign-top" > Megatest command     </th>
+<th class="tableblock halign-left valign-top" >  mtutil</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">Start from scratch</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-rerun-all</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">restart</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">Rerun non-good completed</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-rerun-clean</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">rerunclean</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">Rerun all non-good and not completed yet</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-set-state-status KILLREQ; -rerun-</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">clean</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">killrerun</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Continue run</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-run</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">resume</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Remove run</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-remove-runs</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">clean</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Lock run</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-lock</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">lock</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Unlock run</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-unlock</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">unlock</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">killrun</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">-set-state-status KILLREQ; -kill-run</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect2">
+<h3 id="_config_file_helpers">Config File Helpers</h3>
+<div class="paragraph"><p>Various helpers for more advanced config files.</p></div>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:80%;
+">
+<caption class="title">Table 3. Helpers</caption>
+<col style="width:14%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<thead>
+<tr>
+<th class="tableblock halign-center valign-top" >Helper                      </th>
+<th class="tableblock halign-left valign-top" > Purpose                       </th>
+<th class="tableblock halign-left valign-top" > Valid values            </th>
+<th class="tableblock halign-left valign-top" > Comments</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{scheme (scheme code&#8230;)}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute arbitrary scheme code</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid scheme</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Value returned from the call is converted to a string and processed as part of the config file</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{system command}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute program, inserts exit code</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid Unix command</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Discards the output from the program</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{shell  command} or #{sh &#8230;}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Execute program, inserts result from stdout</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Any valid Unix command</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Value returned from the call is converted to a string and processed as part of the config file</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{realpath path} or #{rp &#8230;}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with normalized path</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Must be a valid path</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{getenv VAR} or #{gv VAR}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with content of env variable</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Must be a valid var</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{get s v} or #{g s v}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with variable v from section s</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Variable must be defined before use</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">#{rget v}</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with variable v from target or default of runconfigs file</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Replace with the path to the megatest testsuite area</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect2">
+<h3 id="_config_file_settings">Config File Settings</h3>
+<div class="paragraph"><p>Settings in megatest.config</p></div>
+</div>
+<div class="sect2">
+<h3 id="_config_file_additional_features">Config File Additional Features</h3>
+<div class="paragraph"><p>Including output from a script as if it was inline to the config file:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[scriptinc myscript.sh]</pre>
+</div></div>
+<div class="paragraph"><p>If the script outputs:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[items]
+A a b c
+B d e f</pre>
+</div></div>
+<div class="paragraph"><p>Then the config file would effectively appear to contain an items section
+exactly like the output from the script. This is useful when dynamically
+creating items, itemstables and other config structures. You can see the
+expansion of the call by looking in the cached files (look in your linktree
+for megatest.config and runconfigs.config cache files and in your test run
+areas for the expanded and cached testconfig).</p></div>
+<div class="paragraph"><p>Wildcards and regexes in Targets</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[a/2/b]
+VAR1 VAL1
+
+[a/%/b]
+VAR1 VAL2</pre>
+</div></div>
+<div class="paragraph"><p>Will result in:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[a/2/b]
+VAR1 VAL2</pre>
+</div></div>
+<div class="paragraph"><p>Can use either wildcard of "%" or a regular expression:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[/abc.*def/]</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_disk_space_checks">Disk Space Checks</h3>
+<div class="paragraph"><p>Some parameters you can put in the [setup] section of megatest.config:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre># minimum space required in a run disk
+minspace 10000000
+
+# minimum space required in dbdir:
+dbdir-space-required 100000
+
+# script that takes path as parameter and returns number of bytes available:
+free-space-script check-space.sh</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_trim_trailing_spaces">Trim trailing spaces</h3>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[configf:settings trim-trailing-spaces yes]</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_job_submission_control">Job Submission Control</h3>
+<div class="sect3">
+<h4 id="_submit_jobs_to_host_types_based_on_test_name_2">Submit jobs to Host Types based on Test Name</h4>
+<div class="listingblock">
+<div class="title">In megatest.config</div>
+<div class="content monospaced">
+<pre>[host-types]
+general   nbfake
+remote    bsub
+
+[launchers]
+runfirst/sum% remote
+% general
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless
+# there is no host-type match.
+flexi-launcher yes</pre>
+</div></div>
+<div class="sect4">
+<h5 id="_host_types">host-types</h5>
+<div class="paragraph"><p>List of host types and the commandline to run a job on that host type.</p></div>
+<div class="listingblock">
+<div class="title">host-type &#8658; launch command</div>
+<div class="content monospaced">
+<pre>general nbfake</pre>
+</div></div>
+</div>
+<div class="sect4">
+<h5 id="_launchers">launchers</h5>
+<div class="listingblock">
+<div class="title">test/itempath &#8658; host-type</div>
+<div class="content monospaced">
+<pre>runfirst/sum% remote</pre>
+</div></div>
+</div>
+<div class="sect4">
+<h5 id="_miscellaneous_setup_items">Miscellaneous Setup Items</h5>
+<div class="paragraph"><p>Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.</p></div>
+<div class="listingblock">
+<div class="title">In megatest.config</div>
+<div class="content monospaced">
+<pre>[setup]
+reruns 5</pre>
+</div></div>
+<div class="paragraph"><p>Replace the default blacklisted environment variables with user supplied
+list.</p></div>
+<div class="paragraph"><p>Default list: USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS MAKEF MAKEOVERRIDES</p></div>
+<div class="paragraph"><div class="title">Add a "bad" variable "PROMPT" to the variables that will be commented out</div><p>in the megatest.sh and megatest.csh files:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[setup]
+blacklistvars USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS PROMPT</pre>
+</div></div>
+</div>
+<div class="sect4">
+<h5 id="_run_time_limit">Run time limit</h5>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[setup]
+# this will automatically kill the test if it runs for more than 1h 2m and 3s
+runtimelim 1h 2m 3s</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_tests_browser_view">Tests browser view</h3>
+<div class="paragraph"><p>The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests.</p></div>
+<div class="olist arabic"><ol class="arabic">
+<li>
+<p>
+Dot (graphviz) based tree
+</p>
+</li>
+<li>
+<p>
+No dot, plain listing
+</p>
+</li>
+</ol></div>
+<div class="paragraph"><p>The default is the graphviz based tree but if your tests don&#8217;t view
+well in that mode then use "nodot" to turn it off.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[setup]
+nodot</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_capturing_test_data">Capturing Test Data</h3>
+<div class="paragraph"><p>In a test you can capture arbitrary variables and roll them up in the
+megatest database for viewing on the dashboard or web app.</p></div>
+<div class="listingblock">
+<div class="title">In a test as a script</div>
+<div class="content monospaced">
+<pre>$MT_MEGATEST -load-test-data &lt;&lt; EOF
+foo,bar,   1.2,  1.9, &gt;
+foo,rab, 1.0e9, 10e9, 1e9
+foo,bla,   1.2,  1.9, &lt;
+foo,bal,   1.2,  1.2, &lt;   ,     ,Check for overload
+foo,alb,   1.2,  1.2, &lt;=  , Amps,This is the high power circuit test
+foo,abl,   1.2,  1.3, 0.1
+foo,bra,   1.2, pass, silly stuff
+faz,bar,    10,  8mA,     ,     ,"this is a comment"
+EOF</pre>
+</div></div>
+<div class="paragraph"><p>Alternatively you can use logpro triggers to capture values and inject them
+into megatest using the -set-values mechanism:</p></div>
+<div class="listingblock">
+<div class="title">Megatest help related to -set-values</div>
+<div class="content monospaced">
+<pre>Test data capture
+  -set-values             : update or set values in the testdata table
+  :category               : set the category field (optional)
+  :variable               : set the variable name (optional)
+  :value                  : value measured (required)
+  :expected               : value expected (required)
+  :tol                    : |value-expect| &lt;= tol (required, can be &lt;, &gt;, &gt;=, &lt;= or number)
+  :units                  : name of the units for value, expected_value etc. (optional)</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_dashboard_settings">Dashboard settings</h3>
+<div class="listingblock">
+<div class="title">Runs tab buttons, font and size</div>
+<div class="content monospaced">
+<pre>[dashboard]
+btn-height x14
+btn-fontsz 10
+cell-width 60</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_database_settings">Database settings</h3>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:70%;
+">
+<caption class="title">Table 4. Database config settings in [setup] section of megatest.config</caption>
+<col style="width:14%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<thead>
+<tr>
+<th class="tableblock halign-center valign-top" >Var                       </th>
+<th class="tableblock halign-left valign-top" > Purpose            </th>
+<th class="tableblock halign-left valign-top" > Valid values            </th>
+<th class="tableblock halign-left valign-top" > Comments</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">delay-on-busy</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Prevent concurrent access issues</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">yes|no or not defined</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Default=no, may help on some network file systems, may slow things down also.</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">faststart</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">All direct file access to sqlite db files</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">yes|no or not defined</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Default=yes, suggest no for central automated systems and yes for interactive use</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">homehost</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Start servers on this host</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">&lt;hostname&gt;</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Defaults to local host</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">hostname</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Hostname to bind to</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">&lt;hostname&gt;|-</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">On multi-homed hosts allows binding to specific hostname</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">lowport</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Start searching for a port at this portnum</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">32768</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">required</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Server required</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">yes|no or not defined</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Default=no, force start of server always</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">server-query-threshold</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Start server when queries take longer than this</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">number in milliseconds</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Default=300</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">timeout</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">http api timeout</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">number in hours</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Default is 1 minute, do not change</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_the_testconfig_file">The testconfig File</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_setup_section">Setup section</h3>
+<div class="sect3">
+<h4 id="_header">Header</h4>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[setup]</pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>runscript main.csh</pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_iteration">Iteration</h3>
+<div class="listingblock">
+<div class="title">Sections for iteration</div>
+<div class="content monospaced">
+<pre># full combinations
+[items]
+A x y
+B 1 2
+
+# Yields: x/1 x/2 y/1 y/2
+
+# tabled
+[itemstable]
+A x y
+B 1 2
+
+# Yields x/1 y/2</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_requirements_section">Requirements section</h3>
+<div class="listingblock">
+<div class="title">Header</div>
+<div class="content monospaced">
+<pre>[requirements]</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_wait_on_other_tests">Wait on Other Tests</h3>
+<div class="listingblock">
+<div class="content monospaced">
+<pre># A normal waiton waits for the prior tests to be COMPLETED
+# and PASS, CHECK or WAIVED
+waiton test1 test2</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_mode">Mode</h3>
+<div class="paragraph"><p>The default (i.e. if mode is not specified) is normal. All pre-dependent tests
+must be COMPLETED and PASS, CHECK or WAIVED before the test will start</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[requirements]
+mode   normal</pre>
+</div></div>
+<div class="paragraph"><p>The toplevel mode requires only that the prior tests are COMPLETED.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[requirements]
+mode toplevel</pre>
+</div></div>
+<div class="paragraph"><p>A item based waiton will start items in a test when the same-named
+item is COMPLETED and PASS, CHECK or WAIVED in the prior test. This
+was historically called "itemwait" mode. The terms "itemwait" and
+"itemmatch" are synonyms.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[requirements]
+mode itemmatch</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_overriding_enviroment_variables">Overriding Enviroment Variables</h3>
+<div class="paragraph"><p>Override variables before starting the test. Can include files (perhaps generated by megatest -envdelta or similar).</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[pre-launch-env-vars]
+VAR1 value1
+
+# Get some generated settings
+[include ../generated-vars.config]
+
+# Use this trick to unset variables
+#{scheme (unsetenv "FOOBAR")}</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_itemmap_handling">Itemmap Handling</h3>
+<div class="paragraph"><p>For cases were the dependent test has a similar but not identical
+itempath to the downstream test an itemmap can allow for itemmatch
+mode</p></div>
+<div class="listingblock">
+<div class="title">example for removing part of itemmap for waiton test (eg: item <span class="monospaced">foo-x/bar</span> depends on waiton&#8217;s item <span class="monospaced">y/bar</span>)</div>
+<div class="content monospaced">
+<pre>[requirements]
+mode itemwait
+# itemmap &lt;item pattern for this test&gt;  &lt;item replacement pattern for waiton test&gt;
+itemmap .*x/ y/</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">example for removing part of itemmap for waiton test (eg: item <span class="monospaced">foo/bar/baz</span> in this test depends on waiton&#8217;s item <span class="monospaced">baz</span>)</div>
+<div class="content monospaced">
+<pre># ## pattern replacement notes
+#
+# ## Example
+# ## Remove everything up to the last /
+[requirements]
+mode itemwait
+# itemmap &lt;item pattern for this test&gt; &lt;nothing here indicates removal&gt;
+itemmap .*/</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">example replacing part of itemmap for (eg: item <span class="monospaced">foo/1234</span> will imply waiton&#8217;s item <span class="monospaced">bar/1234</span>)</div>
+<div class="content monospaced">
+<pre>#
+# ## Example
+# ## Replace foo/ with bar/
+[requirements]
+mode itemwait
+# itemmap &lt;item pattern for this test&gt;  &lt;item replacement pattern for waiton test&gt;
+itemmap foo/ bar/</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">example for backreference (eg: item <span class="monospaced">foo23/thud</span> will imply waiton&#8217;s item <span class="monospaced">num-23/bar/thud</span></div>
+<div class="content monospaced">
+<pre>#
+# ## Example
+# ## can use \{number} in replacement pattern to backreference a (capture) from matching pattern similar to sed or perl
+[requirements]
+mode itemwait
+# itemmap &lt;item pattern for this test&gt;  &lt;item replacement pattern for waiton test&gt;
+itemmap foo(\d+)/ num-\1/bar/</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">example multiple itemmaps</div>
+<div class="content monospaced">
+<pre># multi-line; matches are applied in the listed order
+# The following would map:
+#   a123b321 to b321fooa123 then to 321fooa123p
+#
+[requirements]
+itemmap (a\d+)(b\d+) \2foo\1
+  b(.*) \1p</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_complex_mapping">Complex mapping</h3>
+<div class="paragraph"><p>Complex mappings can be handled with a separate [itemmap] section (instead if an itemmap line in the [requirements] section)</p></div>
+<div class="paragraph"><p>Each line in an itemmap section starts with a waiton test name followed by an itemmap expression</p></div>
+<div class="listingblock">
+<div class="title">eg: The following causes waiton test A item <span class="monospaced">bar/1234</span> to run when our test&#8217;s <span class="monospaced">foo/1234</span> item is requested as well as causing waiton test B&#8217;s <span class="monospaced">blah</span> item to run when our test&#8217;s <span class="monospaced">stuff/blah</span> item is requested</div>
+<div class="content monospaced">
+<pre>[itemmap]
+A foo/ bar/
+B stuff/</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_complex_mapping_example">Complex mapping example</h3>
+<div class="imageblock">
 <div class="content">
-<div class="paragraph"><p>Creating a testsuite or flow and your first test or task.</p></div>
+<img src="complex-itemmap.png" alt="complex-itemmap.png">
+</div>
+</div>
+<div class="paragraph"><p>We accomplish this by configuring the testconfigs of our tests C D and E as follows:</p></div>
+<div class="listingblock">
+<div class="title">Testconfig for Test E has</div>
+<div class="content monospaced">
+<pre>[requirements]
+waiton C
+itemmap (\d+)/res \1/bb</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Testconfig for Test D has</div>
+<div class="content monospaced">
+<pre>[requirements]
+waiton C
+itemmap (\d+)/res \1/aa</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Testconfig for Test C has</div>
+<div class="content monospaced">
+<pre>[requirements]
+waiton A B
+
+[itemmap]
+A (\d+)/aa aa/\1
+B (\d+)/bb bb/\1</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Testconfigs for Test B and Test A have no waiton or itemmap configured</div>
+<div class="content monospaced">
+<pre></pre>
+</div></div>
+<div class="olist arabic"><div class="title">Walk through one item&#8201;&#8212;&#8201;we want the following to happen for testpatt <span class="monospaced">D/1/res</span> (see blue boxes in complex itemmaping figure above):</div><ol class="arabic">
+<li>
+<p>
+eg from command line <span class="monospaced">megatest -run -testpatt D/1/res -target mytarget -runname myrunname</span>
+</p>
+</li>
+<li>
+<p>
+Full list to be run is now: <span class="monospaced">D/1/res</span>
+</p>
+</li>
+<li>
+<p>
+Test D has a waiton - test C.  Test D&#8217;s itemmap rule <span class="monospaced">itemmap (\d&plus;)/res \1/aa</span> &#8594;  causes <span class="monospaced">C/1/aa</span> to run before <span class="monospaced">D/1/res</span>
+</p>
+</li>
+<li>
+<p>
+Full list to be run is now: <span class="monospaced">D/1/res</span>, <span class="monospaced">C/1/aa</span>
+</p>
+</li>
+<li>
+<p>
+Test C was a waiton - test A. Test C&#8217;s rule <span class="monospaced">A (\d&plus;)/aa aa/\1</span> &#8594; causes <span class="monospaced">A/aa/1</span> to run before <span class="monospaced">C/1/aa</span>
+</p>
+</li>
+<li>
+<p>
+Full list to be run is now: <span class="monospaced">D/1/res</span>, <span class="monospaced">C/1/aa</span>, <span class="monospaced">A/aa/1</span>
+</p>
+</li>
+<li>
+<p>
+Test A has no waitons.  All waitons of all tests in full list have been processed.  Full list is finalized.
+</p>
+</li>
+</ol></div>
+</div>
+<div class="sect2">
+<h3 id="_itemstable">itemstable</h3>
+<div class="paragraph"><p>An alternative to defining items is the itemstable section.  This lets you define the itempath in a table format rather than specifying components and relying on getting all permutations of those components.</p></div>
+</div>
+<div class="sect2">
+<h3 id="_dynamic_flow_dependency_tree">Dynamic Flow Dependency Tree</h3>
+<div class="listingblock">
+<div class="title">Autogeneration waiton list for dynamic flow dependency trees</div>
+<div class="content monospaced">
+<pre>[requirements]
+# With a toplevel test you may wish to generate your list
+# of tests to run dynamically
+#
+waiton #{shell get-valid-tests-to-run.sh}</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_run_time_limit_2">Run time limit</h3>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[requirements]
+runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_skip">Skip</h3>
+<div class="paragraph"><p>A test with a skip section will conditional skip running.</p></div>
+<div class="listingblock">
+<div class="title">Skip section example</div>
+<div class="content monospaced">
+<pre>[skip]
+prevrunning x
+# rundelay 30m 15s</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_skip_on_still_running_tests">Skip on Still-running Tests</h3>
+<div class="listingblock">
+<div class="content monospaced">
+<pre># NB// If the prevrunning line exists with *any* value the test will
+# automatically SKIP if the same-named test is currently RUNNING. The
+# "x" can be any string. Comment out the prevrunning line to turn off
+# skip.
+
+[skip]
+prevrunning x</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_skip_if_a_file_exists">Skip if a File Exists</h3>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[skip]
+fileexists /path/to/a/file # skip if /path/to/a/file exists</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_skip_if_test_ran_more_recently_than_specified_time">Skip if test ran more recently than specified time</h3>
+<div class="listingblock">
+<div class="title">Skip if this test has been run in the past fifteen minutes and 15 seconds.</div>
+<div class="content monospaced">
+<pre>[skip]
+rundelay 15m 15s</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_disks">Disks</h3>
+<div class="paragraph"><p>A disks section in testconfig will override the disks section in
+megatest.config. This can be used to allocate disks on a per-test or per item
+basis.</p></div>
+</div>
+<div class="sect2">
+<h3 id="_controlled_waiver_propagation">Controlled waiver propagation</h3>
+<div class="paragraph"><p>If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
+If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED</p></div>
+<div class="paragraph"><p>Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>###### EXAMPLE FROM testconfig #########
+# matching file(s) will be diff'd with previous run and logpro applied
+# if PASS or WARN result from logpro then WAIVER state is set
+#
+[waivers]
+# logpro_file    rulename      input_glob
+waiver_1         logpro        lookittmp.log
+
+[waiver_rules]
+
+# This builtin rule is the default if there is no &lt;waivername&gt;.logpro file
+# diff   diff %file1% %file2%
+
+# This builtin rule is applied if a &lt;waivername&gt;.logpro file exists
+# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_ezsteps">Ezsteps</h3>
+<div class="listingblock">
+<div class="title">Example ezsteps with logpro rules</div>
+<div class="content monospaced">
+<pre>[ezsteps]
+lookittmp   ls /tmp
+
+[logpro]
+lookittmp ;; Note: config file format supports multi-line entries where leading whitespace is removed from each line
+  ;;     a blank line indicates the end of the block of text
+  (expect:required in "LogFileBody" &gt; 0 "A file name that should never exist!" #/This is a awfully stupid file name that should never be found in the temp dir/)</pre>
+</div></div>
+<div class="paragraph"><p>To transfer the environment to the next step you can do the following:</p></div>
+<div class="listingblock">
+<div class="title">Propagate environment to next step</div>
+<div class="content monospaced">
+<pre>$MT_MEGATEST -env2file .ezsteps/${stepname}</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_triggers">Triggers</h3>
+<div class="paragraph"><p>In your testconfig or megatest.config triggers can be specified</p></div>
+<div class="listingblock">
+<div class="title">Triggers spec</div>
+<div class="content monospaced">
+<pre>[triggers]
+
+# Call script running.sh when test goes to state=RUNNING, status=PASS
+RUNNING/PASS running.sh
+
+# Call script running.sh any time state goes to RUNNING
+RUNNING/ running.sh
+
+# Call script onpass.sh any time status goes to PASS
+PASS/ onpass.sh</pre>
+</div></div>
+<div class="paragraph"><p>Scripts called will have; test-id test-rundir trigger test-name item-path state status event-time, added to the commandline.</p></div>
+<div class="paragraph"><p>HINT</p></div>
+<div class="paragraph"><p>To start an xterm (useful for debugging), use a command line like the following:</p></div>
+<div class="listingblock">
+<div class="title">Start an xterm using a trigger for test completed.</div>
+<div class="content monospaced">
+<pre>[triggers]
+COMPLETED/ xterm -e bash -s --</pre>
+</div></div>
+<div class="admonitionblock">
+<table><tr>
+<td class="icon">
+<img src="/usr/images/icons/note.png" alt="Note">
+</td>
+<td class="content">There is a trailing space after the double-dash</td>
+</tr></table>
+</div>
+<div class="paragraph"><p>There are a number of environment variables available to the trigger script
+but since triggers can be called in various contexts not all variables are
+available at all times. The trigger script should check for the variable and
+fail gracefully if it doesn&#8217;t exist.</p></div>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:90%;
+">
+<caption class="title">Table 5. Environment variables visible to the trigger script</caption>
+<col style="width:50%;">
+<col style="width:50%;">
+<thead>
+<tr>
+<th class="tableblock halign-left valign-top" > Variable            </th>
+<th class="tableblock halign-left valign-top" > Purpose</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_TEST_RUN_DIR</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The directory where Megatest ran this test</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_CMDINFO</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">Encoded command data for the test</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_DEBUG_MODE</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">Used to pass the debug mode to nested calls to Megatest</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_RUN_AREA_HOME</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">Megatest home area</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_TESTSUITENAME</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The name of this testsuite or area</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_TEST_NAME</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The name of this test</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_ITEM_INFO</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The variable and values for the test item</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_MEGATEST</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">Which Megatest binary is being used by this area</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_TARGET</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The target variable values, separated by <em>/</em></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_LINKTREE</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The base of the link tree where all run tests can be found</p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_ITEMPATH</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The values of the item path variables, separated by <em>/</em></p></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">MT_RUNNAME</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock">The name of the run</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect2">
+<h3 id="_override_the_toplevel_html_file">Override the Toplevel HTML File</h3>
+<div class="paragraph"><p>Megatest generates a simple html file summary for top level tests of
+iterated tests. The generation can be overridden. NOTE: the output of
+the script is captured from stdout to create the html.</p></div>
+<div class="listingblock">
+<div class="title">For test "runfirst" override the toplevel generation with a script "mysummary.sh"</div>
+<div class="content monospaced">
+<pre># Override the rollup for specific tests
+[testrollup]
+runfirst mysummary.sh</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_archiving_setup">Archiving Setup</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>In megatest.config add the following sections:</p></div>
+<div class="listingblock">
+<div class="title">megatest.config</div>
+<div class="content monospaced">
+<pre>[archive]
+# where to get bup executable
+# bup /path/to/bup
+
+[archive-disks]
+
+# Archives will be organised under these paths like this:
+#  &lt;testsuite&gt;/&lt;creationdate&gt;
+# Within the archive the data is structured like this:
+#  &lt;target&gt;/&lt;runname&gt;/&lt;test&gt;/
+archive0 /mfs/myarchive-data/adisk1</pre>
+</div></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_environment_variables">Environment Variables</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>It is often necessary to capture and or manipulate environment
+variables. Megatest has some facilities built in to help.</p></div>
+<div class="sect2">
+<h3 id="_capture_variables">Capture variables</h3>
+<div class="listingblock">
+<div class="title">Commands</div>
+<div class="content monospaced">
+<pre># capture the current enviroment into a db called envdat.db under
+# the context "before"
+megatest -envcap before
+
+# capture the current environment into a db called startup.db with
+# context "after"
+megatest -envcap after startup.db
+
+# write the diff from before to after
+megatest -envdelta before-after -dumpmode bash</pre>
+</div></div>
+<div class="paragraph"><p>Dump modes include bash, csh and config. You can include config data
+into megatest.config, runconfigs.config and testconfig files. This is
+useful for capturing a complex environment in a special-purpose test
+and then utilizing that environment in downstream tests.</p></div>
+<div class="listingblock">
+<div class="title">Example of generating and using config data</div>
+<div class="content monospaced">
+<pre>megatest -envcap original
+# do some stuff here
+megatest -envcap munged
+megatest -envdelta original-munged -dumpmode ini -o modified.config</pre>
+</div></div>
+<div class="paragraph"><p>Then in runconfigs.config</p></div>
+<div class="listingblock">
+<div class="title">Example of using modified.config in a testconfig</div>
+<div class="content monospaced">
+<pre>[pre-launch-env-vars]
+[include modified.config]</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_managing_old_runs">Managing Old Runs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>It is often desired to keep some older runs around but this must be balanced with the costs of disk space.</p></div>
+<div class="olist arabic"><ol class="arabic">
+<li>
+<p>
+Use -remove-keep
+</p>
+</li>
+<li>
+<p>
+Use -archive (can also be done from the -remove-keep interface)
+</p>
+</li>
+<li>
+<p>
+use -remove-runs with -keep-records
+</p>
+</li>
+</ol></div>
+<div class="listingblock">
+<div class="title">For each target, remove all runs but the most recent 3 if they are over 1 week old</div>
+<div class="content monospaced">
+<pre># use -precmd 'sleep 5;nbfake' to limit overloading the host computer but to allow the removes to run in parallel.
+megatest -actions print,remove-runs -remove-keep 3 -target %/%/%/% -runname % -age 1w -precmd 'sleep 5;nbfake'"</pre>
+</div></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_nested_runs">Nested Runs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>A Megatest test can run a full Megatest run in either the same
+Megatest area or in another area. This is a powerful way of chaining
+complex suites of tests and or actions.</p></div>
+<div class="paragraph"><p>If you are not using the current area you can use ezsteps to retrieve
+and setup the sub-Megatest run area.</p></div>
+<div class="paragraph"><p>In the testconfig:</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>[subrun]
+
+# Required: wait for the run or just launch it
+#           if no then the run will be an automatic PASS irrespective of the actual result
+run-wait yes|no
+
+# Optional: where to execute the run. Default is the current runarea
+run-area /some/path/to/megatest/area
+
+# Optional: method to use to determine pass/fail status of the run
+#   auto (default) - roll up the net state/status of the sub-run
+#   logpro         - use the provided logpro rules, happens automatically if there is a logpro section
+# passfail auto|logpro
+# Example of logpro:
+passfail logpro
+
+# Optional:
+logpro ;; if this section exists then logpro is used to determine pass/fail
+  (expect:required in "LogFileBody" &gt;= 1 "At least one pass" #/PASS/)
+  (expect:error     in "LogFileBody"  = 0 "No FAILs allowed"  #/FAIL/)
+
+# Optional: target translator, default is to use the parent target
+target #{shell somescript.sh}
+
+# Optional: runname translator/generator, default is to use the parent runname
+run-name #{somescript.sh}
+
+# Optional: testpatt spec, default is to first look for TESTPATT spec from runconfigs unless there is a contour spec
+test-patt %/item1,test2
+
+# Optional: contour spec, use the named contour from the megatest.config contour spec
+contour contourname ### NOTE: Not implemented yet! Let us know if you need this feature.
+
+# Optional: mode-patt, use this spec for testpatt from runconfigs
+mode-patt TESTPATT
+
+# Optional: tag-expr, use this tag-expr to select tests
+tag-expr quick
+
+# Optional: (not yet implemented, remove-runs is always propagated at this time), propagate these actions from the parent
+#           test
+#   Note// default is % for all
+propagate remove-runs archive ...</pre>
 </div></div>
 </div>
+</div>
+<div class="sect1">
+<h2 id="_programming_api">Programming API</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
+<table class="tableblock frame-topbot grid-all"
+style="
+width:70%;
+">
+<caption class="title">Table 6. API Keys Related Calls</caption>
+<col style="width:14%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<col style="width:28%;">
+<thead>
+<tr>
+<th class="tableblock halign-center valign-top" >API Call                        </th>
+<th class="tableblock halign-left valign-top" > Purpose comments   </th>
+<th class="tableblock halign-left valign-top" > Returns                 </th>
+<th class="tableblock halign-left valign-top" > Comments</th>
+</tr>
+</thead>
+<tfoot>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">(rmt:get-key-val-pairs run-id)</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">#t=success/#f=fail</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">Works only if the server is still reachable</p></td>
+</tr>
+</tfoot>
+<tbody>
+<tr>
+<td class="tableblock halign-center valign-top" ><p class="tableblock">(rmt:get-keys run-id)</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">( key1 key2 &#8230; )</p></td>
+<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
+</tr>
+</tbody>
+</table>
+</div>
 </div>
 <div class="sect1">
 <h2 id="_megatest_internals">Megatest Internals</h2>
 <div class="sectionbody">
 <div class="imageblock graphviz">
@@ -1292,10 +2865,11 @@
 </div>
 <div id="footnotes"><hr></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.0<br>
-Last updated 2018-10-24 15:52:36 PDT
+Last updated
+ 2018-10-24 19:03:47 MST
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -31,831 +31,836 @@
 |Lock run	         |      -lock	        |  lock	
 |Unlock run	         |      -unlock	        |  unlock	
 |killrun	                 | -set-state-status KILLREQ; -kill-run	| killrun	
 |======================
 
-// Config File Helpers
-// ~~~~~~~~~~~~~~~~~~~
-// 
-// Various helpers for more advanced config files.
-// 
-// .Helpers
-// [width="80%",cols="^,2m,2m,2m",frame="topbot",options="header"]
-// |======================
-// |Helper                      | Purpose                       | Valid values            | Comments
-// | #{scheme (scheme code...)} | Execute arbitrary scheme code | Any valid scheme        | Value returned from the call is converted to a string and processed as part of the config file
-// | #{system command}          | Execute program, inserts exit code  | Any valid Unix command  | Discards the output from the program
-// | #{shell  command} or #{sh ...}  | Execute program, inserts result from stdout | Any valid Unix command | Value returned from the call is converted to a string and processed as part of the config file
-// | #{realpath path} or #{rp ...}   | Replace with normalized path | Must be a valid path |
-// | #{getenv VAR} or #{gv VAR}      | Replace with content of env variable | Must be a valid var |
-// | #{get s v} or #{g s v}     | Replace with variable v from section s | Variable must be defined before use |
-// | #{rget v}                  | Replace with variable v from target or default of runconfigs file | |
-// | #{mtrah}                   | Replace with the path to the megatest testsuite area | | 
-// |======================
-// 
-// Config File Settings
-// ~~~~~~~~~~~~~~~~~~~~
-// 
-// Settings in megatest.config
-// 
-// Config File Additional Features
-// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// 
-// Including output from a script as if it was inline to the config file:
-// 
-// -------------------------
-// [scriptinc myscript.sh]
-// -------------------------
-// 
-// If the script outputs:
-// 
-// -------------------------
-// [items]
-// A a b c
-// B d e f
-// -------------------------
-// 
-// Then the config file would effectively appear to contain an items section
-// exactly like the output from the script. This is useful when dynamically
-// creating items, itemstables and other config structures. You can see the
-// expansion of the call by looking in the cached files (look in your linktree
-// for megatest.config and runconfigs.config cache files and in your test run
-// areas for the expanded and cached testconfig).
-// 
-// Wildcards and regexes in Targets
-// 
-// -------------------------
-// [a/2/b]
-// VAR1 VAL1
-// 
-// [a/%/b]
-// VAR1 VAL2
-// -------------------------
-// 
-// Will result in:
-// 
-// -------------------------
-// [a/2/b]
-// VAR1 VAL2
-// -------------------------
-// 
-// Can use either wildcard of "%" or a regular expression:
-// 
-// -------------------------
-// [/abc.*def/]
-// -------------------------
-// 
-// Disk Space Checks
-// ^^^^^^^^^^^^^^^^^
-// 
-// Some parameters you can put in the [setup] section of megatest.config:
-// 
-// -------------------
-// # minimum space required in a run disk 
-// minspace 10000000
-// 
-// # minimum space required in dbdir:
-// dbdir-space-required 100000
-// 
-// # script that takes path as parameter and returns number of bytes available:
-// free-space-script check-space.sh
-// -------------------
-// 
-// Trim trailing spaces
-// ^^^^^^^^^^^^^^^^^^^^
-// 
-// ------------------
-// [configf:settings trim-trailing-spaces yes]
-// ------------------
-// 
-// Job Submission Control
-// ^^^^^^^^^^^^^^^^^^^^^^
-// 
-// Submit jobs to Host Types based on Test Name
-// ++++++++++++++++++++++++++++++++++++++++++++
-// 
-// .In megatest.config
-// ------------------------
-// [host-types]
-// general   nbfake
-// remote    bsub
-// 
-// [launchers]
-// runfirst/sum% remote
-// % general
-// 
-// [jobtools]
-// launcher bsub
-// # if defined and not "no" flexi-launcher will bypass launcher unless 
-// # there is no host-type match.
-// flexi-launcher yes
-// ------------------------
-// 
-// host-types
-// ++++++++++
-// 
-// List of host types and the commandline to run a job on that host type.
-// 
-// .host-type => launch command
-// ------------
-// general nbfake
-// ------------
-// 
-// launchers
-// +++++++++
-// 
-// .test/itempath => host-type
-// ------------
-// runfirst/sum% remote
-// ------------
-// 
-// Miscellaneous Setup Items
-// +++++++++++++++++++++++++
-// 
-// Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.
-// 
-// .In megatest.config
-// ------------------
-// [setup]
-// reruns 5
-// ------------------
-// 
-// Replace the default blacklisted environment variables with user supplied
-// list.
-// 
-// Default list: USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS MAKEF MAKEOVERRIDES
-// 
-// .Add a "bad" variable "PROMPT" to the variables that will be commented out
-// in the megatest.sh and megatest.csh files:
-// -----------------
-// [setup]
-// blacklistvars USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS PROMPT
-// -----------------
-// 
-// Run time limit
-// ++++++++++++++
-// 
-// -----------------
-// [setup]
-// # this will automatically kill the test if it runs for more than 1h 2m and 3s
-// runtimelim 1h 2m 3s
-// -----------------
-// 
-// Tests browser view
-// ~~~~~~~~~~~~~~~~~~
-// 
-// The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests. 
-// 
-// . Dot (graphviz) based tree
-// . No dot, plain listing
-// 
-// The default is the graphviz based tree but if your tests don't view
-// well in that mode then use "nodot" to turn it off.
-// 
-// -----------------
-// [setup]
-// nodot
-// -----------------
-// 
-// Capturing Test Data
-// ~~~~~~~~~~~~~~~~~~~
-// 
-// In a test you can capture arbitrary variables and roll them up in the
-// megatest database for viewing on the dashboard or web app.
-// 
-// .In a test as a script
-// ------------------------
-// $MT_MEGATEST -load-test-data << EOF
-// foo,bar,   1.2,  1.9, >
-// foo,rab, 1.0e9, 10e9, 1e9
-// foo,bla,   1.2,  1.9, <
-// foo,bal,   1.2,  1.2, <   ,     ,Check for overload
-// foo,alb,   1.2,  1.2, <=  , Amps,This is the high power circuit test
-// foo,abl,   1.2,  1.3, 0.1
-// foo,bra,   1.2, pass, silly stuff
-// faz,bar,    10,  8mA,     ,     ,"this is a comment"
-// EOF
-// ------------------------
-// 
-// Alternatively you can use logpro triggers to capture values and inject them
-// into megatest using the -set-values mechanism:
-// 
-// .Megatest help related to -set-values
-// ------------------------
-// Test data capture
-//   -set-values             : update or set values in the testdata table
-//   :category               : set the category field (optional)
-//   :variable               : set the variable name (optional)
-//   :value                  : value measured (required)
-//   :expected               : value expected (required)
-//   :tol                    : |value-expect| <= tol (required, can be <, >, >=, <= or number)
-//   :units                  : name of the units for value, expected_value etc. (optional)
-// ------------------------
-// 
-// Dashboard settings
-// ~~~~~~~~~~~~~~~~~~
-// 
-// .Runs tab buttons, font and size
-// ------------------
-// [dashboard]
-// btn-height x14
-// btn-fontsz 10
-// cell-width 60
-// ------------------
-// 
-// Database settings
-// ~~~~~~~~~~~~~~~~~
-// 
-// .Database config settings in [setup] section of megatest.config
-// [width="70%",cols="^,2m,2m,2m",frame="topbot",options="header"]
-// |======================
-// |Var                       | Purpose            | Valid values            | Comments
-// |delay-on-busy             | Prevent concurrent access issues | yes\|no or not defined | Default=no, may help on some network file systems, may slow things down also.
-// |faststart		   | All direct file access to sqlite db files | yes\|no or not defined | Default=yes, suggest no for central automated systems and yes for interactive use
-// |homehost 		   | Start servers on this host | <hostname> | Defaults to local host
-// |hostname		   | Hostname to bind to | <hostname>\|-	  | On multi-homed hosts allows binding to specific hostname
-// |lowport		   | Start searching for a port at this portnum| 32768 | 
-// |required		   | Server required    | yes\|no or not defined  | Default=no, force start of server always
-// |server-query-threshold	   | Start server when queries take longer than this | number in milliseconds | Default=300
-// |timeout		   | http api timeout 	| number in hours	  | Default is 1 minute, do not change
-// |======================
-// 
-// The testconfig File
-// -------------------
-// 
-// Setup section
-// ~~~~~~~~~~~~~
-// 
-// Header
-// ^^^^^^
-// 
-// -------------------
-// [setup]
-// -------------------
-// 
-// The runscript method is a brute force way to run scripts where the
-// user is responsible for setting STATE and STATUS
-// 
-// -------------------
-// runscript main.csh
-// -------------------
-// 
-// Iteration
-// ~~~~~~~~~
-// 
-// .Sections for iteration
-// ------------------
-// # full combinations
-// [items]
-// A x y
-// B 1 2
-// 
-// # Yields: x/1 x/2 y/1 y/2
-// 
-// # tabled
-// [itemstable]
-// A x y
-// B 1 2
-// 
-// # Yields x/1 y/2
-// ------------------
-// 
-// 
-// Requirements section
-// ~~~~~~~~~~~~~~~~~~~~
-// 
-// .Header
-// -------------------
-// [requirements]
-// -------------------
-// 
-// Wait on Other Tests
-// ^^^^^^^^^^^^^^^^^^^
-// 
-// -------------------
-// # A normal waiton waits for the prior tests to be COMPLETED 
-// # and PASS, CHECK or WAIVED
-// waiton test1 test2
-// -------------------
-// 
-// Mode
-// ^^^^
-// 
-// The default (i.e. if mode is not specified) is normal. All pre-dependent tests
-// must be COMPLETED and PASS, CHECK or WAIVED before the test will start
-// 
-// -------------------
-// [requirements]
-// mode   normal
-// -------------------
-// 
-// The toplevel mode requires only that the prior tests are COMPLETED.
-// 
-// -------------------
-// [requirements]
-// mode toplevel
-// -------------------
-// 
-// A item based waiton will start items in a test when the same-named
-// item is COMPLETED and PASS, CHECK or WAIVED in the prior test. This
-// was historically called "itemwait" mode. The terms "itemwait" and
-// "itemmatch" are synonyms.
-// 
-// -------------------
-// [requirements]
-// mode itemmatch
-// -------------------
-// 
-// Overriding Enviroment Variables
-// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// 
-// Override variables before starting the test. Can include files (perhaps generated by megatest -envdelta or similar).
-// 
-// --------------------
-// [pre-launch-env-vars]
-// VAR1 value1
-// 
-// # Get some generated settings
-// [include ../generated-vars.config]
-// 
-// # Use this trick to unset variables
-// #{scheme (unsetenv "FOOBAR")}
-// --------------------
-// 
-// Itemmap Handling
-// ~~~~~~~~~~~~~~~~
-// 
-// For cases were the dependent test has a similar but not identical
-// itempath to the downstream test an itemmap can allow for itemmatch
-// mode
-// 
-// .example for removing part of itemmap for waiton test (eg: item +foo-x/bar+ depends on waiton's item +y/bar+)
-// -------------------
-// [requirements]
-// mode itemwait
-// # itemmap <item pattern for this test>  <item replacement pattern for waiton test>
-// itemmap .*x/ y/
-// 
-// -------------------
-// 
-// .example for removing part of itemmap for waiton test (eg: item +foo/bar/baz+ in this test depends on waiton's item +baz+)
-// -------------------
-// 
-// # ## pattern replacement notes
-// #
-// # ## Example
-// # ## Remove everything up to the last /
-// [requirements]
-// mode itemwait
-// # itemmap <item pattern for this test> <nothing here indicates removal>
-// itemmap .*/
-// -------------------
-// 
-// .example replacing part of itemmap for (eg: item +foo/1234+ will imply waiton's item +bar/1234+)
-// -------------------
-// 
-// #
-// # ## Example
-// # ## Replace foo/ with bar/
-// [requirements]
-// mode itemwait
-// # itemmap <item pattern for this test>  <item replacement pattern for waiton test>
-// itemmap foo/ bar/
-// 
-// -------------------
-// 
-// .example for backreference (eg: item +foo23/thud+ will imply waiton's item +num-23/bar/thud+
-// -------------------
-// #
-// # ## Example
-// # ## can use \{number} in replacement pattern to backreference a (capture) from matching pattern similar to sed or perl
-// [requirements]
-// mode itemwait
-// # itemmap <item pattern for this test>  <item replacement pattern for waiton test>
-// itemmap foo(\d+)/ num-\1/bar/
-// 
-// -------------------
-// 
-// .example multiple itemmaps
-// -------------------
-// 
-// # multi-line; matches are applied in the listed order
-// # The following would map:
-// #   a123b321 to b321fooa123 then to 321fooa123p
-// #
-// [requirements]
-// itemmap (a\d+)(b\d+) \2foo\1
-//   b(.*) \1p
-// -------------------
-// 
-// 
-// Complex mapping
-// ^^^^^^^^^^^^^^^
-// Complex mappings can be handled with a separate [itemmap] section (instead if an itemmap line in the [requirements] section)
-// 
-// Each line in an itemmap section starts with a waiton test name followed by an itemmap expression
-// 
-// .eg: The following causes waiton test A item +bar/1234+ to run when our test's +foo/1234+ item is requested as well as causing waiton test B's +blah+ item to run when our test's +stuff/blah+ item is requested
-// --------------
-// [itemmap]
-// A foo/ bar/
-// B stuff/
-// --------------
-// 
-// 
-// Complex mapping example
-// ^^^^^^^^^^^^^^^^^^^^^^^
-// 
-// 
-// 
-// // image::itemmap.png[]
-// image::complex-itemmap.png[]
-// 
-// 
-// We accomplish this by configuring the testconfigs of our tests C D and E as follows:
-// 
-// .Testconfig for Test E has
-// ----------------------
-// [requirements]
-// waiton C
-// itemmap (\d+)/res \1/bb
-// ----------------------
-// 
-// .Testconfig for Test D has
-// ----------------------
-// [requirements]
-// waiton C
-// itemmap (\d+)/res \1/aa
-// ----------------------
-// 
-// .Testconfig for Test C has
-// ----------------------
-// [requirements]
-// waiton A B
-// 
-// [itemmap]
-// A (\d+)/aa aa/\1
-// B (\d+)/bb bb/\1
-// ----------------------
-// 
-// .Testconfigs for Test B and Test A have no waiton or itemmap configured
-// -------------------
-// -------------------
-// 
-// .Walk through one item -- we want the following to happen for testpatt +D/1/res+ (see blue boxes in complex itemmaping figure above):
-// 
-// . eg from command line +megatest -run -testpatt D/1/res -target mytarget -runname myrunname+
-// . Full list to be run is now: +D/1/res+
-// . Test D has a waiton - test C.  Test D's itemmap rule +itemmap (\d&plus;)/res \1/aa+ ->  causes +C/1/aa+ to run before +D/1/res+
-// . Full list to be run is now: +D/1/res+, +C/1/aa+
-// . Test C was a waiton - test A. Test C's rule +A (\d&plus;)/aa aa/\1+ -> causes +A/aa/1+ to run before +C/1/aa+
-// . Full list to be run is now: +D/1/res+, +C/1/aa+, +A/aa/1+
-// . Test A has no waitons.  All waitons of all tests in full list have been processed.  Full list is finalized.
-// 
-// 
-// 
-// itemstable
-// ^^^^^^^^^^
-// An alternative to defining items is the itemstable section.  This lets you define the itempath in a table format rather than specifying components and relying on getting all permutations of those components.
-// 
-// 
-// 
-// 
-// 
-// Dynamic Flow Dependency Tree
-// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-// 
-// .Autogeneration waiton list for dynamic flow dependency trees
-// -------------------
-// [requirements]
-// # With a toplevel test you may wish to generate your list 
-// # of tests to run dynamically
-// #
-// waiton #{shell get-valid-tests-to-run.sh}
-// -------------------
-// 
-// Run time limit
-// ^^^^^^^^^^^^^^
-// 
-// -----------------
-// [requirements]
-// runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s
-// -----------------
-// 
-// Skip
-// ^^^^
-// 
-// A test with a skip section will conditional skip running.
-// 
-// .Skip section example
-// -----------------
-// [skip]
-// prevrunning x
-// # rundelay 30m 15s
-// -----------------
-// 
-// Skip on Still-running Tests
-// ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-// 
-// -----------------
-// # NB// If the prevrunning line exists with *any* value the test will
-// # automatically SKIP if the same-named test is currently RUNNING. The
-// # "x" can be any string. Comment out the prevrunning line to turn off
-// # skip.
-// 
-// [skip]
-// prevrunning x
-// -----------------
-// 
-// Skip if a File Exists
-// ^^^^^^^^^^^^^^^^^^^^^
-// 
-// -----------------
-// [skip]
-// fileexists /path/to/a/file # skip if /path/to/a/file exists
-// -----------------
-// 
-// Skip if test ran more recently than specified time
-// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-// 
-// .Skip if this test has been run in the past fifteen minutes and 15 seconds.
-// -----------------
-// [skip]
-// rundelay 15m 15s
-// -----------------
-// 
-// Disks
-// ^^^^^
-// 
-// A disks section in testconfig will override the disks section in
-// megatest.config. This can be used to allocate disks on a per-test or per item
-// basis.
-// 
-// Controlled waiver propagation
-// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-// 
-// If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
-// If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED
-// 
-// Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)
-// 
-// -----------------
-// ###### EXAMPLE FROM testconfig #########
-// # matching file(s) will be diff'd with previous run and logpro applied
-// # if PASS or WARN result from logpro then WAIVER state is set
-// #
-// [waivers]
-// # logpro_file    rulename      input_glob
-// waiver_1         logpro        lookittmp.log
-// 
-// [waiver_rules]
-// 
-// # This builtin rule is the default if there is no <waivername>.logpro file
-// # diff   diff %file1% %file2%
-// 
-// # This builtin rule is applied if a <waivername>.logpro file exists
-// # logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html
-// -----------------
-// 
-// Ezsteps
-// ~~~~~~~
-// 
-// .Example ezsteps with logpro rules
-// -----------------
-// [ezsteps]
-// lookittmp   ls /tmp
-// 
-// [logpro]
-// lookittmp ;; Note: config file format supports multi-line entries where leading whitespace is removed from each line
-//   ;;     a blank line indicates the end of the block of text 
-//   (expect:required in "LogFileBody" > 0 "A file name that should never exist!" #/This is a awfully stupid file name that should never be found in the temp dir/)
-// 
-// -----------------
-// 
-// To transfer the environment to the next step you can do the following:
-// 
-// ----------------------------
-// $MT_MEGATEST -env2file .ezsteps/${stepname}	  
-// ----------------------------
-// 
-// Triggers
-// ~~~~~~~~
-// 
-// In your testconfig or megatest.config triggers can be specified 
-// 
-// -----------------
-// [triggers]
-// 
-// # Call script running.sh when test goes to state=RUNNING, status=PASS
-// RUNNING/PASS running.sh
-// 
-// # Call script running.sh any time state goes to RUNNING
-// RUNNING/ running.sh
-// 
-// # Call script onpass.sh any time status goes to PASS
-// PASS/ onpass.sh
-// -----------------
-// 
-// Scripts called will have; test-id test-rundir trigger test-name item-path state status event-time, added to the commandline.
-// 
-// HINT
-// 
-// To start an xterm (useful for debugging), use a command line like the following:
-// 
-// -----------------
-// [triggers]
-// COMPLETED/ xterm -e bash -s -- 
-// -----------------
-// 
-// NOTE: There is a trailing space after the --
-// 
-// There are a number of environment variables available to the trigger script
-// but since triggers can be called in various contexts not all variables are
-// available at all times. The trigger script should check for the variable and
-// fail gracefully if it doesn't exist.
-// 
-// .Environment variables visible to the trigger script
-// [width="90%",cols="^,2m",frame="topbot",options="header"]
-// |======================
-// |Variable             | Purpose            
-// | MT_TEST_RUN_DIR     | The directory where Megatest ran this test                   
-// | MT_CMDINFO          | Encoded command data for the test                   
-// | MT_DEBUG_MODE       | Used to pass the debug mode to nested calls to Megatest                   
-// | MT_RUN_AREA_HOME    | Megatest home area 
-// | MT_TESTSUITENAME    | The name of this testsuite or area                   
-// | MT_TEST_NAME        | The name of this test
-// | MT_ITEM_INFO        | The variable and values for the test item
-// | MT_MEGATEST         | Which Megatest binary is being used by this area
-// | MT_TARGET           | The target variable values, separated by '/'
-// | MT_LINKTREE         | The base of the link tree where all run tests can be found
-// | MT_ITEMPATH         | The values of the item path variables, separated by '/'
-// | MT_RUNNAME          | The name of the run
-// |======================
-// 
-// 
-// Override the Toplevel HTML File
-// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// 
-// Megatest generates a simple html file summary for top level tests of
-// iterated tests. The generation can be overridden. NOTE: the output of
-// the script is captured from stdout to create the html.
-// 
-// 
-// .For test "runfirst" override the toplevel generation with a script "mysummary.sh"
-// -----------------
-// # Override the rollup for specific tests
-// [testrollup]
-// runfirst mysummary.sh
-// -----------------
-// 
-// Archiving Setup
-// ---------------
-// 
-// In megatest.config add the following sections:
-// 
-// .megatest.config
-// --------------
-// [archive]
-// # where to get bup executable
-// # bup /path/to/bup
-// 
-// [archive-disks]
-// 
-// # Archives will be organised under these paths like this:
-// #  <testsuite>/<creationdate>
-// # Within the archive the data is structured like this:
-// #  <target>/<runname>/<test>/
-// archive0 /mfs/myarchive-data/adisk1
-// --------------
-// 
-// Environment Variables
-// ---------------------
-// 
-// It is often necessary to capture and or manipulate environment
-// variables. Megatest has some facilities built in to help.
-// 
-// Capture variables
-// ~~~~~~~~~~~~~~~~~
-// 
-// .Commands
-// ------------------------------
-// # capture the current enviroment into a db called envdat.db under
-// # the context "before"
-// megatest -envcap before
-// 
-// # capture the current environment into a db called startup.db with 
-// # context "after"
-// megatest -envcap after startup.db
-// 
-// # write the diff from before to after
-// megatest -envdelta before-after -dumpmode bash
-// ------------------------------
-// 
-// Dump modes include bash, csh and config. You can include config data
-// into megatest.config, runconfigs.config and testconfig files. This is
-// useful for capturing a complex environment in a special-purpose test
-// and then utilizing that environment in downstream tests.
-// 
-// .Example of generating and using config data
-// ------------------------------
-// megatest -envcap original
-// # do some stuff here
-// megatest -envcap munged
-// megatest -envdelta original-munged -dumpmode ini -o modified.config
-// ------------------------------
-// 
-// Then in runconfigs.config
-// 
-// .Example of using modified.config in a testconfig
-// ------------------------------
-// [pre-launch-env-vars]
-// [include modified.config]
-// ------------------------------
-// 
-// Managing Old Runs
-// -----------------
-// 
-// It is often desired to keep some older runs around but this must be balanced with the costs of disk space.
-// 
-// . Use -remove-keep
-// . Use -archive (can also be done from the -remove-keep interface)
-// . use -remove-runs with -keep-records
-// 
-// .For each target, remove all runs but the most recent 3 if they are over 1 week old
-// ---------------------
-// # use -precmd 'sleep 5;nbfake' to limit overloading the host computer but to allow the removes to run in parallel.
-// megatest -actions print,remove-runs -remove-keep 3 -target %/%/%/% -runname % -age 1w -precmd 'sleep 5;nbfake'"
-// ---------------------
-// 
-// Nested Runs
-// -----------
-// 
-// A Megatest test can run a full Megatest run in either the same
-// Megatest area or in another area. This is a powerful way of chaining
-// complex suites of tests and or actions.
-// 
-// If you are not using the current area you can use ezsteps to retrieve
-// and setup the sub-Megatest run area.
-// 
-// In the testconfig:
-// ---------------
-// [subrun]
-// 
-// # Required: wait for the run or just launch it
-// #           if no then the run will be an automatic PASS irrespective of the actual result
-// run-wait yes|no
-// 
-// # Optional: where to execute the run. Default is the current runarea
-// run-area /some/path/to/megatest/area
-// 
-// # Optional: method to use to determine pass/fail status of the run
-// #   auto (default) - roll up the net state/status of the sub-run
-// #   logpro         - use the provided logpro rules, happens automatically if there is a logpro section
-// # passfail auto|logpro
-// # Example of logpro:
-// passfail logpro
-// 
-// # Optional: 
-// logpro ;; if this section exists then logpro is used to determine pass/fail
-//   (expect:required in "LogFileBody" >= 1 "At least one pass" #/PASS/)
-//   (expect:error     in "LogFileBody"  = 0 "No FAILs allowed"  #/FAIL/)
-// 
-// # Optional: target translator, default is to use the parent target
-// target #{shell somescript.sh}
-// 
-// # Optional: runname translator/generator, default is to use the parent runname
-// run-name #{somescript.sh}
-// 
-// # Optional: testpatt spec, default is to first look for TESTPATT spec from runconfigs unless there is a contour spec
-// test-patt %/item1,test2
-// 
-// # Optional: contour spec, use the named contour from the megatest.config contour spec
-// contour contourname ### NOTE: Not implemented yet! Let us know if you need this feature.
-// 
-// # Optional: mode-patt, use this spec for testpatt from runconfigs
-// mode-patt TESTPATT
-// 
-// # Optional: tag-expr, use this tag-expr to select tests
-// tag-expr quick
-// 
-// # Optional: (not yet implemented, remove-runs is always propagated at this time), propagate these actions from the parent
-// #           test
-// #   Note// default is % for all
-// propagate remove-runs archive ...
-// 
-// ---------------
-// 
-// Programming API
-// ---------------
-// 
-// These routines can be called from the megatest repl. 
-// 
-// .API Keys Related Calls
-// [width="70%",cols="^,2m,2m,2m",frame="topbot",options="header,footer"]
-// |======================
-// |API Call                        | Purpose comments   | Returns                 | Comments
-// |(rmt:get-keys run-id)           |                    | ( key1 key2 ... )       | 
-// | (rmt:get-key-val-pairs run-id) |                    | #t=success/#f=fail      | Works only if the server is still reachable
-// |======================
-// 
-// 
-// :numbered!:
+Config File Helpers
+~~~~~~~~~~~~~~~~~~~
+
+Various helpers for more advanced config files.
+
+.Helpers
+[width="80%",cols="^,2m,2m,2m",frame="topbot",options="header"]
+|======================
+|Helper                      | Purpose                       | Valid values            | Comments
+| #{scheme (scheme code...)} | Execute arbitrary scheme code | Any valid scheme        | Value returned from the call is converted to a string and processed as part of the config file
+| #{system command}          | Execute program, inserts exit code  | Any valid Unix command  | Discards the output from the program
+| #{shell  command} or #{sh ...}  | Execute program, inserts result from stdout | Any valid Unix command | Value returned from the call is converted to a string and processed as part of the config file
+| #{realpath path} or #{rp ...}   | Replace with normalized path | Must be a valid path |
+| #{getenv VAR} or #{gv VAR}      | Replace with content of env variable | Must be a valid var |
+| #{get s v} or #{g s v}     | Replace with variable v from section s | Variable must be defined before use |
+| #{rget v}                  | Replace with variable v from target or default of runconfigs file | |
+| #{mtrah}                   | Replace with the path to the megatest testsuite area | | 
+|======================
+
+Config File Settings
+~~~~~~~~~~~~~~~~~~~~
+
+Settings in megatest.config
+
+Config File Additional Features
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Including output from a script as if it was inline to the config file:
+
+-------------------------
+[scriptinc myscript.sh]
+-------------------------
+
+If the script outputs:
+
+-------------------------
+[items]
+A a b c
+B d e f
+-------------------------
+
+Then the config file would effectively appear to contain an items section
+exactly like the output from the script. This is useful when dynamically
+creating items, itemstables and other config structures. You can see the
+expansion of the call by looking in the cached files (look in your linktree
+for megatest.config and runconfigs.config cache files and in your test run
+areas for the expanded and cached testconfig).
+
+Wildcards and regexes in Targets
+
+-------------------------
+[a/2/b]
+VAR1 VAL1
+
+[a/%/b]
+VAR1 VAL2
+-------------------------
+
+Will result in:
+
+-------------------------
+[a/2/b]
+VAR1 VAL2
+-------------------------
+
+Can use either wildcard of "%" or a regular expression:
+
+-------------------------
+[/abc.*def/]
+-------------------------
+
+Disk Space Checks
+~~~~~~~~~~~~~~~~~
+
+Some parameters you can put in the [setup] section of megatest.config:
+
+-------------------
+# minimum space required in a run disk 
+minspace 10000000
+
+# minimum space required in dbdir:
+dbdir-space-required 100000
+
+# script that takes path as parameter and returns number of bytes available:
+free-space-script check-space.sh
+-------------------
+
+Trim trailing spaces
+~~~~~~~~~~~~~~~~~~~~
+
+------------------
+[configf:settings trim-trailing-spaces yes]
+------------------
+
+Job Submission Control
+~~~~~~~~~~~~~~~~~~~~~~
+
+Submit jobs to Host Types based on Test Name
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.In megatest.config
+------------------------
+[host-types]
+general   nbfake
+remote    bsub
+
+[launchers]
+runfirst/sum% remote
+% general
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless 
+# there is no host-type match.
+flexi-launcher yes
+------------------------
+
+host-types
+++++++++++
+
+List of host types and the commandline to run a job on that host type.
+
+.host-type => launch command
+------------
+general nbfake
+------------
+
+launchers
++++++++++
+
+.test/itempath => host-type
+------------
+runfirst/sum% remote
+------------
+
+Miscellaneous Setup Items
++++++++++++++++++++++++++
+
+Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.
+
+.In megatest.config
+------------------
+[setup]
+reruns 5
+------------------
+
+Replace the default blacklisted environment variables with user supplied
+list.
+
+Default list: USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS MAKEF MAKEOVERRIDES
+
+.Add a "bad" variable "PROMPT" to the variables that will be commented out
+in the megatest.sh and megatest.csh files:
+-----------------
+[setup]
+blacklistvars USER HOME DISPLAY LS_COLORS XKEYSYMDB EDITOR MAKEFLAGS PROMPT
+-----------------
+
+Run time limit
+++++++++++++++
+
+-----------------
+[setup]
+# this will automatically kill the test if it runs for more than 1h 2m and 3s
+runtimelim 1h 2m 3s
+-----------------
+
+Tests browser view
+~~~~~~~~~~~~~~~~~~
+
+The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests. 
+
+. Dot (graphviz) based tree
+. No dot, plain listing
+
+The default is the graphviz based tree but if your tests don't view
+well in that mode then use "nodot" to turn it off.
+
+-----------------
+[setup]
+nodot
+-----------------
+
+Capturing Test Data
+~~~~~~~~~~~~~~~~~~~
+
+In a test you can capture arbitrary variables and roll them up in the
+megatest database for viewing on the dashboard or web app.
+
+.In a test as a script
+------------------------
+$MT_MEGATEST -load-test-data << EOF
+foo,bar,   1.2,  1.9, >
+foo,rab, 1.0e9, 10e9, 1e9
+foo,bla,   1.2,  1.9, <
+foo,bal,   1.2,  1.2, <   ,     ,Check for overload
+foo,alb,   1.2,  1.2, <=  , Amps,This is the high power circuit test
+foo,abl,   1.2,  1.3, 0.1
+foo,bra,   1.2, pass, silly stuff
+faz,bar,    10,  8mA,     ,     ,"this is a comment"
+EOF
+------------------------
+
+Alternatively you can use logpro triggers to capture values and inject them
+into megatest using the -set-values mechanism:
+
+.Megatest help related to -set-values
+------------------------
+Test data capture
+  -set-values             : update or set values in the testdata table
+  :category               : set the category field (optional)
+  :variable               : set the variable name (optional)
+  :value                  : value measured (required)
+  :expected               : value expected (required)
+  :tol                    : |value-expect| <= tol (required, can be <, >, >=, <= or number)
+  :units                  : name of the units for value, expected_value etc. (optional)
+------------------------
+
+Dashboard settings
+~~~~~~~~~~~~~~~~~~
+
+.Runs tab buttons, font and size
+------------------
+[dashboard]
+btn-height x14
+btn-fontsz 10
+cell-width 60
+------------------
+
+Database settings
+~~~~~~~~~~~~~~~~~
+
+.Database config settings in [setup] section of megatest.config
+[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header"]
+|======================
+|Var                       | Purpose            | Valid values            | Comments
+|delay-on-busy             | Prevent concurrent access issues | yes\|no or not defined | Default=no, may help on some network file systems, may slow things down also.
+|faststart		   | All direct file access to sqlite db files | yes\|no or not defined | Default=yes, suggest no for central automated systems and yes for interactive use
+|homehost 		   | Start servers on this host | <hostname> | Defaults to local host
+|hostname		   | Hostname to bind to | <hostname>\|-	  | On multi-homed hosts allows binding to specific hostname
+|lowport		   | Start searching for a port at this portnum| 32768 | 
+|required		   | Server required    | yes\|no or not defined  | Default=no, force start of server always
+|server-query-threshold	   | Start server when queries take longer than this | number in milliseconds | Default=300
+|timeout		   | http api timeout 	| number in hours	  | Default is 1 minute, do not change
+|======================
+
+The testconfig File
+-------------------
+
+Setup section
+~~~~~~~~~~~~~
+
+Header
+^^^^^^
+
+-------------------
+[setup]
+-------------------
+
+The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS
+
+-------------------
+runscript main.csh
+-------------------
+
+Iteration
+~~~~~~~~~
+
+.Sections for iteration
+------------------
+# full combinations
+[items]
+A x y
+B 1 2
+
+# Yields: x/1 x/2 y/1 y/2
+
+# tabled
+[itemstable]
+A x y
+B 1 2
+
+# Yields x/1 y/2
+------------------
+
+
+Requirements section
+~~~~~~~~~~~~~~~~~~~~
+
+.Header
+-------------------
+[requirements]
+-------------------
+
+Wait on Other Tests
+~~~~~~~~~~~~~~~~~~~
+
+-------------------
+# A normal waiton waits for the prior tests to be COMPLETED 
+# and PASS, CHECK or WAIVED
+waiton test1 test2
+-------------------
+
+Mode
+~~~~
+
+The default (i.e. if mode is not specified) is normal. All pre-dependent tests
+must be COMPLETED and PASS, CHECK or WAIVED before the test will start
+
+-------------------
+[requirements]
+mode   normal
+-------------------
+
+The toplevel mode requires only that the prior tests are COMPLETED.
+
+-------------------
+[requirements]
+mode toplevel
+-------------------
+
+A item based waiton will start items in a test when the same-named
+item is COMPLETED and PASS, CHECK or WAIVED in the prior test. This
+was historically called "itemwait" mode. The terms "itemwait" and
+"itemmatch" are synonyms.
+
+-------------------
+[requirements]
+mode itemmatch
+-------------------
+
+Overriding Enviroment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Override variables before starting the test. Can include files (perhaps generated by megatest -envdelta or similar).
+
+--------------------
+[pre-launch-env-vars]
+VAR1 value1
+
+# Get some generated settings
+[include ../generated-vars.config]
+
+# Use this trick to unset variables
+#{scheme (unsetenv "FOOBAR")}
+--------------------
+
+Itemmap Handling
+~~~~~~~~~~~~~~~~
+
+For cases were the dependent test has a similar but not identical
+itempath to the downstream test an itemmap can allow for itemmatch
+mode
+
+.example for removing part of itemmap for waiton test (eg: item +foo-x/bar+ depends on waiton's item +y/bar+)
+-------------------
+[requirements]
+mode itemwait
+# itemmap <item pattern for this test>  <item replacement pattern for waiton test>
+itemmap .*x/ y/
+
+-------------------
+
+.example for removing part of itemmap for waiton test (eg: item +foo/bar/baz+ in this test depends on waiton's item +baz+)
+-------------------
+
+# ## pattern replacement notes
+#
+# ## Example
+# ## Remove everything up to the last /
+[requirements]
+mode itemwait
+# itemmap <item pattern for this test> <nothing here indicates removal>
+itemmap .*/
+-------------------
+
+.example replacing part of itemmap for (eg: item +foo/1234+ will imply waiton's item +bar/1234+)
+-------------------
+
+#
+# ## Example
+# ## Replace foo/ with bar/
+[requirements]
+mode itemwait
+# itemmap <item pattern for this test>  <item replacement pattern for waiton test>
+itemmap foo/ bar/
+
+-------------------
+
+.example for backreference (eg: item +foo23/thud+ will imply waiton's item +num-23/bar/thud+
+-------------------
+#
+# ## Example
+# ## can use \{number} in replacement pattern to backreference a (capture) from matching pattern similar to sed or perl
+[requirements]
+mode itemwait
+# itemmap <item pattern for this test>  <item replacement pattern for waiton test>
+itemmap foo(\d+)/ num-\1/bar/
+
+-------------------
+
+.example multiple itemmaps
+-------------------
+
+# multi-line; matches are applied in the listed order
+# The following would map:
+#   a123b321 to b321fooa123 then to 321fooa123p
+#
+[requirements]
+itemmap (a\d+)(b\d+) \2foo\1
+  b(.*) \1p
+-------------------
+
+
+Complex mapping
+~~~~~~~~~~~~~~~
+Complex mappings can be handled with a separate [itemmap] section (instead if an itemmap line in the [requirements] section)
+
+Each line in an itemmap section starts with a waiton test name followed by an itemmap expression
+
+.eg: The following causes waiton test A item +bar/1234+ to run when our test's +foo/1234+ item is requested as well as causing waiton test B's +blah+ item to run when our test's +stuff/blah+ item is requested
+--------------
+[itemmap]
+A foo/ bar/
+B stuff/
+--------------
+
+
+Complex mapping example
+~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
+// image::itemmap.png[]
+image::complex-itemmap.png[]
+
+
+We accomplish this by configuring the testconfigs of our tests C D and E as follows:
+
+.Testconfig for Test E has
+----------------------
+[requirements]
+waiton C
+itemmap (\d+)/res \1/bb
+----------------------
+
+.Testconfig for Test D has
+----------------------
+[requirements]
+waiton C
+itemmap (\d+)/res \1/aa
+----------------------
+
+.Testconfig for Test C has
+----------------------
+[requirements]
+waiton A B
+
+[itemmap]
+A (\d+)/aa aa/\1
+B (\d+)/bb bb/\1
+----------------------
+
+.Testconfigs for Test B and Test A have no waiton or itemmap configured
+-------------------
+-------------------
+
+.Walk through one item -- we want the following to happen for testpatt +D/1/res+ (see blue boxes in complex itemmaping figure above):
+
+. eg from command line +megatest -run -testpatt D/1/res -target mytarget -runname myrunname+
+. Full list to be run is now: +D/1/res+
+. Test D has a waiton - test C.  Test D's itemmap rule +itemmap (\d&plus;)/res \1/aa+ ->  causes +C/1/aa+ to run before +D/1/res+
+. Full list to be run is now: +D/1/res+, +C/1/aa+
+. Test C was a waiton - test A. Test C's rule +A (\d&plus;)/aa aa/\1+ -> causes +A/aa/1+ to run before +C/1/aa+
+. Full list to be run is now: +D/1/res+, +C/1/aa+, +A/aa/1+
+. Test A has no waitons.  All waitons of all tests in full list have been processed.  Full list is finalized.
+
+
+
+itemstable
+~~~~~~~~~~
+An alternative to defining items is the itemstable section.  This lets you define the itempath in a table format rather than specifying components and relying on getting all permutations of those components.
+
+
+
+
+
+Dynamic Flow Dependency Tree
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.Autogeneration waiton list for dynamic flow dependency trees
+-------------------
+[requirements]
+# With a toplevel test you may wish to generate your list 
+# of tests to run dynamically
+#
+waiton #{shell get-valid-tests-to-run.sh}
+-------------------
+
+Run time limit
+~~~~~~~~~~~~~~
+
+-----------------
+[requirements]
+runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s
+-----------------
+
+Skip
+~~~~
+
+A test with a skip section will conditional skip running.
+
+.Skip section example
+-----------------
+[skip]
+prevrunning x
+# rundelay 30m 15s
+-----------------
+
+Skip on Still-running Tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-----------------
+# NB// If the prevrunning line exists with *any* value the test will
+# automatically SKIP if the same-named test is currently RUNNING. The
+# "x" can be any string. Comment out the prevrunning line to turn off
+# skip.
+
+[skip]
+prevrunning x
+-----------------
+
+Skip if a File Exists
+~~~~~~~~~~~~~~~~~~~~~
+
+-----------------
+[skip]
+fileexists /path/to/a/file # skip if /path/to/a/file exists
+-----------------
+
+Skip if test ran more recently than specified time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.Skip if this test has been run in the past fifteen minutes and 15 seconds.
+-----------------
+[skip]
+rundelay 15m 15s
+-----------------
+
+Disks
+~~~~~
+
+A disks section in testconfig will override the disks section in
+megatest.config. This can be used to allocate disks on a per-test or per item
+basis.
+
+Controlled waiver propagation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
+If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED
+
+Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)
+
+-----------------
+###### EXAMPLE FROM testconfig #########
+# matching file(s) will be diff'd with previous run and logpro applied
+# if PASS or WARN result from logpro then WAIVER state is set
+#
+[waivers]
+# logpro_file    rulename      input_glob
+waiver_1         logpro        lookittmp.log
+
+[waiver_rules]
+
+# This builtin rule is the default if there is no <waivername>.logpro file
+# diff   diff %file1% %file2%
+
+# This builtin rule is applied if a <waivername>.logpro file exists
+# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html
+-----------------
+
+Ezsteps
+~~~~~~~
+
+.Example ezsteps with logpro rules
+-----------------
+[ezsteps]
+lookittmp   ls /tmp
+
+[logpro]
+lookittmp ;; Note: config file format supports multi-line entries where leading whitespace is removed from each line
+  ;;     a blank line indicates the end of the block of text 
+  (expect:required in "LogFileBody" > 0 "A file name that should never exist!" #/This is a awfully stupid file name that should never be found in the temp dir/)
+
+-----------------
+
+To transfer the environment to the next step you can do the following:
+
+.Propagate environment to next step
+----------------------------
+$MT_MEGATEST -env2file .ezsteps/${stepname}	  
+----------------------------
+
+Triggers
+~~~~~~~~
+
+In your testconfig or megatest.config triggers can be specified 
+
+.Triggers spec
+-----------------
+[triggers]
+
+# Call script running.sh when test goes to state=RUNNING, status=PASS
+RUNNING/PASS running.sh
+
+# Call script running.sh any time state goes to RUNNING
+RUNNING/ running.sh
+
+# Call script onpass.sh any time status goes to PASS
+PASS/ onpass.sh
+-----------------
+
+Scripts called will have; test-id test-rundir trigger test-name item-path state status event-time, added to the commandline.
+
+HINT
+
+To start an xterm (useful for debugging), use a command line like the following:
+
+.Start an xterm using a trigger for test completed.
+-----------------
+[triggers]
+COMPLETED/ xterm -e bash -s -- 
+-----------------
+
+NOTE: There is a trailing space after the double-dash
+
+There are a number of environment variables available to the trigger script
+but since triggers can be called in various contexts not all variables are
+available at all times. The trigger script should check for the variable and
+fail gracefully if it doesn't exist.
+
+// ,cols="^,2m"
+
+.Environment variables visible to the trigger script
+[width="90%",frame="topbot",options="header"]
+|======================
+| Variable            | Purpose            
+| MT_TEST_RUN_DIR     | The directory where Megatest ran this test                   
+| MT_CMDINFO          | Encoded command data for the test                   
+| MT_DEBUG_MODE       | Used to pass the debug mode to nested calls to Megatest                   
+| MT_RUN_AREA_HOME    | Megatest home area 
+| MT_TESTSUITENAME    | The name of this testsuite or area                   
+| MT_TEST_NAME        | The name of this test
+| MT_ITEM_INFO        | The variable and values for the test item
+| MT_MEGATEST         | Which Megatest binary is being used by this area
+| MT_TARGET           | The target variable values, separated by '/'
+| MT_LINKTREE         | The base of the link tree where all run tests can be found
+| MT_ITEMPATH         | The values of the item path variables, separated by '/'
+| MT_RUNNAME          | The name of the run
+|======================
+
+
+Override the Toplevel HTML File
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Megatest generates a simple html file summary for top level tests of
+iterated tests. The generation can be overridden. NOTE: the output of
+the script is captured from stdout to create the html.
+
+
+.For test "runfirst" override the toplevel generation with a script "mysummary.sh"
+-----------------
+# Override the rollup for specific tests
+[testrollup]
+runfirst mysummary.sh
+-----------------
+
+Archiving Setup
+---------------
+
+In megatest.config add the following sections:
+
+.megatest.config
+--------------
+[archive]
+# where to get bup executable
+# bup /path/to/bup
+
+[archive-disks]
+
+# Archives will be organised under these paths like this:
+#  <testsuite>/<creationdate>
+# Within the archive the data is structured like this:
+#  <target>/<runname>/<test>/
+archive0 /mfs/myarchive-data/adisk1
+--------------
+
+Environment Variables
+---------------------
+
+It is often necessary to capture and or manipulate environment
+variables. Megatest has some facilities built in to help.
+
+Capture variables
+~~~~~~~~~~~~~~~~~
+
+.Commands
+------------------------------
+# capture the current enviroment into a db called envdat.db under
+# the context "before"
+megatest -envcap before
+
+# capture the current environment into a db called startup.db with 
+# context "after"
+megatest -envcap after startup.db
+
+# write the diff from before to after
+megatest -envdelta before-after -dumpmode bash
+------------------------------
+
+Dump modes include bash, csh and config. You can include config data
+into megatest.config, runconfigs.config and testconfig files. This is
+useful for capturing a complex environment in a special-purpose test
+and then utilizing that environment in downstream tests.
+
+.Example of generating and using config data
+------------------------------
+megatest -envcap original
+# do some stuff here
+megatest -envcap munged
+megatest -envdelta original-munged -dumpmode ini -o modified.config
+------------------------------
+
+Then in runconfigs.config
+
+.Example of using modified.config in a testconfig
+------------------------------
+[pre-launch-env-vars]
+[include modified.config]
+------------------------------
+
+Managing Old Runs
+-----------------
+
+It is often desired to keep some older runs around but this must be balanced with the costs of disk space.
+
+. Use -remove-keep
+. Use -archive (can also be done from the -remove-keep interface)
+. use -remove-runs with -keep-records
+
+.For each target, remove all runs but the most recent 3 if they are over 1 week old
+---------------------
+# use -precmd 'sleep 5;nbfake' to limit overloading the host computer but to allow the removes to run in parallel.
+megatest -actions print,remove-runs -remove-keep 3 -target %/%/%/% -runname % -age 1w -precmd 'sleep 5;nbfake'"
+---------------------
+
+Nested Runs
+-----------
+
+A Megatest test can run a full Megatest run in either the same
+Megatest area or in another area. This is a powerful way of chaining
+complex suites of tests and or actions.
+
+If you are not using the current area you can use ezsteps to retrieve
+and setup the sub-Megatest run area.
+
+In the testconfig:
+---------------
+[subrun]
+
+# Required: wait for the run or just launch it
+#           if no then the run will be an automatic PASS irrespective of the actual result
+run-wait yes|no
+
+# Optional: where to execute the run. Default is the current runarea
+run-area /some/path/to/megatest/area
+
+# Optional: method to use to determine pass/fail status of the run
+#   auto (default) - roll up the net state/status of the sub-run
+#   logpro         - use the provided logpro rules, happens automatically if there is a logpro section
+# passfail auto|logpro
+# Example of logpro:
+passfail logpro
+
+# Optional: 
+logpro ;; if this section exists then logpro is used to determine pass/fail
+  (expect:required in "LogFileBody" >= 1 "At least one pass" #/PASS/)
+  (expect:error     in "LogFileBody"  = 0 "No FAILs allowed"  #/FAIL/)
+
+# Optional: target translator, default is to use the parent target
+target #{shell somescript.sh}
+
+# Optional: runname translator/generator, default is to use the parent runname
+run-name #{somescript.sh}
+
+# Optional: testpatt spec, default is to first look for TESTPATT spec from runconfigs unless there is a contour spec
+test-patt %/item1,test2
+
+# Optional: contour spec, use the named contour from the megatest.config contour spec
+contour contourname ### NOTE: Not implemented yet! Let us know if you need this feature.
+
+# Optional: mode-patt, use this spec for testpatt from runconfigs
+mode-patt TESTPATT
+
+# Optional: tag-expr, use this tag-expr to select tests
+tag-expr quick
+
+# Optional: (not yet implemented, remove-runs is always propagated at this time), propagate these actions from the parent
+#           test
+#   Note// default is % for all
+propagate remove-runs archive ...
+
+---------------
+
+Programming API
+---------------
+
+These routines can be called from the megatest repl. 
+
+.API Keys Related Calls
+[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header,footer"]
+|======================
+|API Call                        | Purpose comments   | Returns                 | Comments
+|(rmt:get-keys run-id)           |                    | ( key1 key2 ... )       | 
+| (rmt:get-key-val-pairs run-id) |                    | #t=success/#f=fail      | Works only if the server is still reachable
+|======================
+
+
+:numbered!:

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

