Index: NOTES
==================================================================
--- NOTES
+++ NOTES
@@ -1,5 +1,28 @@
+
+======================================================================
+New way of launching needed to accomodate different target hosttypes
+for items
+======================================================================
+
+[flavors]
+general ssh #{getbgesthost general}
+nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+
+[hosts]
+general cubian xena
+
+[launchers]
+envsetup general
+xor/%/n 4C16G
+% nbgeneral
+
+[jobtools]
+launcher internal
+
+
+
 ======================================================================
 Try writing to in-memory db and every 2-5 seconds syncing to megatest.db
 ======================================================================
 
 First, how much time will it take to write back the changes:

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -391,13 +391,35 @@
   (hash-table-ref/default 
    (or configf (read-config "megatest.config" #f #t))
    "disks" '("none" "")))
 
 ;;======================================================================
-;; T A R G E T S
+;; T A R G E T S  ,   S T A T E ,   S T A T U S ,   
+;;                    R U N N A M E    A N D   T E S T P A T T
 ;;======================================================================
 
+(define (common:args-get-state)
+  (or (args:get-arg "-state")(args:get-arg ":state")))
+
+(define (common:args-get-status)
+  (or (args:get-arg "-status")(args:get-arg ":status")))
+
+(define (common:args-get-testpatt rconf)
+  (let* ((rtestpatt     (if rconf (runconfigs-get rconf "TESTPATT") #f))
+	 (args-testpatt (or (args:get-arg "-testpatt")
+			    (args:get-arg "-runtests")
+			    "%"))
+	 (testpatt    (or (and (equal? args-testpatt "%")
+			       rtestpatt)
+			  args-testpatt)))
+    (if rtestpatt (debug:print-info 0 "TESTPATT from runconfigs: " rtestpatt))
+    testpatt))
+
+(define (common:args-get-runname)
+  (or (args:get-arg "-runname")
+      (args:get-arg ":runname")))
+
 (define (common:args-get-target #!key (split #f))
   (let* ((keys    (keys:config-get-fields *configdat*))
 	 (numkeys (length keys))
 	 (target  (if (args:get-arg "-reqtarg")
 		      (args:get-arg "-reqtarg")
@@ -928,6 +950,55 @@
 (define (mddb:get-dashboards)
   (let ((db (mddb:open-db)))
     (query fetch-column
 	   (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;"))))
     
-	
+;;======================================================================
+;;  T E S T   L A U N C H I N G   P E R   I T E M   W I T H   H O S T   T Y P E S
+;;======================================================================
+;; 
+;; [host-types]
+;; general ssh #{getbgesthost general}
+;; nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+;; 
+;; [hosts]
+;; general cubian xena
+;; 
+;; [launchers]
+;; envsetup general
+;; xor/%/n 4C16G
+;; % nbgeneral
+;; 
+;; [jobtools]
+;; launcher bsub
+;; # if defined and not "no" flexi-launcher will bypass launcher unless there is no
+;; # match.
+;; flexi-launcher yes  
+
+(define (common:get-launcher configdat testname itempath)
+  (let ((fallback-launcher (configf:lookup configdat "jobtools" "launcher")))
+    (if (and (configf:lookup configdat "jobtools" "flexi-launcher") ;; overrides launcher
+	     (not (equal? (configf:lookup configdat "jobtools" "flexi-launcher") "no")))
+	(let* ((launchers         (hash-table-ref/default configdat "launchers" '())))
+	  (if (null? launchers)
+	      fallback-launcher
+	      (let loop ((hed (car launchers))
+			 (tal (cdr launchers)))
+		(let ((patt      (car hed))
+		      (host-type (cadr hed)))
+		  (if (tests:match patt testname itempath)
+		      (begin
+			(debug:print-info 0 "Have flexi-launcher match for " testname "/" itempath " = " host-type)
+			(let ((launcher (configf:lookup configdat "host-types" host-type)))
+			  (if launcher
+			      launcher
+			      (begin
+				(debug:print-info 0 "WARNING: no launcher found for host-type " host-type)
+				(if (null? tal)
+				    fallback-launcher
+				    (loop (car tal)(cdr tal)))))))
+		      ;; no match, try again
+		      (if (null? tal)
+			  fallback-launcher
+			  (loop (car tal)(cdr tal))))))))
+	fallback-launcher)))
+  

Index: configf.scm
==================================================================
--- configf.scm
+++ configf.scm
@@ -118,11 +118,11 @@
 	      (print "ERROR: " cmd " returned bad exit code " status)))
 	  ""))))
 
 ;; Lookup a value in runconfigs based on -reqtarg or -target
 (define (runconfigs-get config var)
-  (let ((targ (or (args:get-arg "-reqtarg")(args:get-arg "-target")(getenv "MT_TARGET"))))
+  (let ((targ (common:args-get-target))) ;; (or (args:get-arg "-reqtarg")(args:get-arg "-target")(getenv "MT_TARGET"))))
     (if targ
 	(or (configf:lookup config targ var)
 	    (configf:lookup config "default" var))
 	(configf:lookup config "default" var))))
 

Index: docs/manual/Makefile
==================================================================
--- docs/manual/Makefile
+++ docs/manual/Makefile
@@ -13,11 +13,11 @@
 #         asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt
 #
 
 all : server.ps megatest_manual.html client.ps
 
-megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt
+megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt
 	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
 #	dos2unix megatest_manual.html
 
 server.ps : server.dot
 	dot -Tps server.dot > server.ps

Index: docs/manual/howto.txt
==================================================================
--- docs/manual/howto.txt
+++ docs/manual/howto.txt
@@ -1,9 +1,83 @@
 
 How To Do Things
 ================
 
+Remove Runs
+-----------
+
+From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that 
+comes up push the clean test button. The command field will be prefilled with a template command for removing 
+that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.
+
+.Remove the test diskperf and all it's items
+----------------
+megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v
+----------------
+
+.Remove all tests for all runs and all targets
+----------------
+megatest -remove-runs -target %/%/% -runname % -testpatt % -v
+----------------
+
+Archive Runs
+------------
+
+Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage
+and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run 
+durations, time stamps etc.) are all preserved in the megatest database.
+
+For setup information see the Archiving topic in the reference section of this manual.
+
+To Archive
+~~~~~~~~~~
+
+Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.
+
+.Archive a full run
+----------------
+megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %
+----------------
+
+To Restore
+~~~~~~~~~~
+
+.Retrieve a single test
+----------------
+megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%
+----------------
+
+Hint: You can browse the archive using bup commands directly.
+
+----------------
+bup -d /path/to/bup/archive ftp
+----------------
+
+Submit jobs to Host Types based on Test Name
+--------------------------------------------
+
+.In megatest.config
+------------------------
+[host-types]
+general ssh #{getbgesthost general}
+nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+
+[hosts]
+general cubian xena
+
+[launchers]
+envsetup general
+xor/%/n 4C16G
+% nbgeneral
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes
+------------------------
+
 Tricks
 ------
 
 This section is a compendium of a various useful tricks for debugging,
 configuring and generally getting the most out of Megatest.
@@ -34,10 +108,18 @@
 Debugging Tricks
 ----------------
 
 Examining The Environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Test Control Panel - xterm
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the 
+window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run
+scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way
+to debug your tests.
 
 During Config File Processing
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 It is often helpful to know the content of variables in various

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="generator" content="AsciiDoc 8.6.7">
+<meta name="generator" content="AsciiDoc 8.6.9">
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -84,14 +84,20 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-pre {
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
   padding: 0;
   margin: 0;
 }
+pre {
+  white-space: pre-wrap;
+}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -216,11 +222,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; }
+span.image img { border-style: none; vertical-align: text-bottom; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -412,16 +418,10 @@
 /*
  * xhtml11 specific
  *
  * */
 
-tt {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 div.tableblock > table {
@@ -451,16 +451,10 @@
 /*
  * html5 specific
  *
  * */
 
-.monospaced {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 thead, p.tableblock.header {
@@ -536,10 +530,12 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
+
+
 @media screen {
   body {
     max-width: 50em; /* approximately 80 characters wide */
     margin-left: 16em;
   }
@@ -876,14 +872,15 @@
 for building Megatest on Linux.</p></div>
 <div class="paragraph"><p><span class="footnote"><br>[An example footnote.]<br></span></p></div>
 </div>
 </div>
 </div>
-<h1 id="_writing_tests">Writing Tests</h1>
 <div class="sect1">
-<h2 id="_creating_a_new_test">Creating a new Test</h2>
+<h2 id="_writing_tests">Writing Tests</h2>
 <div class="sectionbody">
+<div class="sect2">
+<h3 id="_creating_a_new_test">Creating a new Test</h3>
 <div class="paragraph"><p>The following steps will add a test "yourtestname" to your testsuite. This assumes
 starting from a directory where you already have a megatest.config and
 runconfigs.config.</p></div>
 <div class="olist arabic"><ol class="arabic">
 <li>
@@ -912,12 +909,88 @@
 </div></div>
 <div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script
 "stepname.sh". Note that although it is common to put the actions
 needed for a test step into a script it is not necessary.</p></div>
 </div>
+</div>
 </div>
 <h1 id="_how_to_do_things">How To Do Things</h1>
+<div class="sect1">
+<h2 id="_remove_runs">Remove Runs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>From the dashboard click on the button (PASS/FAIL&#8230;) for one of the tests. From the test control panel that
+comes up push the clean test button. The command field will be prefilled with a template command for removing
+that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.</p></div>
+<div class="listingblock">
+<div class="title">Remove the test diskperf and all it&#8217;s items</div>
+<div class="content monospaced">
+<pre>megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v</pre>
+</div></div>
+<div class="listingblock">
+<div class="title">Remove all tests for all runs and all targets</div>
+<div class="content monospaced">
+<pre>megatest -remove-runs -target %/%/% -runname % -testpatt % -v</pre>
+</div></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_archive_runs">Archive Runs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Megatest supports using the bup backup tool (<a href="https://bup.github.io/">https://bup.github.io/</a>) to archive your tests for efficient storage
+and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run
+durations, time stamps etc.) are all preserved in the megatest database.</p></div>
+<div class="paragraph"><p>For setup information see the Archiving topic in the reference section of this manual.</p></div>
+<div class="sect2">
+<h3 id="_to_archive">To Archive</h3>
+<div class="paragraph"><p>Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.</p></div>
+<div class="listingblock">
+<div class="title">Archive a full run</div>
+<div class="content monospaced">
+<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_to_restore">To Restore</h3>
+<div class="listingblock">
+<div class="title">Retrieve a single test</div>
+<div class="content monospaced">
+<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%</pre>
+</div></div>
+<div class="paragraph"><p>Hint: You can browse the archive using bup commands directly.</p></div>
+<div class="listingblock">
+<div class="content monospaced">
+<pre>bup -d /path/to/bup/archive ftp</pre>
+</div></div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h2>
+<div class="sectionbody">
+<div class="listingblock">
+<div class="title">In megatest.config</div>
+<div class="content monospaced">
+<pre>[host-types]
+general ssh #{getbgesthost general}
+nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+
+[hosts]
+general cubian xena
+
+[launchers]
+envsetup general
+xor/%/n 4C16G
+% nbgeneral
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes</pre>
+</div></div>
+</div>
+</div>
 <div class="sect1">
 <h2 id="_tricks">Tricks</h2>
 <div class="sectionbody">
 <div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
 configuring and generally getting the most out of Megatest.</p></div>
@@ -945,10 +1018,17 @@
 <div class="sect1">
 <h2 id="_debugging_tricks">Debugging Tricks</h2>
 <div class="sectionbody">
 <div class="sect2">
 <h3 id="_examining_the_environment">Examining The Environment</h3>
+<div class="sect3">
+<h4 id="_test_control_panel_xterm">Test Control Panel - xterm</h4>
+<div class="paragraph"><p>From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the
+window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run
+scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way
+to debug your tests.</p></div>
+</div>
 <div class="sect3">
 <h4 id="_during_config_file_processing">During Config File Processing</h4>
 <div class="paragraph"><p>It is often helpful to know the content of variables in various
 contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.</p></div>
 <div class="paragraph"><p>For example, if an item list is not being generated as expected you
@@ -1015,19 +1095,58 @@
 </div>
 </div>
 </div>
 <h1 id="_reference">Reference</h1>
 <div class="sect1">
-<h2 id="_config_file_settings">Config File Settings</h2>
+<h2 id="_megatest_config_file_settings">Megatest Config File Settings</h2>
 <div class="sectionbody">
 <div class="sect2">
 <h3 id="_trim_trailing_spaces">Trim trailing spaces</h3>
 <div class="listingblock">
 <div class="content monospaced">
 <pre>[configf:settings trim-trailing-spaces yes]</pre>
 </div></div>
 </div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_submit_jobs_to_host_types_based_on_test_name_2">Submit jobs to Host Types based on Test Name</h2>
+<div class="sectionbody">
+<div class="listingblock">
+<div class="title">In megatest.config</div>
+<div class="content monospaced">
+<pre>[host-types]
+general   nbfake
+remote    bsub
+
+[launchers]
+runfirst/sum% remote
+% general
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes</pre>
+</div></div>
+<div class="sect2">
+<h3 id="_host_types">host-types</h3>
+<div class="paragraph"><p>List of host types and the commandline to run a job on that host type.</p></div>
+<div class="listingblock">
+<div class="title">host-type &#8658; launch command</div>
+<div class="content monospaced">
+<pre>general nbfake</pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_launchers">launchers</h3>
+<div class="listingblock">
+<div class="title">test/itempath &#8658; host-type</div>
+<div class="content monospaced">
+<pre>runfirst/sum% remote</pre>
+</div></div>
+</div>
 </div>
 </div>
 <div class="sect1">
 <h2 id="_the_testconfig_file">The testconfig File</h2>
 <div class="sectionbody">
@@ -1227,11 +1346,11 @@
 COMPLETED/ xterm -e bash -s --</pre>
 </div></div>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.001/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">There is a trailing space after the --</td>
 </tr></table>
 </div>
 </div>
@@ -1247,10 +1366,31 @@
 [testrollup]
 runfirst mysummary.sh</pre>
 </div></div>
 </div>
 </div>
+</div>
+<div class="sect1">
+<h2 id="_archiving_setup">Archiving Setup</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>In megatest.config add the following sections:</p></div>
+<div class="listingblock">
+<div class="title">megatest.config</div>
+<div class="content monospaced">
+<pre>[archive]
+# where to get bup executable
+# bup /path/to/bup
+
+[archive-disks]
+
+# Archives will be organised under these paths like this:
+#  &lt;testsuite&gt;/&lt;creationdate&gt;
+# Within the archive the data is structured like this:
+#  &lt;target&gt;/&lt;runname&gt;/&lt;test&gt;/
+archive0 /mfs/myarchive-data/adisk1</pre>
+</div></div>
+</div>
 </div>
 <div class="sect1">
 <h2 id="_programming_api">Programming API</h2>
 <div class="sectionbody">
 <div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
@@ -1345,11 +1485,11 @@
 <div class="sect2">
 <h3 id="_appendix_sub_section">Appendix Sub-section</h3>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.001/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">Preface and appendix subsections start out of sequence at level
 2 (level 1 is skipped).  This only applies to multi-part book
 documents.</td>
 </tr></table>
@@ -1417,10 +1557,11 @@
 </div>
 <div id="footnotes"><hr></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.0<br>
-Last updated 2015-06-22 12:55:58 PDT
+Last updated
+ 2015-07-07 22:48:26 MST
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -1,18 +1,55 @@
 
 Reference
 =========
 
-Config File Settings
---------------------
+Megatest Config File Settings
+-----------------------------
 
 Trim trailing spaces
 ~~~~~~~~~~~~~~~~~~~~
 
 ------------------
 [configf:settings trim-trailing-spaces yes]
 ------------------
+
+Submit jobs to Host Types based on Test Name
+--------------------------------------------
+
+.In megatest.config
+------------------------
+[host-types]
+general   nbfake
+remote    bsub
+
+[launchers]
+runfirst/sum% remote
+% general
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes
+------------------------
+
+host-types
+~~~~~~~~~~
+
+List of host types and the commandline to run a job on that host type.
+
+.host-type => launch command
+------------
+general nbfake
+------------
+
+launchers
+~~~~~~~~~
+.test/itempath => host-type
+------------
+runfirst/sum% remote
+------------
 
 The testconfig File
 -------------------
 
 Setup section
@@ -239,10 +276,30 @@
 -----------------
 # Override the rollup for specific tests
 [testrollup]
 runfirst mysummary.sh
 -----------------
+
+Archiving Setup
+---------------
+
+In megatest.config add the following sections:
+
+.megatest.config
+--------------
+[archive]
+# where to get bup executable
+# bup /path/to/bup
+
+[archive-disks]
+
+# Archives will be organised under these paths like this:
+#  <testsuite>/<creationdate>
+# Within the archive the data is structured like this:
+#  <target>/<runname>/<test>/
+archive0 /mfs/myarchive-data/adisk1
+--------------
 
 Programming API
 ---------------
 
 These routines can be called from the megatest repl. 

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

Index: docs/manual/writing_tests.txt
==================================================================
--- docs/manual/writing_tests.txt
+++ docs/manual/writing_tests.txt
@@ -1,11 +1,14 @@
 
 Writing Tests
-=============
+-------------
+// =============
 
 Creating a new Test
--------------------
+~~~~~~~~~~~~~~~~~~~
+
+//-------------------
 
 The following steps will add a test "yourtestname" to your testsuite. This assumes
 starting from a directory where you already have a megatest.config and
 runconfigs.config.
 

Index: launch.scm
==================================================================
--- launch.scm
+++ launch.scm
@@ -558,11 +558,12 @@
 
 (define (launch:cache-config)
   ;; if we have a linktree and -runtests and -target and the directory exists dump the config
   ;; to megatest-(current-seconds).cfg and symlink it to megatest.cfg
   (if (and *configdat* 
-	   (args:get-arg "-runtests"))
+	   (or (args:get-arg "-run")
+	       (args:get-arg "-runtests")))
       (let* ((linktree (get-environment-variable "MT_LINKTREE"))
 	     (target   (common:args-get-target))
 	     (runname  (or (args:get-arg "-runname")
 			   (args:get-arg ":runname")))
 	     (fulldir  (conc linktree "/"
@@ -798,11 +799,10 @@
 			    (if ush 
 				(if (equal? ush "no") ;; must use "no" to NOT use shell
 				    #f
 				    ush)
 				#t)))     ;; default is yes
-	 (launcher        (config-lookup *configdat* "jobtools"     "launcher"))
 	 (runscript       (config-lookup test-conf   "setup"        "runscript"))
 	 (ezsteps         (> (length (hash-table-ref/default test-conf "ezsteps" '())) 0)) ;; don't send all the steps, could be big
 	 (diskspace       (config-lookup test-conf   "requirements" "diskspace"))
 	 (memory          (config-lookup test-conf   "requirements" "memory"))
 	 (hosts           (config-lookup *configdat* "jobtools"     "workhosts"))
@@ -820,11 +820,12 @@
 				  (case (string->symbol exe)
 				    ((dboard)    "../megatest")
 				    ((mtest)     "../megatest")
 				    ((dashboard) "megatest")
 				    (else exe)))))
-	 (item-path  (item-list->path itemdat))
+	 (item-path       (item-list->path itemdat))
+	 (launcher        (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools"     "launcher"))
 	 (test-sig   (conc test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
 	 (work-area  #f)
 	 (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
 	 (diskpath   #f)
 	 (cmdparms   #f)

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -434,10 +434,12 @@
 (if (args:get-arg "-itempatt")
     (let ((newval (conc (args:get-arg "-testpatt") "/" (args:get-arg "-itempatt"))))
       (debug:print 0 "WARNING: -itempatt has been deprecated, please use -testpatt testpatt/itempatt method, new testpatt is "newval)
       (hash-table-set! args:arg-hash "-testpatt" newval)
       (hash-table-delete! args:arg-hash "-itempatt")))
+
+
 
 (on-exit std-exit-procedure)
 
 ;;======================================================================
 ;; Misc general calls
@@ -853,14 +855,14 @@
 	    (debug:print 0 "ERROR: Attempted " action "on test(s) but run area config file not found")
 	    (exit 1))
 	  ;; put test parameters into convenient variables
 	  (runs:operate-on  action
 			    target
-			    (or (args:get-arg "-runname")(args:get-arg ":runname"))
-			    (args:get-arg "-testpatt")
-			    state: (or (args:get-arg "-state")(args:get-arg ":state") )
-			    status: (or (args:get-arg "-status")(args:get-arg ":status"))
+			    (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
+			    (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
+			    state: (common:args-get-state)
+			    status: (common:args-get-status)
 			    new-state-status: (args:get-arg "-set-state-status")))
       (set! *didsomething* #t)))))
 	  
 (if (args:get-arg "-remove-runs")
     (general-run-call 
@@ -931,13 +933,14 @@
 (if (or (args:get-arg "-list-runs")
 	(args:get-arg "-list-db-targets"))
     (if (launch:setup-for-run)
 	(let* ((dbstruct    (make-dbr:dbstruct path: *toppath* local: #t))
 	       (runpatt     (args:get-arg "-list-runs"))
-	       (testpatt    (if (args:get-arg "-testpatt") 
-			        (args:get-arg "-testpatt") 
-			        "%"))
+	       (testpatt    (common:args-get-testpatt #f))
+	       ;; (if (args:get-arg "-testpatt") 
+	       ;;  	        (args:get-arg "-testpatt") 
+	       ;;  	        "%"))
 	       (keys        (db:get-keys dbstruct))
 	       ;; (runsda   t  (db:get-runs dbstruct runpatt #f #f '()))
 	       (runsdat     (db:get-runs-by-patt dbstruct keys (or runpatt "%") (common:args-get-target)
 			           	 #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment")))
 	       (runstmp     (db:get-rows runsdat))
@@ -1138,19 +1141,22 @@
 ;;     put task in deferred queue
 ;; if still ok to run tasks
 ;;   process deferred tasks per above steps
 
 ;; run all tests are are Not COMPLETED and PASS or CHECK
-(if (or (args:get-arg "-runall")(args:get-arg "-run"))
+(if (or (args:get-arg "-runall")
+	(args:get-arg "-run")
+	(args:get-arg "-runtests"))
     (general-run-call 
      "-runall"
      "run all tests"
      (lambda (target runname keys keyvals)
        (runs:run-tests target
 		       runname
-		       (or (args:get-arg "-testpatt")
-			   "%")
+		       #f ;; (common:args-get-testpatt #f)
+		       ;; (or (args:get-arg "-testpatt")
+		       ;;     "%")
 		       user
 		       args:arg-hash))))
 
 ;;======================================================================
 ;; run one test
@@ -1167,31 +1173,31 @@
 ;;    - step completed, exit status, timestamp
 ;; 6. test phone home
 ;;    - if test run time > allowed run time then kill job
 ;;    - if cannot access db > allowed disconnect time then kill job
 
-(if (args:get-arg "-runtests")
-  (general-run-call 
-   "-runtests" 
-   "run a test" 
-   (lambda (target runname keys keyvals)
-     ;;
-     ;; May or may not implement it this way ...
-     ;;
-     ;; Insert this run into the tasks queue
-     ;; (open-run-close tasks:add tasks:open-db 
-     ;;    	     "runtests" 
-     ;;    	     user
-     ;;    	     target
-     ;;    	     runname
-     ;;    	     (args:get-arg "-runtests")
-     ;;    	     #f))))
-     (runs:run-tests target
-		     runname
-		     (args:get-arg "-runtests")
-		     user
-		     args:arg-hash))))
+;; == duplicated == (if (or (args:get-arg "-run")(args:get-arg "-runtests"))
+;; == duplicated ==   (general-run-call 
+;; == duplicated ==    "-runtests" 
+;; == duplicated ==    "run a test" 
+;; == duplicated ==    (lambda (target runname keys keyvals)
+;; == duplicated ==      ;;
+;; == duplicated ==      ;; May or may not implement it this way ...
+;; == duplicated ==      ;;
+;; == duplicated ==      ;; Insert this run into the tasks queue
+;; == duplicated ==      ;; (open-run-close tasks:add tasks:open-db 
+;; == duplicated ==      ;;    	     "runtests" 
+;; == duplicated ==      ;;    	     user
+;; == duplicated ==      ;;    	     target
+;; == duplicated ==      ;;    	     runname
+;; == duplicated ==      ;;    	     (args:get-arg "-runtests")
+;; == duplicated ==      ;;    	     #f))))
+;; == duplicated ==      (runs:run-tests target
+;; == duplicated == 		     runname
+;; == duplicated == 		     (common:args-get-testpatt #f) ;; (args:get-arg "-runtests")
+;; == duplicated == 		     user
+;; == duplicated == 		     args:arg-hash))))
 
 ;;======================================================================
 ;; Rollup into a run
 ;;======================================================================
 
@@ -1604,11 +1610,12 @@
 ;;======================================================================
 ;; Wait on a run to complete
 ;;======================================================================
 
 (if (and (args:get-arg "-run-wait")
-	 (not (args:get-arg "-runtests"))) ;; run-wait is built into runtests now
+	 (not (or (args:get-arg "-run")
+		  (args:get-arg "-runtests")))) ;; run-wait is built into runtests now
     (begin
       (if (not (launch:setup-for-run))
 	  (begin
 	    (debug:print 0 "Failed to setup, exiting") 
 	    (exit 1)))
@@ -1675,14 +1682,14 @@
 
 (set! *time-to-exit* #t)
 (thread-join! *watchdog*)
 
 (if (not (eq? *globalexitstatus* 0))
-    (if (or (args:get-arg "-runtests")(args:get-arg "-runall"))
+    (if (or (args:get-arg "-run")(args:get-arg "-runtests")(args:get-arg "-runall"))
         (begin
            (debug:print 0 "NOTE: Subprocesses with non-zero exit code detected: " *globalexitstatus*)
            (exit 0))
         (case *globalexitstatus*
          ((0)(exit 0))
          ((1)(exit 1))
          ((2)(exit 2))
          (else (exit 3)))))

Index: runconfig.scm
==================================================================
--- runconfig.scm
+++ runconfig.scm
@@ -54,11 +54,13 @@
 	  (for-each (lambda (fullkey)
 		      (debug:print 2 (format #f "~20a ~a\n" fullkey (hash-table-ref/default whatfound fullkey 0))))
 		    sections)
 	  (debug:print 2 "---")
 	  (set! *already-seen-runconfig-info* #t)))
-    finaldat))
+    ;; finaldat ;; was returning this "finaldat" which would be good but conflicts with other uses
+    confdat
+    ))
 
 (define (set-run-config-vars run-id keyvals targ-from-db)
   (push-directory *toppath*) ;; the push/pop doesn't appear to do anything ...
   (let ((runconfigf (conc  *toppath* "/runconfigs.config"))
 	(targ       (or (common:args-get-target)

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -47,14 +47,12 @@
 		               (begin
 		                 (debug:print 0 "ERROR: Called setup in a non-megatest area, exiting")
 		                 (exit 1)))))
 	  (runrec      (runs:runrec-make-record))
 	  (target      (common:args-get-target))
-	  (runname     (or (args:get-arg "-runname")
-		           (args:get-arg ":runname")))
-	  (testpatt    (or (args:get-arg "-testpatt")
-		           (args:get-arg "-runtests")))
+	  (runname     (common:args-get-runname))
+	  (testpatt    (common:args-get-testpatt #f))
 	  (keys        (keys:config-get-fields mconfig))
 	  (keyvals     (keys:target->keyval keys target))
 	  (toppath     *toppath*)
 	  (envdat      keyvals) ;; initial values start with keyvals
 	  (runconfig   #f)
@@ -84,10 +82,14 @@
 	(begin
 	  (debug:print 0 "ERROR: [" (args:get-arg "-reqtarg") "] not found in " runconfigf)
 	  (if db (sqlite3:finalize! db))
 	  (exit 1)))
     ;; Now have runconfigs data loaded, set environment vars
+
+    ;; Only now can we calculate the testpatt
+    (set! testpatt (common:args-get-testpatt runconfig))
+    
     (for-each (lambda (section)
 		(for-each (lambda (varval)
 			    (set! envdat (append envdat (list varval)))
 			    (safe-setenv (car varval)(cadr varval)))
 			  (configf:get-section runconfig section)))
@@ -246,18 +248,24 @@
 		       (thread-start! th1)
 		       (thread-join! th2)))))
       (set-signal-handler! signal/int sighand)
       (set-signal-handler! signal/term sighand)
       (set-signal-handler! signal/stop sighand))
+
+    (runs:set-megatest-env-vars run-id inkeys: keys inrunname: runname) ;; these may be needed by the launching process
+    (set! runconf (if (file-exists? runconfigf)
+		      (setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals target)
+		      (begin
+			(debug:print 0 "WARNING: You do not have a run config file: " runconfigf)
+			#f)))
 
     ;; register this run in monitor.db
     (rmt:tasks-add "run-tests" user target runname test-patts task-key) ;; params)
     (rmt:tasks-set-state-given-param-key task-key "running")
-    (runs:set-megatest-env-vars run-id inkeys: keys inrunname: runname) ;; these may be needed by the launching process
-    (if (file-exists? runconfigf)
-	(setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals target)
-	(debug:print 0 "WARNING: You do not have a run config file: " runconfigf))
+
+    (if (not test-patts) ;; first time in - adjust testpatt
+	(set! test-patts (common:args-get-testpatt runconf)))
 
     ;; Now generate all the tests lists
     (set! all-tests-registry (tests:get-all))
     (set! all-test-names     (hash-table-keys all-tests-registry))
     (set! test-names         (tests:filter-test-names all-test-names test-patts))
@@ -429,11 +437,11 @@
 		     )))
 	     waitons)
 	    (let ((remtests (delete-duplicates (append waitons tal))))
 	      (if (not (null? remtests))
 		  (begin
-		    (debug:print-info 0 "Preprocessing continues for " (string-intersperse remtests ", "))
+		    ;; (debug:print-info 0 "Preprocessing continues for " (string-intersperse remtests ", "))
 		    (loop (car remtests)(cdr remtests))))))))
 
     (if (not (null? required-tests))
 	(debug:print-info 1 "Adding \"" (string-intersperse required-tests " ") "\" to the run queue"))
     ;; NOTE: these are all parent tests, items are not expanded yet.

Index: tests/fullrun/megatest.config
==================================================================
--- tests/fullrun/megatest.config
+++ tests/fullrun/megatest.config
@@ -257,6 +257,38 @@
 
 ltest #{scheme (case (string->symbol (conc (getenv "datapath"))) \
                         ((none) "nbfake") \
                         ((openlava) "bsub") \
                         (else "sleeprunner"))}
+
+#================================================================
+# Flexi-launcher
+#================================================================
+#
+# [host-types]
+# general ssh #{getbgesthost general}
+# nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+# 
+# [hosts]
+# general cubian xena
+# 
+# [launchers]
+# envsetup general
+# xor/%/n 4C16G
+# % nbgeneral
+# 
+# [jobtools]
+# launcher bsub
+# # if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# # match.
+# flexi-launcher yes  
+
+[jobtools]
+flexi-launcher yes
+
+[host-types]
+general nbfake
+remote  bsub
+
+[launchers]
+runfirst/sum% remote
 

Index: tests/fullrun/runconfigs.config
==================================================================
--- tests/fullrun/runconfigs.config
+++ tests/fullrun/runconfigs.config
@@ -23,10 +23,13 @@
 
 [default]
 SOMEVAR3 #{rget SOMEVAR}
 SOMEVAR4 #{rget SOMEVAR2}
 SOMEVAR5 #{runconfigs-get SOMEVAR2}
+
+[ubuntu/nfs/all_toplevel]
+TESTPATT all_toplevel
 
 [this/a/test]
 BLAHFOO 123
 
 [ubuntu/nfs/sleep1]

ADDED   tests/release/tests/testpatt_envvar/results.logpro
Index: tests/release/tests/testpatt_envvar/results.logpro
==================================================================
--- /dev/null
+++ tests/release/tests/testpatt_envvar/results.logpro
@@ -0,0 +1,141 @@
+;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com
+;;  
+;;   License GPL.
+
+(define logbody "LogFileBody")
+
+(define pass-specs '( ;; testname num-expected max-runtime
+		     ("exit_0"		        1  20)
+		     ("ezlog_fail_then_pass"    1  20)
+		     ("ezlog_pass"	        1  20)
+		     ("ez_pass"		        1  20)
+		     ("lineitem_pass"	        1  20)
+		     ("priority_1"	        1  20)
+		     ("priority_10"	        1  20)
+		     ("priority_10_waiton_1"    1  20)
+		     ("priority_3"	        1  20)
+		     ("priority_4"	        1  20)
+		     ;; ("priority_5"	        1  20)
+		     ("priority_6"	        1  20)
+;;		     ("priority_7"	        1  20)
+		     ("priority_8"	        1  20)
+		     ("priority_9"	        1  20)
+		     ("runfirst"	        7  20)
+		     ("singletest"	        1  20)
+		     ("singletest2"	        1  20)
+		     ("special"		        1  20)
+		     ("sqlitespeed"	       10  20)
+		     ("test1"		        1  20)
+		     ("test2"		        6  20)
+		     ("test_mt_vars"	        6  20)
+		     ))
+
+(define fail-specs '( ;; testname num-expected max-runtime
+		     ("exit_1"		        1  20)
+		     ("ez_exit2_fail"           1  20)
+		     ("ez_fail"		        1  20)
+		     ("ez_fail_quick"	        1  20)
+		     ("ezlog_fail"	        1  20)
+		     ("lineitem_fail"	        1  20)
+		     ("logpro_required_fail"    1  20)
+		     ("manual_example"	        1  20)
+		     ("neverrun"	        1  20)))
+		     
+(define warn-specs   '(("ezlog_warn"	        1  20)))
+
+(define nost-specs   '(("wait_no_items1"        1  20)
+		       ("wait_no_items2"        1  20)
+		       ("wait_no_items3"        1  20)
+		       ("wait_no_items4"        1  20)
+		       ;; ("no_items"              1  20)
+		       ))
+
+(define (check-one-test estate estatus testname count runtime)
+   (let* ((rxe      (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s")))
+	  (msg1     (conc testname " expecting count of " count))
+	  (msg2     (conc testname " expecting runtime less than " runtime)))
+     (expect:required in logbody = count msg1 rxe)
+     ;;(expect:value    in logbody count < msg2 rxe)
+     ))
+
+;; Special cases
+;;
+(expect:error    in logbody >  0  "blocktestxz not to run"           #/Test: blocktestxz/)
+(expect:ignore   in logbody >= 0  "db_sync test might not have run"  #/Test: db_sync/)
+(expect:ignore   in logbody >= 0  "all_toplevel may not yet be done" #/Test: all_toplevel/)
+(expect:error    in logbody =  0  "tests left in RUNNING state"      #/State: RUNNING/)
+(expect:required in logbody =  1  "priority_2 is KILLED"             #/Test: priority_2\s+State: KILLED\s+Status: KILLED/)
+(expect:required in logbody =  1  "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
+(expect:required in logbody =  1  "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
+(expect:required in logbody =  1  "testxz has 1 NOT_STARTED test"    #/Test: testxz\s+State: NOT_STARTED/)
+(expect:required in logbody =  1  "no items"                         #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/)
+(expect:warning  in logbody =  1  "dynamic waiton"                   #/Test: dynamic_waiton/)
+(expect:required in logbody = 29  "blocktestxz has 29 tests"         #/Test: blocktestxz/)
+
+;; General cases
+;;
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "PASS" testdat))
+ pass-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "FAIL" testdat))
+ fail-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "COMPLETED" "WARN" testdat))
+ warn-specs)
+
+(for-each 
+ (lambda (testdat)
+   (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat))
+ nost-specs)
+
+;; Catch all.
+;;
+(expect:error    in logbody = 0   "Tests not accounted for"     #/Test: /)
+
+
+;; ;; define your hooks
+;; (hook:first-error   "echo \"Error hook activated: #{escaped errmsg}\"")
+;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"")
+;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"")
+;; 
+;; ;; first ensure your run at least started
+;; ;;
+;; (trigger "Init"     #/This is a header/)
+;; (trigger "InitEnd"  #/^\s*$/)
+;; (section "Init" "Init" "InitEnd")
+;; 
+;; (trigger "Body"     #/^.*$/) ;; anything starts the body
+;; ;; (trigger "EndBody"  #/This had better never match/)
+;; 
+;; (section "Body"     "Body" "EndBody")
+;; 
+;; (trigger "Blah2"    #/^begin Blah2/)
+;; (trigger "Blah2End" #/^end Blah2/)
+;; (section "Blah2"    "Blah2" "Blah2End")
+;; 
+;; (expect:required in "Init"  = 1 "Header"      #/This is a header/)
+;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/)
+;; (expect:value    in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/)
+;; (expect:value    in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/)
+;; (expect:value    in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/)
+;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/)
+;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/)
+;; 
+;; ;; Using match number
+;; (expect:value    in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2)
+;; 
+;; ;; Comparison instead of tolerance
+;; (expect:value    in "LogFileBody" 1.9 >   "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2)
+;; 
+;; (expect:ignore   in "Blah2" < 99 "FALSE ERROR" #/ERROR/)
+;; (expect:ignore   in "Body"  < 99 "Ignore the word error in comments" #/^\/\/.*error/)
+;; (expect:warning  in "Body"  = 0 "Any warning" #/WARNING/)
+;; (expect:error    in "Body"  = 0 "ERROR BLAH"  (list #/ERROR/ #/error/)) ;; but disallow any other errors
+;; 
+;; ;(expect in "Init"  < 1 "Junk"        #/This is bogus/)

ADDED   tests/release/tests/testpatt_envvar/testconfig
Index: tests/release/tests/testpatt_envvar/testconfig
==================================================================
--- /dev/null
+++ tests/release/tests/testpatt_envvar/testconfig
@@ -0,0 +1,14 @@
+[var]
+targ -target ubuntu/nfs/all_toplevel
+tp   -testpatt %
+
+[ezsteps]
+cleantop $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -remove-runs #{get var tp}  #{get var targ} -runname release_toplevel
+runall   $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -run         #{get var tp}  #{get var targ} -runname release_toplevel -runwait
+results  $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -list-runs release_toplevel #{get var targ} -runname release_toplevel 
+
+[requirements]
+# waiton  #{getenv ALL_TOPLEVEL_TESTS}
+
+# This is a "toplevel" test, it does not require waitons to be non-FAIL to run
+# mode toplevel