Megatest

======================================================================
New way of launching needed to accomodate different target hosttypes
for items
======================================================================

[flavors]
general ssh #{getbgesthost general}
nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo

[hosts]
general cubian xena

[launchers]
envsetup general
xor/%/n 4C16G
% nbgeneral

[jobtools]
launcher internal



======================================================================
Try writing to in-memory db and every 2-5 seconds syncing to megatest.db
======================================================================

First, how much time will it take to write back the changes:

1. Get the run table

;; '(print (string-intersperse (map cadr (hash-table-ref/default (read-config "megatest.config" \#f \#t) "disks" '"'"'("none" ""))) "\n"))'
(define (common:get-disks #!key (configf #f))
  (hash-table-ref/default 
   (or configf (read-config "megatest.config" #f #t))
   "disks" '("none" "")))

;;======================================================================
;; T A R G E T S  ,   S T A T E ,   S T A T U S ,   
;;                    R U N N A M E    A N D   T E S T P A T T
;;======================================================================

(define (common:args-get-state)
  (or (args:get-arg "-state")(args:get-arg ":state")))

(define (common:args-get-status)
  (or (args:get-arg "-status")(args:get-arg ":status")))

(define (common:args-get-testpatt rconf)
  (let* ((rtestpatt     (if rconf (runconfigs-get rconf "TESTPATT") #f))
	 (args-testpatt (or (args:get-arg "-testpatt")
			    (args:get-arg "-runtests")
			    "%"))
	 (testpatt    (or (and (equal? args-testpatt "%")
			       rtestpatt)
			  args-testpatt)))
    (if rtestpatt (debug:print-info 0 "TESTPATT from runconfigs: " rtestpatt))
    testpatt))

(define (common:args-get-runname)
  (or (args:get-arg "-runname")
      (args:get-arg ":runname")))

(define (common:args-get-target #!key (split #f))
  (let* ((keys    (keys:config-get-fields *configdat*))
	 (numkeys (length keys))
	 (target  (if (args:get-arg "-reqtarg")
		      (args:get-arg "-reqtarg")
		      (if (args:get-arg "-target")
			  (args:get-arg "-target")

;; get registered dashboards
;;
(define (mddb:get-dashboards)
  (let ((db (mddb:open-db)))
    (query fetch-column
	   (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;"))))
    
;;======================================================================
;;  T E S T   L A U N C H I N G   P E R   I T E M   W I T H   H O S T   T Y P E S
;;======================================================================
;; 
;; [host-types]
;; general ssh #{getbgesthost general}
;; nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
;; 
;; [hosts]
;; general cubian xena
;; 
;; [launchers]
;; envsetup general
;; xor/%/n 4C16G
;; % nbgeneral
;; 
;; [jobtools]
;; launcher bsub
;; # if defined and not "no" flexi-launcher will bypass launcher unless there is no
;; # match.
;; flexi-launcher yes  

(define (common:get-launcher configdat testname itempath)
  (let ((fallback-launcher (configf:lookup configdat "jobtools" "launcher")))
    (if (and (configf:lookup configdat "jobtools" "flexi-launcher") ;; overrides launcher
	     (not (equal? (configf:lookup configdat "jobtools" "flexi-launcher") "no")))
	(let* ((launchers         (hash-table-ref/default configdat "launchers" '())))
	  (if (null? launchers)
	      fallback-launcher
	      (let loop ((hed (car launchers))
			 (tal (cdr launchers)))
		(let ((patt      (car hed))
		      (host-type (cadr hed)))
		  (if (tests:match patt testname itempath)
		      (begin
			(debug:print-info 0 "Have flexi-launcher match for " testname "/" itempath " = " host-type)
			(let ((launcher (configf:lookup configdat "host-types" host-type)))
			  (if launcher
			      launcher
			      (begin
				(debug:print-info 0 "WARNING: no launcher found for host-type " host-type)
				(if (null? tal)
				    fallback-launcher
				    (loop (car tal)(cdr tal)))))))
		      ;; no match, try again
		      (if (null? tal)
			  fallback-launcher
			  (loop (car tal)(cdr tal))))))))
	fallback-launcher)))
  

	  (with-output-to-port (current-error-port)
	    (lambda ()
	      (print "ERROR: " cmd " returned bad exit code " status)))
	  ""))))

;; Lookup a value in runconfigs based on -reqtarg or -target
(define (runconfigs-get config var)
  (let ((targ (common:args-get-target))) ;; (or (args:get-arg "-reqtarg")(args:get-arg "-target")(getenv "MT_TARGET"))))
    (if targ
	(or (configf:lookup config targ var)
	    (configf:lookup config "default" var))
	(configf:lookup config "default" var))))

;; this was inline but I'm pretty sure that is a hold over from when it was *very* simple ...
;;

# design_spec.html : $(SRCFILES) $(CSVFILES)
#         asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt
#

all : server.ps megatest_manual.html client.ps

megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt
	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
#	dos2unix megatest_manual.html

server.ps : server.dot
	dot -Tps server.dot > server.ps

client.ps : client.dot

How To Do Things
================

Remove Runs
-----------

From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that 
comes up push the clean test button. The command field will be prefilled with a template command for removing 
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.

.Remove the test diskperf and all it's items
----------------
megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v
----------------

.Remove all tests for all runs and all targets
----------------
megatest -remove-runs -target %/%/% -runname % -testpatt % -v
----------------

Archive Runs
------------

Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run 
durations, time stamps etc.) are all preserved in the megatest database.

For setup information see the Archiving topic in the reference section of this manual.

To Archive
~~~~~~~~~~

Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.

.Archive a full run
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %
----------------

To Restore
~~~~~~~~~~

.Retrieve a single test
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%
----------------

Hint: You can browse the archive using bup commands directly.

----------------
bup -d /path/to/bup/archive ftp
----------------

Submit jobs to Host Types based on Test Name
--------------------------------------------

.In megatest.config
------------------------
[host-types]
general ssh #{getbgesthost general}
nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo

[hosts]
general cubian xena

[launchers]
envsetup general
xor/%/n 4C16G
% nbgeneral

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes
------------------------

Tricks
------

This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.

Limiting your running jobs

Debugging Tricks
----------------

Examining The Environment
~~~~~~~~~~~~~~~~~~~~~~~~~

Test Control Panel - xterm
^^^^^^^^^^^^^^^^^^^^^^^^^^

From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the 
window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run
scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way
to debug your tests.

During Config File Processing
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

It is often helpful to know the content of variables in various
contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.

<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="generator" content="AsciiDoc 8.6.9">
<title>The Megatest Users Manual</title>
<style type="text/css">
/* Shared CSS for AsciiDoc xhtml11 and html5 backends */

/* Default font. */
body {
  font-family: Georgia,serif;

ul, ol, li > p {
  margin-top: 0;
}
ul > li     { color: #aaa; }
ul > li > * { color: black; }

.monospaced, code, pre {
  font-family: "Courier New", Courier, monospace;
  font-size: inherit;
  color: navy;
  padding: 0;
  margin: 0;
}
pre {
  white-space: pre-wrap;
}

#author {
  color: #527bbd;
  font-weight: bold;
  font-size: 1.1em;
}
#email {

div.exampleblock > div.content {
  border-left: 3px solid #dddddd;
  padding-left: 0.5em;
}

div.imageblock div.content { padding-left: 0; }
span.image img { border-style: none; vertical-align: text-bottom; }
a.image:visited { color: white; }

dl {
  margin-top: 0.8em;
  margin-bottom: 0.8em;
}
dt {

/*
 * xhtml11 specific
 *
 * */







div.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
div.tableblock > table {
  border: 3px solid #527bbd;
}

/*
 * html5 specific
 *
 * */







table.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
thead, p.tableblock.header {
  font-weight: bold;
  color: #527bbd;

body.manpage div.sectionbody {
  margin-left: 3em;
}

@media print {
  body.manpage div#toc { display: none; }
}


@media screen {
  body {
    max-width: 50em; /* approximately 80 characters wide */
    margin-left: 16em;
  }

  #toc {

Megatest. See the script installall.sch in the utils directory of the
distribution for a mostly automated way to install everything needed
for building Megatest on Linux.</p></div>
<div class="paragraph"><p><span class="footnote"><br>[An example footnote.]<br></span></p></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_writing_tests">Writing Tests</h2>
<div class="sectionbody">

<div class="sect2">
<h3 id="_creating_a_new_test">Creating a new Test</h3>
<div class="paragraph"><p>The following steps will add a test "yourtestname" to your testsuite. This assumes
starting from a directory where you already have a megatest.config and
runconfigs.config.</p></div>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Create a directory tests/yourtestname

description An example test
reviewed never</pre>
</div></div>
<div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script
"stepname.sh". Note that although it is common to put the actions
needed for a test step into a script it is not necessary.</p></div>
</div>
</div>
</div>
<h1 id="_how_to_do_things">How To Do Things</h1>
<div class="sect1">
<h2 id="_remove_runs">Remove Runs</h2>
<div class="sectionbody">
<div class="paragraph"><p>From the dashboard click on the button (PASS/FAIL&#8230;) for one of the tests. From the test control panel that
comes up push the clean test button. The command field will be prefilled with a template command for removing
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.</p></div>
<div class="listingblock">
<div class="title">Remove the test diskperf and all it&#8217;s items</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v</pre>
</div></div>
<div class="listingblock">
<div class="title">Remove all tests for all runs and all targets</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target %/%/% -runname % -testpatt % -v</pre>
</div></div>
</div>
</div>
<div class="sect1">
<h2 id="_archive_runs">Archive Runs</h2>
<div class="sectionbody">
<div class="paragraph"><p>Megatest supports using the bup backup tool (<a href="https://bup.github.io/">https://bup.github.io/</a>) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run
durations, time stamps etc.) are all preserved in the megatest database.</p></div>
<div class="paragraph"><p>For setup information see the Archiving topic in the reference section of this manual.</p></div>
<div class="sect2">
<h3 id="_to_archive">To Archive</h3>
<div class="paragraph"><p>Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.</p></div>
<div class="listingblock">
<div class="title">Archive a full run</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_to_restore">To Restore</h3>
<div class="listingblock">
<div class="title">Retrieve a single test</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%</pre>
</div></div>
<div class="paragraph"><p>Hint: You can browse the archive using bup commands directly.</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>bup -d /path/to/bup/archive ftp</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h2>
<div class="sectionbody">
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[host-types]
general ssh #{getbgesthost general}
nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo

[hosts]
general cubian xena

[launchers]
envsetup general
xor/%/n 4C16G
% nbgeneral

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes</pre>
</div></div>
</div>
</div>
<div class="sect1">
<h2 id="_tricks">Tricks</h2>
<div class="sectionbody">
<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.</p></div>
</div>
</div>

</div>
</div>
<div class="sect1">
<h2 id="_debugging_tricks">Debugging Tricks</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_examining_the_environment">Examining The Environment</h3>
<div class="sect3">
<h4 id="_test_control_panel_xterm">Test Control Panel - xterm</h4>
<div class="paragraph"><p>From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the
window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run
scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way
to debug your tests.</p></div>
</div>
<div class="sect3">
<h4 id="_during_config_file_processing">During Config File Processing</h4>
<div class="paragraph"><p>It is often helpful to know the content of variables in various
contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.</p></div>
<div class="paragraph"><p>For example, if an item list is not being generated as expected you
can inject the startup of an xterm as if it were an item:</p></div>
<div class="listingblock">

sudo netstat -tulpn</pre>
</div></div>
</div>
</div>
</div>
<h1 id="_reference">Reference</h1>
<div class="sect1">
<h2 id="_megatest_config_file_settings">Megatest Config File Settings</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_trim_trailing_spaces">Trim trailing spaces</h3>
<div class="listingblock">
<div class="content monospaced">
<pre>[configf:settings trim-trailing-spaces yes]</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_submit_jobs_to_host_types_based_on_test_name_2">Submit jobs to Host Types based on Test Name</h2>
<div class="sectionbody">
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes</pre>
</div></div>
<div class="sect2">
<h3 id="_host_types">host-types</h3>
<div class="paragraph"><p>List of host types and the commandline to run a job on that host type.</p></div>
<div class="listingblock">
<div class="title">host-type &#8658; launch command</div>
<div class="content monospaced">
<pre>general nbfake</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_launchers">launchers</h3>
<div class="listingblock">
<div class="title">test/itempath &#8658; host-type</div>
<div class="content monospaced">
<pre>runfirst/sum% remote</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_the_testconfig_file">The testconfig File</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_setup_section">Setup section</h3>

<div class="content monospaced">
<pre>[triggers]
COMPLETED/ xterm -e bash -s --</pre>
</div></div>
<div class="admonitionblock">
<table><tr>
<td class="icon">
<img src="/usr/images/icons/note.png" alt="Note">
</td>
<td class="content">There is a trailing space after the --</td>
</tr></table>
</div>
</div>
<div class="sect2">
<h3 id="_override_the_toplevel_html_file">Override the Toplevel HTML File</h3>
<div class="paragraph"><p>Megatest generates a simple html file summary for top level tests of
iterated tests. The generation can be overridden. NOTE: the output of
the script is captured from stdout to create the html.</p></div>
<div class="listingblock">
<div class="title">For test "runfirst" override the toplevel generation with a script "mysummary.sh"</div>
<div class="content monospaced">
<pre># Override the rollup for specific tests
[testrollup]
runfirst mysummary.sh</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_archiving_setup">Archiving Setup</h2>
<div class="sectionbody">
<div class="paragraph"><p>In megatest.config add the following sections:</p></div>
<div class="listingblock">
<div class="title">megatest.config</div>
<div class="content monospaced">
<pre>[archive]
# where to get bup executable
# bup /path/to/bup

[archive-disks]

# Archives will be organised under these paths like this:
#  &lt;testsuite&gt;/&lt;creationdate&gt;
# Within the archive the data is structured like this:
#  &lt;target&gt;/&lt;runname&gt;/&lt;test&gt;/
archive0 /mfs/myarchive-data/adisk1</pre>
</div></div>
</div>
</div>
<div class="sect1">
<h2 id="_programming_api">Programming API</h2>
<div class="sectionbody">
<div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
<table class="tableblock frame-topbot grid-all"
style="

<div class="sectionbody">
<div class="paragraph"><p>One or more optional appendixes go here at section level zero.</p></div>
<div class="sect2">
<h3 id="_appendix_sub_section">Appendix Sub-section</h3>
<div class="admonitionblock">
<table><tr>
<td class="icon">
<img src="/usr/images/icons/note.png" alt="Note">
</td>
<td class="content">Preface and appendix subsections start out of sequence at level
2 (level 1 is skipped).  This only applies to multi-part book
documents.</td>
</tr></table>
</div>
</div>

</div>
</div>
</div>
<div id="footnotes"><hr></div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated
 2015-07-07 22:48:26 MST
</div>
</div>
</body>
</html>

Reference
=========

Megatest Config File Settings
-----------------------------

Trim trailing spaces
~~~~~~~~~~~~~~~~~~~~

------------------
[configf:settings trim-trailing-spaces yes]
------------------

Submit jobs to Host Types based on Test Name
--------------------------------------------

.In megatest.config
------------------------
[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes
------------------------

host-types
~~~~~~~~~~

List of host types and the commandline to run a job on that host type.

.host-type => launch command
------------
general nbfake
------------

launchers
~~~~~~~~~
.test/itempath => host-type
------------
runfirst/sum% remote
------------

The testconfig File
-------------------

Setup section
~~~~~~~~~~~~~

.For test "runfirst" override the toplevel generation with a script "mysummary.sh"
-----------------
# Override the rollup for specific tests
[testrollup]
runfirst mysummary.sh
-----------------

Archiving Setup
---------------

In megatest.config add the following sections:

.megatest.config
--------------
[archive]
# where to get bup executable
# bup /path/to/bup

[archive-disks]

# Archives will be organised under these paths like this:
#  <testsuite>/<creationdate>
# Within the archive the data is structured like this:
#  <target>/<runname>/<test>/
archive0 /mfs/myarchive-data/adisk1
--------------

Programming API
---------------

These routines can be called from the megatest repl. 

.API Server Management Calls

Writing Tests
-------------
// =============

Creating a new Test
~~~~~~~~~~~~~~~~~~~

//-------------------

The following steps will add a test "yourtestname" to your testsuite. This assumes
starting from a directory where you already have a megatest.config and
runconfigs.config.

. Create a directory tests/yourtestname
. Create a file tests/yourtestname/testconfig

	  )))
  *toppath*)

(define (launch:cache-config)
  ;; if we have a linktree and -runtests and -target and the directory exists dump the config
  ;; to megatest-(current-seconds).cfg and symlink it to megatest.cfg
  (if (and *configdat* 
	   (or (args:get-arg "-run")
	       (args:get-arg "-runtests")))
      (let* ((linktree (get-environment-variable "MT_LINKTREE"))
	     (target   (common:args-get-target))
	     (runname  (or (args:get-arg "-runname")
			   (args:get-arg ":runname")))
	     (fulldir  (conc linktree "/"
			     target "/"
			     runname)))

    ))
  (let* ((useshell        (let ((ush (config-lookup *configdat* "jobtools"     "useshell")))
			    (if ush 
				(if (equal? ush "no") ;; must use "no" to NOT use shell
				    #f
				    ush)
				#t)))     ;; default is yes

	 (runscript       (config-lookup test-conf   "setup"        "runscript"))
	 (ezsteps         (> (length (hash-table-ref/default test-conf "ezsteps" '())) 0)) ;; don't send all the steps, could be big
	 (diskspace       (config-lookup test-conf   "requirements" "diskspace"))
	 (memory          (config-lookup test-conf   "requirements" "memory"))
	 (hosts           (config-lookup *configdat* "jobtools"     "workhosts"))
	 (remote-megatest (config-lookup *configdat* "setup" "executable"))
	 (run-time-limit  (or (configf:lookup  test-conf   "requirements" "runtimelim")

				 (exe (pathname-strip-directory lm)))
			    (conc (if dir (conc dir "/") "")
				  (case (string->symbol exe)
				    ((dboard)    "../megatest")
				    ((mtest)     "../megatest")
				    ((dashboard) "megatest")
				    (else exe)))))
	 (item-path       (item-list->path itemdat))
	 (launcher        (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools"     "launcher"))
	 (test-sig   (conc test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path
	 (work-area  #f)
	 (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all
	 (diskpath   #f)
	 (cmdparms   #f)
	 (fullcmd    #f) ;; (define a (with-output-to-string (lambda ()(write x))))
	 (mt-bindir-path #f)

    (set! open-run-close open-run-close-no-exception-handling))

(if (args:get-arg "-itempatt")
    (let ((newval (conc (args:get-arg "-testpatt") "/" (args:get-arg "-itempatt"))))
      (debug:print 0 "WARNING: -itempatt has been deprecated, please use -testpatt testpatt/itempatt method, new testpatt is "newval)
      (hash-table-set! args:arg-hash "-testpatt" newval)
      (hash-table-delete! args:arg-hash "-itempatt")))



(on-exit std-exit-procedure)

;;======================================================================
;; Misc general calls
;;======================================================================

      (if (not (car *configinfo*))
	  (begin
	    (debug:print 0 "ERROR: Attempted " action "on test(s) but run area config file not found")
	    (exit 1))
	  ;; put test parameters into convenient variables
	  (runs:operate-on  action
			    target
			    (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			    (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			    state: (common:args-get-state)
			    status: (common:args-get-status)
			    new-state-status: (args:get-arg "-set-state-status")))
      (set! *didsomething* #t)))))
	  
(if (args:get-arg "-remove-runs")
    (general-run-call 
     "-remove-runs"
     "remove runs"

;; NOTE: list-runs and list-db-targets operate on local db!!!
;;
(if (or (args:get-arg "-list-runs")
	(args:get-arg "-list-db-targets"))
    (if (launch:setup-for-run)
	(let* ((dbstruct    (make-dbr:dbstruct path: *toppath* local: #t))
	       (runpatt     (args:get-arg "-list-runs"))
	       (testpatt    (common:args-get-testpatt #f))
	       ;; (if (args:get-arg "-testpatt") 
	       ;;  	        (args:get-arg "-testpatt") 
	       ;;  	        "%"))
	       (keys        (db:get-keys dbstruct))
	       ;; (runsda   t  (db:get-runs dbstruct runpatt #f #f '()))
	       (runsdat     (db:get-runs-by-patt dbstruct keys (or runpatt "%") (common:args-get-target)
			           	 #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment")))
	       (runstmp     (db:get-rows runsdat))
	       (header      (db:get-header runsdat))
	       (runs        (if (and (not (null? runstmp))

;;     launch task
;;   else
;;     put task in deferred queue
;; if still ok to run tasks
;;   process deferred tasks per above steps

;; run all tests are are Not COMPLETED and PASS or CHECK
(if (or (args:get-arg "-runall")
	(args:get-arg "-run")
	(args:get-arg "-runtests"))
    (general-run-call 
     "-runall"
     "run all tests"
     (lambda (target runname keys keyvals)
       (runs:run-tests target
		       runname
		       #f ;; (common:args-get-testpatt #f)
		       ;; (or (args:get-arg "-testpatt")
		       ;;     "%")
		       user
		       args:arg-hash))))

;;======================================================================
;; run one test
;;======================================================================

;; 1. find the config file
;; 2. change to the test directory
;; 3. update the db with "test started" status, set running host
;; 4. process launch the test
;;    - monitor the process, update stats in the db every 2^n minutes
;; 5. as the test proceeds internally it calls megatest as each step is
;;    started and completed
;;    - step started, timestamp
;;    - step completed, exit status, timestamp
;; 6. test phone home
;;    - if test run time > allowed run time then kill job
;;    - if cannot access db > allowed disconnect time then kill job

;; == duplicated == (if (or (args:get-arg "-run")(args:get-arg "-runtests"))
;; == duplicated ==   (general-run-call 
;; == duplicated ==    "-runtests" 
;; == duplicated ==    "run a test" 
;; == duplicated ==    (lambda (target runname keys keyvals)
;; == duplicated ==      ;;
;; == duplicated ==      ;; May or may not implement it this way ...
;; == duplicated ==      ;;
;; == duplicated ==      ;; Insert this run into the tasks queue
;; == duplicated ==      ;; (open-run-close tasks:add tasks:open-db 
;; == duplicated ==      ;;    	     "runtests" 
;; == duplicated ==      ;;    	     user
;; == duplicated ==      ;;    	     target
;; == duplicated ==      ;;    	     runname
;; == duplicated ==      ;;    	     (args:get-arg "-runtests")
;; == duplicated ==      ;;    	     #f))))
;; == duplicated ==      (runs:run-tests target
;; == duplicated == 		     runname
;; == duplicated == 		     (common:args-get-testpatt #f) ;; (args:get-arg "-runtests")
;; == duplicated == 		     user
;; == duplicated == 		     args:arg-hash))))

;;======================================================================
;; Rollup into a run
;;======================================================================

(if (args:get-arg "-rollup")
    (general-run-call 

      (set! *didsomething* #t)))

;;======================================================================
;; Wait on a run to complete
;;======================================================================

(if (and (args:get-arg "-run-wait")
	 (not (or (args:get-arg "-run")
		  (args:get-arg "-runtests")))) ;; run-wait is built into runtests now
    (begin
      (if (not (launch:setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting") 
	    (exit 1)))
      (operate-on 'run-wait)
      (set! *didsomething* #t)))

(if (not *didsomething*)
    (debug:print 0 help))

(set! *time-to-exit* #t)
(thread-join! *watchdog*)

(if (not (eq? *globalexitstatus* 0))
    (if (or (args:get-arg "-run")(args:get-arg "-runtests")(args:get-arg "-runall"))
        (begin
           (debug:print 0 "NOTE: Subprocesses with non-zero exit code detected: " *globalexitstatus*)
           (exit 0))
        (case *globalexitstatus*
         ((0)(exit 0))
         ((1)(exit 1))
         ((2)(exit 2))
         (else (exit 3)))))

	(begin
	  (debug:print 2 "Key settings found in runconfig.config:")
	  (for-each (lambda (fullkey)
		      (debug:print 2 (format #f "~20a ~a\n" fullkey (hash-table-ref/default whatfound fullkey 0))))
		    sections)
	  (debug:print 2 "---")
	  (set! *already-seen-runconfig-info* #t)))
    ;; finaldat ;; was returning this "finaldat" which would be good but conflicts with other uses
    confdat
    ))

(define (set-run-config-vars run-id keyvals targ-from-db)
  (push-directory *toppath*) ;; the push/pop doesn't appear to do anything ...
  (let ((runconfigf (conc  *toppath* "/runconfigs.config"))
	(targ       (or (common:args-get-target)
			targ-from-db
			(get-environment-variable "MT_TARGET"))))

		           (if (launch:setup-for-run)
		               *configdat*
		               (begin
		                 (debug:print 0 "ERROR: Called setup in a non-megatest area, exiting")
		                 (exit 1)))))
	  (runrec      (runs:runrec-make-record))
	  (target      (common:args-get-target))
	  (runname     (common:args-get-runname))

	  (testpatt    (common:args-get-testpatt #f))

	  (keys        (keys:config-get-fields mconfig))
	  (keyvals     (keys:target->keyval keys target))
	  (toppath     *toppath*)
	  (envdat      keyvals) ;; initial values start with keyvals
	  (runconfig   #f)
	  (serverdat   (if (args:get-arg "-server")
			   *runremote*

    (set! runconfig (read-config (conc  *toppath* "/runconfigs.config") #f #t sections: (list "default" target)))
    (if (not (hash-table-ref/default runconfig (args:get-arg "-reqtarg") #f))
	(begin
	  (debug:print 0 "ERROR: [" (args:get-arg "-reqtarg") "] not found in " runconfigf)
	  (if db (sqlite3:finalize! db))
	  (exit 1)))
    ;; Now have runconfigs data loaded, set environment vars

    ;; Only now can we calculate the testpatt
    (set! testpatt (common:args-get-testpatt runconfig))
    
    (for-each (lambda (section)
		(for-each (lambda (varval)
			    (set! envdat (append envdat (list varval)))
			    (safe-setenv (car varval)(cadr varval)))
			  (configf:get-section runconfig section)))
	      (list "default" target))
    (vector target runname testpatt keys keyvals envdat mconfig runconfig serverdat transport db toppath run-id)))

					       (exit 4)))))
		       (thread-start! th2)
		       (thread-start! th1)
		       (thread-join! th2)))))
      (set-signal-handler! signal/int sighand)
      (set-signal-handler! signal/term sighand)
      (set-signal-handler! signal/stop sighand))

    (runs:set-megatest-env-vars run-id inkeys: keys inrunname: runname) ;; these may be needed by the launching process
    (set! runconf (if (file-exists? runconfigf)
		      (setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals target)
		      (begin
			(debug:print 0 "WARNING: You do not have a run config file: " runconfigf)
			#f)))

    ;; register this run in monitor.db
    (rmt:tasks-add "run-tests" user target runname test-patts task-key) ;; params)
    (rmt:tasks-set-state-given-param-key task-key "running")

    (if (not test-patts) ;; first time in - adjust testpatt

	(set! test-patts (common:args-get-testpatt runconf)))

    ;; Now generate all the tests lists
    (set! all-tests-registry (tests:get-all))
    (set! all-test-names     (hash-table-keys all-tests-registry))
    (set! test-names         (tests:filter-test-names all-test-names test-patts))

    ;; I think seeding required-tests with all test-names makes sense but lack analysis to back that up.

		     
		     ;; (set! test-names (cons waiton test-names))))) ;; was an append, now a cons
		     )))
	     waitons)
	    (let ((remtests (delete-duplicates (append waitons tal))))
	      (if (not (null? remtests))
		  (begin
		    ;; (debug:print-info 0 "Preprocessing continues for " (string-intersperse remtests ", "))
		    (loop (car remtests)(cdr remtests))))))))

    (if (not (null? required-tests))
	(debug:print-info 1 "Adding \"" (string-intersperse required-tests " ") "\" to the run queue"))
    ;; NOTE: these are all parent tests, items are not expanded yet.
    (debug:print-info 4 "test-records=" (hash-table->alist test-records))
    (let ((reglen (configf:lookup *configdat* "setup" "runqueue")))

VAL1 Foo    
VAL2 ==>#{get test VAL1}Bar<== no spaces between Foo and Bar to pass

ltest #{scheme (case (string->symbol (conc (getenv "datapath"))) \
                        ((none) "nbfake") \
                        ((openlava) "bsub") \
                        (else "sleeprunner"))}

#================================================================
# Flexi-launcher
#================================================================
#
# [host-types]
# general ssh #{getbgesthost general}
# nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
# 
# [hosts]
# general cubian xena
# 
# [launchers]
# envsetup general
# xor/%/n 4C16G
# % nbgeneral
# 
# [jobtools]
# launcher bsub
# # if defined and not "no" flexi-launcher will bypass launcher unless there is no
# # match.
# flexi-launcher yes  

[jobtools]
flexi-launcher yes

[host-types]
general nbfake
remote  bsub

[launchers]
runfirst/sum% remote

SOMEVAR2  This should show up in SOMEVAR4 if the target is ubuntu/nfs/none
VARWITHDOLLARSIGNS The$USER/signs/should/be/replaced/with/variable

[default]
SOMEVAR3 #{rget SOMEVAR}
SOMEVAR4 #{rget SOMEVAR2}
SOMEVAR5 #{runconfigs-get SOMEVAR2}

[ubuntu/nfs/all_toplevel]
TESTPATT all_toplevel

[this/a/test]
BLAHFOO 123

[ubuntu/nfs/sleep1]
SLEEPRUNNER 1

;; (c) 2006,2007,2008,2009 Matthew Welland matt@kiatoa.com
;;  
;;   License GPL.

(define logbody "LogFileBody")

(define pass-specs '( ;; testname num-expected max-runtime
		     ("exit_0"		        1  20)
		     ("ezlog_fail_then_pass"    1  20)
		     ("ezlog_pass"	        1  20)
		     ("ez_pass"		        1  20)
		     ("lineitem_pass"	        1  20)
		     ("priority_1"	        1  20)
		     ("priority_10"	        1  20)
		     ("priority_10_waiton_1"    1  20)
		     ("priority_3"	        1  20)
		     ("priority_4"	        1  20)
		     ;; ("priority_5"	        1  20)
		     ("priority_6"	        1  20)
;;		     ("priority_7"	        1  20)
		     ("priority_8"	        1  20)
		     ("priority_9"	        1  20)
		     ("runfirst"	        7  20)
		     ("singletest"	        1  20)
		     ("singletest2"	        1  20)
		     ("special"		        1  20)
		     ("sqlitespeed"	       10  20)
		     ("test1"		        1  20)
		     ("test2"		        6  20)
		     ("test_mt_vars"	        6  20)
		     ))

(define fail-specs '( ;; testname num-expected max-runtime
		     ("exit_1"		        1  20)
		     ("ez_exit2_fail"           1  20)
		     ("ez_fail"		        1  20)
		     ("ez_fail_quick"	        1  20)
		     ("ezlog_fail"	        1  20)
		     ("lineitem_fail"	        1  20)
		     ("logpro_required_fail"    1  20)
		     ("manual_example"	        1  20)
		     ("neverrun"	        1  20)))
		     
(define warn-specs   '(("ezlog_warn"	        1  20)))

(define nost-specs   '(("wait_no_items1"        1  20)
		       ("wait_no_items2"        1  20)
		       ("wait_no_items3"        1  20)
		       ("wait_no_items4"        1  20)
		       ;; ("no_items"              1  20)
		       ))

(define (check-one-test estate estatus testname count runtime)
   (let* ((rxe      (regexp (conc "^\\s+Test: " testname "(\\(.*|\\s+)\\s+State: " estate "\\s+Status: " estatus "\\s+Runtime:\\s+(\\d+)s")))
	  (msg1     (conc testname " expecting count of " count))
	  (msg2     (conc testname " expecting runtime less than " runtime)))
     (expect:required in logbody = count msg1 rxe)
     ;;(expect:value    in logbody count < msg2 rxe)
     ))

;; Special cases
;;
(expect:error    in logbody >  0  "blocktestxz not to run"           #/Test: blocktestxz/)
(expect:ignore   in logbody >= 0  "db_sync test might not have run"  #/Test: db_sync/)
(expect:ignore   in logbody >= 0  "all_toplevel may not yet be done" #/Test: all_toplevel/)
(expect:error    in logbody =  0  "tests left in RUNNING state"      #/State: RUNNING/)
(expect:required in logbody =  1  "priority_2 is KILLED"             #/Test: priority_2\s+State: KILLED\s+Status: KILLED/)
(expect:required in logbody =  1  "priority_5 is either PASS or SKIP" #/Test: priority_5\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
(expect:required in logbody =  1  "priority_7 is either PASS or SKIP" #/Test: priority_7\s+State: COMPLETED\s+Status: (SKIP|PASS)/)
(expect:required in logbody =  1  "testxz has 1 NOT_STARTED test"    #/Test: testxz\s+State: NOT_STARTED/)
(expect:required in logbody =  1  "no items"                         #/Test: no_items\s+State: NOT_STARTED\s+Status: ZERO_ITEMS/)
(expect:warning  in logbody =  1  "dynamic waiton"                   #/Test: dynamic_waiton/)
(expect:required in logbody = 29  "blocktestxz has 29 tests"         #/Test: blocktestxz/)

;; General cases
;;
(for-each 
 (lambda (testdat)
   (apply check-one-test "COMPLETED" "PASS" testdat))
 pass-specs)

(for-each 
 (lambda (testdat)
   (apply check-one-test "COMPLETED" "FAIL" testdat))
 fail-specs)

(for-each 
 (lambda (testdat)
   (apply check-one-test "COMPLETED" "WARN" testdat))
 warn-specs)

(for-each 
 (lambda (testdat)
   (apply check-one-test "NOT_STARTED" "PREQ_DISCARDED" testdat))
 nost-specs)

;; Catch all.
;;
(expect:error    in logbody = 0   "Tests not accounted for"     #/Test: /)


;; ;; define your hooks
;; (hook:first-error   "echo \"Error hook activated: #{escaped errmsg}\"")
;; (hook:first-warning "echo \"Got warning: #{escaped warnmsg}\"")
;; (hook:value "echo \"Value hook activated: expected=#{expected}, measured=#{measured}, tolerance=#{tolerance}, message=#{message}\"")
;; 
;; ;; first ensure your run at least started
;; ;;
;; (trigger "Init"     #/This is a header/)
;; (trigger "InitEnd"  #/^\s*$/)
;; (section "Init" "Init" "InitEnd")
;; 
;; (trigger "Body"     #/^.*$/) ;; anything starts the body
;; ;; (trigger "EndBody"  #/This had better never match/)
;; 
;; (section "Body"     "Body" "EndBody")
;; 
;; (trigger "Blah2"    #/^begin Blah2/)
;; (trigger "Blah2End" #/^end Blah2/)
;; (section "Blah2"    "Blah2" "Blah2End")
;; 
;; (expect:required in "Init"  = 1 "Header"      #/This is a header/)
;; (expect:required in "LogFileBody" > 0 "Something required but not found" #/This is required but not found/)
;; (expect:value    in "LogFileBody" 1.9 0.1 "Output voltage" #/Measured voltage output:\s*([\d\.\+\-e]+)v/)
;; (expect:value    in "LogFileBody" 0.5 0.1 "Output current" #/Measured output current:\s*([\d\.\+\-e]+)mA/)
;; (expect:value    in "LogFileBody" 110e9 2e9 "A big number (first)" #/Freq:\s*([\d\.\+\-e]+)\s+Hz/)
;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (second), hook not called" #/Freq:\s*([\d\.\+\-e]+)Hz/)
;; (expect:value    in "LogFileBody" 110e9 1e9 "A big number (never activated)" #/Freq:\s*([\d\.\+\-e]+)zH/)
;; 
;; ;; Using match number
;; (expect:value    in "LogFileBody" 1.9 0.1 "Time Voltage" #/out: (\d+)\s+(\d+)/ match: 2)
;; 
;; ;; Comparison instead of tolerance
;; (expect:value    in "LogFileBody" 1.9 >   "Time voltage" #/out: (\d+)\s+(\d+)/ match: 2)
;; 
;; (expect:ignore   in "Blah2" < 99 "FALSE ERROR" #/ERROR/)
;; (expect:ignore   in "Body"  < 99 "Ignore the word error in comments" #/^\/\/.*error/)
;; (expect:warning  in "Body"  = 0 "Any warning" #/WARNING/)
;; (expect:error    in "Body"  = 0 "ERROR BLAH"  (list #/ERROR/ #/error/)) ;; but disallow any other errors
;; 
;; ;(expect in "Init"  < 1 "Junk"        #/This is bogus/)

[var]
targ -target ubuntu/nfs/all_toplevel
tp   -testpatt %

[ezsteps]
cleantop $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -remove-runs #{get var tp}  #{get var targ} -runname release_toplevel
runall   $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -run         #{get var tp}  #{get var targ} -runname release_toplevel -runwait
results  $MTRUNNER $MTTESTDIR/fullrun $MTPATH megatest -list-runs release_toplevel #{get var targ} -runname release_toplevel 

[requirements]
# waiton  #{getenv ALL_TOPLEVEL_TESTS}

# This is a "toplevel" test, it does not require waitons to be non-FAIL to run
# mode toplevel