Index: docs/megatest.lyx ================================================================== --- docs/megatest.lyx +++ docs/megatest.lyx @@ -1,3367 +1,3367 @@ -#LyX 2.0 created this file. For more info see http://www.lyx.org/ -\lyxformat 413 -\begin_document -\begin_header -\textclass docbook-book -\use_default_options false -\maintain_unincluded_children false -\language english -\language_package default -\inputencoding auto -\fontencoding global -\font_roman default -\font_sans default -\font_typewriter default -\font_default_family default -\use_non_tex_fonts false -\font_sc false -\font_osf false -\font_sf_scale 100 -\font_tt_scale 100 - -\graphics default -\default_output_format default -\output_sync 1 -\bibtex_command default -\index_command default -\paperfontsize default -\spacing single -\use_hyperref false -\papersize default -\use_geometry false -\use_amsmath 0 -\use_esint 0 -\use_mhchem 1 -\use_mathdots 1 -\cite_engine basic -\use_bibtopic false -\use_indices false -\paperorientation portrait -\suppress_date false -\use_refstyle 0 -\index Index -\shortcut idx -\color #008000 -\end_index -\secnumdepth 3 -\tocdepth 3 -\paragraph_separation indent -\paragraph_indentation default -\quotes_language english -\papercolumns 1 -\papersides 1 -\paperpagestyle default -\tracking_changes false -\output_changes false -\html_math_output 0 -\html_css_as_file 0 -\html_be_strict false -\end_header - -\begin_body - -\begin_layout Title -Megatest -\end_layout - -\begin_layout Date -5/5/2011 -\end_layout - -\begin_layout Author -Matt Welland -\end_layout - -\begin_layout Abstract -Megatest is a minimalistic tool to help automate and manage the running - of a suite of regression tests or other processes and to roll up the results - for display on a web page or dashboard. -\end_layout - -\begin_layout Standard -\begin_inset CommandInset toc -LatexCommand tableofcontents - -\end_inset - - -\end_layout - -\begin_layout Chapter -About Megatest -\end_layout - -\begin_layout Section -Megatest design philosophy -\end_layout - -\begin_layout Standard -Megatest is intended to provide the minimum needed resources to make writing - a suite of tests for software, design engineering or process control (via - owlfs for example) without being specialized for any specific problem space. - Megatest in of itself does not know what constitutes a PASS or FAIL of - a test. - In most cases megatest is best used in conjunction with logpro or a similar - tool to parse, analyze and decide on the test outcome. - A call to megatest can then be made to record the result. - -\end_layout - -\begin_layout Section -Megatest architecture -\end_layout - -\begin_layout Standard -All data to specify the tests and configure the system is stored in plain - text files. - All system state is stored in an sqlite3 database. - Tests are launched using the launching system available for the distributed - compute platform in use. - A template script is provided which can launch jobs on local and remote - Linux hosts. - Currently megatest uses the network filesystem to -\begin_inset Quotes eld -\end_inset - -call home -\begin_inset Quotes erd -\end_inset - - to your master sqlite3 database. -\end_layout - -\begin_layout Chapter -Installation -\end_layout - -\begin_layout Section -Compilation -\end_layout - -\begin_layout Subsection -Dependencies -\end_layout - -\begin_layout Standard -Chicken scheme and a number of eggs are required for building megatest. - See the file utils/installall.sh for an automated way to install the dependencie -s on Linux. -\end_layout - -\begin_layout Subsection -Build and install -\end_layout - -\begin_layout Standard -Run -\begin_inset Quotes eld -\end_inset - -make test -\begin_inset Quotes erd -\end_inset - - to create the megatest executable. - You may wish to copy the executable to a centrally accessible location. -\end_layout - -\begin_layout Section -Setup -\end_layout - -\begin_layout Subsection -Create megatest.config -\end_layout - -\begin_layout Standard -Create the file megatest.config using the megatest.config template from the - tests directory. - At a minimum you need the following: -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -# Fields are the keys under which your test runs are organized -\end_layout - -\begin_layout Plain Layout - -[fields] -\end_layout - -\begin_layout Plain Layout - -field1 TEXT -\end_layout - -\begin_layout Plain Layout - -field2 TEXT -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -[jobtools] -\end_layout - -\begin_layout Plain Layout - -# The launcher launches jobs to the local or remote hosts, -\end_layout - -\begin_layout Plain Layout - -# the job is managed on the target host by megatest, -\end_layout - -\begin_layout Plain Layout - -# comment out launcher to run local only. - An example launcher -\end_layout - -\begin_layout Plain Layout - -# "nbfake" can be found in the utils directory. - -\end_layout - -\begin_layout Plain Layout - -launcher nbfake -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# The disks section specifies where the tests will be run. - As you -\end_layout - -\begin_layout Plain Layout - -# run out of space in a partition you can add additional disks -\end_layout - -\begin_layout Plain Layout - -# entries. -\end_layout - -\begin_layout Plain Layout - -# Format is: -\end_layout - -\begin_layout Plain Layout - -# name /path/to/area -\end_layout - -\begin_layout Plain Layout - -[disks] -\end_layout - -\begin_layout Plain Layout - -disk1 /tmp -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create runconfigs.config -\end_layout - -\begin_layout Standard -This file is used to set environment variables that are run specific. - You can simply create an empty file to start. -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -# runconfigs.config -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create the tests directory and your first test -\end_layout - -\begin_layout Standard -The structure should look like this: -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -../tests -\end_layout - -\begin_layout Plain Layout - - ├── megatest.config -\end_layout - -\begin_layout Plain Layout - - ├── runconfigs.config -\end_layout - -\begin_layout Plain Layout - - └── tests -\end_layout - -\begin_layout Plain Layout - - └── mytest -\end_layout - -\begin_layout Plain Layout - - ├── main.sh -\end_layout - -\begin_layout Plain Layout - - └── testconfig -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create the testconfig file for your test -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -[setup] -\end_layout - -\begin_layout Plain Layout - -runscript main.sh -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create your test running script, main.sh -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -#!/bin/bash -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -megatest -runstep mystep1 "sleep 20;echo Done" -m "mystep1 is done" -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -megatest -test-status :state COMPLETED :status PASS -m "This is a comment" -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Run megatest and watch your run progress -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -megatest :field1 abc :field2 def :runname 2011week08.4a -runall -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -watch megatest -list-runs % -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# OR use the dashboard -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -dashboard & -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Code - -\end_layout - -\begin_layout Chapter -How to Write Tests -\end_layout - -\begin_layout Section -A Simple Test with one Step -\end_layout - -\begin_layout Subsection -Create your test directory. - The directory name will be the name of the test -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -mkdir simpletest -\end_layout - -\begin_layout Plain Layout - -cd simpletest -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create your testconfig file. - This file contains various specifications for your test. - For our example the test author has chosen to write the test using csh - scripting. -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -# testconfig -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -[setup] -\end_layout - -\begin_layout Plain Layout - -runscript main.csh -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Create your main.csh script -\end_layout - -\begin_layout Standard -Note: Using csh is NOT recommended. - Use bash, perl, ruby, zsh or anything other than csh. - We use csh here because it is popular in some industries. - -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -#!/bin/tcsh -x -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# run the cpu1 simulation. -\end_layout - -\begin_layout Plain Layout - -# The step name is "run_simulation" -\end_layout - -\begin_layout Plain Layout - -# The commandline being run for this step is "runsim cpu1" -\end_layout - -\begin_layout Plain Layout - -# The logpro file to validate the output from the run is "runsim.logpro" -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -runstep run_simulation -logpro runsim.logpro "runsim cpu1" -\end_layout - -\begin_layout Plain Layout - -if ($? == 0) -\end_layout - -\begin_layout Plain Layout - - set status = PASS -\end_layout - -\begin_layout Plain Layout - -else -\end_layout - -\begin_layout Plain Layout - - set status = FAIL -\end_layout - -\begin_layout Plain Layout - -end -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -test-status :state COMPLETED :status $status -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Standard -You can now run megatest and the created test directory will contain the - new files -\begin_inset Quotes eld -\end_inset - -run_simulation.html -\begin_inset Quotes erd -\end_inset - - and -\begin_inset Quotes eld -\end_inset - -run_simulation.log -\begin_inset Quotes erd -\end_inset - -. - If you are using the dashboard you can click on the run and then push the - -\begin_inset Quotes eld -\end_inset - -View log -\begin_inset Quotes erd -\end_inset - - button to view the log file in firefox. -\end_layout - -\begin_layout Section -Simple Test with Multiple Steps -\end_layout - -\begin_layout Standard -To run multiple steps simply add them to the main.csh file. - Here we add a step to test -\begin_inset Quotes eld -\end_inset - -cpu2 -\begin_inset Quotes erd -\end_inset - -. - The second step that tests cpu2 will only run after the step that tested - -\begin_inset Quotes eld -\end_inset - -cpu1 -\begin_inset Quotes erd -\end_inset - - completes. -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -#!/bin/tcsh -x -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# run the cpu1 simulation. -\end_layout - -\begin_layout Plain Layout - -# The step name is "run_simulation" -\end_layout - -\begin_layout Plain Layout - -# The commandline being run for this step is "runsim cpu1" -\end_layout - -\begin_layout Plain Layout - -# The logpro file to validate the output from the run is "runsim.logpro" -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -runstep run_simulation_cpu1 -logpro runsim.logpro "runsim cpu1" - && -\backslash - -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -runstep run_simulation_cpu2 -logpro runsim.logpro "runsim cpu2" -\end_layout - -\begin_layout Plain Layout - -if ($? == 0) -\end_layout - -\begin_layout Plain Layout - - set status = PASS -\end_layout - -\begin_layout Plain Layout - -else -\end_layout - -\begin_layout Plain Layout - - set status = FAIL -\end_layout - -\begin_layout Plain Layout - -end -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -test-status :state COMPLETED :status $status -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Section -Simple Test with Multiple Steps, Some in Parallel -\end_layout - -\begin_layout Subsection -The Makefile -\end_layout - -\begin_layout Standard -A good way to run steps in parallel within a single test, especially when - there are following steps, is to use the Unix Make utility. - Writing Makefiles is beyond the scope of this document but here is a minimal - example that will run -\begin_inset Quotes eld -\end_inset - -runsim cpu1 -\begin_inset Quotes erd -\end_inset - - and -\begin_inset Quotes eld -\end_inset - -runsim cpu2 -\begin_inset Quotes erd -\end_inset - - in parallel. - For more information on make try -\begin_inset Quotes eld -\end_inset - -info make -\begin_inset Quotes erd -\end_inset - - at the Linux command prompt. -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -# Example Makefile to run two steps in parallel -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -RTLDIR=/path/to/rtl -\end_layout - -\begin_layout Plain Layout - -CPUS = cpu1 cpu2 -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -run_simulation_$(CPUS).html : $(RTLDIR)/$(CPUS) -\end_layout - -\begin_layout Plain Layout - - $(MEGATEST) -runstep run_simulation_$(CPUS) -logpro runsim.logpro "runsim - $(CPUS) -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -The main.csh file -\end_layout - -\begin_layout Standard -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -#!/bin/tcsh -x -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# run the cpu1 and cpu2 simulations in parallel. - -\end_layout - -\begin_layout Plain Layout - -# The -j parameter tells make how many jobs it may run in parallel -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -make -j 2 -\end_layout - -\begin_layout Plain Layout - -if ($? == 0) -\end_layout - -\begin_layout Plain Layout - - set status = PASS -\end_layout - -\begin_layout Plain Layout - -else -\end_layout - -\begin_layout Plain Layout - - set status = FAIL -\end_layout - -\begin_layout Plain Layout - -end -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -test-status :state COMPLETED :status $status -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Section -Simple Test with Iteration -\end_layout - -\begin_layout Standard -Since no jobs run after the cpu1 and cpu2 simulations in this test it is - possible to use iterated mode. -\end_layout - -\begin_layout Subsection -Update you testconfig file for iteration -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -[setup] -\end_layout - -\begin_layout Plain Layout - -runscript main.csh -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -[items] -\end_layout - -\begin_layout Plain Layout - -CPU cpu1 cpu2 -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Subsection -Rewrite your main.csh for iteration -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -#!/bin/tcsh -x -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -# run the cpu simulation but now use the environment variable $CPU -\end_layout - -\begin_layout Plain Layout - -# to select what cpu to run the simulation against -\end_layout - -\begin_layout Plain Layout - -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -runstep run_simulation -logpro runsim.logpro "runsim $CPU" -\end_layout - -\begin_layout Plain Layout - -if ($? == 0) -\end_layout - -\begin_layout Plain Layout - - set status = PASS -\end_layout - -\begin_layout Plain Layout - -else -\end_layout - -\begin_layout Plain Layout - - set status = FAIL -\end_layout - -\begin_layout Plain Layout - -end -\end_layout - -\begin_layout Plain Layout - -$MEGATEST -test-status :state COMPLETED :status $status -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Section -Tests with Inter-test dependencies -\end_layout - -\begin_layout Standard -Sometimes a test depends on the output from a previous test or it may not - make sense to run a test is another test does not complete with status - -\begin_inset Quotes eld -\end_inset - -PASS -\begin_inset Quotes erd -\end_inset - -. - In either of these scenarios you can use the -\begin_inset Quotes eld -\end_inset - -waiton -\begin_inset Quotes erd -\end_inset - - keyword in your testconfig file to indicate that this test must wait on - one or more tests to complete before being launched. - In this example there is no point in running the -\begin_inset Quotes eld -\end_inset - -system -\begin_inset Quotes erd -\end_inset - - test if the -\begin_inset Quotes eld -\end_inset - -cpu -\begin_inset Quotes erd -\end_inset - - and -\begin_inset Quotes eld -\end_inset - -mem -\begin_inset Quotes erd -\end_inset - - tests either do not complete or complete but with status -\begin_inset Quotes eld -\end_inset - -FAIL -\begin_inset Quotes erd -\end_inset - -. -\end_layout - -\begin_layout Paragraph -\begin_inset listings -inline false -status open - -\begin_layout Plain Layout - -# testconfig for the "system" test -\end_layout - -\begin_layout Plain Layout - -[setup] -\end_layout - -\begin_layout Plain Layout - -runscript main.csh -\end_layout - -\begin_layout Plain Layout - -waiton cpu mem -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Chapter -Reference -\end_layout - -\begin_layout Section -Configuration file Syntax -\end_layout - -\begin_layout Subsection -Sections -\end_layout - -\begin_layout Standard -[section name] -\end_layout - -\begin_layout Quotation -This creates a section named -\begin_inset Quotes eld -\end_inset - -section name -\begin_inset Quotes erd -\end_inset - - -\end_layout - -\begin_layout Subsection -Variables -\end_layout - -\begin_layout Standard -VARX has this value -\end_layout - -\begin_layout Quotation -The variable -\begin_inset Quotes eld -\end_inset - -VARX -\begin_inset Quotes erd -\end_inset - - will have the value -\begin_inset Quotes eld -\end_inset - -has this value -\begin_inset Quotes erd -\end_inset - - -\end_layout - -\begin_layout Subsection -Includes -\end_layout - -\begin_layout Standard -[include filename] -\end_layout - -\begin_layout Quotation -The file named -\begin_inset Quotes eld -\end_inset - -filename -\begin_inset Quotes erd -\end_inset - - will be included as if part of the calling file. - NOTE: This means no section can be named -\begin_inset Quotes eld -\end_inset - -include -\begin_inset Quotes eld -\end_inset - - (with the whitespace). -\end_layout - -\begin_layout Subsection -Setting a variable by running a command -\end_layout - -\begin_layout Standard -VARNAME [system ls /tmp] -\end_layout - -\begin_layout Quotation -The variable -\begin_inset Quotes eld -\end_inset - -VARNAME -\begin_inset Quotes erd -\end_inset - - will get a value created by the Unix command -\begin_inset Quotes eld -\end_inset - -ls /tmp -\begin_inset Quotes erd -\end_inset - -. - All lines of output from the command will be joined with a space. -\end_layout - -\begin_layout Subsection -Notes -\end_layout - -\begin_layout Itemize -Some variables are infered as lists. - Each token on the line separated by whitespace will be member of the list. -\end_layout - -\begin_layout Itemize -Comments (lines starting with #) and blank lines are ignored. -\end_layout - -\begin_layout Section -Environment variables -\end_layout - -\begin_layout Standard -\begin_inset Tabular - - - - - - - -\begin_inset Text - -\begin_layout Plain Layout -Variable -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Purpose -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Default value/Comment -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_CMDINFO -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Conveys test variables to the megatest test runner. -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Not for use by the end user -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_TEST_RUN_DIR -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Directory assigned by megatest for the test to run. -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Tests can consider this their -\begin_inset Quotes eld -\end_inset - -home -\begin_inset Quotes erd -\end_inset - - directory -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_TEST_NAME -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Name of the test, corrosponds to the directory name under tests. -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_ITEM_INFO -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Iterated tests will set this to a sequence of key/values ((KEY val) ...) -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_RUN_AREA_HOME -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Directory where megatest was launched from and where the tests code can - be found -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_RUNNAME -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Name of this run as set by the :runname parameter -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -MT_MEGATEST -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Path/Filename to megatest executable. - Found either from called path or but using the -\begin_inset Quotes eld -\end_inset - -exectuable -\begin_inset Quotes erd -\end_inset - - keyword in the [setup] section. -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout - .... -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -The field values as set on the megatest -runall command line (e.g. - :field1 abc) -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Planning ahead your field names is important so as to avoid collisions with - other environment variables -\end_layout - -\end_inset - - - - -\end_inset - - -\end_layout - -\begin_layout Section -Configuration files -\end_layout - -\begin_layout Subsection -megatest.config -\end_layout - -\begin_layout Standard -\begin_inset Tabular - - - - - - - - - -\begin_inset Text - -\begin_layout Plain Layout -section -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -variable -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -value -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -required -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -comment -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[setup] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -max_concurrent_jobs -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -if variable is not defined no limit on jobs -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -executable -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -full path to megatest binary -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Use only if necessary, megatest will extract the location from where it - used to launch and add append that to the PATH for test runs. -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -runsdir -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -full path to where the link tree to all runs will be created -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -Because your runs may be spread out over several disk partitions a central - link tree is created to make finding all the runs easy. -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[fields] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -at least one -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[jobtools] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -launcher -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -command line used to launch jobs - the job command (megatest -execute) will - be appended to this -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -workhosts -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -list of hostnames to run jobs on NOT SUPPORTED RIGHT NOW -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -n/a -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[env-override] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -any string -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -These are set on the test launching machine, not the test running machine. - Typical usage is to control the host or run queue for launching tests. - These values will not be seen by the test when it runs. -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[disks] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -a valid path writable by the test launching process and by the test process -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -yes -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -The disk usage balancing algorithm is to choose the disk with the least - space for each test run. - -\end_layout - -\end_inset - - - - -\end_inset - - -\end_layout - -\begin_layout Subsection -runconfigs.config file -\end_layout - -\begin_layout Standard -\begin_inset Tabular - - - - - - - - - -\begin_inset Text - -\begin_layout Plain Layout -section -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -variable -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -value -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -required? -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -comment -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[default] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -any -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -variables set in this section will be available for all runs, defining the - same variable in another section will override the value from the default - section -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[field1value/field2value...] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -string of letters, numbers and underscore -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -any -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -the values in this section will be set for any run where field1 is field1value, - field2 is field2value and fieldN is fieldNvalue. - -\end_layout - -\end_inset - - - - -\end_inset - - -\end_layout - -\begin_layout Standard -Example: a test suite that checks that a piece of software works correctly - for different customer configurations and locations each of which is done - as a separate release regression run. - The fields, CUSTOMER and LOCATION were chosen. - The following runconfigs.config file would set some variables specific to - runs for megacorp in India and femtocorp in the Cook Islands and New Zealand: -\end_layout - -\begin_layout Code -# runconfigs.config -\end_layout - -\begin_layout Code -[default] -\end_layout - -\begin_layout Code -ENCRYTION true -\end_layout - -\begin_layout Code -# -\end_layout - -\begin_layout Code -[megacorp/india] -\end_layout - -\begin_layout Code -TESTPATH /nfs/testing/megacorp_runs -\end_layout - -\begin_layout Code -# -\end_layout - -\begin_layout Code -[femtocorp/cook_islands] -\end_layout - -\begin_layout Code -ENCRYTION false -\end_layout - -\begin_layout Code -TESTPATH /afs/kiatoa/testing/cook_islands -\end_layout - -\begin_layout Code -# -\end_layout - -\begin_layout Code -[femtocorp/new_zealand] -\end_layout - -\begin_layout Code -TESTPATH /afs/kiatao/testing/new_zealand -\end_layout - -\begin_layout Code -# -\end_layout - -\begin_layout Code -[megacorp/new_zealand] -\end_layout - -\begin_layout Code -TESTPATH /nfs/testing/megacorp_runs -\end_layout - -\begin_layout Paragraph* -Running megatest like this: -\end_layout - -\begin_layout Code -megatest :CUSTOMER megacorp :LOCATION new_zealand :runname week12_2011_run1 - -runall -\end_layout - -\begin_layout Paragraph* -Would set: -\end_layout - -\begin_layout Code -ENCRYPTION true -\end_layout - -\begin_layout Code -TESTPATH /nfs/testing/megacorp_runs -\end_layout - -\begin_layout Section -Writing tests -\end_layout - -\begin_layout Subsection -testconfig file -\end_layout - -\begin_layout Standard -\begin_inset Tabular - - - - - - - - - -\begin_inset Text - -\begin_layout Plain Layout -section -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -variable -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -value -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -required? -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -comments -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[setup] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -runscript -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -name of script to execute for this test -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -yes -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -The script must be executable and either provide the full path or put a - copy at the top of your test directory -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[requirements] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -waiton -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -list of valid test names -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -This test will not run until the named tests are state completed and status - PASS -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -[items] -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -any valid -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -list of values -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -no -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -The test will be repeated once for each item with the variable name set - to the value. - If there is more than one variable then the test will be run against all - unique combinations of the values -\end_layout - -\end_inset - - - - -\end_inset - - -\end_layout - -\begin_layout Subsection -Command line -\end_layout - -\begin_layout Standard -\begin_inset Tabular - - - - - - - - -\begin_inset Text - -\begin_layout Plain Layout -switch or param -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -parameter -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -purpose -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -comments -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --h -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -brief help -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --runall -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -run all tests -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --runtests -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -test1,test2,... -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -run one or more tests -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --step -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -stepname -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -record a step -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -requires :state and :status -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --test-status -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -record the test status -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -requires :state and :status -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --setlog -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -logfilename -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -set the logfile name for a test -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -path is assumed to be relative to the test run directory -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --set-toplog -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -logfilename -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -set the logfile name for the top test in an iterated test run -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -each sub test can have its own logfile set -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --m -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -\begin_inset Quotes eld -\end_inset - -comment -\begin_inset Quotes erd -\end_inset - - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout -sets a comment for the step, test or run -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -:runname -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -:state -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout -:status -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --list-runs -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --testpatt -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --itempatt -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --showkeys -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --force -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --xterm -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --remove-runs -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout - -\shape italic -Test helpers -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --runstep -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\begin_inset Text - -\begin_layout Plain Layout --logpro -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - -\begin_inset Text - -\begin_layout Plain Layout - -\end_layout - -\end_inset - - - - -\end_inset - - -\end_layout - -\end_body -\end_document +#LyX 2.0 created this file. For more info see http://www.lyx.org/ +\lyxformat 413 +\begin_document +\begin_header +\textclass docbook-book +\use_default_options false +\maintain_unincluded_children false +\language english +\language_package default +\inputencoding auto +\fontencoding global +\font_roman default +\font_sans default +\font_typewriter default +\font_default_family default +\use_non_tex_fonts false +\font_sc false +\font_osf false +\font_sf_scale 100 +\font_tt_scale 100 + +\graphics default +\default_output_format default +\output_sync 1 +\bibtex_command default +\index_command default +\paperfontsize default +\spacing single +\use_hyperref false +\papersize default +\use_geometry false +\use_amsmath 0 +\use_esint 0 +\use_mhchem 1 +\use_mathdots 1 +\cite_engine basic +\use_bibtopic false +\use_indices false +\paperorientation portrait +\suppress_date false +\use_refstyle 0 +\index Index +\shortcut idx +\color #008000 +\end_index +\secnumdepth 3 +\tocdepth 3 +\paragraph_separation indent +\paragraph_indentation default +\quotes_language english +\papercolumns 1 +\papersides 1 +\paperpagestyle default +\tracking_changes false +\output_changes false +\html_math_output 0 +\html_css_as_file 0 +\html_be_strict false +\end_header + +\begin_body + +\begin_layout Title +Megatest +\end_layout + +\begin_layout Date +5/5/2011 +\end_layout + +\begin_layout Author +Matt Welland +\end_layout + +\begin_layout Abstract +Megatest is a minimalistic tool to help automate and manage the running + of a suite of regression tests or other processes and to roll up the results + for display on a web page or dashboard. +\end_layout + +\begin_layout Standard +\begin_inset CommandInset toc +LatexCommand tableofcontents + +\end_inset + + +\end_layout + +\begin_layout Chapter +About Megatest +\end_layout + +\begin_layout Section +Megatest design philosophy +\end_layout + +\begin_layout Standard +Megatest is intended to provide the minimum needed resources to make writing + a suite of tests for software, design engineering or process control (via + owlfs for example) without being specialized for any specific problem space. + Megatest in of itself does not know what constitutes a PASS or FAIL of + a test. + In most cases megatest is best used in conjunction with logpro or a similar + tool to parse, analyze and decide on the test outcome. + A call to megatest can then be made to record the result. + +\end_layout + +\begin_layout Section +Megatest architecture +\end_layout + +\begin_layout Standard +All data to specify the tests and configure the system is stored in plain + text files. + All system state is stored in an sqlite3 database. + Tests are launched using the launching system available for the distributed + compute platform in use. + A template script is provided which can launch jobs on local and remote + Linux hosts. + Currently megatest uses the network filesystem to +\begin_inset Quotes eld +\end_inset + +call home +\begin_inset Quotes erd +\end_inset + + to your master sqlite3 database. +\end_layout + +\begin_layout Chapter +Installation +\end_layout + +\begin_layout Section +Compilation +\end_layout + +\begin_layout Subsection +Dependencies +\end_layout + +\begin_layout Standard +Chicken scheme and a number of eggs are required for building megatest. + See the file utils/installall.sh for an automated way to install the dependencie +s on Linux. +\end_layout + +\begin_layout Subsection +Build and install +\end_layout + +\begin_layout Standard +Run +\begin_inset Quotes eld +\end_inset + +make test +\begin_inset Quotes erd +\end_inset + + to create the megatest executable. + You may wish to copy the executable to a centrally accessible location. +\end_layout + +\begin_layout Section +Setup +\end_layout + +\begin_layout Subsection +Create megatest.config +\end_layout + +\begin_layout Standard +Create the file megatest.config using the megatest.config template from the + tests directory. + At a minimum you need the following: +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# Fields are the keys under which your test runs are organized +\end_layout + +\begin_layout Plain Layout + +[fields] +\end_layout + +\begin_layout Plain Layout + +field1 TEXT +\end_layout + +\begin_layout Plain Layout + +field2 TEXT +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +[jobtools] +\end_layout + +\begin_layout Plain Layout + +# The launcher launches jobs to the local or remote hosts, +\end_layout + +\begin_layout Plain Layout + +# the job is managed on the target host by megatest, +\end_layout + +\begin_layout Plain Layout + +# comment out launcher to run local only. + An example launcher +\end_layout + +\begin_layout Plain Layout + +# "nbfake" can be found in the utils directory. + +\end_layout + +\begin_layout Plain Layout + +launcher nbfake +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# The disks section specifies where the tests will be run. + As you +\end_layout + +\begin_layout Plain Layout + +# run out of space in a partition you can add additional disks +\end_layout + +\begin_layout Plain Layout + +# entries. +\end_layout + +\begin_layout Plain Layout + +# Format is: +\end_layout + +\begin_layout Plain Layout + +# name /path/to/area +\end_layout + +\begin_layout Plain Layout + +[disks] +\end_layout + +\begin_layout Plain Layout + +disk1 /tmp +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create runconfigs.config +\end_layout + +\begin_layout Standard +This file is used to set environment variables that are run specific. + You can simply create an empty file to start. +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# runconfigs.config +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create the tests directory and your first test +\end_layout + +\begin_layout Standard +The structure should look like this: +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +../tests +\end_layout + +\begin_layout Plain Layout + + ├── megatest.config +\end_layout + +\begin_layout Plain Layout + + ├── runconfigs.config +\end_layout + +\begin_layout Plain Layout + + └── tests +\end_layout + +\begin_layout Plain Layout + + └── mytest +\end_layout + +\begin_layout Plain Layout + + ├── main.sh +\end_layout + +\begin_layout Plain Layout + + └── testconfig +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create the testconfig file for your test +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +[setup] +\end_layout + +\begin_layout Plain Layout + +runscript main.sh +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create your test running script, main.sh +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +#!/bin/bash +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -runstep mystep1 "sleep 20;echo Done" -m "mystep1 is done" +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -test-status :state COMPLETED :status PASS -m "This is a comment" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Run megatest and watch your run progress +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +megatest :field1 abc :field2 def :runname 2011week08.4a -runall +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +watch megatest -list-runs % +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# OR use the dashboard +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +dashboard & +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Code + +\end_layout + +\begin_layout Chapter +How to Write Tests +\end_layout + +\begin_layout Section +A Simple Test with one Step +\end_layout + +\begin_layout Subsection +Create your test directory. + The directory name will be the name of the test +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +mkdir simpletest +\end_layout + +\begin_layout Plain Layout + +cd simpletest +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create your testconfig file. + This file contains various specifications for your test. + For our example the test author has chosen to write the test using csh + scripting. +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# testconfig +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +[setup] +\end_layout + +\begin_layout Plain Layout + +runscript main.csh +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Create your main.csh script +\end_layout + +\begin_layout Standard +Note: Using csh is NOT recommended. + Use bash, perl, ruby, zsh or anything other than csh. + We use csh here because it is popular in some industries. + +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +#!/bin/tcsh -x +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# run the cpu1 simulation. +\end_layout + +\begin_layout Plain Layout + +# The step name is "run_simulation" +\end_layout + +\begin_layout Plain Layout + +# The commandline being run for this step is "runsim cpu1" +\end_layout + +\begin_layout Plain Layout + +# The logpro file to validate the output from the run is "runsim.logpro" +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -runstep run_simulation -logpro runsim.logpro "runsim cpu1" +\end_layout + +\begin_layout Plain Layout + +if ($? == 0) +\end_layout + +\begin_layout Plain Layout + + set status = PASS +\end_layout + +\begin_layout Plain Layout + +else +\end_layout + +\begin_layout Plain Layout + + set status = FAIL +\end_layout + +\begin_layout Plain Layout + +end +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -test-status :state COMPLETED :status $status +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You can now run megatest and the created test directory will contain the + new files +\begin_inset Quotes eld +\end_inset + +run_simulation.html +\begin_inset Quotes erd +\end_inset + + and +\begin_inset Quotes eld +\end_inset + +run_simulation.log +\begin_inset Quotes erd +\end_inset + +. + If you are using the dashboard you can click on the run and then push the + +\begin_inset Quotes eld +\end_inset + +View log +\begin_inset Quotes erd +\end_inset + + button to view the log file in firefox. +\end_layout + +\begin_layout Section +Simple Test with Multiple Steps +\end_layout + +\begin_layout Standard +To run multiple steps simply add them to the main.csh file. + Here we add a step to test +\begin_inset Quotes eld +\end_inset + +cpu2 +\begin_inset Quotes erd +\end_inset + +. + The second step that tests cpu2 will only run after the step that tested + +\begin_inset Quotes eld +\end_inset + +cpu1 +\begin_inset Quotes erd +\end_inset + + completes. +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +#!/bin/tcsh -x +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# run the cpu1 simulation. +\end_layout + +\begin_layout Plain Layout + +# The step name is "run_simulation" +\end_layout + +\begin_layout Plain Layout + +# The commandline being run for this step is "runsim cpu1" +\end_layout + +\begin_layout Plain Layout + +# The logpro file to validate the output from the run is "runsim.logpro" +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -runstep run_simulation_cpu1 -logpro runsim.logpro "runsim cpu1" + && +\backslash + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -runstep run_simulation_cpu2 -logpro runsim.logpro "runsim cpu2" +\end_layout + +\begin_layout Plain Layout + +if ($? == 0) +\end_layout + +\begin_layout Plain Layout + + set status = PASS +\end_layout + +\begin_layout Plain Layout + +else +\end_layout + +\begin_layout Plain Layout + + set status = FAIL +\end_layout + +\begin_layout Plain Layout + +end +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -test-status :state COMPLETED :status $status +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Section +Simple Test with Multiple Steps, Some in Parallel +\end_layout + +\begin_layout Subsection +The Makefile +\end_layout + +\begin_layout Standard +A good way to run steps in parallel within a single test, especially when + there are following steps, is to use the Unix Make utility. + Writing Makefiles is beyond the scope of this document but here is a minimal + example that will run +\begin_inset Quotes eld +\end_inset + +runsim cpu1 +\begin_inset Quotes erd +\end_inset + + and +\begin_inset Quotes eld +\end_inset + +runsim cpu2 +\begin_inset Quotes erd +\end_inset + + in parallel. + For more information on make try +\begin_inset Quotes eld +\end_inset + +info make +\begin_inset Quotes erd +\end_inset + + at the Linux command prompt. +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# Example Makefile to run two steps in parallel +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +RTLDIR=/path/to/rtl +\end_layout + +\begin_layout Plain Layout + +CPUS = cpu1 cpu2 +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +run_simulation_$(CPUS).html : $(RTLDIR)/$(CPUS) +\end_layout + +\begin_layout Plain Layout + + $(MT_MEGATEST) -runstep run_simulation_$(CPUS) -logpro runsim.logpro "runsim + $(CPUS) +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +The main.csh file +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +#!/bin/tcsh -x +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# run the cpu1 and cpu2 simulations in parallel. + +\end_layout + +\begin_layout Plain Layout + +# The -j parameter tells make how many jobs it may run in parallel +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +make -j 2 +\end_layout + +\begin_layout Plain Layout + +if ($? == 0) +\end_layout + +\begin_layout Plain Layout + + set status = PASS +\end_layout + +\begin_layout Plain Layout + +else +\end_layout + +\begin_layout Plain Layout + + set status = FAIL +\end_layout + +\begin_layout Plain Layout + +end +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -test-status :state COMPLETED :status $status +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Section +Simple Test with Iteration +\end_layout + +\begin_layout Standard +Since no jobs run after the cpu1 and cpu2 simulations in this test it is + possible to use iterated mode. +\end_layout + +\begin_layout Subsection +Update you testconfig file for iteration +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +[setup] +\end_layout + +\begin_layout Plain Layout + +runscript main.csh +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +[items] +\end_layout + +\begin_layout Plain Layout + +CPU cpu1 cpu2 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +Rewrite your main.csh for iteration +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +#!/bin/tcsh -x +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +# run the cpu simulation but now use the environment variable $CPU +\end_layout + +\begin_layout Plain Layout + +# to select what cpu to run the simulation against +\end_layout + +\begin_layout Plain Layout + +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -runstep run_simulation -logpro runsim.logpro "runsim $CPU" +\end_layout + +\begin_layout Plain Layout + +if ($? == 0) +\end_layout + +\begin_layout Plain Layout + + set status = PASS +\end_layout + +\begin_layout Plain Layout + +else +\end_layout + +\begin_layout Plain Layout + + set status = FAIL +\end_layout + +\begin_layout Plain Layout + +end +\end_layout + +\begin_layout Plain Layout + +$MT_MEGATEST -test-status :state COMPLETED :status $status +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Section +Tests with Inter-test dependencies +\end_layout + +\begin_layout Standard +Sometimes a test depends on the output from a previous test or it may not + make sense to run a test is another test does not complete with status + +\begin_inset Quotes eld +\end_inset + +PASS +\begin_inset Quotes erd +\end_inset + +. + In either of these scenarios you can use the +\begin_inset Quotes eld +\end_inset + +waiton +\begin_inset Quotes erd +\end_inset + + keyword in your testconfig file to indicate that this test must wait on + one or more tests to complete before being launched. + In this example there is no point in running the +\begin_inset Quotes eld +\end_inset + +system +\begin_inset Quotes erd +\end_inset + + test if the +\begin_inset Quotes eld +\end_inset + +cpu +\begin_inset Quotes erd +\end_inset + + and +\begin_inset Quotes eld +\end_inset + +mem +\begin_inset Quotes erd +\end_inset + + tests either do not complete or complete but with status +\begin_inset Quotes eld +\end_inset + +FAIL +\begin_inset Quotes erd +\end_inset + +. +\end_layout + +\begin_layout Paragraph +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# testconfig for the "system" test +\end_layout + +\begin_layout Plain Layout + +[setup] +\end_layout + +\begin_layout Plain Layout + +runscript main.csh +\end_layout + +\begin_layout Plain Layout + +waiton cpu mem +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Chapter +Reference +\end_layout + +\begin_layout Section +Configuration file Syntax +\end_layout + +\begin_layout Subsection +Sections +\end_layout + +\begin_layout Standard +[section name] +\end_layout + +\begin_layout Quotation +This creates a section named +\begin_inset Quotes eld +\end_inset + +section name +\begin_inset Quotes erd +\end_inset + + +\end_layout + +\begin_layout Subsection +Variables +\end_layout + +\begin_layout Standard +VARX has this value +\end_layout + +\begin_layout Quotation +The variable +\begin_inset Quotes eld +\end_inset + +VARX +\begin_inset Quotes erd +\end_inset + + will have the value +\begin_inset Quotes eld +\end_inset + +has this value +\begin_inset Quotes erd +\end_inset + + +\end_layout + +\begin_layout Subsection +Includes +\end_layout + +\begin_layout Standard +[include filename] +\end_layout + +\begin_layout Quotation +The file named +\begin_inset Quotes eld +\end_inset + +filename +\begin_inset Quotes erd +\end_inset + + will be included as if part of the calling file. + NOTE: This means no section can be named +\begin_inset Quotes eld +\end_inset + +include +\begin_inset Quotes eld +\end_inset + + (with the whitespace). +\end_layout + +\begin_layout Subsection +Setting a variable by running a command +\end_layout + +\begin_layout Standard +VARNAME [system ls /tmp] +\end_layout + +\begin_layout Quotation +The variable +\begin_inset Quotes eld +\end_inset + +VARNAME +\begin_inset Quotes erd +\end_inset + + will get a value created by the Unix command +\begin_inset Quotes eld +\end_inset + +ls /tmp +\begin_inset Quotes erd +\end_inset + +. + All lines of output from the command will be joined with a space. +\end_layout + +\begin_layout Subsection +Notes +\end_layout + +\begin_layout Itemize +Some variables are infered as lists. + Each token on the line separated by whitespace will be member of the list. +\end_layout + +\begin_layout Itemize +Comments (lines starting with #) and blank lines are ignored. +\end_layout + +\begin_layout Section +Environment variables +\end_layout + +\begin_layout Standard +\begin_inset Tabular + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +Variable +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Purpose +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Default value/Comment +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_CMDINFO +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Conveys test variables to the megatest test runner. +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Not for use by the end user +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_TEST_RUN_DIR +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Directory assigned by megatest for the test to run. +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Tests can consider this their +\begin_inset Quotes eld +\end_inset + +home +\begin_inset Quotes erd +\end_inset + + directory +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_TEST_NAME +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Name of the test, corrosponds to the directory name under tests. +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_ITEM_INFO +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Iterated tests will set this to a sequence of key/values ((KEY val) ...) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_RUN_AREA_HOME +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Directory where megatest was launched from and where the tests code can + be found +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_RUNNAME +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Name of this run as set by the :runname parameter +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +MT_MEGATEST +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Path/Filename to megatest executable. + Found either from called path or but using the +\begin_inset Quotes eld +\end_inset + +exectuable +\begin_inset Quotes erd +\end_inset + + keyword in the [setup] section. +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout + .... +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +The field values as set on the megatest -runall command line (e.g. + :field1 abc) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Planning ahead your field names is important so as to avoid collisions with + other environment variables +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +Configuration files +\end_layout + +\begin_layout Subsection +megatest.config +\end_layout + +\begin_layout Standard +\begin_inset Tabular + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +section +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +variable +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +value +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +required +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +comment +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[setup] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +max_concurrent_jobs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +if variable is not defined no limit on jobs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +executable +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +full path to megatest binary +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Use only if necessary, megatest will extract the location from where it + used to launch and add append that to the PATH for test runs. +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +runsdir +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +full path to where the link tree to all runs will be created +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +Because your runs may be spread out over several disk partitions a central + link tree is created to make finding all the runs easy. +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[fields] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +at least one +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[jobtools] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +launcher +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +command line used to launch jobs - the job command (megatest -execute) will + be appended to this +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +workhosts +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +list of hostnames to run jobs on NOT SUPPORTED RIGHT NOW +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +n/a +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[env-override] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +any string +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +These are set on the test launching machine, not the test running machine. + Typical usage is to control the host or run queue for launching tests. + These values will not be seen by the test when it runs. +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[disks] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +a valid path writable by the test launching process and by the test process +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +yes +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +The disk usage balancing algorithm is to choose the disk with the least + space for each test run. + +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Subsection +runconfigs.config file +\end_layout + +\begin_layout Standard +\begin_inset Tabular + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +section +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +variable +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +value +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +required? +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +comment +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[default] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +any +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +variables set in this section will be available for all runs, defining the + same variable in another section will override the value from the default + section +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[field1value/field2value...] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +string of letters, numbers and underscore +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +any +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +the values in this section will be set for any run where field1 is field1value, + field2 is field2value and fieldN is fieldNvalue. + +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Standard +Example: a test suite that checks that a piece of software works correctly + for different customer configurations and locations each of which is done + as a separate release regression run. + The fields, CUSTOMER and LOCATION were chosen. + The following runconfigs.config file would set some variables specific to + runs for megacorp in India and femtocorp in the Cook Islands and New Zealand: +\end_layout + +\begin_layout Code +# runconfigs.config +\end_layout + +\begin_layout Code +[default] +\end_layout + +\begin_layout Code +ENCRYTION true +\end_layout + +\begin_layout Code +# +\end_layout + +\begin_layout Code +[megacorp/india] +\end_layout + +\begin_layout Code +TESTPATH /nfs/testing/megacorp_runs +\end_layout + +\begin_layout Code +# +\end_layout + +\begin_layout Code +[femtocorp/cook_islands] +\end_layout + +\begin_layout Code +ENCRYTION false +\end_layout + +\begin_layout Code +TESTPATH /afs/kiatoa/testing/cook_islands +\end_layout + +\begin_layout Code +# +\end_layout + +\begin_layout Code +[femtocorp/new_zealand] +\end_layout + +\begin_layout Code +TESTPATH /afs/kiatao/testing/new_zealand +\end_layout + +\begin_layout Code +# +\end_layout + +\begin_layout Code +[megacorp/new_zealand] +\end_layout + +\begin_layout Code +TESTPATH /nfs/testing/megacorp_runs +\end_layout + +\begin_layout Paragraph* +Running megatest like this: +\end_layout + +\begin_layout Code +megatest :CUSTOMER megacorp :LOCATION new_zealand :runname week12_2011_run1 + -runall +\end_layout + +\begin_layout Paragraph* +Would set: +\end_layout + +\begin_layout Code +ENCRYPTION true +\end_layout + +\begin_layout Code +TESTPATH /nfs/testing/megacorp_runs +\end_layout + +\begin_layout Section +Writing tests +\end_layout + +\begin_layout Subsection +testconfig file +\end_layout + +\begin_layout Standard +\begin_inset Tabular + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +section +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +variable +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +value +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +required? +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +comments +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[setup] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +runscript +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +name of script to execute for this test +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +yes +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +The script must be executable and either provide the full path or put a + copy at the top of your test directory +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[requirements] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +waiton +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +list of valid test names +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +This test will not run until the named tests are state completed and status + PASS +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +[items] +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +any valid +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +list of values +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +no +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +The test will be repeated once for each item with the variable name set + to the value. + If there is more than one variable then the test will be run against all + unique combinations of the values +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Subsection +Command line +\end_layout + +\begin_layout Standard +\begin_inset Tabular + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +switch or param +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +parameter +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +purpose +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +comments +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-h +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +brief help +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-runall +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +run all tests +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-runtests +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +test1,test2,... +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +run one or more tests +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-step +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +stepname +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +record a step +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +requires :state and :status +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-test-status +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +record the test status +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +requires :state and :status +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-setlog +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +logfilename +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +set the logfile name for a test +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +path is assumed to be relative to the test run directory +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-set-toplog +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +logfilename +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +set the logfile name for the top test in an iterated test run +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +each sub test can have its own logfile set +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-m +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Quotes eld +\end_inset + +comment +\begin_inset Quotes erd +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +sets a comment for the step, test or run +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +:runname +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +:state +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +:status +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-list-runs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-testpatt +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-itempatt +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-showkeys +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-force +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-xterm +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-remove-runs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout + +\shape italic +Test helpers +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-runstep +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +-logpro +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\end_body +\end_document