Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -28,11 +28,11 @@
            ezsteps.scm lock-queue.scm sdb.scm rmt.scm api.scm		\
            subrun.scm portlogger.scm archive.scm env.scm		\
            diff-report.scm cgisetup/models/pgdb.scm
 
 # module source files
-MSRCFILES = 
+MSRCFILES = dbmod.scm servermod.scm apimod.scm commonmod.scm rmtmod.scm
 # ftail.scm rmtmod.scm commonmod.scm removed
 # MSRCFILES = ducttape-lib.scm pkts.scm stml2.scm cookie.scm mutils.scm	\
 #             mtargs.scm commonmod.scm dbmod.scm adjutant.scm ulex.scm	\
 #             rmtmod.scm apimod.scm
 
@@ -45,22 +45,21 @@
 
 MOFILES = $(addprefix mofiles/,$(MSRCFILES:%.scm=%.o))
 # compiled import files
 MOIMPFILES = $(MSRCFILES:%.scm=%.import.o)
 
-%.import.o : %.import.scm
+%.import.o : %.import.scm mofiles/%.o
 	csc $(CSCOPTS) -unit $*.import -c $*.import.scm -o $*.import.o
 
 # I'm not sure the cp is a good idea, changed a lot of things and it may not have been necessary...
 # mofiles/%.o  %.import.scm : %.scm megatest-fossil-hash.scm
 #	@[ -e mofiles ] || mkdir -p mofiles
 #	csc $(CSCOPTS) -I mofiles -I $* -J -c $< -o $*.o
 #	cp $*.o mofiles/$*.o
 #	@touch $*.import.scm # ensure it is touched after the .o is made
 
-mofiles/%.o : %.scm
-	mkdir -p mofiles
+mofiles/%.o %.import.scm : %.scm
 	csc $(CSCOPTS) -J -c $< -o mofiles/$*.o
 
 ADTLSCR=mt_laststep mt_runstep mt_ezstep
 HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR))
 DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR))
@@ -127,11 +126,11 @@
         ezsteps.o
 
 #        mofiles/rmtmod.o \
 #        mofiles/commonmod.o \
 
-tcmt : $(TCMTOBJS) tcmt.scm megatest-version.scm
+tcmt : $(TCMTOBJS) tcmt.scm megatest-version.scm $(MOIMPFILES) $(MOFILES)
 	csc $(CSCOPTS) $(TCMTOBJS) $(MOFILES) $(MOIMPFILES) tcmt.scm -o tcmt
 
 # install documentation to $(PREFIX)/docs
 # DOES NOT REBUILD DOCS
 #
@@ -147,15 +146,17 @@
 
 $(PREFIX)/share/db/mt-pg.sql : mt-pg.sql
 	mkdir -p $(PREFIX)/share/db
 	$(INSTALL) mt-pg.sql $(PREFIX)/share/db/mt-pg.sql
 
-# Special dependencies for the includes
+# Special dependencies for the module includes
 $(MOFILE) $(MOIMPFILES) : megatest-fossil-hash.scm
-
-# common.o : mofiles/commonmod.o megatest-fossil-hash.scm
-
+megatest.o : $(MOIMPFILES)
+mofiles/commonmod.o : megatest-fossil-hash.scm
+mofiles/dbmod.o mofiles/servermod.o mofiles/apimod.o : mofiles/commonmod.o
+mofiles/rmtmod.o : mofiles/apimod.o
+common.o : mofiles/commonmod.o
 # commonmod.o dashboard.o megatest.o tcmt.o apimod.o : megatest-fossil-hash.scm
 
 tests.o db.o launch.o runs.o dashboard-tests.o				\
 dashboard-context-menu.o dashboard-guimonitor.o dashboard-main.o	\
 monitor.o dashboard.o archive.o megatest.o : db_records.scm megatest-fossil-hash.scm

Index: apimod.scm
==================================================================
--- apimod.scm
+++ apimod.scm
@@ -18,19 +18,19 @@
 
 ;;======================================================================
 
 (declare (unit apimod))
 (declare (uses commonmod))
-(declare (uses ulex))
+;; (declare (uses ulex))
 
 (module apimod
 	*
 	
 (import scheme chicken data-structures extras)
 (import (prefix sqlite3 sqlite3:) posix typed-records srfi-18)
 (import commonmod)
-(import (prefix ulex ulex:))
+;; (import (prefix ulex ulex:))
 
 
 (define (api:execute-requests params)
   #f)
 

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -803,10 +803,16 @@
     (8 "DEAD")
     (9 "FAIL")
     (10 "PREQ_FAIL")
     (11 "PREQ_DISCARDED")
     (12 "ABORT")))
+
+(define (common:status>? s1 s2)
+  (let* ((munged (map (lambda (x) `(,(cadr x) . ,(car x))) *common:std-statuses*))
+	 (v1     (alist-ref s1 munged equal?))
+	 (v2     (alist-ref s2 munged equal?)))
+    (> v1 v2)))
 
 (define *common:ended-states*       ;; states which indicate the test is stopped and will not proceed
   '("COMPLETED" "ARCHIVED" "KILLED" "KILLREQ" "STUCK" "INCOMPLETE" ))
 
 (define *common:badly-ended-states* ;; these roll up as CHECK, i.e. results need to be checked

Index: docs/manual/Makefile
==================================================================
--- docs/manual/Makefile
+++ docs/manual/Makefile
@@ -34,13 +34,16 @@
 
 megatest_manual.html : megatest_manual.txt  *.txt installation.txt *png
 	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
 #	dos2unix megatest_manual.html
 
-megatest_manual.pdf : megatest_manual.txt *.txt *png
+megatest_manual.pdf : megatest_manual.txt *.txt *png *.ps
 	a2x -a toc -f pdf megatest_manual.txt
 
+server.pdf : server.dot
+	dot -Tpdf server.dot > server.pdf
+
 server.ps : server.dot
 	dot -Tps server.dot > server.ps
 
 client.ps : client.dot
 	dot -Tps client.dot > client.ps

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="generator" content="AsciiDoc 8.6.7">
+<meta name="generator" content="AsciiDoc 8.6.10">
 <title>The Megatest Users Manual</title>
 <style type="text/css">
 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
 
 /* Default font. */
@@ -84,14 +84,20 @@
   margin-top: 0;
 }
 ul > li     { color: #aaa; }
 ul > li > * { color: black; }
 
-pre {
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
   padding: 0;
   margin: 0;
 }
+pre {
+  white-space: pre-wrap;
+}
 
 #author {
   color: #527bbd;
   font-weight: bold;
   font-size: 1.1em;
@@ -216,11 +222,11 @@
   border-left: 3px solid #dddddd;
   padding-left: 0.5em;
 }
 
 div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; }
+span.image img { border-style: none; vertical-align: text-bottom; }
 a.image:visited { color: white; }
 
 dl {
   margin-top: 0.8em;
   margin-bottom: 0.8em;
@@ -412,16 +418,10 @@
 /*
  * xhtml11 specific
  *
  * */
 
-tt {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 div.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 div.tableblock > table {
@@ -451,16 +451,10 @@
 /*
  * html5 specific
  *
  * */
 
-.monospaced {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-}
-
 table.tableblock {
   margin-top: 1.0em;
   margin-bottom: 1.5em;
 }
 thead, p.tableblock.header {
@@ -536,10 +530,12 @@
 }
 
 @media print {
   body.manpage div#toc { display: none; }
 }
+
+
 @media screen {
   body {
     max-width: 50em; /* approximately 80 characters wide */
     margin-left: 16em;
   }
@@ -2340,11 +2336,11 @@
 <div class="sect2">
 <h3 id="_trim_trailing_spaces">Trim trailing spaces</h3>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">As of Megatest version v1.6548 trim-trailing-spaces defaults to yes.</td>
 </tr></table>
 </div>
 <div class="listingblock">
@@ -2642,11 +2638,11 @@
 waiton test1 test2</pre>
 </div></div>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">Dynamic waiton lists must be capable of being calculated at the
 beginning of a run. This is because Megatest walks the tree of waitons
 to create the list of tests to execute.</td>
 </tr></table>
@@ -3147,11 +3143,11 @@
 COMPLETED/ xterm -e bash -s --</pre>
 </div></div>
 <div class="admonitionblock">
 <table><tr>
 <td class="icon">
-<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
+<img src="/usr/images/icons/note.png" alt="Note">
 </td>
 <td class="content">There is a trailing space after the double-dash</td>
 </tr></table>
 </div>
 <div class="paragraph"><p>There are a number of environment variables available to the trigger script
@@ -3484,10 +3480,11 @@
 </div>
 <div id="footnotes"><hr></div>
 <div id="footer">
 <div id="footer-text">
 Version 1.5<br>
-Last updated 2020-09-08 08:39:29 PDT
+Last updated
+ 2021-01-25 11:33:16 MST
 </div>
 </div>
 </body>
 </html>

Index: docs/manual/server.dot
==================================================================
--- docs/manual/server.dot
+++ docs/manual/server.dot
@@ -14,65 +14,130 @@
 // 
 //     You should have received a copy of the GNU General Public License
 //     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
 digraph G {
 
-    subgraph cluster_1 {
-        node [style=filled,shape=box];
-
-	check_available_queue       -> remove_entries_over_10s_old;
-	remove_entries_over_10s_old -> set_available [label="num_avail < 3"];
-	remove_entries_over_10s_old -> exit [label="num_avail > 2"];
-
-	set_available               -> delay_2s;
-	delay_2s          -> check_place_in_queue;
-
-	check_place_in_queue        -> "http:transport-launch" [label="at head"];
-	check_place_in_queue        -> exit [label="not at head"];
-
-	"client:login"              -> "server:shutdown" [label="login failed"];
-	"server:shutdown"           -> exit;	
-
-	subgraph cluster_2 {
-		"http:transport-launch"       -> "http:transport-run";
-		"http:transport-launch"       -> "http:transport-keep-running";
-
-		"http:transport-keep-running" -> "tests running?";
-		"tests running?"              -> "client:login" [label=yes];
-		"tests running?"              -> "server:shutdown" [label=no];
-		"client:login"                -> delay_5s [label="login ok"];
-		delay_5s                      -> "http:transport-keep-running";
-	}
-
-	// start_server -> "server_running?";
-	// "server_running?" -> set_available [label="no"];
-	// "server_running?" -> delay_2s [label="yes"];
-	// delay_2s -> "still_running?";
-	// "still_running?" -> ping_server [label=yes];
-	// "still_running?" -> set_available [label=no];
-	// ping_server -> exit [label=alive];
-	// ping_server -> remove_server_record [label=dead];
-	// remove_server_record -> set_available;
-	// set_available -> avail_delay [label="delay 3s"];
-	// avail_delay -> "first_in_queue?";
-	// 
-	// "first_in_queue?" -> set_running [label=yes];
-	// set_running -> get_next_port -> handle_requests;
-	// "first_in_queue?" -> "dead_entry_in_queue?" [label=no];
-	// "dead_entry_in_queue?" -> "server_running?" [label=no];
-	// "dead_entry_in_queue?" -> "remove_dead_entries" [label=yes];
-	// remove_dead_entries -> "server_running?";
-	// 
-	// handle_requests -> start_shutdown [label="no traffic\nno running tests"];
-	// handle_requests -> shutdown_request;
-	// start_shutdown -> shutdown_delay;
-	// shutdown_request -> shutdown_delay;
-	// shutdown_delay -> exit;
-	
-        label = "server:launch";
-        color=brown;
-    }
+// connecting to server
+subgraph cluster_1 {
+  label="Connect";
+  
+  node [style=filled,shape=box];
+  dotserver [label="Have .server file?"];
+  connect [label="Connect to server"];
+  // startserver [label="Start Server"];
+  connected [label="Connection Successful"];
+  anylive [label="Any live servers?"];
+  askstart [label="Ask existing server to start a new server"];
+  anydboard [label="Any live dashboards"];
+  askdboard [label="Ask dashboard to start server"];
+  serverlaunch [label="User server launch to start server process"];
+  wait5sec [label="Wait 5 seconds"];
+  
+  dotserver -> connect [label="yes"];
+  connect -> connected [label="yes"];
+  connect -> anylive [label="no"];
+
+  // startserver -> anylive;
+  anylive -> askstart [label="yes"];
+  anylive -> anydboard [label="no"];
+  anydboard -> askdboard [label="yes"];
+  anydboard -> serverlaunch [label="no"];
+  serverlaunch -> wait5sec;
+  askdboard -> wait5sec;
+  askstart -> wait5sec;
+  wait5sec -> dotserver;
+
+}
+
+// server bootstrap
+subgraph cluster_2 {
+  label="Startup";
+  node [style=filled,shape=box];
+  getlock [label="Get file lock"];
+  starthttpcpdb [label="Start http server/copy db to /tmp"];
+  createsrvfile [label="Create .nnn.server file containing host:port"];
+  releaselock [label="Release file lock"];
+  chksrv [label="Check if server already exists"];
+  exitstartup [label="exit/stop"];
+  sleep1 [label="Sleep few seconds"];
+  
+  getlock -> starthttpcpdb [label="yes"];
+  getlock -> chksrv [label="no"];
+  starthttpcpdb -> createsrvfile -> releaselock;
+  chksrv -> exitstartup [label="yes"];
+  chksrv -> sleep1 [label="no"];
+  sleep1 -> getlock;
+}
+
+// shutting down server
+subgraph cluster_3 {
+  label="Shutdown";
+  node [style=filled,shape=box];
+  shutdown [label="Start Shutdown"];
+  rejectmode [label="Put http server into reject requests mode"];
+  syncback [label="Sync db back"];
+  removesrvfile [label="Remove server file containing host:port"];
+  exit [label="Exit process"];
+
+  shutdown -> rejectmode -> syncback -> removesrvfile -> exit;
+}
+
+//     subgraph cluster_1 {
+//         node [style=filled,shape=box];
+// 
+// 	check_available_queue       -> remove_entries_over_10s_old;
+// 	remove_entries_over_10s_old -> set_available [label="num_avail < 3"];
+// 	remove_entries_over_10s_old -> exit [label="num_avail > 2"];
+// 
+// 	set_available               -> delay_2s;
+// 	delay_2s          -> check_place_in_queue;
+// 
+// 	check_place_in_queue        -> "http:transport-launch" [label="at head"];
+// 	check_place_in_queue        -> exit [label="not at head"];
+// 
+// 	"client:login"              -> "server:shutdown" [label="login failed"];
+// 	"server:shutdown"           -> exit;	
+// 
+// 	subgraph cluster_2 {
+// 		"http:transport-launch"       -> "http:transport-run";
+// 		"http:transport-launch"       -> "http:transport-keep-running";
+// 
+// 		"http:transport-keep-running" -> "tests running?";
+// 		"tests running?"              -> "client:login" [label=yes];
+// 		"tests running?"              -> "server:shutdown" [label=no];
+// 		"client:login"                -> delay_5s [label="login ok"];
+// 		delay_5s                      -> "http:transport-keep-running";
+// 	}
+// 
+// 	// start_server -> "server_running?";
+// 	// "server_running?" -> set_available [label="no"];
+// 	// "server_running?" -> delay_2s [label="yes"];
+// 	// delay_2s -> "still_running?";
+// 	// "still_running?" -> ping_server [label=yes];
+// 	// "still_running?" -> set_available [label=no];
+// 	// ping_server -> exit [label=alive];
+// 	// ping_server -> remove_server_record [label=dead];
+// 	// remove_server_record -> set_available;
+// 	// set_available -> avail_delay [label="delay 3s"];
+// 	// avail_delay -> "first_in_queue?";
+// 	// 
+// 	// "first_in_queue?" -> set_running [label=yes];
+// 	// set_running -> get_next_port -> handle_requests;
+// 	// "first_in_queue?" -> "dead_entry_in_queue?" [label=no];
+// 	// "dead_entry_in_queue?" -> "server_running?" [label=no];
+// 	// "dead_entry_in_queue?" -> "remove_dead_entries" [label=yes];
+// 	// remove_dead_entries -> "server_running?";
+// 	// 
+// 	// handle_requests -> start_shutdown [label="no traffic\nno running tests"];
+// 	// handle_requests -> shutdown_request;
+// 	// start_shutdown -> shutdown_delay;
+// 	// shutdown_request -> shutdown_delay;
+// 	// shutdown_delay -> exit;
+// 	
+//         label = "server:launch";
+//         color=brown;
+//     }
 
 //     client_start_server -> start_server;
 //     handle_requests -> read_write;
 //     read_write -> handle_requests;
 }

Index: docs/manual/server.png
==================================================================
--- docs/manual/server.png
+++ docs/manual/server.png
cannot compute difference between binary files

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -41,10 +41,27 @@
 (declare (uses tasks)) ;; only used for debugging.
 (declare (uses env))
 (declare (uses diff-report))
 ;; (declare (uses ftail))
 ;; (import ftail)
+
+;; Needed for repl even if not used here in megatest.scm
+(declare (uses commonmod))
+(import commonmod)
+(declare (uses commonmod.import))
+
+(declare (uses dbmod))
+(import dbmod)
+(declare (uses dbmod.import))
+
+(declare (uses servermod))
+(import servermod)
+(declare (uses servermod.import))
+
+(declare (uses apimod))
+(import apimod)
+(declare (uses apimod.import))
 
 (define *db* #f) ;; this is only for the repl, do not use in general!!!!
 
 (include "common_records.scm")
 (include "key_records.scm")

ADDED   mofiles/README
Index: mofiles/README
==================================================================
--- /dev/null
+++ mofiles/README
@@ -0,0 +1,1 @@
+Built modules go here - an attempt to minimize clutter.

Index: rmtmod.scm
==================================================================
--- rmtmod.scm
+++ rmtmod.scm
@@ -20,11 +20,11 @@
 
 (declare (unit rmtmod))
 (declare (uses commonmod))
 (declare (uses apimod))
 ;; (declare (uses apimod.import))
-(declare (uses ulex))
+;; (declare (uses ulex))
 
 ;; (include "ulex/ulex.scm")
 
 (module rmtmod
 	*
@@ -31,11 +31,11 @@
 	
 (import scheme chicken data-structures extras)
 (import (prefix sqlite3 sqlite3:) posix typed-records srfi-18)
 (import (prefix commonmod cmod:))
 (import apimod)
-(import (prefix ulex ulex:))
+;; (import (prefix ulex ulex:))
 
 (defstruct alldat
   (areapath #f)
   (ulexdat  #f)
   )

ADDED   servermod.scm
Index: servermod.scm
==================================================================
--- /dev/null
+++ servermod.scm
@@ -0,0 +1,39 @@
+;;======================================================================
+;; Copyright 2017, Matthew Welland.
+;; 
+;; This file is part of Megatest.
+;; 
+;;     Megatest is free software: you can redistribute it and/or modify
+;;     it under the terms of the GNU General Public License as published by
+;;     the Free Software Foundation, either version 3 of the License, or
+;;     (at your option) any later version.
+;; 
+;;     Megatest is distributed in the hope that it will be useful,
+;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;     GNU General Public License for more details.
+;; 
+;;     You should have received a copy of the GNU General Public License
+;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
+
+;;======================================================================
+
+(declare (unit servermod))
+
+(module servermod
+	*
+	
+(import scheme chicken data-structures extras)
+(import (prefix sqlite3 sqlite3:) posix typed-records srfi-18)
+
+(define (just-testing)
+  (print "JUST TESTING"))
+
+;; (define (debug:print . params) #f)
+;; (define (debug:print-info . params) #f)
+;; 
+;; (define (set-functions dbgp dbgpinfo)
+;;   (set! debug:print dbgp)
+;;   (set! debug:print-info dbgpinfo))
+
+)