Megatest

Changes On Branch 65618b033ee6cf58
Login

Changes In Branch v1.90-proper-interface-lists Through [65618b033e] Excluding Merge-Ins

This is equivalent to a diff from cee15a9d94 to 65618b033e

2024-02-13
15:17
dashboard runs check-in: 947952bcfb user: mrwellan tags: v1.90-proper-interface-lists
12:40
megatest -repl and -h work check-in: 65618b033e user: mrwellan tags: v1.90-proper-interface-lists
07:27
wip check-in: b7ce99fe0a user: mrwellan tags: v1.90-proper-interface-lists
2024-02-11
19:49
convert to use proper interface lists check-in: bbac9c3682 user: matt tags: v1.90-proper-interface-lists
18:19
Adding uses of .import files back Leaf check-in: cee15a9d94 user: matt tags: v1.90-fix-modes
16:41
Moved remainder of configf into configfmod check-in: c2d750aad9 user: matt tags: v1.90-fix-modes

Modified Makefile from [28690e5571] to [3320918a2c].

24
25
26
27
28
29
30
31

32
33
34


35
36

37
38
39
40
41
42
43
44
45
46
47
48


49
50
51
52
53
54
55
56
57
58
59
60

61
62
63
64
65
66
67
24
25
26
27
28
29
30

31
32


33
34
35

36

37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68







-
+

-
-
+
+

-
+
-










-
+
+












+







	fossil timeline -n 350 -t ci -F "%h,%a,%b,%t,\"%c\"" > recent-commits.csv


SHELL=/bin/bash
PREFIX=$(PWD)
CSCOPTS=
INSTALL=install
SRCFILES = common.scm launch.scm runconfig.scm	\
SRCFILES = launch.scm runconfig.scm	\
           server.scm configf.scm keys.scm		\
           process.scm runs.scm genexample.scm	\
           tdb.scm mt.scm	\
           process.scm runs.scm \
           mt.scm	\
           ezsteps.scm api.scm		\
           subrun.scm archive.scm env.scm		\
           subrun.scm archive.scm env.scm
           diff-report.scm

# cgisetup/models/pgdb.scm

# module source files
MSRCFILES = dbfile.scm debugprint.scm mtargs.scm commonmod.scm dbmod.scm \
            tcp-transportmod.scm rmtmod.scm portlogger.scm apimod.scm \
	    configfmod.scm processmod.scm servermod.scm megatestmod.scm \
	    stml2.scm fsmod.scm cpumod.scm mtmod.scm odsmod.scm \
            pkts.scm testsmod.scm pgdb.scm cookie.scm launchmod.scm \
            subrunmod.scm runsmod.scm tasksmod.scm archivemod.scm \
            ezstepsmod.scm
            ezstepsmod.scm mtbody.scm envmod.scm genexample.scm mutils.scm \
            diff-report.scm tdb.scm

transport-mode.scm : transport-mode.scm.template
	cp transport-mode.scm.template transport-mode.scm

dashboard-transport-mode.scm : dashboard-transport-mode.scm.template
	cp dashboard-transport-mode.scm.template dashboard-transport-mode.scm

mtest : transport-mode.scm
dboard : dashboard-transport-mode.scm

# dbmod.import.o is just a hack here

mofiles/mtbody.o     : mofiles/launchmod.o readline-fix.scm mofiles/envmod.o mofiles/genexample.o mofiles/mutils.o mofiles/diff-report.o mofiles/tdb.o
process.o            : mofiles/processmod.o
mofiles/configfmod.o : mofiles/processmod.o
mofiles/processmod.o : mofiles/commonmod.o
mofiles/servermod.o  : mofiles/commonmod.o
mofiles/rmtmod.o     : mofiles/mtmod.o mofiles/apimod.o
mofiles/dbmod.o      : mofiles/mtmod.o
# mofiles/mtmod.o      : mofiles/tcp-transportmod.o

Modified api.scm from [55795eb2b8] to [185cc0c63f].

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
38
39
40
41
42
43
44




























































































-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
     srfi-18
     posix
     matchable
     s11n
     typed-records)


;; QUEUE METHOD

(define (api:tcp-dispatch-request-make-handler-new dbstruct) ;; cmd run-id params)
  (api:tcp-dispatch-request-make-handler-core dbstruct api:dispatch-request))


;; indat is (cmd run-id params meta)
;;
;; WARNING: Do not print anything in the lambda of this function as it
;;          reads/writes to current in/out port
;;
(define (api:tcp-dispatch-request-make-handler-old dbstruct) ;; cmd run-id params)
  (assert *toppath* "FATAL: api:tcp-dispatch-request-make-handler called but *toppath* not set.")
  (if (not *server-signature*)
      (set! *server-signature* (tt:mk-signature *toppath*)))
  (lambda (indat)
    (api:register-thread (current-thread))
    (let* ((result 
	    (let* ((numthreads (api:get-count-threads-alive))
		   (delay-wait (if (> numthreads 10)
				   (- numthreads 10)
				   0))
		   (normal-proc (lambda (cmd run-id params)
				  (case cmd
				    ((ping) *server-signature*)
				    (else
				     (api:dispatch-request dbstruct cmd run-id params))))))
	      (set! *api-process-request-count* numthreads)
	      (set! *db-last-access* (current-seconds))
;; 	      (if (not (eq? numthreads numthreads))
;; 	      (begin
;; 	      (api:remove-dead-or-terminated)
;; 	      (let ((threads-now (api:get-count-threads-alive)))
;; 	      (debug:print 0 *default-log-port* "WARNING: numthreads="numthreads", numthreads="numthreads", remaining="threads-now)
;; 	      (set! numthreads threads-now))))
	      (match indat
		     ((cmd run-id params meta)
		      (let* ((start-t (current-milliseconds))
			     (db-ok  (let* ((dbfname (dbmod:run-id->dbfname run-id))
					    (ok      (equal? dbfname (dbr:dbstruct-dbfname dbstruct))))
				       (case cmd
					 ((ping) #t) ;; we are fine
					 (else
					  (assert ok "FATAL: database file and run-id not aligned.")))))
			     (ttdat   *server-info*)
			     (server-state (tt-state ttdat))
			     (maxthreads   20) ;; make this a parameter?
			     (status  (cond
				       ((and (> numthreads maxthreads)
					     (> (random 100) 70)) ;; allow a 30% probability to go through so we can figure out what is going wrong in main.db server.
					'busy)
				       ;; ((> numthreads 5) 'loaded) ;; this gets transmitted to the client which calls tt:backoff-incr to slow stuff down.
				       (else 'ok)))
			     (errmsg  (case status
					((busy)   (conc "Server overloaded, "numthreads" threads in flight"))
					((loaded) (conc "Server loaded, "numthreads" threads in flight"))
					(else     #f)))
			     (result  (case status
					((busy)
					 (if (eq? cmd 'ping)
					     (normal-proc cmd run-id params)
					     ;; numthreads must be greater than 5 for busy
					     (* 0.1 (- numthreads maxthreads)) ;; was 15 - return a number for the remote to delay
					     )) ;; (- numthreads 29)) ;; call back in as many seconds
					((loaded)
					 (normal-proc cmd run-id params))
					(else
					 (normal-proc cmd run-id params))))
			     (meta   (case cmd
				       ((ping) `((sstate . ,server-state)))
				       (else   `((wait . ,delay-wait)))))
			     (payload (list status errmsg result meta)))
			;; (cmd run-id params meta)
			(db:add-stats cmd run-id params (- (current-milliseconds) start-t))
			payload))
		     (else
		      (assert #f "FATAL: failed to deserialize indat "indat))))))
      ;; (set! *api-process-request-count* (- *api-process-request-count* 1))
      ;; (serialize payload)
     
      (api:unregister-thread (current-thread))
      result)))

(define api:tcp-dispatch-request-make-handler api:tcp-dispatch-request-make-handler-new) ;; choose -old or -new

Modified apimod.scm from [c5b4d2905e] to [4051ce6944].

23
24
25
26
27
28
29

30









31
32
33
34
35
36
37
23
24
25
26
27
28
29
30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46







+
-
+
+
+
+
+
+
+
+
+







(declare (uses debugprint))
(declare (uses dbmod))
(declare (uses dbfile))
(declare (uses tcp-transportmod))
(declare (uses megatestmod))

(module apimod
	(
	*
	 *server-signature*
	 api:tcp-dispatch-request-make-handler-core
	 api:register-thread
	 api:unregister-thread
	 api:get-count-threads-alive
	 api:print-db-stats
	 api:queue-processor
	 api:dispatch-request
	 )
	
(import scheme chicken data-structures extras)
(import (prefix sqlite3 sqlite3:) posix matchable typed-records srfi-1 srfi-18 srfi-69 )
(import commonmod)
(import debugprint)
(import dbmod)
(import dbfile)

Modified archive.scm from [3972290090] to [e534969391].

17
18
19
20
21
22
23
24

25
26
27
28
29
30
31
17
18
19
20
21
22
23

24
25
26
27
28
29
30
31







-
+







;;

;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')

(declare (unit archive))
(declare (uses debugprint))
(declare (uses mtargs))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses rmtmod))

(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69
     format md5 message-digest srfi-18)

Modified archivemod.scm from [ddced4be70] to [2d74ee0e1f].

35
36
37
38
39
40
41







42


43
44
45
46
47
48
49
35
36
37
38
39
40
41
42
43
44
45
46
47
48

49
50
51
52
53
54
55
56
57







+
+
+
+
+
+
+
-
+
+







(declare (uses mtmod))
(declare (uses dbmod))
(declare (uses dbfile))

(use srfi-69)

(module archivemod
	(
	 archive:get-archive-disks
	 archive:allocate-new-archive-block
	 archive:get-timestamp-dir
	 archive:megatest-db
	 archive:bup-get-data
	 archive:restore-db
	*

	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
119
120
121
122
123
124
125
126

127
128
129
130
131
132
133
127
128
129
130
131
132
133

134
135
136
137
138
139
140
141







-
+







	srfi-13
	srfi-18
	srfi-69
	typed-records
	z3
	)
	
(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")

;;======================================================================
;; 
;;======================================================================

;; ;; NOT CURRENTLY USED

Added attic/codescanlib.scm version [6e625610ce].

















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
;;  Copyright 2006-2017, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

;; gotta compile with csc, doesn't work with csi -s for whatever reason

(use srfi-69)
(use matchable)
(use utils)
(use ports)
(use extras)
(use srfi-1)
(use posix)
(use srfi-12)

;; turn scheme file to a list of sexps, sexps of interest will be in the form of (define (<procname> <args>) <body> )
(define (load-scm-file scm-file)
  ;;(print "load "scm-file)
  (handle-exceptions
   exn
   '()
   (with-input-from-string
       (conc "("
             (with-input-from-file scm-file read-all)
             ")" )
     read)))

;; extract a list of procname, filename, args and body of procedures defined in filename, input from load-scm-file
;;   -- be advised:
;;      * this may be fooled by macros, since this code does not take them into account.
;;      * this code does only checks for form (define (<procname> ... ) <body>)
;;           so it excludes from reckoning
;;               - generated functions, as in things like foo-set! from defstructs,
;;               - define-inline, (
;;               - define procname (lambda ..
;;               - etc...
(define (get-toplevel-procs+file+args+body filename)
  (let* ((scm-tree (load-scm-file filename))
         (procs
          (filter identity
                  (map
                   (match-lambda 
                    [('define ('uses args ...) body ...) #f] ;; filter out (define (uses ...
                    [('define ('unit args ...) body ...) #f] ;; filter out (define (unit ...
                    [('define ('prefix args ...) body ...) #f] ;; filter out (define (prefix ...
                    [('define (defname args ...) body ...) ;; match (define (procname <args>) <body>)
                     (if (atom? defname) ;; filter out things we dont understand (procname is a list, what??)
                         (list defname filename args body)
                         #f)]
                    [else #f] ) scm-tree))))
    procs))


;; given a sexp, return a flat list of atoms in that sexp
(define (get-atoms-in-body body)
  (cond
   ((null? body) '())
   ((atom? body) (list body))
   (else
    (apply append (map get-atoms-in-body body)))))

;;  given a file, return a list of procname, file, list of atoms in said procname
(define (get-procs+file+atoms file)
  (let* ((toplevel-proc-items (get-toplevel-procs+file+args+body file))
         (res
          (map
           (lambda (item)
             (let* ((proc (car item))
                    (file (cadr item))
                    (args (caddr item))
                    (body (cadddr item))
                    (atoms (append (get-atoms-in-body args) (get-atoms-in-body body))))
               (list proc file atoms)))
           toplevel-proc-items)))
    res))

;; uniquify a list of atoms 
(define (unique-atoms lst)
  (let loop ((lst (flatten lst)) (res '()))
    (if (null? lst)
        (reverse res)
        (let ((c (car lst)))
          (loop (cdr lst) (if (member c res) res (cons c res)))))))

;; given a list of procname, filename, list of procs called from procname, cross reference and reverse
;; returning alist mapping procname to procname that calls said procname
(define (get-callers-alist all-procs+file+calls)
  (let* ((all-procs (map car all-procs+file+calls))
         (caller-ht (make-hash-table))) 
    ;; let's cross reference with a hash table
    (for-each (lambda (proc) (hash-table-set! caller-ht proc '())) all-procs)
    (for-each (lambda (item)
               (let* ((proc (car item))
                      (file (cadr item))
                      (calls (caddr item)))
                 (for-each (lambda (callee)
                             (hash-table-set! caller-ht callee
                                              (cons proc
                                                    (hash-table-ref caller-ht callee))))
                           calls)))
              all-procs+file+calls)
    (map (lambda (x)
           (let ((k (car x))
                 (r (unique-atoms (cdr x))))
             (cons k r)))                    
         (hash-table->alist caller-ht))))

;; create a handy cross-reference of callees to callers in the form of an alist.
(define (get-xref all-scm-files)
  (let* ((all-procs+file+atoms
          (apply append (map get-procs+file+atoms all-scm-files)))
         (all-procs (map car all-procs+file+atoms))
         (all-procs+file+calls  ; proc calls things in calls list
          (map (lambda (item)
                 (let* ((proc (car item))
                        (file (cadr item))
                        (atoms (caddr item))
                        (calls
                         (filter identity
                                 (map
                                  (lambda (x)
                                    (if (and ;; (not (equal? x proc))  ;; uncomment to prevent listing self
                                         (member x all-procs))
                                        x
                                        #f))
                                  atoms))))
                   (list proc file calls)))
               all-procs+file+atoms))
         (callers (get-callers-alist all-procs+file+calls))) 
    callers))

Deleted codescanlib.scm version [6e625610ce].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
















































































































































-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
;;  Copyright 2006-2017, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

;; gotta compile with csc, doesn't work with csi -s for whatever reason

(use srfi-69)
(use matchable)
(use utils)
(use ports)
(use extras)
(use srfi-1)
(use posix)
(use srfi-12)

;; turn scheme file to a list of sexps, sexps of interest will be in the form of (define (<procname> <args>) <body> )
(define (load-scm-file scm-file)
  ;;(print "load "scm-file)
  (handle-exceptions
   exn
   '()
   (with-input-from-string
       (conc "("
             (with-input-from-file scm-file read-all)
             ")" )
     read)))

;; extract a list of procname, filename, args and body of procedures defined in filename, input from load-scm-file
;;   -- be advised:
;;      * this may be fooled by macros, since this code does not take them into account.
;;      * this code does only checks for form (define (<procname> ... ) <body>)
;;           so it excludes from reckoning
;;               - generated functions, as in things like foo-set! from defstructs,
;;               - define-inline, (
;;               - define procname (lambda ..
;;               - etc...
(define (get-toplevel-procs+file+args+body filename)
  (let* ((scm-tree (load-scm-file filename))
         (procs
          (filter identity
                  (map
                   (match-lambda 
                    [('define ('uses args ...) body ...) #f] ;; filter out (define (uses ...
                    [('define ('unit args ...) body ...) #f] ;; filter out (define (unit ...
                    [('define ('prefix args ...) body ...) #f] ;; filter out (define (prefix ...
                    [('define (defname args ...) body ...) ;; match (define (procname <args>) <body>)
                     (if (atom? defname) ;; filter out things we dont understand (procname is a list, what??)
                         (list defname filename args body)
                         #f)]
                    [else #f] ) scm-tree))))
    procs))


;; given a sexp, return a flat list of atoms in that sexp
(define (get-atoms-in-body body)
  (cond
   ((null? body) '())
   ((atom? body) (list body))
   (else
    (apply append (map get-atoms-in-body body)))))

;;  given a file, return a list of procname, file, list of atoms in said procname
(define (get-procs+file+atoms file)
  (let* ((toplevel-proc-items (get-toplevel-procs+file+args+body file))
         (res
          (map
           (lambda (item)
             (let* ((proc (car item))
                    (file (cadr item))
                    (args (caddr item))
                    (body (cadddr item))
                    (atoms (append (get-atoms-in-body args) (get-atoms-in-body body))))
               (list proc file atoms)))
           toplevel-proc-items)))
    res))

;; uniquify a list of atoms 
(define (unique-atoms lst)
  (let loop ((lst (flatten lst)) (res '()))
    (if (null? lst)
        (reverse res)
        (let ((c (car lst)))
          (loop (cdr lst) (if (member c res) res (cons c res)))))))

;; given a list of procname, filename, list of procs called from procname, cross reference and reverse
;; returning alist mapping procname to procname that calls said procname
(define (get-callers-alist all-procs+file+calls)
  (let* ((all-procs (map car all-procs+file+calls))
         (caller-ht (make-hash-table))) 
    ;; let's cross reference with a hash table
    (for-each (lambda (proc) (hash-table-set! caller-ht proc '())) all-procs)
    (for-each (lambda (item)
               (let* ((proc (car item))
                      (file (cadr item))
                      (calls (caddr item)))
                 (for-each (lambda (callee)
                             (hash-table-set! caller-ht callee
                                              (cons proc
                                                    (hash-table-ref caller-ht callee))))
                           calls)))
              all-procs+file+calls)
    (map (lambda (x)
           (let ((k (car x))
                 (r (unique-atoms (cdr x))))
             (cons k r)))                    
         (hash-table->alist caller-ht))))

;; create a handy cross-reference of callees to callers in the form of an alist.
(define (get-xref all-scm-files)
  (let* ((all-procs+file+atoms
          (apply append (map get-procs+file+atoms all-scm-files)))
         (all-procs (map car all-procs+file+atoms))
         (all-procs+file+calls  ; proc calls things in calls list
          (map (lambda (item)
                 (let* ((proc (car item))
                        (file (cadr item))
                        (atoms (caddr item))
                        (calls
                         (filter identity
                                 (map
                                  (lambda (x)
                                    (if (and ;; (not (equal? x proc))  ;; uncomment to prevent listing self
                                         (member x all-procs))
                                        x
                                        #f))
                                  atoms))))
                   (list proc file calls)))
               all-procs+file+atoms))
         (callers (get-callers-alist all-procs+file+calls))) 
    callers))

Deleted common.scm version [58be7ce4cf].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117





















































































































-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

(declare (unit common))
(declare (uses commonmod))
(declare (uses processmod))
(declare (uses configfmod))
(declare (uses rmtmod))
(declare (uses debugprint))
(declare (uses mtargs))
        

(use srfi-1 data-structures posix regex-case (prefix base64 base64:)
     format dot-locking csv-xml z3 udp ;; sql-de-lite
     hostinfo md5 message-digest typed-records directory-utils stack
     matchable regex posix (srfi 18) extras ;; tcp 
     (prefix nanomsg nmsg:)
     (prefix sqlite3 sqlite3:)
     pkts (prefix dbi dbi:)
     )
(use posix-extras pathname-expand files)


(import commonmod
	processmod
	debugprint
	configfmod
	rmtmod
	(prefix mtargs args:))

(include "common_records.scm")





;;======================================================================
;; (define *common:telemetry-log-state* 'startup)
;; (define *common:telemetry-log-socket* #f)
;; 
;; (define (common:telemetry-log-open)
;;   (if (eq? *common:telemetry-log-state* 'startup)
;;       (let* ((serverhost (configf:lookup *configdat* "telemetry" "host"))
;;              (serverport (configf:lookup-number *configdat* "telemetry" "port"))
;;              (user (or (get-environment-variable "USER") "unknown"))
;;              (host (or (get-environment-variable "HOST") "unknown")))
;;         (set! *common:telemetry-log-state*
;;               (handle-exceptions
;;                exn
;;                (begin
;;                  (debug:print-info 0 *default-log-port* "common-telemetry-log open udp port failure")
;;                  'broken)
;;                (if (and serverhost serverport user host)
;;                    (let* ((s (udp-open-socket)))
;;                      ;;(udp-bind! s #f 0)
;;                      (udp-connect! s serverhost serverport)
;;                      (set! *common:telemetry-log-socket* s)
;;                      'open)
;;                    'not-needed))))))
;;   
;; (define (common:telemetry-log event #!key (payload '()))
;;   (if (eq? *common:telemetry-log-state* 'startup)
;;       (common:telemetry-log-open))
;; 
;;   (if (eq? 'open *common:telemetry-log-state*)
;;       (handle-exceptions
;;        exn
;;        (begin
;;          (debug:print-info 0 *default-log-port* "common-telemetry-log comms failure ; disabled (no server?)")
;;          ;;(define *common:telemetry-log-state* 'broken-or-no-server-preclose)
;;          ;;(common:telemetry-log-close)
;;          (define *common:telemetry-log-state* 'broken-or-no-server)
;;          (set! *common:telemetry-log-socket* #f)
;;          )
;;        (if (and *common:telemetry-log-socket* event) ;; TODO - filter on event against telemetry.want-events
;;            (let* ((user (or (get-environment-variable "USER") "unknown"))
;;                   (host (or (get-environment-variable "HOST") "unknown"))
;;                   (start (conc "[megatest "event"]"))
;;                   (toppath (or *toppath* "/dev/null"))
;;                   (payload-serialized
;;                    (base64:base64-encode
;;                     (z3:encode-buffer
;;                      (with-output-to-string (lambda () (pp payload))))))
;;                   (msg     (conc user":"host":"start":"(current-process-id)":"(car (argv))":"
;;                                  toppath":"payload-serialized)))
;;              (udp-send *common:telemetry-log-socket* msg))))))
;;   
;; (define (common:telemetry-log-close)
;;   (when (or (member *common:telemetry-log-state* '(broken-or-no-server-preclose open)) *common:telemetry-log-socket*)
;;     (handle-exceptions
;;      exn
;;      (begin
;;        (define *common:telemetry-log-state* 'closed-fail)
;;        (debug:print-info 0 *default-log-port* "common-telemetry-log closure failure")
;;        )
;;      (begin
;;        (define *common:telemetry-log-state* 'closed)
;;        (udp-close-socket *common:telemetry-log-socket*)
;;        (set! *common:telemetry-log-socket* #f)))))

Modified commonmod.scm from [5100c657f0] to [fbed9e11d0].

48
49
50
51
52
53
54










55







56






































































































































































































































57
58
59
60
61
62
63
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309







+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







     typed-records
     z3)

(import stml2
	)

(module commonmod
	(
	 make-sparse-array
	 sparse-array-set!
	 sparse-array-ref
	 keys->valslots
	 item-list->path
	 common:human-time
	 number-of-processes-running
	 get-normalized-cpu-load
	 common:find-local-megatest
	*

	 common:get-intercept
	 common:get-num-cpus
	 common:get-cpu-load
	 common:hms-string->seconds
	 seconds->time-string
	 common:get-megatest-exe

	 megatest-version
	 common:get-toppath
	 common:generic-ssh
	 common:file-exists?
	 common:with-env-vars
	 common:nice-path
	 common:get-fields

	 common:get-normalized-cpu-load
	 common:unix-ping
	 common:get-normalized-cpu-load
	 common:raw-get-remote-host-load
	 common:to-alist

	 ;; globals
	 *common:badly-ended-states*
	 *common:dont-roll-up-states*
	 *common:ended-states*
	 *common:not-started-ok-statuses*
	 *common:running-states*
	 *common:std-states*
	 *common:std-statuses*
	 *common:well-ended-states*
	 *configdat*
	 *configinfo*
	 *db-access-allowed*
	 *db-api-call-time*
	 *db-cache-path*
	 *db-keys*
	 *default-area-tag*
	 *host-loads*
	 *keyvals*
	 *logged-in-clients*
	 *my-client-signature*
	 *on-exit-procs*
	 *pkts-info*
	 *pre-reqs-met-cache*
	 *runremote*
	 *server-id*
	 *server-info*
	 *target*
	 *task-db*
	 *testconfigs*
	 *time-to-exit*
	 *toppath*
	 *transport-type*
	 
	 any->number-if-possible

	 common:special-sort
	 keys:target-set-args

	 getenv
	 setenv
	 safe-setenv

	 commonmod:get-cpu-load

	 get-area-path-signature
	 common:simple-file-lock
	 common:low-noise-print
	 common:get-create-writeable-dir
	 common:real-path
	 val->alist

	 client:get-signature

	 common:get-color-from-status
	 
	 seconds->year-work-week/day-time
	 common:simple-file-release-lock
	 common:simple-file-lock-and-wait
	 tests:lookup-itemmap

	 tests:match->sqlqry
	 runs:get-std-run-fields
	 common:min-max
	 common:max
	 common:sum
	 keys:target->keyval
	 db:patt->like

	 rmt:transport-mode
	 common:version-signature

	 keys->keystr
	 keys:config-get-fields
	 common:make-tmpdir-name

	 db:test-get-status
	 db:test-get-state
	 db:test-get-event_time
	 db:test-get-item-path
	 db:test-get-testname
	 db:test-get-id
	 db:test-get-comment
	 db:test-get-run_duration
	 db:test-get-rundir

	 tests:match
	 patt-list-match
	 common:pkts-spec

	 sdb:qry
	 seconds->work-week/day-time

	 tdb:step-get-comment
	 seconds->hr-min-sec
	 any->number
	 tdb:step-get-logfile
	 tdb:step-get-event_time
	 tdb:step-get-status
	 tdb:step-get-state
	 tdb:step-get-id
	 tdb:step-get-stepname
	 db:test-make-full-name
	 common:htree->html
	 common:list->htree

	 tdb:steps-table-get-log-file
	 tdb:steps-table-get-runtime
	 tdb:steps-table-get-status
	 tdb:steps-table-get-end
	 tdb:steps-table-get-start
	 tdb:steps-table-get-stepname
	 tdb:step-get-last_update
	 tdb:step-get-test_id

	 db:test-get-run_id
	 db:test-get-final_logf

	 tests:testqueue-get-item_path
	 tests:testqueue-get-itemdat
	 tests:testqueue-get-testname
	 tests:testqueue-get-priority
	 tests:testqueue-set-priority!
	 tests:testqueue-get-testconfig
	 tests:testqueue-get-waitons

	 tasks:wait-on-journal
	 common:get-area-path-signature

	 db:test-get-last_update
	 db:test-get-archived
	 db:test-get-uname
	 db:test-get-diskfree
	 db:test-get-cpuload
	 db:test-get-process_id
	 db:test-get-host
	 db:test-data-get-last_update
	 db:test-data-get-type
	 db:test-data-get-status
	 db:test-data-get-comment
	 db:test-data-get-units
	 db:test-data-get-tol
	 db:test-data-get-expected
	 db:test-data-get-value
	 db:test-data-get-variable
	 db:test-data-get-category
	 db:test-data-get-test_id
	 db:test-data-get-id

	 host-last-cpuload
	 host-last-update
	 host-last-cpuload-set!
	 host-last-update-set!
	 host-reachable-set!
	 make-host
	 host-last-used-set!
	 host-reachable
	 host-last-used

	 common:alist-ref/default
	 common:val->alist
	 common:in-running-test?
	 
	 common:without-vars
	 common:get-megatest-exe-path
	 common:get-megatest-exe-dir
	 common:get-param-mapping
	 common:get-mtexe
	 
	 db:test-get-is-toplevel
	 seconds->quarter
	 *globalexitstatus*

	 tests:testqueue-set-items!
	 tests:testqueue-get-items
	 *runconfigdat*
	 *passnum*
	 *already-seen-runconfig-info*
	 common:directory-writable?
	 common:dir-clean-up
	 common:fail-safe
	 common:list-or-null
	 *toptest-paths*
	 common:directory-exists?
	 *configstatus*
	 *last-launch*
	 *launch-setup-mutex*
	 commonmod:is-test-alive
	 alist->env-vars
	 *env-vars-by-run-id*
	 common:get-signature
	 common:join-backgrounded-threads
	 tests:glob-like-match
	 common:send-thunk-to-background-thread
	 db:test-get-fullname
	 common:clear-caches
	 db:mintest-get-event_time
	 *test-meta-updated*
	 tests:testqueue-set-item_path!
	 tests:testqueue-set-itemdat!
	 make-tests:testqueue

	 megatest-fossil-hash

	 common:steps-can-proceed-given-status-sym
	 status-sym->string
	 common:worse-status-sym
	 common:logpro-exit-code->status-sym

	 save-environment-as-files
	 assoc/default
	 common:read-encoded-string
	 common:which

	 stop-the-train
	 )
	
(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
	  (prefix base64 base64:)
118
119
120
121
122
123
124


125
126
127
128
129
130
131
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379







+
+







	  srfi-1
	  srfi-18
	  srfi-69
	  typed-records
	  system-information

	  debugprint
	  megatest-fossil-hash
	  
  )))

;;======================================================================
;; CONTENTS
;;
;;  config file utils
;;  misc conversion, data manipulation functions
383
384
385
386
387
388
389

390
391
392
393
394
395
396
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645







+







(define *fdb* #f)

(define *last-launch* (current-seconds)) ;; use for throttling the launch rate. Would be better to use the db and last time of a test in LAUNCHED state.

;; environment vars handy stuff from common.scm
;;
(define getenv get-environment-variable)

(define (safe-setenv key val)
  (if (or (substring-index "!" key)
	  (substring-index ":" key)  ;; variables containing : are for internal use and cannot be environment variables.
	  (substring-index "." key)) ;; periods are not allowed in environment variables
      (debug:print-error 4 *default-log-port* "skip setting internal use only variables containing \":\" or starting with \"!\"")
      (if (and (string? val)
	       (string? key))
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
810
811
812
813
814
815
816



817
818
819
820
821
822
823







-
-
-







  ;; convert string a=1; b=2; c=a silly thing; d=
  (let ((valstr (lookup cfgdat section var)))
    (if valstr
	(val->alist valstr)
	'()))) ;; should it return empty list or #f to indicate not set?


(define (get-section cfgdat section)
  (hash-table-ref/default cfgdat section '()))

(define (common:make-tmpdir-name areapath tmpadj)
  (let* ((area (pathname-file areapath))
         (dname (conc "/tmp/"(current-user-name)"/megatest_localdb/" area "/" (string-translate areapath "/" ".") tmpadj "/.mtdb")))
    (unless (directory-exists? dname)
      (create-directory dname #t))
    dname))

2734
2735
2736
2737
2738
2739
2740



2741



2742




















2743



































































































































































































2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014

3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209







+
+
+

+
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

(define (common:get-fields cfgdat)
  (let ((fields (hash-table-ref/default cfgdat "fields" '())))
    (map car fields)))

(define keys:config-get-fields common:get-fields)

;;======================================================================
;; db_records.scm
;;======================================================================

;;======================================================================
;; dbstruct
;;======================================================================

(define (make-db:test)(make-vector 20))
(define (db:test-get-id           vec) (vector-ref vec 0))
(define (db:test-get-run_id       vec) (vector-ref vec 1))
(define (db:test-get-testname     vec) (vector-ref vec 2))
(define (db:test-get-state        vec) (vector-ref vec 3))
(define (db:test-get-status       vec) (vector-ref vec 4))
(define (db:test-get-event_time   vec) (vector-ref vec 5))
(define (db:test-get-host         vec) (vector-ref vec 6))
(define (db:test-get-cpuload      vec) (vector-ref vec 7))
(define (db:test-get-diskfree     vec) (vector-ref vec 8))
(define (db:test-get-uname        vec) (vector-ref vec 9))
;; (define (db:test-get-rundir       vec) (sdb:qry 'getstr (vector-ref vec 10)))
(define (db:test-get-rundir       vec) (vector-ref vec 10))
(define (db:test-get-item-path    vec) (vector-ref vec 11))
(define (db:test-get-run_duration vec) (vector-ref vec 12))
(define (db:test-get-final_logf   vec) (vector-ref vec 13))
(define (db:test-get-comment      vec) (vector-ref vec 14))
(define (db:test-get-process_id   vec) (vector-ref vec 16))
(define (db:test-get-archived     vec) (vector-ref vec 17))
(define (db:test-get-last_update     vec) (vector-ref vec 18))
)

;; (define (db:test-get-pass_count   vec) (vector-ref vec 15))
;; (define (db:test-get-fail_count   vec) (vector-ref vec 16))
(define (db:test-get-fullname     vec)
  (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))

;; replace runs:make-full-test-name with this routine
(define (db:test-make-full-name testname itempath)
  (if (equal? itempath "") testname (conc testname "/" itempath)))

;; (define (db:test-get-first_err    vec) (printable (vector-ref vec 15)))
;; (define (db:test-get-first_warn   vec) (printable (vector-ref vec 16))) ;; RADT => reference 16 is repeated

(define (db:test-set-cpuload!  vec val)(vector-set! vec 7 val))
(define (db:test-set-diskfree! vec val)(vector-set! vec 8 val))
(define (db:test-set-testname! vec val)(vector-set! vec 2 val))
(define (db:test-set-state!    vec val)(vector-set! vec 3 val))
(define (db:test-set-status!   vec val)(vector-set! vec 4 val))
(define (db:test-set-run_duration! vec val)(vector-set! vec 12 val))
(define (db:test-set-final_logf! vec val)(vector-set! vec 13 val))

;; Test record utility functions

;; Is a test a toplevel?
;;
(define (db:test-get-is-toplevel vec)
  (and (equal? (db:test-get-item-path vec) "")      ;; test is not an item
       (equal? (db:test-get-uname vec)     "n/a"))) ;; test has never been run

;; make-vector-record "" db mintest id run_id testname state status event_time item_path
;; RADT => purpose of mintest??
;;
(define (make-db:mintest)(make-vector 7))
(define (db:mintest-get-id           vec)    (vector-ref  vec 0))
(define (db:mintest-get-run_id       vec)    (vector-ref  vec 1))
(define (db:mintest-get-testname     vec)    (vector-ref  vec 2))
(define (db:mintest-get-state        vec)    (vector-ref  vec 3))
(define (db:mintest-get-status       vec)    (vector-ref  vec 4))
(define (db:mintest-get-event_time   vec)    (vector-ref  vec 5))
(define (db:mintest-get-item_path    vec)    (vector-ref  vec 6))

;; make-vector-record db testmeta id testname author owner description reviewed iterated avg_runtime avg_disk
(define (make-db:testmeta)(make-vector 10 ""))
(define (db:testmeta-get-id            vec)    (vector-ref  vec 0))
(define (db:testmeta-get-testname      vec)    (vector-ref  vec 1))
(define (db:testmeta-get-author        vec)    (vector-ref  vec 2))
(define (db:testmeta-get-owner         vec)    (vector-ref  vec 3))
(define (db:testmeta-get-description   vec)    (vector-ref  vec 4))
(define (db:testmeta-get-reviewed      vec)    (vector-ref  vec 5))
(define (db:testmeta-get-iterated      vec)    (vector-ref  vec 6))
(define (db:testmeta-get-avg_runtime   vec)    (vector-ref  vec 7))
(define (db:testmeta-get-avg_disk      vec)    (vector-ref  vec 8))
(define (db:testmeta-get-tags          vec)    (vector-ref  vec 9))
(define (db:testmeta-set-id!           vec val)(vector-set! vec 0 val))
(define (db:testmeta-set-testname!     vec val)(vector-set! vec 1 val))
(define (db:testmeta-set-author!       vec val)(vector-set! vec 2 val))
(define (db:testmeta-set-owner!        vec val)(vector-set! vec 3 val))
(define (db:testmeta-set-description!  vec val)(vector-set! vec 4 val))
(define (db:testmeta-set-reviewed!     vec val)(vector-set! vec 5 val))
(define (db:testmeta-set-iterated!     vec val)(vector-set! vec 6 val))
(define (db:testmeta-set-avg_runtime!  vec val)(vector-set! vec 7 val))
(define (db:testmeta-set-avg_disk!     vec val)(vector-set! vec 8 val))

;;======================================================================
;; S I M P L E   R U N
;;======================================================================

;; (defstruct id  "runname" "state" "status" "owner" "event_time"

;;======================================================================
;; T E S T   D A T A 
;;======================================================================
(define (make-db:test-data)(make-vector 10))
(define (db:test-data-get-id               vec)    (vector-ref  vec 0))
(define (db:test-data-get-test_id          vec)    (vector-ref  vec 1))
(define (db:test-data-get-category         vec)    (vector-ref  vec 2))
(define (db:test-data-get-variable         vec)    (vector-ref  vec 3))
(define (db:test-data-get-value            vec)    (vector-ref  vec 4))
(define (db:test-data-get-expected         vec)    (vector-ref  vec 5))
(define (db:test-data-get-tol              vec)    (vector-ref  vec 6))
(define (db:test-data-get-units            vec)    (vector-ref  vec 7))
(define (db:test-data-get-comment          vec)    (vector-ref  vec 8))
(define (db:test-data-get-status           vec)    (vector-ref  vec 9))
(define (db:test-data-get-type             vec)    (vector-ref  vec 10))
(define (db:test-data-get-last_update      vec)    (vector-ref  vec 11))

(define (db:test-data-set-id!              vec val)(vector-set!  vec 0  val))
(define (db:test-data-set-test_id!         vec val)(vector-set!  vec 1  val))
(define (db:test-data-set-category!        vec val)(vector-set!  vec 2  val))
(define (db:test-data-set-variable!        vec val)(vector-set!  vec 3  val))
(define (db:test-data-set-value!           vec val)(vector-set!  vec 4  val))
(define (db:test-data-set-expected!        vec val)(vector-set!  vec 5  val))
(define (db:test-data-set-tol!             vec val)(vector-set!  vec 6  val))
(define (db:test-data-set-units!           vec val)(vector-set!  vec 7  val))
(define (db:test-data-set-comment!         vec val)(vector-set!  vec 8  val))
(define (db:test-data-set-status!          vec val)(vector-set!  vec 9  val))
(define (db:test-data-set-type!            vec val)(vector-set!  vec 10 val))

;;======================================================================
;; S T E P S 
;;======================================================================
;; Run steps
;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time    
(define (make-db:step)(make-vector 9))
(define (tdb:step-get-id              vec)    (vector-ref  vec 0))
(define (tdb:step-get-test_id         vec)    (vector-ref  vec 1))
(define (tdb:step-get-stepname        vec)    (vector-ref  vec 2))
(define (tdb:step-get-state           vec)    (vector-ref  vec 3))
(define (tdb:step-get-status          vec)    (vector-ref  vec 4))
(define (tdb:step-get-event_time      vec)    (vector-ref  vec 5))
(define (tdb:step-get-logfile         vec)    (vector-ref  vec 6))
(define (tdb:step-get-comment         vec)    (vector-ref  vec 7))
(define (tdb:step-get-last_update     vec)    (vector-ref  vec 8))
(define (tdb:step-set-id!             vec val)(vector-set! vec 0 val))
(define (tdb:step-set-test_id!        vec val)(vector-set! vec 1 val))
(define (tdb:step-set-stepname!       vec val)(vector-set! vec 2 val))
(define (tdb:step-set-state!          vec val)(vector-set! vec 3 val))
(define (tdb:step-set-status!         vec val)(vector-set! vec 4 val))
(define (tdb:step-set-event_time!     vec val)(vector-set! vec 5 val))
(define (tdb:step-set-logfile!        vec val)(vector-set! vec 6 val))
(define (tdb:step-set-comment!        vec val)(vector-set! vec 7 val))


;; The steps table
(define (make-db:steps-table)(make-vector 5))
(define (tdb:steps-table-get-stepname   vec)    (vector-ref  vec 0))
(define (tdb:steps-table-get-start      vec)    (vector-ref  vec 1))
(define (tdb:steps-table-get-end        vec)    (vector-ref  vec 2))
(define (tdb:steps-table-get-status     vec)    (vector-ref  vec 3))
(define (tdb:steps-table-get-runtime    vec)    (vector-ref  vec 4))
(define (tdb:steps-table-get-log-file   vec)    (vector-ref  vec 5))

(define (tdb:step-stable-set-stepname!  vec val)(vector-set! vec 0 val))
(define (tdb:step-stable-set-start!     vec val)(vector-set! vec 1 val))
(define (tdb:step-stable-set-end!       vec val)(vector-set! vec 2 val))
(define (tdb:step-stable-set-status!    vec val)(vector-set! vec 3 val))
(define (tdb:step-stable-set-runtime!   vec val)(vector-set! vec 4 val))

;; ;; The data structure for handing off requests via wire
;; (define (make-cdb:packet)(make-vector 6))
;; (define (cdb:packet-get-client-sig   vec)    (vector-ref  vec 0))
;; (define (cdb:packet-get-qtype        vec)    (vector-ref  vec 1))
;; (define (cdb:packet-get-immediate    vec)    (vector-ref  vec 2))
;; (define (cdb:packet-get-query-sig    vec)    (vector-ref  vec 3))
;; (define (cdb:packet-get-params       vec)    (vector-ref  vec 4))
;; (define (cdb:packet-get-qtime        vec)    (vector-ref  vec 5))
;; (define (cdb:packet-set-client-sig!  vec val)(vector-set! vec 0 val))
;; (define (cdb:packet-set-qtype!       vec val)(vector-set! vec 1 val))
;; (define (cdb:packet-set-immediate!   vec val)(vector-set! vec 2 val))
;; (define (cdb:packet-set-query-sig!   vec val)(vector-set! vec 3 val))
;; (define (cdb:packet-set-params!      vec val)(vector-set! vec 4 val))
;; (define (cdb:packet-set-qtime!       vec val)(vector-set! vec 5 val))

;;======================================================================
;; key_records
;;======================================================================

(define (keys->valslots keys) ;; => ?,?,? ....
  (string-intersperse (map (lambda (x) "?") keys) ","))

;; (define (keys->key/field keys . additional)
;;   (string-join (map (lambda (k)(conc k " TEXT"))
;; 		    (append keys additional)) ","))

(define (item-list->path itemdat)
  (if (list? itemdat)
      (string-intersperse  (map cadr itemdat) "/")
      ""))


;;======================================================================
;; test_records
;;======================================================================

;; make-vector-record tests testqueue testname testconfig waitons priority items
(define (make-tests:testqueue)(make-vector 7 #f))
(define (tests:testqueue-get-testname     vec)    (vector-ref  vec 0))
(define (tests:testqueue-get-testconfig   vec)    (vector-ref  vec 1))
(define (tests:testqueue-get-waitons      vec)    (vector-ref  vec 2))
(define (tests:testqueue-get-priority     vec)    (vector-ref  vec 3))
;; items: #f=no items, list=list of items remaining, proc=need to call to get items
(define (tests:testqueue-get-items        vec)    (vector-ref  vec 4))
(define (tests:testqueue-get-itemdat      vec)    (vector-ref  vec 5))
(define (tests:testqueue-get-item_path    vec)    (vector-ref  vec 6))

(define (tests:testqueue-set-testname!    vec val)(vector-set! vec 0 val))
(define (tests:testqueue-set-testconfig!  vec val)(vector-set! vec 1 val))
(define (tests:testqueue-set-waitons!     vec val)(vector-set! vec 2 val))
(define (tests:testqueue-set-priority!    vec val)(vector-set! vec 3 val))
(define (tests:testqueue-set-items!       vec val)(vector-set! vec 4 val))
(define (tests:testqueue-set-itemdat!     vec val)(vector-set! vec 5 val))
(define (tests:testqueue-set-item_path!   vec val)(vector-set! vec 6 val))


)

Modified configf.scm from [59af4a3967] to [912b217ff6].

26
27
28
29
30
31
32
33

34
35
36
37
38
39
40
26
27
28
29
30
31
32

33
34
35
36
37
38
39
40







-
+







(declare (unit configf))
(declare (uses process))
(declare (uses env))
(declare (uses keys))
(declare (uses debugprint))
(declare (uses mtargs))
(declare (uses mtargs.import))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses commonmod.import))
(declare (uses processmod))
(declare (uses processmod.import))
(declare (uses configfmod))
(declare (uses configfmod.import))
(declare (uses dbfile))

Modified configfmod.scm from [8c4c9bcd5b] to [776b4f903b].

23
24
25
26
27
28
29















30












31
32
33
34
35
36
37
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses commonmod))
(declare (uses processmod))
(declare (uses mtargs))

(use regex regex-case)

(module configfmod
	(
	 configf:map-all-hier-alist
	 configf:read-refdb
	 lookup
	 configf:lookup
	 get-section
	 configf:get-section
	 configf:lookup-number
	 read-config
	 runconfigs-get
	 configf:section-vars
	 configf:read-alist
	 configf:config->alist
	 configf:alist->config
	 configf:set-section-var
*	

	 find-and-read-config
	 common:args-get-target
	 configf:eval-string-in-environment

	 read-config-set!
	 configf:read-file

	 configf:system
	 configf:config->ini
	 shell
	)

(import scheme
        chicken
	extras
	files
	matchable
	ports
201
202
203
204
205
206
207


208
209
210
211
212
213
214
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242







+
+







	    #f
	    (let ((match (assoc var sectdat)))
	      (if match ;; (and match (list? match)(> (length match) 1))
		  (cadr match)
		  #f))
	    ))
      #f))

(define lookup configf:lookup)

;; use to have definitive setting:
;;  [foo]
;;  var yes
;;
;;  (configf:var-is? cfgdat "foo" "var" "yes") => #t
;;
232
233
234
235
236
237
238


239
240
241
242
243
244
245
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275







+
+







  (let ((sectdat (hash-table-ref/default cfgdat section '())))
    (if (null? sectdat)
	'()
	(map car sectdat))))

(define (configf:get-section cfgdat section)
  (hash-table-ref/default cfgdat section '()))

(define get-section configf:get-section)

(define (configf:set-section-var cfgdat section var val)
  (let ((sectdat (configf:get-section cfgdat section)))
    (hash-table-set! cfgdat section
                     (configf:assoc-safe-add sectdat var val))))

;;======================================================================
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
535
536
537
538
539
540
541



542
543
544
545
546







-
-
-





	    target)
	(if target
	    (begin
	      (debug:print-error 0 *default-log-port* "Invalid target, spaces or blanks not allowed \"" target "\", target should be: " (string-intersperse keys "/") ", have " tlist " for elements")
	      (if exit-if-bad (exit 1))
	      #f)
	    #f))))




(include "configf-guts.scm")

)

Modified cpumod.scm from [33a302895b] to [338168220c].

27
28
29
30
31
32
33

34

35
36
37
38
39
40
41
27
28
29
30
31
32
33
34

35
36
37
38
39
40
41
42







+
-
+







(declare (unit cpumod))
(declare (uses debugprint))
(declare (uses mtargs))

(use srfi-69)

(module cpumod
	()
	*
       

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified dashboard-context-menu.scm from [a9287541e5] to [2e962c22f5].

20
21
22
23
24
25
26
27

28
29
30
31
32
33
34
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34







-
+








;;======================================================================
;; implementation of context menu that pops up on
;; right click on test cell in Runs & Runs Summary Tabs
;;======================================================================

(declare (unit dashboard-context-menu))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses gutils))
(declare (uses rmtmod))
(declare (uses ezsteps))
;; (declare (uses sdb))
;; (declare (uses filedb))
42
43
44
45
46
47
48
49

50
51
52
53
54
55
56
42
43
44
45
46
47
48

49
50
51
52
53
54
55
56







-
+







(import (prefix iup iup:))

(use canvas-draw)

(use srfi-1 posix regex regex-case srfi-69)
(use (prefix sqlite3 sqlite3:))

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")

(import commonmod
	configfmod
	rmtmod
	testsmod

Modified dashboard-guimonitor.scm from [14af79287f] to [6f526827b3].

28
29
30
31
32
33
34
35

36
37
38
39
40

41
42
43
44
45
46
47
28
29
30
31
32
33
34

35
36
37
38
39

40
41
42
43
44
45
46
47







-
+




-
+








(use canvas-draw)

(use sqlite3 srfi-1 posix regex regex-case srfi-69)
(import (prefix sqlite3 sqlite3:))

(declare (unit dashboard-guimonitor))
(declare (uses common))
;; (declare (uses common))
(declare (uses keys))
(declare (uses commonmod))
(import commonmod)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
(include "task_records.scm")

(define (control-panel db tdb keys)
  (let* ((var-params (make-hash-table)) ;; register all the widgets here for querying on run, rollup, remove?
	 (key-params (make-hash-table))

Modified dashboard-tests.scm from [9f47337a67] to [00c786d383].

19
20
21
22
23
24
25
26

27
28
29
30
31
32
33
19
20
21
22
23
24
25

26
27
28
29
30
31
32
33







-
+







;;======================================================================

;;======================================================================
;; Test info panel
;;======================================================================

(declare (unit dashboard-tests))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses debugprint))
(declare (uses rmtmod))
(declare (uses megatestmod))
(declare (uses dbmod))
(declare (uses dbfile))
59
60
61
62
63
64
65
66

67
68
69
70
71
72
73
59
60
61
62
63
64
65

66
67
68
69
70
71
72
73







-
+







	dbfile
	tasksmod
	testsmod
	runsmod
	subrunmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")

;;======================================================================
;; C O M M O N
;;======================================================================

Modified dashboard.scm from [0974058261] to [21bf76042d].

64
65
66
67
68
69
70
71

72
73
74
75
76
77
78
64
65
66
67
68
69
70

71
72
73
74
75
76
77
78







-
+







(declare (uses archivemod.import))
(declare (uses runsmod))
(declare (uses runsmod.import))
(declare (uses launchmod))
(declare (uses launchmod.import))

(declare (uses configf))
(declare (uses common))
;; (declare (uses common))
(declare (uses keys))

(declare (uses process))
(declare (uses launch))
(declare (uses runs))
(declare (uses dashboard-tests))
(declare (uses tree))
105
106
107
108
109
110
111
112

113
114
115
116

117
118
119
120
121
122
123
105
106
107
108
109
110
111

112
113
114
115

116
117
118
119
120
121
122
123







-
+



-
+







	stml2
	megatestmod
	tasksmod
	runsmod
	testsmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
(include "task_records.scm")
(include "megatest-version.scm")
;; (include "megatest-version.scm")
(include "megatest-fossil-hash.scm")
(include "vg_records.scm")

;; set some parameters here - these need to be put in something that can be loaded from other
;; executables such as dashboard and mtutil
;;
(include "dashboard-transport-mode.scm")

Modified db_records.scm from [d1dae58171] to [1501321c72].

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
11
12
13
14
15
16
17























































































































































































-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================
;; dbstruct
;;======================================================================

(define (make-db:test)(make-vector 20))
(define (db:test-get-id           vec) (vector-ref vec 0))
(define (db:test-get-run_id       vec) (vector-ref vec 1))
(define (db:test-get-testname     vec) (vector-ref vec 2))
(define (db:test-get-state        vec) (vector-ref vec 3))
(define (db:test-get-status       vec) (vector-ref vec 4))
(define (db:test-get-event_time   vec) (vector-ref vec 5))
(define (db:test-get-host         vec) (vector-ref vec 6))
(define (db:test-get-cpuload      vec) (vector-ref vec 7))
(define (db:test-get-diskfree     vec) (vector-ref vec 8))
(define (db:test-get-uname        vec) (vector-ref vec 9))
;; (define (db:test-get-rundir       vec) (sdb:qry 'getstr (vector-ref vec 10)))
(define (db:test-get-rundir       vec) (vector-ref vec 10))
(define (db:test-get-item-path    vec) (vector-ref vec 11))
(define (db:test-get-run_duration vec) (vector-ref vec 12))
(define (db:test-get-final_logf   vec) (vector-ref vec 13))
(define (db:test-get-comment      vec) (vector-ref vec 14))
(define (db:test-get-process_id   vec) (vector-ref vec 16))
(define (db:test-get-archived     vec) (vector-ref vec 17))
(define (db:test-get-last_update     vec) (vector-ref vec 18))

;; (define (db:test-get-pass_count   vec) (vector-ref vec 15))
;; (define (db:test-get-fail_count   vec) (vector-ref vec 16))
(define (db:test-get-fullname     vec)
  (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))

;; replace runs:make-full-test-name with this routine
(define (db:test-make-full-name testname itempath)
  (if (equal? itempath "") testname (conc testname "/" itempath)))

;; (define (db:test-get-first_err    vec) (printable (vector-ref vec 15)))
;; (define (db:test-get-first_warn   vec) (printable (vector-ref vec 16))) ;; RADT => reference 16 is repeated

(define (db:test-set-cpuload!  vec val)(vector-set! vec 7 val))
(define (db:test-set-diskfree! vec val)(vector-set! vec 8 val))
(define (db:test-set-testname! vec val)(vector-set! vec 2 val))
(define (db:test-set-state!    vec val)(vector-set! vec 3 val))
(define (db:test-set-status!   vec val)(vector-set! vec 4 val))
(define (db:test-set-run_duration! vec val)(vector-set! vec 12 val))
(define (db:test-set-final_logf! vec val)(vector-set! vec 13 val))

;; Test record utility functions

;; Is a test a toplevel?
;;
(define (db:test-get-is-toplevel vec)
  (and (equal? (db:test-get-item-path vec) "")      ;; test is not an item
       (equal? (db:test-get-uname vec)     "n/a"))) ;; test has never been run

;; make-vector-record "" db mintest id run_id testname state status event_time item_path
;; RADT => purpose of mintest??
;;
(define (make-db:mintest)(make-vector 7))
(define (db:mintest-get-id           vec)    (vector-ref  vec 0))
(define (db:mintest-get-run_id       vec)    (vector-ref  vec 1))
(define (db:mintest-get-testname     vec)    (vector-ref  vec 2))
(define (db:mintest-get-state        vec)    (vector-ref  vec 3))
(define (db:mintest-get-status       vec)    (vector-ref  vec 4))
(define (db:mintest-get-event_time   vec)    (vector-ref  vec 5))
(define (db:mintest-get-item_path    vec)    (vector-ref  vec 6))

;; make-vector-record db testmeta id testname author owner description reviewed iterated avg_runtime avg_disk
(define (make-db:testmeta)(make-vector 10 ""))
(define (db:testmeta-get-id            vec)    (vector-ref  vec 0))
(define (db:testmeta-get-testname      vec)    (vector-ref  vec 1))
(define (db:testmeta-get-author        vec)    (vector-ref  vec 2))
(define (db:testmeta-get-owner         vec)    (vector-ref  vec 3))
(define (db:testmeta-get-description   vec)    (vector-ref  vec 4))
(define (db:testmeta-get-reviewed      vec)    (vector-ref  vec 5))
(define (db:testmeta-get-iterated      vec)    (vector-ref  vec 6))
(define (db:testmeta-get-avg_runtime   vec)    (vector-ref  vec 7))
(define (db:testmeta-get-avg_disk      vec)    (vector-ref  vec 8))
(define (db:testmeta-get-tags          vec)    (vector-ref  vec 9))
(define (db:testmeta-set-id!           vec val)(vector-set! vec 0 val))
(define (db:testmeta-set-testname!     vec val)(vector-set! vec 1 val))
(define (db:testmeta-set-author!       vec val)(vector-set! vec 2 val))
(define (db:testmeta-set-owner!        vec val)(vector-set! vec 3 val))
(define (db:testmeta-set-description!  vec val)(vector-set! vec 4 val))
(define (db:testmeta-set-reviewed!     vec val)(vector-set! vec 5 val))
(define (db:testmeta-set-iterated!     vec val)(vector-set! vec 6 val))
(define (db:testmeta-set-avg_runtime!  vec val)(vector-set! vec 7 val))
(define (db:testmeta-set-avg_disk!     vec val)(vector-set! vec 8 val))

;;======================================================================
;; S I M P L E   R U N
;;======================================================================

;; (defstruct id  "runname" "state" "status" "owner" "event_time"

;;======================================================================
;; T E S T   D A T A 
;;======================================================================
(define (make-db:test-data)(make-vector 10))
(define (db:test-data-get-id               vec)    (vector-ref  vec 0))
(define (db:test-data-get-test_id          vec)    (vector-ref  vec 1))
(define (db:test-data-get-category         vec)    (vector-ref  vec 2))
(define (db:test-data-get-variable         vec)    (vector-ref  vec 3))
(define (db:test-data-get-value            vec)    (vector-ref  vec 4))
(define (db:test-data-get-expected         vec)    (vector-ref  vec 5))
(define (db:test-data-get-tol              vec)    (vector-ref  vec 6))
(define (db:test-data-get-units            vec)    (vector-ref  vec 7))
(define (db:test-data-get-comment          vec)    (vector-ref  vec 8))
(define (db:test-data-get-status           vec)    (vector-ref  vec 9))
(define (db:test-data-get-type             vec)    (vector-ref  vec 10))
(define (db:test-data-get-last_update      vec)    (vector-ref  vec 11))

(define (db:test-data-set-id!              vec val)(vector-set!  vec 0  val))
(define (db:test-data-set-test_id!         vec val)(vector-set!  vec 1  val))
(define (db:test-data-set-category!        vec val)(vector-set!  vec 2  val))
(define (db:test-data-set-variable!        vec val)(vector-set!  vec 3  val))
(define (db:test-data-set-value!           vec val)(vector-set!  vec 4  val))
(define (db:test-data-set-expected!        vec val)(vector-set!  vec 5  val))
(define (db:test-data-set-tol!             vec val)(vector-set!  vec 6  val))
(define (db:test-data-set-units!           vec val)(vector-set!  vec 7  val))
(define (db:test-data-set-comment!         vec val)(vector-set!  vec 8  val))
(define (db:test-data-set-status!          vec val)(vector-set!  vec 9  val))
(define (db:test-data-set-type!            vec val)(vector-set!  vec 10 val))

;;======================================================================
;; S T E P S 
;;======================================================================
;; Run steps
;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time    
(define (make-db:step)(make-vector 9))
(define (tdb:step-get-id              vec)    (vector-ref  vec 0))
(define (tdb:step-get-test_id         vec)    (vector-ref  vec 1))
(define (tdb:step-get-stepname        vec)    (vector-ref  vec 2))
(define (tdb:step-get-state           vec)    (vector-ref  vec 3))
(define (tdb:step-get-status          vec)    (vector-ref  vec 4))
(define (tdb:step-get-event_time      vec)    (vector-ref  vec 5))
(define (tdb:step-get-logfile         vec)    (vector-ref  vec 6))
(define (tdb:step-get-comment         vec)    (vector-ref  vec 7))
(define (tdb:step-get-last_update     vec)    (vector-ref  vec 8))
(define (tdb:step-set-id!             vec val)(vector-set! vec 0 val))
(define (tdb:step-set-test_id!        vec val)(vector-set! vec 1 val))
(define (tdb:step-set-stepname!       vec val)(vector-set! vec 2 val))
(define (tdb:step-set-state!          vec val)(vector-set! vec 3 val))
(define (tdb:step-set-status!         vec val)(vector-set! vec 4 val))
(define (tdb:step-set-event_time!     vec val)(vector-set! vec 5 val))
(define (tdb:step-set-logfile!        vec val)(vector-set! vec 6 val))
(define (tdb:step-set-comment!        vec val)(vector-set! vec 7 val))


;; The steps table
(define (make-db:steps-table)(make-vector 5))
(define (tdb:steps-table-get-stepname   vec)    (vector-ref  vec 0))
(define (tdb:steps-table-get-start      vec)    (vector-ref  vec 1))
(define (tdb:steps-table-get-end        vec)    (vector-ref  vec 2))
(define (tdb:steps-table-get-status     vec)    (vector-ref  vec 3))
(define (tdb:steps-table-get-runtime    vec)    (vector-ref  vec 4))
(define (tdb:steps-table-get-log-file   vec)    (vector-ref  vec 5))

(define (tdb:step-stable-set-stepname!  vec val)(vector-set! vec 0 val))
(define (tdb:step-stable-set-start!     vec val)(vector-set! vec 1 val))
(define (tdb:step-stable-set-end!       vec val)(vector-set! vec 2 val))
(define (tdb:step-stable-set-status!    vec val)(vector-set! vec 3 val))
(define (tdb:step-stable-set-runtime!   vec val)(vector-set! vec 4 val))

;; ;; The data structure for handing off requests via wire
;; (define (make-cdb:packet)(make-vector 6))
;; (define (cdb:packet-get-client-sig   vec)    (vector-ref  vec 0))
;; (define (cdb:packet-get-qtype        vec)    (vector-ref  vec 1))
;; (define (cdb:packet-get-immediate    vec)    (vector-ref  vec 2))
;; (define (cdb:packet-get-query-sig    vec)    (vector-ref  vec 3))
;; (define (cdb:packet-get-params       vec)    (vector-ref  vec 4))
;; (define (cdb:packet-get-qtime        vec)    (vector-ref  vec 5))
;; (define (cdb:packet-set-client-sig!  vec val)(vector-set! vec 0 val))
;; (define (cdb:packet-set-qtype!       vec val)(vector-set! vec 1 val))
;; (define (cdb:packet-set-immediate!   vec val)(vector-set! vec 2 val))
;; (define (cdb:packet-set-query-sig!   vec val)(vector-set! vec 3 val))
;; (define (cdb:packet-set-params!      vec val)(vector-set! vec 4 val))
;; (define (cdb:packet-set-qtime!       vec val)(vector-set! vec 5 val))

Modified dbmod.scm from [6fddda802c] to [f66568c37d].

24
25
26
27
28
29
30


31



















































































































































32
33
34
35
36
37
38
24
25
26
27
28
29
30
31
32

33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186







+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses debugprint))
(declare (uses mtargs))
(declare (uses mtmod))

(module dbmod
	(
	 dbmod:db-to-db-sync
	*
	 
	 db:test-get-event_time
	 db:test-get-item-path
	 db:test-get-testname
	 db:get-value-by-header
	 
	 db:get-subdb

	 db:multi-db-sync
	 
	 dbmod:open-dbmoddb
	 dbmod:run-id->dbfname

	 db:roll-up-rules
	 db:get-all-state-status-counts-for-test
	 db:test-set-state-status-db
	 db:general-call
	 db:cache-for-read-only
	 db:convert-test-itempath

	 db:test-data-rollup
	 db:keep-trying-until-true
	 db:get-test-info-by-id
	 db:with-db
	 db:get-test-id
	 db:get-test-info

	 dbmod:print-db-stats
	 db:get-keys
	 db:open-no-sync-db
	 db:add-stats

	 ;; dbr:counts record accessors
	 dbr:counts->alist

	 db:add-var
	 db:archive-register-block-name
	 db:archive-register-disk
	 db:create-all-triggers
	 db:csv->test-data
	 db:dec-var
	 db:del-var
	 db:delete-old-deleted-test-records
	 db:delete-run
	 db:delete-steps-for-test!
	 db:delete-test-records
	 db:drop-all-triggers
	 db:get-all-run-ids
	 db:get-all-runids
	 db:get-changed-record-ids
	 db:get-changed-record-run-ids
	 db:get-changed-record-test-ids
	 db:get-count-tests-running
	 db:get-count-tests-running-for-run-id
	 db:get-count-tests-running-for-testname
	 db:get-count-tests-running-in-jobgroup
	 db:get-data-info-by-id
	 db:get-key-val-pairs
	 db:get-key-vals
	 db:get-latest-host-load
	 db:get-main-run-stats
	 db:get-matching-previous-test-run-records
	 db:get-not-completed-cnt
	 db:get-num-runs
	 db:get-prereqs-not-met
	 db:get-prev-run-ids
	 db:get-raw-run-stats
	 db:get-run-ids-matching-target
	 db:get-run-info
	 db:get-run-name-from-id
	 db:get-run-record-ids
	 db:get-run-state
	 db:get-run-state-status
	 db:get-run-stats
	 db:get-run-status
	 db:get-run-times
	 db:get-runs
	 db:get-runs-by-patt
	 db:get-runs-cnt-by-patt
	 db:get-steps-data
	 db:get-steps-for-test
	 db:get-steps-info-by-id
	 db:get-target
	 db:get-targets
	 db:get-test-state-status-by-id
	 db:get-test-times
	 db:get-testinfo-state-status
	 db:get-tests-for-run
	 db:get-tests-for-run-mindata
	 db:get-tests-for-run-state-status
	 db:get-tests-tags
	 db:get-toplevels-and-incompletes
	 db:get-var
	 db:have-incompletes?
	 db:inc-var
	 db:initialize-main-db
	 db:insert-run
	 db:insert-test
	 db:lock/unlock-run
	 db:login
	 db:read-test-data
	 db:read-test-data-varpatt
	 db:register-run
	 db:set-run-state-status
	 db:set-run-status
	 db:set-state-status-and-roll-up-run
	 db:set-var
	 db:simple-get-runs
	 db:test-get-archive-block-info
	 db:test-get-logfile-info
	 db:test-get-paths-matching-keynames-target-new
	 db:test-get-records-for-index-file
	 db:test-get-rundir-from-test-id
	 db:test-get-top-process-pid
	 db:test-set-archive-block-id
	 db:test-set-state-status
	 db:test-set-top-process-pid
	 db:test-toplevel-num-items
	 db:testmeta-add-record
	 db:testmeta-get-record
	 db:testmeta-update-field
	 db:teststep-set-status!
	 db:top-test-set-per-pf-counts
	 db:update-run-event_time
	 db:update-run-stats
	 db:update-tesdata-on-repilcate-db
	 tasks:add
	 tasks:find-task-queue-records
	 tasks:get-last
	 tasks:set-state-given-param-key

	 *db-stats*
	 dbmod:nfs-get-dbstruct
	 *db-stats-mutex*

	 db:get-header
	 db:get-rows
	 db:get-changed-run-ids

	 db:set-sync
	 db:setup
	 db:get-access-mode
	 db:test-record-fields
	 
	 db:logpro-dat->csv
	 std-exit-procedure
	 )
	
(import scheme)
	
(cond-expand
 (chicken-4
  (import chicken
	  data-structures
77
78
79
80
81
82
83
84

85
86
87
88
89
90
91
225
226
227
228
229
230
231

232
233
234
235
236
237
238
239







-
+







	commonmod
	configfmod
	dbfile
	debugprint
	mtmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")
(include "run_records.scm")

(define *number-of-writes* 0)
(define *number-non-write-queries* 0)

Modified dcommon.scm from [d0a5600c78] to [bda06bf5b7].

38
39
40
41
42
43
44
45

46
47
48
49
50
51
52
38
39
40
41
42
43
44

45
46
47
48
49
50
51
52







-
+







	configfmod
	rmtmod
        testsmod
        dbmod
	debugprint)

(include "megatest-version.scm")
(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")
(include "run_records.scm")

;; yes, this is non-ideal 
(define dashboard:update-summary-tab #f)
(define dashboard:update-servers-table #f)

Modified diff-report.scm from [939aa5e4ab] to [4819e11d11].

13
14
15
16
17
18
19
20

21
22
23




24
25
26
27
28
















29
30
31
32
33
34
35
13
14
15
16
17
18
19

20
21
22
23
24
25
26
27





28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50







-
+



+
+
+
+
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

(declare (unit diff-report))
(declare (uses common))
;; (declare (uses common))
(declare (uses debugprint))
(declare (uses rmtmod))
(declare (uses commonmod))
(declare (uses stml2))

(module diff-report
	*
(import commonmod
	rmtmod
	debugprint)
         
(include "common_records.scm")
(import scheme
	chicken
	posix
	debugprint
	ports
	srfi-1
	srfi-13
	srfi-69
	data-structures

	stml2
	commonmod
	rmtmod
	)
         
;; (include "common_records.scm")
(use matchable)
(use fmt)
(use ducttape-lib)
(define css "")

(define (diff:tests-mindat->hash tests-mindat)
  (let* ((res (make-hash-table)))
412
413
414
415
416
417
418

427
428
429
430
431
432
433
434







+
      (debug:print 0 *default-log-port* "No match for source target/runname="dest-target"/"dest-runname)
      (debug:print 0 *default-log-port* "Cannot proceed.")
      #f)
     (else
      (diff:deliver-diff-report src-run-id dest-run-id email-recipients-list: to-list html-output-file: html-file)))))

  
)

Modified env.scm from [84be7d5a91] to [26e16dd2d1].

19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129








































































































130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258


































































































































19
20
21
22
23
24
25








































































































26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

































































































































130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259







-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
;;======================================================================

(declare (unit env))

(declare (uses debugprint))
(declare (uses mtargs))

(import (prefix mtargs args:)
	debugprint)

(use sql-de-lite) ;; srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables)

(define (env:open-db fname)
  (let* ((db-exists (common:file-exists? fname))
	 (db        (open-database fname)))
    (if (not db-exists)
	(begin
	  (exec (sql db "CREATE TABLE envvars (
                    id INTEGER PRIMARY KEY,
                    context TEXT NOT NULL,
                    var TEXT NOT NULL,
                    val TEXT NOT NULL,
                       CONSTRAINT envvars_constraint UNIQUE (context,var))"))))
    (set-busy-handler! db (busy-timeout 10000))
    db))

;; save vars in given context, this is NOT incremental by default
;;
(define (env:save-env-vars db context #!key (incremental #f)(vardat #f))
  (with-transaction
   db
   (lambda ()
     ;; first clear out any vars for this context
     (if (not incremental)(exec (sql db "DELETE FROM envvars WHERE context=?") context))
     (for-each
      (lambda (varval)
	(let ((var (car varval))
	      (val (cdr varval)))
	  (if incremental (exec (sql db "DELETE FROM envvars WHERE context=? AND var=?") context var))
	  (exec (sql db "INSERT INTO envvars (context,var,val) VALUES (?,?,?)") context var val)))
	(if vardat
	    (hash-table->alist vardat)
	    (get-environment-variables))))))

;; merge contexts in the order given
;;  - each context is applied in the given order
;;  - variables in the paths list are split on the separator and the components
;;    merged using simple delta addition
;;    returns a hash of the merged vars
;;
(define (env:merge-contexts db basecontext contexts paths)
  (let ((result (make-hash-table)))
    (for-each
     (lambda (context)
       (query
	(for-each-row
	 (lambda (row)
	   (let ((var  (car row))
		 (val  (cadr row)))
	     (hash-table-set! result var 
			      (if (and (hash-table-ref/default result var #f)
				       (assoc var paths)) ;; this var is a path and there is a previous path
				  (let ((sep (cadr (assoc var paths))))
				    (env:merge-path-envvar sep (hash-table-ref result var) val))
				  val)))))
	(sql db "SELECT var,val FROM envvars WHERE context=?")
	context))
     contexts)
    result))

;;  get list of removed variables between two contexts
;;
(define (env:get-removed db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
     contexta contextb)
    result))

;;  get list of variables added to contextb from contexta
;;
(define (env:get-added db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
     contextb contexta)
    result))

;;  get list of variables in both contexta and contexb that have been changed
;;
(define (env:get-changed db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars AS a WHERE context=? AND val != (SELECT val FROM envvars WHERE var=a.var AND context=?)")
     contextb contexta)
    result))
;; (import (prefix mtargs args:)
;; 	debugprint)
;; 
;; (use sql-de-lite) ;; srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables)
;; 
;; (define (env:open-db fname)
;;   (let* ((db-exists (common:file-exists? fname))
;; 	 (db        (open-database fname)))
;;     (if (not db-exists)
;; 	(begin
;; 	  (exec (sql db "CREATE TABLE envvars (
;;                     id INTEGER PRIMARY KEY,
;;                     context TEXT NOT NULL,
;;                     var TEXT NOT NULL,
;;                     val TEXT NOT NULL,
;;                        CONSTRAINT envvars_constraint UNIQUE (context,var))"))))
;;     (set-busy-handler! db (busy-timeout 10000))
;;     db))
;; 
;; ;; save vars in given context, this is NOT incremental by default
;; ;;
;; (define (env:save-env-vars db context #!key (incremental #f)(vardat #f))
;;   (with-transaction
;;    db
;;    (lambda ()
;;      ;; first clear out any vars for this context
;;      (if (not incremental)(exec (sql db "DELETE FROM envvars WHERE context=?") context))
;;      (for-each
;;       (lambda (varval)
;; 	(let ((var (car varval))
;; 	      (val (cdr varval)))
;; 	  (if incremental (exec (sql db "DELETE FROM envvars WHERE context=? AND var=?") context var))
;; 	  (exec (sql db "INSERT INTO envvars (context,var,val) VALUES (?,?,?)") context var val)))
;; 	(if vardat
;; 	    (hash-table->alist vardat)
;; 	    (get-environment-variables))))))
;; 
;; ;; merge contexts in the order given
;; ;;  - each context is applied in the given order
;; ;;  - variables in the paths list are split on the separator and the components
;; ;;    merged using simple delta addition
;; ;;    returns a hash of the merged vars
;; ;;
;; (define (env:merge-contexts db basecontext contexts paths)
;;   (let ((result (make-hash-table)))
;;     (for-each
;;      (lambda (context)
;;        (query
;; 	(for-each-row
;; 	 (lambda (row)
;; 	   (let ((var  (car row))
;; 		 (val  (cadr row)))
;; 	     (hash-table-set! result var 
;; 			      (if (and (hash-table-ref/default result var #f)
;; 				       (assoc var paths)) ;; this var is a path and there is a previous path
;; 				  (let ((sep (cadr (assoc var paths))))
;; 				    (env:merge-path-envvar sep (hash-table-ref result var) val))
;; 				  val)))))
;; 	(sql db "SELECT var,val FROM envvars WHERE context=?")
;; 	context))
;;      contexts)
;;     result))
;; 
;; ;;  get list of removed variables between two contexts
;; ;;
;; (define (env:get-removed db contexta contextb)
;;   (let ((result (make-hash-table)))
;;     (query
;;      (for-each-row
;;       (lambda (row)
;; 	(let ((var  (car row))
;; 	      (val  (cadr row)))
;; 	  (hash-table-set! result var val))))
;;      (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
;;      contexta contextb)
;;     result))
;; 
;; ;;  get list of variables added to contextb from contexta
;; ;;
;; (define (env:get-added db contexta contextb)
;;   (let ((result (make-hash-table)))
;;     (query
;;      (for-each-row
;;       (lambda (row)
;; 	(let ((var  (car row))
;; 	      (val  (cadr row)))
;; 	  (hash-table-set! result var val))))
;;      (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
;;      contextb contexta)
;;     result))
;; 
;; ;;  get list of variables in both contexta and contexb that have been changed
;; ;;
;; (define (env:get-changed db contexta contextb)
;;   (let ((result (make-hash-table)))
;;     (query
;;      (for-each-row
;;       (lambda (row)
;; 	(let ((var  (car row))
;; 	      (val  (cadr row)))
;; 	  (hash-table-set! result var val))))
;;      (sql db "SELECT var,val FROM envvars AS a WHERE context=? AND val != (SELECT val FROM envvars WHERE var=a.var AND context=?)")
;;      contextb contexta)
;;     result))

;;
(define (env:blind-merge l1 l2)
  (if (null? l1) l2
      (if (null? l2) l1
	  (cons (car l1) (cons (car l2) (env:blind-merge (cdr l1) (cdr l2)))))))

;; given a before and an after envvar calculate a new merged path
;;
(define (env:merge-path-envvar separator patha pathb)
  (let* ((patha-parts  (string-split patha separator))
	 (pathb-parts  (string-split pathb separator))
	 (common-parts (lset-intersection equal? patha-parts pathb-parts))
	 (final        (delete-duplicates ;; env:blind-merge 
			(append pathb-parts common-parts patha-parts))))
;;     (print "BEFORE:   " (string-intersperse patha-parts  "\n       "))
;;     (print "AFTER:    " (string-intersperse pathb-parts  "\n       "))
;;     (print "COMMON:   " (string-intersperse common-parts "\n       "))
    (string-intersperse final separator)))

(define (env:process-path-envvar varname separator patha pathb)
  (let ((newpath (env:merge-path-envvar separator patha pathb)))
    (setenv varname newpath)))

(define (env:have-context db context)
  (> (query fetch-value (sql db "SELECT count(id) FROM envvars WHERE context=?") context)
     0))

;; this is so the calling block does not need to import sql-de-lite
(define (env:close-database db)
  (close-database db))

(define (env:lazy-hash-table->alist indat)
  (if (hash-table? indat)
      (let ((dat (hash-table->alist indat)))
	(if (null? dat)
	    #f 
	    dat))
      #f))

(define (env:inc-path path)
  (print "PATH "
	 (conc "#{scheme (env:min-path \"" path "\" \"#{getenv PATH}\")}")))
;; 	 (conc
;; 	  "#{scheme (string-intersperse "
;; 	  "(delete-duplicates "
;; 	  "(append (string-split \"" path "\" \":\") "
;; 	  "(string-split \"#{getenv PATH}\" \":\")))"
;; 	  " \":\")}")))

(define (env:min-path path1 path2)
  (string-intersperse
   (delete-duplicates
    (append
     (string-split path1 ":")
     (string-split path2 ":")))
   ":"))

;; inc path will set a PATH that is incrementally modified when read - config mode only
;;
(define (env:print added removed changed #!key (inc-path #t))
  (let ((a  (env:lazy-hash-table->alist added))
	(r  (env:lazy-hash-table->alist removed))
	(c  (env:lazy-hash-table->alist changed)))
    (case (if (args:get-arg "-dumpmode")
	      (string->symbol (args:get-arg "-dumpmode"))
	      'bash)
      ((bash)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "unset " (car dat)))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
		  (hash-table->alist changed)))))
      ((csh)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "unsetenv " (car dat)))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
		  (hash-table->alist changed)))))
      ((config ini)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)
		    (let ((var (car dat))
			  (val (cdr dat)))
		      (if (and inc-path
			       (equal? var "PATH"))
			  (env:inc-path val)
			  (print var " " val))))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "#{scheme (unsetenv \"" (car dat) "\")}"))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)
		    (let ((var (car dat))
			  (val (cdr dat)))
		      (if (and inc-path
			       (equal? var "PATH"))
			  (env:inc-path val)
			  (print var " " val))))
		  (hash-table->alist changed)))))
      (else
       (debug:print-error 0 *default-log-port* "No dumpmode specified, use -dumpmode [bash|csh|config]")))))
;; 
;; ;;
;; (define (env:blind-merge l1 l2)
;;   (if (null? l1) l2
;;       (if (null? l2) l1
;; 	  (cons (car l1) (cons (car l2) (env:blind-merge (cdr l1) (cdr l2)))))))
;; 
;; ;; given a before and an after envvar calculate a new merged path
;; ;;
;; (define (env:merge-path-envvar separator patha pathb)
;;   (let* ((patha-parts  (string-split patha separator))
;; 	 (pathb-parts  (string-split pathb separator))
;; 	 (common-parts (lset-intersection equal? patha-parts pathb-parts))
;; 	 (final        (delete-duplicates ;; env:blind-merge 
;; 			(append pathb-parts common-parts patha-parts))))
;; ;;     (print "BEFORE:   " (string-intersperse patha-parts  "\n       "))
;; ;;     (print "AFTER:    " (string-intersperse pathb-parts  "\n       "))
;; ;;     (print "COMMON:   " (string-intersperse common-parts "\n       "))
;;     (string-intersperse final separator)))
;; 
;; (define (env:process-path-envvar varname separator patha pathb)
;;   (let ((newpath (env:merge-path-envvar separator patha pathb)))
;;     (setenv varname newpath)))
;; 
;; (define (env:have-context db context)
;;   (> (query fetch-value (sql db "SELECT count(id) FROM envvars WHERE context=?") context)
;;      0))
;; 
;; ;; this is so the calling block does not need to import sql-de-lite
;; (define (env:close-database db)
;;   (close-database db))
;; 
;; (define (env:lazy-hash-table->alist indat)
;;   (if (hash-table? indat)
;;       (let ((dat (hash-table->alist indat)))
;; 	(if (null? dat)
;; 	    #f 
;; 	    dat))
;;       #f))
;; 
;; (define (env:inc-path path)
;;   (print "PATH "
;; 	 (conc "#{scheme (env:min-path \"" path "\" \"#{getenv PATH}\")}")))
;; ;; 	 (conc
;; ;; 	  "#{scheme (string-intersperse "
;; ;; 	  "(delete-duplicates "
;; ;; 	  "(append (string-split \"" path "\" \":\") "
;; ;; 	  "(string-split \"#{getenv PATH}\" \":\")))"
;; ;; 	  " \":\")}")))
;; 
;; (define (env:min-path path1 path2)
;;   (string-intersperse
;;    (delete-duplicates
;;     (append
;;      (string-split path1 ":")
;;      (string-split path2 ":")))
;;    ":"))
;; 
;; ;; inc path will set a PATH that is incrementally modified when read - config mode only
;; ;;
;; (define (env:print added removed changed #!key (inc-path #t))
;;   (let ((a  (env:lazy-hash-table->alist added))
;; 	(r  (env:lazy-hash-table->alist removed))
;; 	(c  (env:lazy-hash-table->alist changed)))
;;     (case (if (args:get-arg "-dumpmode")
;; 	      (string->symbol (args:get-arg "-dumpmode"))
;; 	      'bash)
;;       ((bash)
;;        (if a
;; 	   (begin
;; 	     (print "# Added vars")
;; 	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
;; 		  (hash-table->alist added))))
;;        (if r
;; 	   (begin
;; 	     (print "# Removed vars")
;; 	     (map (lambda (dat)(print "unset " (car dat)))
;; 		  (hash-table->alist removed))))
;;        (if c
;; 	   (begin
;; 	     (print "# Changed vars")
;; 	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
;; 		  (hash-table->alist changed)))))
;;       ((csh)
;;        (if a
;; 	   (begin
;; 	     (print "# Added vars")
;; 	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
;; 		  (hash-table->alist added))))
;;        (if r
;; 	   (begin
;; 	     (print "# Removed vars")
;; 	     (map (lambda (dat)(print "unsetenv " (car dat)))
;; 		  (hash-table->alist removed))))
;;        (if c
;; 	   (begin
;; 	     (print "# Changed vars")
;; 	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
;; 		  (hash-table->alist changed)))))
;;       ((config ini)
;;        (if a
;; 	   (begin
;; 	     (print "# Added vars")
;; 	     (map (lambda (dat)
;; 		    (let ((var (car dat))
;; 			  (val (cdr dat)))
;; 		      (if (and inc-path
;; 			       (equal? var "PATH"))
;; 			  (env:inc-path val)
;; 			  (print var " " val))))
;; 		  (hash-table->alist added))))
;;        (if r
;; 	   (begin
;; 	     (print "# Removed vars")
;; 	     (map (lambda (dat)(print "#{scheme (unsetenv \"" (car dat) "\")}"))
;; 		  (hash-table->alist removed))))
;;        (if c
;; 	   (begin
;; 	     (print "# Changed vars")
;; 	     (map (lambda (dat)
;; 		    (let ((var (car dat))
;; 			  (val (cdr dat)))
;; 		      (if (and inc-path
;; 			       (equal? var "PATH"))
;; 			  (env:inc-path val)
;; 			  (print var " " val))))
;; 		  (hash-table->alist changed)))))
;;       (else
;;        (debug:print-error 0 *default-log-port* "No dumpmode specified, use -dumpmode [bash|csh|config]")))))
;; 

Added envmod.scm version [72c755440f].




















































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

(use sql-de-lite)

(declare (unit envmod))

(declare (uses debugprint))
(declare (uses mtargs))
(declare (uses commonmod))

(module envmod
	*
	
(import scheme
	chicken

	posix
	srfi-1
	data-structures
	srfi-69)

(import (prefix mtargs args:)
	debugprint
	commonmod)

(import sql-de-lite) ;; srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables)

(define (env:open-db fname)
  (let* ((db-exists (common:file-exists? fname))
	 (db        (open-database fname)))
    (if (not db-exists)
	(begin
	  (exec (sql db "CREATE TABLE envvars (
                    id INTEGER PRIMARY KEY,
                    context TEXT NOT NULL,
                    var TEXT NOT NULL,
                    val TEXT NOT NULL,
                       CONSTRAINT envvars_constraint UNIQUE (context,var))"))))
    (set-busy-handler! db (busy-timeout 10000))
    db))

;; save vars in given context, this is NOT incremental by default
;;
(define (env:save-env-vars db context #!key (incremental #f)(vardat #f))
  (with-transaction
   db
   (lambda ()
     ;; first clear out any vars for this context
     (if (not incremental)(exec (sql db "DELETE FROM envvars WHERE context=?") context))
     (for-each
      (lambda (varval)
	(let ((var (car varval))
	      (val (cdr varval)))
	  (if incremental (exec (sql db "DELETE FROM envvars WHERE context=? AND var=?") context var))
	  (exec (sql db "INSERT INTO envvars (context,var,val) VALUES (?,?,?)") context var val)))
	(if vardat
	    (hash-table->alist vardat)
	    (get-environment-variables))))))

;; merge contexts in the order given
;;  - each context is applied in the given order
;;  - variables in the paths list are split on the separator and the components
;;    merged using simple delta addition
;;    returns a hash of the merged vars
;;
(define (env:merge-contexts db basecontext contexts paths)
  (let ((result (make-hash-table)))
    (for-each
     (lambda (context)
       (query
	(for-each-row
	 (lambda (row)
	   (let ((var  (car row))
		 (val  (cadr row)))
	     (hash-table-set! result var 
			      (if (and (hash-table-ref/default result var #f)
				       (assoc var paths)) ;; this var is a path and there is a previous path
				  (let ((sep (cadr (assoc var paths))))
				    (env:merge-path-envvar sep (hash-table-ref result var) val))
				  val)))))
	(sql db "SELECT var,val FROM envvars WHERE context=?")
	context))
     contexts)
    result))

;;  get list of removed variables between two contexts
;;
(define (env:get-removed db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
     contexta contextb)
    result))

;;  get list of variables added to contextb from contexta
;;
(define (env:get-added db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars WHERE context=? AND var NOT IN (SELECT var FROM envvars WHERE context=?)")
     contextb contexta)
    result))

;;  get list of variables in both contexta and contexb that have been changed
;;
(define (env:get-changed db contexta contextb)
  (let ((result (make-hash-table)))
    (query
     (for-each-row
      (lambda (row)
	(let ((var  (car row))
	      (val  (cadr row)))
	  (hash-table-set! result var val))))
     (sql db "SELECT var,val FROM envvars AS a WHERE context=? AND val != (SELECT val FROM envvars WHERE var=a.var AND context=?)")
     contextb contexta)
    result))

;;
(define (env:blind-merge l1 l2)
  (if (null? l1) l2
      (if (null? l2) l1
	  (cons (car l1) (cons (car l2) (env:blind-merge (cdr l1) (cdr l2)))))))

;; given a before and an after envvar calculate a new merged path
;;
(define (env:merge-path-envvar separator patha pathb)
  (let* ((patha-parts  (string-split patha separator))
	 (pathb-parts  (string-split pathb separator))
	 (common-parts (lset-intersection equal? patha-parts pathb-parts))
	 (final        (delete-duplicates ;; env:blind-merge 
			(append pathb-parts common-parts patha-parts))))
;;     (print "BEFORE:   " (string-intersperse patha-parts  "\n       "))
;;     (print "AFTER:    " (string-intersperse pathb-parts  "\n       "))
;;     (print "COMMON:   " (string-intersperse common-parts "\n       "))
    (string-intersperse final separator)))

(define (env:process-path-envvar varname separator patha pathb)
  (let ((newpath (env:merge-path-envvar separator patha pathb)))
    (setenv varname newpath)))

(define (env:have-context db context)
  (> (query fetch-value (sql db "SELECT count(id) FROM envvars WHERE context=?") context)
     0))

;; this is so the calling block does not need to import sql-de-lite
(define (env:close-database db)
  (close-database db))

(define (env:lazy-hash-table->alist indat)
  (if (hash-table? indat)
      (let ((dat (hash-table->alist indat)))
	(if (null? dat)
	    #f 
	    dat))
      #f))

(define (env:inc-path path)
  (print "PATH "
	 (conc "#{scheme (env:min-path \"" path "\" \"#{getenv PATH}\")}")))
;; 	 (conc
;; 	  "#{scheme (string-intersperse "
;; 	  "(delete-duplicates "
;; 	  "(append (string-split \"" path "\" \":\") "
;; 	  "(string-split \"#{getenv PATH}\" \":\")))"
;; 	  " \":\")}")))

(define (env:min-path path1 path2)
  (string-intersperse
   (delete-duplicates
    (append
     (string-split path1 ":")
     (string-split path2 ":")))
   ":"))

;; inc path will set a PATH that is incrementally modified when read - config mode only
;;
(define (env:print added removed changed #!key (inc-path #t))
  (let ((a  (env:lazy-hash-table->alist added))
	(r  (env:lazy-hash-table->alist removed))
	(c  (env:lazy-hash-table->alist changed)))
    (case (if (args:get-arg "-dumpmode")
	      (string->symbol (args:get-arg "-dumpmode"))
	      'bash)
      ((bash)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "unset " (car dat)))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)(print "export " (car dat) "=" (cdr dat)))
		  (hash-table->alist changed)))))
      ((csh)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "unsetenv " (car dat)))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)(print "setenv " (car dat) " " (cdr dat)))
		  (hash-table->alist changed)))))
      ((config ini)
       (if a
	   (begin
	     (print "# Added vars")
	     (map (lambda (dat)
		    (let ((var (car dat))
			  (val (cdr dat)))
		      (if (and inc-path
			       (equal? var "PATH"))
			  (env:inc-path val)
			  (print var " " val))))
		  (hash-table->alist added))))
       (if r
	   (begin
	     (print "# Removed vars")
	     (map (lambda (dat)(print "#{scheme (unsetenv \"" (car dat) "\")}"))
		  (hash-table->alist removed))))
       (if c
	   (begin
	     (print "# Changed vars")
	     (map (lambda (dat)
		    (let ((var (car dat))
			  (val (cdr dat)))
		      (if (and inc-path
			       (equal? var "PATH"))
			  (env:inc-path val)
			  (print var " " val))))
		  (hash-table->alist changed)))))
      (else
       (debug:print-error 0 *default-log-port* "No dumpmode specified, use -dumpmode [bash|csh|config]")))))

)

Modified ezsteps.scm from [27c83d47a5] to [8e5e826481].

17
18
19
20
21
22
23
24

25
26
27
28
29
30
31
17
18
19
20
21
22
23

24
25
26
27
28
29
30
31







-
+







;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')

(declare (unit ezsteps))
(declare (uses commonmod))
(declare (uses common))
;; (declare (uses common))
(declare (uses configfmod))
(declare (uses debugprint))

(declare (uses runconfig))
(declare (uses rmtmod))
(declare (uses mtargs))
(declare (uses tasksmod))

Modified ezstepsmod.scm from [0f672c5b01] to [ff6068567c].

43
44
45
46
47
48
49
50

51
52
53
54
55
56
57
43
44
45
46
47
48
49

50
51
52
53
54
55
56
57







-
+







(declare (uses testsmod))
(declare (uses runsmod))
(declare (uses fsmod))

(use srfi-69)

(module ezstepsmod
	*
	()

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
124
125
126
127
128
129
130
131

132
133
134
135
136
137
138
124
125
126
127
128
129
130

131
132
133
134
135
136
137
138







-
+







	tasksmod
	subrunmod
	testsmod
	runsmod
	fsmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")


;;(rmt:get-test-info-by-id run-id test-id) -> testdat

Modified fsmod.scm from [57a2c983b3] to [5fe1b052db].

31
32
33
34
35
36
37






38


39
40
41
42
43
44
45
31
32
33
34
35
36
37
38
39
40
41
42
43

44
45
46
47
48
49
50
51
52







+
+
+
+
+
+
-
+
+







(declare (uses configfmod))
(declare (uses commonmod))
(declare (uses processmod))

(use srfi-69)

(module fsmod
	(
	 get-df
	 get-uname
	 common:get-disk-with-most-free-space
	 common:get-disk-space-used
	 common:check-db-dir-and-exit-if-insufficient
	*

	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified genexample.scm from [6229d612cf] to [570b3e311e].

17
18
19
20
21
22
23
24

25
26
27



28
29















30

31
32

33
34



35
36
37
38
39
40
41
17
18
19
20
21
22
23

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

48
49
50
51
52

53
54
55
56
57
58
59
60
61
62







-
+



+
+
+


+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+


+

-
+
+
+







;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

(declare (unit genexample))
(declare (uses mtargs))
(declare (uses debugprint))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses rmtmod))
(declare (uses testsmod))
(declare (uses dbfile))
(declare (uses tasksmod))

(use posix regex matchable)

(module genexample
	*
	
(import scheme
	chicken

	data-structures
	extras
	srfi-1
	srfi-13
	srfi-69
	posix
	regex
	matchable
(import (prefix mtargs args:)
	(prefix mtargs args:)
	commonmod
	configfmod
	testsmod
	rmtmod
	debugprint)
	debugprint
	tasksmod
	dbfile)

;; (include "db_records.scm")

(define genexample:example-logpro
#<<EOF
  ;; You should have at least one expect:required. This ensures that your process ran
  ;; comment out the line below and replace "put pattern here" with a pattern that will
524
525
526
527
528
529
530

545
546
547
548
549
550
551
552







+
		      (if (not (equal? item-path ""))
			  (system (conc "refdb set " miscrdb " itemsinfo " (obfuscate testname) " " (obfuscate item-path) " x")))

		      ))))
	    tests-data)))
       (map (lambda (runrec)(simple-run-id runrec)) runs)))
    ))
)

Modified key_records.scm from [55f6701b87] to [8cb99c889a].

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
14
15
16
17
18
19
20



















-
-
-
-
-
-
-
-
-
-
-
-
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

(define (keys->valslots keys) ;; => ?,?,? ....
  (string-intersperse (map (lambda (x) "?") keys) ","))

;; (define (keys->key/field keys . additional)
;;   (string-join (map (lambda (k)(conc k " TEXT"))
;; 		    (append keys additional)) ","))

(define (item-list->path itemdat)
  (if (list? itemdat)
      (string-intersperse  (map cadr itemdat) "/")
      ""))

Modified keys.scm from [ddf211e0d9] to [727b92d3a4].

18
19
20
21
22
23
24
25

26
27
28
29
30
31
32
18
19
20
21
22
23
24

25
26
27
28
29
30
31
32







-
+







;;
 
;;======================================================================
;; Run keys, these are used to hierarchially organise tests and run areas
;;======================================================================

(declare (unit keys))
(declare (uses common))
;; (declare (uses common))
(declare (uses debugprint))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses mtargs))

(use sqlite3 srfi-1 posix regex regex-case srfi-69)
(import (prefix sqlite3 sqlite3:)

Modified launch.scm from [60c51037a0] to [24a75d7c76].

19
20
21
22
23
24
25
26

27
28
29
30
31
32
33
19
20
21
22
23
24
25

26
27
28
29
30
31
32
33







-
+







;;======================================================================
;; launch a task - this runs on the originating host, tests themselves
;;
;;======================================================================

(declare (unit launch))
(declare (uses subrun))
(declare (uses common))
;; (declare (uses common))
(declare (uses debugprint))
(declare (uses commonmod))
(declare (uses processmod))
(declare (uses configfmod))
(declare (uses configf))
(declare (uses rmtmod))
(declare (uses ezsteps))
44
45
46
47
48
49
50
51

52
53
54
55
56
57
58
44
45
46
47
48
49
50

51
52
53
54
55
56
57
58







-
+







     typed-records pathname-expand matchable)

(import (prefix base64 base64:)
	(prefix sqlite3 sqlite3:)
	(prefix mtargs args:)
)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "megatest-fossil-hash.scm")

(import commonmod
	processmod
	configfmod

Modified launchmod.scm from [46f91d6b1b] to [4708ea7e53].

42
43
44
45
46
47
48
49






50
51
52
53
54
55
56
42
43
44
45
46
47
48

49
50
51
52
53
54
55
56
57
58
59
60
61







-
+
+
+
+
+
+







(declare (uses testsmod))
(declare (uses runsmod))
(declare (uses fsmod))

(use srfi-69)

(module launchmod
	*
	(
	 launch:load-logpro-dat
	 launch:recover-test
	 launch:execute
	 launch:extract-scripts-logpro
	)

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
124
125
126
127
128
129
130
131

132
133
134
135
136
137
138
129
130
131
132
133
134
135

136
137
138
139
140
141
142
143







-
+







	tasksmod
	subrunmod
	testsmod
	runsmod
	fsmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "megatest-fossil-hash.scm")

;;======================================================================
;; ezsteps
;;======================================================================
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
982
983
984
985
986
987
988






















































































989
990
991
992
993
994
995







-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







		     #f)))
    ;; now wait on that process if all is correct
    ;; periodically update the db with runtime
    ;; when the process exits look at the db, if still RUNNING after 10 seconds set
    ;; state/status appropriately
    (process-wait pid)))

;;======================================================================
;; Maintenance
;;======================================================================

(define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
  (let* ((cfg-deadtime             (configf:lookup-number *configdat* "setup" "deadtime"))
	 (test-stats-update-period (configf:lookup-number *configdat* "setup" "test-stats-update-period")))
   (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period)
   ;;call end of eud of run detection for posthook
   (launch:end-of-run-check run-id)))

;;  select end_time-now from
;;      (select testname,item_path,event_time+run_duration as
;;                          end_time,strftime('%s','now') as now from tests where state in
;;      ('RUNNING','REMOTEHOSTSTART','LAUNCHED'));
;;
;; NOT EASY TO MIGRATE TO db{file,mod}
;;
(define (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period)
  (let* ((incompleted '())
	 (oldlaunched '())
	 (toplevels   '())
          ;; The default running-deadtime is 720 seconds = 12 minutes.
          ;; "(running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))" = 200 + (2 * (200 + 30 + 30))
         (deadtime-trim (or ovr-deadtime cfg-deadtime))
         (server-start-allowance 200)
         (server-overloaded-budget 200)
         (launch-monitor-off-time (or test-stats-update-period 30))
         (launch-monitor-on-time-budget 30)
         (launch-monitor-period (+ launch-monitor-off-time launch-monitor-on-time-budget server-overloaded-budget))
         (remotehoststart-deadtime-default (+ server-start-allowance server-overloaded-budget 30))
         (remotehoststart-deadtime (or deadtime-trim remotehoststart-deadtime-default))
         (running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))
         (running-deadtime (or deadtime-trim running-deadtime-default))) ;; two minutes (30 seconds between updates, this leaves 3x grace period)

    (debug:print-info 4  *default-log-port* "running-deadtime = " running-deadtime)
    (debug:print-info 4  *default-log-port* "deadtime-trim = " deadtime-trim)

    (let* ((dat (rmt:get-toplevels-and-incompletes run-id running-deadtime remotehoststart-deadtime)))
      (set! oldlaunched (list-ref dat 1))
      (set! toplevels   (list-ref dat 2))
      (set! incompleted (list-ref dat 0)))

    (debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, "
		      (length toplevels) " old LAUNCHED toplevel tests and "
		      (length incompleted) " tests marked RUNNING but apparently dead.")
  
    ;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE.
    ;;
    ;; (db:delay-if-busy dbdat)
    (let* ((min-incompleted-ids (map car incompleted)) ;; do 'em all
	   (all-ids             (append min-incompleted-ids (map car oldlaunched))))
      (if (> (length all-ids) 0)
	  (begin
	    ;; (launch:is-test-alive "localhost" 435)
	    (debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ")
			 " as DEAD")
	    (for-each
             (lambda (test-id)
               (let* ((tinfo   (rmt:get-test-info-by-id run-id test-id))
		      (run-dir (db:test-get-rundir     tinfo))
		      (host    (db:test-get-host       tinfo))
		      (pid     (db:test-get-process_id tinfo))
		      (result (rmt:get-status-from-final-status-file run-dir)))
		 (if (and (list? result) (> (length result) 1) (equal? "PASS" (cadr result)) (equal? "COMPLETED" (car result))) 
		     (begin
		       (debug:print 0 *default-log-port* "INFO: test " test-id " actually passed, so marking PASS not DEAD")
		       (rmt:set-state-status-and-roll-up-items
			run-id test-id 'foo "COMPLETED" "PASS"
			"Test stopped responding but it has PASSED; marking it PASS in the DB."))
		     (let ((is-alive (and (not (eq? pid 0))  ;; 0 is default in re-used field "attemptnum" where pid stored.
					  (commonmod:is-test-alive host pid))))
		       (if is-alive
			   (debug:print 0 *default-log-port* "INFO: test " test-id " on host " host
					" has a process on pid " pid ", NOT setting to DEAD.")
			   (begin
			     (debug:print 0 *default-log-port* "INFO: test " test-id
					  " final state/status is not COMPLETED/PASS. It is " result)
			     (rmt:set-state-status-and-roll-up-items
			      run-id test-id 'foo "COMPLETED" "DEAD"
			      "Test stopped responding while in RUNNING or REMOTEHOSTSTART; presumed dead.")))))))
	     ;; call end of eud of run detection for posthook - from merge, is it needed?
	     ;; (launch:end-of-run-check run-id)
	     all-ids)
	    )))))


;; TODO: deprecate me in favor of ezsteps.scm
;;
(define (ezsteps:runstep ezstep run-id test-id exit-info m tal testconfig all-steps-dat)
  (let* ((stepname       (car ezstep))  ;; do stuff to run the step
	 (stepinfo       (cadr ezstep))
	;; (let ((info (cadr ezstep)))

Modified megatest.scm from [faeb3396e2] to [a02aadc475].

13
14
15
16
17
18
19
20

21
22
23
24
25

26
27
28
29

30
31
32
33
34
35
36
13
14
15
16
17
18
19

20
21




22
23
24
25

26
27
28
29
30
31
32
33







-
+

-
-
-
-
+



-
+







;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

;; (include "common.scm")
(include "megatest-version.scm")
;; (include "megatest-version.scm")

;; fake out readline usage of toplevel-command
(define (toplevel-command . a) #f)

(declare (uses common))
;; (declare (uses common))
;; (declare (uses megatest-version))
;; (declare (uses margs))
(declare (uses mtargs))
;; (declare (uses mtargs.import))
(declare (uses mtargs.import))
(declare (uses debugprint))
(declare (uses debugprint.import))
(declare (uses cookie))
(declare (uses cookie.import))
(declare (uses stml2))
(declare (uses stml2.import))
(declare (uses commonmod))
82
83
84
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296

1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319

1320
1321
1322
1323

1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900

1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498

2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
79
80
81
82
83
84
85




86











































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































87








88























89




90























































































































































































































































































































































































































































































































































91









































92

















































































































































































































































































































































































































































































































































































































93




94

















































































































































































































































































































































-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
(declare (uses api))
(declare (uses env))
(declare (uses diff-report))
(declare (uses runs))
(declare (uses launch))
(declare (uses server))
(declare (uses genexample))

;; (include "debugmode.scm")

;; (declare (uses daemon))
(declare (uses mtbody))

;; (declare (uses dcommon))

;; (declare (uses debugprint))
;; (declare (uses debugprint.import))

;; (declare (uses ftail))
;; (import ftail)

(import (prefix mtargs args:)
        debugprint
	dbmod
	commonmod
	processmod
	configfmod
	dbfile
	portlogger
	tcp-transportmod
	rmtmod
	apimod
	stml2
	mtmod
	megatestmod
	servermod
	tasksmod
	runsmod
	rmtmod
	launchmod
	fsmod
        )

(define *db* #f) ;; this is only for the repl, do not use in general!!!!

(include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
(include "megatest-fossil-hash.scm")

(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (prefix base64 base64:))
(use readline apropos json http-client directory-utils typed-records)
(use http-client srfi-18 extras format tcp-server tcp)

;; Added for csv stuff - will be removed
;;
(use sparse-vectors)

(require-library mutils)

;; remove when configf fully modularized
(read-config-set! configf:read-file)

(define *usage-log-file* #f)    ;; put path to file for logging usage in this var in the ~/.megatestrc file
(define *usage-use-seconds* #t) ;; for Epoc seconds in usage logging change this to #t in ~/.megatestrc file

;; set some parameters here - these need to be put in something that can be loaded from other
;; executables such as dashboard and mtutil
;;
(include "transport-mode.scm")
(dbfile:db-init-proc db:initialize-main-db)
(debug:enable-timestamp #t) 


(set! rmtmod:send-receive rmt:send-receive)
 ;;(lambda params (apply rmt:send-receive params))) ;; make send-receive available to rmtmod via parameter


;; load the ~/.megatestrc file, put (use trace)(trace-call-sites #t)(trace function-you-want-to-trace) in this file
;;
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.megatestrc")))
  (if (common:file-exists? debugcontrolf)
      (load debugcontrolf)))

;; usage logging, careful with this, it is not designed to deal with all real world challenges!
;;
(if (and *usage-log-file*
         (file-write-access? *usage-log-file*))
    (with-output-to-file
        *usage-log-file*
      (lambda ()
        (print (if *usage-use-seconds*
		   (current-seconds)
		   (time->string
		    (seconds->local-time (current-seconds))
		    "%Yww%V.%w %H:%M:%S"))
               " "
               (current-user-name) " "
               (current-directory) " "
               "\"" (string-intersperse (argv) " ") "\""))
      #:append))

;; Disabled help items
;;  -rollup                 : (currently disabled) fill run (set by :runname)  with latest test(s)
;;                            from prior runs with same keys
;;  -daemonize              : fork into background and disconnect from stdin/out

(define help (conc "
Megatest, documentation at http://www.kiatoa.com/fossils/megatest
  version " megatest-version "
  license GPL, Copyright Matt Welland 2006-2017
 
Usage: megatest [options]
  -h                      : this help
  -manual                 : show the Megatest user manual
  -version                : print megatest version (currently " megatest-version ")

Launching and managing runs
  -run                    : run all tests or as specified by -testpatt
  -remove-runs            : remove the data for a run, requires -runname and -testpatt
                            Optionally use :state and :status, use -keep-records to remove only
                            the run data. Use -kill-wait to override the 10 second
                            per test wait after kill delay (e.g. -kill-wait 0). 
  -kill-runs              : kill existing run(s) (all incomplete tests killed)
  -kill-rerun             : kill an existing run (all incomplete tests killed and run is rerun)
  -set-state-status X,Y   : set state to X and status to Y, requires controls per -remove-runs
  -rerun FAIL,WARN...     : force re-run for tests with specificed status(s)
  -rerun-clean            : set all tests not COMPLETED+PASS,WARN,WAIVED to NOT_STARTED,n/a
                            and then run the specified testpatt with -preclean
  -rerun-all              : set all tests to NOT_STARTED,n/a and run with -preclean
  -lock                   : lock run specified by target and runname
  -unlock                 : unlock run specified by target and runname
  -set-run-status status  : sets status for run to status, requires -target and -runname
  -get-run-status         : gets status for run specified by target and runname
  -run-wait               : wait on run specified by target and runname
  -preclean               : remove the existing test directory before running the test
  -clean-cache            : remove the cached megatest.config and runconfigs.config files
  -no-cache               : do not use the cached config files. 
  -one-pass               : launch as many tests as you can but do not wait for more to be ready
  -remove-keep N          : remove all but N most recent runs per target; use '-actions, -age, -precmd'
  -age <age>              : 120d,3h,20m to apply only to runs older than the 
                                 specified age. NB// M=month, m=minute
  -actions <action>[,...] : actions to take; print,remove-runs,archive,kill-runs
  -precmd                 : insert a wrapper command in front of the commands run

Selectors (e.g. use for -runtests, -remove-runs, -set-state-status, -list-runs etc.)
  -target key1/key2/...   : run for key1, key2, etc.
  -reqtarg key1/key2/...  : run for key1, key2, etc. but key1/key2 must be in runconfigs
  -testpatt patt1/patt2,patt3/...  : % is wildcard
  -runname                : required, name for this particular test run
  -state                  : Applies to runs, tests or steps depending on context
  -status                 : Applies to runs, tests or steps depending on context
  -modepatt key           : load testpatt from <key> in runconfigs instead of default TESTPATT if -testpatt and -tagexpr are not specified
  -tagexpr tag1,tag2%,..  : select tests with tags matching expression
  

Test helpers (for use inside tests)
  -step stepname
  -test-status            : set the state and status of a test (use :state and :status)
  -setlog logfname        : set the path/filename to the final log relative to the test
                            directory. may be used with -test-status
  -set-toplog logfname    : set the overall log for a suite of sub-tests
  -summarize-items        : for an itemized test create a summary html 
  -m comment              : insert a comment for this test

Test data capture
  -set-values             : update or set values in the testdata table
  :category               : set the category field (optional)
  :variable               : set the variable name (optional)
  :value                  : value measured (required)
  :expected               : value expected (required)
  :tol                    : |value-expect| <= tol (required, can be <, >, >=, <= or number)
  :units                  : name of the units for value, expected_value etc. (optional)
  -load-test-data         : read test specific data for storage in the test_data table
                            from standard in. Each line is comma delimited with four
                            fields category,variable,value,comment

Queries
  -list-runs patt         : list runs matching pattern \"patt\", % is the wildcard
  -show-keys              : show the keys used in this megatest setup
  -test-files targpatt    : get the most recent test path/file matching targpatt e.g. %/% or '*.log'
                            returns list sorted by age ascending, see examples below
  -test-paths             : get the test paths matching target, runname, item and test
                            patterns.
  -list-disks             : list the disks available for storing runs
  -list-targets           : list the targets in runconfigs.config
  -list-db-targets        : list the target combinations used in the db
  -show-config            : dump the internal representation of the megatest.config file
  -show-runconfig         : dump the internal representation of the runconfigs.config file
  -dumpmode MODE          : dump in MODE format instead of sexpr, MODE=json,ini,sexp etc. (add -debug 0,9 to see which file contributes each line)
  -show-cmdinfo           : dump the command info for a test (run in test environment)
  -section sectionName
  -var varName            : for config and runconfig lookup value for sectionName varName
  -since N                : get list of runs changed since time N (Unix seconds)
  -fields fieldspec       : fields to include in json dump; runs:id,runame+tests:testname+steps
  -sort fieldname         : in -list-runs sort tests by this field
  -testdata-csv [categorypatt/]varpatt  : dump testdata for given category

Misc 
  -start-dir path         : switch to this directory before running megatest
  -contour cname          : add a level of hierarcy to the linktree and run paths
  -area-tag tagname       : add a tag to an area while syncing to pgdb
  -run-tag tagname        : add a tag to a run while syncing to pgdb
  -rebuild-db             : bring the database schema up to date
  -cleanup-db             : remove any orphan records, vacuum the db
  -import-megatest.db     : push data from megatest.db to cache db files in /tmp/$USER
  -sync-to-megatest.db    : pull data from cache files in /tmp/$USER to megatest.db
  -sync-to dest           : sync to new postgresql central style database
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -adjutant C,M           : start the server/adjutant with allocated cores C and Mem M (Gig), 
                            use 0,0 to auto use full machine
  -transport http|rpc     : use http or rpc for transport (default is http) 
  -log logfile            : send stdout and stderr to logfile
  -list-servers           : list the servers 
  -kill-servers           : kill all servers
  -repl                   : start a repl (useful for extending megatest)
  -load file.scm          : load and run file.scm
  -mark-incompletes       : find and mark incomplete tests
  -ping run-id|host:port  : ping server, exit with 0 if found
  -debug N|N,M,O...       : enable debug 0-N or N and M and O ...
  -debug-noprop N|M,M,O...: enable debug but do not propagate to subprocesses via MT_DEBUG
  -config fname           : override the megatest.config file with fname
  -append-config fname    : append fname to the megatest.config file
  -import-sexpr fname     : import a sexpr file (use -list-runs % -dumpmode sexpr to create)
  -remove-dbs all         : remove Megatest DBs before importing sexpr. (Use only with -import-sexpr)
  -regen-testfiles        : regenerate scripts and logpro files from testconfig, run in test context
  
Utilities
  -env2file fname         : write the environment to fname.csh and fname.sh
  -envcap a               : save current variables labeled as context 'a' in file envdat.db
  -envdelta a-b           : output enviroment delta from context a to context b to -o fname
                            set the output mode with -dumpmode csh, bash or ini
                            note: ini format will use calls to use curr and minimize path
  -refdb2dat refdb        : convert refdb to sexp or to format specified by s-dumpmode
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove, get, replicate-db (use 
                            -dest to set destination), -include path1,path2... to get or save specific files
  -generate-html          : create a simple html dashboard for browsing your runs
  -generate-html-structure  : create a top level html veiw to list targets/runs and a Run view within each run directory.  
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 
                            is $DISPLAY valid 
  -list-waivers           : dump waivers for specified target, runname, testpatt to stdout
  -db2db                  : sync db to db, use -from, -to for dbs, -period and -timeout for continuous sync

Diff report
  -diff-rep               : generate diff report (must include -src-target, -src-runname, -target, -runname
                                                  and either -diff-email or -diff-html)
  -src-target <target>
  -src-runname <target>
  -diff-email <emails>    : comma separated list of email addresses to send diff report
  -diff-html  <rep.html>  : path to html file to generate

Spreadsheet generation
  -extract-ods fname.ods  : extract an open document spreadsheet from the database
  -pathmod path           : insert path, i.e. path/runame/itempath/logfile.html
                            will clear the field if no rundir/testname/itempath/logfile
                            if it contains forward slashes the path will be converted
                            to windows style
Getting started
  -create-megatest-area   : create a skeleton megatest area. You will be prompted for paths
  -create-test testname   : create a skeleton megatest test. You will be prompted for info

Examples

# Get test path, use '.' to get a single path or a specific path/file pattern
megatest -test-files 'logs/*.log' -target ubuntu/n%/no% -runname w49% -testpatt test_mt%

Called as " (string-intersperse (argv) " ") "
Version " megatest-version ", built from " megatest-fossil-hash ))

;;  -gui                    : start a gui interface
;;  -config fname           : override the runconfigs file with fname

;; process args
(define remargs (args:get-args 
		 (argv)
		 (list  "-runtests"  ;; run a specific test
			"-config"    ;; override the config file name
			"-append-config"
			"-execute"   ;; run the command encoded in the base64 parameter
			"-step"
			"-target"
			"-reqtarg"
			":runname"
			"-runname"
			":state"  
			"-state"
			":status"
			"-status"
			"-list-runs"
                        "-testdata-csv"
			"-testpatt"
                        ;; "--modepatt"
                        "-modepatt"
                        "-tagexpr"
			"-itempatt"
			"-setlog"
			"-set-toplog"
			"-runstep"
			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"
			"-from"
			"-to"
			"-dest"
                        "-source" 
                        "-time-stamp" 
			;; values and messages
			":category"
			":variable"
			":value"
			":expected"
			":tol"
			":units"

			;; misc
			"-start-dir"
                        "-run-patt"
                        "-target-patt"   
			"-contour"
                        "-area-tag"  
                        "-area"  
			"-run-tag"
			"-server"
			"-adjutant"
			"-transport"
			"-port"
			"-extract-ods"
			"-pathmod"
			"-env2file"
			"-envcap"
			"-envdelta"
			"-setvars"
			"-set-state-status"
			"-import-sexpr"
                        "-remove-dbs" ;; to be used only with -import-sexpr to remove megatest dbs first.
			"-period"  ;; sync period in seconds
			"-timeout" ;; exit sync if timeout in seconds exceeded since last change

                        ;; move runs stuff here
                        "-remove-keep"           
			"-set-run-status"
			"-age"

			;; archive 
			"-archive"
			"-actions"
			"-precmd"
			"-include"
			"-exclude-rx"
			"-exclude-rx-from"
			
			"-debug" ;; for *verbosity* > 2
			"-debug-noprop"
			"-create-test"
			"-override-timeout"
			"-test-files"  ;; -test-paths is for listing all
			"-load"        ;; load and exectute a scheme file
			"-section"
			"-var"
			"-dumpmode"
			"-run-id"
			"-db"
			"-ping"
			"-refdb2dat"
			"-o"
			"-log"
                        "-sync-log"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			"-target-db"
			"-source-db"
			"-prefix-target"

                        "-src-target"
                        "-src-runname"
                        "-diff-email"
			"-sync-to"			
			"-pgsync"
			"-kill-wait"    ;; wait this long before removing test (default is 10 sec)
                        "-diff-html"

			;; wizards, area capture, setup new ...
			"-extract-skeleton"
			)
 		 (list  "-h" "-help" "--help"
			"-manual"
			"-version"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-show-keys"
		        "-test-status"
			"-set-values"
			"-load-test-data"
			"-summarize-items"
		        "-gui"
			"-daemonize"
			"-preclean"
			"-rerun-clean"
			"-rerun-all"
			"-clean-cache"
			"-no-cache"
			"-cache-db"
			"-cp-eventtime-to-publishtime"
                        "-use-db-cache"
                        "-prepend-contour"


			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
			"-kill-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			"-one-pass"      ;;
			"-local"         ;; run some commands using local db access
			"-generate-html"
			"-generate-html-structure" 
			"-list-run-time"
                        "-list-test-time"
			"-regen-testfiles"
			
			;; misc queries
			"-list-disks"
			"-list-targets"
			"-list-db-targets"
			"-show-runconfig"
			"-show-config"
			"-show-cmdinfo"
			"-get-run-status"
			"-list-waivers"

			;; queries
			"-test-paths" ;; get path(s) to a test, ordered by youngest first

			"-runall"    ;; run all tests, respects -testpatt, defaults to %
			"-run"       ;; alias for -runall
			"-remove-runs"
                        "-kill-runs"
                        "-kill-rerun"
                        "-keep-records" ;; use with -remove-runs to remove only the run data
			"-rebuild-db"
			"-cleanup-db"
			"-rollup"
			"-update-meta"
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"
			"-db2db"
                        "-sync-brute-force"
			"-logging"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only

                        "-diff-rep"

			"-syscheck"
			"-obfuscate"
			;; junk placeholder
			;; "-:p"
			
                        )
		 args:arg-hash
		 0))

;; Add args that use remargs here
;;
(if (and (not (null? remargs))
	 (not (or
	       (args:get-arg "-runstep")
	       (args:get-arg "-envcap")
	       (args:get-arg "-envdelta")
	       )
	      ))
    (debug:print-error 0 *default-log-port* "Unrecognised arguments: " (string-intersperse (if (list? remargs) remargs (argv))  " ")))

;; before doing anything else change to the start-dir if provided
;;
(if (args:get-arg "-start-dir")
    (if (common:file-exists? (args:get-arg "-start-dir"))
        (let ((fullpath (common:real-path (args:get-arg "-start-dir"))))
          (setenv "PWD" fullpath)
          (change-directory fullpath))
	(begin
	  (debug:print-error 0 *default-log-port* "non-existant start dir " (args:get-arg "-start-dir") " specified, exiting.")
	  (exit 1))))

;; immediately set MT_TARGET if -reqtarg or -target are available
;;
(let ((targ (or (args:get-arg "-reqtarg")(args:get-arg "-target"))))
  (if targ (setenv "MT_TARGET" targ)))

;; set the purpose field in procinf

(procinf-purpose-set! *procinf* (get-purpose args:arg-hash))
(procinf-mtversion-set! *procinf* megatest-version)

;; The watchdog is to keep an eye on things like db sync etc.
;;

;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage
;;(define *watchdog* (make-thread
;;		    (lambda ()
;;		      (handle-exceptions
;;			  exn
;;			  (begin
;;			    (print-call-chain)
;;			    (print " message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
;;			(common:watchdog)))
;;		    "Watchdog thread"))

;;(if (not (args:get-arg "-server"))
;;    (thread-start! *watchdog*)) ;; if starting a server; wait till we get to running state before kicking off watchdog
(let* ((no-watchdog-args
       '("-list-runs"
         "-testdata-csv"
         "-list-servers"
         "-server"
	 "-adjutant"
         "-list-disks"
         "-list-targets"
         "-show-runconfig"
         ;;"-list-db-targets"
         "-show-runconfig"
         "-show-config"
         "-show-cmdinfo"
	 "-cleanup-db"
            ))
       (no-watchdog-argvals (list '("-archive" . "replicate-db")))
       (start-watchdog-specail-arg-val (let loop ((hed  (car no-watchdog-argvals))
                                                  (tail (cdr   no-watchdog-argvals)))
                                             ;; (print "hed" hed " arg " (args:get-arg (car hed)) " val:" (cdr hed)  " eql" (equal? (args:get-arg (car hed)) (cdr hed)))  
                                             (if (equal? (args:get-arg (car hed)) (cdr hed))
                                               #f
                                               (if (null? tail)
                                                 #t
                                                 (loop (car tail) (cdr tail))))))      
       (no-watchdog-args-vals (filter (lambda (x) x)
                                      (map args:get-arg no-watchdog-args)))
       (start-watchdog (and (null? no-watchdog-args-vals) start-watchdog-specail-arg-val)))
       ;(print  "no-watchdog-args="no-watchdog-args "no-watchdog-args-vals="no-watchdog-args-vals " start-watchdog-specail-arg-val:" start-watchdog-specail-arg-val " start-watchdog:" start-watchdog) 
;;  (if start-watchdog
;;      (thread-start! *watchdog*))
    #t
)

;; stop the train watchdog
(stop-the-train)

;; bracket open-output-file with code to make leading directory if it does not exist and handle exceptions
(define (open-logfile logpath-in)
  (condition-case
   (let* ((log-dir (or (pathname-directory logpath-in) "."))
	  (fname   (pathname-strip-directory logpath-in))
	  (logpath (if (> (string-length fname) 250)
		       (let ((newlogf (conc log-dir "/" (common:get-signature fname) ".log")))
			 (debug:print 0 *default-log-port* "WARNING: log file " logpath-in " path too long, converted to " newlogf)
			 newlogf)
		       logpath-in)))
     (if (not (directory-exists? log-dir))
         (system (conc "mkdir -p " log-dir)))
     (open-output-file logpath))
   (exn ()
        (debug:print-error 0 *default-log-port* "Could not open log file for write: "logpath)
        (define *didsomething* #t)  
        (exit 1))))

;; this segment will run launch:setup only if -log is not set. This is fairly safe as servers are not
;; manually started and thus should never be started in a non-megatest area. Thus no need to handle situation
;; where (launch:setup) returns #f?
;;
(if (or (args:get-arg "-log")#;(args:get-arg "-server")) ;; redirect the log always when a server
    (handle-exceptions
	exn
      (begin
	(print "ERROR: Failed to switch to log output. " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
      (let* ((tl   (or (args:get-arg "-log")(launch:setup)))   ;; run launch:setup if -server, ensure we do NOT run launch:setup if -log specified
	     (dbname (args:get-arg "-db"))   ;; for the server logfile name
	     (logf (or (args:get-arg "-log") ;; use -log unless we are a server, then craft a logfile name
		       (conc tl "/logs/server-"(or dbname "unk")"-"(current-process-id) "-" (get-host-name) ".log")))
	     (oup  (open-logfile logf)))
	(if (not (args:get-arg "-log"))
	    (hash-table-set! args:arg-hash "-log" logf)) ;; fake out future queries of -log
	(debug:print-info 0 *default-log-port* "Sending log output to " logf)
	(set! *default-log-port* oup))))

(if (or (args:get-arg "-h")
	(args:get-arg "-help")
	(args:get-arg "--help"))
    (begin
      (print help)
      (exit)))

(if (args:get-arg "-manual")
    (let* ((htmlviewercmd (or (configf:lookup *configdat* "setup" "htmlviewercmd")
			      (common:which '("firefox" "arora"))))
	   (install-home  (common:get-install-area))
	   (manual-html   (conc install-home "/share/docs/megatest_manual.html")))
      (if (and install-home
	       (common:file-exists? manual-html))
	  (system (conc "(" htmlviewercmd " " manual-html " ) &"))
	  (system (conc "(" htmlviewercmd " http://www.kiatoa.com/cgi-bin/fossils/megatest/doc/tip/docs/manual/megatest_manual.html ) &")))
      (exit)))

(if (args:get-arg "-version")
    (begin
      (print (common:version-signature)) ;; (print megatest-version)
      (exit)))

(define *didsomething* #f)

;; Overall exit handling setup immediately
;;
(if (or (args:get-arg "-process-reap"))
        ;; (args:get-arg "-runtests")
	;; (args:get-arg "-execute")
	;; (args:get-arg "-remove-runs")
	;; (args:get-arg "-runstep"))
    (let ((original-exit (exit-handler)))
      (exit-handler (lambda (#!optional (exit-code 0))
		      (printf "Preparing to exit with exit code ~A ...\n" exit-code)
		      (for-each
		       
		       (lambda (pid)
			 (handle-exceptions
			     exn
			   (begin
			     (printf "process reap failed. exn=~A\n" exn)
			     #t)
			  (let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
				      (if (or (eq? pid-val pid)
					      (eq? pid-val 0))
					  (begin
					    (printf "Sending signal/term to ~A\n" pid)
					    (process-signal pid signal/term))))))
		       (process:children #f))
		      (original-exit exit-code)))))

;; for some switches always print the command to stderr
;;
(if (args:any-defined? "-run" "-runall" "-remove-runs" "-set-state-status" "-kill-runs" "-kill-rerun")
    (debug:print 0 *default-log-port* (string-intersperse (argv) " ")))


;;======================================================================
;; Misc setup stuff
;;======================================================================

(debug:setup)

(if (args:get-arg "-logging")(set! *logging* #t))

;;(if (debug:debug-mode 3) ;; we are obviously debugging
;;    (set! open-run-close open-run-close-no-exception-handling))

(if (args:get-arg "-itempatt")
    (let ((newval (conc (args:get-arg "-testpatt") "/" (args:get-arg "-itempatt"))))
      (debug:print 0 *default-log-port* "WARNING: -itempatt has been deprecated, please use -testpatt testpatt/itempatt method, new testpatt is "newval)
      (hash-table-set! args:arg-hash "-testpatt" newval)
      (hash-table-delete! args:arg-hash "-itempatt")))

(if (args:get-arg "-runtests")
    (debug:print 0 *default-log-port* "WARNING: \"-runtests\" is deprecated. Use \"-run\" with \"-testpatt\" instead"))

(on-exit std-exit-procedure)

;;======================================================================
;; Misc general calls
;;======================================================================

(if (and (args:get-arg "-cache-db")
         (args:get-arg "-source-db"))
    (let* ((temp-dir (or (args:get-arg "-target-db") (create-directory (conc "/tmp/" (getenv "USER") "/" (string-translate (current-directory) "/" "_")))))
           (target-db (conc temp-dir "/cached.db"))
           (source-db (args:get-arg "-source-db")))        
      (db:cache-for-read-only source-db target-db)
      (set! *didsomething* #t)))

;; handle a clean-cache request as early as possible
;;
(if (args:get-arg "-clean-cache")
    (let ((toppath  (launch:setup)))
      (set! *didsomething* #t) ;; suppress the help output.
      (runs:clean-cache (common:args-get-target)
			(args:get-arg "-runname")
			toppath)))
	  
(if (args:get-arg "-env2file")
    (begin
      (save-environment-as-files (args:get-arg "-env2file"))
      (set! *didsomething* #t)))

(if (args:get-arg "-list-disks")
    (let ((toppath (launch:setup)))
      (print (string-intersperse 
	      (map (lambda (x)
		     (string-intersperse 
		      x
		      " => "))
		   (common:get-disks *configdat*))
	      "\n"))
      (set! *didsomething* #t)))

;; csv processing record
(define (make-refdb:csv)
  (vector 
   (make-sparse-array)
   (make-hash-table)
   (make-hash-table)
   0
   0))
(define-inline (refdb:csv-get-svec     vec)    (vector-ref  vec 0))
(define-inline (refdb:csv-get-rows     vec)    (vector-ref  vec 1))
(define-inline (refdb:csv-get-cols     vec)    (vector-ref  vec 2))
(define-inline (refdb:csv-get-maxrow   vec)    (vector-ref  vec 3))
(define-inline (refdb:csv-get-maxcol   vec)    (vector-ref  vec 4))
(define-inline (refdb:csv-set-svec!    vec val)(vector-set! vec 0 val))
(define-inline (refdb:csv-set-rows!    vec val)(vector-set! vec 1 val))
(define-inline (refdb:csv-set-cols!    vec val)(vector-set! vec 2 val))
(define-inline (refdb:csv-set-maxrow!  vec val)(vector-set! vec 3 val))
(define-inline (refdb:csv-set-maxcol!  vec val)(vector-set! vec 4 val))

(define (get-dat results sheetname)
  (or (hash-table-ref/default results sheetname #f)
      (let ((tmp-vec  (make-refdb:csv)))
	(hash-table-set! results sheetname tmp-vec)
	tmp-vec)))

(if (args:get-arg "-refdb2dat")
    (let* ((input-db (args:get-arg "-refdb2dat"))
	   (out-file (args:get-arg "-o"))
	   (out-fmt  (or (args:get-arg "-dumpmode") "scheme"))
	   (out-port (if (and out-file 
			      (not (member out-fmt '("sqlite3" "csv"))))
			 (open-output-file out-file)
			 (current-output-port)))
	   (res-data (configf:read-refdb input-db))
	   (data     (car res-data))
	   (msg      (cadr res-data)))
      (if (not data)
	  (debug:print 0 *default-log-port* "Bad input? data=" data) ;; some error occurred
	  (with-output-to-port out-port
	    (lambda ()
	      (case (string->symbol out-fmt)
		((scheme)(pp data))
		((perl)
		 ;; (print "%hash = (")
		 ;;        key1 => 'value1',
		 ;;        key2 => 'value2',
		 ;;        key3 => 'value3',
		 ;; );
		 (configf:map-all-hier-alist 
		  data 
		  (lambda (sheetname sectionname varname val)
		    (print "$data{\"" sheetname "\"}{\"" sectionname "\"}{\"" varname "\"} = \"" val "\";"))))
		((python ruby)
		 (print "data={}")
		 (configf:map-all-hier-alist
		  data
		  (lambda (sheetname sectionname varname val)
		    (print "data[\"" sheetname "\"][\"" sectionname "\"][\"" varname "\"] = \"" val "\""))
		  initproc1:
		  (lambda (sheetname)
		    (print "data[\"" sheetname "\"] = {}"))
		  initproc2:
		  (lambda (sheetname sectionname)
		    (print "data[\"" sheetname "\"][\"" sectionname "\"] = {}"))))
		((csv)
		 (let* ((results  (make-hash-table)) ;; (make-sparse-array)))
			(row-cols (make-hash-table))) ;; hash of hashes where section => ht { row-<name> => num or col-<name> => num
		   ;; (print "data=")
		   ;; (pp data)
		   (configf:map-all-hier-alist
		    data
		    (lambda (sheetname sectionname varname val)
		      ;; (print "sheetname: " sheetname ", sectionname: " sectionname ", varname: " varname ", val: " val)
		      (let* ((dat      (get-dat results sheetname))
			     (vec      (refdb:csv-get-svec dat))
			     (rownames (refdb:csv-get-rows dat))
			     (colnames (refdb:csv-get-cols dat))
			     (currrown (hash-table-ref/default rownames varname #f))
			     (currcoln (hash-table-ref/default colnames sectionname #f))
			     (rown     (or currrown 
					   (let* ((lastn   (refdb:csv-get-maxrow dat))
						  (newrown (+ lastn 1)))
					     (refdb:csv-set-maxrow! dat newrown)
					     newrown)))
			     (coln     (or currcoln 
					   (let* ((lastn   (refdb:csv-get-maxcol dat))
						  (newcoln (+ lastn 1)))
					     (refdb:csv-set-maxcol! dat newcoln)
					     newcoln))))
			(if (not (sparse-array-ref vec 0 coln)) ;; (eq? rown 0)
			    (begin
			      (sparse-array-set! vec 0 coln sectionname)
			      ;; (print "sparse-array-ref " 0 "," coln "=" (sparse-array-ref vec 0 coln))
			      ))
			(if (not (sparse-array-ref vec rown 0)) ;; (eq? coln 0)
			    (begin
			      (sparse-array-set! vec rown 0 varname)
			      ;; (print "sparse-array-ref " rown "," 0 "=" (sparse-array-ref vec rown 0))
			      ))
			(if (not currrown)(hash-table-set! rownames varname rown))
			(if (not currcoln)(hash-table-set! colnames sectionname coln))
			;; (print "dat=" dat ", rown=" rown ", coln=" coln)
			(sparse-array-set! vec rown coln val)
			;; (print "sparse-array-ref " rown "," coln "=" (sparse-array-ref vec rown coln))
			)))
		   (for-each
		    (lambda (sheetname)
		      (let* ((sheetdat (get-dat results sheetname))
			     (svec     (refdb:csv-get-svec sheetdat))
			     (maxrow   (refdb:csv-get-maxrow sheetdat))
			     (maxcol   (refdb:csv-get-maxcol sheetdat))
			     (fname    (if out-file 
					   (string-substitute "%s" sheetname out-file) ;; "/foo/bar/%s.csv")
					   (conc sheetname ".csv"))))
			(with-output-to-file fname
			  (lambda ()
			    ;; (print "Sheetname: " sheetname)
			    (let loop ((row       0)
				       (col       0)
				       (curr-row '())
				       (result   '()))
			      (let* ((val (sparse-array-ref svec row col))
				     (disp-val (if val
						   (conc "\"" val "\"")
						   "")))
				(if (> col 0)(display ","))
				(display disp-val)
				(cond
				 ((> row maxrow)(display "\n") result)
				 ((>= col maxcol)
				  (display "\n")
				  (loop (+ row 1) 0 '() (append result (list curr-row))))
				 (else
				  (loop row (+ col 1) (append curr-row (list val)) result)))))))))
		    (hash-table-keys results))))
		((sqlite3)
		 (let* ((db-file   (or out-file (pathname-file input-db)))
			(db-exists (common:file-exists? db-file))
			(db        (sqlite3:open-database db-file)))
		   (if (not db-exists)(sqlite3:execute db "CREATE TABLE data (sheet,section,var,val);"))
		   (configf:map-all-hier-alist
		    data
		    (lambda (sheetname sectionname varname val)
		      (sqlite3:execute db
				       "INSERT OR REPLACE INTO data (sheet,section,var,val) VALUES (?,?,?,?);"
				       sheetname sectionname varname val)))
		   (sqlite3:finalize! db)))
		(else
		 (pp data))))))
      (if out-file (close-output-port out-port))
      (exit) ;; yes, bending the rules here - need to exit since this is a utility
      ))

(if (args:get-arg "-ping")
    (let* ((server-id     (string->number (args:get-arg "-ping"))) ;; extract run-id (i.e. no ":"
	   (host:port     (args:get-arg "-ping")))
      (debug:print 0 *default-log-port* "NOT YET REIMPLEMENTED FOR TCP/INMEM") ;; bug
      (exit)))
      ;; (server:ping (or server-id host:port) #f do-exit: #t)))

;;======================================================================
;; Capture, save and manipulate environments
;;======================================================================

;; NOTE: Keep these above the section where the server or client code is setup

(let ((envcap (args:get-arg "-envcap")))
  (if envcap
      (let* ((db      (env:open-db (if (null? remargs) "envdat.db" (car remargs)))))
	(env:save-env-vars db envcap)
	(env:close-database db)
	(set! *didsomething* #t))))

;; delta "language" will eventually be res=a+b-c but for now it is just res=a-b 
;;
(let ((envdelta (args:get-arg "-envdelta")))
  (if envdelta
      (let ((match (string-split envdelta "-")));; (string-match "([a-z0-9_]+)=([a-z0-9_\\-,]+)" envdelta)))
	(if (not (null? match))
	    (let* ((db        (env:open-db (if (null? remargs) "envdat.db" (car remargs))))
		   ;; (resctx    (cadr match))
		   ;; (equn      (caddr match))
		   (parts     match) ;; (string-split equn "-"))
		   (minuend   (car parts))
		   (subtraend (cadr parts))
		   (added     (env:get-added   db minuend subtraend))
		   (removed   (env:get-removed db minuend subtraend))
		   (changed   (env:get-changed db minuend subtraend)))
	      ;; (pp (hash-table->alist added))
	      ;; (pp (hash-table->alist removed))
	      ;; (pp (hash-table->alist changed))
	      (if (args:get-arg "-o")
		  (with-output-to-file
		      (args:get-arg "-o")
		    (lambda ()
		      (env:print added removed changed)))
		  (env:print added removed changed))
	      (env:close-database db)
	      (set! *didsomething* #t))
	    (debug:print-error 0 *default-log-port* "Parameter to -envdelta should be new=start-end")))))

;;======================================================================
;; Start the server - can be done in conjunction with -runall or -runtests (one day...)
;;   we start the server if not running else start the client thread
;;======================================================================

;; Server? Start up here.
;;
(if (args:get-arg "-server")
    (let* (;; (run-id     (args:get-arg "-run-id"))
	   (dbfname    (args:get-arg "-db"))
	   (tl         (launch:setup))
	   (keys       (keys:config-get-fields *configdat*)))
      (case (rmt:transport-mode)
	((tcp)
	 (let* ((timeout    (server:expiration-timeout)))
	   (debug:print 0 *default-log-port* "INFO: megatest -server starting on " (get-host-name) " for " dbfname " using tcp method with timeout of "timeout)
	   (tt-server-timeout-param timeout)
	   (api:queue-processor)
	   (thread-start! (make-thread api:print-db-stats "print-db-stats"))
	   (if dbfname
	       (tt:start-server tl #f dbfname api:tcp-dispatch-request-make-handler keys)
	       (begin
		 (debug:print 0 *default-log-port* "ERROR: transport mode is tcp - -db is required.")
		 (exit 1)))))
	((nfs)(debug:print 0 *default-log-port* "WARNING: server start called in nfs mode '"(rmt:transport-mode)))
	(else (debug:print 0 *default-log-port* "ERROR: rmt:transport-mode value not recognised "(rmt:transport-mode))))
      (set! *didsomething* #t)))

;; The adjutant is a bit different, it does NOT run (launch:setup) as it is not necessarily tied to
;; a specific Megatest area. Detail are being hashed out and this may change.
;;
(if (args:get-arg "-adjutant")
    (begin
      (adjutant-run)
      (set! *didsomething* #t)))

(if (args:get-arg "-list-servers")
  (let* ((tl (launch:setup)) ;; need this to initialize *toppath*
        (servdir (tt:get-servinfo-dir *toppath*))
        (servfiles (glob (conc servdir "/*:*.db")))
        (fmtstr  "~10a~22a~10a~25a~25a~8a\n")
        (dbfiles (append (glob (conc *toppath* "/.mtdb/main.db")) (glob (conc *toppath* "/.mtdb/?.db"))(glob (conc *toppath* "/.mtdb/??.db"))))
        (ttdat (make-tt areapath: *toppath*))
     )
     (format #t fmtstr "DB" "host:port" "PID" "age" "last mod" "state")
     (for-each
        (lambda (dbfile)
          (let* (
            (dbfname (conc (pathname-file dbfile) ".db"))
            (sfiles   (tt:find-server *toppath* dbfname))
            )
            (for-each 
              (lambda (sfile)
                (let (
                  (sinfos (tt:get-server-info-sorted ttdat dbfname))
                  )
                  (for-each 
                     (lambda (sinfo)
                       (let* (
                         (db (list-ref sinfo 5))
                         (pid (list-ref sinfo 4))
                         (host (list-ref sinfo 0))
                         (port (list-ref sinfo 1))
                         (server-id (list-ref sinfo 3))
                         (age (seconds->hr-min-sec (- (current-seconds) (list-ref sinfo 2))))
                         (last-mod (seconds->string (list-ref sinfo 2)))
                         (status (system (conc "ssh " host " ps " pid " > /dev/null")))
                         (state (if (> status 0)
                                  "dead"
                                  (tt:ping host port server-id 0)
                                ))
                         )
                         (format #t fmtstr db (conc host ":" port) pid age last-mod state)
                       )
                     )
                     sinfos
                  )
                ) 
              )
              sfiles
            )
          )
       )
       dbfiles
     )
     (set! *didsomething* #t)
     (exit)  
  )
)




(if (args:get-arg "-kill-servers")
  
  (let* ((tl (launch:setup)) ;; need this to initialize *toppath*
        (servdir (tt:get-servinfo-dir *toppath*))
        (servfiles (glob (conc servdir "/*:*.db")))
        (fmtstr  "~10a~22a~10a~25a~25a~8a\n")
        (dbfiles (if (file-exists? (conc *toppath* "/.mtdb/main.db")) (append (glob (conc *toppath* "/.mtdb/main.db")) (glob (conc *toppath* "/.mtdb/?.db"))(glob (conc *toppath* "/.mtdb/??.db"))) '()))
        (ttdat (make-tt areapath: *toppath*))
     )
     (format #t fmtstr "DB" "host:port" "PID" "age" "last mod" "state")
     (for-each
        (lambda (dbfile)
          (let* (
            (dbfname (conc (pathname-file dbfile) ".db"))
            (sfiles   (tt:find-server *toppath* dbfname))
            )
            (for-each 
              (lambda (sfile)
                (let (
                  (sinfos (tt:get-server-info-sorted ttdat dbfname))
                  )
                  (for-each 
                     (lambda (sinfo)
                       (let* (
                         (db (list-ref sinfo 5))
                         (pid (list-ref sinfo 4))
                         (host (list-ref sinfo 0))
                         (port (list-ref sinfo 1))
                         (server-id (list-ref sinfo 3))
                         (age (seconds->hr-min-sec (- (current-seconds) (list-ref sinfo 2))))
                         (last-mod (seconds->string (list-ref sinfo 2)))
                         (killed (system (conc "ssh " host " kill " pid " > /dev/null")))
                         (dummy2 (sleep 1))
                         (state (if (> (system (conc "ssh " host " ps " pid " > /dev/null")) 0) "dead" "alive"))
                            )
                         (format #t fmtstr db (conc host ":" port) pid age last-mod state)
                         (system (conc "rm " sfile))
                       )
                     )
                     sinfos
                  )
                ) 
              )
              sfiles
            )
          )
       )
       dbfiles
     )
     ;; remove this db, because otherwise metadata contains records for old servers, and this causes a problem with db:no-sync-get-lock-with-id.
     (if (file-exists? (conc *toppath* "/.mtdb/no-sync.db"))
       (delete-file (conc *toppath* "/.mtdb/no-sync.db"))
     )
     (set! *didsomething* #t)
     (exit)  
  )
)

;;======================================================================
;; Weird special calls that need to run *after* the server has started?
;;======================================================================

(if (args:get-arg "-list-targets")
    (if (launch:setup)
        (let ((targets (common:get-runconfig-targets)))
          ;; (debug:print 1 *default-log-port* "Found "(length targets) " targets")
          (case (string->symbol (or (args:get-arg "-dumpmode") "alist"))
            ((alist)
             (for-each (lambda (x)
                         ;; (print "[" x "]"))
                         (print x))
                       targets))
            ((json)
             (json-write targets))
            (else
             (debug:print-error 0 *default-log-port* "dump output format " (args:get-arg "-dumpmode") " not supported for -list-targets")))
          (set! *didsomething* #t))))

(if (args:get-arg "-show-runconfig")
    (let ((tl (launch:setup)))
      (push-directory *toppath*)
      (let ((data (full-runconfigs-read)))
	;; keep this one local
	(cond
	 ((and (args:get-arg "-section")
	       (args:get-arg "-var"))
	  (let ((val (or (configf:lookup data (args:get-arg "-section")(args:get-arg "-var"))
			 (configf:lookup data "default" (args:get-arg "-var")))))
	    (if val (print val))))
	 ((or (not (args:get-arg "-dumpmode"))
              (string=? (args:get-arg "-dumpmode") "ini"))
	  (configf:config->ini data))
	 ((string=? (args:get-arg "-dumpmode") "sexp")
	  (pp (hash-table->alist data)))
	 ((string=? (args:get-arg "-dumpmode") "json")
	  (json-write data))
	 (else
	  (debug:print-error 0 *default-log-port* "-dumpmode of " (args:get-arg "-dumpmode") " not recognised")))
	(set! *didsomething* #t))
      (pop-directory)))

(if (args:get-arg "-show-config")
    (let ((tl   (launch:setup))
	  (data *configdat*)) ;; (read-config "megatest.config" #f #t)))
      (push-directory *toppath*)
      ;; keep this one local
      (cond 
       ((and (args:get-arg "-section")
	     (args:get-arg "-var"))
	(let ((val (configf:lookup data (args:get-arg "-section")(args:get-arg "-var"))))
	  (if val (print val))))

       ;; print just a section if only -section

       ((equal? (args:get-arg "-dumpmode") "sexp")
	(pp (hash-table->alist data)))
       ((equal? (args:get-arg "-dumpmode") "json")
	(json-write data))
       ((or (not (args:get-arg "-dumpmode"))
	    (string=? (args:get-arg "-dumpmode") "ini"))
	(configf:config->ini data))
       (else
	(debug:print-error 0 *default-log-port* "-dumpmode of " (args:get-arg "-dumpmode") " not recognised")))
      (set! *didsomething* #t)
      (pop-directory)
      (set! *time-to-exit* #t)))

(if (args:get-arg "-show-cmdinfo")
    (if (or (args:get-arg ":value")(getenv "MT_CMDINFO"))
	(let ((data (common:read-encoded-string (or (args:get-arg ":value")(getenv "MT_CMDINFO")))))
	  (if (equal? (args:get-arg "-dumpmode") "json")
	      (json-write data)
	      (pp data))
	  (set! *didsomething* #t))
	(debug:print-info 0 *default-log-port* "environment variable MT_CMDINFO is not set")))

;;======================================================================
;; Remove old run(s)
;;======================================================================

;; since several actions can be specified on the command line the removal
;; is done first
(define (operate-on action #!key (mode #f)(target-in #f)(runname-in #f)(keys-in #f)(keyvals-in #f)) ;; #f is "use default"
  (let* ((runrec (runs:runrec-make-record))
	 (target (or target-in   (common:args-get-target))) ;; eventually get rid of the call to common:args-get-target
	 (runname (or runname-in
		      (args:get-arg "-runname"))) ;; eventually get rid of the get-arg calls
	 (testpatt (or (args:get-arg "-testpatt")
		       (and (eq? action 'archive) ;; if it is an archive command fallback to MT_TEST_NAME and MT_ITEMPATH
			    (common:get-full-test-name))
		       (and (eq? action 'kill-runs)
			    "%/%") ;; I'm just guessing that this is correct :(
		       (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")))
		       ))) ;;
    (cond
     ((not target)
      (debug:print-error 0 *default-log-port* "Missing required parameter for "
			 action ", you must specify -target or -reqtarg")
      (exit 1))
     ((not runname)
      (debug:print-error 0 *default-log-port* "Missing required parameter for "
			 action ", you must specify the run name pattern with -runname patt")
      (exit 2))
     ((not testpatt)
      (debug:print-error 0 *default-log-port* "Missing required parameter for "
			 action ", you must specify the test pattern with -testpatt")
      (exit 3))
     (else
      (if (not (car *configinfo*))
	  (begin
	    (debug:print-error 0 *default-log-port* "Attempted " action "on test(s) but run area config file not found")
	    (exit 1))
	  ;; put test parameters into convenient variables
	  (begin
	    ;; check for correct version, exit with message if not correct
	    (common:exit-on-version-changed)
	    (runs:operate-on  action
			      target
			      runname
			      testpatt
			      state:  (common:args-get-state)
			      status: (common:args-get-status)
			      new-state-status: (args:get-arg "-set-state-status")
                              mode: mode)))
      (set! *didsomething* #t)))))

(if (args:get-arg "-kill-runs")
    (general-run-call 
     "-kill-runs"
     "kill runs"
     (lambda (target runname keys keyvals)
       (operate-on 'kill-runs mode: #f)
       )))

(import csi)
(if (args:get-arg "-kill-rerun")
    (let* ((target-patt (common:args-get-target))
           (runname-patt (args:get-arg "-runname")))
      (cond ((not target-patt)
             (debug:print-error 0 *default-log-port* "Missing target, must specify target for -kill-rerun with -target <target name>")
             (exit 1))
            ((not runname-patt)
             (debug:print-error 0 *default-log-port* "Missing runname, must specify runname for -kill-rerun with -runname <run name>")
             (exit 1))
            ((string-search "[ ,%]" target-patt)
             (debug:print-error 0 *default-log-port* "Invalid target ["target-patt"], must specify exact target (no wildcards) for -kill-rerun with -target <target name>")
             (exit 1))
            ((string-search "[ ,%]" runname-patt)
             (debug:print-error 0 *default-log-port* "Invalid runname ["runname-patt"], must specify exact runname (no wildcards) for -kill-rerun with -runname <runname name>")
             (exit 1))
            (else
             (general-run-call 
              "-kill-runs"
              "kill runs"
              (lambda (target runname keys keyvals)
                (operate-on 'kill-runs mode: #f)
                ))
      
;; fake out readline usage of toplevel-command
             (thread-sleep! 15))
            ;; fall thru and let "-run" loop fire
            )))

(set! toplevel-command (lambda (a b) #f))

(if (args:get-arg "-remove-runs")
    (general-run-call 
     "-remove-runs"
     "remove runs"
     (lambda (target runname keys keyvals)
       (operate-on 'remove-runs mode: (if (args:get-arg "-keep-records")
                                          'remove-data-only
                                          'remove-all)))))

(if (args:get-arg "-remove-keep")
    (general-run-call 
     "-remove-keep"
     "remove keep"
     (lambda (target runname keys keyvals)
       (let ((actions (map string->symbol
                           (string-split
			    (or (args:get-arg "-actions")
				"print")
			    ",")))) ;; default to printing the output
         (runs:remove-all-but-last-n-runs-per-target target runname
						     (string->number (args:get-arg "-remove-keep"))
						     actions: actions)))))

(if (args:get-arg "-set-state-status")
    (general-run-call 
     "-set-state-status"
     "set state and status"
     (lambda (target runname keys keyvals)
       (operate-on 'set-state-status))))

(if (or (args:get-arg "-set-run-status")
	(args:get-arg "-get-run-status"))
    (general-run-call
     "-set-run-status"
     "set run status"
     (lambda (target runname keys keyvals)
       (let* ((runsdat  (rmt:get-runs-by-patt keys runname 
					(common:args-get-target)
					#f #f #f #f))
	      (header   (vector-ref runsdat 0))
	      (rows     (vector-ref runsdat 1)))
	 (if (null? rows)
	     (begin
	       (debug:print-info 0 *default-log-port* "No matching run found.")
	       (exit 1))
	     (let* ((row      (car (vector-ref runsdat 1)))
		    (run-id   (db:get-value-by-header row header "id")))
	       (if (args:get-arg "-set-run-status")
		   (rmt:set-run-status run-id (args:get-arg "-set-run-status") msg: (args:get-arg "-m"))
		   (print (rmt:get-run-status run-id))
		   )))))))

;;======================================================================
;; Query runs
;;======================================================================

;; -fields runs:id,target,runname,comment+tests:id,testname,item_path+steps
;;
;; csi> (extract-fields-constraints "runs:id,target,runname,comment+tests:id,testname,item_path+steps")
;;         => (("runs" "id" "target" "runname" "comment") ("tests" "id" "testname" "item_path") ("steps"))
;;
;;   NOTE: remember that the cdr will be the list you expect (cdr ("runs" "id" "target" "runname" "comment")) => ("id" "target" "runname" "comment")
;;         and so alist-ref will yield what you expect
;;
(define (extract-fields-constraints fields-spec)
  (map (lambda (table-spec) ;; runs:id,target,runname
	 (let ((dat (string-split table-spec ":"))) ;; ("runs" "id,target,runname")
	   (if (> (length dat) 1)
	       (cons (car dat)(string-split (cadr dat) ",")) ;; "id,target,runname"
	       dat)))
       (string-split fields-spec "+")))

(define (get-value-by-fieldname datavec test-field-index fieldname)
  (let ((indx (hash-table-ref/default test-field-index fieldname #f)))
    (if indx
	(if (>= indx (vector-length datavec))
	    #f ;; index too high, should raise an error I suppose
	    (vector-ref datavec indx))
	#f)))





(when (args:get-arg "-testdata-csv")
  (if (launch:setup)
      (let* ((keys        (rmt:get-keys)) ;; (db:get-keys dbstruct))
             (runpatt     (or (args:get-arg "-runname") "%"))
             (testpatt    (common:args-get-testpatt #f))
             (datapatt    (args:get-arg "-testdata-csv"))
             (match-data  (string-match "^([^/]+)/(.*)" (args:get-arg "-testdata-csv")))
             (categorypatt (if match-data (list-ref match-data 1) "%"))
             (setvarpatt  (if match-data
                              (list-ref match-data 2)
                              (args:get-arg "-testdata-csv")))
             (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") 
                                                (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
             (header      (db:get-header runsdat))
             (access-mode (db:get-access-mode))
             (testpatt    (common:args-get-testpatt #f))
             (fields-spec (if (args:get-arg "-fields")
                              (extract-fields-constraints (args:get-arg "-fields"))
                              (list (cons "runs" (append keys (list "id" "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count")))
                                    (cons "tests"  db:test-record-fields) ;; "id" "testname" "test_path")
                                    (list "steps" "id" "stepname"))))
             (tests-spec  (let ((t (alist-ref "tests" fields-spec equal?)))
                            (if (and t (null? t)) ;; all fields
                                db:test-record-fields
                                t)))
             (adj-tests-spec (delete-duplicates (if tests-spec (cons "id" tests-spec) db:test-record-fields))) 
             (test-field-index (make-hash-table))
             (runs (db:get-rows runsdat))
             )
        (if (and tests-spec (not (null? tests-spec))) ;; do some validation and processing of the test-spec
            (let ((invalid-tests-spec (filter (lambda (x)(not (member x db:test-record-fields))) tests-spec)))
              (if (null? invalid-tests-spec)
                  ;; generate the lookup map test-field-name => index-number
                  (let loop ((hed (car adj-tests-spec))
                             (tal (cdr adj-tests-spec))
                             (idx 0))
                    (hash-table-set! test-field-index hed idx)
                    (if (not (null? tal))(loop (car tal)(cdr tal)(+ idx 1))))
                  (begin
                    (debug:print-error 0 *default-log-port* "Invalid test fields specified: " (string-intersperse invalid-tests-spec ", "))
                    (exit)))))
        (let* ((table-header (string-split "target,run,test,itempath,category,var,value,comment" ","))
               (table-rows
                (apply append (map  
                               (lambda (run)
                                 (let* ((target (string-intersperse (map (lambda (x)
							 (db:get-value-by-header run header x))
						       keys) "/"))
                                        (statuses (string-split (or (args:get-arg "-status") "") ","))
                                        (run-id  (db:get-value-by-header run header "id"))
                                        (runname (db:get-value-by-header run header "runname")) 
                                        (states  (string-split (or (args:get-arg "-state") "") ","))
                                        (tests   (if tests-spec
                                                     (rmt:get-tests-for-run run-id testpatt states statuses #f #f #f 'testname 'asc ;; (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc 
                                                                        ;; use qryvals if test-spec provided
                                                                        (if tests-spec
                                                                            (string-intersperse adj-tests-spec ",")
                                                                            ;; db:test-record-fields
                                                                            #f)
                                                                        #f
                                                                        'normal)
                                                     '())))
                                   (apply append
                                          (map
                                           (lambda (test)
                                             (let* (
                                                    (test-id      (if (member "id"           tests-spec)(get-value-by-fieldname test test-field-index "id"          ) #f)) ;; (db:test-get-id         test))
                                                    (testname     (if (member "testname"     tests-spec)(get-value-by-fieldname test test-field-index "testname"    ) #f)) ;; (db:test-get-testname   test))
                                                    (itempath     (if (member "item_path"    tests-spec)(get-value-by-fieldname test test-field-index "item_path"   ) #f)) ;; (db:test-get-item-path  test))
                                                    (fullname     (conc testname
                                                                        (if (equal? itempath "")
                                                                            "" 
                                                                            (conc "/" itempath ))))
                                                    (testdat-raw (map vector->list (rmt:read-test-data-varpatt run-id test-id categorypatt setvarpatt)))
                                                    (testdat (filter
                                                              (lambda (x)
                                                                (not (equal? "logpro"
                                                                             (list-ref x 10))))
                                                              testdat-raw)))
                                               (map 
                                                (lambda (item)
                                                  (receive (id test_id category
                                                               variable value expected
                                                               tol units comment status type)
                                                      (apply values item)
                                                    (list target runname testname itempath category variable value comment)))
                                                testdat)))
                                           tests))))
                               runs))))
          (print (string-join table-header ","))
          (for-each (lambda(table-row)
                      (print (string-join (map ->string table-row) ",")))

                    
                            table-rows))))
  (set! *didsomething* #t)
  (set! *time-to-exit* #t))



;; NOTE: list-runs and list-db-targets operate on local db!!!
;;
;; IDEA: megatest list -runname blah% ...
;;
(if (or (args:get-arg "-list-runs")
	(args:get-arg "-list-db-targets"))
    (if (launch:setup)
	(let* ((runpatt     (args:get-arg "-list-runs"))
               (access-mode (db:get-access-mode))
	       (testpatt    (common:args-get-testpatt #f))
	       ;; (if (args:get-arg "-testpatt") 
	       ;;  	        (args:get-arg "-testpatt") 
	       ;;  	        "%"))
	       (keys        (rmt:get-keys)) ;; (db:get-keys dbstruct))
	       ;; (runsdat  (db:get-runs dbstruct runpatt #f #f '()))
	;; (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") (common:args-get-target) ;; (db:get-runs-by-patt dbstruct keys (or runpatt "%") (common:args-get-target)
	;; 		           	 #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
	       (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") 
                                                  (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
	       (runstmp     (db:get-rows runsdat))
	       (header      (db:get-header runsdat))
	       ;; this is "-since" support. This looks at last mod times of <run-id>.db files
	       ;; and collects those modified since the -since time.
	       (runs        runstmp)
                        ;; (if (and (not (null? runstmp))
			;;        (args:get-arg "-since"))
			;;   (let ((changed-ids (db:get-changed-run-ids (string->number (args:get-arg "-since")))))
			;;     (let loop ((hed (car runstmp))
			;;   	     (tal (cdr runstmp))
			;;   	     (res '()))
			;;       (let ((new-res (if (member (db:get-value-by-header hed header "id") changed-ids)
			;;   		       (cons hed res)
			;;   		       res)))
			;;         (if (null? tal)
			;;   	  (reverse new-res)
			;;   	  (loop (car tal)(cdr tal) new-res)))))
			;;   runstmp))
	       (db-targets  (args:get-arg "-list-db-targets"))
	       (seen        (make-hash-table))
	       (dmode       (let ((d (args:get-arg "-dumpmode"))) ;; json, sexpr
			      (if d (string->symbol d) #f)))
	       (data        (make-hash-table))
	       (fields-spec (if (args:get-arg "-fields")
				(extract-fields-constraints (args:get-arg "-fields"))
				(list (cons "runs" (append keys (list "id" "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count")))
				      (cons "tests"  db:test-record-fields) ;; "id" "testname" "test_path")
				      (list "steps" "id" "stepname"))))
	       (runs-spec   (let ((r (alist-ref "runs"  fields-spec equal?))) ;; the check is now unnecessary
			      (if (and r (not (null? r))) r (list "id" ))))
	       (tests-spec  (let ((t (alist-ref "tests" fields-spec equal?)))
			      (if (and t (null? t)) ;; all fields
				  db:test-record-fields
				  t)))
	       (adj-tests-spec (delete-duplicates (if tests-spec (cons "id" tests-spec) db:test-record-fields))) ;; '("id"))))
	       (steps-spec  (alist-ref "steps" fields-spec equal?))
	       (test-field-index (make-hash-table)))
	  (if (and (args:get-arg "-dumpmode")
		   (not (member (args:get-arg "-dumpmode") '("sexpr" "json" "ods" "list"))))
	      (begin
		(debug:print 0 *default-log-port* "ERROR: dumpmode "(args:get-arg "-dumpmode")" not recognised. Use sexpr, json, ods or list")
		(exit)))
	  (if (and tests-spec (not (null? tests-spec))) ;; do some validation and processing of the test-spec
	      (let ((invalid-tests-spec (filter (lambda (x)(not (member x db:test-record-fields))) tests-spec)))
		(if (null? invalid-tests-spec)
		    ;; generate the lookup map test-field-name => index-number
		    (let loop ((hed (car adj-tests-spec))
			       (tal (cdr adj-tests-spec))
			       (idx 0))
		      (hash-table-set! test-field-index hed idx)
		      (if (not (null? tal))(loop (car tal)(cdr tal)(+ idx 1))))
		    (begin
		      (debug:print-error 0 *default-log-port* "Invalid test fields specified: " (string-intersperse invalid-tests-spec ", "))
		      (exit)))))
	  ;; Each run
	  (for-each 
	   (lambda (run)
	     (let ((targetstr (string-intersperse (map (lambda (x)
							 (db:get-value-by-header run header x))
						       keys) "/")))
	       (if db-targets
		   (if (not (hash-table-ref/default seen targetstr #f))
		       (begin
			 (hash-table-set! seen targetstr #t)
			 ;; (print "[" targetstr "]"))))
			 (if (not dmode)
			     (print targetstr)
			     (hash-table-set! data "targets" (cons targetstr (hash-table-ref/default data "targets" '())))
			     )))
		   (let* ((run-id  (db:get-value-by-header run header "id"))
			  (runname (db:get-value-by-header run header "runname")) 
			  (states  (string-split (or (args:get-arg "-state") "") ","))
			  (statuses (string-split (or (args:get-arg "-status") "") ","))
			  (tests   (if tests-spec
				       (rmt:get-tests-for-run run-id testpatt states statuses #f #f #f 'testname 'asc ;; (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc 
							     ;; use qryvals if test-spec provided
							     (if tests-spec
								 (string-intersperse adj-tests-spec ",")
								 ;; db:test-record-fields
								 #f)
							     #f
							     'normal)
				       '())))
		     (case dmode
		       ((json ods sexpr)
			(if runs-spec
			    (for-each 
			     (lambda (field-name)
			       (mutils:hierhash-set! data (conc (db:get-value-by-header run header field-name)) targetstr runname "meta" field-name))
			     runs-spec)))
			;; (mutils:hierhash-set! data (db:get-value-by-header run header "status")     targetstr runname "meta" "status"     )
			;; (mutils:hierhash-set! data (db:get-value-by-header run header "state")      targetstr runname "meta" "state"      )
			;; (mutils:hierhash-set! data (conc (db:get-value-by-header run header "id"))  targetstr runname "meta" "id"         )
			;; (mutils:hierhash-set! data (db:get-value-by-header run header "event_time") targetstr runname "meta" "event_time" )
			;; (mutils:hierhash-set! data (db:get-value-by-header run header "comment")    targetstr runname "meta" "comment"    )
			;; ;; add last entry twice - seems to be a bug in hierhash?
			;; (mutils:hierhash-set! data (db:get-value-by-header run header "comment")    targetstr runname "meta" "comment"    )
		       ((#f list)
			(if (null? runs-spec)
			    (print "Run: " targetstr "/" runname 
				   " status: " (db:get-value-by-header run header "state")
				   " run-id: " run-id ", number tests: " (length tests)
				   " event_time: " (db:get-value-by-header run header "event_time"))
			    (begin
			      (if (not (member "target" runs-spec))
			          ;; (display (conc "Target: " targetstr))
			          (display (conc "Run: " targetstr "/" runname " ")))
			      (for-each
			       (lambda (field-name)
				 (if (equal? field-name "target")
				     (display (conc "target: " targetstr " "))
				     (display (conc field-name ": " (db:get-value-by-header run header (conc field-name)) " "))))
			       runs-spec)
			      (newline))))
		       (else
			(debug:print 0 *default-log-port* "ERROR: dumpmode "(args:get-arg "-dumpmode")" not recognised. Use sexpr, json, ods or list")
			))
		       
		     (for-each 
		      (lambda (test)
		      	(common:debug-handle-exceptions #f
			 exn
			 (begin
			   (debug:print-error 0 *default-log-port* "Bad data in test record? " test)
			   (debug:print-error 5 *default-log-port* "exn=" (condition->list exn))
			   (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
			   (print-call-chain (current-error-port)))
			 (let* ((test-id      (if (member "id"           tests-spec)(get-value-by-fieldname test test-field-index "id"          ) #f)) ;; (db:test-get-id         test))
				(testname     (if (member "testname"     tests-spec)(get-value-by-fieldname test test-field-index "testname"    ) #f)) ;; (db:test-get-testname   test))
				(itempath     (if (member "item_path"    tests-spec)(get-value-by-fieldname test test-field-index "item_path"   ) #f)) ;; (db:test-get-item-path  test))
				(comment      (if (member "comment"      tests-spec)(get-value-by-fieldname test test-field-index "comment"     ) #f)) ;; (db:test-get-comment    test))
				(tstate       (if (member "state"        tests-spec)(get-value-by-fieldname test test-field-index "state"       ) #f)) ;; (db:test-get-state      test))
				(tstatus      (if (member "status"       tests-spec)(get-value-by-fieldname test test-field-index "status"      ) #f)) ;; (db:test-get-status     test))
				(event-time   (if (member "event_time"   tests-spec)(get-value-by-fieldname test test-field-index "event_time"  ) #f)) ;; (db:test-get-event_time test))
				(rundir       (if (member "rundir"       tests-spec)(get-value-by-fieldname test test-field-index "rundir"      ) #f)) ;; (db:test-get-rundir     test))
				(final_logf   (if (member "final_logf"   tests-spec)(get-value-by-fieldname test test-field-index "final_logf"  ) #f)) ;; (db:test-get-final_logf test))
				(run_duration (if (member "run_duration" tests-spec)(get-value-by-fieldname test test-field-index "run_duration") #f)) ;; (db:test-get-run_duration test))
				(fullname     (conc testname
						    (if (equal? itempath "")
							"" 
							(conc "(" itempath ")")))))
			   (case dmode
			     ((json ods sexpr)
			      (if tests-spec
				  (for-each
				   (lambda (field-name)
				     (mutils:hierhash-set! data  (get-value-by-fieldname test test-field-index field-name) targetstr runname "data" (conc test-id) field-name))
				   tests-spec)))
			     ;; ;; (mutils:hierhash-set! data  fullname   targetstr runname "data" (conc test-id) "tname"     )
			     ;;  (mutils:hierhash-set! data  testname   targetstr runname "data" (conc test-id) "testname"  )
			     ;;  (mutils:hierhash-set! data  itempath   targetstr runname "data" (conc test-id) "itempath"  )
			     ;;  (mutils:hierhash-set! data  comment    targetstr runname "data" (conc test-id) "comment"   )
			     ;;  (mutils:hierhash-set! data  tstate     targetstr runname "data" (conc test-id) "state"     )
			     ;;  (mutils:hierhash-set! data  tstatus    targetstr runname "data" (conc test-id) "status"    )
			     ;;  (mutils:hierhash-set! data  rundir     targetstr runname "data" (conc test-id) "rundir"    )
			     ;;  (mutils:hierhash-set! data  final_logf targetstr runname "data" (conc test-id) "final_logf")
			     ;;  (mutils:hierhash-set! data  run_duration targetstr runname "data" (conc test-id) "run_duration")
			     ;;  (mutils:hierhash-set! data  event-time targetstr runname "data" (conc test-id) "event_time")
			     ;;  ;; add last entry twice - seems to be a bug in hierhash?
			     ;;  (mutils:hierhash-set! data  event-time targetstr runname "data" (conc test-id) "event_time")
			     ;;  )
			     (else
			      (if (and tstate tstatus event-time)
				  (format #t
					  "  Test: ~25a State: ~15a Status: ~15a Runtime: ~5@as Time: ~22a Host: ~10a\n"
					  (if fullname fullname "")
					  (if tstate   tstate   "")
					  (if tstatus  tstatus  "")
					  (get-value-by-fieldname test test-field-index "run_duration");;(if test     (db:test-get-run_duration test) "")
					  (if event-time event-time "")
					  (get-value-by-fieldname test test-field-index "host")) ;;(if test (db:test-get-host test)) "")
				  (print "  Test: " fullname
					 (if tstate  (conc " State: "  tstate)  "")
					 (if tstatus (conc " Status: " tstatus) "")
					 (if (get-value-by-fieldname test test-field-index "run_duration")
					     (conc " Runtime: " (get-value-by-fieldname test test-field-index "run_duration"))
					     "")
					 (if event-time (conc " Time: " event-time) "")
					 (if (get-value-by-fieldname test test-field-index "host")
					     (conc " Host: " (get-value-by-fieldname test test-field-index "host"))
					     "")))
			      (if (not (or (equal? (get-value-by-fieldname test test-field-index "status") "PASS")
					   (equal? (get-value-by-fieldname test test-field-index "status") "WARN")
					   (equal? (get-value-by-fieldname test test-field-index "state")  "NOT_STARTED")))
				  (begin
				    (print   (if (get-value-by-fieldname test test-field-index "cpuload")
						 (conc "         cpuload:  "   (get-value-by-fieldname test test-field-index "cpuload"))
						 "") ;; (db:test-get-cpuload test)
					     (if (get-value-by-fieldname test test-field-index "diskfree")
						 (conc "\n         diskfree: " (get-value-by-fieldname test test-field-index "diskfree")) ;; (db:test-get-diskfree test)
						 "")
					     (if (get-value-by-fieldname test test-field-index "uname")
						 (conc "\n         uname:    " (get-value-by-fieldname test test-field-index "uname")) ;; (db:test-get-uname test)
						 "")
					     (if (get-value-by-fieldname test test-field-index "rundir")
						 (conc "\n         rundir:   " (get-value-by-fieldname test test-field-index "rundir")) ;; (db:test-get-rundir test)
						 "")
;;					     "\n         rundir:   " (get-value-by-fieldname test test-field-index "") ;; (sdb:qry 'getstr ;; (filedb:get-path *fdb* 
;; 					     (db:test-get-rundir test) ;; )
					     )
				    ;; Each test
				    ;; DO NOT remote run
				    (let ((steps (rmt:get-steps-for-test run-id (db:test-get-id test)))) ;; (db:get-steps-for-test dbstruct run-id (db:test-get-id test))))
				      (for-each 
				       (lambda (step)
					 (format #t 
						 "    Step: ~20a State: ~10a Status: ~10a Time ~22a\n"
						 (tdb:step-get-stepname step)
						 (tdb:step-get-state step)
						 (tdb:step-get-status step)
						 (tdb:step-get-event_time step)))
				       steps)))))))))
		      (if (args:get-arg "-sort")
			  (sort tests
				(lambda (a-test b-test)
				  (let* ((key    (args:get-arg "-sort"))
					 (first  (get-value-by-fieldname a-test test-field-index key))
					 (second (get-value-by-fieldname b-test test-field-index key)))
				    ((cond 
				      ((and (number? first)(number? second)) <)
				      ((and (string? first)(string? second)) string<=?)
				      (else equal?))
				     first second))))
			  tests))))))
	   runs)
	  (case dmode
	    ((json)  (json-write data))
	    ((sexpr) (pp (common:to-alist data))))
	  (let* ((metadat-fields (delete-duplicates
				  (append keys '( "runname" "time" "owner" "pass_count" "fail_count" "state" "status" "comment" "id"))))
		 (run-fields    '(
				  "testname"
				  "item_path"
				  "state"
				  "status"
				  "comment"
				  "event_time"
				  "host"
				  "run_id"
				  "run_duration"
				  "attemptnum"
				  "id"
				  "archived"
				  "diskfree"
				  "cpuload"
				  "final_logf"
				  "shortdir"
				  "rundir"
				  "uname"
				  )
				)
		 (newdat          (common:to-alist data))
		 (allrundat       (if (null? newdat)
				      '()
				      (car (map cdr newdat)))) ;; (car (map cdr (car (map cdr newdat)))))
		 (runs            (append
				   (list "runs" ;; sheetname
					 metadat-fields)
				   (map (lambda (run)
					  ;; (print "run: " run)
					  (let* ((runname (car run))
						 (rundat  (cdr run))
						 (metadat (let ((tmp (assoc "meta" rundat)))
							    (if tmp (cdr tmp) #f))))
					    ;; (print "runname: " runname "\n\nrundat: " )(pp rundat)(print "\n\nmetadat: ")(pp metadat)
					    (if metadat
						(map (lambda (field)
						       (let ((tmp (assoc field metadat)))
							 (if tmp (cdr tmp) "")))
						     metadat-fields)
						(begin
						  (debug:print 0 *default-log-port* "WARNING: meta data for run " runname " not found")
						  '()))))
					allrundat)))
		 ;; '( ( "target" ( "runname" ( "data" ( "runid" ( "id . "37" ) ( ... ))))
		 (run-pages      (map (lambda (targdat)
					(let* ((target  (car targdat))
					       (runsdat (cdr targdat)))
					  (if runsdat
					      (map (lambda (rundat)
						     (let* ((runname  (car rundat))
							    (rundat   (cdr rundat))
							    (testsdat (let ((tmp (assoc "data" rundat)))
									(if tmp (cdr tmp) #f))))
						       (if testsdat
							   (let ((tests (map (lambda (test)
									       (let* ((test-id  (car test))
										      (test-dat (cdr test)))
										 (map (lambda (field)
											(let ((tmp (assoc field test-dat)))
											  (if tmp (cdr tmp) "")))
										      run-fields)))
									     testsdat)))
							     ;; (print "Target: " target "/" runname " tests:")
							     ;; (pp tests)
							     (cons (conc target "/" runname)
								   (cons (list (conc target "/" runname))
									 (cons '()
									       (cons run-fields tests)))))
							   (begin
							     (debug:print 4 *default-log-port* "WARNING: run " target "/" runname " appears to have no data")
							     ;; (pp rundat)
							     '()))))
						   runsdat)
					      '())))
				      newdat)) ;; we use newdat to get target
		 (sheets         (filter (lambda (x)
					   (not (null? x)))
					 (cons runs (map car run-pages)))))
	    ;; (print "allrundat:")
	    ;; (pp allrundat)
	    ;; (print "runs:")
	    ;; (pp runs)
	    ;(print "sheets: ")
	    ;; (pp sheets)
	    (if (eq? dmode 'ods)
		(let* ((tempdir    (conc "/tmp/" (current-user-name) "/" (random 10000) "_" (current-process-id)))
		       (outputfile (or (args:get-arg "-o") "out.ods"))
		       (ouf        (if (string-match (regexp "^[/~]+.*") outputfile) ;; full path?
				       outputfile
				       (begin
					 (debug:print 0 *default-log-port* "WARNING: path given, " outputfile " is relative, prefixing with current directory")
					 (conc (current-directory) "/" outputfile)))))
		  (create-directory tempdir #t)
		  (ods:list->ods tempdir ouf sheets))))
	  ;; (system (conc "rm -rf " tempdir))
	  (set! *didsomething* #t)
          (set! *time-to-exit* #t)
          ) ;; end if true branch (end of a let)
        ) ;; end if
    ) ;; end if -list-runs

;; list-waivers
(if (and (args:get-arg "-list-waivers")
	 (launch:setup))
    (let* ((runpatt     (or (args:get-arg "-runname") "%"))
	   (testpatt    (common:args-get-testpatt #f))
	   (keys        (rmt:get-keys)) 
	   (runsdat     (rmt:get-runs-by-patt
			 keys runpatt 
			 (common:args-get-target) #f #f
			 '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
	   (runs        (db:get-rows runsdat))
	   (header      (db:get-header runsdat))
	   (results     (make-hash-table))  ;; [target] ( (testname/itempath . "comment") ... )
	   (addtest     (lambda (target testname itempath comment)
			  (hash-table-set! results target (cons (cons (conc testname "/" itempath) comment)
								(hash-table-ref/default results target '())))))
	   (last-target #f))
      (for-each
       (lambda (run)
	 (let* ((run-id  (db:get-value-by-header run header "id"))
		(target  (rmt:get-target run-id))
		(runname (db:get-value-by-header run header "runname")) 
		(tests   (rmt:get-tests-for-run
			  run-id testpatt '("COMPLETED") '("WAIVED") #f #f #f 'testname 'asc							     ;; use qryvals if test-spec provided
			  #f #f #f)))
	   (if (not (equal? target last-target))
	       (print "[" target "]"))
	   (set! last-target target)
	   (print "# " runname)
	   (for-each
	    (lambda (testdat)
	      (let* ((testfullname (conc (db:test-get-testname testdat)
					 (if (equal? "" (db:test-get-item-path testdat))
					     ""
					     (conc "/" (db:test-get-item-path testdat)))
					 )))
	      (print testfullname " " (db:test-get-comment testdat))))
	    tests)))
       runs)
      (set! *didsomething* #t)))
      
(import mtbody)
;;======================================================================
;; full run
;;======================================================================

(define (handle-run-requests target runname keys keyvals need-clean)	 
  (if (or (args:get-arg "-kill-rerun") (args:get-arg "-rerun-clean")) ;; first set states/statuses correct
      ;; For rerun-clean do we or do we not support the testpatt?
      (let ((states   (or (configf:lookup *configdat* "validvalues" "cleanrerun-states")
			  "KILLREQ,KILLED,UNKNOWN,INCOMPLETE,STUCK,NOT_STARTED"))
	    (statuses (or (configf:lookup *configdat* "validvalues" "cleanrerun-statuses")
			  "FAIL,INCOMPLETE,ABORT,CHECK,DEAD,PREQ_FAIL,PREQ_DISCARDED")))
	(hash-table-set! args:arg-hash "-preclean" #t)
	(runs:operate-on 'set-state-status
			 target
			 (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			 ;; "%" ;; (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			 (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			 state:  states
			 ;; status: statuses
			 new-state-status: "NOT_STARTED,n/a")
	(runs:clean-cache target runname *toppath*)
	(runs:operate-on 'set-state-status
			 target
			 (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			 ;; "%" ;; (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			 (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			 ;; state:  states
			 status: statuses
			 new-state-status: "NOT_STARTED,n/a")))
  ;; RERUN ALL
  (if (args:get-arg "-rerun-all") ;; first set states/statuses correct
      (let* ((rconfig (full-runconfigs-read)))
	(hash-table-set! args:arg-hash "-preclean" #t)
	(runs:operate-on 'set-state-status
			 target
			 (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			 (common:args-get-testpatt rconfig) ;; (args:get-arg "-testpatt")
			 state:  #f
			 ;; status: statuses
			 new-state-status: "NOT_STARTED,n/a")
	(runs:clean-cache target runname *toppath*)
	(runs:operate-on 'set-state-status
			 target
			 (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			 (common:args-get-testpatt rconfig) ;; (args:get-arg "-testpatt")
			 ;; state:  states
			 status: #f
			 new-state-status: "NOT_STARTED,n/a")))
  (let* ((config-reruns      (let ((x (configf:lookup *configdat* "setup" "reruns")))
			       (if x (string->number x) #f)))
	 (rerun-cnt (if config-reruns
			config-reruns
			1)))

    (runs:run-tests target
		    runname
		    #f ;; (common:args-get-testpatt #f)
		    ;; (or (args:get-arg "-testpatt")
		    ;;     "%")
		    user
		    args:arg-hash
		    run-count: rerun-cnt)))

;; get lock in db for full run for this directory
;; for all tests with deps
;;   walk tree of tests to find head tasks
;;   add head tasks to task queue
;;   add dependant tasks to task queue 
;;   add remaining tasks to task queue
;; for each task in task queue
;;   if have adequate resources
;;     launch task
;;   else
;;     put task in deferred queue
;; if still ok to run tasks
;;   process deferred tasks per above steps

;; run all tests are are Not COMPLETED and PASS or CHECK
(if (or (args:get-arg "-runall")
	(args:get-arg "-run")
	(args:get-arg "-rerun-clean")
	(args:get-arg "-rerun-all")
	(args:get-arg "-runtests")
        (args:get-arg "-kill-rerun"))
    (let ((need-clean (or (args:get-arg "-rerun-clean")
                          (args:get-arg "-rerun-all")))
	  (orig-cmdline (string-intersperse (argv) " ")))
      (general-run-call 
       "-runall"
       "run all tests"
       (lambda (target runname keys keyvals)
	 (if (or (string-search "%" target)
		 (string-search "%" runname)) ;; we are being asked to re-run multiple runs
	     (let* ((run-specs (rmt:simple-get-runs runname #f #f target #f))) ;; list of simple-run records
	       (debug:print-info 0 *default-log-port* "Pattern supplied for target or runname with "
				 (length run-specs) " matches found. Running each in turn.")
	       (if (null? run-specs)
		   (debug:print 0 *default-log-port* "WARNING: No runs match target " target " and runname " runname))
	       (for-each (lambda (spec) 
			   (let* ((precmd     (if (args:get-arg "-precmd")(conc (args:get-arg "-precmd") " ") ""))
				  (newcmdline (conc
					       precmd
					       (string-substitute
						(conc "target " target)
						(conc "target " (simple-run-target spec))
						(string-substitute
						 (conc "runname " runname)
						 (conc "runname " (simple-run-runname spec))
						 orig-cmdline)))))
			     (debug:print 0 *default-log-port* "ORIG: " orig-cmdline)
			     (debug:print 0 *default-log-port* "NEW:  " newcmdline)
			     (system newcmdline)))
			 run-specs))
	     (handle-run-requests target runname keys keyvals need-clean))))
      (set! *didsomething* #t)))

;;======================================================================
;; run one test
;;======================================================================

;; 1. find the config file
;; 2. change to the test directory
;; 3. update the db with "test started" status, set running host
;; 4. process launch the test
;;    - monitor the process, update stats in the db every 2^n minutes
;; 5. as the test proceeds internally it calls megatest as each step is
;;    started and completed
;;    - step started, timestamp
;;    - step completed, exit status, timestamp
;; 6. test phone home
;;    - if test run time > allowed run time then kill job
;;    - if cannot access db > allowed disconnect time then kill job

;; == duplicated == (if (or (args:get-arg "-run")(args:get-arg "-runtests"))
;; == duplicated ==   (general-run-call 
;; == duplicated ==    "-runtests" 
;; == duplicated ==    "run a test" 
;; == duplicated ==    (lambda (target runname keys keyvals)
;; == duplicated ==      ;;
;; == duplicated ==      ;; May or may not implement it this way ...
;; == duplicated ==      ;;
;; == duplicated ==      ;; Insert this run into the tasks queue
;; == duplicated ==      ;; (open-run-close tasks:add tasks:open-db 
;; == duplicated ==      ;;    	     "runtests" 
;; == duplicated ==      ;;    	     user
;; == duplicated ==      ;;    	     target
;; == duplicated ==      ;;    	     runname
;; == duplicated ==      ;;    	     (args:get-arg "-runtests")
;; == duplicated ==      ;;    	     #f))))
;; == duplicated ==      (runs:run-tests target
;; == duplicated == 		     runname
;; == duplicated == 		     (common:args-get-testpatt #f) ;; (args:get-arg "-runtests")
;; == duplicated == 		     user
;; == duplicated == 		     args:arg-hash))))

;;======================================================================
;; Rollup into a run
;;======================================================================

(if (args:get-arg "-rollup")
    (general-run-call 
     "-rollup" 
     "rollup tests" 
     (lambda (target runname keys keyvals)
       (runs:rollup-run keys
			keyvals
			(or (args:get-arg "-runname")(args:get-arg ":runname") )
			user))))

;;======================================================================
;; Lock or unlock a run
;;======================================================================

(if (or (args:get-arg "-lock")(args:get-arg "-unlock"))
    (general-run-call 
     (if (args:get-arg "-lock") "-lock" "-unlock")
     "lock/unlock tests" 
     (lambda (target runname keys keyvals)
       (runs:handle-locking 
		  target
		  keys
		  (or (args:get-arg "-runname")(args:get-arg ":runname") )
		  (args:get-arg "-lock")
		  (args:get-arg "-unlock")
		  user))))

;;======================================================================
;; Get paths to tests
;;======================================================================
;; Get test paths matching target, runname, and testpatt
(if (or (args:get-arg "-test-files")(args:get-arg "-test-paths"))
    ;; if we are in a test use the MT_CMDINFO data
    (if (getenv "MT_CMDINFO")
	(let* ((startingdir (current-directory))
	       (cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
	       (transport (assoc/default 'transport cmdinfo))
	       (testpath  (assoc/default 'testpath  cmdinfo))
	       (test-name (assoc/default 'test-name cmdinfo))
	       (runscript (assoc/default 'runscript cmdinfo))
	       (db-host   (assoc/default 'db-host   cmdinfo))
	       (run-id    (assoc/default 'run-id    cmdinfo))
	       (itemdat   (assoc/default 'itemdat   cmdinfo))
	       (state     (args:get-arg ":state"))
	       (status    (args:get-arg ":status"))
	       ;;(target    (args:get-arg "-target"))
	       (target    (common:args-get-target))
	       (toppath   (assoc/default 'toppath   cmdinfo)))
	  (change-directory toppath)
	  (if (not target)
	      (begin
		(debug:print-error 0 *default-log-port* "-target is required.")
		(exit 1)))
	  (if (not (launch:setup))
	      (begin
		(debug:print 0 *default-log-port* "Failed to setup, giving up on -test-paths or -test-files, exiting")
		(exit 1)))
	  (let* ((keys     (rmt:get-keys))
		 ;; db:test-get-paths must not be run remote
		 (paths    (tests:test-get-paths-matching keys target (args:get-arg "-test-files"))))
	    (set! *didsomething* #t)
	    (for-each (lambda (path)
			(if (common:file-exists? path)
			(print path)))	
		      paths)))
	;; else do a general-run-call
	(general-run-call 
	 "-test-files"
	 "Get paths to test"
	 (lambda (target runname keys keyvals)
	   (let* ((db       #f)
		  ;; DO NOT run remote
		  (paths    (tests:test-get-paths-matching keys target (args:get-arg "-test-files"))))
	     (for-each (lambda (path)
			 (print path))
		       paths))))))

;;======================================================================
;; Utils for test areas
;;======================================================================

(if (args:get-arg "-regen-testfiles")
    (if (getenv "MT_TEST_RUN_DIR")
	(begin
	  (launch:setup)
	  (change-directory (getenv "MT_TEST_RUN_DIR"))
	  (let* ((testname (getenv "MT_TEST_NAME"))
		 (itempath (getenv "MT_ITEMPATH")))
	    (launch:extract-scripts-logpro (getenv "MT_TEST_RUN_DIR") testname itempath #f))
	  (set! *didsomething* #t))
	(debug:print 0 *default-log-port* "ERROR: Must run -regen-testfiles in a test environment (i.e. test xterm from dashboard)")))
		 	  
;;======================================================================
;; Archive tests
;;======================================================================
;; Archive tests matching target, runname, and testpatt
(if (equal? (args:get-arg "-archive") "replicate-db")
    (begin
          ;; check if source
          ;; check if megatest.db exist
         (launch:setup)
         (if (not (args:get-arg "-source"))
             (begin 
             (debug:print-info 1 *default-log-port* "Missing required argument -source <archive path>")
             (exit 1)))
         (if (common:file-exists? (conc  *toppath* "/megatest.db"))
             (begin  
               (debug:print-info 1 *default-log-port* "File " (conc  *toppath* "/megatest.db") " already exists. Please remove it before trying to replicate db")
               (exit 1)))
         (if (and (common:make-tmpdir-name *toppath* "") (> (length (directory   (common:make-tmpdir-name *toppath* "") #f)) 0))
           (begin
           (debug:print-info 1 *default-log-port* (common:make-tmpdir-name *toppath* "") " not empty. Please remove it before trying to replicate db")
           (exit 1)))    
          ;; check if timestamp 
          (let* ((source (args:get-arg "-source"))
                (src     (if (not (equal? (substring source 0 1) "/"))
                             (conc (current-directory) "/" source)
                             source))
                (ts (if (args:get-arg "-time-stamp")   (args:get-arg "-time-stamp") "latest")))
              (if  (common:directory-exists? src)
                  (begin 
                  (archive:restore-db src ts)
            (set! *didsomething* #t))
       (begin
         (debug:print-error 1 *default-log-port* "Path " source " not found")
         (exit 1))))))   
    ;; else do a general-run-call
   (if (and (args:get-arg "-archive") (not (equal? (args:get-arg "-archive") "replicate-db"))) 
    (begin
      ;; for the archive get we need to preserve the starting dir as part of the target path
      (if (and (args:get-arg "-dest")
	       (not (equal? (substring (args:get-arg "-dest") 0 1) "/")))
	  (let ((newpath  (conc (current-directory) "/" (args:get-arg "-dest"))))
	    (debug:print-info 1 *default-log-port* "Preserving original path to destination, was " (args:get-arg "-dest") ", now " newpath)
	    (hash-table-set! args:arg-hash "-dest" newpath)))
      (general-run-call 
       "-archive"
       "Archive"
       (lambda (target runname keys keyvals)
	 (operate-on 'archive target-in: target runname-in: runname )))))

;;======================================================================
;; Extract a spreadsheet from the runs database
;;======================================================================

(if (args:get-arg "-extract-ods")
    (general-run-call
     "-extract-ods"
     "Make ods spreadsheet"
     (lambda (target runname keys keyvals)
       (let ((dbstruct   (make-dbr:dbstruct areapath: *toppath* local: #t))
	     (outputfile (args:get-arg "-extract-ods"))
	     (runspatt   (or (args:get-arg "-runname")(args:get-arg ":runname")))
	     (pathmod    (args:get-arg "-pathmod")))
	     ;; (keyvalalist (keys->alist keys "%")))
	 (debug:print 2 *default-log-port* "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvals: " keyvals)
	 (db:extract-ods-file dbstruct outputfile keyvals (if runspatt runspatt "%") pathmod)
	 (db:close-all dbstruct)
	 (set! *didsomething* #t)))))

;;======================================================================
;; execute the test
;;    - gets called on remote host
;;    - receives info from the -execute param
;;    - passes info to steps via MT_CMDINFO env var (future is to use a dot file)
;;    - gathers host info and 
;;======================================================================

(if (args:get-arg "-execute")
    (begin
      (launch:execute (args:get-arg "-execute"))
      (set! *didsomething* #t)))

;;======================================================================
;; recover from a test where the managing mtest was killed but the underlying
;; process might still be salvageable
;;======================================================================

(if (args:get-arg "-recover-test")
    (let* ((params (string-split (args:get-arg "-recover-test") ",")))
      (if (> (length params) 1) ;; run-id and test-id
	  (let ((run-id (string->number (car params)))
		(test-id (string->number (cadr params))))
	    (if (and run-id test-id)
		(begin
		  (launch:recover-test run-id test-id)
		  (set! *didsomething* #t))
		(begin
		  (debug:print-error 0 *default-log-port* "bad run-id or test-id, must be integers")
		  (exit 1)))))))

;;======================================================================
;; Test commands (i.e. for use inside tests)
;;======================================================================

(define (megatest:step step state status logfile msg)
  (if (not (getenv "MT_CMDINFO"))
      (begin
	(debug:print-error 0 *default-log-port* "MT_CMDINFO env var not set, -step must be called *inside* a megatest invoked environment!")
	(exit 5))
      (let* ((cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
	     (transport (assoc/default 'transport cmdinfo))
	     (testpath  (assoc/default 'testpath  cmdinfo))
	     (test-name (assoc/default 'test-name cmdinfo))
	     (runscript (assoc/default 'runscript cmdinfo))
	     (db-host   (assoc/default 'db-host   cmdinfo))
	     (run-id    (assoc/default 'run-id    cmdinfo))
	     (test-id   (assoc/default 'test-id   cmdinfo))
	     (itemdat   (assoc/default 'itemdat   cmdinfo))
	     (work-area (assoc/default 'work-area cmdinfo))
	     (db        #f))
	(change-directory testpath)
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting")
	      (exit 1)))
	(if (and state status)
	    (let ((comment (launch:load-logpro-dat run-id test-id step)))
	      ;; (rmt:test-set-log! run-id test-id (conc stepname ".html"))))
	      (rmt:teststep-set-status! run-id test-id step state status (or comment msg) logfile))
	    (begin
	      (debug:print-error 0 *default-log-port* "You must specify :state and :status with every call to -step")
	      (exit 6))))))

(if (args:get-arg "-step")
    (begin
      (thread-sleep! 1.5)
      (megatest:step 
       (args:get-arg "-step")
       (or (args:get-arg "-state")(args:get-arg ":state"))
       (or (args:get-arg "-status")(args:get-arg ":status"))
       (args:get-arg "-setlog")
       (args:get-arg "-m"))
      ;; (if db (sqlite3:finalize! db))
      (set! *didsomething* #t)
      (thread-sleep! 1.5)))
    
(if (or (args:get-arg "-setlog")       ;; since setting up is so costly lets piggyback on -test-status
	;;     (not (args:get-arg "-step")))  ;; -setlog may have been processed already in the "-step" previous
	;;     NEW POLICY - -setlog sets test overall log on every call.
	(args:get-arg "-set-toplog")
	(args:get-arg "-test-status")
	(args:get-arg "-set-values")
	(args:get-arg "-load-test-data")
	(args:get-arg "-runstep")
	(args:get-arg "-summarize-items"))
    (if (not (getenv "MT_CMDINFO"))
	(begin
	  (debug:print-error 0 *default-log-port* "MT_CMDINFO env var not set, commands -test-status, -runstep and -setlog must be called *inside* a megatest environment!")
	  (exit 5))
	(let* ((startingdir (current-directory))
	       (cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
	       (transport (assoc/default 'transport cmdinfo))
	       (testpath  (assoc/default 'testpath  cmdinfo))
	       (test-name (assoc/default 'test-name cmdinfo))
	       (runscript (assoc/default 'runscript cmdinfo))
	       (db-host   (assoc/default 'db-host   cmdinfo))
	       (run-id    (assoc/default 'run-id    cmdinfo))
	       (test-id   (assoc/default 'test-id   cmdinfo))
	       (itemdat   (assoc/default 'itemdat   cmdinfo))
	       (work-area (assoc/default 'work-area cmdinfo))
	       (db        #f) ;; (open-db))
	       (state     (args:get-arg ":state"))
	       (status    (args:get-arg ":status"))
	       (stepname  (args:get-arg "-step")))
	  (if (not (launch:setup))
	      (begin
		(debug:print 0 *default-log-port* "Failed to setup, exiting")
		(exit 1)))

	  (if (args:get-arg "-runstep")(debug:print-info 1 *default-log-port* "Running -runstep, first change to directory " work-area))
	  (change-directory work-area)
	  ;; can setup as client for server mode now

	  (if (args:get-arg "-load-test-data")
	      ;; has sub commands that are rdb:
	      ;; DO NOT put this one into either rmt: or open-run-close
	      (tdb:load-test-data run-id test-id))
	  (if (args:get-arg "-setlog")
	      (let ((logfname (args:get-arg "-setlog")))
		(rmt:test-set-log! run-id test-id logfname)))
	  (if (args:get-arg "-set-toplog")
	      ;; DO NOT run remote
	      (tests:test-set-toplog! run-id test-name (args:get-arg "-set-toplog")))
	  (if (args:get-arg "-summarize-items")
	      ;; DO NOT run remote
	      (tests:summarize-items run-id test-id test-name #t)) ;; do force here
	  (if (args:get-arg "-runstep")
	      (if (null? remargs)
		  (begin
		    (debug:print-error 0 *default-log-port* "nothing specified to run!")
		    (if db (sqlite3:finalize! db))
		    (exit 6))
		  (let* ((stepname   (args:get-arg "-runstep"))
			 (logprofile (args:get-arg "-logpro"))
			 (logfile    (conc stepname ".log"))
			 (cmd        (if (null? remargs) #f (car remargs)))
			 (params     (if cmd (cdr remargs) '()))
			 (exitstat   #f)
			 (shell      (let ((sh (get-environment-variable "SHELL") ))
				       (if sh 
					   (last (string-split sh "/"))
					   "bash")))
			 (redir      (case (string->symbol shell)
				       ((tcsh csh ksh)    ">&")
				       ((zsh bash sh ash) "2>&1 >")
				       (else ">&")))
			 (fullcmd    (conc "(" (string-intersperse 
						(cons cmd params) " ")
					   ") " redir " " logfile)))
		    ;; mark the start of the test
		    (rmt:teststep-set-status! run-id test-id stepname "start" "n/a" (args:get-arg "-m") logfile)
		    ;; run the test step
		    (debug:print-info 2 *default-log-port* "Running \"" fullcmd "\" in directory \"" startingdir)
		    (change-directory startingdir)
		    (set! exitstat (system fullcmd))
		    (set! *globalexitstatus* exitstat)
		    ;; (change-directory testpath)
		    ;; run logpro if applicable ;; (process-run "ls" (list "/foo" "2>&1" "blah.log"))
		    (if logprofile
			(let* ((htmllogfile (conc stepname ".html"))
			       (oldexitstat exitstat)
			       (cmd         (string-intersperse (list "logpro" logprofile htmllogfile "<" logfile ">" (conc stepname "_logpro.log")) " ")))
			  (debug:print-info 2 *default-log-port* "running \"" cmd "\"")
			  (change-directory startingdir)
			  (set! exitstat (system cmd))
			  (set! *globalexitstatus* exitstat) ;; no necessary
			  (change-directory testpath)
			  (rmt:test-set-log! run-id test-id htmllogfile)))
		    (let ((msg (args:get-arg "-m")))
		      (rmt:teststep-set-status! run-id test-id stepname "end" exitstat msg logfile))
		    )))
	  (if (or (args:get-arg "-test-status")
		  (args:get-arg "-set-values"))
	      (let ((newstatus (cond
				((number? status)       (if (equal? status 0) "PASS" "FAIL"))
				((and (string? status)
				      (string->number status))(if (equal? (string->number status) 0) "PASS" "FAIL"))
				(else status)))
		    ;; transfer relevant keys into a hash to be passed to test-set-status!
		    ;; could use an assoc list I guess. 
		    (otherdata (let ((res (make-hash-table)))
				 (for-each (lambda (key)
					     (if (args:get-arg key)
						 (hash-table-set! res key (args:get-arg key))))
					   (list ":value" ":tol" ":expected" ":first_err" ":first_warn" ":units" ":category" ":variable"))
				 res)))
		(if (and (args:get-arg "-test-status")
			 (or (not state)
			     (not status)))
		    (begin
		      (debug:print-error 0 *default-log-port* "You must specify :state and :status with every call to -test-status\n" help)
		      (if (sqlite3:database? db)(sqlite3:finalize! db))
		      (exit 6)))
		(let* ((msg    (args:get-arg "-m"))
		       (numoth (length (hash-table-keys otherdata))))
		  ;; Convert to rpc inside the tests:test-set-status! call, not here
		  (tests:test-set-status! run-id test-id state newstatus msg otherdata work-area: work-area))))
	  (if (sqlite3:database? db)(sqlite3:finalize! db))
	  (set! *didsomething* #t))))

;;======================================================================
;; Various helper commands can go below here
;;======================================================================

(if (or (args:get-arg "-showkeys")
        (args:get-arg "-show-keys"))
    (let ((db #f)
	  (keys #f))
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting")
	    (exit 1)))
      (set! keys (rmt:get-keys)) ;;  db))
      (debug:print 1 *default-log-port* "Keys: " (string-intersperse keys ", "))
      (if (sqlite3:database? db)(sqlite3:finalize! db))
      (set! *didsomething* #t)))

(if (args:get-arg "-gui")
    (begin
      (debug:print 0 *default-log-port* "Look at the dashboard for now")
      ;; (megatest-gui)
      (set! *didsomething* #t)))

(if (args:get-arg "-create-megatest-area")
    (begin
      (genexample:mk-megatest.config)
      (set! *didsomething* #t)))

(if (args:get-arg "-create-test")
    (let ((testname (args:get-arg "-create-test")))
      (genexample:mk-megatest-test testname)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the database schema, clean up the db
;;======================================================================

(if (args:get-arg "-rebuild-db")
    (begin
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	    (exit 1)))
      ;; keep this one local
      ;; (open-run-close patch-db #f)
      (let ((dbstructs (db:setup)))
        (common:cleanup-db dbstructs full: #t))
      (set! *didsomething* #t)))

(if (args:get-arg "-cleanup-db")
    (begin
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	    (exit 1)))

;;      (if (not (server:choose-server *toppath* 'home?))
;;	  (begin
;;	    (debug:print 0 *default-log-port* "Servers are not running on this host or no servers alive. Cannot run cleanup-db")
;;	    (exit 1)))

      (let ((dbstructs (db:setup)))
        (common:cleanup-db dbstructs))
      (set! *didsomething* #t)))

(if (args:get-arg "-mark-incompletes")
    (begin
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting")
	    (exit 1)))
      (open-run-close db:find-and-mark-incomplete #f)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the tests meta data from the testconfig files
;;======================================================================

(main)
(if (args:get-arg "-update-meta")
    (begin
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	    (exit 1)))
      (runs:update-all-test_meta #f)
      (set! *didsomething* #t)))

;;======================================================================
;; Start a repl
;;======================================================================

;; fakeout readline
(include "readline-fix.scm")


(when (args:get-arg "-diff-rep")
  (when (and
         (not (args:get-arg "-diff-html"))
         (not (args:get-arg "-diff-email")))
    (debug:print 0 *default-log-port* "Must specify -diff-html or -diff-email with -diff-rep")
    (set! *didsomething* 1)
    (exit 1))
  
  (let* ((toppath (launch:setup)))
    (do-diff-report
     (args:get-arg "-src-target")
     (args:get-arg "-src-runname")
     (args:get-arg "-target")
     (args:get-arg "-runname")
     (args:get-arg "-diff-html")
     (args:get-arg "-diff-email"))
    (set! *didsomething* #t)
    (exit 0)))

(if (or (getenv "MT_RUNSCRIPT")
	(args:get-arg "-repl")
	(args:get-arg "-load"))
    (let* ((toppath (launch:setup))
	   (dbstructs (if (and toppath
			       ;; NOTE: server:choose-server is starting a server
			       ;;   either add equivalent for tcp mode or ????
                               #;(server:choose-server toppath 'home?))
                          (db:setup)
                          #f))) ;; make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f)))
      (if *toppath*
	  (cond
	   ((getenv "MT_RUNSCRIPT")
	    ;; How to run megatest scripts
	    ;;
	    ;; #!/bin/bash
	    ;;
	    ;; export MT_RUNSCRIPT=yes
	    ;; megatest << EOF
	    ;; (print "Hello world")
	    ;; (exit)
	    ;; EOF

	    (repl))
	   (else
	    (begin
	      (set! *db* dbstructs)
	      (import extras) ;; might not be needed
	      ;; (import csi)
	      (import readline)
	      (import apropos)
	      (import dbfile)
	      ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ...

	      (if *use-new-readline*
		  (begin
		    (install-history-file (get-environment-variable "HOME") ".megatest_history") ;;  [homedir] [filename] [nlines])
		    (current-input-port (make-readline-port "megatest> ")))
		  (begin
		    (gnu-history-install-file-manager
		     (string-append
		      (or (get-environment-variable "HOME") ".") "/.megatest_history"))
		    (current-input-port (make-gnu-readline-port "megatest> "))))
	      (if (args:get-arg "-repl")
		  (repl)
		  (load (args:get-arg "-load")))
	      ;; (db:close-all dbstruct) <= taken care of by on-exit call
	      )
	    (exit)))
	  (set! *didsomething* #t))))

;;======================================================================
;; Wait on a run to complete
;;======================================================================

(if (and (args:get-arg "-run-wait")
	 (not (or (args:get-arg "-run")
		  (args:get-arg "-runtests")))) ;; run-wait is built into runtests now
    (begin
      (if (not (launch:setup))
	  (begin
	    (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	    (exit 1)))
      (operate-on 'run-wait)
      (set! *didsomething* #t)))

;; ;; ;; redo me ;; Not converted to use dbstruct yet
;; ;; ;; redo me ;;
;; ;; ;; redo me (if (args:get-arg "-convert-to-norm")
;; ;; ;; redo me     (let* ((toppath (setup-for-run))
;; ;; ;; redo me 	   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t))))
;; ;; ;; redo me       (for-each 
;; ;; ;; redo me        (lambda (field)
;; ;; ;; redo me 	 (let ((dat '()))
;; ;; ;; redo me 	   (debug:print-info 0 *default-log-port* "Getting data for field " field)
;; ;; ;; redo me 	   (sqlite3:for-each-row
;; ;; ;; redo me 	    (lambda (id val)
;; ;; ;; redo me 	      (set! dat (cons (list id val) dat)))
;; ;; ;; redo me 	    (db:get-db db run-id)
;; ;; ;; redo me 	    (conc "SELECT id," field " FROM tests;"))
;; ;; ;; redo me 	   (debug:print-info 0 *default-log-port* "found " (length dat) " items for field " field)
;; ;; ;; redo me 	   (let ((qry (sqlite3:prepare db (conc "UPDATE tests SET " field "=? WHERE id=?;"))))
;; ;; ;; redo me 	     (for-each
;; ;; ;; redo me 	      (lambda (item)
;; ;; ;; redo me 		(let ((newval ;; (sdb:qry 'getid 
;; ;; ;; redo me 		       (cadr item))) ;; )
;; ;; ;; redo me 		  (if (not (equal? newval (cadr item)))
;; ;; ;; redo me 		      (debug:print-info 0 *default-log-port* "Converting " (cadr item) " to " newval " for test #" (car item)))
;; ;; ;; redo me 		  (sqlite3:execute qry newval (car item))))
;; ;; ;; redo me 	      dat)
;; ;; ;; redo me 	     (sqlite3:finalize! qry))))
;; ;; ;; redo me        (db:close-all dbstruct)
;; ;; ;; redo me        (list "uname" "rundir" "final_logf" "comment"))
;; ;; ;; redo me       (set! *didsomething* #t)))

(if (args:get-arg "-import-megatest.db")
    (begin
      (launch:setup)
      (db:multi-db-sync 
       (db:setup)
       'killservers
       'dejunk
       'adj-testids
       'old2new
       )
      (set! *didsomething* #t)))

(if (args:get-arg "-import-sexpr")
 (let*(
   (toppath (launch:setup))
   (tmppath (common:make-tmpdir-name toppath "")))
   (if (file-exists? (conc toppath "/.mtdb")) 
     (if (args:get-arg "-remove-dbs")
       (let* ((dbfiles (conc toppath "/.mtdb/* " tmppath "/*")))
        (debug:print 0 *default-log-port* "Removing db files: " dbfiles)
        (system (conc "rm -rvf " dbfiles))
       )
       (begin
         (debug:print 0 *default-log-port* "ERROR: Cannot import sexpr with an existing DB present.")
         (debug:print 0 *default-log-port* "Add '-remove-dbs all'  to remove the current Megatest DBs.")
         (set! *didsomething* #t)
         (exit)
       )
     )
     (debug:print 0 *default-log-port* "Did not find " (conc toppath "/.mtdb"))
   )
   (db:setup)
   (rmt:import-sexpr (args:get-arg "-import-sexpr"))
   (set! *didsomething* #t)))

(if (args:get-arg "-sync-to-megatest.db")
    (let* ((duh      (launch:setup))
	   (dbstruct (db:setup))
	   (tmpdbpth (dbr:dbstruct-tmppath dbstruct))
	   (lockfile (conc tmpdbpth ".lock"))
	   (locked   (common:simple-file-lock lockfile)) 
	   (res      (if locked
			 (db:multi-db-sync 
			  dbstruct
			  'new2old)
			 #f)))
      (if res
	  (begin
	    (common:simple-file-release-lock lockfile)
	    (debug:print 0 *default-log-port* "Synced " res " records to megatest.db"))
	  (debug:print 0 *default-log-port* "Skipping sync, there is a sync in progress."))
      (set! *didsomething* #t)))

(if (args:get-arg "-sync-to")
    (let ((toppath (launch:setup)))
      (tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
      (set! *didsomething* #t)))


;; use with -from and -to
;;
(if (args:get-arg "-db2db")
    (let* ((duh         (launch:setup))
	   (src-db      (args:get-arg "-from"))
	   (dest-db     (args:get-arg "-to"))
	   ;; (sync-period (args:get-arg-number "-period"))
	   ;; (sync-timeout (args:get-arg-number "-timeout"))
	   (sync-period-in  (args:get-arg "-period"))
	   (sync-timeout-in (args:get-arg "-timeout"))
	   (sync-period     (if sync-period-in (string->number sync-period-in) #f))
	   (sync-timeout    (if sync-timeout-in (string->number sync-timeout-in) #f))
	   (lockfile    (conc dest-db".sync-lock"))
	   (keys        (db:get-keys #f))
	   (thesync     (lambda (last-update)
			  (debug:print-info 0 *default-log-port* "Attempting to sync data from "src-db" to "dest-db"...")
			  (debug:print-info 0 *default-log-port* "PID = " (current-process-id))
			  (if (not (file-exists? dest-db))
			      (begin
				(debug:print 0 *default-log-port* "Using copy to create "dest-db" from "src-db)
				(file-copy src-db dest-db)
				1)
			      (let ((res (dbmod:db-to-db-sync src-db dest-db last-update (dbfile:db-init-proc) keys)))
				(if res
				    (debug:print-info 2 *default-log-port* "Synced " res " records from "src-db" to "dest-db)
				    (debug:print-info 0 *default-log-port* "No sync due to permissions or other issue."))
				res))))
	   (start-time  (current-seconds))
           (synclock-mod-time (if (file-exists? lockfile)
             (handle-exceptions
		 exn
	       #f
	       (file-modification-time synclock-file))
	     #f))
            (age (if synclock-mod-time (- (current-seconds) synclock-mod-time) 1000))
           )
      (if (and src-db dest-db)
	  (if (file-exists? src-db)
	      (if (and (file-exists? lockfile) (< age 20))
		    (debug:print 0 *default-log-port* "Lock "lockfile" exists, skipping sync...")
                 (begin
                  (if (file-exists? lockfile)
                    (begin
                    (debug:print 0 *default-log-port* "Deleting old lock file " lockfile)
                    (delete-file lockfile)
                    )
                  )
		  (dbfile:with-simple-file-lock
		   lockfile
		   (lambda ()
		     (let loop ((last-changed (current-seconds))
				(last-update  0))
		       (let* ((changes (handle-exceptions
					   exn
					   (begin
					     (debug:print 0 *default-log-port* "Exception in sync: "(condition->list exn))
					     (delete-file lockfile)
					     (exit))
					 (thesync last-update)))
			      (now-time (current-seconds)))
			 (if (and sync-period sync-timeout) ;; 
			     (if (and (< (- now-time start-time) 600) ;; hard limit on how long we run for
				      (>  sync-timeout (- now-time last-changed)))
				 (begin
				   (if sync-period (thread-sleep! sync-period))
				   (loop (if (> changes 0) now-time last-changed) now-time))))))))
                        (debug:print 0 *default-log-port* "Releasing lock file " lockfile)
                   )
               )
	      (debug:print 0 *default-log-port* "No sync due to unreadble or non-existant source file"src-db))
	  (debug:print 0 *default-log-port* "Usage for -db2db; -to and -from must be specified"))
      (set! *didsomething* #t)))

(if (args:get-arg "-list-test-time")
     (let* ((toppath (launch:setup))) 
     (task:get-test-times)  
     (set! *didsomething* #t)))

(if (args:get-arg "-list-run-time")
     (let* ((toppath (launch:setup))) 
     (task:get-run-times)  
     (set! *didsomething* #t)))
     
(if (args:get-arg "-generate-html")
    (let* ((toppath (launch:setup)))
      (if (tests:create-html-tree #f)
          (debug:print-info 0 *default-log-port* "HTML output created in " toppath "/lt/page0.html")
          (debug:print 0 *default-log-port* "Failed to create HTML output in " toppath "/lt/runs-index.html"))
      (set! *didsomething* #t)))

(if (args:get-arg "-generate-html-structure")
    (let* ((toppath (launch:setup)))
      ;(if (tests:create-html-tree #f)
 				(if (tests:create-html-summary #f)
          (debug:print-info 0 *default-log-port* "HTML output created in " toppath "/lt/targets.html")
          (debug:print 0 *default-log-port* "Failed to create HTML output in " toppath "/lt/runs-index.html"))
      (set! *didsomething* #t)))

(if (args:get-arg "-syscheck")
    (begin
      (mutils:syscheck common:raw-get-remote-host-load
		       server:get-best-guess-address
		       read-config)
      (set! *didsomething* #t)))

(if (args:get-arg "-extract-skeleton")
    (let* ((toppath (launch:setup)))
      (genexample:extract-skeleton-area (args:get-arg "-extract-skeleton"))
      (set! *didsomething* #t)))

;;======================================================================
;; Exit and clean up
;;======================================================================

(if (not *didsomething*)
    (debug:print 0 *default-log-port* help)
    (set! *time-to-exit* #t)
    )
;;(debug:print-info 13 *default-log-port* "thread-join! watchdog")

;; join the watchdog thread if it has been thread-start!ed  (it may not have been started in the case of a server that never enters running state)
;;   (symbols returned by thread-state: created ready running blocked suspended sleeping terminated dead)
;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage
;;(if (thread? *watchdog*)
;;    (case (thread-state *watchdog*)
;;      ((ready running blocked sleeping terminated dead)
;;       (thread-join! *watchdog*))))

(set! *time-to-exit* #t)

(if (not (eq? *globalexitstatus* 0))
    (if (or (args:get-arg "-run")(args:get-arg "-runtests")(args:get-arg "-runall"))
        (begin
           (debug:print 0 *default-log-port* "NOTE: Subprocesses with non-zero exit code detected: " *globalexitstatus*)
           (exit 0))
        (case *globalexitstatus*
         ((0)(exit 0))
         ((1)(exit 1))
         ((2)(exit 2))
         (else (exit 3)))))

Modified megatestmod.scm from [a70a654362] to [43bf1ee100].

36
37
38
39
40
41
42







43

















44
45
46
47
48
49
50
36
37
38
39
40
41
42
43
44
45
46
47
48
49

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73







+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses pkts))
(declare (uses servermod))
(declare (uses fsmod))

(use srfi-69)

(module megatestmod
	(
	 common:get-disks
	 db:set-tests-state-status
	 db:set-state-status-and-roll-up-items
	 common:get-install-area
	 tests:get-all
	 common:use-cache?
	*

	 mt:lazy-read-test-config
	 common:get-full-test-name
	 tests:extend-test-patts
	 tests:get-itemmaps
	 tests:get-items
	 tests:get-global-waitons
	 tests:get-tests-search-path
	 tests:filter-test-names
	 common:args-get-testpatt
	 tests:filter-test-names-not-matched
	 common:args-get-runname
	 common:load-views-config
	 common:args-get-state
	 common:args-get-status
	 common:get-runconfig-targets
	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified monitor.scm from [11b5fa345e] to [5284c87b1f].

17
18
19
20
21
22
23
24

25
26
27
28
29
30

31
32
33
34
17
18
19
20
21
22
23

24
25
26
27
28
29

30
31
32
33
34







-
+





-
+





;;  strftime('%m/%d/%Y %H:%M:%S','now','localtime')

(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking)
(import (prefix sqlite3 sqlite3:))

(declare (unit runs))
(declare (uses common))
;; (declare (uses common))

(declare (uses runconfig))
(declare (uses commonmod))
(import commonmod)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")

Modified mt.scm from [f24a9e55ce] to [715a388a89].

20
21
22
23
24
25
26
27

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

46
47
48
49
50
51
52
53
54







-
+


















-
+








(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) posix-extras directory-utils
     call-with-environment-variables)

(import (prefix sqlite3 sqlite3:))

(declare (unit mt))
(declare (uses debugprint))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses configfmod))

(declare (uses runconfig))
(declare (uses server))
(declare (uses runs))
(declare (uses rmtmod))
(declare (uses megatestmod))

(import debugprint
	commonmod
	configfmod
	rmtmod
	megatestmod)

;; make mt: calls in megatestmod work
;; (read-config-set! read-config)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
;; (include "test_records.scm")

;; This is the Megatest API. All generally "useful" routines will be wrapped or extended
;; here.

Added mtbody.scm version [c8247e48cf].






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
;;======================================================================
;; Copyright 2017, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
;;     (at your option) any later version.
;; 
;;     Megatest is distributed in the hope that it will be useful,
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

;;======================================================================
;;   All the crud that was in megatest.scm
;;======================================================================

(declare (unit mtbody))
(declare (uses debugprint))
(declare (uses mtargs))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses dbmod))
(declare (uses dbfile))
(declare (uses envmod))
(declare (uses apimod))
(declare (uses genexample))
(declare (uses rmtmod))
(declare (uses archivemod))
(declare (uses mutils))
(declare (uses odsmod))
(declare (uses testsmod))
(declare (uses diff-report))
(declare (uses tdb))

(use srfi-69)
(import csi)

(module mtbody
	*
	
(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
	  (prefix base64 base64:)

	  (prefix sqlite3 sqlite3:)
	  data-structures 
	  directory-utils
	  extras
	  files
	  matchable
	  md5
	  message-digest
	  pathname-expand
	  posix
	  posix-extras
	  ;;	  readline
	  regex
	  regex-case
	  sparse-vectors
	  srfi-1
	  srfi-18
	  srfi-69
	  typed-records
	  z3
	  
	  debugprint
	  commonmod
	  configfmod
	  ;; tcp-transportmod
	  (prefix mtargs args:)
	  )
  (use srfi-69))
 (chicken-5
  (import (prefix sqlite3 sqlite3:)
	  ;; data-structures
	  ;; extras
	  ;; files
	  ;; posix
	  ;; posix-extras
	  chicken.base
	  chicken.condition
	  chicken.file
	  chicken.file.posix
	  chicken.io
	  chicken.pathname
	  chicken.port
	  chicken.process
	  chicken.process-context
	  chicken.process-context.posix
	  chicken.sort
	  chicken.string
	  chicken.time
	  chicken.time.posix
	  
	  matchable
	  md5
	  message-digest
	  pathname-expand
	  regex
	  regex-case
	  srfi-1
	  srfi-18
	  srfi-69
	  typed-records
	  system-information

	  debugprint
  )))

;; imports common to chk5 and ck4
(import srfi-13
	csi)

(import (prefix mtargs args:)
        archivemod
	debugprint
	dbmod
	commonmod
	processmod
	configfmod
	dbfile
	dbmod
	portlogger
	tcp-transportmod
	rmtmod
	apimod
	stml2
	mtmod
	megatestmod
	servermod
	tasksmod
	runsmod
	rmtmod
	launchmod
	fsmod
	envmod
	apimod
	genexample
	mutils
	odsmod
	testsmod
	diff-report
	tdb
        )

(include "common_records.scm")

(define *db* #f) ;; this is only for the repl, do not use in general!!!!

;; (set! toplevel-command toplevel-command)

;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
(include "megatest-fossil-hash.scm")

(import (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (prefix base64 base64:))
(import
 ;; readline
 apropos json http-client directory-utils typed-records)
(import http-client srfi-18 extras format tcp-server tcp)

;; Added for csv stuff - will be removed
;;
(use sparse-vectors)

(require-library mutils)

;;======================================================================
;; api handler stuff
;;======================================================================

;; QUEUE METHOD

(define (api:tcp-dispatch-request-make-handler-new dbstruct) ;; cmd run-id params)
  (api:tcp-dispatch-request-make-handler-core dbstruct api:dispatch-request))


;; indat is (cmd run-id params meta)
;;
;; WARNING: Do not print anything in the lambda of this function as it
;;          reads/writes to current in/out port
;;
(define (api:tcp-dispatch-request-make-handler-old dbstruct) ;; cmd run-id params)
  (assert *toppath* "FATAL: api:tcp-dispatch-request-make-handler called but *toppath* not set.")
  (if (not *server-signature*)
      (set! *server-signature* (tt:mk-signature *toppath*)))
  (lambda (indat)
    (api:register-thread (current-thread))
    (let* ((result 
	    (let* ((numthreads (api:get-count-threads-alive))
		   (delay-wait (if (> numthreads 10)
				   (- numthreads 10)
				   0))
		   (normal-proc (lambda (cmd run-id params)
				  (case cmd
				    ((ping) *server-signature*)
				    (else
				     (api:dispatch-request dbstruct cmd run-id params))))))
	      (set! *api-process-request-count* numthreads)
	      (set! *db-last-access* (current-seconds))
;; 	      (if (not (eq? numthreads numthreads))
;; 	      (begin
;; 	      (api:remove-dead-or-terminated)
;; 	      (let ((threads-now (api:get-count-threads-alive)))
;; 	      (debug:print 0 *default-log-port* "WARNING: numthreads="numthreads", numthreads="numthreads", remaining="threads-now)
;; 	      (set! numthreads threads-now))))
	      (match indat
		     ((cmd run-id params meta)
		      (let* ((start-t (current-milliseconds))
			     (db-ok  (let* ((dbfname (dbmod:run-id->dbfname run-id))
					    (ok      (equal? dbfname (dbr:dbstruct-dbfname dbstruct))))
				       (case cmd
					 ((ping) #t) ;; we are fine
					 (else
					  (assert ok "FATAL: database file and run-id not aligned.")))))
			     (ttdat   *server-info*)
			     (server-state (tt-state ttdat))
			     (maxthreads   20) ;; make this a parameter?
			     (status  (cond
				       ((and (> numthreads maxthreads)
					     (> (random 100) 70)) ;; allow a 30% probability to go through so we can figure out what is going wrong in main.db server.
					'busy)
				       ;; ((> numthreads 5) 'loaded) ;; this gets transmitted to the client which calls tt:backoff-incr to slow stuff down.
				       (else 'ok)))
			     (errmsg  (case status
					((busy)   (conc "Server overloaded, "numthreads" threads in flight"))
					((loaded) (conc "Server loaded, "numthreads" threads in flight"))
					(else     #f)))
			     (result  (case status
					((busy)
					 (if (eq? cmd 'ping)
					     (normal-proc cmd run-id params)
					     ;; numthreads must be greater than 5 for busy
					     (* 0.1 (- numthreads maxthreads)) ;; was 15 - return a number for the remote to delay
					     )) ;; (- numthreads 29)) ;; call back in as many seconds
					((loaded)
					 (normal-proc cmd run-id params))
					(else
					 (normal-proc cmd run-id params))))
			     (meta   (case cmd
				       ((ping) `((sstate . ,server-state)))
				       (else   `((wait . ,delay-wait)))))
			     (payload (list status errmsg result meta)))
			;; (cmd run-id params meta)
			(db:add-stats cmd run-id params (- (current-milliseconds) start-t))
			payload))
		     (else
		      (assert #f "FATAL: failed to deserialize indat "indat))))))
      ;; (set! *api-process-request-count* (- *api-process-request-count* 1))
      ;; (serialize payload)
     
      (api:unregister-thread (current-thread))
      result)))

(define api:tcp-dispatch-request-make-handler api:tcp-dispatch-request-make-handler-new) ;; choose -old or -new

;; end api stuff

;; bracket open-output-file with code to make leading directory if it does not exist and handle exceptions
(define (open-logfile logpath-in)
  (let ((lpath #f))
    (condition-case
     (let* ((log-dir (or (pathname-directory logpath-in) "."))
	    (fname   (pathname-strip-directory logpath-in))
	    (logpath (if (> (string-length fname) 250)
			 (let ((newlogf (conc log-dir "/" (common:get-signature fname) ".log")))
			   (debug:print 0 *default-log-port* "WARNING: log file " logpath-in " path too long, converted to " newlogf)
			   newlogf)
			 logpath-in)))
       (set! lpath logpath) ;; just for printing if error
       (if (not (directory-exists? log-dir))
           (system (conc "mkdir -p " log-dir)))
       (open-output-file logpath))
     (exn ()
          (debug:print-error 0 *default-log-port* "Could not open log file for write: "lpath)
          (define *didsomething* #t)  
          (exit 1)))))

(define (main)
  ;; remove when configf fully modularized
  (read-config-set! configf:read-file)

  (define *usage-log-file* #f)    ;; put path to file for logging usage in this var in the ~/.megatestrc file
  (define *usage-use-seconds* #t) ;; for Epoc seconds in usage logging change this to #t in ~/.megatestrc file

  ;; set some parameters here - these need to be put in something that can be loaded from other
  ;; executables such as dashboard and mtutil
  ;;
  (include "transport-mode.scm")
  (dbfile:db-init-proc db:initialize-main-db)
  (debug:enable-timestamp #t) 


  (set! rmtmod:send-receive rmt:send-receive)
  ;;(lambda params (apply rmt:send-receive params))) ;; make send-receive available to rmtmod via parameter


  ;; load the ~/.megatestrc file, put (use trace)(trace-call-sites #t)(trace function-you-want-to-trace) in this file
  ;;
  (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.megatestrc")))
    (if (common:file-exists? debugcontrolf)
	(load debugcontrolf)))

  ;; usage logging, careful with this, it is not designed to deal with all real world challenges!
  ;;
  (if (and *usage-log-file*
           (file-write-access? *usage-log-file*))
      (with-output-to-file
          *usage-log-file*
	(lambda ()
          (print (if *usage-use-seconds*
		     (current-seconds)
		     (time->string
		      (seconds->local-time (current-seconds))
		      "%Yww%V.%w %H:%M:%S"))
		 " "
		 (current-user-name) " "
		 (current-directory) " "
		 "\"" (string-intersperse (argv) " ") "\""))
	#:append))

  ;; Disabled help items
  ;;  -rollup                 : (currently disabled) fill run (set by :runname)  with latest test(s)
  ;;                            from prior runs with same keys
  ;;  -daemonize              : fork into background and disconnect from stdin/out

  (define help (conc "
Megatest, documentation at http://www.kiatoa.com/fossils/megatest
  version " megatest-version "
  license GPL, Copyright Matt Welland 2006-2017
 
Usage: megatest [options]
  -h                      : this help
  -manual                 : show the Megatest user manual
  -version                : print megatest version (currently " megatest-version ")

Launching and managing runs
  -run                    : run all tests or as specified by -testpatt
  -remove-runs            : remove the data for a run, requires -runname and -testpatt
                            Optionally use :state and :status, use -keep-records to remove only
                            the run data. Use -kill-wait to override the 10 second
                            per test wait after kill delay (e.g. -kill-wait 0). 
  -kill-runs              : kill existing run(s) (all incomplete tests killed)
  -kill-rerun             : kill an existing run (all incomplete tests killed and run is rerun)
  -set-state-status X,Y   : set state to X and status to Y, requires controls per -remove-runs
  -rerun FAIL,WARN...     : force re-run for tests with specificed status(s)
  -rerun-clean            : set all tests not COMPLETED+PASS,WARN,WAIVED to NOT_STARTED,n/a
                            and then run the specified testpatt with -preclean
  -rerun-all              : set all tests to NOT_STARTED,n/a and run with -preclean
  -lock                   : lock run specified by target and runname
  -unlock                 : unlock run specified by target and runname
  -set-run-status status  : sets status for run to status, requires -target and -runname
  -get-run-status         : gets status for run specified by target and runname
  -run-wait               : wait on run specified by target and runname
  -preclean               : remove the existing test directory before running the test
  -clean-cache            : remove the cached megatest.config and runconfigs.config files
  -no-cache               : do not use the cached config files. 
  -one-pass               : launch as many tests as you can but do not wait for more to be ready
  -remove-keep N          : remove all but N most recent runs per target; use '-actions, -age, -precmd'
  -age <age>              : 120d,3h,20m to apply only to runs older than the 
                                 specified age. NB// M=month, m=minute
  -actions <action>[,...] : actions to take; print,remove-runs,archive,kill-runs
  -precmd                 : insert a wrapper command in front of the commands run

Selectors (e.g. use for -runtests, -remove-runs, -set-state-status, -list-runs etc.)
  -target key1/key2/...   : run for key1, key2, etc.
  -reqtarg key1/key2/...  : run for key1, key2, etc. but key1/key2 must be in runconfigs
  -testpatt patt1/patt2,patt3/...  : % is wildcard
  -runname                : required, name for this particular test run
  -state                  : Applies to runs, tests or steps depending on context
  -status                 : Applies to runs, tests or steps depending on context
  -modepatt key           : load testpatt from <key> in runconfigs instead of default TESTPATT if -testpatt and -tagexpr are not specified
  -tagexpr tag1,tag2%,..  : select tests with tags matching expression
  

Test helpers (for use inside tests)
  -step stepname
  -test-status            : set the state and status of a test (use :state and :status)
  -setlog logfname        : set the path/filename to the final log relative to the test
                            directory. may be used with -test-status
  -set-toplog logfname    : set the overall log for a suite of sub-tests
  -summarize-items        : for an itemized test create a summary html 
  -m comment              : insert a comment for this test

Test data capture
  -set-values             : update or set values in the testdata table
  :category               : set the category field (optional)
  :variable               : set the variable name (optional)
  :value                  : value measured (required)
  :expected               : value expected (required)
  :tol                    : |value-expect| <= tol (required, can be <, >, >=, <= or number)
  :units                  : name of the units for value, expected_value etc. (optional)
  -load-test-data         : read test specific data for storage in the test_data table
                            from standard in. Each line is comma delimited with four
                            fields category,variable,value,comment

Queries
  -list-runs patt         : list runs matching pattern \"patt\", % is the wildcard
  -show-keys              : show the keys used in this megatest setup
  -test-files targpatt    : get the most recent test path/file matching targpatt e.g. %/% or '*.log'
                            returns list sorted by age ascending, see examples below
  -test-paths             : get the test paths matching target, runname, item and test
                            patterns.
  -list-disks             : list the disks available for storing runs
  -list-targets           : list the targets in runconfigs.config
  -list-db-targets        : list the target combinations used in the db
  -show-config            : dump the internal representation of the megatest.config file
  -show-runconfig         : dump the internal representation of the runconfigs.config file
  -dumpmode MODE          : dump in MODE format instead of sexpr, MODE=json,ini,sexp etc. (add -debug 0,9 to see which file contributes each line)
  -show-cmdinfo           : dump the command info for a test (run in test environment)
  -section sectionName
  -var varName            : for config and runconfig lookup value for sectionName varName
  -since N                : get list of runs changed since time N (Unix seconds)
  -fields fieldspec       : fields to include in json dump; runs:id,runame+tests:testname+steps
  -sort fieldname         : in -list-runs sort tests by this field
  -testdata-csv [categorypatt/]varpatt  : dump testdata for given category

Misc 
  -start-dir path         : switch to this directory before running megatest
  -contour cname          : add a level of hierarcy to the linktree and run paths
  -area-tag tagname       : add a tag to an area while syncing to pgdb
  -run-tag tagname        : add a tag to a run while syncing to pgdb
  -rebuild-db             : bring the database schema up to date
  -cleanup-db             : remove any orphan records, vacuum the db
  -import-megatest.db     : push data from megatest.db to cache db files in /tmp/$USER
  -sync-to-megatest.db    : pull data from cache files in /tmp/$USER to megatest.db
  -sync-to dest           : sync to new postgresql central style database
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -adjutant C,M           : start the server/adjutant with allocated cores C and Mem M (Gig), 
                            use 0,0 to auto use full machine
  -transport http|rpc     : use http or rpc for transport (default is http) 
  -log logfile            : send stdout and stderr to logfile
  -list-servers           : list the servers 
  -kill-servers           : kill all servers
  -repl                   : start a repl (useful for extending megatest)
  -load file.scm          : load and run file.scm
  -mark-incompletes       : find and mark incomplete tests
  -ping run-id|host:port  : ping server, exit with 0 if found
  -debug N|N,M,O...       : enable debug 0-N or N and M and O ...
  -debug-noprop N|M,M,O...: enable debug but do not propagate to subprocesses via MT_DEBUG
  -config fname           : override the megatest.config file with fname
  -append-config fname    : append fname to the megatest.config file
  -import-sexpr fname     : import a sexpr file (use -list-runs % -dumpmode sexpr to create)
  -remove-dbs all         : remove Megatest DBs before importing sexpr. (Use only with -import-sexpr)
  -regen-testfiles        : regenerate scripts and logpro files from testconfig, run in test context
  
Utilities
  -env2file fname         : write the environment to fname.csh and fname.sh
  -envcap a               : save current variables labeled as context 'a' in file envdat.db
  -envdelta a-b           : output enviroment delta from context a to context b to -o fname
                            set the output mode with -dumpmode csh, bash or ini
                            note: ini format will use calls to use curr and minimize path
  -refdb2dat refdb        : convert refdb to sexp or to format specified by s-dumpmode
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove, get, replicate-db (use 
                            -dest to set destination), -include path1,path2... to get or save specific files
  -generate-html          : create a simple html dashboard for browsing your runs
  -generate-html-structure  : create a top level html veiw to list targets/runs and a Run view within each run directory.  
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 
                            is $DISPLAY valid 
  -list-waivers           : dump waivers for specified target, runname, testpatt to stdout
  -db2db                  : sync db to db, use -from, -to for dbs, -period and -timeout for continuous sync

Diff report
  -diff-rep               : generate diff report (must include -src-target, -src-runname, -target, -runname
                                                  and either -diff-email or -diff-html)
  -src-target <target>
  -src-runname <target>
  -diff-email <emails>    : comma separated list of email addresses to send diff report
  -diff-html  <rep.html>  : path to html file to generate

Spreadsheet generation
  -extract-ods fname.ods  : extract an open document spreadsheet from the database
  -pathmod path           : insert path, i.e. path/runame/itempath/logfile.html
                            will clear the field if no rundir/testname/itempath/logfile
                            if it contains forward slashes the path will be converted
                            to windows style
Getting started
  -create-megatest-area   : create a skeleton megatest area. You will be prompted for paths
  -create-test testname   : create a skeleton megatest test. You will be prompted for info

Examples

# Get test path, use '.' to get a single path or a specific path/file pattern
megatest -test-files 'logs/*.log' -target ubuntu/n%/no% -runname w49% -testpatt test_mt%

Called as " (string-intersperse (argv) " ") "
Version " megatest-version ", built from " megatest-fossil-hash ))

  ;;  -gui                    : start a gui interface
  ;;  -config fname           : override the runconfigs file with fname

  ;; process args
  (define remargs (args:get-args 
		   (argv)
		   (list  "-runtests"  ;; run a specific test
			  "-config"    ;; override the config file name
			  "-append-config"
			  "-execute"   ;; run the command encoded in the base64 parameter
			  "-step"
			  "-target"
			  "-reqtarg"
			  ":runname"
			  "-runname"
			  ":state"  
			  "-state"
			  ":status"
			  "-status"
			  "-list-runs"
                          "-testdata-csv"
			  "-testpatt"
                          ;; "--modepatt"
                          "-modepatt"
                          "-tagexpr"
			  "-itempatt"
			  "-setlog"
			  "-set-toplog"
			  "-runstep"
			  "-logpro"
			  "-m"
			  "-rerun"

			  "-days"
			  "-rename-run"
			  "-from"
			  "-to"
			  "-dest"
                          "-source" 
                          "-time-stamp" 
			  ;; values and messages
			  ":category"
			  ":variable"
			  ":value"
			  ":expected"
			  ":tol"
			  ":units"

			  ;; misc
			  "-start-dir"
                          "-run-patt"
                          "-target-patt"   
			  "-contour"
                          "-area-tag"  
                          "-area"  
			  "-run-tag"
			  "-server"
			  "-adjutant"
			  "-transport"
			  "-port"
			  "-extract-ods"
			  "-pathmod"
			  "-env2file"
			  "-envcap"
			  "-envdelta"
			  "-setvars"
			  "-set-state-status"
			  "-import-sexpr"
                          "-remove-dbs" ;; to be used only with -import-sexpr to remove megatest dbs first.
			  "-period"  ;; sync period in seconds
			  "-timeout" ;; exit sync if timeout in seconds exceeded since last change

                          ;; move runs stuff here
                          "-remove-keep"           
			  "-set-run-status"
			  "-age"

			  ;; archive 
			  "-archive"
			  "-actions"
			  "-precmd"
			  "-include"
			  "-exclude-rx"
			  "-exclude-rx-from"
			  
			  "-debug" ;; for *verbosity* > 2
			  "-debug-noprop"
			  "-create-test"
			  "-override-timeout"
			  "-test-files"  ;; -test-paths is for listing all
			  "-load"        ;; load and exectute a scheme file
			  "-section"
			  "-var"
			  "-dumpmode"
			  "-run-id"
			  "-db"
			  "-ping"
			  "-refdb2dat"
			  "-o"
			  "-log"
                          "-sync-log"
			  "-since"
			  "-fields"
			  "-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			  "-sort"
			  "-target-db"
			  "-source-db"
			  "-prefix-target"

                          "-src-target"
                          "-src-runname"
                          "-diff-email"
			  "-sync-to"			
			  "-pgsync"
			  "-kill-wait"    ;; wait this long before removing test (default is 10 sec)
                          "-diff-html"

			  ;; wizards, area capture, setup new ...
			  "-extract-skeleton"
			  )
 		   (list  "-h" "-help" "--help"
			  "-manual"
			  "-version"
		          "-force"
		          "-xterm"
		          "-showkeys"
		          "-show-keys"
		          "-test-status"
			  "-set-values"
			  "-load-test-data"
			  "-summarize-items"
		          "-gui"
			  "-daemonize"
			  "-preclean"
			  "-rerun-clean"
			  "-rerun-all"
			  "-clean-cache"
			  "-no-cache"
			  "-cache-db"
			  "-cp-eventtime-to-publishtime"
                          "-use-db-cache"
                          "-prepend-contour"


			  ;; misc
			  "-repl"
			  "-lock"
			  "-unlock"
			  "-list-servers"
			  "-kill-servers"
                          "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			  "-one-pass"      ;;
			  "-local"         ;; run some commands using local db access
			  "-generate-html"
			  "-generate-html-structure" 
			  "-list-run-time"
                          "-list-test-time"
			  "-regen-testfiles"
			  
			  ;; misc queries
			  "-list-disks"
			  "-list-targets"
			  "-list-db-targets"
			  "-show-runconfig"
			  "-show-config"
			  "-show-cmdinfo"
			  "-get-run-status"
			  "-list-waivers"

			  ;; queries
			  "-test-paths" ;; get path(s) to a test, ordered by youngest first

			  "-runall"    ;; run all tests, respects -testpatt, defaults to %
			  "-run"       ;; alias for -runall
			  "-remove-runs"
                          "-kill-runs"
                          "-kill-rerun"
                          "-keep-records" ;; use with -remove-runs to remove only the run data
			  "-rebuild-db"
			  "-cleanup-db"
			  "-rollup"
			  "-update-meta"
			  "-create-megatest-area"
			  "-mark-incompletes"

			  "-convert-to-norm"
			  "-convert-to-old"
			  "-import-megatest.db"
			  "-sync-to-megatest.db"
			  "-db2db"
                          "-sync-brute-force"
			  "-logging"
			  "-v" ;; verbose 2, more than normal (normal is 1)
			  "-q" ;; quiet 0, errors/warnings only

                          "-diff-rep"

			  "-syscheck"
			  "-obfuscate"
			  ;; junk placeholder
			  ;; "-:p"
			  
                          )
		   args:arg-hash
		   0))

  ;; Add args that use remargs here
  ;;
  (if (and (not (null? remargs))
	   (not (or
		 (args:get-arg "-runstep")
		 (args:get-arg "-envcap")
		 (args:get-arg "-envdelta")
		 )
		))
      (debug:print-error 0 *default-log-port* "Unrecognised arguments: " (string-intersperse (if (list? remargs) remargs (argv))  " ")))

  ;; before doing anything else change to the start-dir if provided
  ;;
  (if (args:get-arg "-start-dir")
      (if (common:file-exists? (args:get-arg "-start-dir"))
          (let ((fullpath (common:real-path (args:get-arg "-start-dir"))))
            (setenv "PWD" fullpath)
            (change-directory fullpath))
	  (begin
	    (debug:print-error 0 *default-log-port* "non-existant start dir " (args:get-arg "-start-dir") " specified, exiting.")
	    (exit 1))))

  ;; immediately set MT_TARGET if -reqtarg or -target are available
  ;;
  (let ((targ (or (args:get-arg "-reqtarg")(args:get-arg "-target"))))
    (if targ (setenv "MT_TARGET" targ)))

  ;; set the purpose field in procinf

  (procinf-purpose-set! *procinf* (get-purpose args:arg-hash))
  (procinf-mtversion-set! *procinf* megatest-version)

  ;; The watchdog is to keep an eye on things like db sync etc.
  ;;

  ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage
  ;;(define *watchdog* (make-thread
  ;;		    (lambda ()
  ;;		      (handle-exceptions
  ;;			  exn
  ;;			  (begin
  ;;			    (print-call-chain)
  ;;			    (print " message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
  ;;			(common:watchdog)))
  ;;		    "Watchdog thread"))

  ;;(if (not (args:get-arg "-server"))
  ;;    (thread-start! *watchdog*)) ;; if starting a server; wait till we get to running state before kicking off watchdog
  (let* ((no-watchdog-args
	  '("-list-runs"
            "-testdata-csv"
            "-list-servers"
            "-server"
	    "-adjutant"
            "-list-disks"
            "-list-targets"
            "-show-runconfig"
            ;;"-list-db-targets"
            "-show-runconfig"
            "-show-config"
            "-show-cmdinfo"
	    "-cleanup-db"
            ))
	 (no-watchdog-argvals (list '("-archive" . "replicate-db")))
	 (start-watchdog-specail-arg-val (let loop ((hed  (car no-watchdog-argvals))
                                                    (tail (cdr   no-watchdog-argvals)))
                                           ;; (print "hed" hed " arg " (args:get-arg (car hed)) " val:" (cdr hed)  " eql" (equal? (args:get-arg (car hed)) (cdr hed)))  
                                           (if (equal? (args:get-arg (car hed)) (cdr hed))
                                               #f
                                               (if (null? tail)
                                                   #t
                                                   (loop (car tail) (cdr tail))))))      
	 (no-watchdog-args-vals (filter (lambda (x) x)
					(map args:get-arg no-watchdog-args)))
	 (start-watchdog (and (null? no-watchdog-args-vals) start-watchdog-specail-arg-val)))
					;(print  "no-watchdog-args="no-watchdog-args "no-watchdog-args-vals="no-watchdog-args-vals " start-watchdog-specail-arg-val:" start-watchdog-specail-arg-val " start-watchdog:" start-watchdog) 
    ;;  (if start-watchdog
    ;;      (thread-start! *watchdog*))
    #t
    )

  ;; stop the train watchdog
  (stop-the-train)

  ;; this segment will run launch:setup only if -log is not set. This is fairly safe as servers are not
  ;; manually started and thus should never be started in a non-megatest area. Thus no need to handle situation
  ;; where (launch:setup) returns #f?
  ;;
  (if (or (args:get-arg "-log")#;(args:get-arg "-server")) ;; redirect the log always when a server
      (handle-exceptions
       exn
       (begin
	 (print "ERROR: Failed to switch to log output. " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
       (let* ((tl   (or (args:get-arg "-log")(launch:setup)))   ;; run launch:setup if -server, ensure we do NOT run launch:setup if -log specified
	      (dbname (args:get-arg "-db"))   ;; for the server logfile name
	      (logf (or (args:get-arg "-log") ;; use -log unless we are a server, then craft a logfile name
			(conc tl "/logs/server-"(or dbname "unk")"-"(current-process-id) "-" (get-host-name) ".log")))
	      (oup  (open-logfile logf)))
	 (if (not (args:get-arg "-log"))
	     (hash-table-set! args:arg-hash "-log" logf)) ;; fake out future queries of -log
	 (debug:print-info 0 *default-log-port* "Sending log output to " logf)
	 (set! *default-log-port* oup))))

  (if (or (args:get-arg "-h")
	  (args:get-arg "-help")
	  (args:get-arg "--help"))
      (begin
	(print help)
	(exit)))

  (if (args:get-arg "-manual")
      (let* ((htmlviewercmd (or (configf:lookup *configdat* "setup" "htmlviewercmd")
				(common:which '("firefox" "arora"))))
	     (install-home  (common:get-install-area))
	     (manual-html   (conc install-home "/share/docs/megatest_manual.html")))
	(if (and install-home
		 (common:file-exists? manual-html))
	    (system (conc "(" htmlviewercmd " " manual-html " ) &"))
	    (system (conc "(" htmlviewercmd " http://www.kiatoa.com/cgi-bin/fossils/megatest/doc/tip/docs/manual/megatest_manual.html ) &")))
	(exit)))

  (if (args:get-arg "-version")
      (begin
	(print (common:version-signature)) ;; (print megatest-version)
	(exit)))

  (define *didsomething* #f)

  ;; Overall exit handling setup immediately
  ;;
  (if (or (args:get-arg "-process-reap"))
      ;; (args:get-arg "-runtests")
      ;; (args:get-arg "-execute")
      ;; (args:get-arg "-remove-runs")
      ;; (args:get-arg "-runstep"))
      (let ((original-exit (exit-handler)))
	(exit-handler (lambda (#!optional (exit-code 0))
			(printf "Preparing to exit with exit code ~A ...\n" exit-code)
			(for-each
			 
			 (lambda (pid)
			   (handle-exceptions
			    exn
			    (begin
			      (printf "process reap failed. exn=~A\n" exn)
			      #t)
			    (let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
			      (if (or (eq? pid-val pid)
				      (eq? pid-val 0))
				  (begin
				    (printf "Sending signal/term to ~A\n" pid)
				    (process-signal pid signal/term))))))
			 (process:children #f))
			(original-exit exit-code)))))

  ;; for some switches always print the command to stderr
  ;;
  (if (args:any-defined? "-run" "-runall" "-remove-runs" "-set-state-status" "-kill-runs" "-kill-rerun")
      (debug:print 0 *default-log-port* (string-intersperse (argv) " ")))


  ;;======================================================================
  ;; Misc setup stuff
  ;;======================================================================

  (debug:setup)

  (if (args:get-arg "-logging")(set! *logging* #t))

  ;;(if (debug:debug-mode 3) ;; we are obviously debugging
  ;;    (set! open-run-close open-run-close-no-exception-handling))

  (if (args:get-arg "-itempatt")
      (let ((newval (conc (args:get-arg "-testpatt") "/" (args:get-arg "-itempatt"))))
	(debug:print 0 *default-log-port* "WARNING: -itempatt has been deprecated, please use -testpatt testpatt/itempatt method, new testpatt is "newval)
	(hash-table-set! args:arg-hash "-testpatt" newval)
	(hash-table-delete! args:arg-hash "-itempatt")))

  (if (args:get-arg "-runtests")
      (debug:print 0 *default-log-port* "WARNING: \"-runtests\" is deprecated. Use \"-run\" with \"-testpatt\" instead"))

  (on-exit std-exit-procedure)

  ;;======================================================================
  ;; Misc general calls
  ;;======================================================================

  (if (and (args:get-arg "-cache-db")
           (args:get-arg "-source-db"))
      (let* ((temp-dir (or (args:get-arg "-target-db") (create-directory (conc "/tmp/" (getenv "USER") "/" (string-translate (current-directory) "/" "_")))))
             (target-db (conc temp-dir "/cached.db"))
             (source-db (args:get-arg "-source-db")))        
	(db:cache-for-read-only source-db target-db)
	(set! *didsomething* #t)))

  ;; handle a clean-cache request as early as possible
  ;;
  (if (args:get-arg "-clean-cache")
      (let ((toppath  (launch:setup)))
	(set! *didsomething* #t) ;; suppress the help output.
	(runs:clean-cache (common:args-get-target)
			  (args:get-arg "-runname")
			  toppath)))
  
  (if (args:get-arg "-env2file")
      (begin
	(save-environment-as-files (args:get-arg "-env2file"))
	(set! *didsomething* #t)))

  (if (args:get-arg "-list-disks")
      (let ((toppath (launch:setup)))
	(print (string-intersperse 
		(map (lambda (x)
		       (string-intersperse 
			x
			" => "))
		     (common:get-disks *configdat*))
		"\n"))
	(set! *didsomething* #t)))

  ;; csv processing record
  (define (make-refdb:csv)
    (vector 
     (make-sparse-array)
     (make-hash-table)
     (make-hash-table)
     0
     0))
  (define-inline (refdb:csv-get-svec     vec)    (vector-ref  vec 0))
  (define-inline (refdb:csv-get-rows     vec)    (vector-ref  vec 1))
  (define-inline (refdb:csv-get-cols     vec)    (vector-ref  vec 2))
  (define-inline (refdb:csv-get-maxrow   vec)    (vector-ref  vec 3))
  (define-inline (refdb:csv-get-maxcol   vec)    (vector-ref  vec 4))
  (define-inline (refdb:csv-set-svec!    vec val)(vector-set! vec 0 val))
  (define-inline (refdb:csv-set-rows!    vec val)(vector-set! vec 1 val))
  (define-inline (refdb:csv-set-cols!    vec val)(vector-set! vec 2 val))
  (define-inline (refdb:csv-set-maxrow!  vec val)(vector-set! vec 3 val))
  (define-inline (refdb:csv-set-maxcol!  vec val)(vector-set! vec 4 val))

  (define (get-dat results sheetname)
    (or (hash-table-ref/default results sheetname #f)
	(let ((tmp-vec  (make-refdb:csv)))
	  (hash-table-set! results sheetname tmp-vec)
	  tmp-vec)))

  (if (args:get-arg "-refdb2dat")
      (let* ((input-db (args:get-arg "-refdb2dat"))
	     (out-file (args:get-arg "-o"))
	     (out-fmt  (or (args:get-arg "-dumpmode") "scheme"))
	     (out-port (if (and out-file 
				(not (member out-fmt '("sqlite3" "csv"))))
			   (open-output-file out-file)
			   (current-output-port)))
	     (res-data (configf:read-refdb input-db))
	     (data     (car res-data))
	     (msg      (cadr res-data)))
	(if (not data)
	    (debug:print 0 *default-log-port* "Bad input? data=" data) ;; some error occurred
	    (with-output-to-port out-port
	      (lambda ()
		(case (string->symbol out-fmt)
		  ((scheme)(pp data))
		  ((perl)
		   ;; (print "%hash = (")
		   ;;        key1 => 'value1',
		   ;;        key2 => 'value2',
		   ;;        key3 => 'value3',
		   ;; );
		   (configf:map-all-hier-alist 
		    data 
		    (lambda (sheetname sectionname varname val)
		      (print "$data{\"" sheetname "\"}{\"" sectionname "\"}{\"" varname "\"} = \"" val "\";"))))
		  ((python ruby)
		   (print "data={}")
		   (configf:map-all-hier-alist
		    data
		    (lambda (sheetname sectionname varname val)
		      (print "data[\"" sheetname "\"][\"" sectionname "\"][\"" varname "\"] = \"" val "\""))
		    initproc1:
		    (lambda (sheetname)
		      (print "data[\"" sheetname "\"] = {}"))
		    initproc2:
		    (lambda (sheetname sectionname)
		      (print "data[\"" sheetname "\"][\"" sectionname "\"] = {}"))))
		  ((csv)
		   (let* ((results  (make-hash-table)) ;; (make-sparse-array)))
			  (row-cols (make-hash-table))) ;; hash of hashes where section => ht { row-<name> => num or col-<name> => num
		     ;; (print "data=")
		     ;; (pp data)
		     (configf:map-all-hier-alist
		      data
		      (lambda (sheetname sectionname varname val)
			;; (print "sheetname: " sheetname ", sectionname: " sectionname ", varname: " varname ", val: " val)
			(let* ((dat      (get-dat results sheetname))
			       (vec      (refdb:csv-get-svec dat))
			       (rownames (refdb:csv-get-rows dat))
			       (colnames (refdb:csv-get-cols dat))
			       (currrown (hash-table-ref/default rownames varname #f))
			       (currcoln (hash-table-ref/default colnames sectionname #f))
			       (rown     (or currrown 
					     (let* ((lastn   (refdb:csv-get-maxrow dat))
						    (newrown (+ lastn 1)))
					       (refdb:csv-set-maxrow! dat newrown)
					       newrown)))
			       (coln     (or currcoln 
					     (let* ((lastn   (refdb:csv-get-maxcol dat))
						    (newcoln (+ lastn 1)))
					       (refdb:csv-set-maxcol! dat newcoln)
					       newcoln))))
			  (if (not (sparse-array-ref vec 0 coln)) ;; (eq? rown 0)
			      (begin
				(sparse-array-set! vec 0 coln sectionname)
				;; (print "sparse-array-ref " 0 "," coln "=" (sparse-array-ref vec 0 coln))
				))
			  (if (not (sparse-array-ref vec rown 0)) ;; (eq? coln 0)
			      (begin
				(sparse-array-set! vec rown 0 varname)
				;; (print "sparse-array-ref " rown "," 0 "=" (sparse-array-ref vec rown 0))
				))
			  (if (not currrown)(hash-table-set! rownames varname rown))
			  (if (not currcoln)(hash-table-set! colnames sectionname coln))
			  ;; (print "dat=" dat ", rown=" rown ", coln=" coln)
			  (sparse-array-set! vec rown coln val)
			  ;; (print "sparse-array-ref " rown "," coln "=" (sparse-array-ref vec rown coln))
			  )))
		     (for-each
		      (lambda (sheetname)
			(let* ((sheetdat (get-dat results sheetname))
			       (svec     (refdb:csv-get-svec sheetdat))
			       (maxrow   (refdb:csv-get-maxrow sheetdat))
			       (maxcol   (refdb:csv-get-maxcol sheetdat))
			       (fname    (if out-file 
					     (string-substitute "%s" sheetname out-file) ;; "/foo/bar/%s.csv")
					     (conc sheetname ".csv"))))
			  (with-output-to-file fname
			    (lambda ()
			      ;; (print "Sheetname: " sheetname)
			      (let loop ((row       0)
					 (col       0)
					 (curr-row '())
					 (result   '()))
				(let* ((val (sparse-array-ref svec row col))
				       (disp-val (if val
						     (conc "\"" val "\"")
						     "")))
				  (if (> col 0)(display ","))
				  (display disp-val)
				  (cond
				   ((> row maxrow)(display "\n") result)
				   ((>= col maxcol)
				    (display "\n")
				    (loop (+ row 1) 0 '() (append result (list curr-row))))
				   (else
				    (loop row (+ col 1) (append curr-row (list val)) result)))))))))
		      (hash-table-keys results))))
		  ((sqlite3)
		   (let* ((db-file   (or out-file (pathname-file input-db)))
			  (db-exists (common:file-exists? db-file))
			  (db        (sqlite3:open-database db-file)))
		     (if (not db-exists)(sqlite3:execute db "CREATE TABLE data (sheet,section,var,val);"))
		     (configf:map-all-hier-alist
		      data
		      (lambda (sheetname sectionname varname val)
			(sqlite3:execute db
					 "INSERT OR REPLACE INTO data (sheet,section,var,val) VALUES (?,?,?,?);"
					 sheetname sectionname varname val)))
		     (sqlite3:finalize! db)))
		  (else
		   (pp data))))))
	(if out-file (close-output-port out-port))
	(exit) ;; yes, bending the rules here - need to exit since this is a utility
	))

  (if (args:get-arg "-ping")
      (let* ((server-id     (string->number (args:get-arg "-ping"))) ;; extract run-id (i.e. no ":"
	     (host:port     (args:get-arg "-ping")))
	(debug:print 0 *default-log-port* "NOT YET REIMPLEMENTED FOR TCP/INMEM") ;; bug
	(exit)))
  ;; (server:ping (or server-id host:port) #f do-exit: #t)))

  ;;======================================================================
  ;; Capture, save and manipulate environments
  ;;======================================================================

  ;; NOTE: Keep these above the section where the server or client code is setup

  (let ((envcap (args:get-arg "-envcap")))
    (if envcap
	(let* ((db      (env:open-db (if (null? remargs) "envdat.db" (car remargs)))))
	  (env:save-env-vars db envcap)
	  (env:close-database db)
	  (set! *didsomething* #t))))

  ;; delta "language" will eventually be res=a+b-c but for now it is just res=a-b 
  ;;
  (let ((envdelta (args:get-arg "-envdelta")))
    (if envdelta
	(let ((match (string-split envdelta "-")));; (string-match "([a-z0-9_]+)=([a-z0-9_\\-,]+)" envdelta)))
	  (if (not (null? match))
	      (let* ((db        (env:open-db (if (null? remargs) "envdat.db" (car remargs))))
		     ;; (resctx    (cadr match))
		     ;; (equn      (caddr match))
		     (parts     match) ;; (string-split equn "-"))
		     (minuend   (car parts))
		     (subtraend (cadr parts))
		     (added     (env:get-added   db minuend subtraend))
		     (removed   (env:get-removed db minuend subtraend))
		     (changed   (env:get-changed db minuend subtraend)))
		;; (pp (hash-table->alist added))
		;; (pp (hash-table->alist removed))
		;; (pp (hash-table->alist changed))
		(if (args:get-arg "-o")
		    (with-output-to-file
			(args:get-arg "-o")
		      (lambda ()
			(env:print added removed changed)))
		    (env:print added removed changed))
		(env:close-database db)
		(set! *didsomething* #t))
	      (debug:print-error 0 *default-log-port* "Parameter to -envdelta should be new=start-end")))))

  ;;======================================================================
  ;; Start the server - can be done in conjunction with -runall or -runtests (one day...)
  ;;   we start the server if not running else start the client thread
  ;;======================================================================

  ;; Server? Start up here.
  ;;
  (if (args:get-arg "-server")
      (let* (;; (run-id     (args:get-arg "-run-id"))
	     (dbfname    (args:get-arg "-db"))
	     (tl         (launch:setup))
	     (keys       (keys:config-get-fields *configdat*)))
	(case (rmt:transport-mode)
	  ((tcp)
	   (let* ((timeout    (server:expiration-timeout)))
	     (debug:print 0 *default-log-port* "INFO: megatest -server starting on " (get-host-name) " for " dbfname " using tcp method with timeout of "timeout)
	     (tt-server-timeout-param timeout)
	     (api:queue-processor)
	     (thread-start! (make-thread api:print-db-stats "print-db-stats"))
	     (if dbfname
		 (tt:start-server tl #f dbfname api:tcp-dispatch-request-make-handler keys)
		 (begin
		   (debug:print 0 *default-log-port* "ERROR: transport mode is tcp - -db is required.")
		   (exit 1)))))
	  ((nfs)(debug:print 0 *default-log-port* "WARNING: server start called in nfs mode '"(rmt:transport-mode)))
	  (else (debug:print 0 *default-log-port* "ERROR: rmt:transport-mode value not recognised "(rmt:transport-mode))))
	(set! *didsomething* #t)))

  ;; The adjutant is a bit different, it does NOT run (launch:setup) as it is not necessarily tied to
  ;; a specific Megatest area. Detail are being hashed out and this may change.
  ;;
  (if (args:get-arg "-adjutant")
      (begin
	;; (adjutant-run)
	(set! *didsomething* #t)))

  (if (args:get-arg "-list-servers")
      (let* ((tl (launch:setup)) ;; need this to initialize *toppath*
             (servdir (tt:get-servinfo-dir *toppath*))
             (servfiles (glob (conc servdir "/*:*.db")))
             (fmtstr  "~10a~22a~10a~25a~25a~8a\n")
             (dbfiles (append (glob (conc *toppath* "/.mtdb/main.db")) (glob (conc *toppath* "/.mtdb/?.db"))(glob (conc *toppath* "/.mtdb/??.db"))))
             (ttdat (make-tt areapath: *toppath*))
	     )
	(format #t fmtstr "DB" "host:port" "PID" "age" "last mod" "state")
	(for-each
         (lambda (dbfile)
           (let* (
		  (dbfname (conc (pathname-file dbfile) ".db"))
		  (sfiles   (tt:find-server *toppath* dbfname))
		  )
             (for-each 
              (lambda (sfile)
                (let (
                      (sinfos (tt:get-server-info-sorted ttdat dbfname))
                      )
                  (for-each 
                   (lambda (sinfo)
                     (let* (
                            (db (list-ref sinfo 5))
                            (pid (list-ref sinfo 4))
                            (host (list-ref sinfo 0))
                            (port (list-ref sinfo 1))
                            (server-id (list-ref sinfo 3))
                            (age (seconds->hr-min-sec (- (current-seconds) (list-ref sinfo 2))))
                            (last-mod (seconds->string (list-ref sinfo 2)))
                            (status (system (conc "ssh " host " ps " pid " > /dev/null")))
                            (state (if (> status 0)
                                       "dead"
                                       (tt:ping host port server-id 0)
                                       ))
                            )
                       (format #t fmtstr db (conc host ":" port) pid age last-mod state)
                       )
                     )
                   sinfos
                   )
                  ) 
		)
              sfiles
              )
             )
	   )
	 dbfiles
	 )
	(set! *didsomething* #t)
	(exit)  
	)
      )




  (if (args:get-arg "-kill-servers")
      
      (let* ((tl (launch:setup)) ;; need this to initialize *toppath*
             (servdir (tt:get-servinfo-dir *toppath*))
             (servfiles (glob (conc servdir "/*:*.db")))
             (fmtstr  "~10a~22a~10a~25a~25a~8a\n")
             (dbfiles (if (file-exists? (conc *toppath* "/.mtdb/main.db")) (append (glob (conc *toppath* "/.mtdb/main.db")) (glob (conc *toppath* "/.mtdb/?.db"))(glob (conc *toppath* "/.mtdb/??.db"))) '()))
             (ttdat (make-tt areapath: *toppath*))
	     )
	(format #t fmtstr "DB" "host:port" "PID" "age" "last mod" "state")
	(for-each
         (lambda (dbfile)
           (let* (
		  (dbfname (conc (pathname-file dbfile) ".db"))
		  (sfiles   (tt:find-server *toppath* dbfname))
		  )
             (for-each 
              (lambda (sfile)
                (let (
                      (sinfos (tt:get-server-info-sorted ttdat dbfname))
                      )
                  (for-each 
                   (lambda (sinfo)
                     (let* (
                            (db (list-ref sinfo 5))
                            (pid (list-ref sinfo 4))
                            (host (list-ref sinfo 0))
                            (port (list-ref sinfo 1))
                            (server-id (list-ref sinfo 3))
                            (age (seconds->hr-min-sec (- (current-seconds) (list-ref sinfo 2))))
                            (last-mod (seconds->string (list-ref sinfo 2)))
                            (killed (system (conc "ssh " host " kill " pid " > /dev/null")))
                            (dummy2 (sleep 1))
                            (state (if (> (system (conc "ssh " host " ps " pid " > /dev/null")) 0) "dead" "alive"))
                            )
                       (format #t fmtstr db (conc host ":" port) pid age last-mod state)
                       (system (conc "rm " sfile))
                       )
                     )
                   sinfos
                   )
                  ) 
		)
              sfiles
              )
             )
	   )
	 dbfiles
	 )
	;; remove this db, because otherwise metadata contains records for old servers, and this causes a problem with db:no-sync-get-lock-with-id.
	(if (file-exists? (conc *toppath* "/.mtdb/no-sync.db"))
	    (delete-file (conc *toppath* "/.mtdb/no-sync.db"))
	    )
	(set! *didsomething* #t)
	(exit)  
	)
      )

  ;;======================================================================
  ;; Weird special calls that need to run *after* the server has started?
  ;;======================================================================

  (if (args:get-arg "-list-targets")
      (if (launch:setup)
          (let ((targets (common:get-runconfig-targets)))
            ;; (debug:print 1 *default-log-port* "Found "(length targets) " targets")
            (case (string->symbol (or (args:get-arg "-dumpmode") "alist"))
              ((alist)
               (for-each (lambda (x)
                           ;; (print "[" x "]"))
                           (print x))
			 targets))
              ((json)
               (json-write targets))
              (else
               (debug:print-error 0 *default-log-port* "dump output format " (args:get-arg "-dumpmode") " not supported for -list-targets")))
            (set! *didsomething* #t))))

  (if (args:get-arg "-show-runconfig")
      (let ((tl (launch:setup)))
	(push-directory *toppath*)
	(let ((data (full-runconfigs-read)))
	  ;; keep this one local
	  (cond
	   ((and (args:get-arg "-section")
		 (args:get-arg "-var"))
	    (let ((val (or (configf:lookup data (args:get-arg "-section")(args:get-arg "-var"))
			   (configf:lookup data "default" (args:get-arg "-var")))))
	      (if val (print val))))
	   ((or (not (args:get-arg "-dumpmode"))
		(string=? (args:get-arg "-dumpmode") "ini"))
	    (configf:config->ini data))
	   ((string=? (args:get-arg "-dumpmode") "sexp")
	    (pp (hash-table->alist data)))
	   ((string=? (args:get-arg "-dumpmode") "json")
	    (json-write data))
	   (else
	    (debug:print-error 0 *default-log-port* "-dumpmode of " (args:get-arg "-dumpmode") " not recognised")))
	  (set! *didsomething* #t))
	(pop-directory)))

  (if (args:get-arg "-show-config")
      (let ((tl   (launch:setup))
	    (data *configdat*)) ;; (read-config "megatest.config" #f #t)))
	(push-directory *toppath*)
	;; keep this one local
	(cond 
	 ((and (args:get-arg "-section")
	       (args:get-arg "-var"))
	  (let ((val (configf:lookup data (args:get-arg "-section")(args:get-arg "-var"))))
	    (if val (print val))))

	 ;; print just a section if only -section

	 ((equal? (args:get-arg "-dumpmode") "sexp")
	  (pp (hash-table->alist data)))
	 ((equal? (args:get-arg "-dumpmode") "json")
	  (json-write data))
	 ((or (not (args:get-arg "-dumpmode"))
	      (string=? (args:get-arg "-dumpmode") "ini"))
	  (configf:config->ini data))
	 (else
	  (debug:print-error 0 *default-log-port* "-dumpmode of " (args:get-arg "-dumpmode") " not recognised")))
	(set! *didsomething* #t)
	(pop-directory)
	(set! *time-to-exit* #t)))

  (if (args:get-arg "-show-cmdinfo")
      (if (or (args:get-arg ":value")(getenv "MT_CMDINFO"))
	  (let ((data (common:read-encoded-string (or (args:get-arg ":value")(getenv "MT_CMDINFO")))))
	    (if (equal? (args:get-arg "-dumpmode") "json")
		(json-write data)
		(pp data))
	    (set! *didsomething* #t))
	  (debug:print-info 0 *default-log-port* "environment variable MT_CMDINFO is not set")))

  ;;======================================================================
  ;; Remove old run(s)
  ;;======================================================================

  ;; since several actions can be specified on the command line the removal
  ;; is done first
  (define (operate-on action #!key (mode #f)(target-in #f)(runname-in #f)(keys-in #f)(keyvals-in #f)) ;; #f is "use default"
    (let* ((runrec (runs:runrec-make-record))
	   (target (or target-in   (common:args-get-target))) ;; eventually get rid of the call to common:args-get-target
	   (runname (or runname-in
			(args:get-arg "-runname"))) ;; eventually get rid of the get-arg calls
	   (testpatt (or (args:get-arg "-testpatt")
			 (and (eq? action 'archive) ;; if it is an archive command fallback to MT_TEST_NAME and MT_ITEMPATH
			      (common:get-full-test-name))
			 (and (eq? action 'kill-runs)
			      "%/%") ;; I'm just guessing that this is correct :(
			 (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")))
			 ))) ;;
      (cond
       ((not target)
	(debug:print-error 0 *default-log-port* "Missing required parameter for "
			   action ", you must specify -target or -reqtarg")
	(exit 1))
       ((not runname)
	(debug:print-error 0 *default-log-port* "Missing required parameter for "
			   action ", you must specify the run name pattern with -runname patt")
	(exit 2))
       ((not testpatt)
	(debug:print-error 0 *default-log-port* "Missing required parameter for "
			   action ", you must specify the test pattern with -testpatt")
	(exit 3))
       (else
	(if (not (car *configinfo*))
	    (begin
	      (debug:print-error 0 *default-log-port* "Attempted " action "on test(s) but run area config file not found")
	      (exit 1))
	    ;; put test parameters into convenient variables
	    (begin
	      ;; check for correct version, exit with message if not correct
	      (common:exit-on-version-changed)
	      (runs:operate-on  action
				target
				runname
				testpatt
				state:  (common:args-get-state)
				status: (common:args-get-status)
				new-state-status: (args:get-arg "-set-state-status")
				mode: mode)))
	(set! *didsomething* #t)))))

  (if (args:get-arg "-kill-runs")
      (general-run-call 
       "-kill-runs"
       "kill runs"
       (lambda (target runname keys keyvals)
	 (operate-on 'kill-runs mode: #f)
	 )))

  (if (args:get-arg "-kill-rerun")
      (let* ((target-patt (common:args-get-target))
             (runname-patt (args:get-arg "-runname")))
	(cond ((not target-patt)
               (debug:print-error 0 *default-log-port* "Missing target, must specify target for -kill-rerun with -target <target name>")
               (exit 1))
              ((not runname-patt)
               (debug:print-error 0 *default-log-port* "Missing runname, must specify runname for -kill-rerun with -runname <run name>")
               (exit 1))
              ((string-search "[ ,%]" target-patt)
               (debug:print-error 0 *default-log-port* "Invalid target ["target-patt"], must specify exact target (no wildcards) for -kill-rerun with -target <target name>")
               (exit 1))
              ((string-search "[ ,%]" runname-patt)
               (debug:print-error 0 *default-log-port* "Invalid runname ["runname-patt"], must specify exact runname (no wildcards) for -kill-rerun with -runname <runname name>")
               (exit 1))
              (else
               (general-run-call 
		"-kill-runs"
		"kill runs"
		(lambda (target runname keys keyvals)
                  (operate-on 'kill-runs mode: #f)
                  ))
	       
               (thread-sleep! 15))
              ;; fall thru and let "-run" loop fire
              )))


  (if (args:get-arg "-remove-runs")
      (general-run-call 
       "-remove-runs"
       "remove runs"
       (lambda (target runname keys keyvals)
	 (operate-on 'remove-runs mode: (if (args:get-arg "-keep-records")
                                            'remove-data-only
                                            'remove-all)))))

  (if (args:get-arg "-remove-keep")
      (general-run-call 
       "-remove-keep"
       "remove keep"
       (lambda (target runname keys keyvals)
	 (let ((actions (map string->symbol
                             (string-split
			      (or (args:get-arg "-actions")
				  "print")
			      ",")))) ;; default to printing the output
           (runs:remove-all-but-last-n-runs-per-target target runname
						       (string->number (args:get-arg "-remove-keep"))
						       actions: actions)))))

  (if (args:get-arg "-set-state-status")
      (general-run-call 
       "-set-state-status"
       "set state and status"
       (lambda (target runname keys keyvals)
	 (operate-on 'set-state-status))))

  (if (or (args:get-arg "-set-run-status")
	  (args:get-arg "-get-run-status"))
      (general-run-call
       "-set-run-status"
       "set run status"
       (lambda (target runname keys keyvals)
	 (let* ((runsdat  (rmt:get-runs-by-patt keys runname 
						(common:args-get-target)
						#f #f #f #f))
		(header   (vector-ref runsdat 0))
		(rows     (vector-ref runsdat 1)))
	   (if (null? rows)
	       (begin
		 (debug:print-info 0 *default-log-port* "No matching run found.")
		 (exit 1))
	       (let* ((row      (car (vector-ref runsdat 1)))
		      (run-id   (db:get-value-by-header row header "id")))
		 (if (args:get-arg "-set-run-status")
		     (rmt:set-run-status run-id (args:get-arg "-set-run-status") msg: (args:get-arg "-m"))
		     (print (rmt:get-run-status run-id))
		     )))))))

  ;;======================================================================
  ;; Query runs
  ;;======================================================================

  ;; -fields runs:id,target,runname,comment+tests:id,testname,item_path+steps
  ;;
  ;; csi> (extract-fields-constraints "runs:id,target,runname,comment+tests:id,testname,item_path+steps")
  ;;         => (("runs" "id" "target" "runname" "comment") ("tests" "id" "testname" "item_path") ("steps"))
  ;;
  ;;   NOTE: remember that the cdr will be the list you expect (cdr ("runs" "id" "target" "runname" "comment")) => ("id" "target" "runname" "comment")
  ;;         and so alist-ref will yield what you expect
  ;;
  (define (extract-fields-constraints fields-spec)
    (map (lambda (table-spec) ;; runs:id,target,runname
	   (let ((dat (string-split table-spec ":"))) ;; ("runs" "id,target,runname")
	     (if (> (length dat) 1)
		 (cons (car dat)(string-split (cadr dat) ",")) ;; "id,target,runname"
		 dat)))
	 (string-split fields-spec "+")))

  (define (get-value-by-fieldname datavec test-field-index fieldname)
    (let ((indx (hash-table-ref/default test-field-index fieldname #f)))
      (if indx
	  (if (>= indx (vector-length datavec))
	      #f ;; index too high, should raise an error I suppose
	      (vector-ref datavec indx))
	  #f)))





  (when (args:get-arg "-testdata-csv")
    (if (launch:setup)
	(let* ((keys        (rmt:get-keys)) ;; (db:get-keys dbstruct))
               (runpatt     (or (args:get-arg "-runname") "%"))
               (testpatt    (common:args-get-testpatt #f))
               (datapatt    (args:get-arg "-testdata-csv"))
               (match-data  (string-match "^([^/]+)/(.*)" (args:get-arg "-testdata-csv")))
               (categorypatt (if match-data (list-ref match-data 1) "%"))
               (setvarpatt  (if match-data
				(list-ref match-data 2)
				(args:get-arg "-testdata-csv")))
               (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") 
                                                  (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
               (header      (db:get-header runsdat))
               (access-mode (db:get-access-mode))
               (testpatt    (common:args-get-testpatt #f))
               (fields-spec (if (args:get-arg "-fields")
				(extract-fields-constraints (args:get-arg "-fields"))
				(list (cons "runs" (append keys (list "id" "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count")))
                                      (cons "tests"  db:test-record-fields) ;; "id" "testname" "test_path")
                                      (list "steps" "id" "stepname"))))
               (tests-spec  (let ((t (alist-ref "tests" fields-spec equal?)))
                              (if (and t (null? t)) ;; all fields
                                  db:test-record-fields
                                  t)))
               (adj-tests-spec (delete-duplicates (if tests-spec (cons "id" tests-spec) db:test-record-fields))) 
               (test-field-index (make-hash-table))
               (runs (db:get-rows runsdat))
               )
          (if (and tests-spec (not (null? tests-spec))) ;; do some validation and processing of the test-spec
              (let ((invalid-tests-spec (filter (lambda (x)(not (member x db:test-record-fields))) tests-spec)))
		(if (null? invalid-tests-spec)
                    ;; generate the lookup map test-field-name => index-number
                    (let loop ((hed (car adj-tests-spec))
                               (tal (cdr adj-tests-spec))
                               (idx 0))
                      (hash-table-set! test-field-index hed idx)
                      (if (not (null? tal))(loop (car tal)(cdr tal)(+ idx 1))))
                    (begin
                      (debug:print-error 0 *default-log-port* "Invalid test fields specified: " (string-intersperse invalid-tests-spec ", "))
                      (exit)))))
          (let* ((table-header (string-split "target,run,test,itempath,category,var,value,comment" ","))
		 (table-rows
                  (apply append (map  
				 (lambda (run)
                                   (let* ((target (string-intersperse (map (lambda (x)
									     (db:get-value-by-header run header x))
									   keys) "/"))
                                          (statuses (string-split (or (args:get-arg "-status") "") ","))
                                          (run-id  (db:get-value-by-header run header "id"))
                                          (runname (db:get-value-by-header run header "runname")) 
                                          (states  (string-split (or (args:get-arg "-state") "") ","))
                                          (tests   (if tests-spec
                                                       (rmt:get-tests-for-run run-id testpatt states statuses #f #f #f 'testname 'asc ;; (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc 
                                                                              ;; use qryvals if test-spec provided
                                                                              (if tests-spec
										  (string-intersperse adj-tests-spec ",")
										  ;; db:test-record-fields
										  #f)
                                                                              #f
                                                                              'normal)
                                                       '())))
                                     (apply append
                                            (map
                                             (lambda (test)
                                               (let* (
                                                      (test-id      (if (member "id"           tests-spec)(get-value-by-fieldname test test-field-index "id"          ) #f)) ;; (db:test-get-id         test))
                                                      (testname     (if (member "testname"     tests-spec)(get-value-by-fieldname test test-field-index "testname"    ) #f)) ;; (db:test-get-testname   test))
                                                      (itempath     (if (member "item_path"    tests-spec)(get-value-by-fieldname test test-field-index "item_path"   ) #f)) ;; (db:test-get-item-path  test))
                                                      (fullname     (conc testname
                                                                          (if (equal? itempath "")
                                                                              "" 
                                                                              (conc "/" itempath ))))
                                                      (testdat-raw (map vector->list (rmt:read-test-data-varpatt run-id test-id categorypatt setvarpatt)))
                                                      (testdat (filter
								(lambda (x)
                                                                  (not (equal? "logpro"
                                                                               (list-ref x 10))))
								testdat-raw)))
						 (map 
                                                  (lambda (item)
                                                    (receive (id test_id category
								 variable value expected
								 tol units comment status type)
							(apply values item)
                                                      (list target runname testname itempath category variable value comment)))
                                                  testdat)))
                                             tests))))
				 runs))))
            (print (string-join table-header ","))
            (for-each (lambda(table-row)
			(print (string-join (map ->string table-row) ",")))

                      
                      table-rows))))
    (set! *didsomething* #t)
    (set! *time-to-exit* #t))



  ;; NOTE: list-runs and list-db-targets operate on local db!!!
  ;;
  ;; IDEA: megatest list -runname blah% ...
  ;;
  (if (or (args:get-arg "-list-runs")
	  (args:get-arg "-list-db-targets"))
      (if (launch:setup)
	  (let* ((runpatt     (args:get-arg "-list-runs"))
		 (access-mode (db:get-access-mode))
		 (testpatt    (common:args-get-testpatt #f))
		 ;; (if (args:get-arg "-testpatt") 
		 ;;  	        (args:get-arg "-testpatt") 
		 ;;  	        "%"))
		 (keys        (rmt:get-keys)) ;; (db:get-keys dbstruct))
		 ;; (runsdat  (db:get-runs dbstruct runpatt #f #f '()))
		 ;; (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") (common:args-get-target) ;; (db:get-runs-by-patt dbstruct keys (or runpatt "%") (common:args-get-target)
		 ;; 		           	 #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
		 (runsdat     (rmt:get-runs-by-patt keys (or runpatt "%") 
                                                    (common:args-get-target) #f #f '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
		 (runstmp     (db:get-rows runsdat))
		 (header      (db:get-header runsdat))
		 ;; this is "-since" support. This looks at last mod times of <run-id>.db files
		 ;; and collects those modified since the -since time.
		 (runs        runstmp)
                 ;; (if (and (not (null? runstmp))
		 ;;        (args:get-arg "-since"))
		 ;;   (let ((changed-ids (db:get-changed-run-ids (string->number (args:get-arg "-since")))))
		 ;;     (let loop ((hed (car runstmp))
		 ;;   	     (tal (cdr runstmp))
		 ;;   	     (res '()))
		 ;;       (let ((new-res (if (member (db:get-value-by-header hed header "id") changed-ids)
		 ;;   		       (cons hed res)
		 ;;   		       res)))
		 ;;         (if (null? tal)
		 ;;   	  (reverse new-res)
		 ;;   	  (loop (car tal)(cdr tal) new-res)))))
		 ;;   runstmp))
		 (db-targets  (args:get-arg "-list-db-targets"))
		 (seen        (make-hash-table))
		 (dmode       (let ((d (args:get-arg "-dumpmode"))) ;; json, sexpr
				(if d (string->symbol d) #f)))
		 (data        (make-hash-table))
		 (fields-spec (if (args:get-arg "-fields")
				  (extract-fields-constraints (args:get-arg "-fields"))
				  (list (cons "runs" (append keys (list "id" "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count")))
					(cons "tests"  db:test-record-fields) ;; "id" "testname" "test_path")
					(list "steps" "id" "stepname"))))
		 (runs-spec   (let ((r (alist-ref "runs"  fields-spec equal?))) ;; the check is now unnecessary
				(if (and r (not (null? r))) r (list "id" ))))
		 (tests-spec  (let ((t (alist-ref "tests" fields-spec equal?)))
				(if (and t (null? t)) ;; all fields
				    db:test-record-fields
				    t)))
		 (adj-tests-spec (delete-duplicates (if tests-spec (cons "id" tests-spec) db:test-record-fields))) ;; '("id"))))
		 (steps-spec  (alist-ref "steps" fields-spec equal?))
		 (test-field-index (make-hash-table)))
	    (if (and (args:get-arg "-dumpmode")
		     (not (member (args:get-arg "-dumpmode") '("sexpr" "json" "ods" "list"))))
		(begin
		  (debug:print 0 *default-log-port* "ERROR: dumpmode "(args:get-arg "-dumpmode")" not recognised. Use sexpr, json, ods or list")
		  (exit)))
	    (if (and tests-spec (not (null? tests-spec))) ;; do some validation and processing of the test-spec
		(let ((invalid-tests-spec (filter (lambda (x)(not (member x db:test-record-fields))) tests-spec)))
		  (if (null? invalid-tests-spec)
		      ;; generate the lookup map test-field-name => index-number
		      (let loop ((hed (car adj-tests-spec))
				 (tal (cdr adj-tests-spec))
				 (idx 0))
			(hash-table-set! test-field-index hed idx)
			(if (not (null? tal))(loop (car tal)(cdr tal)(+ idx 1))))
		      (begin
			(debug:print-error 0 *default-log-port* "Invalid test fields specified: " (string-intersperse invalid-tests-spec ", "))
			(exit)))))
	    ;; Each run
	    (for-each 
	     (lambda (run)
	       (let ((targetstr (string-intersperse (map (lambda (x)
							   (db:get-value-by-header run header x))
							 keys) "/")))
		 (if db-targets
		     (if (not (hash-table-ref/default seen targetstr #f))
			 (begin
			   (hash-table-set! seen targetstr #t)
			   ;; (print "[" targetstr "]"))))
			   (if (not dmode)
			       (print targetstr)
			       (hash-table-set! data "targets" (cons targetstr (hash-table-ref/default data "targets" '())))
			       )))
		     (let* ((run-id  (db:get-value-by-header run header "id"))
			    (runname (db:get-value-by-header run header "runname")) 
			    (states  (string-split (or (args:get-arg "-state") "") ","))
			    (statuses (string-split (or (args:get-arg "-status") "") ","))
			    (tests   (if tests-spec
					 (rmt:get-tests-for-run run-id testpatt states statuses #f #f #f 'testname 'asc ;; (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc 
								;; use qryvals if test-spec provided
								(if tests-spec
								    (string-intersperse adj-tests-spec ",")
								    ;; db:test-record-fields
								    #f)
								#f
								'normal)
					 '())))
		       (case dmode
			 ((json ods sexpr)
			  (if runs-spec
			      (for-each 
			       (lambda (field-name)
				 (mutils:hierhash-set! data (conc (db:get-value-by-header run header field-name)) targetstr runname "meta" field-name))
			       runs-spec)))
			 ;; (mutils:hierhash-set! data (db:get-value-by-header run header "status")     targetstr runname "meta" "status"     )
			 ;; (mutils:hierhash-set! data (db:get-value-by-header run header "state")      targetstr runname "meta" "state"      )
			 ;; (mutils:hierhash-set! data (conc (db:get-value-by-header run header "id"))  targetstr runname "meta" "id"         )
			 ;; (mutils:hierhash-set! data (db:get-value-by-header run header "event_time") targetstr runname "meta" "event_time" )
			 ;; (mutils:hierhash-set! data (db:get-value-by-header run header "comment")    targetstr runname "meta" "comment"    )
			 ;; ;; add last entry twice - seems to be a bug in hierhash?
			 ;; (mutils:hierhash-set! data (db:get-value-by-header run header "comment")    targetstr runname "meta" "comment"    )
			 ((#f list)
			  (if (null? runs-spec)
			      (print "Run: " targetstr "/" runname 
				     " status: " (db:get-value-by-header run header "state")
				     " run-id: " run-id ", number tests: " (length tests)
				     " event_time: " (db:get-value-by-header run header "event_time"))
			      (begin
				(if (not (member "target" runs-spec))
			            ;; (display (conc "Target: " targetstr))
			            (display (conc "Run: " targetstr "/" runname " ")))
				(for-each
				 (lambda (field-name)
				   (if (equal? field-name "target")
				       (display (conc "target: " targetstr " "))
				       (display (conc field-name ": " (db:get-value-by-header run header (conc field-name)) " "))))
				 runs-spec)
				(newline))))
			 (else
			  (debug:print 0 *default-log-port* "ERROR: dumpmode "(args:get-arg "-dumpmode")" not recognised. Use sexpr, json, ods or list")
			  ))
		       
		       (for-each 
			(lambda (test)
		      	  (common:debug-handle-exceptions #f
							  exn
							  (begin
							    (debug:print-error 0 *default-log-port* "Bad data in test record? " test)
							    (debug:print-error 5 *default-log-port* "exn=" (condition->list exn))
							    (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
							    (print-call-chain (current-error-port)))
							  (let* ((test-id      (if (member "id"           tests-spec)(get-value-by-fieldname test test-field-index "id"          ) #f)) ;; (db:test-get-id         test))
								 (testname     (if (member "testname"     tests-spec)(get-value-by-fieldname test test-field-index "testname"    ) #f)) ;; (db:test-get-testname   test))
								 (itempath     (if (member "item_path"    tests-spec)(get-value-by-fieldname test test-field-index "item_path"   ) #f)) ;; (db:test-get-item-path  test))
								 (comment      (if (member "comment"      tests-spec)(get-value-by-fieldname test test-field-index "comment"     ) #f)) ;; (db:test-get-comment    test))
								 (tstate       (if (member "state"        tests-spec)(get-value-by-fieldname test test-field-index "state"       ) #f)) ;; (db:test-get-state      test))
								 (tstatus      (if (member "status"       tests-spec)(get-value-by-fieldname test test-field-index "status"      ) #f)) ;; (db:test-get-status     test))
								 (event-time   (if (member "event_time"   tests-spec)(get-value-by-fieldname test test-field-index "event_time"  ) #f)) ;; (db:test-get-event_time test))
								 (rundir       (if (member "rundir"       tests-spec)(get-value-by-fieldname test test-field-index "rundir"      ) #f)) ;; (db:test-get-rundir     test))
								 (final_logf   (if (member "final_logf"   tests-spec)(get-value-by-fieldname test test-field-index "final_logf"  ) #f)) ;; (db:test-get-final_logf test))
								 (run_duration (if (member "run_duration" tests-spec)(get-value-by-fieldname test test-field-index "run_duration") #f)) ;; (db:test-get-run_duration test))
								 (fullname     (conc testname
										     (if (equal? itempath "")
											 "" 
											 (conc "(" itempath ")")))))
							    (case dmode
							      ((json ods sexpr)
							       (if tests-spec
								   (for-each
								    (lambda (field-name)
								      (mutils:hierhash-set! data  (get-value-by-fieldname test test-field-index field-name) targetstr runname "data" (conc test-id) field-name))
								    tests-spec)))
							      ;; ;; (mutils:hierhash-set! data  fullname   targetstr runname "data" (conc test-id) "tname"     )
							      ;;  (mutils:hierhash-set! data  testname   targetstr runname "data" (conc test-id) "testname"  )
							      ;;  (mutils:hierhash-set! data  itempath   targetstr runname "data" (conc test-id) "itempath"  )
							      ;;  (mutils:hierhash-set! data  comment    targetstr runname "data" (conc test-id) "comment"   )
							      ;;  (mutils:hierhash-set! data  tstate     targetstr runname "data" (conc test-id) "state"     )
							      ;;  (mutils:hierhash-set! data  tstatus    targetstr runname "data" (conc test-id) "status"    )
							      ;;  (mutils:hierhash-set! data  rundir     targetstr runname "data" (conc test-id) "rundir"    )
							      ;;  (mutils:hierhash-set! data  final_logf targetstr runname "data" (conc test-id) "final_logf")
							      ;;  (mutils:hierhash-set! data  run_duration targetstr runname "data" (conc test-id) "run_duration")
							      ;;  (mutils:hierhash-set! data  event-time targetstr runname "data" (conc test-id) "event_time")
							      ;;  ;; add last entry twice - seems to be a bug in hierhash?
							      ;;  (mutils:hierhash-set! data  event-time targetstr runname "data" (conc test-id) "event_time")
							      ;;  )
							      (else
							       (if (and tstate tstatus event-time)
								   (format #t
									   "  Test: ~25a State: ~15a Status: ~15a Runtime: ~5@as Time: ~22a Host: ~10a\n"
									   (if fullname fullname "")
									   (if tstate   tstate   "")
									   (if tstatus  tstatus  "")
									   (get-value-by-fieldname test test-field-index "run_duration");;(if test     (db:test-get-run_duration test) "")
									   (if event-time event-time "")
									   (get-value-by-fieldname test test-field-index "host")) ;;(if test (db:test-get-host test)) "")
								   (print "  Test: " fullname
									  (if tstate  (conc " State: "  tstate)  "")
									  (if tstatus (conc " Status: " tstatus) "")
									  (if (get-value-by-fieldname test test-field-index "run_duration")
									      (conc " Runtime: " (get-value-by-fieldname test test-field-index "run_duration"))
									      "")
									  (if event-time (conc " Time: " event-time) "")
									  (if (get-value-by-fieldname test test-field-index "host")
									      (conc " Host: " (get-value-by-fieldname test test-field-index "host"))
									      "")))
							       (if (not (or (equal? (get-value-by-fieldname test test-field-index "status") "PASS")
									    (equal? (get-value-by-fieldname test test-field-index "status") "WARN")
									    (equal? (get-value-by-fieldname test test-field-index "state")  "NOT_STARTED")))
								   (begin
								     (print   (if (get-value-by-fieldname test test-field-index "cpuload")
										  (conc "         cpuload:  "   (get-value-by-fieldname test test-field-index "cpuload"))
										  "") ;; (db:test-get-cpuload test)
									      (if (get-value-by-fieldname test test-field-index "diskfree")
										  (conc "\n         diskfree: " (get-value-by-fieldname test test-field-index "diskfree")) ;; (db:test-get-diskfree test)
										  "")
									      (if (get-value-by-fieldname test test-field-index "uname")
										  (conc "\n         uname:    " (get-value-by-fieldname test test-field-index "uname")) ;; (db:test-get-uname test)
										  "")
									      (if (get-value-by-fieldname test test-field-index "rundir")
										  (conc "\n         rundir:   " (get-value-by-fieldname test test-field-index "rundir")) ;; (db:test-get-rundir test)
										  "")
									      ;;					     "\n         rundir:   " (get-value-by-fieldname test test-field-index "") ;; (sdb:qry 'getstr ;; (filedb:get-path *fdb* 
									      ;; 					     (db:test-get-rundir test) ;; )
									      )
								     ;; Each test
								     ;; DO NOT remote run
								     (let ((steps (rmt:get-steps-for-test run-id (db:test-get-id test)))) ;; (db:get-steps-for-test dbstruct run-id (db:test-get-id test))))
								       (for-each 
									(lambda (step)
									  (format #t 
										  "    Step: ~20a State: ~10a Status: ~10a Time ~22a\n"
										  (tdb:step-get-stepname step)
										  (tdb:step-get-state step)
										  (tdb:step-get-status step)
										  (tdb:step-get-event_time step)))
									steps)))))))))
			(if (args:get-arg "-sort")
			    (sort tests
				  (lambda (a-test b-test)
				    (let* ((key    (args:get-arg "-sort"))
					   (first  (get-value-by-fieldname a-test test-field-index key))
					   (second (get-value-by-fieldname b-test test-field-index key)))
				      ((cond 
					((and (number? first)(number? second)) <)
					((and (string? first)(string? second)) string<=?)
					(else equal?))
				       first second))))
			    tests))))))
	     runs)
	    (case dmode
	      ((json)  (json-write data))
	      ((sexpr) (pp (common:to-alist data))))
	    (let* ((metadat-fields (delete-duplicates
				    (append keys '( "runname" "time" "owner" "pass_count" "fail_count" "state" "status" "comment" "id"))))
		   (run-fields    '(
				    "testname"
				    "item_path"
				    "state"
				    "status"
				    "comment"
				    "event_time"
				    "host"
				    "run_id"
				    "run_duration"
				    "attemptnum"
				    "id"
				    "archived"
				    "diskfree"
				    "cpuload"
				    "final_logf"
				    "shortdir"
				    "rundir"
				    "uname"
				    )
				  )
		   (newdat          (common:to-alist data))
		   (allrundat       (if (null? newdat)
					'()
					(car (map cdr newdat)))) ;; (car (map cdr (car (map cdr newdat)))))
		   (runs            (append
				     (list "runs" ;; sheetname
					   metadat-fields)
				     (map (lambda (run)
					    ;; (print "run: " run)
					    (let* ((runname (car run))
						   (rundat  (cdr run))
						   (metadat (let ((tmp (assoc "meta" rundat)))
							      (if tmp (cdr tmp) #f))))
					      ;; (print "runname: " runname "\n\nrundat: " )(pp rundat)(print "\n\nmetadat: ")(pp metadat)
					      (if metadat
						  (map (lambda (field)
							 (let ((tmp (assoc field metadat)))
							   (if tmp (cdr tmp) "")))
						       metadat-fields)
						  (begin
						    (debug:print 0 *default-log-port* "WARNING: meta data for run " runname " not found")
						    '()))))
					  allrundat)))
		   ;; '( ( "target" ( "runname" ( "data" ( "runid" ( "id . "37" ) ( ... ))))
		   (run-pages      (map (lambda (targdat)
					  (let* ((target  (car targdat))
						 (runsdat (cdr targdat)))
					    (if runsdat
						(map (lambda (rundat)
						       (let* ((runname  (car rundat))
							      (rundat   (cdr rundat))
							      (testsdat (let ((tmp (assoc "data" rundat)))
									  (if tmp (cdr tmp) #f))))
							 (if testsdat
							     (let ((tests (map (lambda (test)
										 (let* ((test-id  (car test))
											(test-dat (cdr test)))
										   (map (lambda (field)
											  (let ((tmp (assoc field test-dat)))
											    (if tmp (cdr tmp) "")))
											run-fields)))
									       testsdat)))
							       ;; (print "Target: " target "/" runname " tests:")
							       ;; (pp tests)
							       (cons (conc target "/" runname)
								     (cons (list (conc target "/" runname))
									   (cons '()
										 (cons run-fields tests)))))
							     (begin
							       (debug:print 4 *default-log-port* "WARNING: run " target "/" runname " appears to have no data")
							       ;; (pp rundat)
							       '()))))
						     runsdat)
						'())))
					newdat)) ;; we use newdat to get target
		   (sheets         (filter (lambda (x)
					     (not (null? x)))
					   (cons runs (map car run-pages)))))
	      ;; (print "allrundat:")
	      ;; (pp allrundat)
	      ;; (print "runs:")
	      ;; (pp runs)
					;(print "sheets: ")
	      ;; (pp sheets)
	      (if (eq? dmode 'ods)
		  (let* ((tempdir    (conc "/tmp/" (current-user-name) "/" (random 10000) "_" (current-process-id)))
			 (outputfile (or (args:get-arg "-o") "out.ods"))
			 (ouf        (if (string-match (regexp "^[/~]+.*") outputfile) ;; full path?
					 outputfile
					 (begin
					   (debug:print 0 *default-log-port* "WARNING: path given, " outputfile " is relative, prefixing with current directory")
					   (conc (current-directory) "/" outputfile)))))
		    (create-directory tempdir #t)
		    (ods:list->ods tempdir ouf sheets))))
	    ;; (system (conc "rm -rf " tempdir))
	    (set! *didsomething* #t)
            (set! *time-to-exit* #t)
            ) ;; end if true branch (end of a let)
          ) ;; end if
      ) ;; end if -list-runs

  ;; list-waivers
  (if (and (args:get-arg "-list-waivers")
	   (launch:setup))
      (let* ((runpatt     (or (args:get-arg "-runname") "%"))
	     (testpatt    (common:args-get-testpatt #f))
	     (keys        (rmt:get-keys)) 
	     (runsdat     (rmt:get-runs-by-patt
			   keys runpatt 
			   (common:args-get-target) #f #f
			   '("id" "runname" "state" "status" "owner" "event_time" "comment") 0))
	     (runs        (db:get-rows runsdat))
	     (header      (db:get-header runsdat))
	     (results     (make-hash-table))  ;; [target] ( (testname/itempath . "comment") ... )
	     (addtest     (lambda (target testname itempath comment)
			    (hash-table-set! results target (cons (cons (conc testname "/" itempath) comment)
								  (hash-table-ref/default results target '())))))
	     (last-target #f))
	(for-each
	 (lambda (run)
	   (let* ((run-id  (db:get-value-by-header run header "id"))
		  (target  (rmt:get-target run-id))
		  (runname (db:get-value-by-header run header "runname")) 
		  (tests   (rmt:get-tests-for-run
			    run-id testpatt '("COMPLETED") '("WAIVED") #f #f #f 'testname 'asc							     ;; use qryvals if test-spec provided
			    #f #f #f)))
	     (if (not (equal? target last-target))
		 (print "[" target "]"))
	     (set! last-target target)
	     (print "# " runname)
	     (for-each
	      (lambda (testdat)
		(let* ((testfullname (conc (db:test-get-testname testdat)
					   (if (equal? "" (db:test-get-item-path testdat))
					       ""
					       (conc "/" (db:test-get-item-path testdat)))
					   )))
		  (print testfullname " " (db:test-get-comment testdat))))
	      tests)))
	 runs)
	(set! *didsomething* #t)))
  
  ;;======================================================================
  ;; full run
  ;;======================================================================

  (define (handle-run-requests target runname keys keyvals need-clean)	 
    (if (or (args:get-arg "-kill-rerun") (args:get-arg "-rerun-clean")) ;; first set states/statuses correct
	;; For rerun-clean do we or do we not support the testpatt?
	(let ((states   (or (configf:lookup *configdat* "validvalues" "cleanrerun-states")
			    "KILLREQ,KILLED,UNKNOWN,INCOMPLETE,STUCK,NOT_STARTED"))
	      (statuses (or (configf:lookup *configdat* "validvalues" "cleanrerun-statuses")
			    "FAIL,INCOMPLETE,ABORT,CHECK,DEAD,PREQ_FAIL,PREQ_DISCARDED")))
	  (hash-table-set! args:arg-hash "-preclean" #t)
	  (runs:operate-on 'set-state-status
			   target
			   (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			   ;; "%" ;; (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			   (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			   state:  states
			   ;; status: statuses
			   new-state-status: "NOT_STARTED,n/a")
	  (runs:clean-cache target runname *toppath*)
	  (runs:operate-on 'set-state-status
			   target
			   (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			   ;; "%" ;; (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			   (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt")
			   ;; state:  states
			   status: statuses
			   new-state-status: "NOT_STARTED,n/a")))
    ;; RERUN ALL
    (if (args:get-arg "-rerun-all") ;; first set states/statuses correct
	(let* ((rconfig (full-runconfigs-read)))
	  (hash-table-set! args:arg-hash "-preclean" #t)
	  (runs:operate-on 'set-state-status
			   target
			   (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			   (common:args-get-testpatt rconfig) ;; (args:get-arg "-testpatt")
			   state:  #f
			   ;; status: statuses
			   new-state-status: "NOT_STARTED,n/a")
	  (runs:clean-cache target runname *toppath*)
	  (runs:operate-on 'set-state-status
			   target
			   (common:args-get-runname)  ;; (or (args:get-arg "-runname")(args:get-arg ":runname"))
			   (common:args-get-testpatt rconfig) ;; (args:get-arg "-testpatt")
			   ;; state:  states
			   status: #f
			   new-state-status: "NOT_STARTED,n/a")))
    (let* ((config-reruns      (let ((x (configf:lookup *configdat* "setup" "reruns")))
				 (if x (string->number x) #f)))
	   (rerun-cnt (if config-reruns
			  config-reruns
			  1)))

      (runs:run-tests target
		      runname
		      #f ;; (common:args-get-testpatt #f)
		      ;; (or (args:get-arg "-testpatt")
		      ;;     "%")
		      (current-user-name)
		      args:arg-hash
		      run-count: rerun-cnt)))

  ;; get lock in db for full run for this directory
  ;; for all tests with deps
  ;;   walk tree of tests to find head tasks
  ;;   add head tasks to task queue
  ;;   add dependant tasks to task queue 
  ;;   add remaining tasks to task queue
  ;; for each task in task queue
  ;;   if have adequate resources
  ;;     launch task
  ;;   else
  ;;     put task in deferred queue
  ;; if still ok to run tasks
  ;;   process deferred tasks per above steps

  ;; run all tests are are Not COMPLETED and PASS or CHECK
  (if (or (args:get-arg "-runall")
	  (args:get-arg "-run")
	  (args:get-arg "-rerun-clean")
	  (args:get-arg "-rerun-all")
	  (args:get-arg "-runtests")
          (args:get-arg "-kill-rerun"))
      (let ((need-clean (or (args:get-arg "-rerun-clean")
                            (args:get-arg "-rerun-all")))
	    (orig-cmdline (string-intersperse (argv) " ")))
	(general-run-call 
	 "-runall"
	 "run all tests"
	 (lambda (target runname keys keyvals)
	   (if (or (string-search "%" target)
		   (string-search "%" runname)) ;; we are being asked to re-run multiple runs
	       (let* ((run-specs (rmt:simple-get-runs runname #f #f target #f))) ;; list of simple-run records
		 (debug:print-info 0 *default-log-port* "Pattern supplied for target or runname with "
				   (length run-specs) " matches found. Running each in turn.")
		 (if (null? run-specs)
		     (debug:print 0 *default-log-port* "WARNING: No runs match target " target " and runname " runname))
		 (for-each (lambda (spec) 
			     (let* ((precmd     (if (args:get-arg "-precmd")(conc (args:get-arg "-precmd") " ") ""))
				    (newcmdline (conc
						 precmd
						 (string-substitute
						  (conc "target " target)
						  (conc "target " (simple-run-target spec))
						  (string-substitute
						   (conc "runname " runname)
						   (conc "runname " (simple-run-runname spec))
						   orig-cmdline)))))
			       (debug:print 0 *default-log-port* "ORIG: " orig-cmdline)
			       (debug:print 0 *default-log-port* "NEW:  " newcmdline)
			       (system newcmdline)))
			   run-specs))
	       (handle-run-requests target runname keys keyvals need-clean))))
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; run one test
  ;;======================================================================

  ;; 1. find the config file
  ;; 2. change to the test directory
  ;; 3. update the db with "test started" status, set running host
  ;; 4. process launch the test
  ;;    - monitor the process, update stats in the db every 2^n minutes
  ;; 5. as the test proceeds internally it calls megatest as each step is
  ;;    started and completed
  ;;    - step started, timestamp
  ;;    - step completed, exit status, timestamp
  ;; 6. test phone home
  ;;    - if test run time > allowed run time then kill job
  ;;    - if cannot access db > allowed disconnect time then kill job

  ;; == duplicated == (if (or (args:get-arg "-run")(args:get-arg "-runtests"))
  ;; == duplicated ==   (general-run-call 
  ;; == duplicated ==    "-runtests" 
  ;; == duplicated ==    "run a test" 
  ;; == duplicated ==    (lambda (target runname keys keyvals)
  ;; == duplicated ==      ;;
  ;; == duplicated ==      ;; May or may not implement it this way ...
  ;; == duplicated ==      ;;
  ;; == duplicated ==      ;; Insert this run into the tasks queue
  ;; == duplicated ==      ;; (open-run-close tasks:add tasks:open-db 
  ;; == duplicated ==      ;;    	     "runtests" 
  ;; == duplicated ==      ;;    	     user
  ;; == duplicated ==      ;;    	     target
  ;; == duplicated ==      ;;    	     runname
  ;; == duplicated ==      ;;    	     (args:get-arg "-runtests")
  ;; == duplicated ==      ;;    	     #f))))
  ;; == duplicated ==      (runs:run-tests target
  ;; == duplicated == 		     runname
  ;; == duplicated == 		     (common:args-get-testpatt #f) ;; (args:get-arg "-runtests")
  ;; == duplicated == 		     user
  ;; == duplicated == 		     args:arg-hash))))

  ;;======================================================================
  ;; Rollup into a run
  ;;======================================================================

;;   (if (args:get-arg "-rollup")
;;       (general-run-call 
;;        "-rollup" 
;;        "rollup tests" 
;;        (lambda (target runname keys keyvals)
;; 	 (runs:rollup-run keys
;; 			  keyvals
;; 			  (or (args:get-arg "-runname")(args:get-arg ":runname") )
;; 			  user))))

  ;;======================================================================
  ;; Lock or unlock a run
  ;;======================================================================

  (if (or (args:get-arg "-lock")(args:get-arg "-unlock"))
      (general-run-call 
       (if (args:get-arg "-lock") "-lock" "-unlock")
       "lock/unlock tests" 
       (lambda (target runname keys keyvals)
	 (runs:handle-locking 
	  target
	  keys
	  (or (args:get-arg "-runname")(args:get-arg ":runname") )
	  (args:get-arg "-lock")
	  (args:get-arg "-unlock")
	  (current-user-name)))))

  ;;======================================================================
  ;; Get paths to tests
  ;;======================================================================
  ;; Get test paths matching target, runname, and testpatt
  (if (or (args:get-arg "-test-files")(args:get-arg "-test-paths"))
      ;; if we are in a test use the MT_CMDINFO data
      (if (getenv "MT_CMDINFO")
	  (let* ((startingdir (current-directory))
		 (cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
		 (transport (assoc/default 'transport cmdinfo))
		 (testpath  (assoc/default 'testpath  cmdinfo))
		 (test-name (assoc/default 'test-name cmdinfo))
		 (runscript (assoc/default 'runscript cmdinfo))
		 (db-host   (assoc/default 'db-host   cmdinfo))
		 (run-id    (assoc/default 'run-id    cmdinfo))
		 (itemdat   (assoc/default 'itemdat   cmdinfo))
		 (state     (args:get-arg ":state"))
		 (status    (args:get-arg ":status"))
		 ;;(target    (args:get-arg "-target"))
		 (target    (common:args-get-target))
		 (toppath   (assoc/default 'toppath   cmdinfo)))
	    (change-directory toppath)
	    (if (not target)
		(begin
		  (debug:print-error 0 *default-log-port* "-target is required.")
		  (exit 1)))
	    (if (not (launch:setup))
		(begin
		  (debug:print 0 *default-log-port* "Failed to setup, giving up on -test-paths or -test-files, exiting")
		  (exit 1)))
	    (let* ((keys     (rmt:get-keys))
		   ;; db:test-get-paths must not be run remote
		   (paths    (tests:test-get-paths-matching keys target (args:get-arg "-test-files"))))
	      (set! *didsomething* #t)
	      (for-each (lambda (path)
			  (if (common:file-exists? path)
			      (print path)))	
			paths)))
	  ;; else do a general-run-call
	  (general-run-call 
	   "-test-files"
	   "Get paths to test"
	   (lambda (target runname keys keyvals)
	     (let* ((db       #f)
		    ;; DO NOT run remote
		    (paths    (tests:test-get-paths-matching keys target (args:get-arg "-test-files"))))
	       (for-each (lambda (path)
			   (print path))
			 paths))))))

  ;;======================================================================
  ;; Utils for test areas
  ;;======================================================================

  (if (args:get-arg "-regen-testfiles")
      (if (getenv "MT_TEST_RUN_DIR")
	  (begin
	    (launch:setup)
	    (change-directory (getenv "MT_TEST_RUN_DIR"))
	    (let* ((testname (getenv "MT_TEST_NAME"))
		   (itempath (getenv "MT_ITEMPATH")))
	      (launch:extract-scripts-logpro (getenv "MT_TEST_RUN_DIR") testname itempath #f))
	    (set! *didsomething* #t))
	  (debug:print 0 *default-log-port* "ERROR: Must run -regen-testfiles in a test environment (i.e. test xterm from dashboard)")))
  
  ;;======================================================================
  ;; Archive tests
  ;;======================================================================
  ;; Archive tests matching target, runname, and testpatt
  (if (equal? (args:get-arg "-archive") "replicate-db")
      (begin
        ;; check if source
        ;; check if megatest.db exist
        (launch:setup)
        (if (not (args:get-arg "-source"))
            (begin 
              (debug:print-info 1 *default-log-port* "Missing required argument -source <archive path>")
              (exit 1)))
        (if (common:file-exists? (conc  *toppath* "/megatest.db"))
            (begin  
              (debug:print-info 1 *default-log-port* "File " (conc  *toppath* "/megatest.db") " already exists. Please remove it before trying to replicate db")
              (exit 1)))
        (if (and (common:make-tmpdir-name *toppath* "") (> (length (directory   (common:make-tmpdir-name *toppath* "") #f)) 0))
            (begin
              (debug:print-info 1 *default-log-port* (common:make-tmpdir-name *toppath* "") " not empty. Please remove it before trying to replicate db")
              (exit 1)))    
        ;; check if timestamp 
        (let* ((source (args:get-arg "-source"))
               (src     (if (not (equal? (substring source 0 1) "/"))
                            (conc (current-directory) "/" source)
                            source))
               (ts (if (args:get-arg "-time-stamp")   (args:get-arg "-time-stamp") "latest")))
          (if  (common:directory-exists? src)
               (begin 
                 (archive:restore-db src ts)
		 (set! *didsomething* #t))
	       (begin
		 (debug:print-error 1 *default-log-port* "Path " source " not found")
		 (exit 1))))))   
  ;; else do a general-run-call
  (if (and (args:get-arg "-archive") (not (equal? (args:get-arg "-archive") "replicate-db"))) 
      (begin
	;; for the archive get we need to preserve the starting dir as part of the target path
	(if (and (args:get-arg "-dest")
		 (not (equal? (substring (args:get-arg "-dest") 0 1) "/")))
	    (let ((newpath  (conc (current-directory) "/" (args:get-arg "-dest"))))
	      (debug:print-info 1 *default-log-port* "Preserving original path to destination, was " (args:get-arg "-dest") ", now " newpath)
	      (hash-table-set! args:arg-hash "-dest" newpath)))
	(general-run-call 
	 "-archive"
	 "Archive"
	 (lambda (target runname keys keyvals)
	   (operate-on 'archive target-in: target runname-in: runname )))))

  ;;======================================================================
  ;; Extract a spreadsheet from the runs database
  ;;======================================================================

  (if (args:get-arg "-extract-ods")
      (general-run-call
       "-extract-ods"
       "Make ods spreadsheet"
       (lambda (target runname keys keyvals)
	 (let ((dbstruct   (make-dbr:dbstruct areapath: *toppath* local: #t))
	       (outputfile (args:get-arg "-extract-ods"))
	       (runspatt   (or (args:get-arg "-runname")(args:get-arg ":runname")))
	       (pathmod    (args:get-arg "-pathmod")))
	   ;; (keyvalalist (keys->alist keys "%")))
	   (debug:print 2 *default-log-port* "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvals: " keyvals)
	   (db:extract-ods-file dbstruct outputfile keyvals (if runspatt runspatt "%") pathmod)
	   (db:close-all dbstruct)
	   (set! *didsomething* #t)))))

  ;;======================================================================
  ;; execute the test
  ;;    - gets called on remote host
  ;;    - receives info from the -execute param
  ;;    - passes info to steps via MT_CMDINFO env var (future is to use a dot file)
  ;;    - gathers host info and 
  ;;======================================================================

  (if (args:get-arg "-execute")
      (begin
	(launch:execute (args:get-arg "-execute"))
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; recover from a test where the managing mtest was killed but the underlying
  ;; process might still be salvageable
  ;;======================================================================

  (if (args:get-arg "-recover-test")
      (let* ((params (string-split (args:get-arg "-recover-test") ",")))
	(if (> (length params) 1) ;; run-id and test-id
	    (let ((run-id (string->number (car params)))
		  (test-id (string->number (cadr params))))
	      (if (and run-id test-id)
		  (begin
		    (launch:recover-test run-id test-id)
		    (set! *didsomething* #t))
		  (begin
		    (debug:print-error 0 *default-log-port* "bad run-id or test-id, must be integers")
		    (exit 1)))))))

  ;;======================================================================
  ;; Test commands (i.e. for use inside tests)
  ;;======================================================================

  (define (megatest:step step state status logfile msg)
    (if (not (getenv "MT_CMDINFO"))
	(begin
	  (debug:print-error 0 *default-log-port* "MT_CMDINFO env var not set, -step must be called *inside* a megatest invoked environment!")
	  (exit 5))
	(let* ((cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
	       (transport (assoc/default 'transport cmdinfo))
	       (testpath  (assoc/default 'testpath  cmdinfo))
	       (test-name (assoc/default 'test-name cmdinfo))
	       (runscript (assoc/default 'runscript cmdinfo))
	       (db-host   (assoc/default 'db-host   cmdinfo))
	       (run-id    (assoc/default 'run-id    cmdinfo))
	       (test-id   (assoc/default 'test-id   cmdinfo))
	       (itemdat   (assoc/default 'itemdat   cmdinfo))
	       (work-area (assoc/default 'work-area cmdinfo))
	       (db        #f))
	  (change-directory testpath)
	  (if (not (launch:setup))
	      (begin
		(debug:print 0 *default-log-port* "Failed to setup, exiting")
		(exit 1)))
	  (if (and state status)
	      (let ((comment (launch:load-logpro-dat run-id test-id step)))
		;; (rmt:test-set-log! run-id test-id (conc stepname ".html"))))
		(rmt:teststep-set-status! run-id test-id step state status (or comment msg) logfile))
	      (begin
		(debug:print-error 0 *default-log-port* "You must specify :state and :status with every call to -step")
		(exit 6))))))

  (if (args:get-arg "-step")
      (begin
	(thread-sleep! 1.5)
	(megatest:step 
	 (args:get-arg "-step")
	 (or (args:get-arg "-state")(args:get-arg ":state"))
	 (or (args:get-arg "-status")(args:get-arg ":status"))
	 (args:get-arg "-setlog")
	 (args:get-arg "-m"))
	;; (if db (sqlite3:finalize! db))
	(set! *didsomething* #t)
	(thread-sleep! 1.5)))
  
  (if (or (args:get-arg "-setlog")       ;; since setting up is so costly lets piggyback on -test-status
	  ;;     (not (args:get-arg "-step")))  ;; -setlog may have been processed already in the "-step" previous
	  ;;     NEW POLICY - -setlog sets test overall log on every call.
	  (args:get-arg "-set-toplog")
	  (args:get-arg "-test-status")
	  (args:get-arg "-set-values")
	  (args:get-arg "-load-test-data")
	  (args:get-arg "-runstep")
	  (args:get-arg "-summarize-items"))
      (if (not (getenv "MT_CMDINFO"))
	  (begin
	    (debug:print-error 0 *default-log-port* "MT_CMDINFO env var not set, commands -test-status, -runstep and -setlog must be called *inside* a megatest environment!")
	    (exit 5))
	  (let* ((startingdir (current-directory))
		 (cmdinfo   (common:read-encoded-string (getenv "MT_CMDINFO")))
		 (transport (assoc/default 'transport cmdinfo))
		 (testpath  (assoc/default 'testpath  cmdinfo))
		 (test-name (assoc/default 'test-name cmdinfo))
		 (runscript (assoc/default 'runscript cmdinfo))
		 (db-host   (assoc/default 'db-host   cmdinfo))
		 (run-id    (assoc/default 'run-id    cmdinfo))
		 (test-id   (assoc/default 'test-id   cmdinfo))
		 (itemdat   (assoc/default 'itemdat   cmdinfo))
		 (work-area (assoc/default 'work-area cmdinfo))
		 (db        #f) ;; (open-db))
		 (state     (args:get-arg ":state"))
		 (status    (args:get-arg ":status"))
		 (stepname  (args:get-arg "-step")))
	    (if (not (launch:setup))
		(begin
		  (debug:print 0 *default-log-port* "Failed to setup, exiting")
		  (exit 1)))

	    (if (args:get-arg "-runstep")(debug:print-info 1 *default-log-port* "Running -runstep, first change to directory " work-area))
	    (change-directory work-area)
	    ;; can setup as client for server mode now

	    (if (args:get-arg "-load-test-data")
		;; has sub commands that are rdb:
		;; DO NOT put this one into either rmt: or open-run-close
		(tdb:load-test-data run-id test-id))
	    (if (args:get-arg "-setlog")
		(let ((logfname (args:get-arg "-setlog")))
		  (rmt:test-set-log! run-id test-id logfname)))
	    (if (args:get-arg "-set-toplog")
		;; DO NOT run remote
		(tests:test-set-toplog! run-id test-name (args:get-arg "-set-toplog")))
	    (if (args:get-arg "-summarize-items")
		;; DO NOT run remote
		(tests:summarize-items run-id test-id test-name #t)) ;; do force here
	    (if (args:get-arg "-runstep")
		(if (null? remargs)
		    (begin
		      (debug:print-error 0 *default-log-port* "nothing specified to run!")
		      (if db (sqlite3:finalize! db))
		      (exit 6))
		    (let* ((stepname   (args:get-arg "-runstep"))
			   (logprofile (args:get-arg "-logpro"))
			   (logfile    (conc stepname ".log"))
			   (cmd        (if (null? remargs) #f (car remargs)))
			   (params     (if cmd (cdr remargs) '()))
			   (exitstat   #f)
			   (shell      (let ((sh (get-environment-variable "SHELL") ))
					 (if sh 
					     (last (string-split sh "/"))
					     "bash")))
			   (redir      (case (string->symbol shell)
					 ((tcsh csh ksh)    ">&")
					 ((zsh bash sh ash) "2>&1 >")
					 (else ">&")))
			   (fullcmd    (conc "(" (string-intersperse 
						  (cons cmd params) " ")
					     ") " redir " " logfile)))
		      ;; mark the start of the test
		      (rmt:teststep-set-status! run-id test-id stepname "start" "n/a" (args:get-arg "-m") logfile)
		      ;; run the test step
		      (debug:print-info 2 *default-log-port* "Running \"" fullcmd "\" in directory \"" startingdir)
		      (change-directory startingdir)
		      (set! exitstat (system fullcmd))
		      (set! *globalexitstatus* exitstat)
		      ;; (change-directory testpath)
		      ;; run logpro if applicable ;; (process-run "ls" (list "/foo" "2>&1" "blah.log"))
		      (if logprofile
			  (let* ((htmllogfile (conc stepname ".html"))
				 (oldexitstat exitstat)
				 (cmd         (string-intersperse (list "logpro" logprofile htmllogfile "<" logfile ">" (conc stepname "_logpro.log")) " ")))
			    (debug:print-info 2 *default-log-port* "running \"" cmd "\"")
			    (change-directory startingdir)
			    (set! exitstat (system cmd))
			    (set! *globalexitstatus* exitstat) ;; no necessary
			    (change-directory testpath)
			    (rmt:test-set-log! run-id test-id htmllogfile)))
		      (let ((msg (args:get-arg "-m")))
			(rmt:teststep-set-status! run-id test-id stepname "end" exitstat msg logfile))
		      )))
	    (if (or (args:get-arg "-test-status")
		    (args:get-arg "-set-values"))
		(let ((newstatus (cond
				  ((number? status)       (if (equal? status 0) "PASS" "FAIL"))
				  ((and (string? status)
					(string->number status))(if (equal? (string->number status) 0) "PASS" "FAIL"))
				  (else status)))
		      ;; transfer relevant keys into a hash to be passed to test-set-status!
		      ;; could use an assoc list I guess. 
		      (otherdata (let ((res (make-hash-table)))
				   (for-each (lambda (key)
					       (if (args:get-arg key)
						   (hash-table-set! res key (args:get-arg key))))
					     (list ":value" ":tol" ":expected" ":first_err" ":first_warn" ":units" ":category" ":variable"))
				   res)))
		  (if (and (args:get-arg "-test-status")
			   (or (not state)
			       (not status)))
		      (begin
			(debug:print-error 0 *default-log-port* "You must specify :state and :status with every call to -test-status\n" help)
			(if (sqlite3:database? db)(sqlite3:finalize! db))
			(exit 6)))
		  (let* ((msg    (args:get-arg "-m"))
			 (numoth (length (hash-table-keys otherdata))))
		    ;; Convert to rpc inside the tests:test-set-status! call, not here
		    (tests:test-set-status! run-id test-id state newstatus msg otherdata work-area: work-area))))
	    (if (sqlite3:database? db)(sqlite3:finalize! db))
	    (set! *didsomething* #t))))

  ;;======================================================================
  ;; Various helper commands can go below here
  ;;======================================================================

  (if (or (args:get-arg "-showkeys")
          (args:get-arg "-show-keys"))
      (let ((db #f)
	    (keys #f))
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting")
	      (exit 1)))
	(set! keys (rmt:get-keys)) ;;  db))
	(debug:print 1 *default-log-port* "Keys: " (string-intersperse keys ", "))
	(if (sqlite3:database? db)(sqlite3:finalize! db))
	(set! *didsomething* #t)))

  (if (args:get-arg "-gui")
      (begin
	(debug:print 0 *default-log-port* "Look at the dashboard for now")
	;; (megatest-gui)
	(set! *didsomething* #t)))

  (if (args:get-arg "-create-megatest-area")
      (begin
	(genexample:mk-megatest.config)
	(set! *didsomething* #t)))

  (if (args:get-arg "-create-test")
      (let ((testname (args:get-arg "-create-test")))
	(genexample:mk-megatest-test testname)
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; Update the database schema, clean up the db
  ;;======================================================================

  (if (args:get-arg "-rebuild-db")
      (begin
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	      (exit 1)))
	;; keep this one local
	;; (open-run-close patch-db #f)
	(let ((dbstructs (db:setup)))
          (common:cleanup-db dbstructs full: #t))
	(set! *didsomething* #t)))

  (if (args:get-arg "-cleanup-db")
      (begin
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	      (exit 1)))

	;;      (if (not (server:choose-server *toppath* 'home?))
	;;	  (begin
	;;	    (debug:print 0 *default-log-port* "Servers are not running on this host or no servers alive. Cannot run cleanup-db")
	;;	    (exit 1)))

	(let ((dbstructs (db:setup)))
          (common:cleanup-db dbstructs))
	(set! *didsomething* #t)))

  #;(if (args:get-arg "-mark-incompletes")
      (begin
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting")
	      (exit 1)))
	(open-run-close db:find-and-mark-incomplete #f)
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; Update the tests meta data from the testconfig files
  ;;======================================================================

  (if (args:get-arg "-update-meta")
      (begin
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	      (exit 1)))
	(runs:update-all-test_meta #f)
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; Start a repl
  ;;======================================================================

  ;; fakeout readline
  (include "readline-fix.scm")

  (when (args:get-arg "-diff-rep")
    (when (and
           (not (args:get-arg "-diff-html"))
           (not (args:get-arg "-diff-email")))
      (debug:print 0 *default-log-port* "Must specify -diff-html or -diff-email with -diff-rep")
      (set! *didsomething* 1)
      (exit 1))
    
    (let* ((toppath (launch:setup)))
      (do-diff-report
       (args:get-arg "-src-target")
       (args:get-arg "-src-runname")
       (args:get-arg "-target")
       (args:get-arg "-runname")
       (args:get-arg "-diff-html")
       (args:get-arg "-diff-email"))
      (set! *didsomething* #t)
      (exit 0)))

  (if (or (getenv "MT_RUNSCRIPT")
	  (args:get-arg "-repl")
	  (args:get-arg "-load"))
      (let* ((toppath (launch:setup))
	     (dbstructs (if (and toppath
				 ;; NOTE: server:choose-server is starting a server
				 ;;   either add equivalent for tcp mode or ????
				 #;(server:choose-server toppath 'home?))
                            (db:setup)
                            #f))) ;; make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f)))
	(if *toppath*
	    (cond
	     ((getenv "MT_RUNSCRIPT")
	      ;; How to run megatest scripts
	      ;;
	      ;; #!/bin/bash
	      ;;
	      ;; export MT_RUNSCRIPT=yes
	      ;; megatest << EOF
	      ;; (print "Hello world")
	      ;; (exit)
	      ;; EOF
    
	      (repl))
	     (else
	      (begin
		(define toplevel-command (lambda (a b)(print a " "b)))
		(set! *db* dbstructs)
		(import extras) ;; might not be needed
		;; (import csi)
		;; (import readline)
		(import apropos)
		(import dbfile)
		
		;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ...

		(if *use-new-readline*
		    (begin
		      #;(install-history-file (get-environment-variable "HOME") ".megatest_history") ;;  [homedir] [filename] [nlines])
		      #;(current-input-port (make-readline-port "megatest> ")))
		    #;(begin
		      (gnu-history-install-file-manager
		       (string-append
			(or (get-environment-variable "HOME") ".") "/.megatest_history"))
		      (current-input-port (make-gnu-readline-port "megatest> "))))
		(if (args:get-arg "-repl")
		    (repl)
		    (load (args:get-arg "-load")))
		;; (db:close-all dbstruct) <= taken care of by on-exit call
		)
	      (exit)))
	    (set! *didsomething* #t))))

  ;;======================================================================
  ;; Wait on a run to complete
  ;;======================================================================

  (if (and (args:get-arg "-run-wait")
	   (not (or (args:get-arg "-run")
		    (args:get-arg "-runtests")))) ;; run-wait is built into runtests now
      (begin
	(if (not (launch:setup))
	    (begin
	      (debug:print 0 *default-log-port* "Failed to setup, exiting") 
	      (exit 1)))
	(operate-on 'run-wait)
	(set! *didsomething* #t)))

  ;; ;; ;; redo me ;; Not converted to use dbstruct yet
  ;; ;; ;; redo me ;;
  ;; ;; ;; redo me (if (args:get-arg "-convert-to-norm")
  ;; ;; ;; redo me     (let* ((toppath (setup-for-run))
  ;; ;; ;; redo me 	   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t))))
  ;; ;; ;; redo me       (for-each 
  ;; ;; ;; redo me        (lambda (field)
  ;; ;; ;; redo me 	 (let ((dat '()))
  ;; ;; ;; redo me 	   (debug:print-info 0 *default-log-port* "Getting data for field " field)
  ;; ;; ;; redo me 	   (sqlite3:for-each-row
  ;; ;; ;; redo me 	    (lambda (id val)
  ;; ;; ;; redo me 	      (set! dat (cons (list id val) dat)))
  ;; ;; ;; redo me 	    (db:get-db db run-id)
  ;; ;; ;; redo me 	    (conc "SELECT id," field " FROM tests;"))
  ;; ;; ;; redo me 	   (debug:print-info 0 *default-log-port* "found " (length dat) " items for field " field)
  ;; ;; ;; redo me 	   (let ((qry (sqlite3:prepare db (conc "UPDATE tests SET " field "=? WHERE id=?;"))))
  ;; ;; ;; redo me 	     (for-each
  ;; ;; ;; redo me 	      (lambda (item)
  ;; ;; ;; redo me 		(let ((newval ;; (sdb:qry 'getid 
  ;; ;; ;; redo me 		       (cadr item))) ;; )
  ;; ;; ;; redo me 		  (if (not (equal? newval (cadr item)))
  ;; ;; ;; redo me 		      (debug:print-info 0 *default-log-port* "Converting " (cadr item) " to " newval " for test #" (car item)))
  ;; ;; ;; redo me 		  (sqlite3:execute qry newval (car item))))
  ;; ;; ;; redo me 	      dat)
  ;; ;; ;; redo me 	     (sqlite3:finalize! qry))))
  ;; ;; ;; redo me        (db:close-all dbstruct)
  ;; ;; ;; redo me        (list "uname" "rundir" "final_logf" "comment"))
  ;; ;; ;; redo me       (set! *didsomething* #t)))

  (if (args:get-arg "-import-megatest.db")
      (begin
	(launch:setup)
	(db:multi-db-sync 
	 (db:setup)
	 'killservers
	 'dejunk
	 'adj-testids
	 'old2new
	 )
	(set! *didsomething* #t)))

  (if (args:get-arg "-import-sexpr")
      (let*(
	    (toppath (launch:setup))
	    (tmppath (common:make-tmpdir-name toppath "")))
	(if (file-exists? (conc toppath "/.mtdb")) 
	    (if (args:get-arg "-remove-dbs")
		(let* ((dbfiles (conc toppath "/.mtdb/* " tmppath "/*")))
		  (debug:print 0 *default-log-port* "Removing db files: " dbfiles)
		  (system (conc "rm -rvf " dbfiles))
		  )
		(begin
		  (debug:print 0 *default-log-port* "ERROR: Cannot import sexpr with an existing DB present.")
		  (debug:print 0 *default-log-port* "Add '-remove-dbs all'  to remove the current Megatest DBs.")
		  (set! *didsomething* #t)
		  (exit)
		  )
		)
	    (debug:print 0 *default-log-port* "Did not find " (conc toppath "/.mtdb"))
	    )
	(db:setup)
	(rmt:import-sexpr (args:get-arg "-import-sexpr"))
	(set! *didsomething* #t)))

  (if (args:get-arg "-sync-to-megatest.db")
      (let* ((duh      (launch:setup))
	     (dbstruct (db:setup))
	     (tmpdbpth (dbr:dbstruct-tmppath dbstruct))
	     (lockfile (conc tmpdbpth ".lock"))
	     (locked   (common:simple-file-lock lockfile)) 
	     (res      (if locked
			   (db:multi-db-sync 
			    dbstruct
			    'new2old)
			   #f)))
	(if res
	    (begin
	      (common:simple-file-release-lock lockfile)
	      (debug:print 0 *default-log-port* "Synced " res " records to megatest.db"))
	    (debug:print 0 *default-log-port* "Skipping sync, there is a sync in progress."))
	(set! *didsomething* #t)))

  (if (args:get-arg "-sync-to")
      (let ((toppath (launch:setup)))
	(tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
	(set! *didsomething* #t)))


  ;; use with -from and -to
  ;;
  (if (args:get-arg "-db2db")
      (let* ((duh         (launch:setup))
	     (src-db      (args:get-arg "-from"))
	     (dest-db     (args:get-arg "-to"))
	     ;; (sync-period (args:get-arg-number "-period"))
	     ;; (sync-timeout (args:get-arg-number "-timeout"))
	     (sync-period-in  (args:get-arg "-period"))
	     (sync-timeout-in (args:get-arg "-timeout"))
	     (sync-period     (if sync-period-in (string->number sync-period-in) #f))
	     (sync-timeout    (if sync-timeout-in (string->number sync-timeout-in) #f))
	     (synclock-file   (conc dest-db".sync-lock"))
	     (keys        (db:get-keys #f))
	     (thesync     (lambda (last-update)
			    (debug:print-info 0 *default-log-port* "Attempting to sync data from "src-db" to "dest-db"...")
			    (debug:print-info 0 *default-log-port* "PID = " (current-process-id))
			    (if (not (file-exists? dest-db))
				(begin
				  (debug:print 0 *default-log-port* "Using copy to create "dest-db" from "src-db)
				  (file-copy src-db dest-db)
				  1)
				(let ((res (dbmod:db-to-db-sync src-db dest-db last-update (dbfile:db-init-proc) keys)))
				  (if res
				      (debug:print-info 2 *default-log-port* "Synced " res " records from "src-db" to "dest-db)
				      (debug:print-info 0 *default-log-port* "No sync due to permissions or other issue."))
				  res))))
	     (start-time  (current-seconds))
             (synclock-mod-time (if (file-exists? synclock-file)
				    (handle-exceptions
				     exn
				     #f
				     (file-modification-time synclock-file))
				    #f))
             (age (if synclock-mod-time (- (current-seconds) synclock-mod-time) 1000))
             )
	(if (and src-db dest-db)
	    (if (file-exists? src-db)
		(if (and (file-exists? synclock-file) (< age 20))
		    (debug:print 0 *default-log-port* "Lock "synclock-file" exists, skipping sync...")
                    (begin
                      (if (file-exists? synclock-file)
			  (begin
			    (debug:print 0 *default-log-port* "Deleting old lock file " synclock-file)
			    (delete-file synclock-file)
			    )
			  )
		      (dbfile:with-simple-file-lock
		       synclock-file
		       (lambda ()
			 (let loop ((last-changed (current-seconds))
				    (last-update  0))
			   (let* ((changes (handle-exceptions
					    exn
					    (begin
					      (debug:print 0 *default-log-port* "Exception in sync: "(condition->list exn))
					      (delete-file synclock-file)
					      (exit))
					    (thesync last-update)))
				  (now-time (current-seconds)))
			     (if (and sync-period sync-timeout) ;; 
				 (if (and (< (- now-time start-time) 600) ;; hard limit on how long we run for
					  (>  sync-timeout (- now-time last-changed)))
				     (begin
				       (if sync-period (thread-sleep! sync-period))
				       (loop (if (> changes 0) now-time last-changed) now-time))))))))
                      (debug:print 0 *default-log-port* "Releasing lock file " synclock-file)
                      )
		    )
		(debug:print 0 *default-log-port* "No sync due to unreadble or non-existant source file"src-db))
	    (debug:print 0 *default-log-port* "Usage for -db2db; -to and -from must be specified"))
	(set! *didsomething* #t)))

  (if (args:get-arg "-list-test-time")
      (let* ((toppath (launch:setup))) 
	(task:get-test-times)  
	(set! *didsomething* #t)))

  (if (args:get-arg "-list-run-time")
      (let* ((toppath (launch:setup))) 
	(task:get-run-times)  
	(set! *didsomething* #t)))
  
  (if (args:get-arg "-generate-html")
      (let* ((toppath (launch:setup)))
	(if (tests:create-html-tree #f)
            (debug:print-info 0 *default-log-port* "HTML output created in " toppath "/lt/page0.html")
            (debug:print 0 *default-log-port* "Failed to create HTML output in " toppath "/lt/runs-index.html"))
	(set! *didsomething* #t)))

  (if (args:get-arg "-generate-html-structure")
      (let* ((toppath (launch:setup)))
					;(if (tests:create-html-tree #f)
 	(if (tests:create-html-summary #f)
            (debug:print-info 0 *default-log-port* "HTML output created in " toppath "/lt/targets.html")
            (debug:print 0 *default-log-port* "Failed to create HTML output in " toppath "/lt/runs-index.html"))
	(set! *didsomething* #t)))

  (if (args:get-arg "-syscheck")
      (begin
	(mutils:syscheck common:raw-get-remote-host-load
			 server:get-best-guess-address
			 read-config)
	(set! *didsomething* #t)))

  (if (args:get-arg "-extract-skeleton")
      (let* ((toppath (launch:setup)))
	(genexample:extract-skeleton-area (args:get-arg "-extract-skeleton"))
	(set! *didsomething* #t)))

  ;;======================================================================
  ;; Exit and clean up
  ;;======================================================================

  (if (not *didsomething*)
      (debug:print 0 *default-log-port* help)
      (set! *time-to-exit* #t)
      )
  ;;(debug:print-info 13 *default-log-port* "thread-join! watchdog")

  ;; join the watchdog thread if it has been thread-start!ed  (it may not have been started in the case of a server that never enters running state)
  ;;   (symbols returned by thread-state: created ready running blocked suspended sleeping terminated dead)
  ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage
  ;;(if (thread? *watchdog*)
  ;;    (case (thread-state *watchdog*)
  ;;      ((ready running blocked sleeping terminated dead)
  ;;       (thread-join! *watchdog*))))

  (set! *time-to-exit* #t)

  (if (not (eq? *globalexitstatus* 0))
      (if (or (args:get-arg "-run")(args:get-arg "-runtests")(args:get-arg "-runall"))
          (begin
            (debug:print 0 *default-log-port* "NOTE: Subprocesses with non-zero exit code detected: " *globalexitstatus*)
            (exit 0))
          (case *globalexitstatus*
            ((0)(exit 0))
            ((1)(exit 1))
            ((2)(exit 2))
            (else (exit 3)))))
  ) ;; main
)

Modified mtexec.scm from [8cf589213b] to [e107d3437d].

34
35
36
37
38
39
40
41

42
43
44
45
46
47
48
34
35
36
37
38
39
40

41
42
43
44
45
46
47
48







-
+







(declare (uses configfmod))

(import commonmod
	configfmod
	(prefix mtargs args:))

;; (use ducttape-lib)
(include "megatest-version.scm")
;; (include "megatest-version.scm")
(include "megatest-fossil-hash.scm")

;; (require-library stml)

(define help (conc "
mtutil, part of the Megatest tool suite, documentation at http://www.kiatoa.com/fossils/megatest
  version " megatest-version "

Modified mtmod.scm from [b742c926fe] to [4d23e65eeb].

30
31
32
33
34
35
36







37





38
39
40
41
42
43
44
30
31
32
33
34
35
36
37
38
39
40
41
42
43

44
45
46
47
48
49
50
51
52
53
54
55







+
+
+
+
+
+
+
-
+
+
+
+
+







(declare (uses commonmod))
(declare (uses configfmod))
;; (declare (uses tcp-transportmod)) ;; we don't want mtmod depending on tcp

(use srfi-69)

(module mtmod
	(
	 keys:make-key/field-string
	 common:get-testsuite-name
	 items:get-items-from-config
	 mt:run-trigger
	 common:get-linktree
	 common:get-area-name
	*
	 
	 items:check-valid-items
	 mt:discard-blocked-tests

	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified mtut.scm from [73522346db] to [f402ce6bd2].

12
13
14
15
16
17
18
19

20
21
22
23
24
25
26
12
13
14
15
16
17
18

19
20
21
22
23
24
25
26







-
+







;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;
;
(declare (uses common))
;; (declare (uses common))
(declare (uses mtargs))
(declare (uses mtargs.import))
(declare (uses debugprint))
(declare (uses debugprint.import))
(declare (uses commonmod))
(declare (uses commonmod.import))
(declare (uses configfmod))

Modified odsmod.scm from [9072959ec4] to [aabfb04b69].

14
15
16
17
18
19
20
21

22
23
24
25
26
27
28




29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

45
46
47
48
49
50
51
14
15
16
17
18
19
20

21
22
23
24
25
26
27

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55







-
+






-
+
+
+
+
















+







;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

(use csv-xml regex)
(declare (unit odsmod))
(declare (uses common))
;; (declare (uses common))
(declare (uses commonmod))
(declare (uses debugprint))
(declare (uses dbfile))
(declare (uses dbmod))

(module odsmod
	*
	(
	 db:extract-ods-file
	 ods:list->ods
	 )
	
(import scheme
	chicken
	data-structures
	extras
	posix
	ports
	regex
	srfi-1
	srfi-13
	(prefix sqlite3 sqlite3:)
	
	commonmod
	debugprint
	dbfile
	dbmod
	
	)

(define ods:dirs
  '("Configurations2"
    "Configurations2/toolpanel"
    "Configurations2/menubar"
    "Configurations2/toolbar"

Modified processmod.scm from [1cce7c0878] to [85887c9778].

21
22
23
24
25
26
27


28









29
30
31
32
33
34
35
21
22
23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45







+
+
-
+
+
+
+
+
+
+
+
+







(declare (unit processmod))
(declare (uses debugprint))
(declare (uses commonmod))

(use srfi-69)

(module processmod
	(
	 process:children
	*

	 process:cmd-run->list
	 process:alive?
	 run-n-wait
	 process:cmd-run-with-stderr-and-exitcode->list

	 process:alive-on-host?
	 process:get-sub-pids
	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified rmtmod.scm from [f16c2416fe] to [981fa22127].

26
27
28
29
30
31
32






33


















































































































34
35
36
37
38
39
40
26
27
28
29
30
31
32
33
34
35
36
37
38

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159







+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses dbmod))
(declare (uses mtmod))
(declare (uses tcp-transportmod))
(declare (uses apimod))
(declare (uses servermod))

(module rmtmod
	(
	 rmt:test-data-rollup
	 rmt:import-sexpr
	 rmt:read-test-data-varpatt
	 rmt:get-run-status
	 rmt:set-run-status
	*

	 rmtmod:send-receive
	 rmt:send-receive
	 rmt:no-sync-get-lock
	 rmt:no-sync-del!
	 rmt:no-sync-set
	 rmt:no-sync-get/default

	 rmt:get-runs-by-patt
	 rmt:get-testinfo-state-status
	 rmt:get-test-id
	 rmt:set-state-status-and-roll-up-items

	 rmt:get-prereqs-not-met
	 rmt:get-tests-for-run

	 rmt:get-keys
	 rmt:test-get-records-for-index-file
	 tests:test-set-toplog!
	 rmt:test-get-logfile-info
	 rmt:general-call
	 rmt:test-get-paths-matching-keynames-target-new
	 rmt:get-test-info-by-id
	 rmt:get-steps-for-test
	 rmt:get-num-runs
	 rmt:get-runs-cnt-by-patt
	 rmt:get-runs

	 rmt:get-latest-host-load
	 rmt:get-changed-record-test-ids
	 rmt:get-all-runids
	 rmt:get-changed-record-run-ids
	 rmt:get-run-record-ids
	 rmt:get-data-info-by-id
	 rmt:get-steps-info-by-id
	 rmt:get-target

	 rmt:get-run-name-from-id
	 rmt:get-run-info
	 rmt:get-test-times
	 rmt:get-run-times

	 rmt:tasks-find-task-queue-records
	 
	 common:api-changed?
	 rmt:on-homehost?

	 rmt:get-var
	 rmt:csv->test-data
	 rmt:get-previous-test-run-record

	 common:cleanup-db
	 common:get-last-run-version

	 rmt:get-key-val-pairs
	 rmt:create-all-triggers
	 rmt:update-tesdata-on-repilcate-db
	 rmt:drop-all-triggers
	 rmt:test-get-archive-block-info
	 rmt:test-toplevel-num-items
	 rmt:archive-get-allocations
	 rmt:archive-register-disk
	 rmt:archive-register-block-name

	 mt:get-runs-by-patt
	 rmt:simple-get-runs
	 rmt:get-tests-for-runs-mindata
	 rmt:test-get-top-process-pid
	 rmt:set-state-status-and-roll-up-run
	 rmt:get-run-state-status
	 rmt:get-not-completed-cnt
	 rmt:get-tests-tags
	 rmt:testmeta-update-field
	 rmt:testmeta-add-record
	 rmt:testmeta-get-record
	 rmt:lock/unlock-run
	 rmt:delete-old-deleted-test-records
	 rmt:delete-run
	 rmt:get-raw-run-stats
	 rmt:update-run-stats
	 rmt:delete-test-records
	 rmt:test-set-archive-block-id
	 mt:get-tests-for-run
	 mt:test-set-state-status-by-testname
	 mt:test-set-state-status-by-testname-unless-completed
	 rmt:register-test
	 mt:test-set-state-status-by-id-unless-completed
	 rmt:get-all-run-ids

	 rmt:set-run-state-status
	 rmt:set-var
	 rmt:set-tests-state-status
	 rmt:tasks-add
	 rmt:tasks-set-state-given-param-key
	 rmt:register-run
	 rmt:get-count-tests-running-in-jobgroup
	 rmt:get-count-tests-running-for-run-id
	 
	 rmt:test-set-state-status-by-id
	 mt:test-set-state-status-by-id

	 rmt:get-status-from-final-status-file
	 rmt:get-toplevels-and-incompletes 

	 rmt:test-set-log!
	 rmt:teststep-set-status!

	 rmt:delete-steps-for-test!
	 rmt:test-set-state-status
	 rmt:get-test-state-status-by-id
	 rmt:test-set-top-process-pid

	 )
	
	
(import scheme
	chicken
	data-structures
	regex
	extras
	matchable
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
281
282
283
284
285
286
287








288
289
290
291
292
293
294







-
-
-
-
-
-
-
-








(define (rmt:get-test-state-status-by-id run-id test-id)
  (rmtmod:send-receive 'get-test-state-status-by-id run-id (list run-id test-id)))

(define (rmt:test-get-rundir-from-test-id run-id test-id)
  (rmtmod:send-receive 'test-get-rundir-from-test-id run-id (list run-id test-id)))

;; (define (rmt:open-test-db-by-test-id run-id test-id #!key (work-area #f))
;;   (assert (number? run-id) "FATAL: Run id required.")
;;   (let* ((test-path (if (string? work-area)
;; 			work-area
;; 			(rmt:test-get-rundir-from-test-id run-id test-id))))
;;     (debug:print 3 *default-log-port* "TEST PATH: " test-path)
;;     (open-test-db test-path)))

;; WARNING: This currently bypasses the transaction wrapped writes system
(define (rmt:test-set-state-status-by-id run-id test-id newstate newstatus newcomment)
  (assert (number? run-id) "FATAL: Run id required.")
  (rmtmod:send-receive 'test-set-state-status-by-id run-id (list run-id test-id newstate newstatus newcomment)))

(define (rmt:set-tests-state-status run-id testnames currstate currstatus newstate newstatus)
  (assert (number? run-id) "FATAL: Run id required.")
703
704
705
706
707
708
709
710
711
712
713




714
715
716
717
718
719
720
814
815
816
817
818
819
820




821
822
823
824
825
826
827
828
829
830
831







-
-
-
-
+
+
+
+







(define (rmt:update-run-event_time run-id)
  (assert (number? run-id) "FATAL: Run id required.")
  (rmt:send-receive 'update-run-event_time #f (list run-id)))

(define (rmt:get-runs-by-patt  keys runnamepatt targpatt offset limit fields last-runs-update  #!key  (sort-order "asc")) ;; fields of #f uses default
  (rmt:send-receive 'get-runs-by-patt #f (list keys runnamepatt targpatt offset limit fields last-runs-update sort-order)))

(define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
  (assert (number? run-id) "FATAL: Run id required.")
  ;; (if (rmt:send-receive 'have-incompletes? run-id (list run-id ovr-deadtime))
  (rmt:send-receive 'mark-incomplete run-id (list run-id ovr-deadtime))) ;; )
;; (define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
;;   (assert (number? run-id) "FATAL: Run id required.")
;;   ;; (if (rmt:send-receive 'have-incompletes? run-id (list run-id ovr-deadtime))
;;   (rmt:send-receive 'mark-incomplete run-id (list run-id ovr-deadtime))) ;; )

(define (rmt:get-main-run-stats run-id)
  (assert (number? run-id) "FATAL: Run id required.")
  (rmt:send-receive 'get-main-run-stats #f (list run-id)))

(define (rmt:get-var varname)
  (rmt:send-receive 'get-var #f (list varname)))
735
736
737
738
739
740
741
742
743
744
745
746





747
748
749
750
751
752
753
846
847
848
849
850
851
852





853
854
855
856
857
858
859
860
861
862
863
864







-
-
-
-
-
+
+
+
+
+







  (rmt:send-receive 'add-var #f (list varname value)))

;;======================================================================
;; M U L T I R U N   Q U E R I E S
;;======================================================================

;; Need to move this to multi-run section and make associated changes
(define (rmt:find-and-mark-incomplete-all-runs #!key (ovr-deadtime #f))
  (let ((run-ids (rmt:get-all-run-ids)))
    (for-each (lambda (run-id)
	       (rmt:find-and-mark-incomplete run-id ovr-deadtime))
	     run-ids)))
;; (define (rmt:find-and-mark-incomplete-all-runs #!key (ovr-deadtime #f))
;;   (let ((run-ids (rmt:get-all-run-ids)))
;;     (for-each (lambda (run-id)
;; 	       (rmt:find-and-mark-incomplete run-id ovr-deadtime))
;; 	     run-ids)))

;; get the previous record for when this test was run where all keys match but runname
;; returns #f if no such test found, returns a single test record if found
;; 
;; Run this at the client end since we have to connect to multiple run-id dbs
;;
(define (rmt:get-previous-test-run-record run-id test-name item-path)

Modified runconfig.scm from [5664220be3] to [5190dce4cd].

19
20
21
22
23
24
25
26

27
28
29
30
31
32
33

34
19
20
21
22
23
24
25

26
27
28
29
30
31
32

33
34







-
+






-
+

;; read a config file, loading only the section pertinent
;; to this run field1val/field2val/field3val ...
;;======================================================================

(use format directory-utils)

(declare (unit runconfig))
(declare (uses common))
;; (declare (uses common))
(declare (uses debugprint))
(declare (uses commonmod))

(import commonmod
	debugprint)

(include "common_records.scm")
;; (include "common_records.scm")

Modified runs.scm from [647460790c] to [9c12515ed9].

27
28
29
30
31
32
33
34

35
36
37
38
39
40
41
42
43
44
45
46
47

48
49
50
51
52
53
54
27
28
29
30
31
32
33

34
35
36
37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54







-
+












-
+







(declare (uses dbfile))
(declare (uses dbmod))
(declare (uses megatestmod))
(declare (uses mtmod))
(declare (uses tasksmod))
(declare (uses servermod))

(declare (uses common))
;; (declare (uses common))

(declare (uses runconfig))
(declare (uses server))
(declare (uses mt))
(declare (uses archive))

(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (srfi 18) 
     posix-extras directory-utils pathname-expand typed-records format  sxml-serializer
     sxml-modifications matchable)



(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
;; (include "test_records.scm")

;; (include "debugger.scm")

Modified runsmod.scm from [251bedfaeb] to [63f920fb0b].

42
43
44
45
46
47
48






49













50
51
52
53
54
55
56
42
43
44
45
46
47
48
49
50
51
52
53
54

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74







+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses subrunmod))
(declare (uses archivemod))
(declare (uses fsmod))

(use srfi-69)

(module runsmod
	(
	 runs:clean-cache
	 rmt:find-and-mark-incomplete
	 launch:setup
	 launch:end-of-run-check
	 launch:test-copy
	*

	 set-item-env-vars
	 runs:set-megatest-env-vars
	 full-runconfigs-read
	 runs:operate-on

	 runs:update-all-test_meta
	 runs:handle-locking
	 ;; runs:rollup-run ;; not ported
	 runs:run-tests
	 runs:remove-all-but-last-n-runs-per-target
	 general-run-call
	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
125
126
127
128
129
130
131
132

133
134
135
136
137
138
139
143
144
145
146
147
148
149

150
151
152
153
154
155
156
157







-
+







	tasksmod
	testsmod
	subrunmod
	archivemod
	fsmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
;; (include "test_records.scm")

;; use this struct to facilitate refactoring
;;
4538
4539
4540
4541
4542
4543
4544
4545



4546






















































































4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566

4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652








+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
		(if (member (symbol->string archive-command) '("save-remove"))
                    (begin 
                     (debug:print-info 0 *default-log-port* "remove testdat")
		    (runs:remove-test-directory test-dat 'archive-remove)))))
	    (hash-table-ref test-groups test-base)))))
       (hash-table-keys disk-groups))
    #t))

;;======================================================================
;; Maintenance
;;======================================================================
)

(define (rmt:find-and-mark-incomplete run-id ovr-deadtime)
  (let* ((cfg-deadtime             (configf:lookup-number *configdat* "setup" "deadtime"))
	 (test-stats-update-period (configf:lookup-number *configdat* "setup" "test-stats-update-period")))
   (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period)
   ;;call end of eud of run detection for posthook
   (launch:end-of-run-check run-id)))

;;  select end_time-now from
;;      (select testname,item_path,event_time+run_duration as
;;                          end_time,strftime('%s','now') as now from tests where state in
;;      ('RUNNING','REMOTEHOSTSTART','LAUNCHED'));
;;
;; NOT EASY TO MIGRATE TO db{file,mod}
;;
(define (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period)
  (let* ((incompleted '())
	 (oldlaunched '())
	 (toplevels   '())
          ;; The default running-deadtime is 720 seconds = 12 minutes.
          ;; "(running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))" = 200 + (2 * (200 + 30 + 30))
         (deadtime-trim (or ovr-deadtime cfg-deadtime))
         (server-start-allowance 200)
         (server-overloaded-budget 200)
         (launch-monitor-off-time (or test-stats-update-period 30))
         (launch-monitor-on-time-budget 30)
         (launch-monitor-period (+ launch-monitor-off-time launch-monitor-on-time-budget server-overloaded-budget))
         (remotehoststart-deadtime-default (+ server-start-allowance server-overloaded-budget 30))
         (remotehoststart-deadtime (or deadtime-trim remotehoststart-deadtime-default))
         (running-deadtime-default (+ server-start-allowance (* 2 launch-monitor-period)))
         (running-deadtime (or deadtime-trim running-deadtime-default))) ;; two minutes (30 seconds between updates, this leaves 3x grace period)

    (debug:print-info 4  *default-log-port* "running-deadtime = " running-deadtime)
    (debug:print-info 4  *default-log-port* "deadtime-trim = " deadtime-trim)

    (let* ((dat (rmt:get-toplevels-and-incompletes run-id running-deadtime remotehoststart-deadtime)))
      (set! oldlaunched (list-ref dat 1))
      (set! toplevels   (list-ref dat 2))
      (set! incompleted (list-ref dat 0)))

    (debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, "
		      (length toplevels) " old LAUNCHED toplevel tests and "
		      (length incompleted) " tests marked RUNNING but apparently dead.")
  
    ;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE.
    ;;
    ;; (db:delay-if-busy dbdat)
    (let* ((min-incompleted-ids (map car incompleted)) ;; do 'em all
	   (all-ids             (append min-incompleted-ids (map car oldlaunched))))
      (if (> (length all-ids) 0)
	  (begin
	    ;; (launch:is-test-alive "localhost" 435)
	    (debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ")
			 " as DEAD")
	    (for-each
             (lambda (test-id)
               (let* ((tinfo   (rmt:get-test-info-by-id run-id test-id))
		      (run-dir (db:test-get-rundir     tinfo))
		      (host    (db:test-get-host       tinfo))
		      (pid     (db:test-get-process_id tinfo))
		      (result (rmt:get-status-from-final-status-file run-dir)))
		 (if (and (list? result) (> (length result) 1) (equal? "PASS" (cadr result)) (equal? "COMPLETED" (car result))) 
		     (begin
		       (debug:print 0 *default-log-port* "INFO: test " test-id " actually passed, so marking PASS not DEAD")
		       (rmt:set-state-status-and-roll-up-items
			run-id test-id 'foo "COMPLETED" "PASS"
			"Test stopped responding but it has PASSED; marking it PASS in the DB."))
		     (let ((is-alive (and (not (eq? pid 0))  ;; 0 is default in re-used field "attemptnum" where pid stored.
					  (commonmod:is-test-alive host pid))))
		       (if is-alive
			   (debug:print 0 *default-log-port* "INFO: test " test-id " on host " host
					" has a process on pid " pid ", NOT setting to DEAD.")
			   (begin
			     (debug:print 0 *default-log-port* "INFO: test " test-id
					  " final state/status is not COMPLETED/PASS. It is " result)
			     (rmt:set-state-status-and-roll-up-items
			      run-id test-id 'foo "COMPLETED" "DEAD"
			      "Test stopped responding while in RUNNING or REMOTEHOSTSTART; presumed dead.")))))))
	     ;; call end of eud of run detection for posthook - from merge, is it needed?
	     ;; (launch:end-of-run-check run-id)
	     all-ids)
	    )))))



)

Modified server.scm from [3cd1085ec7] to [d483f05a8e].

14
15
16
17
18
19
20
21

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

40
41
42
43
44
45
46
14
15
16
17
18
19
20

21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

39
40
41
42
43
44
45
46







-
+

















-
+







;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
;;

(declare (unit server))

(declare (uses common))
;; (declare (uses common))
(declare (uses debugprint))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses rmtmod))
(declare (uses launch))
(declare (uses mtargs))

(use (srfi 18) extras s11n)
(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
(use directory-utils posix-extras matchable utils)
(use spiffy uri-common intarweb http-client spiffy-request-vars)

(import commonmod
	configfmod
	debugprint
	(prefix mtargs args:))

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")

(define (db:kill-servers)
  (let* ((tl (launch:setup)) ;; need this to initialize *toppath*
        (servdir (conc *toppath* "/.servinfo"))
        (servfiles (glob (conc servdir "/*:*.db")))
        (fmtstr  "~10a~22a~10a~25a~25a~8a\n")

Modified servermod.scm from [5384b281b4] to [cad826e462].

21
22
23
24
25
26
27






28


29
30
31
32
33
34
35
21
22
23
24
25
26
27
28
29
30
31
32
33

34
35
36
37
38
39
40
41
42







+
+
+
+
+
+
-
+
+







(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses mtmod))
(declare (uses debugprint))
(declare (uses mtargs))

(module servermod
	(
	 remote-hh-dat
	 server:mk-signature
	 common:wait-for-normalized-load
	 server:expiration-timeout
	 server:get-best-guess-address
	*
	 
	 )

(import scheme
	chicken)
	
(use (srfi 18) extras s11n)
(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
(use directory-utils posix-extras matchable utils)
44
45
46
47
48
49
50
51

52
53
54
55
56
57
58
51
52
53
54
55
56
57

58
59
60
61
62
63
64
65







-
+







	commonmod
	configfmod
	debugprint
	(prefix mtargs args:)
	mtmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")

(define (server:make-server-url hostport)
  (if (not hostport)
      #f
      (conc "http://" (car hostport) ":" (cadr hostport))))

Modified subrun.scm from [479d716ad5] to [66d6a52eda].

22
23
24
25
26
27
28
29

30
31
32
33
34
35
36
22
23
24
25
26
27
28

29
30
31
32
33
34
35
36







-
+








(declare (uses debugprint))
(declare (uses commonmod))
(declare (uses configfmod))
(declare (uses tasksmod))

(declare (uses mt))
(declare (uses common))
;; (declare (uses common))

(use (prefix sqlite3 sqlite3:) srfi-1 posix regex regex-case srfi-69 (srfi 18) 
     posix-extras directory-utils pathname-expand typed-records format
     call-with-environment-variables)

(import commonmod
	configfmod

Modified subrunmod.scm from [ddf54f1377] to [f63d1179cd].

38
39
40
41
42
43
44
45










46
47
48
49
50
51
52
38
39
40
41
42
43
44

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61







-
+
+
+
+
+
+
+
+
+
+







(declare (uses mtmod))
(declare (uses megatestmod))
(declare (uses tasksmod))

(use srfi-69)

(module subrunmod
	*
	(
	 subrun:set-state-status
	 subrun:kill-subrun
	 subrun:get-log-path
	 subrun:remove-subrun
	 subrun:subrun-removed?
	 subrun:subrun-test-initialized?
	 subrun:launch-cmd
	 subrun:initialize-toprun-test
	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified tasksmod.scm from [381a26e6c2] to [7361eb58d0].

37
38
39
40
41
42
43









44








45
46
47
48
49
50
51
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

53
54
55
56
57
58
59
60
61
62
63
64
65
66
67







+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+







(declare (uses pgdb))
(declare (uses mtmod))
(declare (uses megatestmod))

(use srfi-69)

(module tasksmod
	(
	 configf:write-alist
	 common:simple-unlock
	 common:simple-lock
	 tests:test-set-status!
	 common:get-launcher
	 tasks:kill-runner
	 tests:get-testconfig
	 tests:get-waitons
	*

	 tests:get-test-path-from-environment
	 common:exit-on-version-changed
	 task:get-run-times
	 task:get-test-times
	 tasks:sync-to-postgres

	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports

Modified tcmt.scm from [114f35b4de] to [80d6c74c63].

22
23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38
39
40
41
42

43
44
45
46
47
48
49
22
23
24
25
26
27
28

29
30
31
32
33
34
35
36
37
38
39
40
41

42
43
44
45
46
47
48
49







-
+












-
+







;;  1. Run the megatest process and pass it all the needed parameters
;;  2. Every five seconds check for state/status changes and print the info
;;

(declare (uses mtargs))
(declare (uses rmt))
(declare (uses rmtmod))
(declare (uses common))
;; (declare (uses common))
;; (declare (uses megatest-version))
(declare (uses commonmod))

(use srfi-1 posix srfi-69 srfi-18 regex defstruct)

(use trace)
;; (trace-call-sites #t)

(import commonmod
	rmtmod
	(prefix mtargs args:))

(include "megatest-version.scm")
;; (include "megatest-version.scm")
(include "megatest-fossil-hash.scm")
;; (include "db_records.scm")

(define origargs (cdr (argv)))
(define remargs (args:get-args
		 (argv)
		 `( "-target"

Modified tcp-transportmod.scm from [1a862ecee9] to [9bf965443b].

25
26
27
28
29
30
31
32



















33
34
35
36
37
38
39
25
26
27
28
29
30
31

32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57







-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses dbmod))
(declare (uses portlogger))
(declare (uses mtmod))

(use address-info tcp)

(module tcp-transportmod
	*
	(
	 make-tt
	 tt:get-server-info-sorted
	 tt:ping
	 tt:find-server
	 tt:start-server
	 tt:get-servinfo-dir
	 tt-server-timeout-param
	 tt:mk-signature
	 tt-state
	 tt:server-process-run
	 tt:make-remote
	 tt-ro-mode-checked-set!
	 tt-ro-mode-set!
	 tt-ro-mode
	 tt-ro-mode-checked
	 tt:handler
	 tt:get-conn
	 )
	
(import scheme)

(cond-expand
 (chicken-4
  (import (prefix sqlite3 sqlite3:)
	  chicken

Modified tdb.scm from [e7e7aee13a] to [6bf4733c7a].

1

2
3
4
5
6
7
8

1
2
3
4
5
6
7
8
-
+







;;======================================================================
>;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;; This file is part of Megatest.
;; 
;;     Megatest is free software: you can redistribute it and/or modify
;;     it under the terms of the GNU General Public License as published by
;;     the Free Software Foundation, either version 3 of the License, or
20
21
22
23
24
25
26
27

28
29
30
31
32
33








34

35



36
37
38
39
40
41
42
43
44

45
46
47

48
49
50
51
52
53
54
55
56
57









58
59
60
61
62
63
64
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

44
45
46
47
48
49
50
51
52
53
54

55
56
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84







-
+






+
+
+
+
+
+
+
+

+
-
+
+
+








-
+


-
+










+
+
+
+
+
+
+
+
+








;;======================================================================
;; Database access
;;======================================================================

(declare (unit tdb))
(declare (uses debugprint))
(declare (uses common))
;; (declare (uses common))
(declare (uses keys))
(declare (uses mt))
(declare (uses commonmod))
(declare (uses mtargs))
(declare (uses rmtmod))

(module tdb
	*

(import scheme
	chicken
	data-structures
	)

(require-extension (srfi 18) extras tcp)

(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64)
(import srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5
	message-digest base64)

(import (prefix sqlite3 sqlite3:))
(import (prefix base64 base64:))

(import commonmod
	debugprint
	rmtmod
	(prefix mtargs args:))

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")
(include "run_records.scm")
;; (include "run_records.scm")

;;======================================================================
;;
;; T E S T   D A T A B A S E S
;;
;;======================================================================

;;======================================================================
;; T E S T   S P E C I F I C   D B 
;;======================================================================

;; (define (rmt:open-test-db-by-test-id run-id test-id #!key (work-area #f))
;;   (assert (number? run-id) "FATAL: Run id required.")
;;   (let* ((test-path (if (string? work-area)
;; 			work-area
;; 			(rmt:test-get-rundir-from-test-id run-id test-id))))
;;     (debug:print 3 *default-log-port* "TEST PATH: " test-path)
;;     (open-test-db test-path)))


;; =not-used= ;; Create the sqlite db for the individual test(s)
;; =not-used= ;;
;; =not-used= ;; Moved these tables into <runid>.db
;; =not-used= ;; THIS CODE TO BE REMOVED
;; =not-used= ;;
;; =not-used= (define (open-test-db work-area) 
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249













250
251
252
253
254
255
256
250
251
252
253
254
255
256













257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276







-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+







          (rmt:csv->test-data run-id test-id lin)
          ;;)
	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))

;; NOTE: Run this local with #f for db !!!
(define (tdb:load-logpro-data run-id test-id)
  (let loop ((lin (read-line)))
    (if (not (eof-object? lin))
	(begin
	  (debug:print 4 *default-log-port* lin)
          ;;(when lin  ;; this when blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
          (rmt:csv->test-data run-id test-id lin)
          ;;)
	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))
;; ;; NOTE: Run this local with #f for db !!!
;; (define (tdb:load-logpro-data run-id test-id)
;;   (let loop ((lin (read-line)))
;;     (if (not (eof-object? lin))
;; 	(begin
;; 	  (debug:print 4 *default-log-port* lin)
;;           ;;(when lin  ;; this when blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
;;           (rmt:csv->test-data run-id test-id lin)
;;           ;;)
;; 	  (loop (read-line)))))
;;   ;; roll up the current results.
;;   ;; FIXME: Add the status too 
;;   (rmt:test-data-rollup run-id test-id #f))

;;======================================================================
;; S T E P S 
;;======================================================================

(define (tdb:step-get-time-as-string vec)
  (seconds->time-string (tdb:step-get-event_time vec)))
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416











421
422
423
424
425
426
427









428
429
430
431
432
433
434
435
436
437
438







-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
			 (if (eq? time-a time-b)
			     (string<? (conc (vector-ref a 2))
				       (conc (vector-ref b 2)))
			     #f))
		     (string<? (conc time-a)(conc time-b))))))))

;; 
(define (tdb:remote-update-testdat-meta-info run-id test-id work-area cpuload diskfree minutes)
  (let ((tdb         (rmt:open-test-db-by-test-id run-id test-id work-area: work-area)))
    (if (sqlite3:database? tdb)
	(begin
	  (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
			   cpuload diskfree minutes)
	  (sqlite3:finalize! tdb))
	(debug:print 2 *default-log-port* "Can't update testdat.db for test " test-id " read-only or non-existant"))))
    
;; (define (tdb:remote-update-testdat-meta-info run-id test-id work-area
;; 					     cpuload diskfree minutes)
;;   (let ((tdb         (rmt:open-test-db-by-test-id run-id test-id work-area: work-area)))
;;     (if (sqlite3:database? tdb)
;; 	(begin
;; 	  (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
;; 			   cpuload diskfree minutes)
;; 	  (sqlite3:finalize! tdb))
;; 	(debug:print 2 *default-log-port* "Can't update testdat.db for test " test-id " read-only or non-existant"))))
;;     
)

Modified test_records.scm from [d106f3911c] to [1501321c72].

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
11
12
13
14
15
16
17


























-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;     GNU General Public License for more details.
;; 
;;     You should have received a copy of the GNU General Public License
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;; make-vector-record tests testqueue testname testconfig waitons priority items
(define (make-tests:testqueue)(make-vector 7 #f))
(define (tests:testqueue-get-testname     vec)    (vector-ref  vec 0))
(define (tests:testqueue-get-testconfig   vec)    (vector-ref  vec 1))
(define (tests:testqueue-get-waitons      vec)    (vector-ref  vec 2))
(define (tests:testqueue-get-priority     vec)    (vector-ref  vec 3))
;; items: #f=no items, list=list of items remaining, proc=need to call to get items
(define (tests:testqueue-get-items        vec)    (vector-ref  vec 4))
(define (tests:testqueue-get-itemdat      vec)    (vector-ref  vec 5))
(define (tests:testqueue-get-item_path    vec)    (vector-ref  vec 6))

(define (tests:testqueue-set-testname!    vec val)(vector-set! vec 0 val))
(define (tests:testqueue-set-testconfig!  vec val)(vector-set! vec 1 val))
(define (tests:testqueue-set-waitons!     vec val)(vector-set! vec 2 val))
(define (tests:testqueue-set-priority!    vec val)(vector-set! vec 3 val))
(define (tests:testqueue-set-items!       vec val)(vector-set! vec 4 val))
(define (tests:testqueue-set-itemdat!     vec val)(vector-set! vec 5 val))
(define (tests:testqueue-set-item_path!   vec val)(vector-set! vec 6 val))

Modified testsmod.scm from [342c5ad45d] to [db63cb4f1d].

37
38
39
40
41
42
43




44












45
46
47
48
49
50
51
37
38
39
40
41
42
43
44
45
46
47

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66







+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+







(declare (uses mtmod))
(declare (uses servermod))
(declare (uses fsmod))

(use srfi-69)

(module testsmod
	(
	 tests:summarize-items
	 tests:filter-non-runnable
	 tests:sort-by-priority-and-waiton
	*

	 tests:summarize-test
	 tests:save-final-status
	 tests:update-central-meta-info
	 tests:set-full-meta-info
	 tests:get-compressed-steps
	 tests:create-html-summary
	 tests:create-html-summary
	 tests:create-html-tree
	 tests:summarize-items
	 tests:test-get-paths-matching
	 )

(import scheme)
(cond-expand
 (chicken-4
  
  (import chicken
	  ports
123
124
125
126
127
128
129
130

131
132
133
134
135
136
137
138
139
140
141
142
143
144

145
146
147
148
149
150
151
152







-
+







	rmtmod
	stml2
	mtmod
	servermod
	fsmod
	)

(include "common_records.scm")
;; (include "common_records.scm")
;; (include "key_records.scm")
;; (include "db_records.scm")
(include "run_records.scm")
;; (include "test_records.scm")
(include "js-path.scm")

(define (init-java-script-lib)

Modified tree.scm from [5b26f8b9f9] to [ee0f2b29cf].

33
34
35
36
37
38
39
40
41


42
43
44
45
46
47
48
33
34
35
36
37
38
39


40
41
42
43
44
45
46
47
48







-
-
+
+








(use sqlite3 srfi-1 posix regex regex-case srfi-69)
(import (prefix sqlite3 sqlite3:))

(import (prefix mtargs args:)
	debugprint)

(include "megatest-version.scm")
(include "common_records.scm")
;; (include "megatest-version.scm")
;; (include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")

;;======================================================================
;; T R E E   S T U F F 
;;======================================================================

Added utils/extract-export-list.sh version [d50045d1ff].














1
2
3
4
5
6
7
8
9
10
11
12
13
+
+
+
+
+
+
+
+
+
+
+
+
+
#!/bin/bash

LAST_PARENT=foobar

for fn in $(grep 'Warning: refer' typescript |tr '`' ' '|tr "'" " "|awk '{print $7}');do
    PARENT=$(grep $fn *mod.scm|grep define|cut -d: -f1)
    if [[ $PARENT != $LAST_PARENT ]];then
	echo
	echo $PARENT
	LAST_PARENT=$PARENT
    fi
    echo $fn
done