Megatest

Changes On Branch 39fc4b3bd914800d
Login

Changes In Branch v1.62 Through [39fc4b3bd9] Excluding Merge-Ins

This is equivalent to a diff from 1a6bb460f9 to 39fc4b3bd9

2016-11-03
09:26
Added cxt and global hash check-in: d256b07495 user: mrwellan tags: v1.62
2016-11-02
23:26
Added inter-test launch delay of 5 seconds. Override with setting: [setup]\nlaunch-delay D where D is delay in seconds. check-in: 39fc4b3bd9 user: matt tags: v1.62
21:13
Pulled fix for endless loop from e851. Minor clean up of redundant code for opening databases. check-in: fc06e165b8 user: matt tags: v1.62
2016-09-15
09:15
Brute force fix for the disappearing tests issue in the run summary view check-in: 232b64b7dd user: mrwellan tags: v1.61, v1.6105
2016-09-14
17:17
synced with v1.61 check-in: 9e1c71c37f user: bjbarcla tags: runs-summary-context-menu
17:09
Updated pop-up menu check-in: 1a6bb460f9 user: ritikaag tags: v1.61
2016-09-13
16:26
Fixed horizontal scrollbar and change update handling. check-in: 07fdd25a9d user: mrwellan tags: v1.61

Modified Makefile from [867a54f75f] to [83b5fe2a28].

1


2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)'


PREFIX=$(PWD)
CSCOPTS= 
INSTALL=install
SRCFILES = common.scm items.scm launch.scm \
   ods.scm runconfig.scm server.scm configf.scm \
   db.scm keys.scm margs.scm megatest-version.scm \
   process.scm runs.scm tasks.scm tests.scm genexample.scm \
   http-transport.scm nmsg-transport.scm filedb.scm \
   client.scm synchash.scm daemon.scm mt.scm \
   ezsteps.scm lock-queue.scm sdb.scm \
   rmt.scm api.scm tdb.scm rpc-transport.scm \
   portlogger.scm archive.scm env.scm

# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \

>
>







|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)'
# rm <files>.o ; make install CSCOPTS='-profile' ; ... ;  chicken-profile | less

PREFIX=$(PWD)
CSCOPTS= 
INSTALL=install
SRCFILES = common.scm items.scm launch.scm \
   ods.scm runconfig.scm server.scm configf.scm \
   db.scm keys.scm margs.scm megatest-version.scm \
   process.scm runs.scm tasks.scm tests.scm genexample.scm \
   http-transport.scm filedb.scm \
   client.scm synchash.scm daemon.scm mt.scm \
   ezsteps.scm lock-queue.scm sdb.scm \
   rmt.scm api.scm tdb.scm rpc-transport.scm \
   portlogger.scm archive.scm env.scm

# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
31
32
33
34
35
36
37


38
39
40
41
42
43
44
45
46
47
48
49








50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
CSIPATH=$(shell which csi)
CKPATH=$(shell dirname $(shell dirname $(CSIPATH)))
# ARCHSTR=$(shell uname -m)_$(shell uname -r)
# BASH_MACHTYPE=$(shell bash -c "echo \$$MACHTYPE")
# ARCHSTR=$(BASH_MACHTYPE)_$(shell lsb_release -sr)
ARCHSTR=$(shell lsb_release -sr)
# ARCHSTR=$(shell bash -c "echo \$$MACHTYPE")



all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard 

mtest: $(OFILES) readline-fix.scm megatest.o
	csc $(CSCOPTS) $(OFILES) megatest.o -o mtest

dboard : $(OFILES) $(GOFILES) dashboard.scm
	csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o dboard

ndboard : newdashboard.scm $(OFILES) $(GOFILES)
	csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard









multi-dboard : multi-dboard.scm $(OFILES) $(GOFILES)
	csc $(CSCOPTS) $(OFILES) $(GOFILES) multi-dboard.scm -o multi-dboard

# 
# $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm
#	csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl

# Special dependencies for the includes
tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o  \
archive.o megatest.o : db_records.scm
tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o  : run_records.scm
db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm
tests.o tasks.o dashboard-tasks.o : task_records.scm
runs.o : test_records.scm
megatest.o : megatest-fossil-hash.scm
client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm rpc-transport.scm
common_records.scm : altdb.scm
vg.o dashboard.o : vg_records.scm

# Temporary while transitioning to new routine
# runs.o : run-tests-queue-classic.scm  run-tests-queue-new.scm

megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm
	echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new
	if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi








>
>












>
>
>
>
>
>
>
>
|
|
















|







33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
CSIPATH=$(shell which csi)
CKPATH=$(shell dirname $(shell dirname $(CSIPATH)))
# ARCHSTR=$(shell uname -m)_$(shell uname -r)
# BASH_MACHTYPE=$(shell bash -c "echo \$$MACHTYPE")
# ARCHSTR=$(BASH_MACHTYPE)_$(shell lsb_release -sr)
ARCHSTR=$(shell lsb_release -sr)
# ARCHSTR=$(shell bash -c "echo \$$MACHTYPE")

PNGFILES = $(shell cd docs/manual;ls *png)

all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard 

mtest: $(OFILES) readline-fix.scm megatest.o
	csc $(CSCOPTS) $(OFILES) megatest.o -o mtest

dboard : $(OFILES) $(GOFILES) dashboard.scm
	csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o dboard

ndboard : newdashboard.scm $(OFILES) $(GOFILES)
	csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard

# install documentation to $(PREFIX)/docs
# DOES NOT REBUILD DOCS
#
$(PREFIX)/share/docs/megatest_manual.html : docs/manual/megatest_manual.html
	mkdir -p $(PREFIX)/share/docs
	$(INSTALL) docs/manual/megatest_manual.html $(PREFIX)/share/docs/megatest_manual.html
	for png in $(PNGFILES);do $(INSTALL) docs/manual/$$png $(PREFIX)/share/docs/$$png;done

#multi-dboard : multi-dboard.scm $(OFILES) $(GOFILES)
#	csc $(CSCOPTS) $(OFILES) $(GOFILES) multi-dboard.scm -o multi-dboard

# 
# $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm
#	csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl

# Special dependencies for the includes
tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o  \
archive.o megatest.o : db_records.scm
tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o  : run_records.scm
db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm
tests.o tasks.o dashboard-tasks.o : task_records.scm
runs.o : test_records.scm
megatest.o : megatest-fossil-hash.scm
client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm rpc-transport.scm
common_records.scm : altdb.scm
vg.o dashboard.o : vg_records.scm
dcommon.o : run_records.scm
# Temporary while transitioning to new routine
# runs.o : run-tests-queue-classic.scm  run-tests-queue-new.scm

megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm
	echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new
	if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
$(PREFIX)/bin/.$(ARCHSTR)/ndboard : ndboard
	$(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard

$(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper
	utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard
	chmod a+x $(PREFIX)/bin/newdashboard

$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard
	$(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard

$(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard  utils/mk_wrapper
	utils/mk_wrapper $(PREFIX) mdboard $(PREFIX)/bin/mdboard
	chmod a+x $(PREFIX)/bin/mdboard

# $(HELPERS) : utils/%
# 	$(INSTALL) $< $@
# 	chmod a+x $@

$(PREFIX)/bin/mt_laststep : utils/mt_laststep
	$(INSTALL) $< $@







|
|

|
|
|







99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
$(PREFIX)/bin/.$(ARCHSTR)/ndboard : ndboard
	$(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard

$(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper
	utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard
	chmod a+x $(PREFIX)/bin/newdashboard

#$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard
#	$(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard

# $(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard  utils/mk_wrapper
# 	utils/mk_wrapper $(PREFIX) mdboard $(PREFIX)/bin/mdboard
# 	chmod a+x $(PREFIX)/bin/mdboard

# $(HELPERS) : utils/%
# 	$(INSTALL) $< $@
# 	chmod a+x $@

$(PREFIX)/bin/mt_laststep : utils/mt_laststep
	$(INSTALL) $< $@
117
118
119
120
121
122
123




124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
$(PREFIX)/bin/mt_xterm : utils/mt_xterm
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/nbfake : utils/nbfake
	$(INSTALL) $< $@
	chmod a+x $@





$(PREFIX)/bin/nbfind : utils/nbfind
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/loadrunner : utils/loadrunner
	$(INSTALL) $< $@
	chmod a+x $@

# $(PREFIX)/bin/refdb : refdb
# 	$(INSTALL) $< $@
# 	chmod a+x $@

deploytarg/nbfake : utils/nbfake
	$(INSTALL) $< $@
	chmod a+x $@

deploytarg/nbfind : utils/nbfind
	$(INSTALL) $< $@
	chmod a+x $@





# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper
	utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
	chmod a+x $(PREFIX)/bin/dashboard
	$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard

install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
          $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
	  $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/mt_xterm \
          $(PREFIX)/bin/newdashboard $(PREFIX)/bin/mdboard

$(PREFIX)/bin/.$(ARCHSTR) : 
	mkdir -p $(PREFIX)/bin/.$(ARCHSTR)

test: tests/tests.scm
	cd tests;csi -I .. -b -n tests.scm








>
>
>
>

















|



>
>
>









|
|







129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
$(PREFIX)/bin/mt_xterm : utils/mt_xterm
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/nbfake : utils/nbfake
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/viewscreen : utils/viewscreen
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/nbfind : utils/nbfind
	$(INSTALL) $< $@
	chmod a+x $@

$(PREFIX)/bin/loadrunner : utils/loadrunner
	$(INSTALL) $< $@
	chmod a+x $@

# $(PREFIX)/bin/refdb : refdb
# 	$(INSTALL) $< $@
# 	chmod a+x $@

deploytarg/nbfake : utils/nbfake
	$(INSTALL) $< $@
	chmod a+x $@

deploytarg/viewscreen : utils/viewscreen
	$(INSTALL) $< $@
	chmod a+x $@

deploytarg/nbfind : utils/nbfind
	$(INSTALL) $< $@
	chmod a+x $@

# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper
	utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
	chmod a+x $(PREFIX)/bin/dashboard
	$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard

install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
          $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
	  $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \
	  $(PREFIX)/share/docs/megatest_manual.html 

$(PREFIX)/bin/.$(ARCHSTR) : 
	mkdir -p $(PREFIX)/bin/.$(ARCHSTR)

test: tests/tests.scm
	cd tests;csi -I .. -b -n tests.scm

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204

#	for i in apropos base64 canvas-draw csv-xml directory-utils dot-locking extras fmt format hostinfo http-client intarweb json md5 message-digest posix posix-extras readline regex regex-case s11n spiffy spiffy-request-vars sqlite3 srfi-1 srfi-18 srfi-69 tcp test uri-common check-errors synch matchable sql-null tcp-server rpc blob-utils string-utils variable-item defstruct uri-generic sendfile opensll openssl lookup-table list-utils stack; do \
#	chicken-install -prefix deploytarg -deploy $$i;done

# deploytarg/libsqlite3.so : 
# 	CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3

deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/nbfind deploytarg/apropos.so

# deploytarg/libiupcd.so : $(CKPATH)/lib/libiupcd.so
# 	for i in iup im cd av call sqlite; do \
# 	  cp $(CKPATH)/lib/lib$$i* deploytarg/ ; \
# 	done
# 	cp $(CKPATH)/include/*.h deploytarg








|







209
210
211
212
213
214
215
216
217
218
219
220
221
222
223

#	for i in apropos base64 canvas-draw csv-xml directory-utils dot-locking extras fmt format hostinfo http-client intarweb json md5 message-digest posix posix-extras readline regex regex-case s11n spiffy spiffy-request-vars sqlite3 srfi-1 srfi-18 srfi-69 tcp test uri-common check-errors synch matchable sql-null tcp-server rpc blob-utils string-utils variable-item defstruct uri-generic sendfile opensll openssl lookup-table list-utils stack; do \
#	chicken-install -prefix deploytarg -deploy $$i;done

# deploytarg/libsqlite3.so : 
# 	CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3

deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/viewsceen deploytarg/nbfind deploytarg/apropos.so

# deploytarg/libiupcd.so : $(CKPATH)/lib/libiupcd.so
# 	for i in iup im cd av call sqlite; do \
# 	  cp $(CKPATH)/lib/lib$$i* deploytarg/ ; \
# 	done
# 	cp $(CKPATH)/include/*.h deploytarg

255
256
257
258
259
260
261
262
263
	if  csi -ne '(use mysql-client)';then \
           echo "(use mysql-client)(hash-table-set! *available-db* 'mysql #t)" >> altdb.scm; \
	fi
	if csi -ne '(use postgresql)';then \
	   echo "(use postgresql)(hash-table-set! *available-db* 'postgresql #t)" >> altdb.scm;\
	fi

portlogger-example : portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o nmsg-transport.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o
	csc $(CSCOPTS) portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o nmsg-transport.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o







|
|
274
275
276
277
278
279
280
281
282
	if  csi -ne '(use mysql-client)';then \
           echo "(use mysql-client)(hash-table-set! *available-db* 'mysql #t)" >> altdb.scm; \
	fi
	if csi -ne '(use postgresql)';then \
	   echo "(use postgresql)(hash-table-set! *available-db* 'postgresql #t)" >> altdb.scm;\
	fi

portlogger-example : portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o
	csc $(CSCOPTS) portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o

Modified api.scm from [d744d47aad] to [bcdab13d33].

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
;;    - returns #( flag result )
;;
(define (api:execute-requests dbstruct dat)
  (handle-exceptions
   exn
   (let ((call-chain (get-call-chain)))
     (print-call-chain (current-error-port))
     (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))       
     (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
   (if (not (vector? dat))                    ;; it is an error to not receive a vector
       (vector #f #f "remote must be called with a vector")       
       (vector                                   ;; return a vector + the returned data structure
	#t 
	(let ((cmd    (vector-ref dat 0))
	      (params (vector-ref dat 1)))







|







105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
;;    - returns #( flag result )
;;
(define (api:execute-requests dbstruct dat)
  (handle-exceptions
   exn
   (let ((call-chain (get-call-chain)))
     (print-call-chain (current-error-port))
     (debug:print 0 *default-log-port* " message: "  ((condition-property-accessor 'exn 'message) exn))       
     (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
   (if (not (vector? dat))                    ;; it is an error to not receive a vector
       (vector #f #f "remote must be called with a vector")       
       (vector                                   ;; return a vector + the returned data structure
	#t 
	(let ((cmd    (vector-ref dat 0))
	      (params (vector-ref dat 1)))

Modified client.scm from [3a2fa3c3cb] to [b597605018].

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

;;======================================================================
;; C L I E N T S
;;======================================================================

(require-extension (srfi 18) extras tcp s11n)

(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
;; (use zmq)

(import (prefix sqlite3 sqlite3:))

(use spiffy uri-common intarweb http-client spiffy-request-vars uri-common intarweb directory-utils)

(declare (unit client))

(declare (uses common))
(declare (uses db))







|


|







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

;;======================================================================
;; C L I E N T S
;;======================================================================

(require-extension (srfi 18) extras tcp s11n)

(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
;; (use zmq)

(use (prefix sqlite3 sqlite3:))

(use spiffy uri-common intarweb http-client spiffy-request-vars uri-common intarweb directory-utils)

(declare (unit client))

(declare (uses common))
(declare (uses db))
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    ((rpc)  (rpc:client-connect  iface port))
    ((http) (http:client-connect iface port))
    ((zmq)  (zmq:client-connect  iface port))
    (else   (rpc:client-connect  iface port))))

(define (client:setup  run-id #!key (remaining-tries 10) (failed-connects 0))
  (case (server:get-transport)
    ((rpc) (rpc-transport:client-setup run-id)) ;;(client:setup-rpc run-id))
    ((http)(client:setup-http run-id))
    (else  (rpc-transport:client-setup run-id)))) ;; (client:setup-rpc run-id))))

;; (define (client:login-no-auto-setup server-info run-id)
;;   (case (server:get-transport)
;;     ((rpc)  (rpc:login-no-auto-client-setup server-info run-id))
;;     ((http) (rmt:login-no-auto-client-setup server-info run-id))
;;     (else   (rpc:login-no-auto-client-setup server-info run-id))))
;; 







|
|
|







48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    ((rpc)  (rpc:client-connect  iface port))
    ((http) (http:client-connect iface port))
    ((zmq)  (zmq:client-connect  iface port))
    (else   (rpc:client-connect  iface port))))

(define (client:setup  run-id #!key (remaining-tries 10) (failed-connects 0))
  (case (server:get-transport)
    ((rpc) (rpc-transport:client-setup run-id remaining-tries: remaining-tries failed-connects: failed-connects)) ;;(client:setup-rpc run-id))
    ((http)(client:setup-http run-id remaining-tries: remaining-tries failed-connects: failed-connects))
    (else  (rpc-transport:client-setup run-id remaining-tries: remaining-tries failed-connects: failed-connects)))) ;; (client:setup-rpc run-id))))

;; (define (client:login-no-auto-setup server-info run-id)
;;   (case (server:get-transport)
;;     ((rpc)  (rpc:login-no-auto-client-setup server-info run-id))
;;     ((http) (rmt:login-no-auto-client-setup server-info run-id))
;;     (else   (rpc:login-no-auto-client-setup server-info run-id))))
;; 
165
166
167
168
169
170
171
172

173
174
175
176
177


178
179
180
181
182
183
184
185
	  (debug:print-info 4 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
	  (if server-dat
	      (let* ((iface     (tasks:hostinfo-get-interface server-dat))
		     (hostname  (tasks:hostinfo-get-hostname  server-dat))
		     (port      (tasks:hostinfo-get-port      server-dat))
		     (start-res (case *transport-type*
				  ((http)(http-transport:client-connect iface port))
				  ((nmsg)(nmsg-transport:client-connect hostname port))))

		     (ping-res  (case *transport-type* 
				  ((http)(rmt:login-no-auto-client-setup start-res run-id))
				  ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
 					   (if logininfo
 					       (car (vector-ref logininfo 1))


 					       #f))))))
		(if (and start-res
			 ping-res)
		    (begin
		      (hash-table-set! *runremote* run-id start-res)
		      (debug:print-info 2 *default-log-port* "connected to " (http-transport:server-dat-make-url start-res))
		      start-res)
		    (begin    ;; login failed but have a server record, clean out the record and try again







|
>


|
|
|
>
>
|







165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
	  (debug:print-info 4 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
	  (if server-dat
	      (let* ((iface     (tasks:hostinfo-get-interface server-dat))
		     (hostname  (tasks:hostinfo-get-hostname  server-dat))
		     (port      (tasks:hostinfo-get-port      server-dat))
		     (start-res (case *transport-type*
				  ((http)(http-transport:client-connect iface port))
				  ;;((nmsg)(nmsg-transport:client-connect hostname port))
                                  ))
		     (ping-res  (case *transport-type* 
				  ((http)(rmt:login-no-auto-client-setup start-res run-id))
				  ;; ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
 				  ;;          (if logininfo
 				  ;;              (car (vector-ref logininfo 1))
 				  ;;              #f)))

                                  )))
		(if (and start-res
			 ping-res)
		    (begin
		      (hash-table-set! *runremote* run-id start-res)
		      (debug:print-info 2 *default-log-port* "connected to " (http-transport:server-dat-make-url start-res))
		      start-res)
		    (begin    ;; login failed but have a server record, clean out the record and try again

Modified common.scm from [14a3d2fe92] to [d577dfee02].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 nanomsg sql-de-lite hostinfo)
(require-extension regex posix)

(require-extension (srfi 18) extras tcp rpc)

(import (prefix sqlite3 sqlite3:))
(import (prefix base64 base64:))












|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 sql-de-lite hostinfo)
(require-extension regex posix)

(require-extension (srfi 18) extras tcp rpc)

(import (prefix sqlite3 sqlite3:))
(import (prefix base64 base64:))

129
130
131
132
133
134
135


136
137
138
139
140
141
142
143
144
145
146
147
148
149
150




151
152
153
154
155
156
157
158

159
160
161
162
163
164
165
166
167
168
169
170
171


172
173
174
175
176
177
178
179
180
181
182
183
  (set! *test-id-cache*      (make-hash-table)))

;; Generic string database
(define sdb:qry #f) ;; (make-sdb:qry)) ;;  'init #f)
;; Generic path database
(define *fdb* #f)



;;======================================================================
;; V E R S I O N
;;======================================================================

(define (common:get-full-version)
  (conc megatest-version "-" megatest-fossil-hash))

(define (common:version-signature)
  (conc megatest-version "-" (substring megatest-fossil-hash 0 4)))

;; from metadat lookup MEGATEST_VERSION
;;
(define (common:get-last-run-version)
  (rmt:get-var "MEGATEST_VERSION"))





(define (common:set-last-run-version)
  (rmt:set-var "MEGATEST_VERSION" (common:version-signature)))

(define (common:version-changed?)
  (not (equal? (common:get-last-run-version)
	       (common:version-signature))))

;; Move me elsewhere ...

;;
(define (common:cleanup-db)
  (db:multi-db-sync 
   #f ;; do all run-ids
   ;; 'new2old
   'killservers
   'dejunk
   ;; 'adj-testids
   ;; 'old2new
   'new2old)
  (if (common:version-changed?)
      (common:set-last-run-version)))



(define (common:exit-on-version-changed)
  (if (common:version-changed?)
      (let ((mtconf (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.config")))
        (debug:print 0 *default-log-port*
		     "ERROR: Version mismatch!\n"
		     "   expected: " (common:version-signature) "\n"
		     "   got:      " (common:get-last-run-version))
	(if (and (file-exists? mtconf)
		 (eq? (current-user-id)(file-owner mtconf))) ;; safe to run -cleanup-db
	    (begin
	      (debug:print 0 *default-log-port* "   I see you are the owner of megatest.config, attempting to cleanup and reset to new version")
	      (handle-exceptions







>
>












|


>
>
>
>








>













>
>




|







129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
  (set! *test-id-cache*      (make-hash-table)))

;; Generic string database
(define sdb:qry #f) ;; (make-sdb:qry)) ;;  'init #f)
;; Generic path database
(define *fdb* #f)

(define *last-launch* (current-seconds)) ;; use for throttling the launch rate. Would be better to use the db and last time of a test in LAUNCHED state.

;;======================================================================
;; V E R S I O N
;;======================================================================

(define (common:get-full-version)
  (conc megatest-version "-" megatest-fossil-hash))

(define (common:version-signature)
  (conc megatest-version "-" (substring megatest-fossil-hash 0 4)))

;; from metadat lookup MEGATEST_VERSION
;;
(define (common:get-last-run-version) ;; RADT => How does this work in send-receive function??; assume it is the value saved in some DB
  (rmt:get-var "MEGATEST_VERSION"))

(define (common:get-last-run-version-number)
  (string->number 
   (substring (common:get-last-run-version) 0 6)))

(define (common:set-last-run-version)
  (rmt:set-var "MEGATEST_VERSION" (common:version-signature)))

(define (common:version-changed?)
  (not (equal? (common:get-last-run-version)
	       (common:version-signature))))

;; Move me elsewhere ...
;; RADT => Why do we meed the version check here, this is called only if version misma
;;
(define (common:cleanup-db)
  (db:multi-db-sync 
   #f ;; do all run-ids
   ;; 'new2old
   'killservers
   'dejunk
   ;; 'adj-testids
   ;; 'old2new
   'new2old)
  (if (common:version-changed?)
      (common:set-last-run-version)))

;; Force a megatest cleanup-db if version is changed and skip-version-check not specified
;;
(define (common:exit-on-version-changed)
  (if (common:version-changed?)
      (let ((mtconf (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.config")))
        (debug:print 0 *default-log-port*
		     "WARNING: Version mismatch!\n"
		     "   expected: " (common:version-signature) "\n"
		     "   got:      " (common:get-last-run-version))
	(if (and (file-exists? mtconf)
		 (eq? (current-user-id)(file-owner mtconf))) ;; safe to run -cleanup-db
	    (begin
	      (debug:print 0 *default-log-port* "   I see you are the owner of megatest.config, attempting to cleanup and reset to new version")
	      (handle-exceptions
351
352
353
354
355
356
357

358

359
360
361
362
363
364
365

366
367
368
369
370
371
372

(define (assoc/default key lst . default)
  (let ((res (assoc key lst)))
    (if res (cadr res)(if (null? default) #f (car default)))))

(define (common:get-testsuite-name)
  (or (configf:lookup *configdat* "setup" "testsuite" )

      (pathname-file *toppath*)))


;;======================================================================
;; E X I T   H A N D L I N G
;;======================================================================

(define (common:legacy-sync-recommended)
  (or (args:get-arg "-runtests")

      (args:get-arg "-server")
      ;; (args:get-arg "-set-run-status")
      (args:get-arg "-remove-runs")
      ;; (args:get-arg "-get-run-status")
      ))

(define (common:legacy-sync-required)







>
|
>







>







360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384

(define (assoc/default key lst . default)
  (let ((res (assoc key lst)))
    (if res (cadr res)(if (null? default) #f (car default)))))

(define (common:get-testsuite-name)
  (or (configf:lookup *configdat* "setup" "testsuite" )
      (if *toppath* 
          (pathname-file *toppath*)
          (pathname-file (current-directory)))))

;;======================================================================
;; E X I T   H A N D L I N G
;;======================================================================

(define (common:legacy-sync-recommended)
  (or (args:get-arg "-runtests")
      (args:get-arg "-run")
      (args:get-arg "-server")
      ;; (args:get-arg "-set-run-status")
      (args:get-arg "-remove-runs")
      ;; (args:get-arg "-get-run-status")
      ))

(define (common:legacy-sync-required)
482
483
484
485
486
487
488


























489
490
491
492
493
494
495
	targs)))

;; '(print (string-intersperse (map cadr (hash-table-ref/default (read-config "megatest.config" \#f \#t) "disks" '"'"'("none" ""))) "\n"))'
(define (common:get-disks #!key (configf #f))
  (hash-table-ref/default 
   (or configf (read-config "megatest.config" #f #t))
   "disks" '("none" "")))



























;;======================================================================
;; T A R G E T S  ,   S T A T E ,   S T A T U S ,   
;;                    R U N N A M E    A N D   T E S T P A T T
;;======================================================================

;; Lookup a value in runconfigs based on -reqtarg or -target







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
	targs)))

;; '(print (string-intersperse (map cadr (hash-table-ref/default (read-config "megatest.config" \#f \#t) "disks" '"'"'("none" ""))) "\n"))'
(define (common:get-disks #!key (configf #f))
  (hash-table-ref/default 
   (or configf (read-config "megatest.config" #f #t))
   "disks" '("none" "")))

;; return first command that exists, else #f
;;
(define (common:which cmds)
  (if (null? cmds)
      #f
      (let loop ((hed (car cmds))
		 (tal (cdr cmds)))
	(let ((res (with-input-from-pipe (conc "which " hed) read-line)))
	  (if (and (string? res)
		   (file-exists? res))
	      res
	      (if (null? tal)
		  #f
		  (loop (car tal)(cdr tal))))))))
  
(define (common:get-install-area)
  (let ((exe-path (car (argv))))
    (if (file-exists? exe-path)
	(handle-exceptions
	 exn
	 #f
	 (pathname-directory
	  (pathname-directory 
	   (pathname-directory exe-path))))
	#f)))

;;======================================================================
;; T A R G E T S  ,   S T A T E ,   S T A T U S ,   
;;                    R U N N A M E    A N D   T E S T P A T T
;;======================================================================

;; Lookup a value in runconfigs based on -reqtarg or -target
585
586
587
588
589
590
591


































































592
593
594
595
596
597
598
	     (tal     (cdr inlst)))
    (if (not (null? tal))
	(loop (max hed max-val)
	      (car tal)
	      (cdr tal))
	(max hed max-val))))




































































;;======================================================================
;; M U N G E   D A T A   I N T O   N I C E   F O R M S
;;======================================================================

;; Generate an index for a sparse list of key values
;;   ( (rowname1 colname1 val1)(rowname2 colname2 val2) )







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
	     (tal     (cdr inlst)))
    (if (not (null? tal))
	(loop (max hed max-val)
	      (car tal)
	      (cdr tal))
	(max hed max-val))))

;; get min or max, use > for max and < for min, this works around the limits on apply
;;
(define (common:min-max comp lst)
  (if (null? lst)
      #f ;; better than an exception for my needs
      (fold (lambda (a b)
	      (if (comp a b) a b))
	    (car lst)
	    lst)))

;; path list to hash-table tree
;;   ((a b c)(a b d)(e b c)) => ((a (b (d) (c))) (e (b (c))))
;;
(define (common:list->htree lst)
  (let ((resh (make-hash-table)))
    (for-each
     (lambda (inlst)
       (let loop ((ht  resh)
		  (hed (car inlst))
		  (tal (cdr inlst)))
	 (if (hash-table-ref/default ht hed #f)
	     (if (not (null? tal))
		 (loop (hash-table-ref ht hed)
		       (car tal)
		       (cdr tal)))
	     (begin
	       (hash-table-set! ht hed (make-hash-table))
	       (loop ht hed tal)))))
     lst)
    resh))

;; hash-table tree to html list tree
;;
;;   tipfunc takes two parameters: y the tip value and path the path to that point
;;
(define (common:htree->html ht path tipfunc)
  (let ((datlist 	(sort (hash-table->alist ht)
                              (lambda (a b)
                                (string< (car a)(car b))))))
    (if (null? datlist)
    	(tipfunc #f path) ;; really shouldn't get here
	(s:ul
	 (map (lambda (x)
		(let* ((levelname (car x))
		       (y         (cdr x))
		       (newpath   (append path (list levelname)))
		       (leaf      (or (not (hash-table? y))
				      (null? (hash-table-keys y)))))
		  (if leaf
		      (s:li (tipfunc y newpath))
		      (s:li
		       (list 
			levelname
			(common:htree->html y newpath tipfunc))))))
	      datlist)))))

;; hash-table tree to alist tree
;;
(define (common:htree->atree ht)
  (map (lambda (x)
	 (cons (car x)
	       (let ((y (cdr x)))
		 (if (hash-table? y)
		     (common:htree->atree y)
		     y))))
       (hash-table->alist ht)))

;;======================================================================
;; M U N G E   D A T A   I N T O   N I C E   F O R M S
;;======================================================================

;; Generate an index for a sparse list of key values
;;   ( (rowname1 colname1 val1)(rowname2 colname2 val2) )
632
633
634
635
636
637
638








639
640
641
642
643
644
645
		    (if (> curr-rownum rownum) curr-rownum rownum)
		    (if (> curr-colnum colnum) curr-colnum colnum)
		    ))))))

;;======================================================================
;; S Y S T E M   S T U F F
;;======================================================================









;; return a nice clean pathname made absolute
(define (common:nice-path dir)
  (let ((match (string-match "^(~[^\\/]*)(\\/.*|)$" dir)))
    (if match ;; using ~ for home?
	(common:nice-path (conc (common:read-link-f (cadr match)) "/" (caddr match)))
	(normalize-pathname (if (absolute-pathname? dir)







>
>
>
>
>
>
>
>







736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
		    (if (> curr-rownum rownum) curr-rownum rownum)
		    (if (> curr-colnum colnum) curr-colnum colnum)
		    ))))))

;;======================================================================
;; S Y S T E M   S T U F F
;;======================================================================

;; lazy-safe get file mod time. on any error (file not existing etc.) return 0
;;
(define (common:lazy-modification-time fpath)
  (handle-exceptions
   exn
   0
   (file-modification-time fpath)))

;; return a nice clean pathname made absolute
(define (common:nice-path dir)
  (let ((match (string-match "^(~[^\\/]*)(\\/.*|)$" dir)))
    (if match ;; using ~ for home?
	(common:nice-path (conc (common:read-link-f (cadr match)) "/" (caddr match)))
	(normalize-pathname (if (absolute-pathname? dir)
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679






680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711





712
713
714
715
716
717
718
719
720
721
722
723
	(debug:print-error 0 *default-log-port* "command \"/bin/readlink -f " path "\" failed.")
	path) ;; just give up
    (with-input-from-pipe
	(conc "/bin/readlink -f " path)
      (lambda ()
	(read-line)))))

(define (get-cpu-load)
  (car (common:get-cpu-load)))
;;   (let* ((load-res (process:cmd-run->list "uptime"))
;; 	 (load-rx  (regexp "load average:\\s+(\\d+)"))
;; 	 (cpu-load #f))
;;     (for-each (lambda (l)
;; 		(let ((match (string-search load-rx l)))
;; 		  (if match
;; 		      (let ((newval (string->number (cadr match))))
;; 			(if (number? newval)
;; 			    (set! cpu-load newval))))))
;; 	      (car load-res))
;;     cpu-load))

;; get cpu load by reading from /proc/loadavg, return all three values
;;
(define (common:get-cpu-load)






  (with-input-from-file "/proc/loadavg" 
    (lambda ()(list (read)(read)(read)))))

(define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000) (msg #f))
  (let* ((loadavg (common:get-cpu-load))
	 (first   (car loadavg))
	 (next    (cadr loadavg))
	 (adjload (* maxload numcpus))
	 (loadjmp (- first next)))
    (cond
     ((and (> first adjload)
	   (> count 0))
      (debug:print-info 0 *default-log-port* "waiting " waitdelay " seconds due to load " first " exceeding max of " adjload (if msg msg ""))
      (thread-sleep! waitdelay)
      (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1)))
     ((and (> loadjmp numcpus)
	   (> count 0))
      (debug:print-info 0 *default-log-port* "waiting " waitdelay " seconds due to load jump " loadjmp " > numcpus " numcpus (if msg msg ""))
      (thread-sleep! waitdelay)
      (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1))))))

(define (common:get-num-cpus)
  (with-input-from-file "/proc/cpuinfo"
    (lambda ()
      (let loop ((numcpu 0)
		 (inl    (read-line)))
	(if (eof-object? inl)
	    numcpu
	    (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl)
		      (+ numcpu 1)
		      numcpu)
		  (read-line)))))))






;; wait for normalized cpu load to drop below maxload
;;
(define (common:wait-for-normalized-load maxload #!key (msg #f))
  (let ((num-cpus (common:get-num-cpus)))
    (common:wait-for-cpuload maxload num-cpus 15 msg: msg)))

(define (get-uname . params)
  (let* ((uname-res (process:cmd-run->list (conc "uname " (if (null? params) "-a" (car params)))))
	 (uname #f))
    (if (null? (car uname-res))
	"unknown"







|
|














|
>
>
>
>
>
>
|
|

|
|
















|
<
|
|
|
|
|
|
|
|
|
>
>
>
>
>



|
|







768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819

820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
	(debug:print-error 0 *default-log-port* "command \"/bin/readlink -f " path "\" failed.")
	path) ;; just give up
    (with-input-from-pipe
	(conc "/bin/readlink -f " path)
      (lambda ()
	(read-line)))))

(define (get-cpu-load #!key (remote-host #f))
  (car (common:get-cpu-load remote-host)))
;;   (let* ((load-res (process:cmd-run->list "uptime"))
;; 	 (load-rx  (regexp "load average:\\s+(\\d+)"))
;; 	 (cpu-load #f))
;;     (for-each (lambda (l)
;; 		(let ((match (string-search load-rx l)))
;; 		  (if match
;; 		      (let ((newval (string->number (cadr match))))
;; 			(if (number? newval)
;; 			    (set! cpu-load newval))))))
;; 	      (car load-res))
;;     cpu-load))

;; get cpu load by reading from /proc/loadavg, return all three values
;;
(define (common:get-cpu-load remote-host)
  (if remote-host
      (map (lambda (res)
	     (if (eof-object? res) 9e99 res))
	   (with-input-from-pipe 
	    (conc "ssh " remote-host " cat /proc/loadavg")
	    (lambda ()(list (read)(read)(read)))))
      (with-input-from-file "/proc/loadavg" 
	(lambda ()(list (read)(read)(read))))))

(define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000) (msg #f)(remote-host #f))
  (let* ((loadavg (common:get-cpu-load remote-host))
	 (first   (car loadavg))
	 (next    (cadr loadavg))
	 (adjload (* maxload numcpus))
	 (loadjmp (- first next)))
    (cond
     ((and (> first adjload)
	   (> count 0))
      (debug:print-info 0 *default-log-port* "waiting " waitdelay " seconds due to load " first " exceeding max of " adjload (if msg msg ""))
      (thread-sleep! waitdelay)
      (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1)))
     ((and (> loadjmp numcpus)
	   (> count 0))
      (debug:print-info 0 *default-log-port* "waiting " waitdelay " seconds due to load jump " loadjmp " > numcpus " numcpus (if msg msg ""))
      (thread-sleep! waitdelay)
      (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1))))))

(define (common:get-num-cpus remote-host)

  (let ((proc (lambda ()
		(let loop ((numcpu 0)
			   (inl    (read-line)))
		  (if (eof-object? inl)
		      numcpu
		      (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl)
				(+ numcpu 1)
				numcpu)
			    (read-line)))))))
    (if remote-host
	(with-input-from-pipe 
	 (conc "ssh " remote-host " cat /proc/cpuinfo")
	 proc)
	(with-input-from-file "/proc/cpuinfo" proc))))

;; wait for normalized cpu load to drop below maxload
;;
(define (common:wait-for-normalized-load maxload #!key (msg #f)(remote-host #f))
  (let ((num-cpus (common:get-num-cpus remote-host)))
    (common:wait-for-cpuload maxload num-cpus 15 msg: msg)))

(define (get-uname . params)
  (let* ((uname-res (process:cmd-run->list (conc "uname " (if (null? params) "-a" (car params)))))
	 (uname #f))
    (if (null? (car uname-res))
	"unknown"
916
917
918
919
920
921
922
923
924
925
926



927


928
929
930
931
932
933
934
935
     (proc          (proc)))
    (hash-table-for-each
     vars
     (lambda (var val)
       (setenv var val)))
    vars))

(define (common:run-a-command cmd)
  (let ((fullcmd  (conc (dtests:get-pre-command)
			cmd 
			(dtests:get-post-command))))



    (debug:print-info 02 *default-log-port* "Running command: " fullcmd)


    (common:without-vars fullcmd "MT_.*")))
		  
;;======================================================================
;; T I M E   A N D   D A T E
;;======================================================================

;; Convert strings like "5s 2h 3m" => 60x60x2 + 3x60 + 5
(define (common:hms-string->seconds tstr)







|
|
<
|
>
>
>

>
>
|







1038
1039
1040
1041
1042
1043
1044
1045
1046

1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
     (proc          (proc)))
    (hash-table-for-each
     vars
     (lambda (var val)
       (setenv var val)))
    vars))

(define (common:run-a-command cmd #!key (with-vars #f))
  (let* ((pre-cmd  (dtests:get-pre-command))

         (post-cmd (dtests:get-post-command))
         (fullcmd  (if (or pre-cmd post-cmd)
                       (conc pre-cmd cmd post-cmd)
                       (conc "viewscreen " cmd))))
    (debug:print-info 02 *default-log-port* "Running command: " fullcmd)
    (if with-vars
        (common:without-vars cmd)
        (common:without-vars fullcmd "MT_.*"))))
		  
;;======================================================================
;; T I M E   A N D   D A T E
;;======================================================================

;; Convert strings like "5s 2h 3m" => 60x60x2 + 3x60 + 5
(define (common:hms-string->seconds tstr)
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
     ;; NOTE: This can fail when there is no mention of the host in /etc/hosts. FIXME
     (vector->list (hostinfo-addresses (hostname->hostinfo hostname))))
    (string-intersperse 
     (map number->string
	  (u8vector->list
	   (if res res (hostname->ip hostname)))) ".")))

(define (common:open-nm-req addr)
  (let* ((req (nn-socket 'req))
	 (res (nn-connect req addr)))
    req))

;; (with-output-to-string (lambda ()(serialize obj)))
(define (common:nm-send-receive soc msg)
  (nn-send soc msg)
  (nn-recv soc))

(define (common:close-nm-req soc)
  (nn-close soc))

(define (common:send-dboard-main-changed)
  (let* ((dashboard-ips (mddb:get-dashboards)))
    (for-each
     (lambda (ipadr)
       (let* ((soc (common:open-nm-req (conc "tcp://" ipadr)))
	      (msg (conc "main " *toppath*))
	      (res (common:nm-send-receive-timeout soc msg)))
	 (if (not res) ;; couldn't reach that dashboard - remove it from db
	     (print "ERROR: couldn't reach dashboard " ipadr))
	 res))
     dashboard-ips)))
    
(define (common:nm-send-receive-timeout req msg)
  (let* ((key     "ping")
	 (success #f)
	 (keepwaiting #t)
	 (result  #f)
	 (sendrec (make-thread
		   (lambda ()
		     (nn-send req msg)
		     (set! result (nn-recv req))
		     (set! success #t))
		   "send-receive"))
	 (timeout (make-thread (lambda ()
				 (let loop ((count 0))
				   (thread-sleep! 1)
				   (print "still waiting after count seconds...")
				   (if (and keepwaiting (< count 10))
				       (loop (+ count 1))))
				 (if keepwaiting
				     (begin
				       (print "timeout waiting for reply")
				       (thread-terminate! sendrec))))
			       "timeout")))
    (handle-exceptions
     exn
     (begin
       (print-call-chain)
       (print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
       (print "exn=" (condition->list exn)))
     (thread-start! timeout)
     (thread-start! sendrec)
     (thread-join!  sendrec)
     (if success (thread-terminate! timeout)))
    result))
    
(define (common:ping-nm req)
  ;; send a random number and check that we get it back
  (let* ((key     "ping")
	 (success #f)
	 (keepwaiting #t)
	 (ping    (make-thread
		   (lambda ()
		     (print "ping: sending string \"" key "\", expecting " (current-process-id))
		     (nn-send req key)
		     (let ((result  (nn-recv req)))
		       (if (equal? (conc (current-process-id)) result)
			   (begin
			     (print "ping, success: received \"" result "\"")
			     (set! success #t))
			   (begin
			     (print "ping, failed: received key \"" result "\"")
			     (set! keepwaiting #f)
			     (set! success #f)))))
		   "ping"))
	 (timeout (make-thread (lambda ()
				 (let loop ((count 0))
				   (thread-sleep! 1)
				   (print "still waiting after count seconds...")
				   (if (and keepwaiting (< count 10))
				       (loop (+ count 1))))
				 (if keepwaiting
				     (begin
				       (print "timeout waiting for ping")
				       (thread-terminate! ping))))
			       "timeout")))
    (handle-exceptions
     exn
     (begin
       (print-call-chain)
       (print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
       (print "exn=" (condition->list exn))
       (print "ping failed to connect to tcp://" hostport))
     (thread-start! timeout)
     (thread-start! ping)
     (thread-join! ping)
     (if success (thread-terminate! timeout)))
    (if return-socket
	(if success req #f)
	(begin
	  (nn-close req)
	  success))))

;;======================================================================
;; D A S H B O A R D   D B 
;;======================================================================

(define (mddb:open-db)
  (let* ((db (open-database (conc (get-environment-variable "HOME") "/.dashboard.db"))))
    (set-busy-handler! db (busy-timeout 10000))







<
<
<
<
<
<
<
<
<
<
<
<













<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







1210
1211
1212
1213
1214
1215
1216












1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229

































1230















































1231
1232
1233
1234
1235
1236
1237
     ;; NOTE: This can fail when there is no mention of the host in /etc/hosts. FIXME
     (vector->list (hostinfo-addresses (hostname->hostinfo hostname))))
    (string-intersperse 
     (map number->string
	  (u8vector->list
	   (if res res (hostname->ip hostname)))) ".")))














(define (common:send-dboard-main-changed)
  (let* ((dashboard-ips (mddb:get-dashboards)))
    (for-each
     (lambda (ipadr)
       (let* ((soc (common:open-nm-req (conc "tcp://" ipadr)))
	      (msg (conc "main " *toppath*))
	      (res (common:nm-send-receive-timeout soc msg)))
	 (if (not res) ;; couldn't reach that dashboard - remove it from db
	     (print "ERROR: couldn't reach dashboard " ipadr))
	 res))
     dashboard-ips)))
    

































    















































;;======================================================================
;; D A S H B O A R D   D B 
;;======================================================================

(define (mddb:open-db)
  (let* ((db (open-database (conc (get-environment-variable "HOME") "/.dashboard.db"))))
    (set-busy-handler! db (busy-timeout 10000))

Modified common_records.scm from [a408794bcc] to [6bf211fc41].

101
102
103
104
105
106
107














108
109
110
111
112
113
114
  (if (debug:debug-mode n)
      (with-output-to-port (or e (current-error-port))
	(lambda ()
	  (if *logging*
	      (db:log-event (apply conc params))
	      (apply print params)
	      )))))















(define (debug:print-error n e . params)
  ;; normal print
  (if (debug:debug-mode n)
      (with-output-to-port (or e (current-error-port))
	(lambda ()
	  (if *logging*







>
>
>
>
>
>
>
>
>
>
>
>
>
>







101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  (if (debug:debug-mode n)
      (with-output-to-port (or e (current-error-port))
	(lambda ()
	  (if *logging*
	      (db:log-event (apply conc params))
	      (apply print params)
	      )))))

;; Brandon's debug printer shortcut (indulge me :)
(define (BB> . in-args)
  (let* ((stack (get-call-chain))
         (location #f))
    (for-each
     (lambda (frame)
       (let* ((this-loc (vector-ref frame 0))
              (this-func (cadr (string-split this-loc " "))))
         (if (equal? this-func "BB>")
             (set! location this-loc))))
     stack)
    (let ((dp-args (append (list 0 *default-log-port* location"   "  ) in-args)))
      (apply debug:print dp-args))))

(define (debug:print-error n e . params)
  ;; normal print
  (if (debug:debug-mode n)
      (with-output-to-port (or e (current-error-port))
	(lambda ()
	  (if *logging*

Modified dashboard-tests.scm from [3a6a535f7d] to [269ce18d09].

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
;; C O M M O N
;;======================================================================

(define *dashboard-comment-share-slot* #f)

(define (dtests:get-pre-command #!key (default-override #f))
  (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "pre-command")))
    (or cfg-ovrd default-override "xterm -geometry 180x20 -e \"")))

(define (dtests:get-post-command #!key (default-override #f))
  (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "post-command")))
    (or cfg-ovrd default-override ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &")))


(define (test-info-panel testdat store-label widgets)
  (iup:frame 
   #:title "Test Info" ; #:expand "YES"
   (iup:hbox ; #:expand "YES"
    (apply iup:vbox ; #:expand "YES"







|



|







39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
;; C O M M O N
;;======================================================================

(define *dashboard-comment-share-slot* #f)

(define (dtests:get-pre-command #!key (default-override #f))
  (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "pre-command")))
    (or cfg-ovrd default-override "viewscreen "))) ;; "xterm -geometry 180x20 -e \"")))

(define (dtests:get-post-command #!key (default-override #f))
  (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "post-command")))
    (or cfg-ovrd default-override ""))) ;; ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &")))


(define (test-info-panel testdat store-label widgets)
  (iup:frame 
   #:title "Test Info" ; #:expand "YES"
   (iup:hbox ; #:expand "YES"
    (apply iup:vbox ; #:expand "YES"
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
					 (iup:destroy! dlog)))))))
    dlog))


;;======================================================================
;;
;;======================================================================
(define (examine-test run-id test-id) ;; run-id run-key origtest)
  (let* ((db-path       (db:dbfile-path run-id)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/db/" run-id ".db"))
	 (dbstruct      (make-dbr:dbstruct path:  (db:dbfile-path #f) ;; (configf:lookup *configdat* "setup" "linktree") 
					   local: #t))
	 (testdat        (rmt:get-test-info-by-id run-id test-id)) ;; (db:get-test-info-by-id dbstruct run-id test-id))
	 (db-mod-time   0) ;; (file-modification-time db-path))
	 (last-update   0) ;; (current-seconds))
	 (request-update #t))







|







412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
					 (iup:destroy! dlog)))))))
    dlog))


;;======================================================================
;;
;;======================================================================
(define (dashboard-tests:examine-test run-id test-id) ;; run-id run-key origtest)
  (let* ((db-path       (db:dbfile-path run-id)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/db/" run-id ".db"))
	 (dbstruct      (make-dbr:dbstruct path:  (db:dbfile-path #f) ;; (configf:lookup *configdat* "setup" "linktree") 
					   local: #t))
	 (testdat        (rmt:get-test-info-by-id run-id test-id)) ;; (db:get-test-info-by-id dbstruct run-id test-id))
	 (db-mod-time   0) ;; (file-modification-time db-path))
	 (last-update   0) ;; (current-seconds))
	 (request-update #t))

Modified dashboard.scm from [d651eae42e] to [ef1ffd321d].

84
85
86
87
88
89
90





91
92
93
94
95
96


97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

114
115
116
117
118
119
120
121
122
123
124
125


126
127
128
129
130
131


132
133
134
135
136
137
138

139
140
141
142
143
144
145
146
147
148
149
150
151
152

153
154
155
156
157
158
159
			"-v"
			"-q"
			"-use-local"
			"-skip-version-check"
			)
		 args:arg-hash
		 0))






(if (args:get-arg "-h")
    (begin
      (print help)
      (exit)))



(if (not (launch:setup))
    (begin
      (print "Failed to find megatest.config, exiting") 
      (exit 1)))

;; data common to all tabs goes here
;;
(defstruct dboard:commondat
  ((curr-tab-num 0) : number)
  please-update  
  tabdats
  update-mutex
  updaters 
  updating
  uidat ;; needs to move to tabdat at some time
  hide-not-hide-tabs
  )


(define (dboard:commondat-make)
  (make-dboard:commondat
   curr-tab-num:         0
   tabdats:              (make-hash-table)
   please-update:        #t
   update-mutex:         (make-mutex)
   updaters:             (make-hash-table)
   updating:             #f
   hide-not-hide-tabs:   #f
   ))



(define (dboard:common-get-tabdat commondat #!key (tab-num #f))
  (hash-table-ref/default 
   (dboard:commondat-tabdats commondat)
   (or tab-num (dboard:commondat-curr-tab-num commondat))
   #f))



(define (dboard:common-set-tabdat! commondat tabnum tabdat)
  (hash-table-set!
   (dboard:commondat-tabdats commondat)
   tabnum
   tabdat))

;; gets and calls updater based on curr-tab-num

(define (dboard:common-run-curr-updaters commondat #!key (tab-num #f))
  (if (dboard:common-get-tabdat commondat tab-num: tab-num) ;; only update if there is a tabdat
      (let* ((tnum     (or tab-num (dboard:commondat-curr-tab-num commondat)))
	     (updaters (hash-table-ref/default (dboard:commondat-updaters commondat)
					       tnum
					       '())))
	(debug:print 4 *default-log-port* "Found these updaters: " updaters " for tab-num: " tnum)
	(for-each
	 (lambda (updater)
	   ;; (debug:print 3 *default-log-port* "Running " updater)
	   (updater))
	 updaters))))

;; if tab-num passed in then use it, otherwise look in commondat at curr-tab-num

;;
(define (dboard:commondat-add-updater commondat updater #!key (tab-num #f))
  (let* ((tnum          (or tab-num
			     (dboard:commondat-curr-tab-num commondat)))
	 (curr-updaters (hash-table-ref/default (dboard:commondat-updaters commondat) tnum '())))
    (hash-table-set! (dboard:commondat-updaters commondat)
		     tnum







>
>
>
>
>






>
>

















>












>
>



|


>
>






|
>







|






>







84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
			"-v"
			"-q"
			"-use-local"
			"-skip-version-check"
			)
		 args:arg-hash
		 0))

(if (not (null? remargs))
    (begin
      (print "Unrecognised arguments: " (string-intersperse remargs " "))
      (exit)))

(if (args:get-arg "-h")
    (begin
      (print help)
      (exit)))

;; TODO: Move this inside (main)
;;
(if (not (launch:setup))
    (begin
      (print "Failed to find megatest.config, exiting") 
      (exit 1)))

;; data common to all tabs goes here
;;
(defstruct dboard:commondat
  ((curr-tab-num 0) : number)
  please-update  
  tabdats
  update-mutex
  updaters 
  updating
  uidat ;; needs to move to tabdat at some time
  hide-not-hide-tabs
  )


(define (dboard:commondat-make)
  (make-dboard:commondat
   curr-tab-num:         0
   tabdats:              (make-hash-table)
   please-update:        #t
   update-mutex:         (make-mutex)
   updaters:             (make-hash-table)
   updating:             #f
   hide-not-hide-tabs:   #f
   ))

;; RA => returns the tabdat stored at hashkey passed in commondat-tabdats table (e.g. 0 gives summary)
;;
(define (dboard:common-get-tabdat commondat #!key (tab-num #f))
  (hash-table-ref/default 
   (dboard:commondat-tabdats commondat)
   (or tab-num (dboard:commondat-curr-tab-num commondat)) ;; tab-num value is curr-tab-num value in passed commondat
   #f))

;; RA => sets the tabdat passed to the hashkey at commondat:tabdats hash table
;;
(define (dboard:common-set-tabdat! commondat tabnum tabdat)
  (hash-table-set!
   (dboard:commondat-tabdats commondat)
   tabnum
   tabdat))

;; gets and calls updater list based on curr-tab-num
;;
(define (dboard:common-run-curr-updaters commondat #!key (tab-num #f))
  (if (dboard:common-get-tabdat commondat tab-num: tab-num) ;; only update if there is a tabdat
      (let* ((tnum     (or tab-num (dboard:commondat-curr-tab-num commondat)))
	     (updaters (hash-table-ref/default (dboard:commondat-updaters commondat)
					       tnum
					       '())))
	(debug:print 4 *default-log-port* "Found these updaters: " updaters " for tab-num: " tnum)
	(for-each ;; perform the function calls for the complete updaters list
	 (lambda (updater)
	   ;; (debug:print 3 *default-log-port* "Running " updater)
	   (updater))
	 updaters))))

;; if tab-num passed in then use it, otherwise look in commondat at curr-tab-num
;; adds the updater passed in the updaters list at that hashkey
;;
(define (dboard:commondat-add-updater commondat updater #!key (tab-num #f))
  (let* ((tnum          (or tab-num
			     (dboard:commondat-curr-tab-num commondat)))
	 (curr-updaters (hash-table-ref/default (dboard:commondat-updaters commondat) tnum '())))
    (hash-table-set! (dboard:commondat-updaters commondat)
		     tnum
198
199
200
201
202
203
204








205
206
207

208
209
210
211
212
213
214
215

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241

242
243
244
245
246








247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
  ((max-row           0)                 : number)
  ((running-layout    #f)                : boolean)
  (originx            #f)
  (originy            #f)
  ((layout-update-ok  #t)                : boolean)
  ((compact-layout    #t)                : boolean)









  ;; Controls used to launch runs etc.
  ((command          "")                 : string)      ;; for run control this is the command being built up
  (command-tb        #f)			         

  (key-listboxes     #f)			         
  (key-lbs           #f)			         
  run-name                                              ;; from run name setting widget
  states                                                ;; states for -state s1,s2 ...
  statuses                                              ;; statuses for -status s1,s2 ...
						         
  ;; Selector variables				         
  curr-run-id                                           ;; current row to display in Run summary view

  curr-test-ids                                         ;; used only in dcommon:run-update which is used in newdashboard
  ((filters-changed  #f)                  : boolean)    ;; to to indicate that the user changed filters for this tab
  ((last-filter-str  "")                  : string)      ;; conc the target runname and testpatt for a signature of changed filters
  ((hide-empty-runs  #f)                  : boolean)     
  ((hide-not-hide    #t)                  : boolean)     ;; toggle for hide/not hide empty runs
  (hide-not-hide-button #f)
  ((searchpatts        (make-hash-table)) : hash-table)  ;;
  ((state-ignore-hash  (make-hash-table)) : hash-table)  ;; hash of  STATE => #t/#f for display control
  ((status-ignore-hash (make-hash-table)) : hash-table)  ;; hash of STATUS => #t/#f
  (target              #f)
  (test-patts          #f)

  ;; db info to file the .db files for the area
  (dbdir               #f)
  (dbfpath             #f)
  (dbkeys              #f)
  ((last-db-update     0)                : number)      ;; last db file timestamp
  (monitor-db-path     #f)                              ;; where to find monitor.db
  ro                                                    ;; is the database read-only?

  ;; tests data
  ((num-tests          10)               : number)      ;; total number of tests to show (used in the old runs display)

  ;; runs tree
  ((path-run-ids       (make-hash-table)) : hash-table) ;; path (target / runname) => id
  (runs-tree           #f)


  ;; tab data
  ((view-changed       #t)                : boolean)   
  ((xadj               0)                 : number)     ;; x slider number (if using canvas)
  ((yadj               0)                 : number)     ;; y slider number (if using canvas)









  tests-tree       ;; used in newdashboard
  )

(define (dboard:tabdat-target-string vec)
  (let ((targ (dboard:tabdat-target vec)))
    (if (list? targ)(string-intersperse targ "/") "no-target-specified")))

(define (dboard:tabdat-test-patts-use vec)    
  (let ((val (dboard:tabdat-test-patts vec)))(if val val "")))

;; additional setters for dboard:data
(define (dboard:tabdat-test-patts-set!-use    vec val)
  (dboard:tabdat-test-patts-set! vec (if (equal? val "") #f val)))

(define (dboard:tabdat-make-data)
  (let ((dat (make-dboard:tabdat)))







>
>
>
>
>
>
>
>


|
>








>

|














|









>





>
>
>
>
>
>
>
>
|








|







212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
  ((max-row           0)                 : number)
  ((running-layout    #f)                : boolean)
  (originx            #f)
  (originy            #f)
  ((layout-update-ok  #t)                : boolean)
  ((compact-layout    #t)                : boolean)

  ;; Run times layout
  ;; (graph-button-box #f) ;; RA => Think it is not referenced anywhere
  (graph-matrix     #f)
  ((graph-matrix-table (make-hash-table)) : hash-table) ;; graph-dats referenced thru graph name info
  ((graph-cell-table (make-hash-table)) : hash-table) ;; graph-dats referenced thru matrix cell info
  ((graph-matrix-row 1) : number)
  ((graph-matrix-col 1) : number)

  ;; Controls used to launch runs etc.
  ((command          "")                 : string)      ;; for run control this is the command being built up
  (command-tb        #f)	                        ;; widget for the type of command; run, remove-runs etc.
  (test-patterns-textbox #f)                            ;; text box widget for editing a list of test patterns
  (key-listboxes     #f)			         
  (key-lbs           #f)			         
  run-name                                              ;; from run name setting widget
  states                                                ;; states for -state s1,s2 ...
  statuses                                              ;; statuses for -status s1,s2 ...
						         
  ;; Selector variables				         
  curr-run-id                                           ;; current row to display in Run summary view
  prev-run-id                                           ;; previous runid selected before current runid was selected (used in xor-two-runs runs summary mode
  curr-test-ids                                         ;; used only in dcommon:run-update which is used in newdashboard
  ((filters-changed  #t)                  : boolean)    ;; to to indicate that the user changed filters for this tab
  ((last-filter-str  "")                  : string)      ;; conc the target runname and testpatt for a signature of changed filters
  ((hide-empty-runs  #f)                  : boolean)     
  ((hide-not-hide    #t)                  : boolean)     ;; toggle for hide/not hide empty runs
  (hide-not-hide-button #f)
  ((searchpatts        (make-hash-table)) : hash-table)  ;;
  ((state-ignore-hash  (make-hash-table)) : hash-table)  ;; hash of  STATE => #t/#f for display control
  ((status-ignore-hash (make-hash-table)) : hash-table)  ;; hash of STATUS => #t/#f
  (target              #f)
  (test-patts          #f)

  ;; db info to file the .db files for the area
  (dbdir               #f)
  (dbfpath             #f)
  (dbkeys              #f)
  ((last-db-update     (make-hash-table)) : hash-table) ;; last db file timestamp
  (monitor-db-path     #f)                              ;; where to find monitor.db
  ro                                                    ;; is the database read-only?

  ;; tests data
  ((num-tests          10)               : number)      ;; total number of tests to show (used in the old runs display)

  ;; runs tree
  ((path-run-ids       (make-hash-table)) : hash-table) ;; path (target / runname) => id
  (runs-tree           #f)
  ((runs-tree-ht       (make-hash-table)) : hash-table) ;; track which targests added to tree (merge functionality with path-run-ids?)

  ;; tab data
  ((view-changed       #t)                : boolean)   
  ((xadj               0)                 : number)     ;; x slider number (if using canvas)
  ((yadj               0)                 : number)     ;; y slider number (if using canvas)
  ;; runs-summary tab state
  ((runs-summary-modes '((one-run . "Show One Run") (xor-two-runs . "XOR Two Runs") (xor-two-runs-hide-clean . "XOR; Hide Clean")) )   : list)
  ((runs-summary-mode-buttons '())               : list)
  ((runs-summary-mode  'one-run)            : symbol)
  ((runs-summary-mode-change-callbacks '()) : list)
  (runs-summary-source-runname-label #f)
  (runs-summary-dest-runname-label #f)
  ;; runs summary view
  
  tests-tree       ;; used in newdashboard
  )

(define (dboard:tabdat-target-string vec)
  (let ((targ (dboard:tabdat-target vec)))
    (if (list? targ)(string-intersperse targ "/") "no-target-specified")))

(define (dboard:tabdat-test-patts-use vec)    
  (let ((val (dboard:tabdat-test-patts vec)))(if val val ""))) ;;RADT => What is the if for?

;; additional setters for dboard:data
(define (dboard:tabdat-test-patts-set!-use    vec val)
  (dboard:tabdat-test-patts-set! vec (if (equal? val "") #f val)))

(define (dboard:tabdat-make-data)
  (let ((dat (make-dboard:tabdat)))
273
274
275
276
277
278
279








280
281
282
283
284
285
286
  ;; HACK ALERT: this is a hack, please fix.
  (dboard:tabdat-ro-set! tabdat (not (file-read-access? (dboard:tabdat-dbfpath tabdat))))
  
  (dboard:tabdat-keys-set! tabdat (rmt:get-keys))
  (dboard:tabdat-dbkeys-set! tabdat (append (dboard:tabdat-keys tabdat) (list "runname")))
  (dboard:tabdat-tot-runs-set! tabdat (rmt:get-num-runs "%"))
  )









;; data for runs, tests etc. was used in run summary?
;;
(defstruct dboard:runsdat
  ;; new system
  runs-index    ;; target/runname => colnum
  tests-index   ;; testname/itempath => rownum







>
>
>
>
>
>
>
>







306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
  ;; HACK ALERT: this is a hack, please fix.
  (dboard:tabdat-ro-set! tabdat (not (file-read-access? (dboard:tabdat-dbfpath tabdat))))
  
  (dboard:tabdat-keys-set! tabdat (rmt:get-keys))
  (dboard:tabdat-dbkeys-set! tabdat (append (dboard:tabdat-keys tabdat) (list "runname")))
  (dboard:tabdat-tot-runs-set! tabdat (rmt:get-num-runs "%"))
  )

;; RADT => Matrix defstruct addition
(defstruct dboard:graph-dat
    ((id           #f) : string)
    ((color        #f) : vector)
    ((flag         #t) : boolean)
    ((cell         #f) : number)
    )

;; data for runs, tests etc. was used in run summary?
;;
(defstruct dboard:runsdat
  ;; new system
  runs-index    ;; target/runname => colnum
  tests-index   ;; testname/itempath => rownum
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
(define (dboard:rundat-make-init #!key (run #f)(key-vals #f)(tests #f));; -100 is before time began
  (make-dboard:rundat 
   run: run
   tests: (or tests (make-hash-table))
   key-vals: key-vals 
   )) 

(define (dboard:rundat-copy-tests-to-by-name rundat)
  (let ((src-ht (dboard:rundat-tests rundat))
	(trg-ht (dboard:rundat-tests-by-name rundat)))
    (if (and (hash-table? src-ht)(hash-table? trg-ht))
	(begin
	  (hash-table-clear! trg-ht)
	  (for-each
	   (lambda (testdat)
	     (hash-table-set! trg-ht (test:test-get-fullname testdat) testdat))
	   (hash-table-values src-ht)))
	(debug:print 0 *default-log-port* "WARNING: src-ht " src-ht " trg-ht " trg-ht))))
  
(defstruct dboard:testdat
  id       ;; testid
  state    ;; test state
  status   ;; test status
  )

(define (dboard:runsdat-get-col-num dat target runname force-set)
  (let* ((runs-index (dboard:runsdat-runs-index dat))
	 (col-name   (conc target "/" runname))
	 (res        (hash-table-ref/default runs-index col-name #f)))
    (if res
	res
	(if force-set
	    (let ((max-col-num (+ 1 (apply max -1 (hash-table-values runs-index)))))
	      (hash-table-set! runs-index col-name max-col-num)
	      max-col-num)))))

(define (dboard:runsdat-get-row-num dat testname itempath force-set)
  (let* ((tests-index (dboard:runsdat-runs-index dat))
	 (row-name    (conc testname "/" itempath))
	 (res         (hash-table-ref/default runs-index row-name #f)))
    (if res
	res
	(if force-set
	    (let ((max-row-num (+ 1 (apply max -1 (hash-table-values tests-index)))))
	      (hash-table-set! runs-index row-name max-row-num)
	      max-row-num)))))

;; default is to NOT set the cell if the column and row names are not pre-existing
;;
(define (dboard:runsdat-set-test-cell dat target runname testname itempath test-id state status #!key (force-set #f))
  (let* ((col-num  (dboard:runsdat-get-col-num dat target runname force-set))
	 (row-num  (dboard:runsdat-get-row-num dat testname itempath force-set)))
    (if (and row-num col-num)
	(let ((tdat (dboard:testdat 
		     id: test-id
		     state: state
		     status: status)))
	  (sparse-array-set! (dboard:runsdat-matrix-dat dat) col-num row-num tdat)
	  tdat)







<
<
<
<
<
<
<
<
<
<
<
<






<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<



|
|







355
356
357
358
359
360
361












362
363
364
365
366
367






















368
369
370
371
372
373
374
375
376
377
378
379
(define (dboard:rundat-make-init #!key (run #f)(key-vals #f)(tests #f));; -100 is before time began
  (make-dboard:rundat 
   run: run
   tests: (or tests (make-hash-table))
   key-vals: key-vals 
   )) 













(defstruct dboard:testdat
  id       ;; testid
  state    ;; test state
  status   ;; test status
  )























;; default is to NOT set the cell if the column and row names are not pre-existing
;;
(define (dboard:runsdat-set-test-cell dat target runname testname itempath test-id state status #!key (force-set #f))
  (let* ((col-num  (dcommon:runsdat-get-col-num dat target runname force-set))
	 (row-num  (dcommon:runsdat-get-row-num dat testname itempath force-set)))
    (if (and row-num col-num)
	(let ((tdat (dboard:testdat 
		     id: test-id
		     state: state
		     status: status)))
	  (sparse-array-set! (dboard:runsdat-matrix-dat dat) col-num row-num tdat)
	  tdat)
470
471
472
473
474
475
476
477






478
479


480
481
482
483
484
485
486
487
488
489
490
491



492



493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
;; This is roughly the same as dboard:get-tests-dat, should merge them if possible
;;
;; gets all the tests for run-id that match testnamepatt and key-vals, merges them
;;
;;    NOTE: Yes, this is used
;;
(define (dboard:get-tests-for-run-duplicate tabdat run-id run testnamepatt key-vals)
  (let* ((num-to-get  100)






	 (states      (hash-table-keys (dboard:tabdat-state-ignore-hash tabdat)))
	 (statuses    (hash-table-keys (dboard:tabdat-status-ignore-hash tabdat)))


	 (sort-info   (get-curr-sort))
	 (sort-by     (vector-ref sort-info 1))
	 (sort-order  (vector-ref sort-info 2))
	 (bubble-type (if (member sort-order '(testname))
			  'testname
			  'itempath))
	 ;; note: the rundat is normally created in "update-rundat". 
	 (run-dat    (or (hash-table-ref/default (dboard:tabdat-allruns-by-id tabdat) run-id #f)
			 (let ((rd (dboard:rundat-make-init run: run key-vals: key-vals)))
			   (hash-table-set! (dboard:tabdat-allruns-by-id tabdat) run-id rd)
			   rd)))
	 ;; (prev-tests  (dboard:rundat-tests prev-dat)) ;; (vector-ref prev-dat 1))



	 (last-update (dboard:rundat-last-update run-dat)) ;; (vector-ref prev-dat 3))



	 (db-path     (or (dboard:rundat-db-path run-dat)
			  (let* ((db-dir (tasks:get-task-db-path))
				 (db-pth (conc db-dir "/" run-id ".db")))
			    (dboard:rundat-db-path-set! run-dat db-pth)
			    db-pth)))
	 (tmptests    (if (or (configf:lookup *configdat* "setup" "do-not-use-db-file-timestamps")
			      (>= (file-modification-time db-path) last-update))
			  (rmt:get-tests-for-run run-id testnamepatt states statuses  ;; run-id testpatt states statuses
						 (dboard:rundat-run-data-offset run-dat)
						 num-to-get
						 (dboard:tabdat-hide-not-hide tabdat) ;; no-in
						 sort-by                              ;; sort-by
						 sort-order                           ;; sort-order
						 #f ;; 'shortlist                           ;; qrytype
						 (if (dboard:tabdat-filters-changed tabdat) 







|
>
>
>
>
>
>


>
>












>
>
>
|
>
>
>





|
|
|







477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
;; This is roughly the same as dboard:get-tests-dat, should merge them if possible
;;
;; gets all the tests for run-id that match testnamepatt and key-vals, merges them
;;
;;    NOTE: Yes, this is used
;;
(define (dboard:get-tests-for-run-duplicate tabdat run-id run testnamepatt key-vals)
  (let* ((num-to-get
          (let ((num-tests-from-config (configf:lookup *configdat* "setup" "num-tests-to-get")))
            (if num-tests-from-config
                (begin
                  (BB> "override num-tests 100 -> "num-tests-from-config)
                  (string->number num-tests-from-config))
                100)))
	 (states      (hash-table-keys (dboard:tabdat-state-ignore-hash tabdat)))
	 (statuses    (hash-table-keys (dboard:tabdat-status-ignore-hash tabdat)))
         (do-not-use-db-file-timestamps (configf:lookup *configdat* "setup" "do-not-use-db-file-timestamps")) ;; this still hosts runs-summary-tab
         (do-not-use-query-timestamps (configf:lookup *configdat* "setup" "do-not-use-query-timestamps")) ;; this no longer troubles runs-summary-tab
	 (sort-info   (get-curr-sort))
	 (sort-by     (vector-ref sort-info 1))
	 (sort-order  (vector-ref sort-info 2))
	 (bubble-type (if (member sort-order '(testname))
			  'testname
			  'itempath))
	 ;; note: the rundat is normally created in "update-rundat". 
	 (run-dat    (or (hash-table-ref/default (dboard:tabdat-allruns-by-id tabdat) run-id #f)
			 (let ((rd (dboard:rundat-make-init run: run key-vals: key-vals)))
			   (hash-table-set! (dboard:tabdat-allruns-by-id tabdat) run-id rd)
			   rd)))
	 ;; (prev-tests  (dboard:rundat-tests prev-dat)) ;; (vector-ref prev-dat 1))
         (last-update
          (if do-not-use-query-timestamps
              0
              (dboard:rundat-last-update run-dat)
              ;;(hash-table-ref/default (dboard:tabdat-run-update-times tabdat) run-id 0)
              ))

	 (db-path     (or (dboard:rundat-db-path run-dat)
			  (let* ((db-dir (tasks:get-task-db-path))
				 (db-pth (conc db-dir "/" run-id ".db")))
			    (dboard:rundat-db-path-set! run-dat db-pth)
			    db-pth)))
	 (tmptests    (if (or do-not-use-db-file-timestamps
			      (>=  (common:lazy-modification-time db-path) last-update))
                          (rmt:get-tests-for-run run-id testnamepatt states statuses  ;; run-id testpatt states statuses
						 (dboard:rundat-run-data-offset run-dat)
						 num-to-get
						 (dboard:tabdat-hide-not-hide tabdat) ;; no-in
						 sort-by                              ;; sort-by
						 sort-order                           ;; sort-order
						 #f ;; 'shortlist                           ;; qrytype
						 (if (dboard:tabdat-filters-changed tabdat) 
537
538
539
540
541
542
543

544


545
546
547
548
549
550
551
     tmptests)
    
    ;; set last-update to 0 if still getting data incrementally

    (if (> (dboard:rundat-run-data-offset run-dat) 0)
	(begin
	  ;; (print "run-data-offset: " (dboard:rundat-run-data-offset run-dat) ", setting last-update to 0")

	  (dboard:rundat-last-update-set! run-dat 0))


	(dboard:rundat-last-update-set! run-dat (- (current-seconds) 2))) ;; go back two seconds in time to ensure all changes are captured.

    ;; (debug:print-info 0 *default-log-port* "tests-ht: " (hash-table-keys tests-ht))
    tests-ht))

;; tmptests   - new tests data
;; prev-tests - old tests data







>
|
>
>







558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
     tmptests)
    
    ;; set last-update to 0 if still getting data incrementally

    (if (> (dboard:rundat-run-data-offset run-dat) 0)
	(begin
	  ;; (print "run-data-offset: " (dboard:rundat-run-data-offset run-dat) ", setting last-update to 0")
	  ;; (dboard:rundat-last-update-set! run-dat 0)
          (dboard:rundat-last-update-set! run-dat 0))
        ;; (hash-table-set! (dboard:tabdat-run-update-times tabdat) run-id (- start-time 3))
        
	(dboard:rundat-last-update-set! run-dat (- (current-seconds) 2))) ;; go back two seconds in time to ensure all changes are captured.

    ;; (debug:print-info 0 *default-log-port* "tests-ht: " (hash-table-keys tests-ht))
    tests-ht))

;; tmptests   - new tests data
;; prev-tests - old tests data
635
636
637
638
639
640
641



642
643
644
645
646
647
648
		    (dboard:tabdat-allruns-set! tabdat new-res)
		    maxtests)
		  (if (> (dboard:rundat-run-data-offset run-struct) 0)
		      (loop run tal new-res newmaxtests) ;; not done getting data for this run
		      (loop (car tal)(cdr tal) new-res newmaxtests)))))))
    (dboard:tabdat-filters-changed-set! tabdat #f)
    (dboard:update-tree tabdat runs-hash header tb)))




(define *collapsed* (make-hash-table))

(define (toggle-hide lnum uidat) ; fulltestname)
  (let* ((btn (vector-ref (dboard:uidat-get-lftcol uidat) lnum))
	 (fulltestname (iup:attribute btn "TITLE"))
	 (parts        (string-split fulltestname "("))







>
>
>







659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
		    (dboard:tabdat-allruns-set! tabdat new-res)
		    maxtests)
		  (if (> (dboard:rundat-run-data-offset run-struct) 0)
		      (loop run tal new-res newmaxtests) ;; not done getting data for this run
		      (loop (car tal)(cdr tal) new-res newmaxtests)))))))
    (dboard:tabdat-filters-changed-set! tabdat #f)
    (dboard:update-tree tabdat runs-hash header tb)))




(define *collapsed* (make-hash-table))

(define (toggle-hide lnum uidat) ; fulltestname)
  (let* ((btn (vector-ref (dboard:uidat-get-lftcol uidat) lnum))
	 (fulltestname (iup:attribute btn "TITLE"))
	 (parts        (string-split fulltestname "("))
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
			  (take-right (dboard:tabdat-allruns tabdat) numruns)
			  (pad-list (dboard:tabdat-allruns tabdat) numruns)))
	 (lftcol      (dboard:uidat-get-lftcol uidat))
	 (tableheader (dboard:uidat-get-header uidat))
	 (table       (dboard:uidat-get-runsvec uidat))
	 (coln        0)
	 (all-test-names (make-hash-table)))

    ;; create a concise list of test names
    ;;
    (for-each
     (lambda (rundat)
       (if rundat
	   (let* ((testdats  (dboard:rundat-tests rundat))
		  (testnames (map test:test-get-fullname (hash-table-values testdats))))
	     (dboard:rundat-copy-tests-to-by-name rundat)
	     ;; for the normalized list of testnames (union of all runs)
	     (if (not (and (dboard:tabdat-hide-empty-runs tabdat)
			   (null? testnames)))
		 (for-each (lambda (testname)
			     (hash-table-set! all-test-names testname #t))
			   testnames)))))
     runs)







<







|







815
816
817
818
819
820
821

822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
			  (take-right (dboard:tabdat-allruns tabdat) numruns)
			  (pad-list (dboard:tabdat-allruns tabdat) numruns)))
	 (lftcol      (dboard:uidat-get-lftcol uidat))
	 (tableheader (dboard:uidat-get-header uidat))
	 (table       (dboard:uidat-get-runsvec uidat))
	 (coln        0)
	 (all-test-names (make-hash-table)))

    ;; create a concise list of test names
    ;;
    (for-each
     (lambda (rundat)
       (if rundat
	   (let* ((testdats  (dboard:rundat-tests rundat))
		  (testnames (map test:test-get-fullname (hash-table-values testdats))))
	     (dcommon:rundat-copy-tests-to-by-name rundat)
	     ;; for the normalized list of testnames (union of all runs)
	     (if (not (and (dboard:tabdat-hide-empty-runs tabdat)
			   (null? testnames)))
		 (for-each (lambda (testname)
			     (hash-table-set! all-test-names testname #t))
			   testnames)))))
     runs)
923
924
925
926
927
928
929


930
931
932
933
934
935
936
937
938
939
    (dboard:tabdat-filters-changed-set! tabdat #t)))

(define (update-search commondat tabdat x val)
  (hash-table-set! (dboard:tabdat-searchpatts tabdat) x val)
  (dboard:tabdat-filters-changed-set! tabdat #t)
  (set-bg-on-filter commondat tabdat))



(define (mark-for-update tabdat)
  ;; (dboard:tabdat-filters-changed-set! tabdat #t)
  (dboard:tabdat-last-db-update-set! tabdat 0))

;;======================================================================
;; R U N C O N T R O L
;;======================================================================

;; target populating logic
;;  







>
>

<
|







949
950
951
952
953
954
955
956
957
958

959
960
961
962
963
964
965
966
    (dboard:tabdat-filters-changed-set! tabdat #t)))

(define (update-search commondat tabdat x val)
  (hash-table-set! (dboard:tabdat-searchpatts tabdat) x val)
  (dboard:tabdat-filters-changed-set! tabdat #t)
  (set-bg-on-filter commondat tabdat))

;; force ALL updates to zero (effectively)
;;
(define (mark-for-update tabdat)

  (dboard:tabdat-last-db-update-set! tabdat (make-hash-table)))

;;======================================================================
;; R U N C O N T R O L
;;======================================================================

;; target populating logic
;;  
1060
1061
1062
1063
1064
1065
1066

1067


1068
1069
1070
1071
1072
1073
1074

;; Extract the various bits of data from tabdat and create the command line equivalent that will be displayed
;;
(define (dashboard:update-run-command tabdat)
  (let* ((cmd-tb       (dboard:tabdat-command-tb tabdat))
	 (cmd          (dboard:tabdat-command    tabdat))
	 (test-patt    (let ((tp (dboard:tabdat-test-patts tabdat)))

			 (if (equal? tp "") "%" tp)))


	 (states       (dboard:tabdat-states     tabdat))
	 (statuses     (dboard:tabdat-statuses   tabdat))
	 (target       (let ((targ-list (dboard:tabdat-target     tabdat)))
			 (if targ-list (string-intersperse targ-list "/") "no-target-selected")))
	 (run-name     (dboard:tabdat-run-name   tabdat))
	 (states-str   (if (or (not states)
			       (null? states))







>
|
>
>







1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104

;; Extract the various bits of data from tabdat and create the command line equivalent that will be displayed
;;
(define (dashboard:update-run-command tabdat)
  (let* ((cmd-tb       (dboard:tabdat-command-tb tabdat))
	 (cmd          (dboard:tabdat-command    tabdat))
	 (test-patt    (let ((tp (dboard:tabdat-test-patts tabdat)))
			 (if (or (not tp)
                                 (equal? tp ""))
                             "%"
                             tp)))
	 (states       (dboard:tabdat-states     tabdat))
	 (statuses     (dboard:tabdat-statuses   tabdat))
	 (target       (let ((targ-list (dboard:tabdat-target     tabdat)))
			 (if targ-list (string-intersperse targ-list "/") "no-target-selected")))
	 (run-name     (dboard:tabdat-run-name   tabdat))
	 (states-str   (if (or (not states)
			       (null? states))
1147
1148
1149
1150
1151
1152
1153

1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168




1169

1170
1171


1172
1173
1174
1175
1176
1177
1178

;; used by run-controls
;;
(define (dashboard:update-tree-selector tabdat #!key (action-proc #f))
  (let* ((tb            (dboard:tabdat-runs-tree tabdat))
	 (runconf-targs (common:get-runconfig-targets))
	 (db-target-dat (rmt:get-targets))

	 (header        (vector-ref db-target-dat 0))
	 (db-targets    (vector-ref db-target-dat 1))
	 (munge-target  (lambda (x)            ;; create a target vector from a string. Pad with na if needed.
			  (take (append (string-split x "/")
					(make-list (length header) "na"))
				(length header))))
	 (all-targets   (append (list (munge-target (string-intersperse 
						     (map (lambda (x) "%") header)
						     "/")))
				(map vector->list db-targets)
				(map munge-target
				     runconf-targs)
				)))
    (for-each
     (lambda (target)




       (tree:add-node tb "Runs" target)) ;; (append key-vals (list run-name))

     all-targets)))



(define (dashboard:run-controls commondat tabdat #!key (tab-num #f))
  (let* ((targets       (make-hash-table))
	 (test-records  (make-hash-table))
	 (all-tests-registry (tests:get-all)) ;; (tests:get-valid-tests *toppath* '()))
	 (test-names    (hash-table-keys all-tests-registry))
	 (sorted-testnames #f)
	 (action        "-run")







>















>
>
>
>
|
>


>
>







1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216

;; used by run-controls
;;
(define (dashboard:update-tree-selector tabdat #!key (action-proc #f))
  (let* ((tb            (dboard:tabdat-runs-tree tabdat))
	 (runconf-targs (common:get-runconfig-targets))
	 (db-target-dat (rmt:get-targets))
         (runs-tree-ht  (dboard:tabdat-runs-tree-ht tabdat))
	 (header        (vector-ref db-target-dat 0))
	 (db-targets    (vector-ref db-target-dat 1))
	 (munge-target  (lambda (x)            ;; create a target vector from a string. Pad with na if needed.
			  (take (append (string-split x "/")
					(make-list (length header) "na"))
				(length header))))
	 (all-targets   (append (list (munge-target (string-intersperse 
						     (map (lambda (x) "%") header)
						     "/")))
				(map vector->list db-targets)
				(map munge-target
				     runconf-targs)
				)))
    (for-each
     (lambda (target)
       (if (not (hash-table-ref/default runs-tree-ht target #f))
           ;; (let ((existing (tree:find-node tb target)))
           ;;   (if (not existing)
           (begin
             (tree:add-node tb "Runs" target) ;; (append key-vals (list run-name))
             (hash-table-set! runs-tree-ht target #t))))
     all-targets)))

;; Run controls panel
;;
(define (dashboard:run-controls commondat tabdat #!key (tab-num #f))
  (let* ((targets       (make-hash-table))
	 (test-records  (make-hash-table))
	 (all-tests-registry (tests:get-all)) ;; (tests:get-valid-tests *toppath* '()))
	 (test-names    (hash-table-keys all-tests-registry))
	 (sorted-testnames #f)
	 (action        "-run")
1209
1210
1211
1212
1213
1214
1215

1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227












1228
1229
1230
1231
1232

1233
1234
1235
1236
1237
1238
1239
1240


1241
1242
1243




1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
	      (dcommon:command-testname-selector commondat tabdat update-keyvals))
	     ;;  key-listboxes))
	     (dcommon:command-tests-tasks-canvas tabdat test-records sorted-testnames tests-draw-state))))
	   (tb (dboard:tabdat-runs-tree tabdat)))
      (dboard:commondat-add-updater 
       commondat 
       (lambda ()

	 (dashboard:update-tree-selector tabdat))
       tab-num: tab-num)
      result)))

 ;;(iup:frame
 ;; #:title "Logs" ;; To be replaced with tabs
 ;; (let ((logs-tb (iup:textbox #:expand "YES"
 ;;				   #:multiline "YES")))
 ;;	 (dboard:tabdat-logs-textbox-set! tabdat logs-tb)
 ;;	 logs-tb))

(define (dboard:runs-tree-browser commondat tabdat)












  (let* ((tb      (iup:treebox
		   #:value 0
		   #:name "Runs"
		   #:expand "YES"
		   #:addexpanded "NO"

		   #:selection-cb
		   (lambda (obj id state)
		     (debug:catch-and-dump
		      (lambda ()
			(let* ((run-path (tree:node->path obj id))
			       (run-id    (tree-path->run-id tabdat (cdr run-path))))
                          ;; (dboard:tabdat-view-changed-set! tabdat #t) ;; ?? done below when run-id is a number
			  (dboard:tabdat-target-set! tabdat (cdr run-path)) ;; (print "run-path: " run-path)			    


			  (dboard:tabdat-layout-update-ok-set! tabdat #f)
			  (if (number? run-id)
			      (begin




				(dboard:tabdat-curr-run-id-set! tabdat run-id)
				(dboard:tabdat-view-changed-set! tabdat #t))
			      (debug:print-error 5 *default-log-port* "tree-path->run-id returned non-number " run-id))))
		      "treebox"))
		   ;; (print "path: " (tree:node->path obj id) " run-id: " run-id)
		   )))
    (dboard:tabdat-runs-tree-set! tabdat tb)
    tb))

;;======================================================================
;; R U N   C O N T R O L S
;;======================================================================
;;
;; A gui for launching tests
;;







>
|











>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
>
|
|
|
|
|
|
|
|
>
>
|
|
|
>
>
>
>
|
|
|
|
|
|

|







1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
	      (dcommon:command-testname-selector commondat tabdat update-keyvals))
	     ;;  key-listboxes))
	     (dcommon:command-tests-tasks-canvas tabdat test-records sorted-testnames tests-draw-state))))
	   (tb (dboard:tabdat-runs-tree tabdat)))
      (dboard:commondat-add-updater 
       commondat 
       (lambda ()
         (if (dashboard:database-changed? commondat tabdat context-key: 'run-control)
             (dashboard:update-tree-selector tabdat)))
       tab-num: tab-num)
      result)))

 ;;(iup:frame
 ;; #:title "Logs" ;; To be replaced with tabs
 ;; (let ((logs-tb (iup:textbox #:expand "YES"
 ;;				   #:multiline "YES")))
 ;;	 (dboard:tabdat-logs-textbox-set! tabdat logs-tb)
 ;;	 logs-tb))

(define (dboard:runs-tree-browser commondat tabdat)
  (let* ((txtbox (iup:textbox #:action (lambda (val a b)
					 (debug:catch-and-dump
					  (lambda ()
					    (if b (dboard:tabdat-target-set! tabdat (string-split b "/")))
					    (dashboard:update-run-command tabdat))
					  "command-testname-selector tb action"))
			      #:value (dboard:test-patt->lines
				       (dboard:tabdat-test-patts-use tabdat))
			      #:expand "HORIZONTAL"
			      ;; #:size "10x30"
			      ))
	 (tb
          (iup:treebox
           #:value 0
           #:name "Runs"
           #:expand "YES"
           #:addexpanded "NO"
           #:size "10x"
           #:selection-cb
           (lambda (obj id state)
             (debug:catch-and-dump
              (lambda ()
                (let* ((run-path (tree:node->path obj id))
                       (run-id    (tree-path->run-id tabdat (cdr run-path))))
                  ;; (dboard:tabdat-view-changed-set! tabdat #t) ;; ?? done below when run-id is a number
                  (dboard:tabdat-target-set! tabdat (cdr run-path)) ;; (print "run-path: " run-path)
		  (iup:attribute-set! txtbox "VALUE" (string-intersperse (cdr run-path) "/"))
		  (dashboard:update-run-command tabdat)
                  (dboard:tabdat-layout-update-ok-set! tabdat #f)
                  (if (number? run-id)
                      (begin
                        ;; capture last two in tabdat.
                        (dboard:tabdat-prev-run-id-set!
                         tabdat
                         (dboard:tabdat-curr-run-id tabdat))
                        (dboard:tabdat-curr-run-id-set! tabdat run-id)
                        (dboard:tabdat-view-changed-set! tabdat #t))
                      (debug:print-error 5 *default-log-port* "tree-path->run-id returned non-number " run-id))))
              "treebox"))
           ;; (print "path: " (tree:node->path obj id) " run-id: " run-id)
           )))
    (dboard:tabdat-runs-tree-set! tabdat tb)
    (iup:vbox tb txtbox)))

;;======================================================================
;; R U N   C O N T R O L S
;;======================================================================
;;
;; A gui for launching tests
;;
1309
1310
1311
1312
1313
1314
1315



1316
1317
1318
1319
1320
1321
1322
			  (dboard:tabdat-compact-layout-set! tabdat #t))
		      (dboard:tabdat-last-filter-str-set! tabdat "")
		      )
		    "text-list-toggle-box"))))
      (dcommon:command-runname-selector commondat tabdat tab-num: tab-num)
      (dcommon:command-testname-selector commondat tabdat update-keyvals))
     (iup:vbox



      (let* ((cnv-obj (iup:canvas 
		       ;; #:size "500x400"
		       #:expand "YES"
		       #:scrollbar "YES"
		       #:posx "0.5"
		       #:posy "0.5"
		       #:action (make-canvas-action







>
>
>







1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
			  (dboard:tabdat-compact-layout-set! tabdat #t))
		      (dboard:tabdat-last-filter-str-set! tabdat "")
		      )
		    "text-list-toggle-box"))))
      (dcommon:command-runname-selector commondat tabdat tab-num: tab-num)
      (dcommon:command-testname-selector commondat tabdat update-keyvals))
     (iup:vbox
      (iup:split
       #:orientation "HORIZONTAL"
       #:value 800
      (let* ((cnv-obj (iup:canvas 
		       ;; #:size "500x400"
		       #:expand "YES"
		       #:scrollbar "YES"
		       #:posx "0.5"
		       #:posy "0.5"
		       #:action (make-canvas-action
1349
1350
1351
1352
1353
1354
1355
1356


















































1357
1358
1359
1360
1361
1362
1363
					  (vg:drawing-scalex-set! drawing
								  (+ scalex
								     (if (> step 0)
									 (* scalex  0.02)
									 (* scalex -0.02))))))
				      "wheel-cb"))
		       )))
	cnv-obj)))))



















































;;======================================================================
;; R U N
;;======================================================================
;;
;; display and manage a single run at a time








|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
					  (vg:drawing-scalex-set! drawing
								  (+ scalex
								     (if (> step 0)
									 (* scalex  0.02)
									 (* scalex -0.02))))))
				      "wheel-cb"))
		       )))
	cnv-obj)
      (let* ((hb1 (iup:hbox))
             (graph-cell-table (dboard:tabdat-graph-cell-table tabdat))
             (changed #f)
             (graph-matrix (iup:matrix
                           #:alignment1 "ALEFT"
                           #:expand "YES" ;; "HORIZONTAL"
                           #:scrollbar "YES"
                           #:numcol 10
                           #:numlin 20
                           #:numcol-visible (min 8)
                           #:numlin-visible 1
                           #:click-cb
                           (lambda (obj row col status)
                             (let*
                                 ((graph-cell (conc row ":" col))
                                 (graph-dat   (hash-table-ref/default graph-cell-table graph-cell #f))
                                 (graph-flag  (dboard:graph-dat-flag graph-dat)))
                               (if graph-flag
                                   (dboard:graph-dat-flag-set! graph-dat #f)
                                   (dboard:graph-dat-flag-set! graph-dat #t))
                               (if (not (dboard:tabdat-running-layout tabdat))
						     (begin
						       (dashboard:run-times-tab-run-data-updater commondat tabdat tab-num)
						       (dboard:tabdat-last-data-update-set! tabdat (current-seconds))
						       (thread-start! (make-thread
								       (lambda ()
									 (dboard:tabdat-running-layout-set! tabdat #t)
									 (dashboard:run-times-tab-layout-updater commondat tabdat tab-num)
									 (dboard:tabdat-running-layout-set! tabdat #f))
								       "run-times-tab-layout-updater"))))
                               ;;(dboard:tabdat-view-changed-set! tabdat #t)
                               )))))
        (dboard:tabdat-graph-matrix-set! tabdat graph-matrix)
        (iup:attribute-set! graph-matrix "WIDTH0" 0)
        (iup:attribute-set! graph-matrix "HEIGHT0" 0)
        graph-matrix))
      (iup:hbox
       (iup:vbox
        (iup:button "Show All" #:action (lambda (obj)
                                          (for-each (lambda (graph-cell)
                                                      (let* ((graph-dat   (hash-table-ref (dboard:tabdat-graph-cell-table tabdat) graph-cell)))
                                                        (dboard:graph-dat-flag-set! graph-dat #t)))
                                                    (hash-table-keys (dboard:tabdat-graph-cell-table tabdat))))))
       (iup:hbox
        (iup:button "Hide All" #:action (lambda (obj)
                                          (for-each (lambda (graph-cell)
                                                      (let* ((graph-dat   (hash-table-ref (dboard:tabdat-graph-cell-table tabdat) graph-cell)))
                                                        (dboard:graph-dat-flag-set! graph-dat #f)))
                                                    (hash-table-keys (dboard:tabdat-graph-cell-table tabdat)))))))
      ))))

;;======================================================================
;; R U N
;;======================================================================
;;
;; display and manage a single run at a time

1411
1412
1413
1414
1415
1416
1417
1418

1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433














1434





























1435
1436
1437

1438
1439
1440
1441











1442






1443


1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459

1460

1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526

1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
    (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2))
    (for-each (lambda (run-id)
		(let* ((run-record (hash-table-ref/default runs-hash run-id #f))
		       (key-vals   (map (lambda (key)(db:get-value-by-header run-record runs-header key))
					(dboard:tabdat-keys tabdat)))
		       (run-name   (db:get-value-by-header run-record runs-header "runname"))
		       (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
		       (run-path   (append key-vals (list run-name)))

		       (existing   (tree:find-node tb run-path)))
		  (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f))
		      (begin
			(hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path)
			;; (iup:attribute-set! (dboard:tabdat-runs-matrix tabdat)
			;;    		 (conc rownum ":" colnum) col-name)
			;; (hash-table-set! runid-to-col run-id (list colnum run-record))
			;; Here we update the tests treebox and tree keys
			(tree:add-node tb "Runs" run-path ;; (append key-vals (list run-name))
				       userdata: (conc "run-id: " run-id))
			(hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id)
			;; (set! colnum (+ colnum 1))
			))))
	      run-ids)))
  














(define (dashboard:one-run-updater commondat tabdat tb cell-lookup run-matrix)





























  (let* ((last-runs-update  (dboard:tabdat-last-runs-update tabdat))
	 (runs-dat     (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update))
	 (runs-header  (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records

	 (run-id       (dboard:tabdat-curr-run-id tabdat))
         (runs-hash    (let ((ht (make-hash-table)))
			 (for-each (lambda (run)
				     (hash-table-set! ht (db:get-value-by-header run runs-header "id") run))











				   (vector-ref runs-dat 1))






			 ht)))


    (dboard:update-tree tabdat runs-hash runs-header tb)
    (if run-id
        (let* (
               
               (last-update  (hash-table-ref/default (dboard:tabdat-run-update-times tabdat) run-id 0))
               (db-path      (or (hash-table-ref/default (dboard:tabdat-run-db-paths tabdat) run-id #f)
                                 (let* ((db-dir (tasks:get-task-db-path))
                                        (db-pth (conc db-dir "/" run-id ".db")))
                                   (hash-table-set! (dboard:tabdat-run-db-paths tabdat) run-id db-pth)
                                   db-pth)))
               (tests-dat    (if (or (not run-id)
                                     (configf:lookup *configdat* "setup" "do-not-use-db-file-timestamps")
                                     (not (hash-table-exists? (dboard:tabdat-last-test-dat tabdat) run-id))
                                     (>= (file-modification-time db-path) last-update))
                                 (dboard:get-tests-dat tabdat run-id last-update)
                                 (hash-table-ref (dboard:tabdat-last-test-dat tabdat) run-id)))

               (tests-mindat (dcommon:minimize-test-data tests-dat))

               (indices      (common:sparse-list-generate-index tests-mindat)) ;;  proc: set-cell))
               (row-indices  (cadr indices))
               (col-indices  (car indices))
               (max-row      (if (null? row-indices) 1 (common:max (map cadr row-indices))))
               (max-col      (if (null? col-indices) 1 (common:max (map cadr col-indices))))
               (max-visible  (max (- (dboard:tabdat-num-tests tabdat) 15) 3)) ;; (dboard:tabdat-num-tests tabdat) is proportional to the size of the window
               (numrows      1)
               (numcols      1)
               (changed      #f)
               )
    (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2))
          (hash-table-set! (dboard:tabdat-last-test-dat tabdat) run-id tests-dat)
          (hash-table-set! (dboard:tabdat-run-update-times tabdat) run-id (- (current-seconds) 10))
          (dboard:tabdat-filters-changed-set! tabdat #f)
          (let loop ((pass-num 0)
                     (changed  #f))
            ;; Update the runs tree
            (dboard:update-tree tabdat runs-hash runs-header tb)
            
            (if (eq? pass-num 1)
                (begin ;; big reset
                  (iup:attribute-set! run-matrix "CLEARVALUE" "ALL") ;; NOTE: Was CONTENTS
                  (iup:attribute-set! run-matrix "CLEARATTRIB" "CONTENTS")
                  (iup:attribute-set! run-matrix "RESIZEMATRIX" "YES")))

            (if (> max-col (string->number (iup:attribute run-matrix "NUMCOL")))
                (iup:attribute-set! run-matrix "NUMCOL" max-col ))

            (let ((effective-max-row (if (< max-row max-visible) max-visible max-row)))
              (if (> effective-max-row (string->number (iup:attribute run-matrix "NUMLIN")))
                (iup:attribute-set! run-matrix "NUMLIN" effective-max-row )))
            
            ;; Row labels
            (for-each (lambda (ind)
                        (let* ((name (car ind))
                               (num  (cadr ind))
                               (key  (conc num ":0")))
                          (if (not (equal? (iup:attribute run-matrix key) name))
                              (begin
                                (set! changed #t)
                                (iup:attribute-set! run-matrix key name)))))
                      row-indices)
            ;; (print "row-indices: " row-indices " col-indices: " col-indices)
            (if (and (eq? pass-num 0) changed)
                (loop 1 #t)) ;; force second pass
            
            ;; Cell contents
            (for-each (lambda (entry)
                        ;; (print "entry: " entry)
                        (let* ((row-name  (cadr entry))
                               (col-name  (car entry))
                               (valuedat  (caddr entry))
                               (test-id   (list-ref valuedat 0))
                               (test-name row-name) ;; (list-ref valuedat 1))
                               (item-path col-name) ;; (list-ref valuedat 2))
                               (state     (list-ref valuedat 1))
                               (status    (list-ref valuedat 2))
                               (value     (gutils:get-color-for-state-status state status))
                               (row-num   (cadr (assoc row-name row-indices)))
                               (col-num   (cadr (assoc col-name col-indices)))
                               (key       (conc row-num ":" col-num)))
                          (hash-table-set! cell-lookup key test-id)
                          (if (not (equal? (iup:attribute run-matrix key) (cadr value)))
                              (begin
                                (set! changed #t)
                                (iup:attribute-set! run-matrix key (cadr value))

                                (iup:attribute-set! run-matrix (conc "BGCOLOR" key) (car value))))))
                      tests-mindat)
            
            ;; Col labels - do after setting Cell contents so they are accounted for in the size calc.
            
            (for-each (lambda (ind)
                        (let* ((name (car ind))
                               (num  (cadr ind))
                               (key  (conc "0:" num)))
                          (if (not (equal? (iup:attribute run-matrix key) name))
                              (begin
                                (set! changed #t)
                                (iup:attribute-set! run-matrix key name)
                                (if (<= num max-col)
                                    (iup:attribute-set! run-matrix "FITTOTEXT" (conc "C" num)))))))
                      col-indices)
            
            (if (and (eq? pass-num 0) changed)
                (loop 1 #t)) ;; force second pass due to column labels changing
            
            ;; (debug:print 0 *default-log-port* "one-run-updater, changed: " changed " pass-num: " pass-num)
            ;; (print "one-run-updater, changed: " changed " pass-num: " pass-num)
            (if changed (iup:attribute-set! run-matrix "REDRAW" "ALL"))))))

  )

;;======================================================================
;; S U M M A R Y 
;;======================================================================
;;
;; General info about the run(s) and megatest area
(define (dashboard:summary commondat tabdat #!key (tab-num #f))







|
>
|
|
|
|
|
|
|
|
|
|
|
|
|

|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|


>




>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
|
>
>
|

|
<
<
<
<
<
|
<
|
<
<
<
|
<
>
|
>
|
|
|
|
|
|
|
|
|
|
|
<
<
|
|
|
|
|
|
|
|
|
|
|

|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<
<







1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621





1622

1623



1624

1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638


1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715


1716
1717
1718
1719
1720
1721
1722
    (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2))
    (for-each (lambda (run-id)
		(let* ((run-record (hash-table-ref/default runs-hash run-id #f))
		       (key-vals   (map (lambda (key)(db:get-value-by-header run-record runs-header key))
					(dboard:tabdat-keys tabdat)))
		       (run-name   (db:get-value-by-header run-record runs-header "runname"))
		       (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
		       (run-path   (append key-vals (list run-name))))
		  (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f))
                      ;; (let ((existing   (tree:find-node tb run-path)))
                      ;;   (if (not existing)
                      (begin
                        (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path)
                        ;; (iup:attribute-set! (dboard:tabdat-runs-matrix tabdat)
                        ;;    		 (conc rownum ":" colnum) col-name)
                        ;; (hash-table-set! runid-to-col run-id (list colnum run-record))
                        ;; Here we update the tests treebox and tree keys
                        (tree:add-node tb "Runs" run-path) ;; (append key-vals (list run-name))
                        ;;                                             userdata: (conc "run-id: " run-id))))
                        (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id)
                        ;; (set! colnum (+ colnum 1))
                        ))))
	      run-ids)))

(define (dashboard:tests-ht->tests-dat tests-ht)
  (reverse
   (sort
    (hash-table-values tests-ht)
    (lambda (a b) 
      (let ((a-test-name  (db:test-get-testname a))
            (a-item-path  (db:test-get-item-path a))
            (b-test-name  (db:test-get-testname b))
            (b-item-path  (db:test-get-item-path b)))
        (cond
         ((< 0 (string-compare3 a-test-name b-test-name)) #t)
         ((> 0 (string-compare3 a-test-name b-test-name)) #f)
         ((< 0 (string-compare3 a-item-path b-item-path)) #t)
         (else #f)))))))


(define (dashboard:run-id->tests-mindat run-id tabdat runs-hash)
  (let* ((run          (hash-table-ref/default runs-hash run-id #f))
         (key-vals     (rmt:get-key-vals run-id))
         (testnamepatt (or (dboard:tabdat-test-patts tabdat) "%/%"))
         (tests-ht     (dboard:get-tests-for-run-duplicate tabdat run-id run testnamepatt key-vals))
         (tests-dat    (dashboard:tests-ht->tests-dat tests-ht)) 
         (tests-mindat (dcommon:minimize-test-data tests-dat)))  ;; reduces data for display
    (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2))
    (hash-table-set! (dboard:tabdat-last-test-dat tabdat) run-id tests-dat)
    (hash-table-set! (dboard:tabdat-run-update-times tabdat) run-id (- (current-seconds) 10))
    (when (not run)
        (BB> "ERROR: NO RUN FOR RUN-ID run-id="run-id)
        (BB> "runs-hash-> " (hash-table->alist runs-hash))
        )
    tests-mindat))

(define (dashboard:runs-summary-xor-matrix-content tabdat runs-hash #!key (hide-clean #f))
  (let* ((src-run-id (dboard:tabdat-prev-run-id tabdat))
         (dest-run-id (dboard:tabdat-curr-run-id tabdat)))
    (if (and src-run-id dest-run-id)
        (dcommon:xor-tests-mindat 
         (dashboard:run-id->tests-mindat src-run-id tabdat runs-hash)
         (dashboard:run-id->tests-mindat dest-run-id tabdat runs-hash)
         hide-clean: hide-clean)
        #f)))


(define (dashboard:get-runs-hash tabdat)
  (let* ((last-runs-update  0);;(dboard:tabdat-last-runs-update tabdat))
	 (runs-dat     (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update))
	 (runs-header  (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records
         (runs         (vector-ref runs-dat 1))
	 (run-id       (dboard:tabdat-curr-run-id tabdat))
         (runs-hash    (let ((ht (make-hash-table)))
			 (for-each (lambda (run)
				     (hash-table-set! ht (db:get-value-by-header run runs-header "id") run))
				   runs) ht)))
    runs-hash))
         

(define (dashboard:runs-summary-updater commondat tabdat tb cell-lookup run-matrix)
  (if (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-rundat)
      (dashboard:do-update-rundat tabdat))
  (dboard:runs-summary-control-panel-updater tabdat)
  (let* ((last-runs-update  (dboard:tabdat-last-runs-update tabdat))
	 (runs-dat     (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update))
	 (runs-header  (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records
         (runs         (vector-ref runs-dat 1))
	 (run-id       (dboard:tabdat-curr-run-id tabdat))
         (runs-hash (dashboard:get-runs-hash tabdat))
         ;; (runs-hash    (let ((ht (make-hash-table)))
	 ;;        	 (for-each (lambda (run)
	 ;;        		     (hash-table-set! ht (db:get-value-by-header run runs-header "id") run))
	 ;;        		   runs)
	 ;;        	 ht))
         )
    (if (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-tree)
        (dboard:update-tree tabdat runs-hash runs-header tb))
    (if run-id
        (let* ((matrix-content





                (case (dboard:tabdat-runs-summary-mode tabdat) 

                  ((one-run) (dashboard:run-id->tests-mindat run-id tabdat runs-hash))



                  ((xor-two-runs) (dashboard:runs-summary-xor-matrix-content tabdat runs-hash))

                  ((xor-two-runs-hide-clean) (dashboard:runs-summary-xor-matrix-content tabdat runs-hash hide-clean: #t))
                  (else (dashboard:run-id->tests-mindat run-id tabdat runs-hash)))))
          (when matrix-content
            (let* ((indices      (common:sparse-list-generate-index matrix-content)) ;;  proc: set-cell))
                   (row-indices  (cadr indices))
                   (col-indices  (car indices))
                   (max-row      (if (null? row-indices) 1 (common:max (map cadr row-indices))))
                   (max-col      (if (null? col-indices) 1 (common:max (map cadr col-indices))))
                   (max-visible  (max (- (dboard:tabdat-num-tests tabdat) 15) 3)) ;; (dboard:tabdat-num-tests tabdat) is proportional to the size of the window
                   (numrows      1)
                   (numcols      1)
                   (changed      #f)
                   )
              


              (dboard:tabdat-filters-changed-set! tabdat #f)
              (let loop ((pass-num 0)
                         (changed  #f))
                ;; Update the runs tree
                ;; (dboard:update-tree tabdat runs-hash runs-header tb)
                
                (if (eq? pass-num 1)
                    (begin ;; big reset
                      (iup:attribute-set! run-matrix "CLEARVALUE" "ALL") ;; NOTE: Was CONTENTS
                      (iup:attribute-set! run-matrix "CLEARATTRIB" "CONTENTS")
                      (iup:attribute-set! run-matrix "RESIZEMATRIX" "YES")))

                (if (> max-col (string->number (iup:attribute run-matrix "NUMCOL")))
                    (iup:attribute-set! run-matrix "NUMCOL" max-col ))

                (let ((effective-max-row (if (< max-row max-visible) max-visible max-row)))
                  (if (> effective-max-row (string->number (iup:attribute run-matrix "NUMLIN")))
                      (iup:attribute-set! run-matrix "NUMLIN" effective-max-row )))
                
                ;; Row labels
                (for-each (lambda (ind)
                            (let* ((name (car ind))
                                   (num  (cadr ind))
                                   (key  (conc num ":0")))
                              (if (not (equal? (iup:attribute run-matrix key) name))
                                  (begin
                                    (set! changed #t)
                                    (iup:attribute-set! run-matrix key name)))))
                          row-indices)
                ;; (print "row-indices: " row-indices " col-indices: " col-indices)
                (if (and (eq? pass-num 0) changed)
                    (loop 1 #t)) ;; force second pass
                
                ;; Cell contents
                (for-each (lambda (entry)
                            ;; (print "entry: " entry)
                            (let* ((row-name  (cadr entry))
                                   (col-name  (car entry))
                                   (valuedat  (caddr entry))
                                   (test-id   (list-ref valuedat 0))
                                   (test-name row-name) ;; (list-ref valuedat 1))
                                   (item-path col-name) ;; (list-ref valuedat 2))
                                   (state     (list-ref valuedat 1))
                                   (status    (list-ref valuedat 2))
                                   (value     (gutils:get-color-for-state-status state status))
                                   (row-num   (cadr (assoc row-name row-indices)))
                                   (col-num   (cadr (assoc col-name col-indices)))
                                   (key       (conc row-num ":" col-num)))
                              (hash-table-set! cell-lookup key test-id)
                              (if (not (equal? (iup:attribute run-matrix key) (cadr value)))
                                  (begin
                                    (set! changed #t)
                                    (iup:attribute-set! run-matrix key (cadr value))
                                    ;; (print "RA=> value" (car value))
                                    (iup:attribute-set! run-matrix (conc "BGCOLOR" key) (car value))))))
                          matrix-content)
                
                ;; Col labels - do after setting Cell contents so they are accounted for in the size calc.
                
                (for-each (lambda (ind)
                            (let* ((name (car ind))
                                   (num  (cadr ind))
                                   (key  (conc "0:" num)))
                              (if (not (equal? (iup:attribute run-matrix key) name))
                                  (begin
                                    (set! changed #t)
                                    (iup:attribute-set! run-matrix key name)
                                    (if (<= num max-col)
                                        (iup:attribute-set! run-matrix "FITTOTEXT" (conc "C" num)))))))
                          col-indices)
                
                (if (and (eq? pass-num 0) changed)
                    (loop 1 #t)) ;; force second pass due to column labels changing
                
                ;; (debug:print 0 *default-log-port* "runs-summary-updater, changed: " changed " pass-num: " pass-num)
                ;; (print "runs-summary-updater, changed: " changed " pass-num: " pass-num)
                (if changed (iup:attribute-set! run-matrix "REDRAW" "ALL")))))))))



;;======================================================================
;; S U M M A R Y 
;;======================================================================
;;
;; General info about the run(s) and megatest area
(define (dashboard:summary commondat tabdat #!key (tab-num #f))
1641
1642
1643
1644
1645
1646
1647
1648



















































































1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672




1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685

1686









1687
1688
1689


















1690



1691
1692



















1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703


1704
1705
1706
1707

1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721

1722
1723
1724
1725
1726
1727
1728
1729
					 ((eval (string->symbol updater)) commondat tabs tab-num view-name views-cfgdat *configdat*)))
				      tab-num: tab-num))
    (if success
	(begin
	  ;; (iup:attribute-set! tabs (conc "TABTITLE" tab-num) view-name)
	  (dboard:common-set-tabdat! commondat tab-num (dboard:tabdat-make-data))))
    result-child))




















































































;;======================================================================
;; R U N
;;======================================================================
;;
;; display and manage a single run at a time

;; This is the Run Summary tab
;; 
(define (dashboard:one-run commondat tabdat #!key (tab-num #f))
  (let* ((update-mutex (dboard:commondat-update-mutex commondat))
	 (tb      (iup:treebox
		   #:value 0
		   #:name "Runs"
		   #:expand "YES"
		   #:addexpanded "NO"
		   #:selection-cb
		   (lambda (obj id state)
		     (debug:catch-and-dump
		      (lambda ()
			;; (print "obj: " obj ", id: " id ", state: " state)
			(let* ((run-path (tree:node->path obj id))
			       (run-id   (tree-path->run-id tabdat (cdr run-path))))
			  (if (number? run-id)
			      (begin




				(dboard:tabdat-curr-run-id-set! tabdat run-id)
				(dboard:tabdat-layout-update-ok-set! tabdat #f)
				;; (dashboard:update-run-summary-tab)
				)
			      ;; (debug:print-error 0 *default-log-port* "tree-path->run-id returned non-number " run-id)
			      )))
		      "selection-cb in one-run")
		     ;; (print "path: " (tree:node->path obj id) " run-id: " run-id)
		     )))
	 (cell-lookup (make-hash-table))
	 (run-matrix (iup:matrix
		      #:expand "YES"
		      #:click-cb

		      (lambda (obj lin col status)









			(let* ((toolpath (car (argv)))
			       (key      (conc lin ":" col))
			       (test-id  (hash-table-ref/default cell-lookup key -1))


















			       (cmd      (conc toolpath " -test " (dboard:tabdat-curr-run-id tabdat) "," test-id "&")))



			  (system cmd)))))
	 (one-run-updater  



















          (lambda ()
	    (mutex-lock! update-mutex)
            (if  (or (dashboard:database-changed? commondat tabdat)
                     (dboard:tabdat-view-changed tabdat))
                 (debug:catch-and-dump
                  (lambda () ;; check that run-matrix is initialized before calling the updater
		    (if run-matrix 
			(dashboard:one-run-updater commondat tabdat tb cell-lookup run-matrix)))
                  "dashboard:one-run-updater")
                 )
	    (mutex-unlock! update-mutex))))


    (dboard:commondat-add-updater commondat one-run-updater tab-num: tab-num)
    (dboard:tabdat-runs-tree-set! tabdat tb)
    (iup:vbox
     (iup:split

      tb
      run-matrix)
     (dboard:make-controls commondat tabdat))))

;;======================================================================
;; R U N S 
;;======================================================================

(define (dboard:make-controls commondat tabdat)
  (let ((btn-fontsz  (dboard:tabdat-runs-btn-fontsz tabdat)))
    (iup:hbox
     (iup:vbox
      (iup:frame 
       #:title "filter test and items"

       (iup:hbox
	(iup:vbox
	 (iup:textbox #:size "120x15" #:fontsize "10" #:value "%"
		      #:expand "NO"
		      #:action (lambda (obj unk val)
				 (debug:catch-and-dump
				  (lambda ()
				    (mark-for-update tabdat)








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








|















>
>
>
>






|






>

>
>
>
>
>
>
>
>
>
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|




|
|

|
>
>
|



>


|





|





>
|







1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
					 ((eval (string->symbol updater)) commondat tabs tab-num view-name views-cfgdat *configdat*)))
				      tab-num: tab-num))
    (if success
	(begin
	  ;; (iup:attribute-set! tabs (conc "TABTITLE" tab-num) view-name)
	  (dboard:common-set-tabdat! commondat tab-num (dboard:tabdat-make-data))))
    result-child))



(define (dboard:runs-summary-buttons-updater tabdat)
  (let loop ((buttons-left (dboard:tabdat-runs-summary-mode-buttons tabdat))
             (modes-left (dboard:tabdat-runs-summary-modes tabdat)))
    (if (or (null? buttons-left) (null? modes-left))
        #t
        (let* ((this-button (car buttons-left))
               (mode-item (car modes-left))
               (this-mode (car mode-item))
               (sel-color    "180 100 100")
               (nonsel-color "170 170 170")
               (current-mode (dboard:tabdat-runs-summary-mode tabdat)))
          (if (eq? this-mode current-mode)
              (iup:attribute-set! this-button "BGCOLOR" sel-color)
              (iup:attribute-set! this-button "BGCOLOR" nonsel-color))
          (loop (cdr buttons-left) (cdr modes-left))))))

(define (dboard:runs-summary-xor-labels-updater tabdat)
  (let ((source-runname-label (dboard:tabdat-runs-summary-source-runname-label tabdat))
        (dest-runname-label (dboard:tabdat-runs-summary-dest-runname-label tabdat))
        (mode (dboard:tabdat-runs-summary-mode tabdat)))
    (when (and source-runname-label dest-runname-label)
      (case mode
        ((xor-two-runs xor-two-runs-hide-clean)
         (let* ((curr-run-id          (dboard:tabdat-curr-run-id tabdat))
                (prev-run-id          (dboard:tabdat-prev-run-id tabdat))
                (curr-runname (if curr-run-id
                                  (rmt:get-run-name-from-id curr-run-id)
                                  "None"))
                (prev-runname (if prev-run-id
                                  (rmt:get-run-name-from-id prev-run-id)
                                  "None")))
           (iup:attribute-set! source-runname-label "TITLE" (conc " SRC: "prev-runname"  "))
           (iup:attribute-set! dest-runname-label "TITLE" (conc "DEST: "curr-runname"  "))))
        (else
         (iup:attribute-set! source-runname-label "TITLE" "")
         (iup:attribute-set! dest-runname-label "TITLE" ""))))))

(define (dboard:runs-summary-control-panel-updater tabdat)
  (dboard:runs-summary-xor-labels-updater tabdat)
  (dboard:runs-summary-buttons-updater tabdat))


;; setup buttons and callbacks to switch between modes in runs summary tab
;;
(define (dashboard:runs-summary-control-panel tabdat)
  (let* ((summary-buttons ;; build buttons
          (map
           (lambda (mode-item)
             (let* ((this-mode (car mode-item))
                    (this-mode-label (cdr mode-item)))
               (iup:button this-mode-label
                           #:action
                           (lambda (obj)
                             (debug:catch-and-dump
                              (lambda ()
                                (dboard:tabdat-runs-summary-mode-set! tabdat this-mode)
                                (dboard:runs-summary-control-panel-updater tabdat))
                              "runs summary control panel updater")))))
           (dboard:tabdat-runs-summary-modes tabdat)))
         (summary-buttons-hbox (apply iup:hbox summary-buttons))
         (xor-runname-labels-hbox
          (iup:hbox
           (let ((temp-label
                  (iup:label "" #:size "125x15" #:fontsize "10" )))
             (dboard:tabdat-runs-summary-source-runname-label-set! tabdat temp-label)
             temp-label
             )
           (let ((temp-label
                  (iup:label "" #:size "125x15" #:fontsize "10")))
             (dboard:tabdat-runs-summary-dest-runname-label-set! tabdat temp-label)
             temp-label))))
    (dboard:tabdat-runs-summary-mode-buttons-set! tabdat summary-buttons)

    ;; maybe wrap in a frame
    (let ((res (iup:vbox summary-buttons-hbox xor-runname-labels-hbox )))
      (dboard:runs-summary-control-panel-updater tabdat)
      res
      )))



;;======================================================================
;; R U N
;;======================================================================
;;
;; display and manage a single run at a time

;; This is the Run Summary tab
;; 
(define (dashboard:runs-summary commondat tabdat #!key (tab-num #f))
  (let* ((update-mutex (dboard:commondat-update-mutex commondat))
	 (tb      (iup:treebox
		   #:value 0
		   #:name "Runs"
		   #:expand "YES"
		   #:addexpanded "NO"
		   #:selection-cb
		   (lambda (obj id state)
		     (debug:catch-and-dump
		      (lambda ()
			;; (print "obj: " obj ", id: " id ", state: " state)
			(let* ((run-path (tree:node->path obj id))
			       (run-id   (tree-path->run-id tabdat (cdr run-path))))
			  (if (number? run-id)
			      (begin
                                (dboard:tabdat-prev-run-id-set!
                                 tabdat
                                 (dboard:tabdat-curr-run-id tabdat))

				(dboard:tabdat-curr-run-id-set! tabdat run-id)
				(dboard:tabdat-layout-update-ok-set! tabdat #f)
				;; (dashboard:update-run-summary-tab)
				)
			      ;; (debug:print-error 0 *default-log-port* "tree-path->run-id returned non-number " run-id)
			      )))
		      "selection-cb in runs-summary")
		     ;; (print "path: " (tree:node->path obj id) " run-id: " run-id)
		     )))
	 (cell-lookup (make-hash-table))
	 (run-matrix (iup:matrix
		      #:expand "YES"
		      #:click-cb
                      
		      (lambda (obj lin col status)
                        (debug:catch-and-dump
                         (lambda ()

                           ;; Bummer - we dont have the global get/set api mapped in chicken
                           ;; (let* ((modkeys (iup:global "MODKEYSTATE")))
                           ;;   (BB> "modkeys="modkeys))

                           (BB> "click-cb: obj="obj" lin="lin" col="col" status="status)
                           ;; status is corrupted on Brandon's home machine.  will have to wait until after shutdown to see if it is still broken in PDX SLES
                           (let* ((toolpath (car (argv)))
                                  (key      (conc lin ":" col))
                                  (test-id   (hash-table-ref/default cell-lookup key -1))
                                  (run-id   (dboard:tabdat-curr-run-id tabdat))
                                  (run-info (rmt:get-run-info run-id))
                                  (target   (rmt:get-target run-id))
                                  (runname  (db:get-value-by-header (db:get-rows run-info)
                                                                    (db:get-header run-info) "runname"))
                                  (test-name (db:test-get-testname (rmt:get-test-info-by-id run-id test-id)))
                                  (testpatt  (let ((tlast (rmt:tasks-get-last target runname)))
                                                (if tlast
                                                    (let ((tpatt (tasks:task-get-testpatt tlast)))
                                                      (if (member tpatt '("0" 0)) ;; known bad historical value - remove in 2017
                                                          "%"
                                                          tpatt))
                                                    "%")))
                                  (item-path (db:test-get-item-path (rmt:get-test-info-by-id run-id test-id)))
                                  (item-test-path (conc test-name "/" (if (equal? item-path "")
									"%" 
									item-path)))
                                  (status-chars (char-set->list (string->char-set status)))
                                  (testpanel-cmd      (conc toolpath " -test " (dboard:tabdat-curr-run-id tabdat) "," test-id " &")))
                             (BB> "status-chars=["status-chars"] status=["status"]")
                             (cond
                              ((member #\1 status-chars) ;; 1 is left mouse button
                               (system testpanel-cmd))
                              
                              ((member #\2 status-chars) ;; 2 is middle mouse button
                               
                               (BB> "mmb- test-name="test-name" testpatt="testpatt)
                               (iup:show (dashboard:popup-menu run-id test-id target runname test-name testpatt item-test-path) ;; popup-menu
                                         #:x 'mouse
                                         #:y 'mouse
                                         #:modal? "NO")
                               )
                              (else
                               (BB> "unhandled status in run-summary-click-cb.  Doing right click action. (status is corrupted on Brandon's ubuntu host - bad/buggy  iup install??" )
                               (iup:show (dashboard:popup-menu run-id test-id target runname test-name testpatt item-test-path) ;; popup-menu
                                         #:x 'mouse
                                         #:y 'mouse
                                         #:modal? "NO")
                               )
                              )
                            
                             )) "runs-summary-click-callback"))))
	 (runs-summary-updater  
          (lambda ()
	    (mutex-lock! update-mutex)
            (if  (or (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-updater)
                     (dboard:tabdat-view-changed tabdat))
                 (debug:catch-and-dump
                  (lambda () ;; check that run-matrix is initialized before calling the updater
		    (if run-matrix 
			(dashboard:runs-summary-updater commondat tabdat tb cell-lookup run-matrix)))
                  "dashboard:runs-summary-updater")
                 )
	    (mutex-unlock! update-mutex)))
         (runs-summary-control-panel (dashboard:runs-summary-control-panel tabdat))
         )
    (dboard:commondat-add-updater commondat runs-summary-updater tab-num: tab-num)
    (dboard:tabdat-runs-tree-set! tabdat tb)
    (iup:vbox
     (iup:split
      #:value 200
      tb
      run-matrix)
     (dboard:make-controls commondat tabdat extra-widget: runs-summary-control-panel))))

;;======================================================================
;; R U N S 
;;======================================================================

(define (dboard:make-controls commondat tabdat #!key (extra-widget #f) )
  (let ((btn-fontsz  (dboard:tabdat-runs-btn-fontsz tabdat)))
    (iup:hbox
     (iup:vbox
      (iup:frame 
       #:title "filter test and items"
       (iup:vbox
        (iup:hbox
	(iup:vbox
	 (iup:textbox #:size "120x15" #:fontsize "10" #:value "%"
		      #:expand "NO"
		      #:action (lambda (obj unk val)
				 (debug:catch-and-dump
				  (lambda ()
				    (mark-for-update tabdat)
1770
1771
1772
1773
1774
1775
1776

1777
1778
1779
1780
1781
1782
1783
		(sort-lb    (iup:listbox #:expand "NO" ;; "HORIZONTAL"
					 #:size   "80x15"
					 #:dropdown "YES"
					 #:action (lambda (obj val index lbstate)
						    (set! *tests-sort-reverse* index)
						    (mark-for-update tabdat))))
		(default-cmd (car (list-ref *tests-sort-type-index* *tests-sort-reverse*))))

	   (iuplistbox-fill-list sort-lb cmds-list selected-item: default-cmd)
	   
	   (set! hide-empty (iup:button "HideEmpty"
					;; #:expand HORIZONTAL"
					#:expand "NO" #:size "80x15"
					#:action (lambda (obj)
						   (dboard:tabdat-hide-empty-runs-set! tabdat (not (dboard:tabdat-hide-empty-runs tabdat)))







>







2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
		(sort-lb    (iup:listbox #:expand "NO" ;; "HORIZONTAL"
					 #:size   "80x15"
					 #:dropdown "YES"
					 #:action (lambda (obj val index lbstate)
						    (set! *tests-sort-reverse* index)
						    (mark-for-update tabdat))))
		(default-cmd (car (list-ref *tests-sort-type-index* *tests-sort-reverse*))))
                
	   (iuplistbox-fill-list sort-lb cmds-list selected-item: default-cmd)
	   
	   (set! hide-empty (iup:button "HideEmpty"
					;; #:expand HORIZONTAL"
					#:expand "NO" #:size "80x15"
					#:action (lambda (obj)
						   (dboard:tabdat-hide-empty-runs-set! tabdat (not (dboard:tabdat-hide-empty-runs tabdat)))
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806










1807

1808
1809
1810
1811
1812
1813
1814
					     (dboard:tabdat-hide-not-hide-set! tabdat #f) ;; (not (dboard:tabdat-hide-not-hide tabdat)))
					     (iup:attribute-set! show "BGCOLOR" sel-color)
					     (iup:attribute-set! hide "BGCOLOR" nonsel-color)
					     (mark-for-update tabdat))))
	   (iup:attribute-set! hide "BGCOLOR" sel-color)
	   (iup:attribute-set! show "BGCOLOR" nonsel-color)
	   ;; (dboard:tabdat-hide-not-hide-button-set! tabdat hideit) ;; never used, can eliminate ...
	   (iup:vbox
	    (iup:hbox hide show)
	    hide-empty sort-lb)))










	)))

     (iup:frame 
      #:title "state/status filter"
      (iup:vbox
       (apply 
	iup:hbox
	(map (lambda (status)
	       (iup:toggle (conc status "  ")







|
|
|
>
>
>
>
>
>
>
>
>
>
|
>







2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
					     (dboard:tabdat-hide-not-hide-set! tabdat #f) ;; (not (dboard:tabdat-hide-not-hide tabdat)))
					     (iup:attribute-set! show "BGCOLOR" sel-color)
					     (iup:attribute-set! hide "BGCOLOR" nonsel-color)
					     (mark-for-update tabdat))))
	   (iup:attribute-set! hide "BGCOLOR" sel-color)
	   (iup:attribute-set! show "BGCOLOR" nonsel-color)
	   ;; (dboard:tabdat-hide-not-hide-button-set! tabdat hideit) ;; never used, can eliminate ...
           (iup:vbox
            (iup:hbox hide show)
            sort-lb))) 
	)

        ;; insert extra widget here
        (if extra-widget
            extra-widget
            (iup:hbox)) ;; empty widget

        

        
        )))

     (iup:frame 
      #:title "state/status filter"
      (iup:vbox
       (apply 
	iup:hbox
	(map (lambda (status)
	       (iup:toggle (conc status "  ")
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854










































1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
					   (mark-for-update tabdat)
					   (debug:print 6 *default-log-port* "(dboard:tabdat-start-run-offset tabdat) " (dboard:tabdat-start-run-offset tabdat) " maxruns: " maxruns ", val: " val " oldmax: " oldmax)
					   (iup:attribute-set! obj "MAX" (* maxruns 10))))
		     #:expand "HORIZONTAL"
		     #:max (* 10 (max (hash-table-size (dboard:tabdat-allruns-by-id tabdat)) 10))
		     #:min 0
		     #:step 0.01)))
					;(iup:button "inc rows" #:action (lambda (obj)(dboard:tabdat-num-tests-set! tabdat (+ (dboard:tabdat-num-tests tabdat) 1))))
					;(iup:button "dec rows" #:action (lambda (obj)(dboard:tabdat-num-tests-set! tabdat (if (> (dboard:tabdat-num-tests tabdat) 0)(- (dboard:tabdat-num-tests tabdat) 1) 0))))
     )))

(define (dashboard:popup-menu buttndat run-id test-id target runname test-name testpatt item-test-path)
  (iup:menu 










































   (iup:menu-item
    "Run"
    (iup:menu              
     (iup:menu-item
      (conc "Rerun " testpatt)
      #:action
      (lambda (obj)
        ;; (print "buttndat: " buttndat " run-id: " run-id " test-id: " test-id " target: " target " runname: " runname " test-name: " test-name " testpatt: " testpatt "item-path : " item-path)
	(common:run-a-command
	 (conc "megatest -run -target " target
	       " -runname " runname
	       " -testpatt " testpatt
	       " -preclean -clean-cache")
	 )))
     (iup:menu-item







|



|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







|







2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
					   (mark-for-update tabdat)
					   (debug:print 6 *default-log-port* "(dboard:tabdat-start-run-offset tabdat) " (dboard:tabdat-start-run-offset tabdat) " maxruns: " maxruns ", val: " val " oldmax: " oldmax)
					   (iup:attribute-set! obj "MAX" (* maxruns 10))))
		     #:expand "HORIZONTAL"
		     #:max (* 10 (max (hash-table-size (dboard:tabdat-allruns-by-id tabdat)) 10))
		     #:min 0
		     #:step 0.01)))
     ;;(iup:button "inc rows" #:action (lambda (obj)(dboard:tabdat-num-tests-set! tabdat (+ (dboard:tabdat-num-tests tabdat) 1))))
					;(iup:button "dec rows" #:action (lambda (obj)(dboard:tabdat-num-tests-set! tabdat (if (> (dboard:tabdat-num-tests tabdat) 0)(- (dboard:tabdat-num-tests tabdat) 1) 0))))
     )))

(define (dashboard:popup-menu  run-id test-id target runname test-name testpatt item-test-path)
  (iup:menu 
   (iup:menu-item
    "Test Control Panel"
    #:action
    (lambda (obj)
      (let* ((toolpath (car (argv)))
             (testpanel-cmd
              (conc toolpath " -test " run-id "," test-id " &")))
        (system testpanel-cmd)
        )))
   
   (iup:menu-item
    (conc "View Log (not yet implemented) " item-test-path)
    )
   
   (iup:menu-item
    (conc "Rerun " item-test-path)
    #:action
    (lambda (obj)
      (common:run-a-command
       (conc "megatest -set-state-status NOT_STARTED,n/a -run -target " target
             " -runname " runname
             " -testpatt " item-test-path
             " -preclean -clean-cache"))))
   
   (iup:menu-item
    "Start xterm"
    #:action
    (lambda (obj)
      (dcommon:examine-xterm run-id test-id)))

   (iup:menu-item
    (conc "Kill " item-test-path)
    #:action
    (lambda (obj)
      ;; (rmt:test-set-state-status-by-id run-id test-id "KILLREQ" #f #f)
      (common:run-a-command
       (conc "megatest -set-state-status KILLREQ,n/a -target " target
             " -runname " runname
             " -testpatt " item-test-path 
             " -state RUNNING,REMOTEHOSTSTART,LAUNCHED"))))

   
   (iup:menu-item
    "Run"
    (iup:menu              
     (iup:menu-item
      (conc "Rerun " testpatt)
      #:action
      (lambda (obj)
        ;; (print  " run-id: " run-id " test-id: " test-id " target: " target " runname: " runname " test-name: " test-name " testpatt: " testpatt "item-path : " item-path)
	(common:run-a-command
	 (conc "megatest -run -target " target
	       " -runname " runname
	       " -testpatt " testpatt
	       " -preclean -clean-cache")
	 )))
     (iup:menu-item
1878
1879
1880
1881
1882
1883
1884
1885









1886
1887
1888
1889
1890
1891
1892
     (iup:menu-item
      "Clean Complete Run"
      #:action
      (lambda (obj)
        (common:run-a-command
         (conc "megatest -remove-runs -target " target
               " -runname " runname
               " -testpatt % "))))))









   (iup:menu-item
    "Test"
    (iup:menu 
     (iup:menu-item
      (conc "Rerun " item-test-path)
      #:action
      (lambda (obj)







|
>
>
>
>
>
>
>
>
>







2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
     (iup:menu-item
      "Clean Complete Run"
      #:action
      (lambda (obj)
        (common:run-a-command
         (conc "megatest -remove-runs -target " target
               " -runname " runname
               " -testpatt % "))))
     (iup:menu-item 
      "Kill Complete Run"
      #:action
      (lambda (obj)
        (common:run-a-command
         (conc "megatest -set-state-status KILLREQ,n/a -target " target
               " -runname " runname
               " -testpatt % "
               "  -state RUNNING,REMOTEHOSTSTART,LAUNCHED"))))))
   (iup:menu-item
    "Test"
    (iup:menu 
     (iup:menu-item
      (conc "Rerun " item-test-path)
      #:action
      (lambda (obj)
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
			  " " tconfig " &")))
	  (system cmd))))
     ))))

(define (make-dashboard-buttons commondat) ;;  runs-sum-dat new-view-dat)
  (let* ((stats-dat       (dboard:tabdat-make-data))
	 (runs-dat        (dboard:tabdat-make-data))
	 (onerun-dat      (dboard:tabdat-make-data))
	 (runcontrols-dat (dboard:tabdat-make-data))
	 (runtimes-dat    (dboard:tabdat-make-data))
	 (nruns           (dboard:tabdat-numruns runs-dat))
	 (ntests          (dboard:tabdat-num-tests runs-dat))
	 (keynames        (dboard:tabdat-dbkeys runs-dat))
	 (nkeys           (length keynames))
	 (runsvec         (make-vector nruns))







|







2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
			  " " tconfig " &")))
	  (system cmd))))
     ))))

(define (make-dashboard-buttons commondat) ;;  runs-sum-dat new-view-dat)
  (let* ((stats-dat       (dboard:tabdat-make-data))
	 (runs-dat        (dboard:tabdat-make-data))
	 (onerun-dat      (dboard:tabdat-make-data)) ;; name for run-summary structure 
	 (runcontrols-dat (dboard:tabdat-make-data))
	 (runtimes-dat    (dboard:tabdat-make-data))
	 (nruns           (dboard:tabdat-numruns runs-dat))
	 (ntests          (dboard:tabdat-num-tests runs-dat))
	 (keynames        (dboard:tabdat-dbkeys runs-dat))
	 (nkeys           (length keynames))
	 (runsvec         (make-vector nruns))
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
								     "%"
								     tpatt))
							       "%")))
                                              (item-path (db:test-get-item-path (rmt:get-test-info-by-id run-id test-id)))
                                              (item-test-path (conc test-name "/" (if (equal? item-path "")
									"%" 
									item-path))))
					 (iup:show (dashboard:popup-menu buttndat run-id test-id target runname test-name testpatt item-test-path) ;; popup-menu
						   #:x 'mouse
						   #:y 'mouse
						   #:modal? "NO")
					 ;; (print "got here")
					 ))
				   (if (eq? pressed 0)
				       (let* ((toolpath (car (argv)))







|







2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
								     "%"
								     tpatt))
							       "%")))
                                              (item-path (db:test-get-item-path (rmt:get-test-info-by-id run-id test-id)))
                                              (item-test-path (conc test-name "/" (if (equal? item-path "")
									"%" 
									item-path))))
					 (iup:show (dashboard:popup-menu run-id test-id target runname test-name testpatt item-test-path) ;; popup-menu
						   #:x 'mouse
						   #:y 'mouse
						   #:modal? "NO")
					 ;; (print "got here")
					 ))
				   (if (eq? pressed 0)
				       (let* ((toolpath (car (argv)))
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
						 (let* ((tab-num (dboard:commondat-curr-tab-num commondat))
							(tabdat  (dboard:common-get-tabdat commondat tab-num: tab-num)))
						   (dboard:commondat-please-update-set! commondat #t)
						   (dboard:tabdat-layout-update-ok-set! tabdat #t)))
					       "tabchangepos"))
			  (dashboard:summary commondat stats-dat tab-num: 0)
			  runs-view
			  (dashboard:one-run commondat onerun-dat tab-num: 2)
			  ;; (dashboard:new-view db data new-view-dat tab-num: 3)
			  (dashboard:run-controls commondat runcontrols-dat tab-num: 3)
			  (dashboard:run-times commondat runtimes-dat tab-num: 4)
			  additional-views)))
	;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c)))
	(iup:attribute-set! tabs "TABTITLE0" "Summary")
	(iup:attribute-set! tabs "TABTITLE1" "Runs")







|







2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
						 (let* ((tab-num (dboard:commondat-curr-tab-num commondat))
							(tabdat  (dboard:common-get-tabdat commondat tab-num: tab-num)))
						   (dboard:commondat-please-update-set! commondat #t)
						   (dboard:tabdat-layout-update-ok-set! tabdat #t)))
					       "tabchangepos"))
			  (dashboard:summary commondat stats-dat tab-num: 0)
			  runs-view
			  (dashboard:runs-summary commondat onerun-dat tab-num: 2)
			  ;; (dashboard:new-view db data new-view-dat tab-num: 3)
			  (dashboard:run-controls commondat runcontrols-dat tab-num: 3)
			  (dashboard:run-times commondat runtimes-dat tab-num: 4)
			  additional-views)))
	;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c)))
	(iup:attribute-set! tabs "TABTITLE0" "Summary")
	(iup:attribute-set! tabs "TABTITLE1" "Runs")
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243






2244
2245
2246
2247





2248
2249
2250
2251
2252
2253
2254
2255


2256
2257
2258
2259
2260
2261
2262
(define *tim* (iup:timer))
(define *ord* #f)
(iup:attribute-set! *tim* "TIME" 300)
(iup:attribute-set! *tim* "RUN" "YES")

(define *last-recalc-ended-time* 0)

(define (dashboard:been-changed tabdat)
  (> (file-modification-time (dboard:tabdat-dbfpath tabdat)) (dboard:tabdat-last-db-update tabdat)))

(define (dashboard:set-db-update-time tabdat)
  (dboard:tabdat-last-db-update-set! tabdat (file-modification-time (dboard:tabdat-dbfpath tabdat))))

(define (dashboard:recalc modtime please-update-buttons last-db-update-time)
  (or please-update-buttons
      (and (> (current-milliseconds)(+ *last-recalc-ended-time* 150))
	   (> modtime last-db-update-time)
	   (> (current-seconds)(+ last-db-update-time 1)))))

;; (define *monitor-db-path* #f)
(define *last-monitor-update-time* 0)

;; Force creation of the db in case it isn't already there.
(tasks:open-db)

(define (dashboard:get-youngest-run-db-mod-time tabdat)
  (handle-exceptions
   exn
   (begin
     (debug:print 0 *default-log-port* "WARNING: error in accessing databases in get-youngest-run-db-mod-time: " ((condition-property-accessor 'exn 'message) exn))
     (current-seconds)) ;; something went wrong - just print an error and return current-seconds
   (apply max (map (lambda (filen)
		     (file-modification-time filen))
		   (glob (conc (dboard:tabdat-dbdir tabdat) "/*.db"))))))

(define (dashboard:monitor-changed? commondat tabdat)
  (let* ((run-update-time (current-seconds))
	 (monitor-db-path (dboard:tabdat-monitor-db-path tabdat))
	 (monitor-modtime (if (and monitor-db-path (file-exists? monitor-db-path))
			      (file-modification-time monitor-db-path)
			      -1)))
    (if (and (eq? (dboard:commondat-curr-tab-num commondat) 0)
	     (or (> monitor-modtime *last-monitor-update-time*)
		 (> (- run-update-time *last-monitor-update-time*) 5))) ;; update every 1/2 minute just in case
	(begin
	  (set! *last-monitor-update-time* run-update-time) ;; monitor-modtime)
	  #t)
	#f)))







(define (dashboard:database-changed? commondat tabdat)
  (let* ((run-update-time (current-seconds))
	 (modtime         (dashboard:get-youngest-run-db-mod-time tabdat)) ;; NOTE: ensure this is tabdat!! 
	 (recalc          (dashboard:recalc modtime (dboard:commondat-please-update commondat) (dboard:tabdat-last-db-update tabdat))))





     (dboard:commondat-please-update-set! commondat #f)
     recalc))

;; point inside line
;;
(define-inline (dashboard:px-between px lx1 lx2)
  (and (< lx1 px)(> lx2 px)))



;; can a bar be placed in row "rownum" covering x1 to x2 without overlapping with existing 
;; bars? Use num-rows to check that a block will fit from rownum to (+ rownum num-rows)
;;
(define (dashboard:row-collision rowhash rownum x1 x2 #!key (num-rows #f))
  (let ((lastrow   (if num-rows (+ rownum num-rows) rownum)))
    (let loop ((i      0)
	       (rowdat (hash-table-ref/default rowhash rownum '())))







<
<
<
<
<
<


















|

















>
>
>
>
>
>
|


|
>
>
>
>
>
|
|






>
>







2563
2564
2565
2566
2567
2568
2569






2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
(define *tim* (iup:timer))
(define *ord* #f)
(iup:attribute-set! *tim* "TIME" 300)
(iup:attribute-set! *tim* "RUN" "YES")

(define *last-recalc-ended-time* 0)







(define (dashboard:recalc modtime please-update-buttons last-db-update-time)
  (or please-update-buttons
      (and (> (current-milliseconds)(+ *last-recalc-ended-time* 150))
	   (> modtime last-db-update-time)
	   (> (current-seconds)(+ last-db-update-time 1)))))

;; (define *monitor-db-path* #f)
(define *last-monitor-update-time* 0)

;; Force creation of the db in case it isn't already there.
(tasks:open-db)

(define (dashboard:get-youngest-run-db-mod-time tabdat)
  (handle-exceptions
   exn
   (begin
     (debug:print 0 *default-log-port* "WARNING: error in accessing databases in get-youngest-run-db-mod-time: " ((condition-property-accessor 'exn 'message) exn))
     (current-seconds)) ;; something went wrong - just print an error and return current-seconds
   (common:max (map (lambda (filen)
		     (file-modification-time filen))
		   (glob (conc (dboard:tabdat-dbdir tabdat) "/*.db"))))))

(define (dashboard:monitor-changed? commondat tabdat)
  (let* ((run-update-time (current-seconds))
	 (monitor-db-path (dboard:tabdat-monitor-db-path tabdat))
	 (monitor-modtime (if (and monitor-db-path (file-exists? monitor-db-path))
			      (file-modification-time monitor-db-path)
			      -1)))
    (if (and (eq? (dboard:commondat-curr-tab-num commondat) 0)
	     (or (> monitor-modtime *last-monitor-update-time*)
		 (> (- run-update-time *last-monitor-update-time*) 5))) ;; update every 1/2 minute just in case
	(begin
	  (set! *last-monitor-update-time* run-update-time) ;; monitor-modtime)
	  #t)
	#f)))

(define (dboard:get-last-db-update tabdat context)
  (hash-table-ref/default (dboard:tabdat-last-db-update tabdat) context 0))

(define (dboard:set-last-db-update! tabdat context newtime)
  (hash-table-set! (dboard:tabdat-last-db-update tabdat) context newtime))

(define (dashboard:database-changed? commondat tabdat #!key (context-key 'default))
  (let* ((run-update-time (current-seconds))
	 (modtime         (dashboard:get-youngest-run-db-mod-time tabdat)) ;; NOTE: ensure this is tabdat!! 
	 (recalc          (dashboard:recalc modtime 
					    (dboard:commondat-please-update commondat) 
                                            (dboard:get-last-db-update tabdat context-key))))
					    ;; (dboard:tabdat-last-db-update tabdat))))
    (if recalc 
        (dboard:set-last-db-update! tabdat context-key run-update-time))
    (dboard:commondat-please-update-set! commondat #f)
    recalc))

;; point inside line
;;
(define-inline (dashboard:px-between px lx1 lx2)
  (and (< lx1 px)(> lx2 px)))

;;Not reference anywhere
;;
;; can a bar be placed in row "rownum" covering x1 to x2 without overlapping with existing 
;; bars? Use num-rows to check that a block will fit from rownum to (+ rownum num-rows)
;;
(define (dashboard:row-collision rowhash rownum x1 x2 #!key (num-rows #f))
  (let ((lastrow   (if num-rows (+ rownum num-rows) rownum)))
    (let loop ((i      0)
	       (rowdat (hash-table-ref/default rowhash rownum '())))
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
    (hash-table-set! rowhash 
		     (+ i rownum)
		     (cons (cons x1 x2) 
			   (hash-table-ref/default rowhash (+ i rownum) '())))
    (if (< i num-rows)
	(loop (+ i 1)))))

;; get min or max, use > for max and < for min, this works around the limits on apply
;;
(define (dboard:min-max comp lst)
  (if (null? lst)
      #f ;; better than an exception for my needs
      (fold (lambda (a b)
	      (if (comp a b) a b))
	    (car lst)
	    lst)))

;; sort a list of test-ids by the event _time using a hash table of id => testdat
;;
(define-inline (dboard:sort-testsdat-by-event-time test-ids tests-ht)
  (sort test-ids
	(lambda (a b)
	  (< (db:test-get-event_time (hash-table-ref tests-ht a))
	     (db:test-get-event_time (hash-table-ref tests-ht b))))))







<
<
<
<
<
<
<
<
<
<







2658
2659
2660
2661
2662
2663
2664










2665
2666
2667
2668
2669
2670
2671
    (hash-table-set! rowhash 
		     (+ i rownum)
		     (cons (cons x1 x2) 
			   (hash-table-ref/default rowhash (+ i rownum) '())))
    (if (< i num-rows)
	(loop (+ i 1)))))











;; sort a list of test-ids by the event _time using a hash table of id => testdat
;;
(define-inline (dboard:sort-testsdat-by-event-time test-ids tests-ht)
  (sort test-ids
	(lambda (a b)
	  (< (db:test-get-event_time (hash-table-ref tests-ht a))
	     (db:test-get-event_time (hash-table-ref tests-ht b))))))
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405

2406

2407
2408
2409
2410
2411
2412
2413
2414
2415
	(for-each
	 (lambda (run-id)
	   (let* ((run-record (hash-table-ref/default runs-hash run-id #f))
		  (key-vals   (map (lambda (key)(db:get-value-by-header run-record runs-header key))
				   (dboard:tabdat-keys tabdat)))
		  (run-name   (db:get-value-by-header run-record runs-header "runname"))
		  (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
		  (run-path   (append key-vals (list run-name)))
		  (existing   (tree:find-node tb run-path)))
	     (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f))
		 (begin
		   (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path)
		   ;; Here we update the tests treebox and tree keys
		   (tree:add-node tb "Runs" run-path ;; (append key-vals (list run-name))
				  userdata: (conc "run-id: " run-id))
		   (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id)
		   ;; (set! colnum (+ colnum 1))
		   ))))
	 run-ids))
    ;; (print "Updating rundat")
    (if (dboard:tabdat-keys tabdat) ;; have keys yet?
	(let* ((num-keys (length (dboard:tabdat-keys tabdat)))
	       (targpatt (map (lambda (k v)
				(list k v))
			      (dboard:tabdat-keys tabdat)
			      (take (append (or (dboard:tabdat-target tabdat);; (string-split (dboard: "/")
						'("%" "%"))
					    (make-list num-keys "%"))
				    num-keys)
			      ))

	       (runpatt   (if (dboard:tabdat-target tabdat)

			     (last (dboard:tabdat-target tabdat))
			     "%"))
	       (testpatt  (or (dboard:tabdat-test-patts tabdat) "%"))
	       (filtrstr  (conc targpatt "/" runpatt "/" testpatt)))
	  ;; (print "targpatt: " targpatt " runpatt: " runpatt " testpatt: " testpatt)

	  (if (not (equal? (dboard:tabdat-last-filter-str tabdat) filtrstr))
	      (let ((dwg (dboard:tabdat-drawing tabdat)))
		(print "reseting drawing")







|
|




|
|















>
|
>
|
|







2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
	(for-each
	 (lambda (run-id)
	   (let* ((run-record (hash-table-ref/default runs-hash run-id #f))
		  (key-vals   (map (lambda (key)(db:get-value-by-header run-record runs-header key))
				   (dboard:tabdat-keys tabdat)))
		  (run-name   (db:get-value-by-header run-record runs-header "runname"))
		  (col-name   (conc (string-intersperse key-vals "\n") "\n" run-name))
		  (run-path   (append key-vals (list run-name))))
             ;; 		  (existing   (tree:find-node tb run-path)))
	     (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f))
		 (begin
		   (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path)
		   ;; Here we update the tests treebox and tree keys
		   (tree:add-node tb "Runs" run-path) ;; (append key-vals (list run-name))
                   ;;				  userdata: (conc "run-id: " run-id))
		   (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id)
		   ;; (set! colnum (+ colnum 1))
		   ))))
	 run-ids))
    ;; (print "Updating rundat")
    (if (dboard:tabdat-keys tabdat) ;; have keys yet?
	(let* ((num-keys (length (dboard:tabdat-keys tabdat)))
	       (targpatt (map (lambda (k v)
				(list k v))
			      (dboard:tabdat-keys tabdat)
			      (take (append (or (dboard:tabdat-target tabdat);; (string-split (dboard: "/")
						'("%" "%"))
					    (make-list num-keys "%"))
				    num-keys)
			      ))
	       (runpatt   (if (and (dboard:tabdat-target tabdat)
                                   (list? (dboard:tabdat-target tabdat))
                                   (not (null? (dboard:tabdat-target tabdat))))
                              (last (dboard:tabdat-target tabdat))
                              "%"))
	       (testpatt  (or (dboard:tabdat-test-patts tabdat) "%"))
	       (filtrstr  (conc targpatt "/" runpatt "/" testpatt)))
	  ;; (print "targpatt: " targpatt " runpatt: " runpatt " testpatt: " testpatt)

	  (if (not (equal? (dboard:tabdat-last-filter-str tabdat) filtrstr))
	      (let ((dwg (dboard:tabdat-drawing tabdat)))
		(print "reseting drawing")
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532




2533
2534
2535
2536
2537
2538
2539
					    fieldname
					    (cons
					     (apply vector tstart (cdr zeropt))						    
					     (hash-table-ref/default res-ht fieldname '())))))))
		 fields)
		res-ht)
	      #f)))))
  
;; graph data 
;;  tsc=timescale, tfn=function; time->x
;;
(define (dboard:graph commondat tabdat tabnum llx lly ulx uly tstart tend tsc tfn compname cmargin)
  (let* ((dwg      (dboard:tabdat-drawing tabdat))
	 (lib      (vg:get/create-lib dwg "runslib"))
	 (cnv      (dboard:tabdat-cnv tabdat))
	 (dur      (- tstart tend)) ;; time duration
	 (cmp      (vg:get-component dwg "runslib" compname))
	 (cfg      (configf:get-section *configdat* "graph"))
	 (stdcolor (vg:rgb->number 120 130 140))
	 (delta-y  (- uly lly)))




    (vg:add-obj-to-comp
     cmp 
     (vg:make-rect-obj llx lly ulx uly))
    (vg:add-obj-to-comp
     cmp
     (vg:make-text-obj (- (tfn tstart) 10)(- lly 10)(seconds->year-week/day-time tstart)))
    (let*-values (((span timeunit time-blk first timesym) (common:find-start-mark-and-mark-delta tstart tend)))







|











|
>
>
>
>







2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
					    fieldname
					    (cons
					     (apply vector tstart (cdr zeropt))						    
					     (hash-table-ref/default res-ht fieldname '())))))))
		 fields)
		res-ht)
	      #f)))))

;; graph data 
;;  tsc=timescale, tfn=function; time->x
;;
(define (dboard:graph commondat tabdat tabnum llx lly ulx uly tstart tend tsc tfn compname cmargin)
  (let* ((dwg      (dboard:tabdat-drawing tabdat))
	 (lib      (vg:get/create-lib dwg "runslib"))
	 (cnv      (dboard:tabdat-cnv tabdat))
	 (dur      (- tstart tend)) ;; time duration
	 (cmp      (vg:get-component dwg "runslib" compname))
	 (cfg      (configf:get-section *configdat* "graph"))
	 (stdcolor (vg:rgb->number 120 130 140))
	 (delta-y  (- uly lly))
         (graph-matrix-table (dboard:tabdat-graph-matrix-table tabdat))
         (graph-cell-table (dboard:tabdat-graph-cell-table tabdat))
         (graph-matrix (dboard:tabdat-graph-matrix tabdat))
         (changed      #f))
    (vg:add-obj-to-comp
     cmp 
     (vg:make-rect-obj llx lly ulx uly))
    (vg:add-obj-to-comp
     cmp
     (vg:make-text-obj (- (tfn tstart) 10)(- lly 10)(seconds->year-week/day-time tstart)))
    (let*-values (((span timeunit time-blk first timesym) (common:find-start-mark-and-mark-delta tstart tend)))
2554
2555
2556
2557
2558
2559
2560
2561





2562


































2563
2564
2565
2566
2567
2568
2569

2570
2571



2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618

2619
2620
2621
2622
2623
2624
2625
			 (loop (+ mark time-blk)(+ count 1))))))
    (for-each 
     (lambda (cf)
       (let* ((alldat  (dboard:graph-read-data (cadr cf) tstart tend)))
	 (if alldat
	     (for-each
	      (lambda (fieldn)
		(let* ((dat     (hash-table-ref alldat fieldn))





		       (vals    (map (lambda (x)(vector-ref x 2)) dat)))


































		  (if (not (null? vals))
		      (let* ((maxval   (apply max vals))
			     (minval   (min 0 (apply min vals)))
			     (yoff     (- minval lly)) ;;  minval))
			     (deltaval (- maxval minval))
			     (yscale   (/ delta-y (if (zero? deltaval) 1 deltaval)))
			     (yfunc    (lambda (y)(+ lly (* yscale (- y minval))))) ;; (lambda (y)(* (+ y yoff) yscale))))

                             (graph-color (vg:generate-color)))
			;; (print (car cf) "; maxval: " maxval " minval: " minval " deltaval: " deltaval " yscale: " yscale)



			(vg:add-obj-to-comp
			 cmp 
			 (vg:make-text-obj (- llx 10)(yfunc maxval) (conc maxval)))
			(vg:add-obj-to-comp
			 cmp 
			 (vg:make-text-obj (- llx 10)(yfunc minval) (conc minval)))
			(fold 
			 (lambda (next prev)  ;; #(time ? val) #(time ? val)
			   (if prev
			       (let* ((yval        (vector-ref prev 2))
                                      (yval-next   (vector-ref next 2))
				      (last-tval   (tfn   (vector-ref prev 0)))
				      (last-yval   (yfunc yval)) ;; (+ lly (* yscale (vector-ref prev 2))))
                                      (next-yval   (yfunc yval-next))
				      (curr-tval   (tfn   (vector-ref next 0))))
				 (if (>= curr-tval last-tval)
                                     (begin
                                       (vg:add-obj-to-comp
                                        cmp 
                                        ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale))
                                        (vg:make-line-obj last-tval last-yval curr-tval last-yval
                                                          line-color: graph-color))
                                       (vg:add-obj-to-comp
                                        cmp 
                                        ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale))
                                        (vg:make-line-obj curr-tval last-yval curr-tval next-yval
                                                 line-color: graph-color)))         
				     (print "ERROR: curr-tval is not > last-tval; curr-tval " curr-tval ", last-tval " last-tval))))
			   next)
			 ;; for init create vector tstart,0
			 #f ;; (vector tstart minval minval)
			 dat)
			   
			 ;; (for-each
			;;  (lambda (dpt)
			;;    (let* ((tval  (vector-ref dpt 0))
			 ;; 	  (yval  (vector-ref dpt 2))
			;; 	  (stval (tfn tval))
			;; 	  (syval (yfunc yval)))
			;;      (vg:add-obj-to-comp
			;;       cmp 
			;;       (vg:make-rect-obj (- stval 2) lly (+ stval 2)(+ lly (* yval yscale))
			;; 			fill-color: stdcolor))))
			;;  dat)
			)))) ;; for each data point in the series
	      (hash-table-keys alldat)))))
     cfg)))

	 
;; run times tab
;;
(define (dashboard:run-times-tab-layout-updater commondat tabdat tab-num)
  ;; each test is an object in the run component
  ;; each run is a component
  ;; all runs stored in runslib library







|
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
>
|
<
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<
|
|
<
<
<
<
<
<
<
<
<
<
<
<
|

|
>







2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981

2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013

3014
3015












3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
			 (loop (+ mark time-blk)(+ count 1))))))
    (for-each 
     (lambda (cf)
       (let* ((alldat  (dboard:graph-read-data (cadr cf) tstart tend)))
	 (if alldat
	     (for-each
	      (lambda (fieldn)
		(let*-values (((dat)                (hash-table-ref alldat fieldn))
                              ((vals minval maxval) (if (null? dat)
                                                        (values '() #f #f)
                                                        (let loop ((hed (car dat))
                                                                   (tal (cdr dat))
                                                                   (res '())
                                                                   (min (vector-ref (car dat) 2))
                                                                   (max (vector-ref (car dat) 2)))
                                                          (let* ((val    (vector-ref hed 2))
                                                                 (newmin (if (< val min) val min))
                                                                 (newmax (if (> val max) val max))
                                                                 (newres (cons val res)))
                                                            (if (null? tal)
                                                                (values (reverse res) newmin newmax)
                                                                (loop (car tal)(cdr tal) newres newmin newmax)))))))
                  (if (not (hash-table-exists? graph-matrix-table fieldn))
                      (begin
                        (let* ((graph-color-rgb (vg:generate-color-rgb))
                               (graph-color (vg:iup-color->number graph-color-rgb))
                               (graph-matrix-col (dboard:tabdat-graph-matrix-col tabdat))
                               (graph-matrix-row (dboard:tabdat-graph-matrix-row tabdat))
                               (graph-cell       (conc graph-matrix-row ":" graph-matrix-col)) 
                               (graph-dat (make-dboard:graph-dat
                                                  id: fieldn
                                                  color: graph-color
                                                  flag: #t
                                                  cell: graph-cell
                                                  )))
                          (hash-table-set! graph-matrix-table fieldn graph-dat)
                          (hash-table-set! graph-cell-table graph-cell graph-dat)
                          ;; (print "Graph data " graph-matrix-row " " graph-matrix-col " " fieldn " " graph-color " " graph-color-rgb " ")
                          ;; (print "Graph data " graph-matrix-row " " graph-matrix-col " " fieldn " " graph-color " " graph-color-rgb " ")
                          (set! changed #t)
                          (iup:attribute-set! graph-matrix (conc graph-matrix-row ":"  graph-matrix-col) fieldn)
                          (iup:attribute-set! graph-matrix (conc "BGCOLOR" (conc graph-matrix-row ":"  graph-matrix-col)) graph-color-rgb)
                          (if (> graph-matrix-col 10)
                              (begin
                                (dboard:tabdat-graph-matrix-col-set! tabdat 1)
                                (dboard:tabdat-graph-matrix-row-set! tabdat (+ graph-matrix-row 1)))
                              (dboard:tabdat-graph-matrix-col-set! tabdat (+ graph-matrix-col 1)))
                          )))
		  (if (not (null? vals)) 
		      (let* (;; (maxval   (apply max vals))
			     ;; (minval   (min 0 (apply min vals)))
			     (yoff        (- minval lly)) ;;  minval))
			     (deltaval    (- maxval minval))
			     (yscale      (/ delta-y (if (zero? deltaval) 1 deltaval)))
			     (yfunc       (lambda (y)(+ lly (* yscale (- y minval))))) ;; (lambda (y)(* (+ y yoff) yscale))))
                             (graph-dat   (hash-table-ref graph-matrix-table fieldn))
                             (graph-color (dboard:graph-dat-color graph-dat))

                             (graph-flag (dboard:graph-dat-flag graph-dat)))
                        (if graph-flag
                            (begin
                              (vg:add-obj-to-comp
                               cmp 
                               (vg:make-text-obj (- llx 10)(yfunc maxval) (conc maxval)))
                              (vg:add-obj-to-comp
                               cmp 
                               (vg:make-text-obj (- llx 10)(yfunc minval) (conc minval)))
                              (fold 
                               (lambda (next prev)  ;; #(time ? val) #(time ? val)
                                 (if prev
                                     (let* ((yval        (vector-ref prev 2))
                                            (yval-next   (vector-ref next 2))
                                            (last-tval   (tfn   (vector-ref prev 0)))
                                            (last-yval   (yfunc yval)) ;; (+ lly (* yscale (vector-ref prev 2))))
                                            (next-yval   (yfunc yval-next))
                                            (curr-tval   (tfn   (vector-ref next 0))))
                                       (if (>= curr-tval last-tval)
                                           (begin
                                             (vg:add-obj-to-comp
                                              cmp 
                                              ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale))
                                              (vg:make-line-obj last-tval last-yval curr-tval last-yval
                                                                line-color: graph-color))
                                             (vg:add-obj-to-comp
                                              cmp 
                                              ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale))
                                              (vg:make-line-obj curr-tval last-yval curr-tval next-yval
                                                                line-color: graph-color)))         
                                           (print "ERROR: curr-tval is not > last-tval; curr-tval " curr-tval ", last-tval " last-tval))))
                                 next)

                               #f ;; (vector tstart minval minval)
                               dat)












                              )))))) ;; for each data point in the series
	      (hash-table-keys alldat)))))
     cfg)
    (if changed (iup:attribute-set! graph-matrix "REDRAW" "ALL"))))
	 
;; run times tab
;;
(define (dashboard:run-times-tab-layout-updater commondat tabdat tab-num)
  ;; each test is an object in the run component
  ;; each run is a component
  ;; all runs stored in runslib library
2635
2636
2637
2638
2639
2640
2641
2642

2643
2644
2645
2646
2647
2648
2649
	       (num-runs     (length allruns))
	       (cnv          (dboard:tabdat-cnv tabdat))
	       (compact-layout (dboard:tabdat-compact-layout tabdat))
	       (row-height     (if compact-layout 2 10))
	       (graph-height 120)
	       (run-to-run-margin 25))
	  (dboard:tabdat-layout-update-ok-set! tabdat #t)
	  (if (canvas? cnv)

	      (let*-values (((sizex sizey sizexmm sizeymm) (canvas-size cnv))
			    ((originx originy)             (canvas-origin cnv))
			    ((calc-y)                      (lambda (rownum)
							     (- (/ sizey 2)
								(* rownum row-height))))
			    ((fixed-originx)               (if (dboard:tabdat-originx tabdat)
							       (dboard:tabdat-originx tabdat)







|
>







3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
	       (num-runs     (length allruns))
	       (cnv          (dboard:tabdat-cnv tabdat))
	       (compact-layout (dboard:tabdat-compact-layout tabdat))
	       (row-height     (if compact-layout 2 10))
	       (graph-height 120)
	       (run-to-run-margin 25))
	  (dboard:tabdat-layout-update-ok-set! tabdat #t)
	  (if (and (canvas? cnv)
                   (not (null? allruns))) ;; allruns can go null when browsing the runs tree
	      (let*-values (((sizex sizey sizexmm sizeymm) (canvas-size cnv))
			    ((originx originy)             (canvas-origin cnv))
			    ((calc-y)                      (lambda (rownum)
							     (- (/ sizey 2)
								(* rownum row-height))))
			    ((fixed-originx)               (if (dboard:tabdat-originx tabdat)
							       (dboard:tabdat-originx tabdat)
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
					      (dboard:rundat-hierdat rundat)))
			       (tests-ht  (dboard:rundat-tests rundat))
			       (all-tids  (hash-table-keys   tests-ht)) ;; (apply append hierdat)) ;; was testsdat
			       (testsdat  (hash-table-values tests-ht))
			       (runcomp   (vg:comp-new));; new component for this run
			       (rows-used (make-hash-table)) ;; keep track of what parts of the rows are used here row1 = (obj1 obj2 ...)
			       ;; (row-height 4)
			       (run-start  (dboard:min-max < (map db:test-get-event_time testsdat)))
			       (run-end    (let ((re (dboard:min-max > (map (lambda (t)(+ (db:test-get-event_time t)(db:test-get-run_duration t))) testsdat))))
					     (max re (+ 1 run-start)))) ;; use run-start+1 if run-start == run-end so delta is not zero
			       (timeoffset (- run-start)) ;; (+ fixed-originx canvas-margin) run-start))
			       (run-duration (- run-end run-start))
			       (timescale  (/ (- sizex (* 2 canvas-margin))
					      (if (> run-duration 0)
						  run-duration
						  (current-seconds)))) ;; a least lously guess







|
|







3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
					      (dboard:rundat-hierdat rundat)))
			       (tests-ht  (dboard:rundat-tests rundat))
			       (all-tids  (hash-table-keys   tests-ht)) ;; (apply append hierdat)) ;; was testsdat
			       (testsdat  (hash-table-values tests-ht))
			       (runcomp   (vg:comp-new));; new component for this run
			       (rows-used (make-hash-table)) ;; keep track of what parts of the rows are used here row1 = (obj1 obj2 ...)
			       ;; (row-height 4)
			       (run-start  (common:min-max < (map db:test-get-event_time testsdat)))
			       (run-end    (let ((re (common:min-max > (map (lambda (t)(+ (db:test-get-event_time t)(db:test-get-run_duration t))) testsdat))))
					     (max re (+ 1 run-start)))) ;; use run-start+1 if run-start == run-end so delta is not zero
			       (timeoffset (- run-start)) ;; (+ fixed-originx canvas-margin) run-start))
			       (run-duration (- run-end run-start))
			       (timescale  (/ (- sizex (* 2 canvas-margin))
					      (if (> run-duration 0)
						  run-duration
						  (current-seconds)))) ;; a least lously guess
2840
2841
2842
2843
2844
2845
2846
2847


2848
2849



2850
2851


2852
2853

2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872










2873

2874
2875
2876
2877



2878



















2879



2880
2881
2882


















2883




2884
2885
2886





2887
2888

2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
				      (begin
					(if (dboard:tabdat-layout-update-ok tabdat)
					    (runloop (car runtal)(cdr runtal) (+ run-num 1) newdoneruns)
					    (escapeloop #t) ;; (dboard:tabdat-layout-update-ok tabdat)
					    ))))))))) ;;  new-run-start-row
		)))
	(debug:print 2 *default-log-port* "no tabdat for run-times-tab-updater"))))



(define (dashboard:runs-tab-updater commondat tab-num)
  (debug:catch-and-dump 



   (lambda ()
     (let* ((tabdat (dboard:common-get-tabdat commondat tab-num: tab-num))


	    (dbkeys (dboard:tabdat-dbkeys tabdat)))
       (update-rundat tabdat

		      (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "runname" "%")
		      (dboard:tabdat-numruns tabdat)
		      (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "test-name" "%/%")
		      ;; (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "item-name" "%")
		      (let* ((dbkeys (dboard:tabdat-dbkeys tabdat)))
			;; (print "dbkeys: " dbkeys)
			(let ((fres   (if (dboard:tabdat-target tabdat)
					 (let ((ptparts (append (dboard:tabdat-target tabdat)(make-list (length dbkeys) "%"))))
					   (map (lambda (k v)(list k v)) dbkeys ptparts))
					 (let ((res '()))
					   ;; (print "target: " (dboard:tabdat-target tabdat))
					   (for-each (lambda (key)
						       (if (not (equal? key "runname"))
							   (let ((val (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) key #f)))
							     (if val (set! res (cons (list key val) res))))))
						     dbkeys)
					   res))))
			  ;; (debug:print 0 *default-log-port* "fres: " fres)
			  fres)))










       (let ((uidat (dboard:commondat-uidat commondat)))

	 (update-buttons tabdat uidat (dboard:tabdat-numruns tabdat) (dboard:tabdat-num-tests tabdat)))
       ))
   "dashboard:runs-tab-updater"))




;; ((2)



















;;  (dashboard:update-run-summary-tab))



;; ((3)
;;  (dashboard:update-new-view-tab))
;; (else


















;;  (dboard:common-run-curr-updater commondat)))




;; (set! *last-recalc-ended-time* (current-milliseconds))))))))

;;======================================================================





;; The heavy lifting starts here
;;======================================================================


;; ease debugging by loading ~/.dashboardrc
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc")))
  (if (file-exists? debugcontrolf)
      (load debugcontrolf)))

(define (main)
  (if (not (args:get-arg "-skip-version-check"))(common:exit-on-version-changed))
  (let* ((commondat       (dboard:commondat-make)))
    ;; Move this stuff to db.scm? I'm not sure that is the right thing to do...
    (cond 
     ((args:get-arg "-test") ;; run-id,test-id
      (let* ((dat     (let ((d (map string->number (string-split (args:get-arg "-test") ","))))
			(if (> (length d) 1)
			    d
			    (list #f #f))))
	     (run-id  (car dat))
	     (test-id (cadr dat)))
	(if (and (number? run-id)
		 (number? test-id)
		 (>= test-id 0))
	    (examine-test run-id test-id)
	    (begin
	      (debug:print 3 *default-log-port* "INFO: tried to open test with invalid run-id,test-id. " (args:get-arg "-test"))
	      (exit 1)))))
     ;; ((args:get-arg "-guimonitor")
     ;;  (gui-monitor (dboard:tabdat-dblocal tabdat)))
     (else
      (dboard:commondat-uidat-set! commondat (make-dashboard-buttons commondat)) ;; (dboard:tabdat-dblocal data)
					  ;; (dboard:tabdat-numruns tabdat)
					  ;; (dboard:tabdat-num-tests tabdat)
					  ;; (dboard:tabdat-dbkeys tabdat)
					  ;; runs-sum-dat new-view-dat))
      ;; legacy setup of updaters for summary tab and runs tab
      ;; summary tab
      ;; (dboard:commondat-add-updater 
      ;;  commondat 
      ;;  (lambda ()
      ;; 	 (dashboard:summary-tab-updater commondat 0))
      ;;  tab-num: 0)
      ;; runs tab
      (dboard:commondat-curr-tab-num-set! commondat 0)
      (dboard:commondat-add-updater 
       commondat 
       (lambda ()
      	 (dashboard:runs-tab-updater commondat 1))
       tab-num: 1)
      (iup:callback-set! *tim*
			 "ACTION_CB"
			 (lambda (time-obj)
			   (let ((update-is-running #f))
			     (mutex-lock! (dboard:commondat-update-mutex commondat))
			     (set! update-is-running (dboard:commondat-updating commondat))
			     (if (not update-is-running)
				 (dboard:commondat-updating-set! commondat #t))
			     (mutex-unlock! (dboard:commondat-update-mutex commondat))
			     (if (not update-is-running) ;; we know that the update was not running and we now have a lock on doing an update
				 (begin
				   (dboard:common-run-curr-updaters commondat) ;; (dashboard:run-update commondat)
				   (mutex-lock! (dboard:commondat-update-mutex commondat))
				   (dboard:commondat-updating-set! commondat #f)
				   (mutex-unlock! (dboard:commondat-update-mutex commondat)))
				 ))
			   1))))
    
    (let ((th1 (make-thread (lambda ()
			      (thread-sleep! 1)
			      (dboard:common-run-curr-updaters commondat 0) ;; force update of summary tab 
			      ;; (dboard:commondat-please-update-set! commondat #t) ;; MRW: ww36.3 - why was please update set true here? Removing it for now.
			      ;; (dashboard:run-update commondat)
			      ) "update buttons once"))
	  (th2 (make-thread iup:main-loop "Main loop")))
      ;; (thread-start! th1)
      (thread-start! th2)
      (thread-join! th2))))

(main)









>
>
|
<
>
>
>
|
|
>
>
|
|
>
|
|
|
<
|
<
|
|
|
|
<
|
|
|
|
|
|
<
|
>
>
>
>
>
>
>
>
>
>

>




>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
|
|
<
>
>
>
>
>
|
<
>






<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252

3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265

3266

3267
3268
3269
3270

3271
3272
3273
3274
3275
3276

3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348

3349
3350
3351
3352
3353
3354

3355
3356
3357
3358
3359
3360
3361






































































3362
3363
				      (begin
					(if (dboard:tabdat-layout-update-ok tabdat)
					    (runloop (car runtal)(cdr runtal) (+ run-num 1) newdoneruns)
					    (escapeloop #t) ;; (dboard:tabdat-layout-update-ok tabdat)
					    ))))))))) ;;  new-run-start-row
		)))
	(debug:print 2 *default-log-port* "no tabdat for run-times-tab-updater"))))

;; handy trick for printing a record
;;
;;   (pp (dboard:tabdat->alist tabdat))

;; 
;;  removing the tabdat-values proc 
;;
;; (define (tabdat-values tabdat)

;; runs update-rundat using the various filters from the gui
;;
(define (dashboard:do-update-rundat tabdat)
  (update-rundat
   tabdat
   (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "runname" "%")
   (dboard:tabdat-numruns tabdat)
   (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "test-name" "%/%")

   (let* ((dbkeys (dboard:tabdat-dbkeys tabdat)))

     (let ((fres   (if (dboard:tabdat-target tabdat)
                       (let ((ptparts (append (dboard:tabdat-target tabdat)(make-list (length dbkeys) "%"))))
                         (map (lambda (k v)(list k v)) dbkeys ptparts))
                       (let ((res '()))

                         (for-each (lambda (key)
                                     (if (not (equal? key "runname"))
                                         (let ((val (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) key #f)))
                                           (if val (set! res (cons (list key val) res))))))
                                   dbkeys)
                         res))))

       fres))))

(define (dashboard:runs-tab-updater commondat tab-num)
  (debug:catch-and-dump 
   (lambda ()
     (let* ((tabdat (dboard:common-get-tabdat commondat tab-num: tab-num))
	    (dbkeys (dboard:tabdat-dbkeys tabdat)))
       ;;(print "RA => calling runs-tab-updater with commondat " commondat " tab-num " tab-num)
       ;;(tabdat-values tabdat) ;;RA added 
       ;; (pp (dboard:tabdat->alist tabdat))
       (dashboard:do-update-rundat tabdat)
       (let ((uidat (dboard:commondat-uidat commondat)))
         ;;(print "RA => Calling update-buttons with tabdat : " tabdat " uidat " uidat)
	 (update-buttons tabdat uidat (dboard:tabdat-numruns tabdat) (dboard:tabdat-num-tests tabdat)))
       ))
   "dashboard:runs-tab-updater"))

;;======================================================================
;; The heavy lifting starts here
;;======================================================================

(define (main)
  (let ((mtdb-path (conc *toppath* "/megatest.db"))) ;; 
    (if (and (file-exists? mtdb-path)
	     (file-write-access? mtdb-path))
	(if (not (args:get-arg "-skip-version-check"))
            (common:exit-on-version-changed)))
    (let* ((commondat       (dboard:commondat-make)))
      ;; Move this stuff to db.scm? I'm not sure that is the right thing to do...
      (cond 
       ((args:get-arg "-test") ;; run-id,test-id
      (let* ((dat     (let ((d (map string->number (string-split (args:get-arg "-test") ",")))) 
			  (if (> (length d) 1)
			      d
			      (list #f #f))))
	       (run-id  (car dat))
	       (test-id (cadr dat)))
	  (if (and (number? run-id)
		   (number? test-id)
		   (>= test-id 0))
	      (dashboard-tests:examine-test run-id test-id)
	      (begin
		(debug:print 3 *default-log-port* "INFO: tried to open test with invalid run-id,test-id. " (args:get-arg "-test"))
		(exit 1)))))
       ;; ((args:get-arg "-guimonitor")
       ;;  (gui-monitor (dboard:tabdat-dblocal tabdat)))
       (else
	(dboard:commondat-uidat-set! commondat (make-dashboard-buttons commondat)) ;; (dboard:tabdat-dblocal data)
	(dboard:commondat-curr-tab-num-set! commondat 0)
	(dboard:commondat-add-updater 
	 commondat 
	 (lambda ()
	   (dashboard:runs-tab-updater commondat 1))
	 tab-num: 1)
	(iup:callback-set! *tim*
			   "ACTION_CB"
			   (lambda (time-obj)
			     (let ((update-is-running #f))
			       (mutex-lock! (dboard:commondat-update-mutex commondat))
			       (set! update-is-running (dboard:commondat-updating commondat))
			       (if (not update-is-running)
				   (dboard:commondat-updating-set! commondat #t))
			       (mutex-unlock! (dboard:commondat-update-mutex commondat))
			       (if (not update-is-running) ;; we know that the update was not running and we now have a lock on doing an update
				   (begin
				     (dboard:common-run-curr-updaters commondat) ;; (dashboard:run-update commondat)
				     (mutex-lock! (dboard:commondat-update-mutex commondat))
				     (dboard:commondat-updating-set! commondat #f)
				     (mutex-unlock! (dboard:commondat-update-mutex commondat)))
				   ))
			     1))))
      

      (let ((th1 (make-thread (lambda ()
				(thread-sleep! 1)
				(dboard:common-run-curr-updaters commondat 0) ;; force update of summary tab 
				) "update buttons once"))
	    (th2 (make-thread iup:main-loop "Main loop")))
	(thread-start! th2)

	(thread-join! th2)))))

;; ease debugging by loading ~/.dashboardrc
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc")))
  (if (file-exists? debugcontrolf)
      (load debugcontrolf)))







































































(main)

Modified db.scm from [1c6bc853bb] to [bd53297b84].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15


16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35



















36
37
38
39
40
41
42
43
44
45
46
47
48

49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

80
81
82
83
84
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

;;======================================================================
;; Database access
;;======================================================================



(require-extension (srfi 18) extras tcp)
(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3)
(import (prefix sqlite3 sqlite3:))
(import (prefix base64 base64:))

(declare (unit db))
(declare (uses common))
(declare (uses keys))
(declare (uses ods))
(declare (uses client))
(declare (uses mt))

(include "common_records.scm")
(include "db_records.scm")
(include "key_records.scm")
(include "run_records.scm")

(define *rundb-mutex* (make-mutex)) ;; prevent problems opening/closing rundb's
(define *number-of-writes* 0)
(define *number-non-write-queries* 0)




















;;======================================================================
;; SQLITE3 HELPERS
;;======================================================================

(define (db:general-sqlite-error-dump exn stmt . params)
  (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
    ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
    (print "err-status: " err-status)
    (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
    (print-call-chain (current-error-port))))

;; convert to -inline

(define (db:first-result-default db stmt default . params)
  (handle-exceptions
   exn
   (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
     ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
     (if (eq? err-status 'done)
	 default
	 (begin
	   (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
	   (print-call-chain (current-error-port))
	   default)))
   (apply sqlite3:first-result db stmt params)))

;; Get/open a database
;;    if run-id => get run specific db
;;    if #f     => get main db
;;    if db already open - return inmem
;;    if db not open, open inmem, rundb and sync then return inmem
;;    inuse gets set automatically for rundb's
;;
(define (db:get-db dbstruct run-id)
  (if (sqlite3:database? dbstruct) ;; pass sqlite3 databases on through
      dbstruct
      (begin
	(let ((dbdat (if (or (not run-id)
			     (eq? run-id 0))
			 (db:open-main dbstruct)
			 (db:open-rundb dbstruct run-id)
			 )))
	  dbdat))))


(define (db:dbdat-get-db dbdat)
  (if (pair? dbdat)
      (car dbdat)
      dbdat))

(define (db:dbdat-get-path dbdat)
  (if (pair? dbdat)
      (cdr dbdat)
      #f))

;; mod-read:
;;     'mod   modified data
;;     'read  read data

;;
(define (db:done-with dbstruct run-id mod-read)
  (if (not (sqlite3:database? dbstruct))
      (begin
	(mutex-lock! *rundb-mutex*)
	(if (eq? mod-read 'mod)
	    (dbr:dbstruct-set-mtime! dbstruct (current-milliseconds))
	    (dbr:dbstruct-set-rtime! dbstruct (current-milliseconds)))
	(dbr:dbstruct-set-inuse! dbstruct #f)
	(mutex-unlock! *rundb-mutex*))))

;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;")
;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
;;
(define (db:with-db dbstruct run-id r/w proc . params)
  (let* ((dbdat (if (vector? dbstruct)
		    (db:get-db dbstruct run-id)
		    dbstruct)) ;; cheat, allow for passing in a dbdat
	 (db    (db:dbdat-get-db dbdat)))
    (db:delay-if-busy dbdat)
    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn))
       (print-call-chain (current-error-port)))
     (let ((res (apply proc db params)))
       (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))

|













>
>
|
|

|
















>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>






|





|
>




















|










>













>






|
|
|






|


|
<







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

136
137
138
139
140
141
142
;;======================================================================
;; Copyright 2006-2016, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

;;======================================================================
;; Database access
;;======================================================================

;; dbstruct vector containing all the relevant dbs like main.db, megatest.db, run.db etc

(require-extension (srfi 18) extras tcp) ;; RADT => use of require-extension?
(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3 typed-records)
(import (prefix sqlite3 sqlite3:))
(import (prefix base64 base64:)) ;; RADT => prefix??

(declare (unit db))
(declare (uses common))
(declare (uses keys))
(declare (uses ods))
(declare (uses client))
(declare (uses mt))

(include "common_records.scm")
(include "db_records.scm")
(include "key_records.scm")
(include "run_records.scm")

(define *rundb-mutex* (make-mutex)) ;; prevent problems opening/closing rundb's
(define *number-of-writes* 0)
(define *number-non-write-queries* 0)

;;======================================================================
;;  R E C O R D S
;;======================================================================

(defstruct dbr:dbstruct 
  main
  strdb
  ((path #f)  : string)
  ((local #f) : boolean)
  rundb
  inmem
  mtime
  rtime 
  stime
  inuse
  refdb
  ((locdbs (make-hash-table)) : hash-table)
  olddb)

;;======================================================================
;; SQLITE3 HELPERS
;;======================================================================

(define (db:general-sqlite-error-dump exn stmt . params)
  (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn))) ;; RADT ... how does this work?
    ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
    (print "err-status: " err-status)
    (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
    (print-call-chain (current-error-port))))

;; convert to -inline 
;;
(define (db:first-result-default db stmt default . params)
  (handle-exceptions
   exn
   (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
     ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
     (if (eq? err-status 'done)
	 default
	 (begin
	   (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
	   (print-call-chain (current-error-port))
	   default)))
   (apply sqlite3:first-result db stmt params)))

;; Get/open a database
;;    if run-id => get run specific db
;;    if #f     => get main db
;;    if db already open - return inmem
;;    if db not open, open inmem, rundb and sync then return inmem
;;    inuse gets set automatically for rundb's
;;
(define (db:get-db dbstruct run-id) 
  (if (sqlite3:database? dbstruct) ;; pass sqlite3 databases on through
      dbstruct
      (begin
	(let ((dbdat (if (or (not run-id)
			     (eq? run-id 0))
			 (db:open-main dbstruct)
			 (db:open-rundb dbstruct run-id)
			 )))
	  dbdat))))

;; legacy handling of structure for managing db's. Refactor this into dbr:?
(define (db:dbdat-get-db dbdat)
  (if (pair? dbdat)
      (car dbdat)
      dbdat))

(define (db:dbdat-get-path dbdat)
  (if (pair? dbdat)
      (cdr dbdat)
      #f))

;; mod-read:
;;     'mod   modified data
;;     'read  read data
;; Locks the mutex and depending on 'mod or 'read passed, sets the last timestamp in dbstruct
;;
(define (db:done-with dbstruct run-id mod-read)
  (if (not (sqlite3:database? dbstruct))
      (begin
	(mutex-lock! *rundb-mutex*)
	(if (eq? mod-read 'mod)
	    (dbr:dbstruct-mtime-set! dbstruct (current-milliseconds))
	    (dbr:dbstruct-rtime-set! dbstruct (current-milliseconds)))
	(dbr:dbstruct-inuse-set! dbstruct #f)
	(mutex-unlock! *rundb-mutex*))))

;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;")
;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
;;
(define (db:with-db dbstruct run-id r/w proc . params)
  (let* ((dbdat (if (dbr:dbstruct? dbstruct)
		    (db:get-db dbstruct run-id)
		    dbstruct)) ;; cheat, allow for passing in a dbdat
	 (db    (db:dbdat-get-db dbdat))) 

    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn))
       (print-call-chain (current-error-port)))
     (let ((res (apply proc db params)))
       (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
141
142
143
144
145
146
147


148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163


164
165
166
167
168
169
170
171
172
173
174

175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193





194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
;; ;;
;; (define (db:get-path dbstruct id)
;;   (let ((fdb (db:get-filedb dbstruct)))
;;     (filedb:get-path db id)))

;; NB// #f => return dbdir only
;;      (was planned to be;  zeroth db with name=main.db)


;;
(define (db:dbfile-path run-id)
  (let* ((dbdir           (db:get-dbdir))
	 (fname           (if run-id
			      (if (eq? run-id 0) "main.db" (conc run-id ".db"))
			      #f)))
    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir)
       (exit 1))
     (if (not (directory? dbdir))(create-directory dbdir #t)))
    (if fname
	(conc dbdir "/" fname)
	dbdir)))



(define (db:get-dbdir)
  (or (configf:lookup *configdat* "setup" "dbdir")
      (conc (configf:lookup *configdat* "setup" "linktree") "/.db")))
	       
(define (db:set-sync db)
  (let ((syncprag (configf:lookup *configdat* "setup" "sychronous")))
    (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 1) ";"))))

;; open an sql database inside a file lock
;;
;; returns: db existed-prior-to-opening

;;
(define (db:lock-create-open fname initproc)
  ;; (if (file-exists? fname)
  ;;     (let ((db (sqlite3:open-database fname)))
  ;;       (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
  ;;       (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;")
  ;;       db)
  (let* ((parent-dir   (pathname-directory fname))
	 (dir-writable (file-write-access? parent-dir))
	 (file-exists  (file-exists? fname))
	 (file-write   (if file-exists
			   (file-write-access? fname)
			   dir-writable )))
    (if file-write ;; dir-writable
	(let (;; (lock    (obtain-dot-lock fname 1 5 10))
	      (db      (sqlite3:open-database fname)))
	  (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
	  (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;")
	  (if (not file-exists)(initproc db))





	  ;; (release-dot-lock fname)
	  db)
	(begin
	  (debug:print 2 *default-log-port* "WARNING: opening db in non-writable dir " fname)
	  (sqlite3:open-database fname))))) ;; )

;; This routine creates the db. It is only called if the db is not already opened
;; 
(define (db:open-rundb dbstruct run-id #!key (attemptnum 0)(do-not-open #f)) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
  (let* ((local  (dbr:dbstruct-get-local dbstruct))
	 (rdb    (if local
		     (dbr:dbstruct-get-localdb dbstruct run-id)
		     (dbr:dbstruct-get-inmem dbstruct)))) ;; (dbr:dbstruct-get-runrec dbstruct run-id 'inmem)))
    (if (or rdb
	    do-not-open)
	rdb
	(begin
	  (mutex-lock! *rundb-mutex*)
	  (let* ((dbpath       (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db"))
		 (dbexists     (file-exists? dbpath))







>
>













|


>
>






|


<

>


















|
>
>
>
>
>









|

|
|







164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
;; ;;
;; (define (db:get-path dbstruct id)
;;   (let ((fdb (db:get-filedb dbstruct)))
;;     (filedb:get-path db id)))

;; NB// #f => return dbdir only
;;      (was planned to be;  zeroth db with name=main.db)
;; 
;; If run-id is #f return to create and retrieve the path where the db will live.
;;
(define (db:dbfile-path run-id)
  (let* ((dbdir           (db:get-dbdir))
	 (fname           (if run-id
			      (if (eq? run-id 0) "main.db" (conc run-id ".db"))
			      #f)))
    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir)
       (exit 1))
     (if (not (directory? dbdir))(create-directory dbdir #t)))
    (if fname
	(conc dbdir "/" fname) 
	dbdir)))

;; Returns the database location as specified in config file
;;
(define (db:get-dbdir)
  (or (configf:lookup *configdat* "setup" "dbdir")
      (conc (configf:lookup *configdat* "setup" "linktree") "/.db")))
	       
(define (db:set-sync db)
  (let ((syncprag (configf:lookup *configdat* "setup" "sychronous")))
    (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 1) ";")))) 

;; open an sql database inside a file lock

;; returns: db existed-prior-to-opening
;; RA => Returns a db handler; sets the lock if opened in writable mode
;;
(define (db:lock-create-open fname initproc)
  ;; (if (file-exists? fname)
  ;;     (let ((db (sqlite3:open-database fname)))
  ;;       (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
  ;;       (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;")
  ;;       db)
  (let* ((parent-dir   (pathname-directory fname))
	 (dir-writable (file-write-access? parent-dir))
	 (file-exists  (file-exists? fname))
	 (file-write   (if file-exists
			   (file-write-access? fname)
			   dir-writable )))
    (if file-write ;; dir-writable
	(let (;; (lock    (obtain-dot-lock fname 1 5 10))
	      (db      (sqlite3:open-database fname)))
	  (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
	  (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;")
	  (if (not file-exists)
	      (begin
		(if (string-match "^/tmp/.*" fname) ;; this is a file in /tmp
		    (sqlite3:execute db "PRAGMA journal_mode=WAL;")
		    (print "Creating " fname " in NON-WAL mode."))
		(initproc db)))
	  ;; (release-dot-lock fname)
	  db)
	(begin
	  (debug:print 2 *default-log-port* "WARNING: opening db in non-writable dir " fname)
	  (sqlite3:open-database fname))))) ;; )

;; This routine creates the db. It is only called if the db is not already opened
;; 
(define (db:open-rundb dbstruct run-id #!key (attemptnum 0)(do-not-open #f)) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
  (let* ((local  (dbr:dbstruct-local dbstruct))
	 (rdb    (if local
		     (dbr:dbstruct-localdb dbstruct run-id)
		     (dbr:dbstruct-inmem dbstruct)))) ;; (dbr:dbstruct-runrec dbstruct run-id 'inmem)))
    (if (or rdb
	    do-not-open)
	rdb
	(begin
	  (mutex-lock! *rundb-mutex*)
	  (let* ((dbpath       (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db"))
		 (dbexists     (file-exists? dbpath))
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292


293
294
295
296
297








298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
				     (set! *megatest-db* db)
				     db)))
		 (write-access (file-write-access? dbpath))
		 ;; (handler      (make-busy-timeout 136000))
		 )
	    (if (and dbexists (not write-access))
		(set! *db-write-access* #f)) ;; only unset so other db's also can use this control
	    (dbr:dbstruct-set-rundb!  dbstruct (cons db dbpath))
	    (dbr:dbstruct-set-inuse!  dbstruct #t)
	    (dbr:dbstruct-set-olddb!  dbstruct olddb)
	    ;; (dbr:dbstruct-set-run-id! dbstruct run-id)
	    (mutex-unlock! *rundb-mutex*)
	    (if local
		(begin
		  (dbr:dbstruct-set-localdb! dbstruct run-id db) ;; (dbr:dbstruct-set-inmem! dbstruct db) ;; direct access ...
		  db)
		(begin
		  (dbr:dbstruct-set-inmem!  dbstruct inmem)
		  ;; dec 14, 2014 - keep deleted records available. hunch is that they are needed for id placeholders
		  ;; (sqlite3:execute db "DELETE FROM tests WHERE state='DELETED';") ;; they just slow us down in this context
		  (db:sync-tables db:sync-tests-only db inmem)
		  (db:delay-if-busy refdb) ;; dbpath: (db:dbdat-get-path refdb)) ;; What does delaying here achieve? 
		  (dbr:dbstruct-set-refdb!  dbstruct refdb)
		  (db:sync-tables db:sync-tests-only inmem refdb) ;; use inmem as the reference, don't read again from db
		  ;; sync once more to deal with delays?
		  ;; (db:sync-tables db:sync-tests-only db inmem)
		  ;; (db:sync-tables db:sync-tests-only inmem refdb)
		  inmem)))))))

;; This routine creates the db. It is only called if the db is not already ls opened
;;
(define (db:open-main dbstruct) ;;  (conc *toppath* "/megatest.db") (car *configinfo*)))
  (let ((mdb (dbr:dbstruct-get-main dbstruct)))
    (if mdb
	mdb
	(begin
	  (mutex-lock! *rundb-mutex*)
	  (let* ((dbpath       (db:dbfile-path 0))
		 (dbexists     (file-exists? dbpath))
		 (db           (db:lock-create-open dbpath db:initialize-main-db))
		 (olddb        (db:open-megatest-db))
		 (write-access (file-write-access? dbpath))
		 (dbdat        (cons db dbpath)))
	    (if (and dbexists (not write-access))
		(set! *db-write-access* #f))
	    (dbr:dbstruct-set-main!   dbstruct dbdat)
	    (dbr:dbstruct-set-olddb!  dbstruct olddb) ;; olddb is already a (cons db path)
	    (mutex-unlock! *rundb-mutex*)
	    (if (and (not dbexists)
		     *db-write-access*) ;; did not have a prior db and do have write access
		(db:multi-db-sync #f 'old2new))  ;; migrate data from megatest.db automatically
	    dbdat)))))

;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;


(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))
    dbstruct))









;; Open the classic megatest.db file in toppath
;;
(define (db:open-megatest-db)
  (let* ((dbpath       (conc *toppath* "/megatest.db"))
	 (dbexists     (file-exists? dbpath))
	 (db           (db:lock-create-open dbpath
					    (lambda (db)
					      (db:initialize-main-db db)
					      (db:initialize-run-id-db db))))
	 (write-access (file-write-access? dbpath)))
    (if (and dbexists (not write-access))
	(set! *db-write-access* #f))
    (cons db dbpath)))

;; sync run to disk if touched
;;
(define (db:sync-touched dbstruct run-id #!key (force-sync #f))
  (let ((mtime  (dbr:dbstruct-get-mtime dbstruct))
	(stime  (dbr:dbstruct-get-stime dbstruct))
	(rundb  (dbr:dbstruct-get-rundb dbstruct))
	(inmem  (dbr:dbstruct-get-inmem dbstruct))
	(maindb (dbr:dbstruct-get-main  dbstruct))
	(refdb  (dbr:dbstruct-get-refdb dbstruct))
	(olddb  (dbr:dbstruct-get-olddb dbstruct))
	;; (runid  (dbr:dbstruct-get-run-id dbstruct))
	)
    (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id)
    ;; (mutex-lock! *http-mutex*)
    (if (eq? run-id 0)
	;; runid equal to 0 is main.db
	(if maindb
	    (if (or (not (number? mtime))
		    (not (number? stime))
		    (> mtime stime)
		    force-sync)
		(begin
		  (db:delay-if-busy maindb)
		  (db:delay-if-busy olddb)
		  (let ((num-synced (db:sync-tables (db:sync-main-list maindb) maindb olddb)))
		    (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
		    num-synced)
		  0))
	    (begin
	      ;; this can occur when using local access (i.e. not in a server)
	      ;; need a flag to turn it off.
	      ;;
	      (debug:print 3 *default-log-port* "WARNING: call to sync main.db to megatest.db but main not initialized")
	      0))
	;; any other runid is a run
	(if (or (not (number? mtime))
		(not (number? stime))
		(> mtime stime)
		force-sync)
	    (begin
	      (db:delay-if-busy rundb)
	      (db:delay-if-busy olddb)
	      (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
	      (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb olddb)))
		;; (mutex-unlock! *http-mutex*)
		num-synced)
	      (begin
		;; (mutex-unlock! *http-mutex*)
		0))))))

(define (db:close-main dbstruct)
  (let ((maindb (dbr:dbstruct-get-main dbstruct)))
    (if maindb
	(begin
	  (sqlite3:finalize! (db:dbdat-get-db maindb))
	  (dbr:dbstruct-set-main! dbstruct #f)))))

(define (db:close-run-db dbstruct run-id)
  (let ((rdb (db:open-rundb dbstruct run-id do-not-open: #t)))
    (if (and rdb
	     (sqlite3:database? rdb))
	(begin
	  (sqlite3:finalize! rdb)
	  (dbr:dbstruct-set-localdb! dbstruct run-id #f)
	  (dbr:dbstruct-set-inmem! dbstruct #f)))))

;; close all opened run-id dbs
(define (db:close-all dbstruct)
  ;; finalize main.db
  (db:sync-touched dbstruct 0 force-sync: #t)
  ;;(common:db-block-further-queries)
  ;; (mutex-lock! *db-sync-mutex*) ;; with this perhaps it isn't necessary to use the block-further-queries mechanism?

  (db:close-main dbstruct)
  
  (let ((locdbs (dbr:dbstruct-get-locdbs dbstruct)))
    (if (hash-table? locdbs)
	(for-each (lambda (run-id)
		    (db:close-run-db dbstruct run-id))
		  (hash-table-keys locdbs)))))

(define (db:open-inmem-db)
  (let* ((db      (sqlite3:open-database ":memory:"))







|
|
|
|



|


|




|






|

|
|












|
|








>
>





>
>
>
>
>
>
>
>

















|
|
|
|
|
|
|
|














|
















|








|



|







|
|










|







270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
				     (set! *megatest-db* db)
				     db)))
		 (write-access (file-write-access? dbpath))
		 ;; (handler      (make-busy-timeout 136000))
		 )
	    (if (and dbexists (not write-access))
		(set! *db-write-access* #f)) ;; only unset so other db's also can use this control
	    (dbr:dbstruct-rundb-set!  dbstruct (cons db dbpath))
	    (dbr:dbstruct-inuse-set!  dbstruct #t)
	    (dbr:dbstruct-olddb-set!  dbstruct olddb)
	    ;; (dbr:dbstruct-run-id-set! dbstruct run-id)
	    (mutex-unlock! *rundb-mutex*)
	    (if local
		(begin
		  (dbr:dbstruct-localdb-set! dbstruct run-id db) ;; (dbr:dbstruct-inmem-set! dbstruct db) ;; direct access ...
		  db)
		(begin
		  (dbr:dbstruct-inmem-set!  dbstruct inmem)
		  ;; dec 14, 2014 - keep deleted records available. hunch is that they are needed for id placeholders
		  ;; (sqlite3:execute db "DELETE FROM tests WHERE state='DELETED';") ;; they just slow us down in this context
		  (db:sync-tables db:sync-tests-only db inmem)
		  (db:delay-if-busy refdb) ;; dbpath: (db:dbdat-get-path refdb)) ;; What does delaying here achieve? 
		  (dbr:dbstruct-refdb-set!  dbstruct refdb)
		  (db:sync-tables db:sync-tests-only inmem refdb) ;; use inmem as the reference, don't read again from db
		  ;; sync once more to deal with delays?
		  ;; (db:sync-tables db:sync-tests-only db inmem)
		  ;; (db:sync-tables db:sync-tests-only inmem refdb)
		  inmem)))))))

;; This routine creates the db if not already present. It is only called if the db is not already ls opened
;;
(define (db:open-main dbstruct) ;;  (conc *toppath* "/megatest.db") (car *configinfo*))) 
  (let ((mdb (dbr:dbstruct-main dbstruct))) ;; RA => Returns the first reference in dbstruct
    (if mdb
	mdb
	(begin
	  (mutex-lock! *rundb-mutex*)
	  (let* ((dbpath       (db:dbfile-path 0))
		 (dbexists     (file-exists? dbpath))
		 (db           (db:lock-create-open dbpath db:initialize-main-db))
		 (olddb        (db:open-megatest-db))
		 (write-access (file-write-access? dbpath))
		 (dbdat        (cons db dbpath)))
	    (if (and dbexists (not write-access))
		(set! *db-write-access* #f))
	    (dbr:dbstruct-main-set!   dbstruct dbdat)
	    (dbr:dbstruct-olddb-set!  dbstruct olddb) ;; olddb is already a (cons db path)
	    (mutex-unlock! *rundb-mutex*)
	    (if (and (not dbexists)
		     *db-write-access*) ;; did not have a prior db and do have write access
		(db:multi-db-sync #f 'old2new))  ;; migrate data from megatest.db automatically
	    dbdat)))))

;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;
;; called in http-transport and replicated in rmt.scm for *local* access. 
;;
(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))
    dbstruct))

;; open the local db for direct access (no server)
;;
(define (db:open-local-db-handle)
  (or *dbstruct-db*
      (let ((dbstruct (db:setup #f local: #t)))
	(set! *dbstruct-db* dbstruct)
	dbstruct)))
	  
;; Open the classic megatest.db file in toppath
;;
(define (db:open-megatest-db)
  (let* ((dbpath       (conc *toppath* "/megatest.db"))
	 (dbexists     (file-exists? dbpath))
	 (db           (db:lock-create-open dbpath
					    (lambda (db)
					      (db:initialize-main-db db)
					      (db:initialize-run-id-db db))))
	 (write-access (file-write-access? dbpath)))
    (if (and dbexists (not write-access))
	(set! *db-write-access* #f))
    (cons db dbpath)))

;; sync run to disk if touched
;;
(define (db:sync-touched dbstruct run-id #!key (force-sync #f))
  (let ((mtime  (dbr:dbstruct-mtime dbstruct))
	(stime  (dbr:dbstruct-stime dbstruct))
	(rundb  (dbr:dbstruct-rundb dbstruct))
	(inmem  (dbr:dbstruct-inmem dbstruct))
	(maindb (dbr:dbstruct-main  dbstruct))
	(refdb  (dbr:dbstruct-refdb dbstruct))
	(olddb  (dbr:dbstruct-olddb dbstruct))
	;; (runid  (dbr:dbstruct-run-id dbstruct))
	)
    (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id)
    ;; (mutex-lock! *http-mutex*)
    (if (eq? run-id 0)
	;; runid equal to 0 is main.db
	(if maindb
	    (if (or (not (number? mtime))
		    (not (number? stime))
		    (> mtime stime)
		    force-sync)
		(begin
		  (db:delay-if-busy maindb)
		  (db:delay-if-busy olddb)
		  (let ((num-synced (db:sync-tables (db:sync-main-list maindb) maindb olddb)))
		    (dbr:dbstruct-stime-set! dbstruct (current-milliseconds))
		    num-synced)
		  0))
	    (begin
	      ;; this can occur when using local access (i.e. not in a server)
	      ;; need a flag to turn it off.
	      ;;
	      (debug:print 3 *default-log-port* "WARNING: call to sync main.db to megatest.db but main not initialized")
	      0))
	;; any other runid is a run
	(if (or (not (number? mtime))
		(not (number? stime))
		(> mtime stime)
		force-sync)
	    (begin
	      (db:delay-if-busy rundb)
	      (db:delay-if-busy olddb)
	      (dbr:dbstruct-stime-set! dbstruct (current-milliseconds))
	      (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb olddb)))
		;; (mutex-unlock! *http-mutex*)
		num-synced)
	      (begin
		;; (mutex-unlock! *http-mutex*)
		0))))))

(define (db:close-main dbstruct)
  (let ((maindb (dbr:dbstruct-main dbstruct)))
    (if maindb
	(begin
	  (sqlite3:finalize! (db:dbdat-get-db maindb))
	  (dbr:dbstruct-main-set! dbstruct #f)))))

(define (db:close-run-db dbstruct run-id)
  (let ((rdb (db:open-rundb dbstruct run-id do-not-open: #t)))
    (if (and rdb
	     (sqlite3:database? rdb))
	(begin
	  (sqlite3:finalize! rdb)
	  (dbr:dbstruct-localdb-set! dbstruct run-id #f)
	  (dbr:dbstruct-inmem-set! dbstruct #f)))))

;; close all opened run-id dbs
(define (db:close-all dbstruct)
  ;; finalize main.db
  (db:sync-touched dbstruct 0 force-sync: #t)
  ;;(common:db-block-further-queries)
  ;; (mutex-lock! *db-sync-mutex*) ;; with this perhaps it isn't necessary to use the block-further-queries mechanism?

  (db:close-main dbstruct)
  
  (let ((locdbs (dbr:dbstruct-locdbs dbstruct)))
    (if (hash-table? locdbs)
	(for-each (lambda (run-id)
		    (db:close-run-db dbstruct run-id))
		  (hash-table-keys locdbs)))))

(define (db:open-inmem-db)
  (let* ((db      (sqlite3:open-database ":memory:"))
689
690
691
692
693
694
695
































































696
697
698
699
700
701
702
	      (set! tot-count (+ tot-count count))
	      (if (> count 0)
		  (if should-print (debug:print 0 *default-log-port* (format #f "    ~10a ~5a" tblname count))))))
	  (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
       tot-count)))
   (mutex-unlock! *db-sync-mutex*)))

































































;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db records to .db/{main,1,2 ...}.db
;;  'new2old      - sync .db/{main,1,2,3 ...}.db to megatest.db







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
	      (set! tot-count (+ tot-count count))
	      (if (> count 0)
		  (if should-print (debug:print 0 *default-log-port* (format #f "    ~10a ~5a" tblname count))))))
	  (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
       tot-count)))
   (mutex-unlock! *db-sync-mutex*)))


(define (db:patch-schema-rundb run-id frundb)
  ;;
  ;; remove this some time after September 2016 (added in version v1.6031
  ;;
  (for-each
   (lambda (table-name)
     (handle-exceptions
      exn
      (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn))
          (debug:print 0 *default-log-port* "Column last_update already added to " table-name " table")
          (db:general-sqlite-error-dump exn "alter table " table-name " ..." #f "none"))
      (sqlite3:execute
       frundb
       (conc "ALTER TABLE " table-name " ADD COLUMN last_update INTEGER DEFAULT 0")))
     (sqlite3:execute
      frundb
      (conc "DROP TRIGGER IF EXISTS update_" table-name "_trigger;"))
     (sqlite3:execute
      frundb
      (conc "CREATE TRIGGER IF NOT EXISTS update_" table-name "_trigger AFTER UPDATE ON " table-name "
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE " table-name " SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;"))
     )
   '("tests" "test_steps" "test_data")))

(define (db:patch-schema-maindb run-id maindb)
  ;;
  ;; remove all these some time after september 2016 (added in v1.6031
  ;;
  (handle-exceptions
   exn
   (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn))
       (debug:print 0 *default-log-port* "Column last_update already added to runs table")
       (db:general-sqlite-error-dump exn "alter table runs ..." run-id "none"))
   (sqlite3:execute
    maindb
    "ALTER TABLE runs ADD COLUMN last_update INTEGER DEFAULT 0"))
  ;; these schema changes don't need exception handling
  (sqlite3:execute
   maindb
   "CREATE TRIGGER IF NOT EXISTS update_runs_trigger AFTER UPDATE ON runs
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE runs SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;")
  (sqlite3:execute maindb "CREATE TABLE IF NOT EXISTS run_stats (
                              id     INTEGER PRIMARY KEY,
                              run_id INTEGER,
                              state  TEXT,
                              status TEXT,
                              count  INTEGER,
                              last_update INTEGER DEFAULT (strftime('%s','now')))")
  (sqlite3:execute maindb "CREATE TRIGGER  IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE run_stats SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;"))

;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db records to .db/{main,1,2 ...}.db
;;  'new2old      - sync .db/{main,1,2,3 ...}.db to megatest.db
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766

767



768
769


770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787

788
789
790
791
792
793
794


795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811


812




813
814
815



816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
	  (for-each 
	   (lambda (run-id)
	     (db:delay-if-busy mtdb)
	     (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id))
		   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)))
	       (debug:print 0 *default-log-port* "INFO: Propagating " (length testrecs) " records for run-id=" run-id " to run specific db")
	       (db:replace-test-records dbstruct run-id testrecs)
	       (sqlite3:finalize! (db:dbdat-get-db (dbr:dbstruct-get-rundb dbstruct)))))
	   run-ids)))

    ;; now ensure all newdb data are synced to megatest.db
    ;; do not use the run-ids list passed in to the function
    ;;
    (if (member 'new2old options)
	(let* ((maindb      (make-dbr:dbstruct path: toppath local: #t))
	       (src-run-ids (if run-ids run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0)))))
	       (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <))
	       (count       1)
	       (total       (length all-run-ids))
	       (dead-runs  '()))

	  (for-each



	   (lambda (run-id)
	     (debug:print 0 *default-log-port* "Processing run " (if (eq? run-id 0) " main.db " run-id) ", " count " of " total)


	     (set! count (+ count 1))
	     (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))
		    (frundb (db:dbdat-get-db (db:get-db fromdb run-id))))
	       ;; (db:delay-if-busy frundb)
	       ;; (db:delay-if-busy mtdb)
	       ;; (db:clean-up frundb)
	       (if (eq? run-id 0)
		   (let ((maindb  (db:dbdat-get-db (db:get-db fromdb #f))))
		     (db:sync-tables (db:sync-main-list dbstruct) (db:get-db fromdb #f) mtdb)
		     (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f)))
		     ;; 
		     ;; Feb 18, 2016: add field last_update to runs table
		     ;;
		     ;; remove all these some time after september 2016 (added in v1.6031
		     ;;
		     (handle-exceptions
		      exn
		      (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn))

			  (debug:print 0 *default-log-port* "Column last_update already added to runs table")
			  (db:general-sqlite-error-dump exn "alter table runs ..." run-id "none"))
		      (sqlite3:execute
		       maindb
		       "ALTER TABLE runs ADD COLUMN last_update INTEGER DEFAULT 0"))
		     ;; these schema changes don't need exception handling
		     (sqlite3:execute


		      maindb
		      "CREATE TRIGGER IF NOT EXISTS update_runs_trigger AFTER UPDATE ON runs
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE runs SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;")
		     (sqlite3:execute maindb "CREATE TABLE IF NOT EXISTS run_stats (
                              id     INTEGER PRIMARY KEY,
                              run_id INTEGER,
                              state  TEXT,
                              status TEXT,
                              count  INTEGER,
                              last_update INTEGER DEFAULT (strftime('%s','now')))")
		     (sqlite3:execute maindb "CREATE TRIGGER  IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats
                             FOR EACH ROW
                               BEGIN 


                                 UPDATE run_stats SET last_update=(strftime('%s','now'))




                                   WHERE id=old.id;
                               END;")
		     )



		   (begin
		     ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db
		     (db:sync-tables db:sync-tests-only (db:get-db fromdb run-id) mtdb)
		     (db:clean-up-rundb (db:get-db fromdb run-id))
		     ;;
		     ;; Feb 18, 2016: add field last_update to tests, test_steps and test_data
		     ;;
		     ;; remove this some time after September 2016 (added in version v1.6031
		     ;;
		     (for-each
		      (lambda (table-name)
			(handle-exceptions
			 exn
			 (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn))
			     (debug:print 0 *default-log-port* "Column last_update already added to " table-name " table")
			     (db:general-sqlite-error-dump exn "alter table " table-name " ..." #f "none"))
			 (sqlite3:execute
			  frundb
			  (conc "ALTER TABLE " table-name " ADD COLUMN last_update INTEGER DEFAULT 0")))
			(sqlite3:execute
			 frundb
			 (conc "DROP TRIGGER IF EXISTS update_" table-name "_trigger;"))
			(sqlite3:execute
			 frundb
			 (conc "CREATE TRIGGER IF NOT EXISTS update_" table-name "_trigger AFTER UPDATE ON " table-name "
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE " table-name " SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;"))
			)
		      '("tests" "test_steps" "test_data"))))))
	   all-run-ids)
	  ;; removed deleted runs
	  (let ((dbdir (tasks:get-task-db-path)))
	    (for-each (lambda (run-id)
			(let ((fullname (conc dbdir "/" run-id ".db")))
			  (if (file-exists? fullname)
			      (begin
				(debug:print 0 *default-log-port* "Removing database file for deleted run " fullname)
				(delete-file fullname)))))







|












>
|
>
>
>
|
<
>
>
|
|
|
<
<
<
|
|
<
|
<
|
<
<
<
<
<
<
>
|
|
<
|
<
<
|
>
>
|
<
<
<
<
<
<
<
<
<
<
<
<
|
|
<
<
>
>
|
>
>
>
>
|
<
<
>
>
>
|
|
|
|
<
<
<
<
<
|
<
<
<
<
|
<
<
|
<
<
<
|
<
|
<
<
<
<
<
<
|
<
<
|







853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878

879
880
881
882
883



884
885

886

887






888
889
890

891


892
893
894
895












896
897


898
899
900
901
902
903
904
905


906
907
908
909
910
911
912





913




914


915



916

917






918


919
920
921
922
923
924
925
926
	  (for-each 
	   (lambda (run-id)
	     (db:delay-if-busy mtdb)
	     (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id))
		   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)))
	       (debug:print 0 *default-log-port* "INFO: Propagating " (length testrecs) " records for run-id=" run-id " to run specific db")
	       (db:replace-test-records dbstruct run-id testrecs)
	       (sqlite3:finalize! (db:dbdat-get-db (dbr:dbstruct-rundb dbstruct)))))
	   run-ids)))

    ;; now ensure all newdb data are synced to megatest.db
    ;; do not use the run-ids list passed in to the function
    ;;
    (if (member 'new2old options)
	(let* ((maindb      (make-dbr:dbstruct path: toppath local: #t))
	       (src-run-ids (if run-ids run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0)))))
	       (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <))
	       (count       1)
	       (total       (length all-run-ids))
	       (dead-runs  '()))
          ;; first fix schema if needed
          (map
           (lambda (th)
             (thread-join! th))
           (map
            (lambda (run-id)

              (thread-start! 
               (make-thread
                (lambda ()
                  (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))
                         (frundb (db:dbdat-get-db (db:get-db fromdb run-id))))



                    (if (eq? run-id 0)
                        (let ((maindb  (db:dbdat-get-db (db:get-db fromdb #f))))

                          (db:patch-schema-maindb run-id maindb))

                        (db:patch-schema-rundb run-id frundb)))






                  (set! count (+ count 1))
                  (debug:print 0 *default-log-port* "Finished patching schema for " (if (eq? run-id 0) " main.db " (conc run-id ".db")) ", " count " of " total)))))
            all-run-ids))

          ;; Then sync and fix db's


          (set! count 0)
          (process-fork
           (lambda ()
             (map












              (lambda (th)
                (thread-join! th))


              (map
               (lambda (run-id)
                 (thread-start! 
                  (make-thread
                   (lambda ()
                     (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))
                            (frundb (db:dbdat-get-db (db:get-db fromdb run-id))))
                       (if (eq? run-id 0)


                           (let ((maindb  (db:dbdat-get-db (db:get-db fromdb #f))))
                             (db:sync-tables (db:sync-main-list dbstruct) (db:get-db fromdb #f) mtdb)
                             (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f))))
                           (begin
                             ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db
                             (db:sync-tables db:sync-tests-only (db:get-db fromdb run-id) mtdb)
                             (db:clean-up-rundb (db:get-db fromdb run-id)))))





                     (set! count (+ count 1))




                     (debug:print 0 *default-log-port* "Finished clean up of "


                                  (if (eq? run-id 0)



                                      " main.db " (conc run-id ".db")) ", " count " of " total)))))

               all-run-ids))))









          ;; removed deleted runs
	  (let ((dbdir (tasks:get-task-db-path)))
	    (for-each (lambda (run-id)
			(let ((fullname (conc dbdir "/" run-id ".db")))
			  (if (file-exists? fullname)
			      (begin
				(debug:print 0 *default-log-port* "Removing database file for deleted run " fullname)
				(delete-file fullname)))))
1332
1333
1334
1335
1336
1337
1338





1339
1340
1341
1342
1343
1344
1345
    
    (debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, " (length toplevels) " old LAUNCHED toplevel tests and " (length incompleted) " tests marked RUNNING but apparently dead.")
    (if (and (null? incompleted)
	     (null? oldlaunched)
	     (null? toplevels))
	#f
	#t)))






;;  select end_time-now from
;;      (select testname,item_path,event_time+run_duration as
;;                          end_time,strftime('%s','now') as now from tests where state in
;;      ('RUNNING','REMOTEHOSTSTART','LAUNCED'));

(define (db:find-and-mark-incomplete dbstruct run-id ovr-deadtime)







>
>
>
>
>







1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
    
    (debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, " (length toplevels) " old LAUNCHED toplevel tests and " (length incompleted) " tests marked RUNNING but apparently dead.")
    (if (and (null? incompleted)
	     (null? oldlaunched)
	     (null? toplevels))
	#f
	#t)))

;; given a launch delay (minimum time from last launch) return amount of time to wait
;;
;; (define (db:launch-delay-left dbstruct run-id launch-delay)
  

;;  select end_time-now from
;;      (select testname,item_path,event_time+run_duration as
;;                          end_time,strftime('%s','now') as now from tests where state in
;;      ('RUNNING','REMOTEHOSTSTART','LAUNCED'));

(define (db:find-and-mark-incomplete dbstruct run-id ovr-deadtime)
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
;; Use: (db:get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
;;  to extract info from the structure returned
;;
(define (db:get-runs-by-patt dbstruct keys runnamepatt targpatt offset limit fields last-update) ;; test-name)
  (let* ((tmp      (runs:get-std-run-fields keys (or fields '("id" "runname" "state" "status" "owner" "event_time"))))
	 (keystr   (car tmp))
	 (header   (cadr tmp))
	 (res     '())
	 (key-patt "")
	 (runwildtype (if (substring-index "%" runnamepatt) "like" "glob"))
	 (qry-str  #f)
	 (keyvals  (if targpatt (keys:target->keyval keys targpatt) '())))
    (for-each (lambda (keyval)
		(let* ((key    (car keyval))
		       (patt   (cadr keyval))







<







2039
2040
2041
2042
2043
2044
2045

2046
2047
2048
2049
2050
2051
2052
;; Use: (db:get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
;;  to extract info from the structure returned
;;
(define (db:get-runs-by-patt dbstruct keys runnamepatt targpatt offset limit fields last-update) ;; test-name)
  (let* ((tmp      (runs:get-std-run-fields keys (or fields '("id" "runname" "state" "status" "owner" "event_time"))))
	 (keystr   (car tmp))
	 (header   (cadr tmp))

	 (key-patt "")
	 (runwildtype (if (substring-index "%" runnamepatt) "like" "glob"))
	 (qry-str  #f)
	 (keyvals  (if targpatt (keys:target->keyval keys targpatt) '())))
    (for-each (lambda (keyval)
		(let* ((key    (car keyval))
		       (patt   (cadr keyval))
1989
1990
1991
1992
1993
1994
1995


1996
1997
1998
1999
2000

2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
			       (conc " AND last_update >= " last-update " ")
			       " ")
			" ORDER BY event_time "
			(if limit  (conc " LIMIT " limit)   "")
			(if offset (conc " OFFSET " offset) "")
			";"))
    (debug:print-info 4 *default-log-port* "runs:get-runs-by-patt qry=" qry-str " " runnamepatt)


    (db:with-db dbstruct #f #f ;; reads db, does not write to it.
		(lambda (db)
		  (sqlite3:for-each-row
		   (lambda (a . r)
		     (set! res (cons (list->vector (cons a r)) res)))

		   db
		   qry-str
		   runnamepatt)))
    (vector header res)))

;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
(define (db:get-run-info dbstruct run-id)
  ;;(if (hash-table-ref/default *run-info-cache* run-id #f)
  ;;    (hash-table-ref *run-info-cache* run-id)
  (let* ((dbdat     (db:get-db dbstruct #f))
	 (db        (db:dbdat-get-db dbdat))







>
>
|
|
|
|
|
>
|
|
|
<







2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080

2081
2082
2083
2084
2085
2086
2087
			       (conc " AND last_update >= " last-update " ")
			       " ")
			" ORDER BY event_time "
			(if limit  (conc " LIMIT " limit)   "")
			(if offset (conc " OFFSET " offset) "")
			";"))
    (debug:print-info 4 *default-log-port* "runs:get-runs-by-patt qry=" qry-str " " runnamepatt)
    (vector header 
            (reverse
             (db:with-db dbstruct #f #f ;; reads db, does not write to it.
                         (lambda (db)
                           (sqlite3:fold-row
                            (lambda (res . r)
                              (cons (list->vector r) res))
                            '()
                            db
                            qry-str
                            runnamepatt)))))))


;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
(define (db:get-run-info dbstruct run-id)
  ;;(if (hash-table-ref/default *run-info-cache* run-id #f)
  ;;    (hash-table-ref *run-info-cache* run-id)
  (let* ((dbdat     (db:get-db dbstruct #f))
	 (db        (db:dbdat-get-db dbdat))
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
    ;; ((fs) obj)
    ((http fs)
     (string-substitute
      (regexp "=") "_"
      (base64:base64-encode 
       (z3:encode-buffer
	(with-output-to-string
	  (lambda ()(serialize obj)))))
      #t))
    ((zmq nmsg)(with-output-to-string (lambda ()(serialize obj))))
    (else obj)))

(define (db:string->obj msg #!key (transport 'http))
  (case transport
    ;; ((fs) msg)
    ((http fs)
     (if (string? msg)
	 (with-input-from-string 
	     (z3:decode-buffer
	      (base64:base64-decode
	       (string-substitute 
		(regexp "_") "=" msg #t)))
	   (lambda ()(deserialize)))
	 (begin
	   (debug:print-error 0 *default-log-port* "reception failed. Received " msg " but cannot translate it.")
	   msg))) ;; crude reply for when things go awry
    ((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize))))
    (else msg)))

(define (db:test-set-status-state dbstruct run-id test-id status state msg)
  (let ((dbdat  (db:get-db dbstruct run-id)))
    (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
	(db:general-call dbdat 'set-test-start-time (list test-id)))
    (if msg
	(db:general-call dbdat 'state-status-msg (list state status msg test-id))







|


|
















|







3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
    ;; ((fs) obj)
    ((http fs)
     (string-substitute
      (regexp "=") "_"
      (base64:base64-encode 
       (z3:encode-buffer
	(with-output-to-string
	  (lambda ()(serialize obj))))) ;; BB: serialize - this is what causes problems between different builds of megatest communicating.  serialize is sensitive to binary image of mtest.
      #t))
    ((zmq nmsg)(with-output-to-string (lambda ()(serialize obj))))
    (else obj))) ;; rpc

(define (db:string->obj msg #!key (transport 'http))
  (case transport
    ;; ((fs) msg)
    ((http fs)
     (if (string? msg)
	 (with-input-from-string 
	     (z3:decode-buffer
	      (base64:base64-decode
	       (string-substitute 
		(regexp "_") "=" msg #t)))
	   (lambda ()(deserialize)))
	 (begin
	   (debug:print-error 0 *default-log-port* "reception failed. Received " msg " but cannot translate it.")
	   msg))) ;; crude reply for when things go awry
    ((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize))))
    (else msg))) ;; rpc

(define (db:test-set-status-state dbstruct run-id test-id status state msg)
  (let ((dbdat  (db:get-db dbstruct run-id)))
    (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))
	(db:general-call dbdat 'set-test-start-time (list test-id)))
    (if msg
	(db:general-call dbdat 'state-status-msg (list state status msg test-id))
3347
3348
3349
3350
3351
3352
3353



3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
			   ;; this test is younger, store it in the hash
			   (hash-table-set! tests-hash full-testname testdat))))
		   results)
		  (if (null? tal)
		      (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests
		      (loop (car tal)(cdr tal))))))))))




(define (db:delay-if-busy dbdat #!key (count 6))
  (if (not (configf:lookup *configdat* "server" "delay-on-busy"))
      (and dbdat (db:dbdat-get-db dbdat))
      (if dbdat
	  (let* ((dbpath (db:dbdat-get-path dbdat))
		 (db     (db:dbdat-get-db   dbdat)) ;; we'll return this so (db:delay--if-busy can be called inline
		 (dbfj   (conc dbpath "-journal")))
	    (if (handle-exceptions
		 exn
		 (begin
		   (debug:print-info 0 *default-log-port* "WARNING: failed to test for existance of " dbfj)
		   (thread-sleep! 1)
		   (db:delay-if-busy count (- count 1)))
		 (file-exists? dbfj))
		(case count
		  ((6)
		   (thread-sleep! 0.2)
		   (db:delay-if-busy count: 5))
		  ((5)
		   (thread-sleep! 0.4)







>
>
>

|










|







3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
			   ;; this test is younger, store it in the hash
			   (hash-table-set! tests-hash full-testname testdat))))
		   results)
		  (if (null? tal)
		      (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests
		      (loop (car tal)(cdr tal))))))))))

;; Function recursively checks if <db>.journal exists; if yes means db busy; call itself after delayed interval
;; return the sqlite3 db handle if possible
;; 
(define (db:delay-if-busy dbdat #!key (count 6))
  (if (not (configf:lookup *configdat* "server" "delay-on-busy")) 
      (and dbdat (db:dbdat-get-db dbdat))
      (if dbdat
	  (let* ((dbpath (db:dbdat-get-path dbdat))
		 (db     (db:dbdat-get-db   dbdat)) ;; we'll return this so (db:delay--if-busy can be called inline
		 (dbfj   (conc dbpath "-journal")))
	    (if (handle-exceptions
		 exn
		 (begin
		   (debug:print-info 0 *default-log-port* "WARNING: failed to test for existance of " dbfj)
		   (thread-sleep! 1)
		   (db:delay-if-busy count (- count 1))) 
		 (file-exists? dbfj))
		(case count
		  ((6)
		   (thread-sleep! 0.2)
		   (db:delay-if-busy count: 5))
		  ((5)
		   (thread-sleep! 0.4)
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
		   (db:delay-if-busy count: 1))
		  ((1)
		   (thread-sleep! 6.4)
		   (db:delay-if-busy count: 0))
		  (else
		   (debug:print-info 0 *default-log-port* "delaying db access due to high database load.")
		   (thread-sleep! 12.8))))
	    db)
	  "bogus result from db:delay-if-busy")))

(define (db:test-get-records-for-index-file dbstruct run-id test-name)
  (let ((res '()))
    (db:with-db
     dbstruct
     run-id







|







3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
		   (db:delay-if-busy count: 1))
		  ((1)
		   (thread-sleep! 6.4)
		   (db:delay-if-busy count: 0))
		  (else
		   (debug:print-info 0 *default-log-port* "delaying db access due to high database load.")
		   (thread-sleep! 12.8))))
	    db) 
	  "bogus result from db:delay-if-busy")))

(define (db:test-get-records-for-index-file dbstruct run-id test-name)
  (let ((res '()))
    (db:with-db
     dbstruct
     run-id

Modified db_records.scm from [f90e27c50c] to [ebae0b2ffd].

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


62
63
64
65
66
67
68
69
70
71
72
73
;;            |-1.db
;;            |-<N>.db
;;
;;
;; Accessors for a dbstruct
;;

(define-inline (dbr:dbstruct-get-main    vec)    (vector-ref  vec 0)) ;; ( db path )
(define-inline (dbr:dbstruct-get-strdb   vec)    (vector-ref  vec 1)) ;; ( db path )
(define-inline (dbr:dbstruct-get-path    vec)    (vector-ref  vec 2)) 
(define-inline (dbr:dbstruct-get-local   vec)    (vector-ref  vec 3))
(define-inline (dbr:dbstruct-get-rundb   vec)    (vector-ref  vec 4)) ;; ( db path )
(define-inline (dbr:dbstruct-get-inmem   vec)    (vector-ref  vec 5)) ;; ( db #f )
(define-inline (dbr:dbstruct-get-mtime   vec)    (vector-ref  vec 6))
(define-inline (dbr:dbstruct-get-rtime   vec)    (vector-ref  vec 7))
(define-inline (dbr:dbstruct-get-stime   vec)    (vector-ref  vec 8))
(define-inline (dbr:dbstruct-get-inuse   vec)    (vector-ref  vec 9))
(define-inline (dbr:dbstruct-get-refdb   vec)    (vector-ref  vec 10)) ;; ( db path )
(define-inline (dbr:dbstruct-get-locdbs  vec)    (vector-ref  vec 11))
(define-inline (dbr:dbstruct-get-olddb   vec)    (vector-ref  vec 12)) ;; ( db path )
;; (define-inline (dbr:dbstruct-get-main-path vec)  (vector-ref  vec 13))
;; (define-inline (dbr:dbstruct-get-rundb-path vec) (vector-ref  vec 14))
;; (define-inline (dbr:dbstruct-get-run-id  vec)    (vector-ref  vec 13))

(define-inline (dbr:dbstruct-set-main!   vec val)(vector-set! vec 0 val))
(define-inline (dbr:dbstruct-set-strdb!  vec val)(vector-set! vec 1 val))
(define-inline (dbr:dbstruct-set-path!   vec val)(vector-set! vec 2 val))
(define-inline (dbr:dbstruct-set-local!  vec val)(vector-set! vec 3 val))
(define-inline (dbr:dbstruct-set-rundb!  vec val)(vector-set! vec 4 val))
(define-inline (dbr:dbstruct-set-inmem!  vec val)(vector-set! vec 5 val))
(define-inline (dbr:dbstruct-set-mtime!  vec val)(vector-set! vec 6 val))
(define-inline (dbr:dbstruct-set-rtime!  vec val)(vector-set! vec 7 val))
(define-inline (dbr:dbstruct-set-stime!  vec val)(vector-set! vec 8 val))
(define-inline (dbr:dbstruct-set-inuse!  vec val)(vector-set! vec 9 val))
(define-inline (dbr:dbstruct-set-refdb!  vec val)(vector-set! vec 10 val))
(define-inline (dbr:dbstruct-set-locdbs! vec val)(vector-set! vec 11 val))
(define-inline (dbr:dbstruct-set-olddb!  vec val)(vector-set! vec 12 val))
(define-inline (dbr:dbstruct-set-main-path! vec val)(vector-set! vec 13 val))
(define-inline (dbr:dbstruct-set-rundb-path! vec val)(vector-set! vec 14 val))

; (define-inline (dbr:dbstruct-set-run-id! vec val)(vector-set! vec 13 val))

;; constructor for dbstruct
;;
(define (make-dbr:dbstruct #!key (path #f)(local #f))
  (let ((v (make-vector 15 #f)))
    (dbr:dbstruct-set-path! v path)
    (dbr:dbstruct-set-local! v local)
    (dbr:dbstruct-set-locdbs! v (make-hash-table))
    v))



(define (dbr:dbstruct-get-localdb v run-id)
  (hash-table-ref/default (dbr:dbstruct-get-locdbs v) run-id #f))

(define (dbr:dbstruct-set-localdb! v run-id db)
  (hash-table-set! (dbr:dbstruct-get-locdbs v) run-id db))


(define (make-db:test)(make-vector 20))
(define-inline (db:test-get-id           vec) (vector-ref vec 0))
(define-inline (db:test-get-run_id       vec) (vector-ref vec 1))
(define-inline (db:test-get-testname     vec) (vector-ref vec 2))
(define-inline (db:test-get-state        vec) (vector-ref vec 3))







|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|



|
|
|
|
|
|

>
>
|
|

|
|







11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
;;            |-1.db
;;            |-<N>.db
;;
;;
;; Accessors for a dbstruct
;;

;; (define-inline (dbr:dbstruct-main    vec)    (vector-ref  vec 0)) ;; ( db path )
;; (define-inline (dbr:dbstruct-strdb   vec)    (vector-ref  vec 1)) ;; ( db path )
;; (define-inline (dbr:dbstruct-path    vec)    (vector-ref  vec 2)) 
;; (define-inline (dbr:dbstruct-local   vec)    (vector-ref  vec 3))
;; (define-inline (dbr:dbstruct-rundb   vec)    (vector-ref  vec 4)) ;; ( db path )
;; (define-inline (dbr:dbstruct-inmem   vec)    (vector-ref  vec 5)) ;; ( db #f )
;; (define-inline (dbr:dbstruct-mtime   vec)    (vector-ref  vec 6))
;; (define-inline (dbr:dbstruct-rtime   vec)    (vector-ref  vec 7))
;; (define-inline (dbr:dbstruct-stime   vec)    (vector-ref  vec 8))
;; (define-inline (dbr:dbstruct-inuse   vec)    (vector-ref  vec 9))
;; (define-inline (dbr:dbstruct-refdb   vec)    (vector-ref  vec 10)) ;; ( db path )
;; (define-inline (dbr:dbstruct-locdbs  vec)    (vector-ref  vec 11))
;; (define-inline (dbr:dbstruct-olddb   vec)    (vector-ref  vec 12)) ;; ( db path )
;; ;; (define-inline (dbr:dbstruct-main-path vec)  (vector-ref  vec 13))
;; ;; (define-inline (dbr:dbstruct-rundb-path vec) (vector-ref  vec 14))
;; ;; (define-inline (dbr:dbstruct-run-id  vec)    (vector-ref  vec 13))
;; 
;; (define-inline (dbr:dbstruct-main-set!   vec val)(vector-set! vec 0 val))
;; (define-inline (dbr:dbstruct-strdb-set!  vec val)(vector-set! vec 1 val))
;; (define-inline (dbr:dbstruct-path-set!   vec val)(vector-set! vec 2 val))
;; (define-inline (dbr:dbstruct-local-set!  vec val)(vector-set! vec 3 val))
;; (define-inline (dbr:dbstruct-rundb-set!  vec val)(vector-set! vec 4 val))
;; (define-inline (dbr:dbstruct-inmem-set!  vec val)(vector-set! vec 5 val))
;; (define-inline (dbr:dbstruct-mtime-set!  vec val)(vector-set! vec 6 val))
;; (define-inline (dbr:dbstruct-rtime-set!  vec val)(vector-set! vec 7 val))
;; (define-inline (dbr:dbstruct-stime-set!  vec val)(vector-set! vec 8 val))
;; (define-inline (dbr:dbstruct-inuse-set!  vec val)(vector-set! vec 9 val))
;; (define-inline (dbr:dbstruct-refdb-set!  vec val)(vector-set! vec 10 val))
;; (define-inline (dbr:dbstruct-locdbs-set! vec val)(vector-set! vec 11 val))
;; (define-inline (dbr:dbstruct-olddb-set!  vec val)(vector-set! vec 12 val))
;; (define-inline (dbr:dbstruct-main-path-set! vec val)(vector-set! vec 13 val))
;; (define-inline (dbr:dbstruct-rundb-path-set! vec val)(vector-set! vec 14 val))
;; 
; (define-inline (dbr:dbstruct-run-id-set! vec val)(vector-set! vec 13 val))

;; constructor for dbstruct
;;
;; (define (make-dbr:dbstruct #!key (path #f)(local #f))
;;   (let ((v (make-vector 15 #f)))
;;     (dbr:dbstruct-path-set! v path)
;;     (dbr:dbstruct-local-set! v local)
;;     (dbr:dbstruct-locdbs-set! v (make-hash-table))
;;     v))

;; Returns the database for a particular run-id fron the dbstruct:localdbs
;;
(define (dbr:dbstruct-localdb v run-id)
  (hash-table-ref/default (dbr:dbstruct-locdbs v) run-id #f))

(define (dbr:dbstruct-localdb-set! v run-id db)
  (hash-table-set! (dbr:dbstruct-locdbs v) run-id db))


(define (make-db:test)(make-vector 20))
(define-inline (db:test-get-id           vec) (vector-ref vec 0))
(define-inline (db:test-get-run_id       vec) (vector-ref vec 1))
(define-inline (db:test-get-testname     vec) (vector-ref vec 2))
(define-inline (db:test-get-state        vec) (vector-ref vec 3))
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

118
119
120
121
122
123
124
  (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))

;; replace runs:make-full-test-name with this routine
(define (db:test-make-full-name testname itempath)
  (if (equal? itempath "") testname (conc testname "/" itempath)))

(define-inline (db:test-get-first_err    vec) (printable (vector-ref vec 15)))
(define-inline (db:test-get-first_warn   vec) (printable (vector-ref vec 16)))

(define-inline (db:test-set-cpuload!  vec val)(vector-set! vec 7 val))
(define-inline (db:test-set-diskfree! vec val)(vector-set! vec 8 val))
(define-inline (db:test-set-testname! vec val)(vector-set! vec 2 val))
(define-inline (db:test-set-state!    vec val)(vector-set! vec 3 val))
(define-inline (db:test-set-status!   vec val)(vector-set! vec 4 val))
(define-inline (db:test-set-run_duration! vec val)(vector-set! vec 12 val))
(define-inline (db:test-set-final_logf! vec val)(vector-set! vec 13 val))

;; Test record utility functions

;; Is a test a toplevel?
;;
(define (db:test-get-is-toplevel vec)
  (and (equal? (db:test-get-item-path vec) "")      ;; test is not an item
       (equal? (db:test-get-uname vec)     "n/a"))) ;; test has never been run

;; make-vector-record "" db mintest id run_id testname state status event_time item_path

;;
(define (make-db:mintest)(make-vector 7))
(define-inline (db:mintest-get-id           vec)    (vector-ref  vec 0))
(define-inline (db:mintest-get-run_id       vec)    (vector-ref  vec 1))
(define-inline (db:mintest-get-testname     vec)    (vector-ref  vec 2))
(define-inline (db:mintest-get-state        vec)    (vector-ref  vec 3))
(define-inline (db:mintest-get-status       vec)    (vector-ref  vec 4))







|


















>







94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
  (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))

;; replace runs:make-full-test-name with this routine
(define (db:test-make-full-name testname itempath)
  (if (equal? itempath "") testname (conc testname "/" itempath)))

(define-inline (db:test-get-first_err    vec) (printable (vector-ref vec 15)))
(define-inline (db:test-get-first_warn   vec) (printable (vector-ref vec 16))) ;; RADT => reference 16 is repeated

(define-inline (db:test-set-cpuload!  vec val)(vector-set! vec 7 val))
(define-inline (db:test-set-diskfree! vec val)(vector-set! vec 8 val))
(define-inline (db:test-set-testname! vec val)(vector-set! vec 2 val))
(define-inline (db:test-set-state!    vec val)(vector-set! vec 3 val))
(define-inline (db:test-set-status!   vec val)(vector-set! vec 4 val))
(define-inline (db:test-set-run_duration! vec val)(vector-set! vec 12 val))
(define-inline (db:test-set-final_logf! vec val)(vector-set! vec 13 val))

;; Test record utility functions

;; Is a test a toplevel?
;;
(define (db:test-get-is-toplevel vec)
  (and (equal? (db:test-get-item-path vec) "")      ;; test is not an item
       (equal? (db:test-get-uname vec)     "n/a"))) ;; test has never been run

;; make-vector-record "" db mintest id run_id testname state status event_time item_path
;; RADT => purpose of mintest??
;;
(define (make-db:mintest)(make-vector 7))
(define-inline (db:mintest-get-id           vec)    (vector-ref  vec 0))
(define-inline (db:mintest-get-run_id       vec)    (vector-ref  vec 1))
(define-inline (db:mintest-get-testname     vec)    (vector-ref  vec 2))
(define-inline (db:mintest-get-state        vec)    (vector-ref  vec 3))
(define-inline (db:mintest-get-status       vec)    (vector-ref  vec 4))

Modified dcommon.scm from [4022829d20] to [eb6ea73393].

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

29
30
31
32
33
34
35
;;======================================================================

(use format)
(require-library iup)
(import (prefix iup iup:))
(use canvas-draw)
(import canvas-draw-iup)
(use regex defstruct)

(declare (unit dcommon))

(declare (uses megatest-version))
(declare (uses gutils))
(declare (uses db))
(declare (uses synchash))

(include "common_records.scm")
(include "db_records.scm")
(include "key_records.scm")


;; yes, this is non-ideal 
(define dashboard:update-summary-tab #f)
(define dashboard:update-servers-table #f)

;;======================================================================
;; C O M M O N   D A T A   S T R U C T U R E







|











>







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
;;======================================================================

(use format)
(require-library iup)
(import (prefix iup iup:))
(use canvas-draw)
(import canvas-draw-iup)
(use regex typed-records)

(declare (unit dcommon))

(declare (uses megatest-version))
(declare (uses gutils))
(declare (uses db))
(declare (uses synchash))

(include "common_records.scm")
(include "db_records.scm")
(include "key_records.scm")
(include "run_records.scm")

;; yes, this is non-ideal 
(define dashboard:update-summary-tab #f)
(define dashboard:update-servers-table #f)

;;======================================================================
;; C O M M O N   D A T A   S T R U C T U R E
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
				  ;; (iup:attribute-set! tb (conc "COLOR" node-num) color)
				  )
				(hash-table-set! (dboard:tabdat-path-test-ids data) test-path test-id)
				(if (not rownum)
				    (let ((rownums (hash-table-values testname-to-row)))
				      (set! rownum (if (null? rownums)
						       1
						       (+ 1 (apply max rownums))))
				      (hash-table-set! testname-to-row fullname rownum)
				      ;; create the label
				      (set! changed (dcommon:modifiy-if-different 
						     (dboard:tabdat-runs-matrix data)
						     (conc rownum ":" 0)
						     dispname
						     changed))







|







214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
				  ;; (iup:attribute-set! tb (conc "COLOR" node-num) color)
				  )
				(hash-table-set! (dboard:tabdat-path-test-ids data) test-path test-id)
				(if (not rownum)
				    (let ((rownums (hash-table-values testname-to-row)))
				      (set! rownum (if (null? rownums)
						       1
						       (+ 1 (common:max rownums))))
				      (hash-table-set! testname-to-row fullname rownum)
				      ;; create the label
				      (set! changed (dcommon:modifiy-if-different 
						     (dboard:tabdat-runs-matrix data)
						     (conc rownum ":" 0)
						     dispname
						     changed))
257
258
259
260
261
262
263



































264
265
266
267
268
269
270
    (let ((updater (hash-table-ref/default  (dboard:commondat-updaters commondat) window-id #f)))
      (if updater (updater (hash-table-ref/default data get-details-sig #f))))

    (if changed (iup:attribute-set! (dboard:tabdat-runs-matrix data) "REDRAW" "ALL"))
    ;; (debug:print 2 *default-log-port* "run-changes: " run-changes)
    ;; (debug:print 2 *default-log-port* "test-changes: " test-changes)
    (list run-changes all-test-changes)))




































;;======================================================================
;; TESTS DATA
;;======================================================================

;; Produce a list of lists ready for common:sparse-list-generate-index
;;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    (let ((updater (hash-table-ref/default  (dboard:commondat-updaters commondat) window-id #f)))
      (if updater (updater (hash-table-ref/default data get-details-sig #f))))

    (if changed (iup:attribute-set! (dboard:tabdat-runs-matrix data) "REDRAW" "ALL"))
    ;; (debug:print 2 *default-log-port* "run-changes: " run-changes)
    ;; (debug:print 2 *default-log-port* "test-changes: " test-changes)
    (list run-changes all-test-changes)))

(define (dcommon:runsdat-get-col-num dat target runname force-set)
  (let* ((runs-index (dboard:runsdat-runs-index dat))
	 (col-name   (conc target "/" runname))
	 (res        (hash-table-ref/default runs-index col-name #f)))
    (if res
	res
	(if force-set
	    (let ((max-col-num (+ 1 (common:max (cons-1 (hash-table-values runs-index))))))
	      (hash-table-set! runs-index col-name max-col-num)
	      max-col-num)))))

(define (dcommon:runsdat-get-row-num dat testname itempath force-set)
  (let* ((tests-index (dboard:runsdat-runs-index dat))
	 (row-name    (conc testname "/" itempath))
	 (res         (hash-table-ref/default runs-index row-name #f)))
    (if res
	res
	(if force-set
	    (let ((max-row-num (+ 1 (common:max (cons -1 (hash-table-values tests-index))))))
	      (hash-table-set! runs-index row-name max-row-num)
	      max-row-num)))))

(define (dcommon:rundat-copy-tests-to-by-name rundat)
  (let ((src-ht (dboard:rundat-tests rundat))
	(trg-ht (dboard:rundat-tests-by-name rundat)))
    (if (and (hash-table? src-ht)(hash-table? trg-ht))
	(begin
	  (hash-table-clear! trg-ht)
	  (for-each
	   (lambda (testdat)
	     (hash-table-set! trg-ht (test:test-get-fullname testdat) testdat))
	   (hash-table-values src-ht)))
	(debug:print 0 *default-log-port* "WARNING: src-ht " src-ht " trg-ht " trg-ht))))
  

;;======================================================================
;; TESTS DATA
;;======================================================================

;; Produce a list of lists ready for common:sparse-list-generate-index
;;
279
280
281
282
283
284
285
286












































































































287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
	       (item-path  (db:test-get-item-path hed))
	       (state      (db:test-get-state hed))
	       (status     (db:test-get-status hed))
	       (newitem    (list test-name item-path (list test-id state status))))
	  (if (null? tal)
	      (reverse (cons newitem res))
	      (loop (car tal)(cdr tal)(cons newitem res)))))))













































































































(define (dcommon:examine-xterm run-id test-id)
  (let* ((testdat (rmt:get-test-info-by-id run-id test-id)))
    (if (not testdat)
	(begin
	  (debug:print 2 "ERROR: No test data found for test " test-id ", exiting")
	  (exit 1))
        (let*
            ((rundir        (if testdat 
				(db:test-get-rundir testdat)
				  logfile))
             (testfullname  (if testdat (db:test-get-fullname testdat) "Gathering data ..."))
             (xterm      (lambda ()
                           (if (directory-exists? rundir)
                               (let* ((shell (if (get-environment-variable "SHELL") 
                                                (conc "-e " (get-environment-variable "SHELL"))
                                                ""))
                                      (command (conc "cd " rundir 
                                                     ";mt_xterm -T \"" (string-translate testfullname "()" "  ") "\" " shell "&")))
                                 (print "Command =" command)
                                 (common:without-vars
                                  command
                                  "MT_.*"))
                               (message-window  (conc "Directory " rundir " not found"))))))
          (xterm)
          (print "Adding xterm code")))))

;;======================================================================
;; D A T A   T A B L E S
;;======================================================================

;; Table of keys
(define (dcommon:keys-matrix rawconfig)
  (let* ((curr-row-num 1)
	 (key-vals     (configf:section-vars rawconfig "fields"))
	 (keys-matrix  (iup:matrix
			#:alignment1 "ALEFT"
			#:expand "YES" ;; "HORIZONTAL" ;; "VERTICAL"
			;; #:scrollbar "YES"
			#:numcol 1
			#:numlin (length key-vals)
			#:numcol-visible 1
			#:numlin-visible (length key-vals)
			#:click-cb (lambda (obj lin col status)
				     (print "obj: " obj " lin: " lin " col: " col " status: " status)))))
    ;; (iup:attribute-set! keys-matrix "0:0" "Run Keys")
    (iup:attribute-set! keys-matrix "WIDTH0" 0)
    (iup:attribute-set! keys-matrix "0:1" "Key Name")
    ;; (iup:attribute-set! keys-matrix "WIDTH1" "100")
    ;; fill in keys
    (for-each 
     (lambda (var)
       ;; (iup:attribute-set! keys-matrix "ADDLIN" (conc curr-row-num))
       (iup:attribute-set! keys-matrix (conc curr-row-num ":0") curr-row-num)
       (iup:attribute-set! keys-matrix (conc curr-row-num ":1") var)
       (set! curr-row-num (+ 1 curr-row-num))) ;; (config-lookup *configdat* "fields" var)))
     key-vals)
    (iup:attribute-set! keys-matrix "WIDTHDEF" "40")
    keys-matrix))

;; Section to table
(define (dcommon:section-matrix rawconfig sectionname varcolname valcolname #!key (title #f))
  (let* ((curr-row-num    1)
	 (key-vals        (configf:section-vars rawconfig sectionname))
	 (section-matrix  (iup:matrix
			   #:alignment1 "ALEFT"
			   #:expand "YES" ;; "HORIZONTAL"
			   #:numcol 1
			   #:numlin (length key-vals)
			   #:numcol-visible 1
			   #:numlin-visible (min 10 (length key-vals))
			   #:scrollbar "YES")))
    (iup:attribute-set! section-matrix "0:0" varcolname)
    (iup:attribute-set! section-matrix "0:1" valcolname)
    (iup:attribute-set! section-matrix "WIDTH1" "200")
    ;; fill in keys
    (for-each 
     (lambda (var)








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



|
|
|


|
|




|
|

















|
|
|
|
|
|
|
|
|
|
|


















|
|
|
|
|
|
|
|







315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
	       (item-path  (db:test-get-item-path hed))
	       (state      (db:test-get-state hed))
	       (status     (db:test-get-status hed))
	       (newitem    (list test-name item-path (list test-id state status))))
	  (if (null? tal)
	      (reverse (cons newitem res))
	      (loop (car tal)(cdr tal)(cons newitem res)))))))

(define (dcommon:tests-mindat->hash tests-mindat)
  (let* ((res (make-hash-table)))
    (for-each
     (lambda (item)
       (let* ((test-name+item-path (cons (list-ref item 0) (list-ref item 1)))
              (value (list-ref item 2)))
         (hash-table-set! res test-name+item-path value)))
     tests-mindat)
    res))

;; return 1 if status1 is better
;; return 0 if status1 and 2 are equally good
;; return -1 if status2 is better
(define (dcommon:status-compare3 status1 status2)
  (let*
      ((status-goodness-ranking  (list "PASS" "WARN" "WAIVED" "SKIP" "FAIL" "ABORT" #f))
       (mem1 (member status1 status-goodness-ranking))
       (mem2 (member status2 status-goodness-ranking))
       )
    (cond
     ((and (not mem1) (not mem2)) 0)
     ((not mem1) -1)
     ((not mem2) 1)
     ((= (length mem1) (length mem2)) 0)
     ((> (length mem1) (length mem2)) 1)
     (else -1))))
     
(define (dcommon:xor-tests-mindat src-tests-mindat dest-tests-mindat #!key (hide-clean #f))
  (let* ((src-hash (dcommon:tests-mindat->hash src-tests-mindat))
         (dest-hash (dcommon:tests-mindat->hash dest-tests-mindat))
         (all-keys
          (reverse (sort 
           (delete-duplicates
            (append (hash-table-keys src-hash) (hash-table-keys dest-hash)))

           (lambda (a b) 
             (cond
              ((< 0 (string-compare3 (car a) (car b))) #t)
              ((> 0 (string-compare3 (car a) (car b))) #f)
              ((< 0 (string-compare3 (cdr a) (cdr b))) #t)
              (else #f)))

           ))))
    (let ((res
           (map ;; TODO: rename xor to delta globally in dcommon and dashboard
            (lambda (key)
              (let* ((test-name (car key))
                     (item-path (cdr key))

                     (dest-value (hash-table-ref/default dest-hash key #f)) ;; (list test-id state status)
                     (dest-test-id  (if dest-value (list-ref dest-value 0) #f))
                     (dest-state    (if dest-value (list-ref dest-value 1) #f))
                     (dest-status   (if dest-value (list-ref dest-value 2) #f))

                     (src-value     (hash-table-ref/default src-hash key #f))   ;; (list test-id state status)
                     (src-test-id   (if src-value (list-ref src-value 0) #f))
                     (src-state     (if src-value (list-ref src-value 1) #f))
                     (src-status    (if src-value (list-ref src-value 2) #f))

                     (incomplete-statuses '("DELETED" "INCOMPLETE" "STUCK/DEAD" "N/A")) ;; if any of these statuses apply, treat test as incomplete

                     (dest-complete
                      (and dest-value dest-state dest-status
                           (equal? dest-state "COMPLETED")
                           (not (member dest-status incomplete-statuses))))
                     (src-complete
                      (and src-value src-state src-status
                           (equal? src-state "COMPLETED")
                           (not (member src-status incomplete-statuses))))
                     (status-compare-result (dcommon:status-compare3 src-status dest-status))
                     (xor-new-item
                      (cond
                       ;; complete, for this case means: state=compelte AND status not in ( deleted uncomplete stuck/dead n/a )
                       ;; neither complete -> bad

                       ;; src !complete, dest complete -> better
                       ((and (not dest-complete) (not src-complete))
                        (list dest-test-id "BOTH-BAD" "BOTH-INCOMPLETE"))
                       ((not dest-complete)
                        (list src-test-id "DIFF-MISSING" "DEST-INCOMPLETE"))  
                       ((not src-complete)
                        (list dest-test-id "DIFF-NEW" "SRC-INCOMPLETE"))      
                       ((and
                         (equal? src-state dest-state)
                         (equal? src-status dest-status))
                        (list dest-test-id  (conc "CLEAN") (conc "CLEAN-" dest-status) )) 
                       ;;    better or worse: pass > warn > waived > skip > fail > abort
                       ;;     pass > warn > waived > skip > fail > abort
                       
                       ((= 1 status-compare-result) ;; src is better, dest is worse
                        (list dest-test-id "DIRTY-WORSE" (conc src-status "->" dest-status)))
                       (else
                        (list dest-test-id "DIRTY-BETTER" (conc src-status "->" dest-status)))
                       )))
                (list test-name item-path  xor-new-item)))
            all-keys)))

      (if hide-clean
          (filter
           (lambda (item)
             ;;(print item)
             (not
              (equal?
               "CLEAN"
               (list-ref (list-ref item 2) 1))))
           res)
          res))))

(define (dcommon:examine-xterm run-id test-id)
  (let* ((testdat (rmt:get-test-info-by-id run-id test-id)))
    (if (not testdat)
        (begin
          (debug:print 2 "ERROR: No test data found for test " test-id ", exiting")
          (exit 1))
        (let*
            ((rundir        (if testdat 
                                (db:test-get-rundir testdat)
                                logfile))
             (testfullname  (if testdat (db:test-get-fullname testdat) "Gathering data ..."))
             (xterm      (lambda ()
                           (if (directory-exists? rundir)
                               (let* ((shell (if (get-environment-variable "SHELL") 
                                                 (conc "-e " (get-environment-variable "SHELL"))
                                                 ""))
                                      (command (conc "cd " rundir 
                                                     ";mt_xterm -T \"" (string-translate testfullname "()" "  ") "\" " shell "&")))
                                 (print "Command =" command)
                                 (common:without-vars
                                  command
                                  "MT_.*"))
                               (message-window  (conc "Directory " rundir " not found"))))))
          (xterm)
          (print "Adding xterm code")))))

;;======================================================================
;; D A T A   T A B L E S
;;======================================================================

;; Table of keys
(define (dcommon:keys-matrix rawconfig)
  (let* ((curr-row-num 1)
         (key-vals     (configf:section-vars rawconfig "fields"))
         (keys-matrix  (iup:matrix
                        #:alignment1 "ALEFT"
                        #:expand "YES" ;; "HORIZONTAL" ;; "VERTICAL"
                        ;; #:scrollbar "YES"
                        #:numcol 1
                        #:numlin (length key-vals)
                        #:numcol-visible 1
                        #:numlin-visible (length key-vals)
                        #:click-cb (lambda (obj lin col status)
                                     (print "obj: " obj " lin: " lin " col: " col " status: " status)))))
    ;; (iup:attribute-set! keys-matrix "0:0" "Run Keys")
    (iup:attribute-set! keys-matrix "WIDTH0" 0)
    (iup:attribute-set! keys-matrix "0:1" "Key Name")
    ;; (iup:attribute-set! keys-matrix "WIDTH1" "100")
    ;; fill in keys
    (for-each 
     (lambda (var)
       ;; (iup:attribute-set! keys-matrix "ADDLIN" (conc curr-row-num))
       (iup:attribute-set! keys-matrix (conc curr-row-num ":0") curr-row-num)
       (iup:attribute-set! keys-matrix (conc curr-row-num ":1") var)
       (set! curr-row-num (+ 1 curr-row-num))) ;; (config-lookup *configdat* "fields" var)))
     key-vals)
    (iup:attribute-set! keys-matrix "WIDTHDEF" "40")
    keys-matrix))

;; Section to table
(define (dcommon:section-matrix rawconfig sectionname varcolname valcolname #!key (title #f))
  (let* ((curr-row-num    1)
         (key-vals        (configf:section-vars rawconfig sectionname))
         (section-matrix  (iup:matrix
                           #:alignment1 "ALEFT"
                           #:expand "YES" ;; "HORIZONTAL"
                           #:numcol 1
                           #:numlin (length key-vals)
                           #:numcol-visible 1
                           #:numlin-visible (min 10 (length key-vals))
			   #:scrollbar "YES")))
    (iup:attribute-set! section-matrix "0:0" varcolname)
    (iup:attribute-set! section-matrix "0:1" valcolname)
    (iup:attribute-set! section-matrix "WIDTH1" "200")
    ;; fill in keys
    (for-each 
     (lambda (var)
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418

    general-matrix))

(define (dcommon:run-stats commondat tabdat #!key (tab-num #f))
  (let* ((stats-matrix (iup:matrix expand: "YES"))
	 (changed      #f)
	 (stats-updater (lambda ()
			 (if (dashboard:database-changed? commondat tabdat)
			     (let* ((run-stats    (rmt:get-run-stats))
				    (indices      (common:sparse-list-generate-index run-stats)) ;;  proc: set-cell))
				    (row-indices  (car indices))
				    (col-indices  (cadr indices))
				    (max-row      (if (null? row-indices) 1 (apply max (map cadr row-indices))))
				    (max-col      (if (null? col-indices) 1 
						      (apply max (map cadr col-indices))))
				    (max-visible  (max (- (dboard:tabdat-num-tests tabdat) 15) 3))
				    (max-col-vis  (if (> max-col 10) 10 max-col))
				    (numrows      1)
				    (numcols      1))
			       (iup:attribute-set! stats-matrix "CLEARVALUE" "CONTENTS")
			       (iup:attribute-set! stats-matrix "NUMCOL" max-col )
			       (iup:attribute-set! stats-matrix "NUMLIN" (if (< max-row max-visible) max-visible max-row)) ;; min of 20







|




|

|







541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562

    general-matrix))

(define (dcommon:run-stats commondat tabdat #!key (tab-num #f))
  (let* ((stats-matrix (iup:matrix expand: "YES"))
	 (changed      #f)
	 (stats-updater (lambda ()
			 (if (dashboard:database-changed? commondat tabdat context-key: 'run-stats)
			     (let* ((run-stats    (rmt:get-run-stats))
				    (indices      (common:sparse-list-generate-index run-stats)) ;;  proc: set-cell))
				    (row-indices  (car indices))
				    (col-indices  (cadr indices))
				    (max-row      (if (null? row-indices) 1 (common:max (map cadr row-indices))))
				    (max-col      (if (null? col-indices) 1 
						      (common:max (map cadr col-indices))))
				    (max-visible  (max (- (dboard:tabdat-num-tests tabdat) 15) 3))
				    (max-col-vis  (if (> max-col 10) 10 max-col))
				    (numrows      1)
				    (numcols      1))
			       (iup:attribute-set! stats-matrix "CLEARVALUE" "CONTENTS")
			       (iup:attribute-set! stats-matrix "NUMCOL" max-col )
			       (iup:attribute-set! stats-matrix "NUMLIN" (if (< max-row max-visible) max-visible max-row)) ;; min of 20
450
451
452
453
454
455
456
457



458
459
460
461
462
463
464
465
						  (col-num  (cadr (assoc col-name col-indices)))
						  (key      (conc row-num ":" col-num)))
					     (if (not (equal? (iup:attribute stats-matrix key) value))
						 (begin
						   (set! changed #t)
						   (iup:attribute-set! stats-matrix key value)))))
					 run-stats)
			       (if changed (iup:attribute-set! stats-matrix "REDRAW" "ALL")))))))



    (stats-updater)
    (dboard:commondat-add-updater commondat stats-updater tab-num: tab-num)
    ;; (set! dashboard:update-summary-tab updater)
    (iup:attribute-set! stats-matrix "WIDTHDEF" "40")
    (iup:vbox
     ;; (iup:label "Run statistics"  #:expand "HORIZONTAL")
     stats-matrix)))








|
>
>
>
|







594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
						  (col-num  (cadr (assoc col-name col-indices)))
						  (key      (conc row-num ":" col-num)))
					     (if (not (equal? (iup:attribute stats-matrix key) value))
						 (begin
						   (set! changed #t)
						   (iup:attribute-set! stats-matrix key value)))))
					 run-stats)
			       (if changed (iup:attribute-set! stats-matrix "REDRAW" "ALL")))
                             ))))
    ;; (dboard:commondat-please-update-set! commondat #t) ;; force redraw on first pass 
    ;; (mark-for-update tabdat)
    ;; (stats-updater)
    (dboard:commondat-add-updater commondat stats-updater tab-num: tab-num)
    ;; (set! dashboard:update-summary-tab updater)
    (iup:attribute-set! stats-matrix "WIDTHDEF" "40")
    (iup:vbox
     ;; (iup:label "Run statistics"  #:expand "HORIZONTAL")
     stats-matrix)))

889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
	       #:readonly "YES"
	       #:font "Courier New, -12"
	       )))
      (dboard:tabdat-command-tb-set! data tb)
      tb)
    (iup:button "Execute" #:size "50x"
		#:action (lambda (obj)
			   (let ((cmd (conc "xterm -geometry 180x20 -e \""
					    (iup:attribute (dboard:tabdat-command-tb data) "VALUE")
					    ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &")))
			     (system cmd)))))))

(define (dcommon:command-action-selector commondat tabdat #!key (tab-num #f))
  (iup:frame
   #:title "Set the action to take"
   (iup:hbox
    ;; (iup:label "Command to run" #:expand "HORIZONTAL" #:size "70x" #:alignment "LEFT:ACENTER")
    (let* ((cmds-list '("run" "remove-runs" "set-state-status" "lock-runs" "unlock-runs"))
	   (lb         (iup:listbox #:expand "HORIZONTAL"
				    #:dropdown "YES"
				    #:action (lambda (obj val index lbstate)
					       ;; (print obj " " val " " index " " lbstate)
					       (dboard:tabdat-command-set! tabdat val)
					       (dashboard:update-run-command tabdat))))
	   (default-cmd (car cmds-list)))







|
|
|
|






|







1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
	       #:readonly "YES"
	       #:font "Courier New, -12"
	       )))
      (dboard:tabdat-command-tb-set! data tb)
      tb)
    (iup:button "Execute" #:size "50x"
		#:action (lambda (obj)
			   ;; (let ((cmd (conc ;; "xterm -geometry 180x20 -e \""
                           (common:run-a-command (iup:attribute (dboard:tabdat-command-tb data) "VALUE")))))))
    ;; ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &")))
    ;; (system cmd)))))))

(define (dcommon:command-action-selector commondat tabdat #!key (tab-num #f))
  (iup:frame
   #:title "Set the action to take"
   (iup:hbox
    ;; (iup:label "Command to run" #:expand "HORIZONTAL" #:size "70x" #:alignment "LEFT:ACENTER")
    (let* ((cmds-list '("run" "remove-runs")) ;;  "set-state-status" "lock-runs" "unlock-runs"))
	   (lb         (iup:listbox #:expand "HORIZONTAL"
				    #:dropdown "YES"
				    #:action (lambda (obj val index lbstate)
					       ;; (print obj " " val " " index " " lbstate)
					       (dboard:tabdat-command-set! tabdat val)
					       (dashboard:update-run-command tabdat))))
	   (default-cmd (car cmds-list)))
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951

952
953
954
955
956
957
958
959
960
961
962
963
					 (if (not (equal? val ""))
					     (begin
					       (iup:attribute-set! tb "VALUE" val)
					       (dboard:tabdat-run-name-set! tabdat val)
					       (dashboard:update-run-command tabdat))))
				       "command-runname-selector lb action"))))
	  (refresh-runs-list (lambda ()
			       (if (dashboard:database-changed? commondat tabdat)
				   (let* ((target        (dboard:tabdat-target-string tabdat))
					  (runs-for-targ (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" target #f #f #f 0))
					  (runs-header   (vector-ref runs-for-targ 0))
					  (runs-dat      (vector-ref runs-for-targ 1))
					  (run-names     (cons default-run-name 
							       (map (lambda (x)
								      (db:get-value-by-header x runs-header "runname"))
								    runs-dat))))

				     ;; (iup:attribute-set! lb "REMOVEITEM" "ALL")
				     (iuplistbox-fill-list lb run-names selected-item: default-run-name))))))
     ;; (dboard:tabdat-updater-for-runs-set! tabdat refresh-runs-list)
     (dboard:commondat-add-updater commondat refresh-runs-list tab-num: tab-num)
     (refresh-runs-list)
     (dboard:tabdat-run-name-set! tabdat default-run-name)
     (iup:hbox
      tb
      lb))))

(define (dcommon:command-testname-selector commondat tabdat update-keyvals) ;;  key-listboxes)
  (iup:vbox







|
|
|






>




|







1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
					 (if (not (equal? val ""))
					     (begin
					       (iup:attribute-set! tb "VALUE" val)
					       (dboard:tabdat-run-name-set! tabdat val)
					       (dashboard:update-run-command tabdat))))
				       "command-runname-selector lb action"))))
	  (refresh-runs-list (lambda ()
			       (if (dashboard:database-changed? commondat tabdat context-key: 'runname-selector-runs-list)
				   (let* (;; (target        (dboard:tabdat-target-string tabdat))
					  (runs-for-targ (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f 0))
					  (runs-header   (vector-ref runs-for-targ 0))
					  (runs-dat      (vector-ref runs-for-targ 1))
					  (run-names     (cons default-run-name 
							       (map (lambda (x)
								      (db:get-value-by-header x runs-header "runname"))
								    runs-dat))))
				     ;; (print "DEBUGINFO: run-names=" run-names)
				     ;; (iup:attribute-set! lb "REMOVEITEM" "ALL")
				     (iuplistbox-fill-list lb run-names selected-item: default-run-name))))))
     ;; (dboard:tabdat-updater-for-runs-set! tabdat refresh-runs-list)
     (dboard:commondat-add-updater commondat refresh-runs-list tab-num: tab-num)
     ;; (refresh-runs-list)
     (dboard:tabdat-run-name-set! tabdat default-run-name)
     (iup:hbox
      tb
      lb))))

(define (dcommon:command-testname-selector commondat tabdat update-keyvals) ;;  key-listboxes)
  (iup:vbox
971
972
973
974
975
976
977
978
979
980

981
982
983
984
985
986
987
					  tabdat
					  (dboard:lines->test-patt b))
					 (dashboard:update-run-command tabdat))
				       "command-testname-selector tb action"))
			   #:value (dboard:test-patt->lines
				    (dboard:tabdat-test-patts-use tabdat))
			   #:expand "YES"
			   #:size "10x30"
			   #:multiline "YES")))
      (set! test-patterns-textbox tb)

      tb))
;; (iup:frame
;;  #:title "Target"
;;  ;; Target selectors
;;  (apply iup:hbox
;; 	   (let* ((dat      (dashboard:update-target-selector tabdat action-proc: update-keyvals))
;; 		  (key-lb   (car dat))







|


>







1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
					  tabdat
					  (dboard:lines->test-patt b))
					 (dashboard:update-run-command tabdat))
				       "command-testname-selector tb action"))
			   #:value (dboard:test-patt->lines
				    (dboard:tabdat-test-patts-use tabdat))
			   #:expand "YES"
			   #:size "x30" ;; was 10x30
			   #:multiline "YES")))
      (set! test-patterns-textbox tb)
      (dboard:tabdat-test-patterns-textbox-set! tabdat tb)
      tb))
;; (iup:frame
;;  #:title "Target"
;;  ;; Target selectors
;;  (apply iup:hbox
;; 	   (let* ((dat      (dashboard:update-target-selector tabdat action-proc: update-keyvals))
;; 		  (key-lb   (car dat))
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
     #:title "Statuses"
     (dashboard:text-list-toggle-box 
      (map cadr *common:std-statuses*) ;; '("PASS" "FAIL" "n/a" "CHECK" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD")
      (lambda (all)
	(dboard:tabdat-statuses-set! tabdat all)
	(dashboard:update-run-command tabdat)))))))

(define (dcommon:command-tests-tasks-canvas data test-records sorted-testnames tests-draw-state)
  (iup:frame
   #:title "Tests and Tasks"
   (let* ((updater #f)
	  (last-xadj 0)
	  (last-yadj 0)
	  (the-cnv   #f)
	  (canvas-obj 







|







1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
     #:title "Statuses"
     (dashboard:text-list-toggle-box 
      (map cadr *common:std-statuses*) ;; '("PASS" "FAIL" "n/a" "CHECK" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD")
      (lambda (all)
	(dboard:tabdat-statuses-set! tabdat all)
	(dashboard:update-run-command tabdat)))))))

(define (dcommon:command-tests-tasks-canvas tabdat test-records sorted-testnames tests-draw-state)
  (iup:frame
   #:title "Tests and Tasks"
   (let* ((updater #f)
	  (last-xadj 0)
	  (last-yadj 0)
	  (the-cnv   #f)
	  (canvas-obj 
1048
1049
1050
1051
1052
1053
1054
1055

1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071




1072
1073
1074
1075
1076

1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
				     ;; (print "canvas-origin: " xx " " yy " click at " x " " y))
				     (let* ((tests-info     (hash-table-ref tests-draw-state 'tests-info))
					    (selected-tests (hash-table-ref tests-draw-state 'selected-tests))
					    (scalef         (hash-table-ref tests-draw-state 'scalef))
					    (sizey          (hash-table-ref tests-draw-state 'sizey))
					    (xoffset        (dcommon:get-xoffset tests-draw-state #f #f))
					    (yoffset        (dcommon:get-yoffset tests-draw-state #f #f))
					    (new-y          (- sizey y)))

				       ;; (print "xoffset=" xoffset ", yoffset=" yoffset)
				       ;; (print "\tx\ty\tllx\tlly\turx\tury")
				       (for-each (lambda (test-name)
						   (let* ((rec-coords (hash-table-ref tests-info test-name))
							  (llx        (dcommon:x->canvas (list-ref rec-coords 0) scalef xoffset))
							  (lly        (dcommon:y->canvas (list-ref rec-coords 1) scalef yoffset))
							  (urx        (dcommon:x->canvas (list-ref rec-coords 2) scalef xoffset))
							  (ury        (dcommon:y->canvas (list-ref rec-coords 3) scalef yoffset)))
						     ;; (if (eq? pressed 1)
						     ;;    (print "\tx=" x "\ty=" y "\tnew-y=" new-y "\tllx=" llx "\tlly=" lly "\turx=" urx "\tury=" ury "\t" test-name " "))
						     (if (and (eq? pressed 1)
							      (>= x llx)
							      (>= new-y lly)
							      (<= x urx)
							      (<= new-y ury))
							 (let ((patterns (string-split (iup:attribute test-patterns-textbox "VALUE"))))




							   (let* ((selected     (not (member test-name patterns)))
								  (newpatt-list (if selected
										    (cons test-name patterns)
										    (delete test-name patterns)))
								  (newpatt      (string-intersperse newpatt-list "\n")))

							     (iup:attribute-set! obj "REDRAW" "ALL")
							     (hash-table-set! selected-tests test-name selected)
							     (iup:attribute-set! test-patterns-textbox "VALUE" newpatt)
							     (dboard:tabdat-test-patts-set!-use data (dboard:lines->test-patt newpatt))
							     (dashboard:update-run-command data)
							     (if updater (updater last-xadj last-yadj)))))))
						 (hash-table-keys tests-info)))))))
     canvas-obj)))

;;======================================================================
;;  S T E P S
;;======================================================================







|
>















|
>
>
>
>





>


<
|
|







1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233

1234
1235
1236
1237
1238
1239
1240
1241
1242
				     ;; (print "canvas-origin: " xx " " yy " click at " x " " y))
				     (let* ((tests-info     (hash-table-ref tests-draw-state 'tests-info))
					    (selected-tests (hash-table-ref tests-draw-state 'selected-tests))
					    (scalef         (hash-table-ref tests-draw-state 'scalef))
					    (sizey          (hash-table-ref tests-draw-state 'sizey))
					    (xoffset        (dcommon:get-xoffset tests-draw-state #f #f))
					    (yoffset        (dcommon:get-yoffset tests-draw-state #f #f))
					    (new-y          (- sizey y))
					    (test-patterns-textbox (dboard:tabdat-test-patterns-textbox tabdat)))
				       ;; (print "xoffset=" xoffset ", yoffset=" yoffset)
				       ;; (print "\tx\ty\tllx\tlly\turx\tury")
				       (for-each (lambda (test-name)
						   (let* ((rec-coords (hash-table-ref tests-info test-name))
							  (llx        (dcommon:x->canvas (list-ref rec-coords 0) scalef xoffset))
							  (lly        (dcommon:y->canvas (list-ref rec-coords 1) scalef yoffset))
							  (urx        (dcommon:x->canvas (list-ref rec-coords 2) scalef xoffset))
							  (ury        (dcommon:y->canvas (list-ref rec-coords 3) scalef yoffset)))
						     ;; (if (eq? pressed 1)
						     ;;    (print "\tx=" x "\ty=" y "\tnew-y=" new-y "\tllx=" llx "\tlly=" lly "\turx=" urx "\tury=" ury "\t" test-name " "))
						     (if (and (eq? pressed 1)
							      (>= x llx)
							      (>= new-y lly)
							      (<= x urx)
							      (<= new-y ury))
							 (let* ((box-patterns (string-split (iup:attribute test-patterns-textbox "VALUE")))
                                                                (test-patts   (string-split (or (dboard:tabdat-test-patts tabdat)
                                                                                                "")
                                                                                            ","))
                                                                (patterns     (delete-duplicates (append box-patterns test-patts)))) 
							   (let* ((selected     (not (member test-name patterns)))
								  (newpatt-list (if selected
										    (cons test-name patterns)
										    (delete test-name patterns)))
								  (newpatt      (string-intersperse newpatt-list "\n")))
							     (iup:attribute-set! test-patterns-textbox "VALUE" newpatt)
							     (iup:attribute-set! obj "REDRAW" "ALL")
							     (hash-table-set! selected-tests test-name selected)

							     (dboard:tabdat-test-patts-set!-use tabdat (dboard:lines->test-patt newpatt))
							     (dashboard:update-run-command tabdat)
							     (if updater (updater last-xadj last-yadj)))))))
						 (hash-table-keys tests-info)))))))
     canvas-obj)))

;;======================================================================
;;  S T E P S
;;======================================================================

Added defunct/multi-dboard.scm version [de11d53f46].



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use format numbers sql-de-lite srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables)
(require-library iup)
(import (prefix iup iup:))
(use canvas-draw)

(declare (uses margs))
(declare (uses megatest-version))
(declare (uses gutils))
(declare (uses tree))
(declare (uses configf))
(declare (uses portlogger))
(declare (uses keys))
(declare (uses common))

(include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")

(define help (conc 
	      "Megatest Dashboard, documentation at http://www.kiatoa.com/fossils/megatest
  version " megatest-version "
  license GPL, Copyright (C) Matt Welland 2011

Usage: dashboard [options]
  -h                : this help
  -group groupname  : display this group of areas
  -test testid      : control test identified by testid
  -guimonitor       : control panel for runs

Misc
  -rows N         : set number of rows
"))

;; process args
(define remargs (args:get-args 
		 (argv)
		 (list  "-group" ;; display this group of areas
			"-debug"
			) 
		 (list  "-h"
			"-v"
			"-q"
			)
		 args:arg-hash
		 0))

(if (args:get-arg "-h")
    (begin
      (print help)
      (exit)))

;; (if (args:get-arg "-host")
;;     (begin
;;       (set! (common:get-remote remote) (string-split (args:get-arg "-host" ":")))
;;       (client:launch))
;;     (client:launch))

(define *runremote* #f)
(define *windows* (make-hash-table))
(define *changed-main* (make-hash-table)) ;; set path/... => #t
(define *changed-mutex* (make-mutex))     ;; use for all incoming change requests
(define *searchpatts*   (make-hash-table))

(debug:setup)

(define *tim* (iup:timer))
(define *ord* #f)

(iup:attribute-set! *tim* "TIME" 300)
(iup:attribute-set! *tim* "RUN" "YES")

(define (message-window msg)
  (iup:show
   (iup:dialog
    (iup:vbox 
     (iup:label msg #:margin "40x40")))))

(define (iuplistbox-fill-list lb items . default)
  (let ((i 1)
	(selected-item (if (null? default) #f (car default))))
    (iup:attribute-set! lb "VALUE" (if selected-item selected-item ""))
    (for-each (lambda (item)
		(iup:attribute-set! lb (number->string i) item)
		(if selected-item
		    (if (equal? selected-item item)
			(iup:attribute-set! lb "VALUE" item))) ;; (number->string i))))
		(set! i (+ i 1)))
	      items)
    i))

(define (pad-list l n)(append l (make-list (- n (length l)))))


(define (mkstr . x)
  (string-intersperse (map conc x) ","))

(define (update-search x val)
  (hash-table-set! *searchpatts* x val))


;;======================================================================
;; R E C O R D S
;;======================================================================

;; NOTE: Consider switching to defstruct.

;; data for an area (regression or testsuite)
;;
(define-record areadat
  name               ;; area name
  path               ;; mt run area home
  configdat          ;; megatest config
  denoise            ;; focal point for not putting out same messages over and over
  client-signature   ;; key for client-server conversation
  remote             ;; hash of all the client side connnections
  run-keys           ;; target keys for this area
  runs               ;; used in dashboard, hash of run-ids -> rundat
  read-only          ;; can I write to this area?
  monitordb          ;; db handle for monitor.db
  maindb             ;; db handle for main.db
  )

;; rundat, basic run data
;;
(define-record rundat
  id                 ;; the run-id
  target             ;; val1/val2 ... corrosponding to run-keys in areadat
  runname
  state              ;; state of the run, symbol 
  status             ;; status of the run, symbol
  event-time         ;; when the run was initiated
  tests              ;; hash of test-id -> testdat, QUESTION: separate by run-id?
  db                 ;; db handle
  )

;; testdat, basic test data
(define-record testdat
  run-id             ;; what run is this from
  id                 ;; test id
  testname           ;; test name
  itempath           ;; item path
  state              ;; test state, symbol
  status             ;; test status, symbol
  event-time         ;; when the test started
  duration           ;; how long the test took
  )

;; general data for the dboard application
;;
(define-record data
  cfgdat             ;; data from ~/.megatest/<group>.dat
  areas              ;; hash of areaname -> area-rec
  current-window-id  ;; 
  current-tab-id     ;; 
  update-needed      ;; flag to indicate that the tab pointed to by current tab id needs refreshing immediately
  tabs               ;; hash of tab-id -> areaname (??) should be of type "tab"
  )

;; all the components of an area display, all fits into a tab but
;; parts may be swapped in/out as needed
;;
(define-record tab
  tree
  matrix    ;; the spreadsheet 
  areadat   ;; the one-structure (one day dbstruct will be put in here)
  view-path ;; <target/path>/<runname>/...
  view-type ;; standard, etc.
  controls  ;; the controls
  data      ;; all the data kept in sync with db
  filters   ;; user filters, alist name -> filter record, eventually store these in ~/.megatest/<group>.dat?
  run-id    ;; the current run-id
  test-ids  ;; the current test id hash, run-id => test-id
  command   ;; the command from the entry field
  headers   ;; hash of header  -> colnum
  rows      ;; hash of rowname -> rownum
  )

(define-record filter
  target    ;; hash of widgets for the target
  runname   ;; the runname widget
  testpatt  ;; the testpatt widget
  )

;;======================================================================
;; D B
;;======================================================================

;; These are all using sql-de-lite and independent of area so cannot use stuff 
;; from db.scm

;; NB// run-id=#f => return dbdir only
;;
(define (areadb:dbfile-path areadat run-id)
  (let* ((cfgdat  (areadat-configdat areadat))
	 (dbdir   (or (configf:lookup cfgdat "setup" "dbdir")
		      (conc (configf:lookup cfgdat "setup" "linktree") "/.db")))
	 (fname   (if run-id
		      (case run-id
			((-1) "monitor.db")
			((0) "main.db")
			(else (conc run-id ".db")))
		      #f)))
    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir)
       (exit 1))
     (if (not (directory? dbdir))(create-directory dbdir #t)))
    (if fname
	(conc dbdir "/" fname)
	dbdir)))

;; -1 => monitor.db
;;  0 => main.db
;; >1 => <run-id>.db
;;
(define (areadb:open areadat run-id)
  (let* ((runs   (areadat-runs areadat))
	 (rundat (if (> run-id 0) ;; it is a run
		     (hash-table-ref/default runs run-id #f)
		     #f))
	 (db     (case run-id ;; if already opened, get the db and return it
		   ((-1) (areadat-monitordb areadat))
		   ((0)  (areadat-maindb    areadat))
		   (else (if rundat
			     (rundat-db rundat)
			     #f)))))
    (if db
	db ;; merely return the already opened db
	(let* ((dbfile (areadb:dbfile-path areadat run-id)) ;; not already opened, so open it
	       (db     (if (file-exists? dbfile)
			   (open-database dbfile)
			   (begin
			     (debug:print-error 0 *default-log-port* "I was asked to open " dbfile ", but file does not exist or is not readable.")
			     #f))))
	  (case run-id
	    ((-1)(areadat-monitordb-set! areadat db))
	    ((0) (areadat-maindb-set!    areadat db))
	    (else (rundat-db-set!        rundat  db)))
	  db))))

;; populate the areadat tests info, does NOT fill the tests data itself unless asked
;;
(define (areadb:populate-run-info areadat)
  (let* ((runs   (or (areadat-runs areadat) (make-hash-table)))
	 (keys   (areadat-run-keys areadat))
	 (maindb (areadb:open areadat 0)))
    (if maindb
	(query (for-each-row (lambda (row)
			       (let ((id  (list-ref row 0))
				     (dat (apply make-rundat (append row (list #f #f))))) ;; add placeholders for tests and db
				 (print row)
				 (hash-table-set! runs id dat))))
	       (sql maindb (conc "SELECT id,"
				 (string-intersperse keys "||'/'||")
				 ",runname,state,status,event_time FROM runs WHERE state != 'deleted';")))
	(debug:print-error 0 *default-log-port* "no main.db found at "  (areadb:dbfile-path areadat 0)))
    areadat))

;; given an areadat and target/runname patt fill up runs data
;;
;; ?????/

;; given a list of run-ids refresh/retrieve runs data into areadat
;;
(define (areadb:fill-tests areadat #!key (run-ids #f))
  (let* ((runs   (or (areadat-runs areadat) (make-hash-table))))
    (for-each
     (lambda (run-id)
       (let* ((rundat (hash-table-ref/default runs run-id #f))
	      (tests  (if (and rundat
			       (rundat-tests rundat)) ;; re-use existing hash table?
			  (rundat-tests rundat)
			  (let ((ht (make-hash-table)))
			    (rundat-tests-set! rundat ht)
			    ht)))
	      (rundb  (areadb:open areadat run-id)))
	 (query (for-each-row (lambda (row)
				(let* ((id         (list-ref row 0))
				       (testname   (list-ref row 1))
				       (itempath   (list-ref row 2))
				       (state      (list-ref row 3))
				       (status     (list-ref row 4))
				       (eventtim   (list-ref row 5))
				       (duration   (list-ref row 6)))
				  (hash-table-set! tests id
						   (make-testdat run-id id testname itempath state status eventtim duration)))))
		(sql rundb "SELECT id,testname,item_path,state,status,event_time,run_duration FROM tests WHERE state != 'DELETED';"))))
     (or run-ids (hash-table-keys runs)))
    areadat))
    

;; initialize and refresh data
;;		
(define (dboard:general-updater con port)
  (for-each
   (lambda (window-id)
     ;; (print "Processing for window-id " window-id)
     (let* ((window-dat     (hash-table-ref *windows* window-id))
	    (areas          (data-areas     window-dat))
	    ;; (keys           (areadat-run-keys area-dat))
	    (tabs           (data-tabs      window-dat))
	    (tab-ids        (hash-table-keys tabs))
	    (current-tab    (if (null? tab-ids)
				#f
				(hash-table-ref tabs (car tab-ids))))
	    (current-tree   (if (null? tab-ids) #f (tab-tree   current-tab)))
	    (current-node   (if (null? tab-ids) 0  (string->number (iup:attribute current-tree "VALUE"))))
	    (current-path   (if (eq? current-node 0)
				"Areas"
				(string-intersperse (tree:node->path current-tree current-node) "/")))
	    (current-matrix (if (null? tab-ids) #f (tab-matrix current-tab)))
	    (seen-nodes     (make-hash-table))
	    (path-changed   (if current-tab
				(equal? current-path (tab-view-path current-tab))
				#t)))
       ;; (debug:print-info 0 *default-log-port* "Current path: " current-path)
       ;; now for each area in the window gather the data
       (if path-changed
	   (begin
	     (debug:print-info 0 *default-log-port* "clearing matrix - path changed")
	     (dboard:clear-matrix current-tab)))
       (for-each
	(lambda (area-name)
	  ;; (print "Processing for area-name " area-name)
	  (let* ((area-dat  (hash-table-ref areas area-name))
		 (area-path (areadat-path   area-dat))
		 (runs      (areadat-runs   area-dat)))
	    (if (hash-table-ref/default *changed-main* area-path 'processed)
		(begin
		  (print "Processing " area-dat " for area-name " area-name)
		  (hash-table-set! *changed-main* area-path #f)
		  (areadb:populate-run-info area-dat)
		  (for-each 
		   (lambda (run-id)
		     (let* ((run     (hash-table-ref runs run-id))
			    (target  (rundat-target run))
			    (runname (rundat-runname run)))
		       (if current-tree
			   (let* ((partial-path (append (string-split target "/")(list runname)))
				  (full-path    (cons area-name partial-path)))
			     (if (not (hash-table-exists? seen-nodes full-path))
				 (begin
				   (print "INFO: Adding node " partial-path " to section " area-name)
				   (tree:add-node current-tree "Areas" full-path)
				   (areadb:fill-tests area-dat run-ids: (list run-id))))
				   (hash-table-set! seen-nodes full-path #t)))))
		   (hash-table-keys runs))))
	    (if (or (equal? "Areas" current-path)
		    (string-match (conc "^Areas/" area-name "(|\\/.*)$") current-path))
		(dboard:redraw-area area-name area-dat current-tab current-matrix current-path))))
	(hash-table-keys areas))))
   (hash-table-keys *windows*)))

;;======================================================================
;; D A S H B O A R D   D B 
;;======================================================================

;; All moved to common.scm		

;;======================================================================
;; T R E E 
;;======================================================================

;; <area> - <target - ... > - <runname> - <test> - <itempath - ...>

(define (dashboard:tree-browser data adat window-id)
  ;; (iup:split
  (let* ((tb      (iup:treebox
		   #:value 0
		   #:title "Areas"
		   #:expand "YES"
		   #:addexpanded "NO"
		   #:selection-cb
		   (lambda (obj id state)
		     ;; (print "obj: " obj ", id: " id ", state: " state)
		     (let* ((tree-path (tree:node->path obj id))
			    (area      (car tree-path))
			    (areadat-path (cdr tree-path)))
		       #f
		       ;; (test-id  (tree-path->test-id (cdr run-path))))
		       ;; (if test-id
		       ;;    (hash-table-set! (dboard:data-curr-test-ids *data*)
		       ;;		     window-id test-id))
		       ;; (print "path: " (tree:node->path obj id) " test-id: " test-id))))))
		       )))))
    ;; (iup:attribute-set! tb "VALUE" "0")
    ;; (iup:attribute-set! tb "NAME" "Runs")
    ;; (iup:attribute-set! tb "ADDEXPANDED" "NO")
    ;; (dboard:data-tests-tree-set! *data* tb)
    tb))

;;======================================================================
;; M A I N   M A T R I X
;;======================================================================

;; General displayer
;;
(define (dashboard:main-matrix data adat window-id)
  (let* (;; (tab-dat         (areadat-
	 (view-matrix     (iup:matrix
			   ;; (runs-for-targ (db:get-runs-by-patt *dbstruct-local* *keys* "%" target #f #f #f))
			   #:expand "YES"
			   ;; #:fittosize "YES"
			   #:resizematrix "YES"
			   #:scrollbar "YES"
			   #:numcol 100
			   #:numlin 100
			   #:numcol-visible 3
			   #:numlin-visible 20
			   #:click-cb (lambda (obj lin col status)
					(print "obj: " obj " lin: " lin " col: " col " status: " status " value: " (iup:attribute obj "VALUE"))))))
    
    ;; (iup:attribute-set! view-matrix "RESIZEMATRIX" "YES")
    (iup:attribute-set! view-matrix "WIDTH0" "100")
    ;; (dboard:data-runs-matrix-set! *data* runs-matrix)
    ;; (iup:hbox
    ;;  (iup:frame 
    ;;   #:title "Runs browser"
    ;;   (iup:vbox
    view-matrix))

;;======================================================================
;; A R E A S
;;======================================================================

(define (dashboard:init-area data area-name apath)
  (let* ((mtconf      (dboard:read-mtconf apath))
	 (area-dat    (let ((ad (make-areadat
				 area-name ;; area name
				 apath     ;; path to area
				 ;; 'http     ;; transport
				 mtconf    ;; megatest.config
				 (make-hash-table) ;; denoise hash
				 #f        ;; client-signature
				 #f        ;; remote connections
				 (keys:config-get-fields mtconf) ;; run keys
				 (make-hash-table) ;; run-id -> (hash of test-ids => dat)
				 (and (file-exists? apath)(file-write-access? apath)) ;; read-only
				 #f
				 #f
				 )))
			(hash-table-set! (data-areas data) area-name ad)
			ad)))
    area-dat))

;; given the keys for an area and a path from the tree browser
;; return the level: areas area runs run tests test
;;
(define (dboard:get-view-type keys current-path)
  (let* ((path-parts (string-split current-path "/"))
	 (path-len   (length path-parts)))
    (cond
     ((equal? current-path "Areas")     'areas)
     ((eq? path-len 2)                  'area)
     ((<= (+ (length keys) 2) path-len) 'runs)
     (else                              'run))))

(define (dboard:clear-matrix tab)
  (if tab
      (begin
	(iup:attribute-set! (tab-matrix tab) "CLEARVALUE" "ALL")
	(tab-headers-set! tab (make-hash-table))
	(tab-rows-set!    tab (make-hash-table)))))

;; full redraw of a given area
;;
(define (dboard:redraw-area area-name area-dat tab-dat current-matrix current-path)
  (let* ((keys      (areadat-run-keys area-dat))
	 (runs      (areadat-runs     area-dat))
	 (headers   (tab-headers   tab-dat))
	 (rows      (tab-rows      tab-dat))
	 (used-cols (hash-table-values headers))
	 (used-rows (hash-table-values rows))
	 (touched   (make-hash-table)) ;; (vector row col) ==> true, touched cell
	 (view-type (dboard:get-view-type keys current-path))
	 (changed   #f)
	 (state-statuses  (list "PASS" "FAIL" "WARN" "CHECK" "SKIP" "RUNNING" "LAUNCHED")))
    ;; (debug:print 0 *default-log-port* "current-matrix=" current-matrix)
    (case view-type
      ((areas) ;; find row for this area, if not found, create new entry
       (let* ((curr-rownum (hash-table-ref/default rows area-name #f))
	      (next-rownum (+ (apply max (cons 0 used-rows)) 1))
	      (rownum      (or curr-rownum next-rownum))
	      (coord       (conc rownum ":0")))
	 (if (not curr-rownum)(hash-table-set! rows area-name rownum))
	 (if (not (equal? (iup:attribute current-matrix coord) area-name))
	     (begin
	       (let loop ((hed  (car state-statuses))
			  (tal  (cdr state-statuses))
			  (count 1))
		 (if (not (equal? (iup:attribute current-matrix (conc "0:" count)) hed))
		     (iup:attribute-set! current-matrix (conc "0:" count) hed))
		 (iup:attribute-set! current-matrix (conc rownum ":" count) "0")
		 (if (not (null? tal))
		     (loop (car tal)(cdr tal)(+ count 1))))
	       (debug:print-info 0 *default-log-port* "view-type=" view-type ", rownum=" rownum ", curr-rownum=" curr-rownum ", next-rownum=" next-rownum ", coord=" coord ", area-name=" area-name)
	       (iup:attribute-set! current-matrix coord area-name)
	       (set! changed #t))))))
    (if changed (iup:attribute-set! current-matrix "REDRAW" "ALL"))))
	     

       
   ;; (dboard:clear-matrix current-matrix used-cols used-rows touched) ;; clear all
    
	
  
;;======================================================================
;; D A S H B O A R D
;;======================================================================

(define (dashboard:area-panel aname data window-id)
  (let* ((apath      (configf:lookup (data-cfgdat data) aname "path")) ;;  (hash-table-ref (dboard:data-cfgdat data) area-name))
	 ;;          (hash-table-ref (dboard:data-cfgdat data) aname))
	 (area-dat   (dashboard:init-area data aname apath))
	 (tb         (dashboard:tree-browser data area-dat window-id)) ;; (dboard:areas-tree-browser data)
	 (ad         (dashboard:main-matrix  data area-dat window-id))
	 (areas      (data-areas data))
	 (dboard-dat (make-tab
		      #f           ;; tree
		      #f           ;; matrix
		      area-dat     ;;
		      #f           ;; view path
		      'default     ;; view type
		      #f           ;; controls
		      (make-hash-table) ;; cached data? not sure how to use this yet :)
		      #f           ;; filters
		      #f           ;; the run-id
		      (make-hash-table) ;; run-id -> test-id, for current test id
		      ""
		      (make-hash-table) ;; headername -> colnum
		      (make-hash-table) ;; rowname    -> rownum
		      )))
    (hash-table-set! (data-areas data) aname area-dat) ;; dboard-dat)
    (hash-table-set! (data-tabs data) window-id dboard-dat)
    (tab-tree-set!   dboard-dat tb)
    (tab-matrix-set! dboard-dat ad)
    (iup:split
     #:value 200
     tb ad)))


;; Main Panel
;;
(define (dashboard:main-panel data window-id)
  (iup:dialog
   #:title "Megatest Control Panel"
;;   #:menu (dcommon:main-menu data)
   #:shrink "YES"
   (iup:vbox
    (let* ((area-names  (hash-table-keys (data-cfgdat data)))
	   (area-panels (map (lambda (aname)
			       (dashboard:area-panel aname data window-id))
			     area-names))
	   (tabtop      (apply iup:tabs  
			       #:tabchangepos-cb (lambda (obj curr prev)
						   (data-current-tab-id-set! data curr)
						   (data-update-needed-set!  data #t)
						   (print "Tab is: " curr ", prev was " prev))
			       area-panels))
	   (tabs     (data-tabs data)))
      (if (not (null? area-names))
	  (let loop ((index 0)
		     (hed   (car area-names))
		     (tal   (cdr area-names)))
	    ;; (hash-table-set! tabs index hed)
	    (debug:print 0 *default-log-port* "Adding area " hed " with index " index " to dashboard")
	    (iup:attribute-set! tabtop (conc "TABTITLE" index) hed)
	    (if (not (null? tal))
		(loop (+ index 1)(car tal)(cdr tal)))))
      tabtop))))


;;======================================================================
;; N A N O M S G   S E R V E R
;;======================================================================

(define (dboard:server-service soc port)
  (print "server starting")
  (let loop ((msg-in (nn-recv soc))
	     (count  0))
    (if (eq? 0 (modulo count 1000))
	(print "server received: " msg-in ", count=" count))
    (cond
     ;;
     ;; quit
     ;;
     ((equal? msg-in "quit")
      (nn-send soc "Ok, quitting"))
     ;;
     ;; ping
     ;;
     ((and (>= (string-length msg-in) 4)
	   (equal? (substring msg-in 0 4) "ping"))
      (nn-send soc (conc (current-process-id)))
      (loop (nn-recv soc)(+ count 1)))
     ;;
     ;; main changed
     ;;
     ((and (>= (string-length msg-in) 4)
	   (equal? (substring msg-in 0 4) "main"))
      (let ((parts (string-split msg-in " ")))
	(hash-table-set! *changed-main* (cadr parts) #t)
	(nn-send soc "got it!")))
     ;;
     ;; ??
     ;;
     (else
      (nn-send soc "hello " msg-in " you got to the else clause!")))
    (loop (nn-recv soc)(if (> count 20000000)
			   0
			   (+ count 1)))))

(define (dboard:one-time-ping-receive soc port)
  (let ((msg-in (nn-recv soc)))
    (if (and (>= (string-length msg-in) 4)
	     (equal? (substring msg-in 0 4) "ping"))
	(nn-send soc (conc (current-process-id))))))

(define (dboard:server-start given-port #!key (num-tries 200))
  (let* ((rep (nn-socket 'rep))
	 (port (or given-port  (portlogger:main "find")))
	 (con (conc "tcp://*:" port)))
    ;; register this connect here ....
    (nn-bind rep con)
    (thread-start! 
     (make-thread (lambda ()
		    (dboard:one-time-ping-receive rep port))
		  "one time receive thread"))
    (if (dboard:ping-self "localhost" port)
	(begin
	  (print "INFO: dashboard nanomsg server started on " port)
	  (values rep port))
	(begin
	  (print "WARNING: couldn't create server on port " port)
	  (portlogger:main "set" "failed")
	  (if (> num-tries 0)
	      (dboard:server-start #f (- num-tries 1))
	      (begin
		(print "ERROR: failed to start nanomsg server")
		(values #f #f)))))))

(define (dboard:server-close con port)
  (nn-close con)
  (portlogger:main "set" port "released"))

(define (dboard:ping-self host port #!key (return-socket #t))
  ;; send a random number along with pid and check that we get it back
  (let* ((req     (nn-socket 'req))
	 (key     "ping")
	 (success #f)
	 (keepwaiting #t)
	 (ping    (make-thread
		   (lambda ()
		     (print "ping: sending string \"" key "\", expecting " (current-process-id))
		     (nn-send req key)
		     (let ((result  (nn-recv req)))
		       (if (equal? (conc (current-process-id)) result)
			   (begin
			     (print "ping, success: received \"" result "\"")
			     (set! success #t))
			   (begin
			     (print "ping, failed: received key \"" result "\"")
			     (set! keepwaiting #f)
			     (set! success #f)))))
		   "ping"))
	 (timeout (make-thread (lambda ()
				 (let loop ((count 0))
				   (thread-sleep! 1)
				   (print "still waiting after " count " seconds...")
				   (if (and keepwaiting (< count 10))
				       (loop (+ count 1))))
				 (if keepwaiting
				     (begin
				       (print "timeout waiting for ping")
				       (thread-terminate! ping))))
			       "timeout")))
    (nn-connect req (conc "tcp://" host ":" port))
    (handle-exceptions
     exn
     (begin
       (print-call-chain)
       (print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
       (print "exn=" (condition->list exn))
       (print "ping failed to connect to " host ":" port))
     (thread-start! timeout)
     (thread-start! ping)
     (thread-join! ping)
     (if success (thread-terminate! timeout)))
    (if return-socket
	(if success req #f)
	(begin
	  (nn-close req)
	  success))))

;;======================================================================
;; C O N F I G U R A T I O N 
;;======================================================================

;; Get the configuration file for a group name, if the group name is "default" and it doesn't 
;; exist, create it and add the current path if it contains megatest.config
;;
(define (dboard:get-config group-name)
  (let* ((fname (conc (getenv "HOME") "/.megatest/" group-name ".dat")))
    (if (file-exists? fname)
	(read-config fname (make-hash-table) #t)
	(if (dboard:create-config fname)
	    (dboard:get-config group-name)
	    (make-hash-table)))))

(define (dboard:create-config fname)
  ;; (handle-exceptions
  ;;  exn
  ;;  
  ;;  #f ;; failed to create - just give up
   (let* ((dirname       (pathname-directory fname))
	  (file-name     (pathname-strip-directory fname))
	  (curr-mtcfgdat (find-config "megatest.config"
				      toppath: (or (get-environment-variable "MT_RUN_AREA_HOME")(current-directory))))
	  (curr-mtcfg    (if (and curr-mtcfgdat (not (null? curr-mtcfgdat)))(cadr curr-mtcfgdat) #f))
	  (curr-mtpath   (if curr-mtcfg (car curr-mtcfgdat) #f)))
     (if curr-mtpath
	 (begin
	   (debug:print-info 0 *default-log-port* "Creating config file " fname)
	   (if (not (file-exists? dirname))
	       (create-directory dirname #t))
	   (with-output-to-file fname
	     (lambda ()
	       (let ((aname (pathname-strip-directory curr-mtpath)))
		 (print "[" aname "]")
		 (print  "path " curr-mtpath))))
	   #t)
	 (begin
	   (debug:print-info 0 *default-log-port* "Need to create a config but no megatest.config found: " curr-mtcfgdat)
	   #f))))
;; )

(define (dboard:read-mtconf apath)
  (let* ((mtconffile  (conc apath "/megatest.config")))
    (call-with-environment-variables
     (list (cons "MT_RUN_AREA_HOME" apath))
     (lambda ()
       (read-config mtconffile (make-hash-table) #f)) ;; megatest.config
     )))
	 

;;======================================================================
;; G U I   S T U F F 
;;======================================================================

;;; main. Theoretically could have multiple windows (each with a group of tags, thus window-id
;;;
(define (dboard:make-window window-id)
  (let* (;; (window-id 0)
	 (groupn    (or (args:get-arg "-group") "default"))
	 (cfgdat    (dboard:get-config groupn))
	 ;; (cfgdat    (if (file-exists? cfname)(read-config cfname (make-hash-table) #t)(make-hash-table)))
	 (data      (make-data
		     cfgdat ;; this is the data from ~/.megatest for the selected group
		     (make-hash-table) ;; areaname -> area-rec
		     0                 ;; current window id
		     0                 ;; current tab id
		     #f                ;; redraw needed for current tab id
		     (make-hash-table) ;; tab-id -> areaname
		     )))
    (hash-table-set! *windows* window-id data)
    (iup:show (dashboard:main-panel data window-id))
    (iup:main-loop)))

;; ease debugging by loading ~/.dashboardrc
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc")))
  (if (file-exists? debugcontrolf)
      (load debugcontrolf)))

(define (main)
  (let-values 
      (((con port)(dboard:server-start #f)))
    (let ((portnum   (if (string? port)(string->number port) port)))
      ;; got here, monitor/dashboard was started
      (mddb:register-dashboard portnum)
      (thread-start! (make-thread (lambda ()(dboard:server-service con portnum)) "server service"))
      (thread-start! (make-thread (lambda ()
				    (let loop ()
				      (dboard:general-updater con portnum)
				      (thread-sleep! 1)
				      (loop))) "general updater"))
      (dboard:make-window 0)
      (mddb:unregister-dashboard (get-host-name) portnum)
      (dboard:server-close con port))))

Added defunct/nmsg-transport.scm version [b30844cb1a].













































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358

;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

(require-extension (srfi 18) extras tcp s11n)

(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
(import (prefix sqlite3 sqlite3:))

;; (use nanomsg)

(declare (unit nmsg-transport))

(declare (uses common))
(declare (uses db))
(declare (uses tests))
(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
(declare (uses server))

(include "common_records.scm")
(include "db_records.scm")

;; Transition to pub --> sub with pull <-- push
;;
;;   1. client sends request to server via push to the pull port
;;   2. server puts request in queue or processes immediately as appropriate
;;   3. server puts responses from completed requests into pub port 
;;
;; TODO
;;
;; Done Tested
;; [x]  [ ]    1. Add columns pullport pubport to servers table
;; [x]  [ ]    2. Add rm of monitor.db if older than 11/12/2012 
;; [x]  [ ]    3. Add create of pullport and pubport with finding of available ports
;; [x]  [ ]    4. Add client compose of request
;; [x]  [ ]        - name of client: testname/itempath-test_id-hostname 
;; [x]  [ ]        - name of request: callname, params
;; [x]  [ ]        - request key: f(clientname, callname, params)
;; [x]  [ ]    5. Add processing of subscription hits
;; [x]  [ ]        - done when get key 
;; [x]  [ ]        - return results
;; [x]  [ ]    6. Add timeout processing
;; [x]  [ ]        - after 60 seconds
;; [ ]  [ ]            i. check server alive, connect to new if necessary
;; [ ]  [ ]           ii. resend request
;; [ ]  [ ]    7. Turn self ping back on

(define (nmsg-transport:make-server-url hostport #!key (bindall #f))
  (if (not hostport)
      #f
      (conc "tcp://" (if bindall "*" (car hostport)) ":" (cadr hostport))))

(define *server-loop-heart-beat* (current-seconds))
(define *heartbeat-mutex* (make-mutex))

;;======================================================================
;; S E R V E R
;;======================================================================

(define (nmsg-transport:run dbstruct hostn run-id server-id #!key (retrynum 1000))
  (debug:print 2 *default-log-port* "Attempting to start the server ...")
  (let* ((start-port      (portlogger:open-run-close portlogger:find-port))
	 (server-thread   (make-thread (lambda ()
					 (nmsg-transport:try-start-server dbstruct run-id start-port server-id))
				       "server thread"))
	 (tdbdat          (tasks:open-db)))
    (thread-start! server-thread)
    (thread-sleep! 0.1)
    (if (nmsg-transport:ping hostn start-port timeout: 2 expected-key: (current-process-id))
	(let ((interface (if (equal? hostn "-")(get-host-name) hostn)))
	  (tasks:server-set-interface-port (db:delay-if-busy tdbdat) server-id interface start-port)
	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep")
	  (set! *server-info* (list hostn start-port)) ;; probably not needed anymore? currently used by keep-running
	  (thread-sleep! 3) ;; give some margin for queries to complete before switching from file based access to server based access
	  ;; (set! *inmemdb*  dbstruct)
	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running")
	  (thread-start! (make-thread
			  (lambda ()(nmsg-transport:keep-running server-id run-id))
			  "keep running"))
	  (thread-join! server-thread))
	(if (> retrynum 0)
	    (begin
	      (debug:print 0 *default-log-port* "WARNING: Failed to connect to server (self) on host " hostn ":" start-port ", trying again.")
	      (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id "failed to start, never received server alive signature")
	      (portlogger:open-run-close portlogger:set-failed start-port)
	      (nmsg-transport:run dbstruct hostn run-id server-id))
	    (begin
	      (debug:print-error 0 *default-log-port* "could not find an open port to start server on. Giving up")
	      (exit 1))))))

(define (nmsg-transport:try-start-server dbstruct run-id portnum server-id)
  (let ((repsoc (nn-socket 'rep)))
    (nn-bind repsoc (conc "tcp://*:" portnum))
    (let loop ((msg-in (nn-recv repsoc)))
      (let* ((dat    (db:string->obj msg-in transport: 'nmsg)))
	(debug:print 0 *default-log-port* "server, received: " dat)
	(let ((result (api:execute-requests dbstruct dat)))
	  (debug:print 0 *default-log-port* "server, sending: " result)
	  (nn-send repsoc (db:obj->string result  transport: 'nmsg)))
	(loop (nn-recv repsoc))))))

;; all routes though here end in exit ...
;;
(define (nmsg-transport:launch run-id)
  (let* ((tdbdat   (tasks:open-db))
	 (dbstruct (db:setup run-id))
	 (hostn    (or (args:get-arg "-server") "-")))
    (set! *run-id*   run-id)
    (set! *inmemdb* dbstruct)
    ;; with nbfake daemonize isn't really needed
    ;;
    ;; (if (args:get-arg "-daemonize")
    ;;     (begin
    ;;       (daemon:ize)
    ;;       (if *alt-log-file* ;; we should re-connect to this port, I think daemon:ize disrupts it
    ;;           (begin
    ;;     	(current-error-port *alt-log-file*)
    ;;     	(current-output-port *alt-log-file*)))))
    (if (server:check-if-running run-id)
	(begin
	  (debug:print-info 0 *default-log-port* "Server for run-id " run-id " already running")
	  (exit 0)))
    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
	       (remtries  4))
      (if (not server-id)
	  (if (> remtries 0)
	      (begin
		(thread-sleep! 2)
		(if (not (server:check-if-running run-id))
		    (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
			  (- remtries 1))
		    (begin
		      (debug:print-info 0 *default-log-port* "Another server took the slot, exiting")
		      (exit 0))))
	      (begin
		;; since we didn't get the server lock we are going to clean up and bail out
		(debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
		(tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " http-transport:launch")
		))
	  ;; locked in a server id, try to start up
	  (nmsg-transport:run dbstruct hostn run-id server-id))
      (set! *didsomething* #t)
      (exit))))

;;======================================================================
;; S E R V E R   U T I L I T I E S 
;;======================================================================

(define (nmsg-transport:mk-signature)
  (message-digest-string (md5-primitive) 
			 (with-output-to-string
			   (lambda ()
			     (write (list (current-directory)
					  (argv)))))))

;;======================================================================
;; C L I E N T S
;;======================================================================

;; ping the server at host:port
;;   return the open socket if successful (return-socket == #t)
;;   expect the key expected-key returned in payload
;;   send our-key or #f as payload
;;
(define (nmsg-transport:ping hostn port #!key (timeout 3)(return-socket #t)(expected-key #f)(our-key #f)(socket #f))
  ;; send a random number along with pid and check that we get it back
  (let* ((host    (if (or (not hostn)
			  (equal? hostn "-")) ;; use localhost
		      (get-host-name)
		      hostn))
	 (req     (or socket
		      (let ((soc (nn-socket 'req)))
			(nn-connect soc (conc "tcp://" host ":" port))
			soc)))
	 (success #t)
	 (dat     (vector "ping" our-key))
	 (result  (condition-case 
		   (nmsg-transport:client-api-send-receive-raw req dat timeout: timeout)
		   ((timeout)(set! success #f) #f)))
	 (key     (if success 
		      (vector-ref result 1)
		      #f)))
    (debug:print 0 *default-log-port* "success=" success ", key=" key ", expected-key=" expected-key ", equal? " (equal? key expected-key))
    (if (and success
	     (or (not expected-key) ;; just getting a reply is good enough then
		 (equal? key expected-key)))
	(if return-socket
	    req
	    (begin
	      (if (not socket)(nn-close req)) ;; don't want a side effect of closing socket if handed it
	      #t))
	(begin
	  (if (not socket)(nn-close req)) ;; failed to ping, close socket as side effect
	  #f))))

;; send data to server, wait max of timeout seconds for a response.
;; return #( success/fail result )
;;
;; for effiency it is easier to do the obj->string and string->obj here.
;;
(define (nmsg-transport:client-api-send-receive-raw socreq indat #!key (enable-send #t)(timeout 25))
  (let* ((success     #f)
	 (result      #f)
	 (keepwaiting #t)
	 (dat         (db:obj->string indat transport: 'nmsg))
	 (send-recv   (make-thread
		       (lambda ()
			 (nn-send socreq dat)
			 (let* ((res (nn-recv socreq)))
			   (set! success #t)
			   (set! result (db:string->obj res transport: 'nmsg))))
		       "send-recv"))
	 (timeout     (make-thread
		       (lambda ()
			 (let loop ((count 0))
			   (thread-sleep! 1)
			   (debug:print-info 1 *default-log-port* "send-receive-raw, still waiting after " count " seconds...")
			   (if (and keepwaiting (< count timeout)) ;; yes, this is very aproximate
			       (loop (+ count 1))))
			 (if keepwaiting
			     (begin
			       (print "timeout waiting for ping")
			       (thread-terminate! send-recv))))
		       "timeout")))
    ;; replace with condition-case?
    (handle-exceptions
     exn
     (set! result "timeout")
     (thread-start! timeout)
     (thread-start! send-recv)
     (thread-join! send-recv)
     (if success (thread-terminate! timeout)))
    ;; raise timeout error if timed out
    (if success
	(if (and (vector? result)
		 (vector-ref result 0)) ;; did it fail at the server?
	    result                ;; nope, all good
	    (begin
	      (debug:print-error 0 *default-log-port* "error occured at server, info=" (vector-ref result 2))
	      (debug:print 0 *default-log-port* " client call chain:")
	      (print-call-chain (current-error-port))
	      (debug:print 0 *default-log-port* " server call chain:")
	      (pp (vector-ref result 1) (current-error-port))
	      (signal (vector-ref result 0))))
	(signal (make-composite-condition
		 (make-property-condition 'timeout 'message "nmsg-transport:client-api-send-receive-raw timed out talking to server"))))))

;; run nmsg-transport:keep-running in a parallel thread to monitor that the db is being 
;; used and to shutdown after sometime if it is not.
;;
(define (nmsg-transport:keep-running server-id run-id)
  ;; if none running or if > 20 seconds since 
  ;; server last used then start shutdown
  ;; This thread waits for the server to come alive
  (let* ((server-info (let loop ()
                        (let ((sdat #f))
                          (mutex-lock! *heartbeat-mutex*)
                          (set! sdat *server-info*)
                          (mutex-unlock! *heartbeat-mutex*)
                          (if sdat 
			      (begin
				(debug:print-info 0 *default-log-port* "keep-running got sdat=" sdat)
				sdat)
                              (begin
                                (thread-sleep! 0.5)
                                (loop))))))
         (iface       (car server-info))
         (port        (cadr server-info))
         (last-access 0)
	 (tdbdat      (tasks:open-db))
	 (server-timeout (let ((tmo (configf:lookup  *configdat* "server" "timeout")))
			   (if (and (string? tmo)
				    (string->number tmo))
			       (* 60 60 (string->number tmo))
			       ;; (* 3 24 60 60) ;; default to three days
			       (* 60 1)         ;; default to one minute
			       ;; (* 60 60 25)      ;; default to 25 hours
			       ))))
    (print "Keep-running got server pid " server-id ", using iface " iface " and port " port)
    (let loop ((count 0))
      (thread-sleep! 4) ;; no need to do this very often
      ;; NB// sync currently does NOT return queue-length
      (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1)))
      ;; (print "Server running, count is " count)
        (if (< count 1) ;; 3x3 = 9 secs aprox
            (loop (+ count 1)))
        
        (mutex-lock! *heartbeat-mutex*)
        (set! last-access *last-db-access*)
        (mutex-unlock! *heartbeat-mutex*)
	(db:sync-touched *inmemdb* run-id force-sync: #t)
        (if (and *server-run*
	       (> (+ last-access server-timeout)
		  (current-seconds)))
            (begin
              (debug:print-info 0 *default-log-port* "Server continuing, seconds since last db access: " (- (current-seconds) last-access))
              (loop 0))
            (begin
              (debug:print-info 0 *default-log-port* "Starting to shutdown the server.")
              (set! *time-to-exit* #t)
	      (db:sync-touched *inmemdb* run-id force-sync: #t)
              (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running")
              (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting")
              (exit)
	      ))))))

;;======================================================================
;; C L I E N T S
;;======================================================================

(define (nmsg-transport:client-connect iface portnum)
  (let* ((reqsoc      (nmsg-transport:ping iface portnum return-socket: #t)))
    (vector iface portnum #f #f #f (current-seconds) reqsoc)))

;; returns result, there is no sucess/fail flag - handled via excpections
;;
(define (nmsg-transport:client-api-send-receive run-id connection-info cmd param #!key (remtries 5))
  ;; NB// In the html version of this routine there is a call to 
  ;;      tasks:kill-server-run-id when there is an exception
  (mutex-lock! *http-mutex*)
  (let* ((packet  (vector cmd param))
	 (reqsoc  (http-transport:server-dat-get-socket connection-info))
	 (res     (nmsg-transport:client-api-send-receive-raw reqsoc packet)))
;;	 (status  (vector-ref rawres 0))
;;	 (result  (vector-ref rawres 1)))
    (mutex-unlock! *http-mutex*)
    res)) ;; (vector status (if status (db:string->obj result transport: 'nmsg) result))))
	
;;======================================================================
;; J U N K 
;;======================================================================

;; DO NOT USE
;;
(define (nmsg-transport:client-signal-handler signum)
  (handle-exceptions
   exn
   (debug:print 0 *default-log-port* " ... exiting ...")
   (let ((th1 (make-thread (lambda ()
			     (if (not *received-response*)
				 (receive-message* *runremote*))) ;; flush out last call if applicable
			   "eat response"))
	 (th2 (make-thread (lambda ()
			     (debug:print-error 0 *default-log-port* "Received ^C, attempting clean exit. Please be patient and wait a few seconds before hitting ^C again.")
			     (thread-sleep! 3) ;; give the flush three seconds to do it's stuff
			     (debug:print 0 *default-log-port* "       Done.")
			     (exit 4))
			   "exit on ^C timer")))
     (thread-start! th2)
     (thread-start! th1)
     (thread-join! th2))))

Modified docs/api.html from [c2276e6b3d] to [b7a1558ae4].

823
824
825
826
827
828
829

830
831
832
833
834
835
836
</div></div>
</div>
<div class="sect2">
<h3 id="_get_list_of_runs">1.2. Get List of Runs</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Filter Params: target, testpatt, offset, limit</p></div>

<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
  {
    "<span class="red">run_id</span>" : "1",
    "<span class="red">name</span>"   : "runname1",







>







823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
</div></div>
</div>
<div class="sect2">
<h3 id="_get_list_of_runs">1.2. Get List of Runs</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Filter Params: target, testpatt, offset, limit</p></div>
<div class="paragraph"><p>Megatest Cmd: megatest -start-dir &lt;path to  megatest area&gt; -list-runs % -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id</p></div>
<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
  {
    "<span class="red">run_id</span>" : "1",
    "<span class="red">name</span>"   : "runname1",
862
863
864
865
866
867
868

869
870
871
872
873
874
875
]</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_trigger_a_new_run">1.3. Trigger a new Run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs</p></div>
<div class="paragraph"><p>Method: POST</p></div>

<div class="paragraph"><p>Request Params:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>{"<span class="blue">target</span>": "target_value", "<span class="blue">runname</span>" : "runname", "<span class="blue">test_pattern</span>": "optional test pattern"}</p></div>
</div></div>
<div class="paragraph"><p>Response:</p></div>
<div class="paragraph"><p>If Error</p></div>







>







863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
]</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_trigger_a_new_run">1.3. Trigger a new Run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs</p></div>
<div class="paragraph"><p>Method: POST</p></div>
<div class="paragraph"><p>Megatest Cmd:  megatest -runtests % -target &lt;target&gt; :runname &lt;run_name&gt; -run</p></div>
<div class="paragraph"><p>Request Params:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>{"<span class="blue">target</span>": "target_value", "<span class="blue">runname</span>" : "runname", "<span class="blue">test_pattern</span>": "optional test pattern"}</p></div>
</div></div>
<div class="paragraph"><p>Response:</p></div>
<div class="paragraph"><p>If Error</p></div>
899
900
901
902
903
904
905

906
907
908
909
910
911
912
</div></div>
</div>
<div class="sect2">
<h3 id="_get_perticular_run">1.4. Get perticular Run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Filter Params: testpatt</p></div>

<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
   {
    "<span class="red">run_id</span>" : "2",
    "<span class="red">name</span>"   : "runname2",







>







901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
</div></div>
</div>
<div class="sect2">
<h3 id="_get_perticular_run">1.4. Get perticular Run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Filter Params: testpatt</p></div>
<div class="paragraph"><p>Megatest Cmd: megatest -start-dir &lt;path to  megatest area&gt; -list-runs &lt;runname&gt; -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id</p></div>
<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
   {
    "<span class="red">run_id</span>" : "2",
    "<span class="red">name</span>"   : "runname2",
950
951
952
953
954
955
956

957
958
959
960
961
962
963
]</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_get_list_of_tests_within_a_run">1.6. Get List of tests within a run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id/tests</p></div>
<div class="paragraph"><p>Method: GET</p></div>

<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
     "<span class="red">tests</span>"  :
              [
                   {"<span class="blue">id</span>": 4, "<span class="blue">name</span>":[blue]test1, "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}







>







953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
]</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_get_list_of_tests_within_a_run">1.6. Get List of tests within a run</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id/tests</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Megatest Cmd: megatest -start-dir &lt;path to  megatest area&gt; -list-runs &lt;runname&gt; -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id</p></div>
<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>[
     "<span class="red">tests</span>"  :
              [
                   {"<span class="blue">id</span>": 4, "<span class="blue">name</span>":[blue]test1, "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}
977
978
979
980
981
982
983

984
985
986
987
988
989
990
<div class="paragraph"><p>{"<span class="blue">id</span>": "4", "<span class="blue">name</span>":"test1", "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_get_perticular_test_that_belongs_to_a_runs">1.8. Get perticular test that belongs to a Runs</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id/tests/:id</p></div>
<div class="paragraph"><p>Method: GET</p></div>

<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>{"<span class="blue">id</span>": "4", "<span class="blue">name</span>":"test1", "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}</p></div>
</div></div>
</div>
</div>







>







981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
<div class="paragraph"><p>{"<span class="blue">id</span>": "4", "<span class="blue">name</span>":"test1", "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}</p></div>
</div></div>
</div>
<div class="sect2">
<h3 id="_get_perticular_test_that_belongs_to_a_runs">1.8. Get perticular test that belongs to a Runs</h3>
<div class="paragraph"><p>URL: &lt;base&gt;/runs/:id/tests/:id</p></div>
<div class="paragraph"><p>Method: GET</p></div>
<div class="paragraph"><p>Megatest Cmd: megatest -start-dir &lt;path to  megatest area&gt; -list-runs &lt;runname&gt; -target % -testpattern &lt;pattern&gt; -dumpmode json  -fields runs:runname,id+tests:state,status:id</p></div>
<div class="paragraph"><p>Response:</p></div>
<div class="exampleblock">
<div class="content">
<div class="paragraph"><p>{"<span class="blue">id</span>": "4", "<span class="blue">name</span>":"test1", "<span class="blue">item_path</span>": "", "<span class="blue">shortdir</span>": "/temp/foo/bar/target2/runname2/test1", "<span class="blue">final_logf</span>": "megatest-rollup-test1.html",  "<span class="blue">status</span>": "PASS"}</p></div>
</div></div>
</div>
</div>
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
</div>
</div>
</div>
<div id="footnotes"><hr></div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated 2016-08-04 09:33:43 PDT
</div>
</div>
</body>
</html>







|




1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
</div>
</div>
</div>
<div id="footnotes"><hr></div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated 2016-08-22 14:16:39 PDT
</div>
</div>
</body>
</html>

Modified docs/api.txt from [bcdb746fea] to [22dea6c059].

38
39
40
41
42
43
44


45
46
47
48
49
50
51

URL: <base>/runs

Method: GET

Filter Params: target, testpatt, offset, limit



Response:


==================
[
  {
    "[red]#run_id#" : "1",







>
>







38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

URL: <base>/runs

Method: GET

Filter Params: target, testpatt, offset, limit

Megatest Cmd: megatest -start-dir <path to  megatest area> -list-runs % -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id

Response:


==================
[
  {
    "[red]#run_id#" : "1",
82
83
84
85
86
87
88


89
90
91
92
93
94
95
Trigger a new Run
~~~~~~~~~~~~~~~~~~

URL: <base>/runs

Method: POST



Request Params: 
==================
{"[blue]#target#": "target_value", "[blue]#runname#" : "runname", "[blue]#test_pattern#": "optional test pattern"}
==================

Response: 








>
>







84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
Trigger a new Run
~~~~~~~~~~~~~~~~~~

URL: <base>/runs

Method: POST

Megatest Cmd:  megatest -runtests % -target <target> :runname <run_name> -run

Request Params: 
==================
{"[blue]#target#": "target_value", "[blue]#runname#" : "runname", "[blue]#test_pattern#": "optional test pattern"}
==================

Response: 

125
126
127
128
129
130
131



132
133
134
135
136
137
138
~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id

Method: GET

Filter Params: testpatt




Response: 

==================
[
   {
    "[red]#run_id#" : "2",







>
>
>







129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id

Method: GET

Filter Params: testpatt

Megatest Cmd: megatest -start-dir <path to  megatest area> -list-runs <runname> -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id


Response: 

==================
[
   {
    "[red]#run_id#" : "2",
186
187
188
189
190
191
192



193
194
195
196
197
198
199

Get List of tests within a run 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id/tests

Method: GET




Response: 
==================
[
     "[red]#tests#"  :
              [
                   {"[blue]#id#": 4, "[blue]#name#":[blue]#test1#, "[blue]#item_path#": "", "[blue]#shortdir#": "/temp/foo/bar/target2/runname2/test1", "[blue]#final_logf#": "megatest-rollup-test1.html",  "[blue]#status#": "PASS"}







>
>
>







193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209

Get List of tests within a run 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id/tests

Method: GET

Megatest Cmd: megatest -start-dir <path to  megatest area> -list-runs <runname> -target % -dumpmode json  -fields runs:runname,id+tests:state,status:id


Response: 
==================
[
     "[red]#tests#"  :
              [
                   {"[blue]#id#": 4, "[blue]#name#":[blue]#test1#, "[blue]#item_path#": "", "[blue]#shortdir#": "/temp/foo/bar/target2/runname2/test1", "[blue]#final_logf#": "megatest-rollup-test1.html",  "[blue]#status#": "PASS"}
220
221
222
223
224
225
226



227
228
229
230
231
232
233

Get perticular test that belongs to a Runs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id/tests/:id

Method: GET




Response: 

==================
{"[blue]#id#": "4", "[blue]#name#":"test1", "[blue]#item_path#": "", "[blue]#shortdir#": "/temp/foo/bar/target2/runname2/test1", "[blue]#final_logf#": "megatest-rollup-test1.html",  "[blue]#status#": "PASS"}
==================








>
>
>







230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246

Get perticular test that belongs to a Runs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

URL: <base>/runs/:id/tests/:id

Method: GET

Megatest Cmd: megatest -start-dir <path to  megatest area> -list-runs <runname> -target % -testpattern <pattern> -dumpmode json  -fields runs:runname,id+tests:state,status:id


Response: 

==================
{"[blue]#id#": "4", "[blue]#name#":"test1", "[blue]#item_path#": "", "[blue]#shortdir#": "/temp/foo/bar/target2/runname2/test1", "[blue]#final_logf#": "megatest-rollup-test1.html",  "[blue]#status#": "PASS"}
==================

Modified docs/dashboard.png from [5ee6b0359b] to [76099a5fb7].

cannot compute difference between binary files

Added docs/inprogress/megatest-architecture-2.fig version [a394cb2a13].

























































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#FIG 3.2  Produced by xfig version 3.2.5-alpha5
Landscape
Center
Inches
Letter  
100.00
Single
-2
1200 2
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 750 975 5850 975 5850 7425 750 7425 750 975
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 6000 975 9975 975 9975 7425 6000 7425 6000 975
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 900 1500 5250 1500 5250 2475 900 2475 900 1500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 900 2625 5250 2625 5250 3675 900 3675 900 2625
2 3 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 8
	 5325 1500 5325 5850 900 5850 900 7275 5700 7275 5700 1425
	 5250 1425 5325 1500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 900 3750 5250 3750 5250 4725 900 4725 900 3750
4 0 0 50 -1 0 12 0.0000 4 165 1170 975 1275 megatest.exe\001
4 0 0 50 -1 0 12 0.0000 4 150 1275 6150 1275 dashboard.exe\001
4 0 0 50 -1 0 12 0.0000 4 195 900 1050 1725 run engine\001
4 0 0 50 -1 0 12 0.0000 4 150 780 975 2850 database\001
4 0 0 50 -1 0 12 0.0000 4 195 2025 1050 6075 data transport - use http\001
4 0 0 50 -1 0 12 0.0000 4 195 2325 975 3900 test or launch management\001

Added docs/inprogress/megatest-architecture-proposed.fig version [9938b93bd4].

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
#FIG 3.2  Produced by xfig version 3.2.5-alpha5
Landscape
Center
Inches
Letter  
100.00
Single
-2
1200 2
6 600 1350 1575 2400
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1125 1500 450 150 1125 1500 1575 1650
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1124 2177 450 150 1124 2177 1574 2327
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 675 1575 675 2175
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1575 1500 1575 2175
-6
6 1875 825 2850 1875
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2400 975 450 150 2400 975 2850 1125
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2399 1652 450 150 2399 1652 2849 1802
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1950 1050 1950 1650
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 2850 975 2850 1650
-6
6 3225 450 4200 1500
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3750 600 450 150 3750 600 4200 750
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3749 1277 450 150 3749 1277 4199 1427
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3300 675 3300 1275
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 4200 600 4200 1275
-6
6 3075 2925 4050 3975
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 3075 450 150 3600 3075 4050 3225
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3599 3752 450 150 3599 3752 4049 3902
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3150 3150 3150 3750
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 4050 3075 4050 3750
-6
6 7275 4050 12825 9675
6 8175 4125 8400 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4125 8400 4125 8400 4350 8175 4350 8175 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4350 8400 4350 8400 4575 8175 4575 8175 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4575 8400 4575 8400 4800 8175 4800 8175 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4800 8400 4800 8400 5025 8175 5025 8175 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5025 8400 5025 8400 5250 8175 5250 8175 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5250 8400 5250 8400 5475 8175 5475 8175 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5475 8400 5475 8400 5700 8175 5700 8175 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5700 8400 5700 8400 5925 8175 5925 8175 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5925 8400 5925 8400 6150 8175 6150 8175 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6150 8400 6150 8400 6375 8175 6375 8175 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6375 8400 6375 8400 6600 8175 6600 8175 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6600 8400 6600 8400 6825 8175 6825 8175 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6825 8400 6825 8400 7050 8175 7050 8175 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7050 8400 7050 8400 7275 8175 7275 8175 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7275 8400 7275 8400 7500 8175 7500 8175 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7500 8400 7500 8400 7725 8175 7725 8175 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7725 8400 7725 8400 7950 8175 7950 8175 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7950 8400 7950 8400 8175 8175 8175 8175 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 8175 8400 8175 8400 8400 8175 8400 8175 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 8400 8400 8400 8400 8625 8175 8625 8175 8400
-6
6 8475 4125 8700 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4125 8700 4125 8700 4350 8475 4350 8475 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4350 8700 4350 8700 4575 8475 4575 8475 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4575 8700 4575 8700 4800 8475 4800 8475 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4800 8700 4800 8700 5025 8475 5025 8475 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5025 8700 5025 8700 5250 8475 5250 8475 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5250 8700 5250 8700 5475 8475 5475 8475 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5475 8700 5475 8700 5700 8475 5700 8475 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5700 8700 5700 8700 5925 8475 5925 8475 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5925 8700 5925 8700 6150 8475 6150 8475 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6150 8700 6150 8700 6375 8475 6375 8475 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6375 8700 6375 8700 6600 8475 6600 8475 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6600 8700 6600 8700 6825 8475 6825 8475 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6825 8700 6825 8700 7050 8475 7050 8475 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7050 8700 7050 8700 7275 8475 7275 8475 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7275 8700 7275 8700 7500 8475 7500 8475 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7500 8700 7500 8700 7725 8475 7725 8475 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7725 8700 7725 8700 7950 8475 7950 8475 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7950 8700 7950 8700 8175 8475 8175 8475 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 8175 8700 8175 8700 8400 8475 8400 8475 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 8400 8700 8400 8700 8625 8475 8625 8475 8400
-6
6 8775 4125 9000 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4125 9000 4125 9000 4350 8775 4350 8775 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4350 9000 4350 9000 4575 8775 4575 8775 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4575 9000 4575 9000 4800 8775 4800 8775 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4800 9000 4800 9000 5025 8775 5025 8775 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5025 9000 5025 9000 5250 8775 5250 8775 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5250 9000 5250 9000 5475 8775 5475 8775 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5475 9000 5475 9000 5700 8775 5700 8775 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5700 9000 5700 9000 5925 8775 5925 8775 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5925 9000 5925 9000 6150 8775 6150 8775 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6150 9000 6150 9000 6375 8775 6375 8775 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6375 9000 6375 9000 6600 8775 6600 8775 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6600 9000 6600 9000 6825 8775 6825 8775 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6825 9000 6825 9000 7050 8775 7050 8775 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7050 9000 7050 9000 7275 8775 7275 8775 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7275 9000 7275 9000 7500 8775 7500 8775 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7500 9000 7500 9000 7725 8775 7725 8775 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7725 9000 7725 9000 7950 8775 7950 8775 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7950 9000 7950 9000 8175 8775 8175 8775 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 8175 9000 8175 9000 8400 8775 8400 8775 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 8400 9000 8400 9000 8625 8775 8625 8775 8400
-6
6 9075 4125 9300 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4125 9300 4125 9300 4350 9075 4350 9075 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4350 9300 4350 9300 4575 9075 4575 9075 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4575 9300 4575 9300 4800 9075 4800 9075 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4800 9300 4800 9300 5025 9075 5025 9075 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5025 9300 5025 9300 5250 9075 5250 9075 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5250 9300 5250 9300 5475 9075 5475 9075 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5475 9300 5475 9300 5700 9075 5700 9075 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5700 9300 5700 9300 5925 9075 5925 9075 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5925 9300 5925 9300 6150 9075 6150 9075 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6150 9300 6150 9300 6375 9075 6375 9075 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6375 9300 6375 9300 6600 9075 6600 9075 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6600 9300 6600 9300 6825 9075 6825 9075 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6825 9300 6825 9300 7050 9075 7050 9075 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7050 9300 7050 9300 7275 9075 7275 9075 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7275 9300 7275 9300 7500 9075 7500 9075 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7500 9300 7500 9300 7725 9075 7725 9075 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7725 9300 7725 9300 7950 9075 7950 9075 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7950 9300 7950 9300 8175 9075 8175 9075 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 8175 9300 8175 9300 8400 9075 8400 9075 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 8400 9300 8400 9300 8625 9075 8625 9075 8400
-6
6 9375 4125 9600 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4125 9600 4125 9600 4350 9375 4350 9375 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4350 9600 4350 9600 4575 9375 4575 9375 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4575 9600 4575 9600 4800 9375 4800 9375 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4800 9600 4800 9600 5025 9375 5025 9375 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5025 9600 5025 9600 5250 9375 5250 9375 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5250 9600 5250 9600 5475 9375 5475 9375 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5475 9600 5475 9600 5700 9375 5700 9375 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5700 9600 5700 9600 5925 9375 5925 9375 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5925 9600 5925 9600 6150 9375 6150 9375 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6150 9600 6150 9600 6375 9375 6375 9375 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6375 9600 6375 9600 6600 9375 6600 9375 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6600 9600 6600 9600 6825 9375 6825 9375 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6825 9600 6825 9600 7050 9375 7050 9375 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7050 9600 7050 9600 7275 9375 7275 9375 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7275 9600 7275 9600 7500 9375 7500 9375 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7500 9600 7500 9600 7725 9375 7725 9375 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7725 9600 7725 9600 7950 9375 7950 9375 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7950 9600 7950 9600 8175 9375 8175 9375 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 8175 9600 8175 9600 8400 9375 8400 9375 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 8400 9600 8400 9600 8625 9375 8625 9375 8400
-6
# Dimension line: 1-1/16 in
6 7875 9375 9150 9675
# main dimension line
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2
	1 1 1.00 60.00 120.00
	1 1 1.00 60.00 120.00
	 7875 9525 9150 9525
# text box
2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5
	 8085 9375 8085 9675 8939 9675 8939 9375 8085 9375
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7875 9375 7875 9675
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 9150 9375 9150 9675
4 1 0 48 -1 0 12 -0.0000 4 180 735 8512 9585 1-1/16 in\001
-6
# Dimension line: 1-11/16 in
6 7425 4125 7725 6150
# main dimension line
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2
	1 1 1.00 60.00 120.00
	1 1 1.00 60.00 120.00
	 7575 4125 7575 6150
# text box
2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5
	 7425 5617 7725 5617 7725 4657 7425 4657 7425 5617
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7425 6150 7725 6150
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7425 4125 7725 4125
4 1 0 48 -1 0 12 1.5708 4 180 840 7635 5137 1-11/16 in\001
-6
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 7800 4050 12825 4050 12825 8925 7800 8925 7800 4050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9225 6225 12450 6225 12450 8325 9225 8325 9225 6225
2 2 0 1 4 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9225 6150 9675 6150 9675 8400 9225 8400 9225 6150
4 0 0 50 -1 0 12 0.0000 4 150 150 8475 9300 X\001
4 0 0 50 -1 0 12 0.0000 4 150 135 7275 6975 Y\001
-6
6 14100 150 19950 6075
6 14850 1350 15825 2400
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 1500 450 150 15375 1500 15825 1650
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 2177 450 150 15374 2177 15824 2327
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 14925 1575 14925 2175
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 15825 1500 15825 2175
-6
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 17025 4050 17025 3450 15750 3450 15750 4050 17025 4050
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16050 3375 15525 2400
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 15825 5325 17175 5325 17175 5850 15825 5850 15825 5325
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16350 4050 16350 5325
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 17550 4800 18900 4800 18900 5325 17550 5325 17550 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 18375 3900 19725 3900 19725 4425 18375 4425 18375 3900
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16725 4050 17850 4800
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 17025 3750 18375 4125
2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 18975 3900 18075 2625 15900 1875
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 14100 150 19950 150 19950 6075 14100 6075 14100 150
4 0 0 50 -1 0 12 0.0000 4 150 900 15825 3675 rpc-server\001
4 0 0 50 -1 0 12 0.0000 4 165 270 17475 3825 tcp\001
4 0 0 50 -1 0 12 0.0000 4 120 315 18525 4125 test\001
4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 1200 megatest.db\001
4 0 0 50 -1 0 12 0.0000 4 150 1020 14325 525 basic model\001
-6
6 14850 7425 15825 8475
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 7575 450 150 15375 7575 15825 7725
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 8252 450 150 15374 8252 15824 8402
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 14925 7650 14925 8250
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 15825 7575 15825 8250
-6
6 17775 6675 18750 7725
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18300 6825 450 150 18300 6825 18750 6975
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18299 7502 450 150 18299 7502 18749 7652
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 17850 6900 17850 7500
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 18750 6825 18750 7500
-6
6 4875 6075 5850 7125
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5400 6225 450 150 5400 6225 5850 6375
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5399 6902 450 150 5399 6902 5849 7052
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 4950 6300 4950 6900
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 5850 6225 5850 6900
-6
6 5400 7425 7350 8925
6 5475 7650 6450 8700
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 6000 7800 450 150 6000 7800 6450 7950
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5999 8477 450 150 5999 8477 6449 8627
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 5550 7875 5550 8475
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 6450 7800 6450 8475
-6
4 0 0 50 -1 0 12 0.0000 4 195 1905 5400 8850 pointers to the servers\001
4 0 0 50 -1 0 12 0.0000 4 150 930 5550 7575 monitor.db\001
-6
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 1725 5025 1275 2475
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 5550 4500 5550 225 225 225 225 4500 5550 4500
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 1875 7725 1875 5775
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 2775 5400 7125 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 1425 7725 2775 7725 2775 8250 1425 8250 1425 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 3375 7725 4725 7725 4725 8250 3375 8250 3375 7725
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 3675 7725 2175 5775
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 2775 5700 2775 5100 1500 5100 1500 5700 2775 5700
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 6600 3300 2925 5025
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 17025 10125 17025 9525 15750 9525 15750 10125 17025 10125
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16050 9450 15525 8475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 15825 11400 17175 11400 17175 11925 15825 11925 15825 11400
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16350 10125 16350 11400
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 17550 10875 18900 10875 18900 11400 17550 11400 17550 10875
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 18375 9975 19725 9975 19725 10500 18375 10500 18375 9975
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16725 10125 17850 10875
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 17025 9825 18375 10200
2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 18975 9975 18075 8700 15900 7950
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 14100 6225 19950 6225 19950 12150 14100 12150 14100 6225
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16575 9375 17850 7950
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2100 10425 6150 10425 6150 14400 2100 14400 2100 10425
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2325 10875 5925 10875 5925 13800 2325 13800 2325 10875
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2400 10950 3975 10950 3975 11625 2400 11625 2400 10950
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 5325 12675 5325 12075 4050 12075 4050 12675 5325 12675
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 3975 11250 4575 12075
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 2175 5025 3075 3750
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 6150 2700 7500 2700 7500 3225 6150 3225 6150 2700
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 4800 6375 2850 5550
4 0 0 50 -1 0 12 0.0000 4 195 990 1800 2625 last_update\001
4 0 0 50 -1 0 12 0.0000 4 150 690 3150 4125 main.db\001
4 0 0 50 -1 0 12 0.0000 4 195 990 4200 3600 last_update\001
4 0 0 50 -1 0 12 0.0000 4 195 330 1950 6825 http\001
4 0 0 50 -1 0 12 0.0000 4 180 870 1575 7950 run1/test1\001
4 0 0 50 -1 0 12 0.0000 4 150 720 1650 5400 server-1\001
4 0 0 50 -1 0 12 0.0000 4 150 375 2175 2025 2.db\001
4 0 0 50 -1 0 12 0.0000 4 150 375 750 2550 1.db\001
4 0 0 50 -1 0 12 0.0000 4 180 870 3450 7950 run1/test2\001
4 0 0 50 -1 0 12 0.0000 4 150 1110 9675 3750 Dashboardm\001
4 0 0 50 -1 0 12 1.5708 4 150 390 8325 3975 run1\001
4 0 0 50 -1 0 12 0.0000 4 150 900 15825 9750 rpc-server\001
4 0 0 50 -1 0 12 0.0000 4 165 270 17475 9900 tcp\001
4 0 0 50 -1 0 12 0.0000 4 120 315 18525 10200 test\001
4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 7275 megatest.db\001
4 0 0 50 -1 0 12 0.0000 4 195 1305 17700 7875 mysql/postgres\001
4 0 0 50 -1 0 12 0.0000 4 195 1875 14325 6600 mysql/postgres model\001
4 0 0 50 -1 0 12 0.0000 4 195 4065 600 9300 Current state, no bypass - (if we switch to rpc?)\001
4 0 0 50 -1 0 12 0.0000 4 195 1125 2175 10650 prev try RPC\001
4 0 0 50 -1 0 12 0.0000 4 165 1095 2475 11100 rmt:get-tests\001
4 0 0 50 -1 0 12 0.0000 4 195 2205 450 525 /tmp/<user>/??? /.db/*.db\001
4 0 0 50 -1 0 12 0.0000 4 180 870 6225 2925 run2/test1\001
4 0 0 50 -1 0 12 0.0000 4 195 1065 4800 7350 megatest.db\001
4 0 0 50 -1 0 12 0.0000 4 150 1785 600 8775 Possible Future state\001
4 0 0 50 -1 0 12 0.0000 4 150 1110 8025 450 CHANGES:\001
4 0 0 50 -1 0 12 0.0000 4 195 2145 8025 705 1. http -> rcp or nanomsg\001
4 0 0 50 -1 0 12 0.0000 4 195 3330 8025 960 2. cache db moves from inmem to /tmp\001

Added docs/inprogress/megatest-architecture.fig version [e8481e20cc].

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
#FIG 3.2  Produced by xfig version 3.2.5-alpha5
Landscape
Center
Inches
Letter  
100.00
Single
-2
1200 2
6 600 1350 1575 2400
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1125 1500 450 150 1125 1500 1575 1650
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1124 2177 450 150 1124 2177 1574 2327
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 675 1575 675 2175
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1575 1500 1575 2175
-6
6 1875 825 2850 1875
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2400 975 450 150 2400 975 2850 1125
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2399 1652 450 150 2399 1652 2849 1802
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1950 1050 1950 1650
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 2850 975 2850 1650
-6
6 3225 450 4200 1500
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3750 600 450 150 3750 600 4200 750
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3749 1277 450 150 3749 1277 4199 1427
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3300 675 3300 1275
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 4200 600 4200 1275
-6
6 3075 2925 4050 3975
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 3075 450 150 3600 3075 4050 3225
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3599 3752 450 150 3599 3752 4049 3902
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3150 3150 3150 3750
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 4050 3075 4050 3750
-6
6 7275 4050 12825 9675
6 8175 4125 8400 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4125 8400 4125 8400 4350 8175 4350 8175 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4350 8400 4350 8400 4575 8175 4575 8175 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4575 8400 4575 8400 4800 8175 4800 8175 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 4800 8400 4800 8400 5025 8175 5025 8175 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5025 8400 5025 8400 5250 8175 5250 8175 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5250 8400 5250 8400 5475 8175 5475 8175 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5475 8400 5475 8400 5700 8175 5700 8175 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5700 8400 5700 8400 5925 8175 5925 8175 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 5925 8400 5925 8400 6150 8175 6150 8175 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6150 8400 6150 8400 6375 8175 6375 8175 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6375 8400 6375 8400 6600 8175 6600 8175 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6600 8400 6600 8400 6825 8175 6825 8175 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 6825 8400 6825 8400 7050 8175 7050 8175 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7050 8400 7050 8400 7275 8175 7275 8175 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7275 8400 7275 8400 7500 8175 7500 8175 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7500 8400 7500 8400 7725 8175 7725 8175 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7725 8400 7725 8400 7950 8175 7950 8175 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 7950 8400 7950 8400 8175 8175 8175 8175 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 8175 8400 8175 8400 8400 8175 8400 8175 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8175 8400 8400 8400 8400 8625 8175 8625 8175 8400
-6
6 8475 4125 8700 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4125 8700 4125 8700 4350 8475 4350 8475 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4350 8700 4350 8700 4575 8475 4575 8475 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4575 8700 4575 8700 4800 8475 4800 8475 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 4800 8700 4800 8700 5025 8475 5025 8475 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5025 8700 5025 8700 5250 8475 5250 8475 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5250 8700 5250 8700 5475 8475 5475 8475 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5475 8700 5475 8700 5700 8475 5700 8475 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5700 8700 5700 8700 5925 8475 5925 8475 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 5925 8700 5925 8700 6150 8475 6150 8475 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6150 8700 6150 8700 6375 8475 6375 8475 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6375 8700 6375 8700 6600 8475 6600 8475 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6600 8700 6600 8700 6825 8475 6825 8475 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 6825 8700 6825 8700 7050 8475 7050 8475 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7050 8700 7050 8700 7275 8475 7275 8475 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7275 8700 7275 8700 7500 8475 7500 8475 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7500 8700 7500 8700 7725 8475 7725 8475 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7725 8700 7725 8700 7950 8475 7950 8475 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 7950 8700 7950 8700 8175 8475 8175 8475 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 8175 8700 8175 8700 8400 8475 8400 8475 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8475 8400 8700 8400 8700 8625 8475 8625 8475 8400
-6
6 8775 4125 9000 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4125 9000 4125 9000 4350 8775 4350 8775 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4350 9000 4350 9000 4575 8775 4575 8775 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4575 9000 4575 9000 4800 8775 4800 8775 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 4800 9000 4800 9000 5025 8775 5025 8775 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5025 9000 5025 9000 5250 8775 5250 8775 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5250 9000 5250 9000 5475 8775 5475 8775 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5475 9000 5475 9000 5700 8775 5700 8775 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5700 9000 5700 9000 5925 8775 5925 8775 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 5925 9000 5925 9000 6150 8775 6150 8775 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6150 9000 6150 9000 6375 8775 6375 8775 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6375 9000 6375 9000 6600 8775 6600 8775 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6600 9000 6600 9000 6825 8775 6825 8775 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 6825 9000 6825 9000 7050 8775 7050 8775 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7050 9000 7050 9000 7275 8775 7275 8775 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7275 9000 7275 9000 7500 8775 7500 8775 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7500 9000 7500 9000 7725 8775 7725 8775 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7725 9000 7725 9000 7950 8775 7950 8775 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 7950 9000 7950 9000 8175 8775 8175 8775 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 8175 9000 8175 9000 8400 8775 8400 8775 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8775 8400 9000 8400 9000 8625 8775 8625 8775 8400
-6
6 9075 4125 9300 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4125 9300 4125 9300 4350 9075 4350 9075 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4350 9300 4350 9300 4575 9075 4575 9075 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4575 9300 4575 9300 4800 9075 4800 9075 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 4800 9300 4800 9300 5025 9075 5025 9075 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5025 9300 5025 9300 5250 9075 5250 9075 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5250 9300 5250 9300 5475 9075 5475 9075 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5475 9300 5475 9300 5700 9075 5700 9075 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5700 9300 5700 9300 5925 9075 5925 9075 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 5925 9300 5925 9300 6150 9075 6150 9075 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6150 9300 6150 9300 6375 9075 6375 9075 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6375 9300 6375 9300 6600 9075 6600 9075 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6600 9300 6600 9300 6825 9075 6825 9075 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 6825 9300 6825 9300 7050 9075 7050 9075 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7050 9300 7050 9300 7275 9075 7275 9075 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7275 9300 7275 9300 7500 9075 7500 9075 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7500 9300 7500 9300 7725 9075 7725 9075 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7725 9300 7725 9300 7950 9075 7950 9075 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 7950 9300 7950 9300 8175 9075 8175 9075 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 8175 9300 8175 9300 8400 9075 8400 9075 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9075 8400 9300 8400 9300 8625 9075 8625 9075 8400
-6
6 9375 4125 9600 8625
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4125 9600 4125 9600 4350 9375 4350 9375 4125
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4350 9600 4350 9600 4575 9375 4575 9375 4350
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4575 9600 4575 9600 4800 9375 4800 9375 4575
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 4800 9600 4800 9600 5025 9375 5025 9375 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5025 9600 5025 9600 5250 9375 5250 9375 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5250 9600 5250 9600 5475 9375 5475 9375 5250
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5475 9600 5475 9600 5700 9375 5700 9375 5475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5700 9600 5700 9600 5925 9375 5925 9375 5700
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 5925 9600 5925 9600 6150 9375 6150 9375 5925
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6150 9600 6150 9600 6375 9375 6375 9375 6150
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6375 9600 6375 9600 6600 9375 6600 9375 6375
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6600 9600 6600 9600 6825 9375 6825 9375 6600
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 6825 9600 6825 9600 7050 9375 7050 9375 6825
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7050 9600 7050 9600 7275 9375 7275 9375 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7275 9600 7275 9600 7500 9375 7500 9375 7275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7500 9600 7500 9600 7725 9375 7725 9375 7500
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7725 9600 7725 9600 7950 9375 7950 9375 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 7950 9600 7950 9600 8175 9375 8175 9375 7950
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 8175 9600 8175 9600 8400 9375 8400 9375 8175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9375 8400 9600 8400 9600 8625 9375 8625 9375 8400
-6
# Dimension line: 1-1/16 in
6 7875 9375 9150 9675
# main dimension line
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2
	1 1 1.00 60.00 120.00
	1 1 1.00 60.00 120.00
	 7875 9525 9150 9525
# text box
2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5
	 8085 9375 8085 9675 8939 9675 8939 9375 8085 9375
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7875 9375 7875 9675
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 9150 9375 9150 9675
4 1 0 48 -1 0 12 -0.0000 4 180 735 8512 9585 1-1/16 in\001
-6
# Dimension line: 1-11/16 in
6 7425 4125 7725 6150
# main dimension line
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2
	1 1 1.00 60.00 120.00
	1 1 1.00 60.00 120.00
	 7575 4125 7575 6150
# text box
2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5
	 7425 5617 7725 5617 7725 4657 7425 4657 7425 5617
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7425 6150 7725 6150
# tick
2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2
	 7425 4125 7725 4125
4 1 0 48 -1 0 12 1.5708 4 180 840 7635 5137 1-11/16 in\001
-6
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 7800 4050 12825 4050 12825 8925 7800 8925 7800 4050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9225 6225 12450 6225 12450 8325 9225 8325 9225 6225
2 2 0 1 4 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9225 6150 9675 6150 9675 8400 9225 8400 9225 6150
4 0 0 50 -1 0 12 0.0000 4 150 150 8475 9300 X\001
4 0 0 50 -1 0 12 0.0000 4 150 135 7275 6975 Y\001
-6
6 975 5100 1500 5700
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1258 5186 242 86 1258 5186 1500 5271
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1257 5573 242 86 1257 5573 1499 5658
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1015 5229 1015 5571
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 1500 5186 1500 5571
-6
6 3000 6075 3525 6675
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3283 6161 242 86 3283 6161 3525 6246
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3282 6548 242 86 3282 6548 3524 6633
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3040 6204 3040 6546
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 3525 6161 3525 6546
-6
6 7575 2625 8100 3225
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 7858 2711 242 86 7858 2711 8100 2796
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 7857 3098 242 86 7857 3098 8099 3183
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 7615 2754 7615 3096
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 8100 2711 8100 3096
-6
6 9525 450 10500 1500
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 10050 600 450 150 10050 600 10500 750
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 10049 1277 450 150 10049 1277 10499 1427
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 9600 675 9600 1275
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 10500 600 10500 1275
-6
6 14100 150 19950 6075
6 14850 1350 15825 2400
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 1500 450 150 15375 1500 15825 1650
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 2177 450 150 15374 2177 15824 2327
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 14925 1575 14925 2175
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 15825 1500 15825 2175
-6
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 17025 4050 17025 3450 15750 3450 15750 4050 17025 4050
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16050 3375 15525 2400
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 15825 5325 17175 5325 17175 5850 15825 5850 15825 5325
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16350 4050 16350 5325
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 17550 4800 18900 4800 18900 5325 17550 5325 17550 4800
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 18375 3900 19725 3900 19725 4425 18375 4425 18375 3900
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16725 4050 17850 4800
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 17025 3750 18375 4125
2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 18975 3900 18075 2625 15900 1875
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 14100 150 19950 150 19950 6075 14100 6075 14100 150
4 0 0 50 -1 0 12 0.0000 4 150 900 15825 3675 rpc-server\001
4 0 0 50 -1 0 12 0.0000 4 165 270 17475 3825 tcp\001
4 0 0 50 -1 0 12 0.0000 4 120 315 18525 4125 test\001
4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 1200 megatest.db\001
4 0 0 50 -1 0 12 0.0000 4 150 1020 14325 525 basic model\001
-6
6 14850 7425 15825 8475
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 7575 450 150 15375 7575 15825 7725
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 8252 450 150 15374 8252 15824 8402
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 14925 7650 14925 8250
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 15825 7575 15825 8250
-6
6 17775 6675 18750 7725
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18300 6825 450 150 18300 6825 18750 6975
1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18299 7502 450 150 18299 7502 18749 7652
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 17850 6900 17850 7500
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 18750 6825 18750 7500
-6
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 1725 5025 1275 2475
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 5550 4500 5550 225 225 225 225 4500 5550 4500
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 1875 7725 1875 5775
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 2775 5400 7125 5700
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 2400 7725 3900 6675
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 4275 6075 3825 4200
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 7125 5850 4800 6300
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 1425 7725 2775 7725 2775 8250 1425 8250 1425 7725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 3375 7725 4725 7725 4725 8250 3375 8250 3375 7725
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 3675 7725 2175 5775
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 3975 7725 3975 6750
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 7125 825 8475 825 8475 1350 7125 1350 7125 825
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 2775 5700 2775 5100 1500 5100 1500 5700 2775 5700
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 4800 6675 4800 6075 3525 6075 3525 6675 4800 6675
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 7575 3225 7575 2625 6300 2625 6300 3225 7575 3225
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 7650 1425 6975 2625
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 6300 2925 2850 1725
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 6975 3300 7725 3900
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 4
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 7200 1350 5925 2550 5925 4875 4650 6000
2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 4
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 1425 8025 825 6750 825 2850 900 2700
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 17025 10125 17025 9525 15750 9525 15750 10125 17025 10125
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16050 9450 15525 8475
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 15825 11400 17175 11400 17175 11925 15825 11925 15825 11400
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16350 10125 16350 11400
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 17550 10875 18900 10875 18900 11400 17550 11400 17550 10875
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 18375 9975 19725 9975 19725 10500 18375 10500 18375 9975
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16725 10125 17850 10875
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 17025 9825 18375 10200
2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 18975 9975 18075 8700 15900 7950
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 14100 6225 19950 6225 19950 12150 14100 12150 14100 6225
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2
	0 0 1.00 60.00 120.00
	0 0 1.00 60.00 120.00
	 16575 9375 17850 7950
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2100 10425 6150 10425 6150 14400 2100 14400 2100 10425
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2325 10875 5925 10875 5925 13800 2325 13800 2325 10875
2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5
	 2400 10950 3975 10950 3975 11625 2400 11625 2400 10950
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 5325 12675 5325 12075 4050 12075 4050 12675 5325 12675
2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 3975 11250 4575 12075
4 0 0 50 -1 0 12 0.0000 4 195 990 1800 2625 last_update\001
4 0 0 50 -1 0 12 0.0000 4 150 690 3150 4125 main.db\001
4 0 0 50 -1 0 12 0.0000 4 195 990 4200 3600 last_update\001
4 0 0 50 -1 0 12 0.0000 4 180 1500 450 525 link tree /.db/*.db\001
4 0 0 50 -1 0 12 0.0000 4 195 330 1950 6825 http\001
4 0 0 50 -1 0 12 0.0000 4 180 870 1575 7950 run1/test1\001
4 0 0 50 -1 0 12 0.0000 4 150 720 1650 5400 server-1\001
4 0 0 50 -1 0 12 0.0000 4 150 1035 3675 6375 server-main\001
4 0 0 50 -1 0 12 0.0000 4 150 375 2175 2025 2.db\001
4 0 0 50 -1 0 12 0.0000 4 150 375 750 2550 1.db\001
4 0 0 50 -1 0 12 0.0000 4 180 870 3450 7950 run1/test2\001
4 0 0 50 -1 0 12 0.0000 4 150 1110 9675 3750 Dashboardm\001
4 0 0 50 -1 0 12 0.0000 4 180 870 7275 1050 run2/test1\001
4 0 0 50 -1 0 12 0.0000 4 150 720 6375 2850 server-2\001
4 0 0 50 -1 0 12 1.5708 4 150 390 8325 3975 run1\001
4 0 0 50 -1 0 12 0.0000 4 195 1905 9450 1650 pointers to the servers\001
4 0 0 50 -1 0 12 0.0000 4 150 930 9600 375 monitor.db\001
4 0 0 50 -1 0 12 0.0000 4 150 900 15825 9750 rpc-server\001
4 0 0 50 -1 0 12 0.0000 4 165 270 17475 9900 tcp\001
4 0 0 50 -1 0 12 0.0000 4 120 315 18525 10200 test\001
4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 7275 megatest.db\001
4 0 0 50 -1 0 12 0.0000 4 195 1305 17700 7875 mysql/postgres\001
4 0 0 50 -1 0 12 0.0000 4 195 1875 14325 6600 mysql/postgres model\001
4 0 0 50 -1 0 12 0.0000 4 150 1140 600 8775 Current state\001
4 0 0 50 -1 0 12 0.0000 4 195 4065 600 9300 Current state, no bypass - (if we switch to rpc?)\001
4 0 0 50 -1 0 12 0.0000 4 195 1125 2175 10650 prev try RPC\001
4 0 0 50 -1 0 12 0.0000 4 165 1095 2475 11100 rmt:get-tests\001

Added docs/inprogress/megatest-query-view.fig version [b760721f42].























































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#FIG 3.2  Produced by xfig version 3.2.5-alpha5
Landscape
Center
Inches
Letter  
100.00
Single
-2
1200 2
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 900 675 4350 675 4350 1650 900 1650 900 675
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 4350 1200 6975 1725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 6975 1350 10725 1350 10725 3075 6975 3075 6975 1350
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 7125 1800 8925 2025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 8850 1875 10125 1875 10125 2550 8850 2550 8850 1875
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 9825 2550 10125 6000
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 11400 7350 11400 6000 9000 6000 9000 7350 11400 7350
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 9750 6000 9375 2550
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 8775 2250 7125 2025
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 6975 1800 4350 1275
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 975 3600 4350 3600 4350 4575 975 4575 975 3600
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 5
	0 0 1.00 60.00 120.00
	 1050 1650 1050 2025 4575 2025 4575 1050 4350 1050
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 4350 3825 6975 2700
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2
	0 0 1.00 60.00 120.00
	 9000 2625 4350 4200
4 0 0 50 -1 0 12 0.0000 4 150 390 1050 975 Test\001
4 0 0 50 -1 0 12 0.0000 4 150 585 7125 1650 Server\001
4 0 0 50 -1 0 12 0.0000 4 195 1020 4800 1125 http request\001
4 0 0 50 -1 0 12 0.0000 4 195 750 9075 2100 db query\001
4 0 0 50 -1 0 12 0.0000 4 150 345 9525 6375 disk\001
4 0 0 50 -1 0 12 0.0000 4 195 1905 1725 1200 (rmt:tests-get-info ....)\001
4 0 0 50 -1 0 12 0.0000 4 195 1905 1800 4125 (rmt:tests-get-info ....)\001
4 0 0 50 -1 0 12 0.0000 4 195 1110 5100 3300 send-request\001
4 0 0 50 -1 0 12 0.0000 4 150 2145 5475 3900 call-back with result data\001
4 0 0 50 -1 0 12 0.0000 4 150 390 1125 3750 Test\001
4 0 0 50 -1 0 12 0.0000 4 195 675 7350 2550 api.scm\001
4 0 0 50 -1 0 12 0.0000 4 195 990 7350 2805 recieve-req\001
4 0 0 50 -1 0 12 0.0000 4 150 1515 7350 3060 send-res-callback\001

Added docs/inprogress/megatest_qa.fig version [b36c055ff1].













































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
#FIG 3.2  Produced by xfig version 3.2.5-alpha5
Landscape
Center
Inches
Letter  
100.00
Single
-2
1200 2
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 6000 300 6000 9675
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 525 675 4500 675 4500 2550 525 2550 525 675
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 900 1125 2325 1125 2325 1575 900 1575 900 1125
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
	 225 150 225 7050
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 525 3150 3750 3150 3750 4275 525 4275 525 3150
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 12750 5025 12750 750 6450 750 6450 5025 12750 5025
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9300 1725 10800 1725 10800 2100 9300 2100 9300 1725
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 9300 2175 10800 2175 10800 2550 9300 2550 9300 2175
2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
	 6750 1275 12600 1275 12600 2925 6750 2925 6750 1275
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
	 12450 2700 12450 1650 6975 1650 6975 2700 12450 2700
4 0 0 50 -1 0 12 0.0000 4 120 405 675 900 tests\001
4 0 0 50 -1 0 12 0.0000 4 195 900 975 1425 nested_mt\001
4 0 0 50 -1 0 12 0.0000 4 195 2790 675 3375 nested_mt (a full megatest suite)\001
4 0 0 50 -1 0 12 0.0000 4 165 1110 375 5100 megatest_qa\001
4 0 0 50 -1 0 12 0.0000 4 150 420 525 300 code\001
4 0 0 50 -1 0 12 0.0000 4 150 1005 6750 375 Actual runs\001
4 0 0 50 -1 0 12 0.0000 4 165 1710 6675 1050 outer megatest runs\001
4 0 0 50 -1 0 12 0.0000 4 195 1785 6900 1500 test (e.g. nested_mt)\001
4 0 0 50 -1 0 12 0.0000 4 195 1665 7125 1875 nested_mt testsuite\001

Modified docs/manual/Makefile from [a64e790e18] to [bf9f3b126b].

11
12
13
14
15
16
17
18
19
20



21
22
23
24
25
26
27

# design_spec.html : $(SRCFILES) $(CSVFILES)
#         asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt
#

all : server.ps megatest_manual.html client.ps complex-itemmap.png

megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt *png
	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
#	dos2unix megatest_manual.html




server.ps : server.dot
	dot -Tps server.dot > server.ps

client.ps : client.dot
	dot -Tps client.dot > client.ps








|


>
>
>







11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# design_spec.html : $(SRCFILES) $(CSVFILES)
#         asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt
#

all : server.ps megatest_manual.html client.ps complex-itemmap.png

megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt installation.txt *png
	asciidoc  -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt
#	dos2unix megatest_manual.html

megatest.pdf : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt *png
	a2x -a toc -f pdf megatest_manual.txt

server.ps : server.dot
	dot -Tps server.dot > server.ps

client.ps : client.dot
	dot -Tps client.dot > client.ps

Modified docs/manual/getting_started.txt from [d4e6b12deb] to [de12299551].

1
2
3

4
5
6
7
8
9
10
11
12


13
14
15
16
17
18

19





20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52


53
54
55
56
57
58
59
60
61

62
63




64



65
66
67
68
69
70
71
72
73
74
75
76


77






78


79



80



81























Getting Started
===============


[partintro]
.Getting started with Megatest
--
How to install Megatest and set it up for running your regressions and continuous integration process.
--

Installation
------------



Dependencies
~~~~~~~~~~~~

Chicken scheme and a number of "eggs" are required for building
Megatest. See the script installall.sch in the utils directory of the

distribution for a mostly automated way to install everything needed





for building Megatest on Linux.

footnote:[An example footnote.]
indexterm:[Example index entry]

// 
// 
// And now for something completely different: ((monkeys)), lions and
// tigers (Bengal and Siberian) using the alternative syntax index
// entries.
// (((Big cats,Lions)))
// (((Big cats,Tigers,Bengal Tiger)))
// (((Big cats,Tigers,Siberian Tiger)))
// Note that multi-entry terms generate separate index entries.
// 
// Here are a couple of image examples: an image:images/smallnew.png[]
// example inline image followed by an example block image:
// 
// .Tiger block image
// image::images/tiger.png[Tiger image]
// 
// Followed by an example table:
// 
// .An example table
// [width="60%",options="header"]
// |==============================================
// | Option          | Description
// | -a 'USER GROUP' | Add 'USER' to 'GROUP'.
// | -R 'GROUP'      | Disables access to 'GROUP'.
// |==============================================
// 
// .An example example
// ===============================================


// Lorum ipum...
// ===============================================
// 
// [[X1]]
// Sub-section with Anchor
// ~~~~~~~~~~~~~~~~~~~~~~~
// Sub-section at level 2.
// 
// Chapter Sub-section

// ^^^^^^^^^^^^^^^^^^^
// Sub-section at level 3.




// 



// Chapter Sub-section
// +++++++++++++++++++
// Sub-section at level 4.
// 
// This is the maximum sub-section depth supported by the distributed
// AsciiDoc configuration.
// footnote:[A second example footnote.]
// 
// 
// The Second Chapter
// ------------------
// An example link to anchor at start of the <<X1,first sub-section>>.


// indexterm:[Second example index entry]






// 


// An example link to a bibliography entry <<taoup>>.



// 



// 
























<
>




|


|
<
>
>

|
|

<
|
>
|
>
>
>
>
>
|

<
|

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
|
|
|
<
<
<
<
|
<
>
>
|
|
|
<
<
<
<
<
<
>
|
|
>
>
>
>
|
>
>
>
|
<
|
<
<
|
<
<
<
|
|
|
>
>
|
>
>
>
>
>
>
|
>
>
|
>
>
>
|
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2

3
4
5
6
7
8
9
10
11

12
13
14
15
16
17

18
19
20
21
22
23
24
25
26
27

28
29


















30
31
32
33




34

35
36
37
38
39






40
41
42
43
44
45
46
47
48
49
50
51

52


53



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

Getting Started

---------------

[partintro]
.Getting started with Megatest
--
Creating a testsuite or flow and your first test or task.
--

After installing Megatest you can create a flow or testsuite and add some

tests using the helpers. Here is a quickstart sequence to get you up and
running your first automated testsuite.

Creating a Megatest Area
~~~~~~~~~~~~~~~~~~~~~~~~


Choose Target Keys
^^^^^^^^^^^^^^^^^^

First choose your "target" keys. These are used to organise your runs in a
way that is meaningful to your project. If you are unsure about what to use
for keys just use a single generic key such as "RUNTYPE". These keys will be
used to hand values to your tests via environment variables so ensure they
are unique. Prefixing them with something such as PROJKEYS_ is a good
strategy. 


Examples of keys:



















.Example keys
[width="60%",options="header"]
|==============================================
| Option            | Description




| RELEASE/ITERATION | This example is used by Megatest for its internal QA.

| ARCH/OS/RELEASE   | For a software project targeting multiple platforms
| UCTRLR/NODETYPE   | Microcontroller project with different controllers
running same software
|==============================================







Create Area Config Files
^^^^^^^^^^^^^^^^^^^^^^^^

You will need to choose locations for your runs (the data generated every
time you run the testsuite) and link tree. For getting started answer the
prompts with "runs" and "links". We use the Unix editor "vi" in the examples
below but you can use any plain text editor.

.Using the helper to create a Megatest area
------------------
megatest -create-megatest-area


# optional: verify that the settings are ok


vi megatest.config



vi runconfigs.config
------------------

Creating a Test
~~~~~~~~~~~~~~~

Choose the test name for your first test and run the helper. You can edit
the files after the initial creation. You will need to enter names and
scripts for the steps to be run and then edit the
tests/<testname>/testconfig file and modify the logpro rules to properly
process the log output from your steps. For your first test just hit enter
for the "waiton", "priority" and iteration variable prompts.

Hint: for geting started make your logpro rules very liberal. expect:error
patterns should match nothing and comment out expect:required rules.

.Using the helper to create a Megatest test
---------------
megatest -create-test myfirsttest

# then edit the generated config
vi tests/myfirsttest/testconfig
---------------

Running your test
~~~~~~~~~~~~~~~~~

First choose a target and runname. If you have a two-place target such as
RELEASE/ITERATION a target would look like v1.0/aff3 where v1.0 is the
RELEASE and aff3 is the ITERATION. For a run name just use something like
run1. 

.Running all tests (testpatt of "%" matches all tests)
---------------
megatest -run -target v1.0/aff3 -runname run1 -testpatt % -log run1.log
---------------

Viewing the results
~~~~~~~~~~~~~~~~~~~

Start the dashboard and browse your run in the "Runs" tab.

.Starting dashboard
----------------
dashboard -rows 24
----------------

Modified docs/manual/howto.txt from [8d66a8b9b7] to [b7c69b99ac].

1
2
3

4
5
6

7
8
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34
35

36
37
38
39
40
41
42
43
44
45

46
47
48
49
50
51
52

How To Do Things
================


Process Runs
------------


Remove Runs
~~~~~~~~~~~


From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that 
comes up push the clean test button. The command field will be prefilled with a template command for removing 
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.

.Remove the test diskperf and all it's items
----------------
megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v
----------------

.Remove all tests for all runs and all targets
----------------
megatest -remove-runs -target %/%/% -runname % -testpatt % -v
----------------

Archive Runs
~~~~~~~~~~~~


Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run 
durations, time stamps etc.) are all preserved in the megatest database.

For setup information see the Archiving topic in the reference section of this manual.

To Archive
^^^^^^^^^^


Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.

.Archive a full run
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %
----------------

To Restore
^^^^^^^^^^


.Retrieve a single test
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%
----------------

Hint: You can browse the archive using bup commands directly.


<
>


<
>


<
>
















<
>








<
>









<
>







1
2

3
4
5

6
7
8

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

26
27
28
29
30
31
32
33
34

35
36
37
38
39
40
41
42
43
44

45
46
47
48
49
50
51
52

How To Do Things

----------------

Process Runs

~~~~~~~~~~~~

Remove Runs

^^^^^^^^^^^

From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that 
comes up push the clean test button. The command field will be prefilled with a template command for removing 
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.

.Remove the test diskperf and all it's items
----------------
megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v
----------------

.Remove all tests for all runs and all targets
----------------
megatest -remove-runs -target %/%/% -runname % -testpatt % -v
----------------

Archive Runs

^^^^^^^^^^^^

Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run 
durations, time stamps etc.) are all preserved in the megatest database.

For setup information see the Archiving topic in the reference section of this manual.

To Archive

++++++++++

Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.

.Archive a full run
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %
----------------

To Restore

++++++++++

.Retrieve a single test
----------------
megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%
----------------

Hint: You can browse the archive using bup commands directly.
76
77
78
79
80
81
82
83

84
85
86
87
88
89

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes
------------------------

Tricks
======


This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.

Limiting your running jobs
--------------------------


The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.

In your testconfig:

----------------
[test_meta]
jobgroup group1
----------------

In your megatest.config:

---------------
[jobgroups]
group1 10
custdes 4
---------------




Debugging Tricks
----------------

Examining The Environment
~~~~~~~~~~~~~~~~~~~~~~~~~








<
>





<
>

















<
<
<







76
77
78
79
80
81
82

83
84
85
86
87
88

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106



107
108
109
110
111
112
113
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes
------------------------

Tricks

------

This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.

Limiting your running jobs

~~~~~~~~~~~~~~~~~~~~~~~~~~

The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.

In your testconfig:

----------------
[test_meta]
jobgroup group1
----------------

In your megatest.config:

---------------
[jobgroups]
group1 10
custdes 4
---------------




Debugging Tricks
----------------

Examining The Environment
~~~~~~~~~~~~~~~~~~~~~~~~~

Added docs/manual/installation.txt version [6d6c5f2d2a].





















>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
Installation
------------

Dependencies
~~~~~~~~~~~~

Chicken scheme and a number of "eggs" are required for building
Megatest. See the script installall.sh in the utils directory of the
source distribution for an automated way to install everything
needed for building Megatest on Linux.

Modified docs/manual/megatest_manual.html from [77414f8407] to [2d6199dc08].

1
2
3
4
5
6
7
8
9
10
11
12
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="generator" content="AsciiDoc 8.6.9">
<title>The Megatest Users Manual</title>
<style type="text/css">
/* Shared CSS for AsciiDoc xhtml11 and html5 backends */

/* Default font. */
body {
  font-family: Georgia,serif;




|







1
2
3
4
5
6
7
8
9
10
11
12
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="generator" content="AsciiDoc 8.6.7">
<title>The Megatest Users Manual</title>
<style type="text/css">
/* Shared CSS for AsciiDoc xhtml11 and html5 backends */

/* Default font. */
body {
  font-family: Georgia,serif;
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

ul, ol, li > p {
  margin-top: 0;
}
ul > li     { color: #aaa; }
ul > li > * { color: black; }

.monospaced, code, pre {
  font-family: "Courier New", Courier, monospace;
  font-size: inherit;
  color: navy;
  padding: 0;
  margin: 0;
}
pre {
  white-space: pre-wrap;
}

#author {
  color: #527bbd;
  font-weight: bold;
  font-size: 1.1em;
}
#email {







|
<
<
<



<
<
<







82
83
84
85
86
87
88
89



90
91
92



93
94
95
96
97
98
99

ul, ol, li > p {
  margin-top: 0;
}
ul > li     { color: #aaa; }
ul > li > * { color: black; }

pre {



  padding: 0;
  margin: 0;
}




#author {
  color: #527bbd;
  font-weight: bold;
  font-size: 1.1em;
}
#email {
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

div.exampleblock > div.content {
  border-left: 3px solid #dddddd;
  padding-left: 0.5em;
}

div.imageblock div.content { padding-left: 0; }
span.image img { border-style: none; vertical-align: text-bottom; }
a.image:visited { color: white; }

dl {
  margin-top: 0.8em;
  margin-bottom: 0.8em;
}
dt {







|







214
215
216
217
218
219
220
221
222
223
224
225
226
227
228

div.exampleblock > div.content {
  border-left: 3px solid #dddddd;
  padding-left: 0.5em;
}

div.imageblock div.content { padding-left: 0; }
span.image img { border-style: none; }
a.image:visited { color: white; }

dl {
  margin-top: 0.8em;
  margin-bottom: 0.8em;
}
dt {
415
416
417
418
419
420
421






422
423
424
425
426
427
428
div.unbreakable { page-break-inside: avoid; }


/*
 * xhtml11 specific
 *
 * */







div.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
div.tableblock > table {
  border: 3px solid #527bbd;







>
>
>
>
>
>







409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
div.unbreakable { page-break-inside: avoid; }


/*
 * xhtml11 specific
 *
 * */

tt {
  font-family: "Courier New", Courier, monospace;
  font-size: inherit;
  color: navy;
}

div.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
div.tableblock > table {
  border: 3px solid #527bbd;
448
449
450
451
452
453
454






455
456
457
458
459
460
461
}


/*
 * html5 specific
 *
 * */







table.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
thead, p.tableblock.header {
  font-weight: bold;







>
>
>
>
>
>







448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
}


/*
 * html5 specific
 *
 * */

.monospaced {
  font-family: "Courier New", Courier, monospace;
  font-size: inherit;
  color: navy;
}

table.tableblock {
  margin-top: 1.0em;
  margin-bottom: 1.5em;
}
thead, p.tableblock.header {
  font-weight: bold;
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
body.manpage div.sectionbody {
  margin-left: 3em;
}

@media print {
  body.manpage div#toc { display: none; }
}


@media screen {
  body {
    max-width: 50em; /* approximately 80 characters wide */
    margin-left: 16em;
  }

  #toc {







<
<







534
535
536
537
538
539
540


541
542
543
544
545
546
547
body.manpage div.sectionbody {
  margin-left: 3em;
}

@media print {
  body.manpage div#toc { display: none; }
}


@media screen {
  body {
    max-width: 50em; /* approximately 80 characters wide */
    margin-left: 16em;
  }

  #toc {
777
778
779
780
781
782
783


784
785

786
787
788
789
790
791
792
793
794

795
796

797
798
799
800
801
802
803
</div>
</div>
<div id="content">
<div class="sect1">
<h2 id="_preface">Preface</h2>
<div class="sectionbody">
<div class="paragraph"><p>This book is organised as three sub-books; getting started, writing tests and reference.</p></div>


<div class="sect2">
<h3 id="_why_megatest">Why Megatest?</h3>

<div class="paragraph"><p>The Megatest project was started for two reasons, the first was an
immediate and pressing need for a generalized tool to manage a suite
of regression tests and the second was the fact that the author had
written or maintained several such tools at different companies over
the years and it seemed a good thing to have a single open source
tool, flexible enough to meet the needs of any team doing continuous
integrating and or running a complex suite of tests for release
qualification.</p></div>
</div>

<div class="sect2">
<h3 id="_megatest_design_philosophy">Megatest Design Philosophy</h3>

<div class="paragraph"><p>Megatest is intended to provide the minimum needed resources to make
writing a suite of tests and tasks for implementing continuous build
for software, design engineering or process control (via owlfs for
example) without being specialized for any specific problem
space. Megatest in of itself does not know what constitutes a PASS or
FAIL of a test or task. In most cases megatest is best used in
conjunction with logpro or a similar tool to parse, analyze and decide







>
>
|
|
>









>
|
|
>







781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
</div>
</div>
<div id="content">
<div class="sect1">
<h2 id="_preface">Preface</h2>
<div class="sectionbody">
<div class="paragraph"><p>This book is organised as three sub-books; getting started, writing tests and reference.</p></div>
</div>
</div>
<div class="sect1">
<h2 id="_why_megatest">Why Megatest?</h2>
<div class="sectionbody">
<div class="paragraph"><p>The Megatest project was started for two reasons, the first was an
immediate and pressing need for a generalized tool to manage a suite
of regression tests and the second was the fact that the author had
written or maintained several such tools at different companies over
the years and it seemed a good thing to have a single open source
tool, flexible enough to meet the needs of any team doing continuous
integrating and or running a complex suite of tests for release
qualification.</p></div>
</div>
</div>
<div class="sect1">
<h2 id="_megatest_design_philosophy">Megatest Design Philosophy</h2>
<div class="sectionbody">
<div class="paragraph"><p>Megatest is intended to provide the minimum needed resources to make
writing a suite of tests and tasks for implementing continuous build
for software, design engineering or process control (via owlfs for
example) without being specialized for any specific problem
space. Megatest in of itself does not know what constitutes a PASS or
FAIL of a test or task. In most cases megatest is best used in
conjunction with logpro or a similar tool to parse, analyze and decide
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846

847
848

849
850
851
852
853
854
855
856
857
858
859

860
861

862
863



























864








































865


866
867
868
869
870
871
872
873
<li>
<p>
Relocatable - the testsuite or automation area can be checked out and the tests run anywhere
</p>
</li>
<li>
<p>
Encapsulated - the area where the tests run are self-contained and all inputs
   and outputs to the process can be found in the run areas.
</p>
</li>
<li>
<p>
Deployable - anyone on the team, at any site, at any time can run the flow
</p>
</li>
</ul></div>
</div>

<div class="sect2">
<h3 id="_megatest_architecture">Megatest Architecture</h3>

<div class="paragraph"><p>All data to specify the tests and configure the system is stored in
plain text files. All system state is stored in an sqlite3
database. Tests are launched using the launching system available for
the distributed compute platform in use. A template script is provided
which can launch jobs on local and remote Linux hosts. Currently
megatest uses the network filesystem to call home to your master
sqlite3 database.</p></div>
</div>
</div>
</div>
<h1 id="_road_map">Road Map</h1>

<div class="paragraph"><p>Note 1: This road-map is tentative and subject to change without notice.</p></div>
<div class="paragraph"><p>Note 2: Starting over. Old plan is commented out.</p></div>

<div class="sect1">
<h2 id="_current_items">Current Items</h2>



























<div class="sectionbody">








































<div class="sect2">


<h3 id="_ww05_migrate_to_inmem_db">ww05 - migrate to inmem-db</h3>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Switch to inmem db with fast sync to on disk db&#8217;s [DONE]
</p>
</li>
<li>







|










>
|
|
>









|
|
>
|
|
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>
|







838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
<li>
<p>
Relocatable - the testsuite or automation area can be checked out and the tests run anywhere
</p>
</li>
<li>
<p>
Encapsulated - the tests run in self-contained directories and all inputs
   and outputs to the process can be found in the run areas.
</p>
</li>
<li>
<p>
Deployable - anyone on the team, at any site, at any time can run the flow
</p>
</li>
</ul></div>
</div>
</div>
<div class="sect1">
<h2 id="_megatest_architecture">Megatest Architecture</h2>
<div class="sectionbody">
<div class="paragraph"><p>All data to specify the tests and configure the system is stored in
plain text files. All system state is stored in an sqlite3
database. Tests are launched using the launching system available for
the distributed compute platform in use. A template script is provided
which can launch jobs on local and remote Linux hosts. Currently
megatest uses the network filesystem to call home to your master
sqlite3 database.</p></div>
</div>
</div>
<div class="sect1">
<h2 id="_road_map">Road Map</h2>
<div class="sectionbody">
<div class="paragraph"><p>Note 1: This road-map is still evolving and subject to change without notice.</p></div>
<div class="sect2">
<h3 id="_architecture_refactor">Architecture Refactor</h3>
<div class="sect3">
<h4 id="_goals">Goals</h4>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Reduce load on the file system. Sqlite3 files on network filesystem can be
  a burden.
</p>
</li>
<li>
<p>
Reduce number of servers and frequency of start/stop. This is mostly an
  issue of clutter but also a reduction in "moving parts".
</p>
</li>
<li>
<p>
Coalesce activities to a single home host where possible. Give the user
  feedback that they have started the dashboard on a host other than the
  home host.
</p>
</li>
<li>
<p>
Reduce number of processes involved in managing running tests.
</p>
</li>
</ol></div>
</div>
<div class="sect3">
<h4 id="_changes_needed">Changes Needed</h4>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
ACID compliant db will be on /tmp and synced to megatest.db with a five
  second max delay.
</p>
</li>
<li>
<p>
Read/writes to db for processes on homehost will go direct to /tmp
  megatest.db file.
</p>
</li>
<li>
<p>
Read/wites fron non-homehost processes will go through one server. Bulk
  reads (e.g. for dashboard or list-runs) will be cached on the current host
  in /tmp and synced from the home megatest.db in the testsuite area.
</p>
</li>
<li>
<p>
Db syncs rely on the target db file timestame minus some margin.
</p>
</li>
<li>
<p>
Since bulk reads do not use the server we can switch to simple RPC for the
  network transport.
</p>
</li>
<li>
<p>
Test running manager process extended to manage multiple running tests.
</p>
</li>
</ol></div>
</div>
</div>
<div class="sect2">
<h3 id="_current_items">Current Items</h3>
<div class="sect3">
<h4 id="_ww05_migrate_to_inmem_db">ww05 - migrate to inmem-db</h4>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Switch to inmem db with fast sync to on disk db&#8217;s [DONE]
</p>
</li>
<li>
889
890
891
892
893
894
895

896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
























914






















































































915
916
917
918
919
920
921
<p>
Task table used for queueing runner actions (remove runs, cleanRunExecute, etc)
</p>
</li>
</ol></div>
</li>
</ol></div>

</div>
</div>
</div>
<h1 id="_getting_started">Getting Started</h1>
<div class="openblock">
<div class="title">Getting started with Megatest</div>
<div class="content">
<div class="paragraph"><p>How to install Megatest and set it up for running your regressions and continuous integration process.</p></div>
</div></div>
<div class="sect1">
<h2 id="_installation">Installation</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_dependencies">Dependencies</h3>
<div class="paragraph"><p>Chicken scheme and a number of "eggs" are required for building
Megatest. See the script installall.sch in the utils directory of the
distribution for a mostly automated way to install everything needed
for building Megatest on Linux.</p></div>
























<div class="paragraph"><p><span class="footnote"><br>[An example footnote.]<br></span></p></div>






















































































</div>
</div>
</div>
<div class="sect1">
<h2 id="_writing_tests">Writing Tests</h2>
<div class="sectionbody">
<div class="sect2">







>



<
<
<
<
<
|






|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







971
972
973
974
975
976
977
978
979
980
981





982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
<p>
Task table used for queueing runner actions (remove runs, cleanRunExecute, etc)
</p>
</li>
</ol></div>
</li>
</ol></div>
<div class="paragraph"><p>shifting, note that the preceding blank line is needed.</p></div>
</div>
</div>
</div>





</div>
<div class="sect1">
<h2 id="_installation">Installation</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_dependencies">Dependencies</h3>
<div class="paragraph"><p>Chicken scheme and a number of "eggs" are required for building
Megatest. See the script installall.sh in the utils directory of the
source distribution for an automated way to install everything
needed for building Megatest on Linux.</p></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_getting_started">Getting Started</h2>
<div class="sectionbody">
<div class="openblock">
<div class="title">Getting started with Megatest</div>
<div class="content">
<div class="paragraph"><p>Creating a testsuite or flow and your first test or task.</p></div>
</div></div>
<div class="paragraph"><p>After installing Megatest you can create a flow or testsuite and add some
tests using the helpers. Here is a quickstart sequence to get you up and
running your first automated testsuite.</p></div>
<div class="sect2">
<h3 id="_creating_a_megatest_area">Creating a Megatest Area</h3>
<div class="sect3">
<h4 id="_choose_target_keys">Choose Target Keys</h4>
<div class="paragraph"><p>First choose your "target" keys. These are used to organise your runs in a
way that is meaningful to your project. If you are unsure about what to use
for keys just use a single generic key such as "RUNTYPE". These keys will be
used to hand values to your tests via environment variables so ensure they
are unique. Prefixing them with something such as PROJKEYS_ is a good
strategy.</p></div>
<div class="paragraph"><p>Examples of keys:</p></div>
<table class="tableblock frame-all grid-all"
style="
width:60%;
">
<caption class="title">Table 1. Example keys</caption>
<col style="width:50%;">
<col style="width:50%;">
<thead>
<tr>
<th class="tableblock halign-left valign-top" > Option            </th>
<th class="tableblock halign-left valign-top" > Description</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top" ><p class="tableblock">RELEASE/ITERATION</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock">This example is used by Megatest for its internal QA.</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top" ><p class="tableblock">ARCH/OS/RELEASE</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock">For a software project targeting multiple platforms</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top" ><p class="tableblock">UCTRLR/NODETYPE</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock">Microcontroller project with different controllers
running same software</p></td>
</tr>
</tbody>
</table>
</div>
<div class="sect3">
<h4 id="_create_area_config_files">Create Area Config Files</h4>
<div class="paragraph"><p>You will need to choose locations for your runs (the data generated every
time you run the testsuite) and link tree. For getting started answer the
prompts with "runs" and "links". We use the Unix editor "vi" in the examples
below but you can use any plain text editor.</p></div>
<div class="listingblock">
<div class="title">Using the helper to create a Megatest area</div>
<div class="content monospaced">
<pre>megatest -create-megatest-area

# optional: verify that the settings are ok
vi megatest.config
vi runconfigs.config</pre>
</div></div>
</div>
</div>
<div class="sect2">
<h3 id="_creating_a_test">Creating a Test</h3>
<div class="paragraph"><p>Choose the test name for your first test and run the helper. You can edit
the files after the initial creation. You will need to enter names and
scripts for the steps to be run and then edit the
tests/&lt;testname&gt;/testconfig file and modify the logpro rules to properly
process the log output from your steps. For your first test just hit enter
for the "waiton", "priority" and iteration variable prompts.</p></div>
<div class="paragraph"><p>Hint: for geting started make your logpro rules very liberal. expect:error
patterns should match nothing and comment out expect:required rules.</p></div>
<div class="listingblock">
<div class="title">Using the helper to create a Megatest test</div>
<div class="content monospaced">
<pre>megatest -create-test myfirsttest

# then edit the generated config
vi tests/myfirsttest/testconfig</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_running_your_test">Running your test</h3>
<div class="paragraph"><p>First choose a target and runname. If you have a two-place target such as
RELEASE/ITERATION a target would look like v1.0/aff3 where v1.0 is the
RELEASE and aff3 is the ITERATION. For a run name just use something like
run1.</p></div>
<div class="listingblock">
<div class="title">Running all tests (testpatt of "%" matches all tests)</div>
<div class="content monospaced">
<pre>megatest -run -target v1.0/aff3 -runname run1 -testpatt % -log run1.log</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_viewing_the_results">Viewing the results</h3>
<div class="paragraph"><p>Start the dashboard and browse your run in the "Runs" tab.</p></div>
<div class="listingblock">
<div class="title">Starting dashboard</div>
<div class="content monospaced">
<pre>dashboard -rows 24</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_writing_tests">Writing Tests</h2>
<div class="sectionbody">
<div class="sect2">
950
951
952
953
954
955
956

957
958
959
960

961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003

1004
1005
1006
1007
1008
1009
1010
</div></div>
<div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script
"stepname.sh". Note that although it is common to put the actions
needed for a test step into a script it is not necessary.</p></div>
</div>
</div>
</div>

<h1 id="_how_to_do_things">How To Do Things</h1>
<div class="sect1">
<h2 id="_process_runs">Process Runs</h2>
<div class="sectionbody">

<div class="sect2">
<h3 id="_remove_runs">Remove Runs</h3>
<div class="paragraph"><p>From the dashboard click on the button (PASS/FAIL&#8230;) for one of the tests. From the test control panel that
comes up push the clean test button. The command field will be prefilled with a template command for removing
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.</p></div>
<div class="listingblock">
<div class="title">Remove the test diskperf and all it&#8217;s items</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v</pre>
</div></div>
<div class="listingblock">
<div class="title">Remove all tests for all runs and all targets</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target %/%/% -runname % -testpatt % -v</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_archive_runs">Archive Runs</h3>
<div class="paragraph"><p>Megatest supports using the bup backup tool (<a href="https://bup.github.io/">https://bup.github.io/</a>) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run
durations, time stamps etc.) are all preserved in the megatest database.</p></div>
<div class="paragraph"><p>For setup information see the Archiving topic in the reference section of this manual.</p></div>
<div class="sect3">
<h4 id="_to_archive">To Archive</h4>
<div class="paragraph"><p>Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.</p></div>
<div class="listingblock">
<div class="title">Archive a full run</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_to_restore">To Restore</h4>
<div class="listingblock">
<div class="title">Retrieve a single test</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%</pre>
</div></div>
<div class="paragraph"><p>Hint: You can browse the archive using bup commands directly.</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>bup -d /path/to/bup/archive ftp</pre>
</div></div>

</div>
</div>
<div class="sect2">
<h3 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h3>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">







>
|
|
<
|
>
|
|














|
|




|
|







|
|










>







1138
1139
1140
1141
1142
1143
1144
1145
1146
1147

1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
</div></div>
<div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script
"stepname.sh". Note that although it is common to put the actions
needed for a test step into a script it is not necessary.</p></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_how_to_do_things">How To Do Things</h2>
<div class="sectionbody">

<div class="sect2">
<h3 id="_process_runs">Process Runs</h3>
<div class="sect3">
<h4 id="_remove_runs">Remove Runs</h4>
<div class="paragraph"><p>From the dashboard click on the button (PASS/FAIL&#8230;) for one of the tests. From the test control panel that
comes up push the clean test button. The command field will be prefilled with a template command for removing
that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.</p></div>
<div class="listingblock">
<div class="title">Remove the test diskperf and all it&#8217;s items</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v</pre>
</div></div>
<div class="listingblock">
<div class="title">Remove all tests for all runs and all targets</div>
<div class="content monospaced">
<pre>megatest -remove-runs -target %/%/% -runname % -testpatt % -v</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_archive_runs">Archive Runs</h4>
<div class="paragraph"><p>Megatest supports using the bup backup tool (<a href="https://bup.github.io/">https://bup.github.io/</a>) to archive your tests for efficient storage
and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run
durations, time stamps etc.) are all preserved in the megatest database.</p></div>
<div class="paragraph"><p>For setup information see the Archiving topic in the reference section of this manual.</p></div>
<div class="sect4">
<h5 id="_to_archive">To Archive</h5>
<div class="paragraph"><p>Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.</p></div>
<div class="listingblock">
<div class="title">Archive a full run</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %</pre>
</div></div>
</div>
<div class="sect4">
<h5 id="_to_restore">To Restore</h5>
<div class="listingblock">
<div class="title">Retrieve a single test</div>
<div class="content monospaced">
<pre>megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%</pre>
</div></div>
<div class="paragraph"><p>Hint: You can browse the archive using bup commands directly.</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>bup -d /path/to/bup/archive ftp</pre>
</div></div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h3>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
1025
1026
1027
1028
1029
1030
1031

1032

1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051

1052
1053
1054
1055
1056
1057
1058
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes</pre>
</div></div>
</div>
</div>
</div>

<h1 id="_tricks">Tricks</h1>

<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.</p></div>
<div class="sect1">
<h2 id="_limiting_your_running_jobs">Limiting your running jobs</h2>
<div class="sectionbody">
<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
<div class="paragraph"><p>In your testconfig:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[test_meta]
jobgroup group1</pre>
</div></div>
<div class="paragraph"><p>In your megatest.config:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[jobgroups]
group1 10
custdes 4</pre>
</div></div>

</div>
</div>
<div class="sect1">
<h2 id="_debugging_tricks">Debugging Tricks</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_examining_the_environment">Examining The Environment</h3>







>
|
>


|
|
<














>







1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228

1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
# if defined and not "no" flexi-launcher will bypass launcher unless there is no
# match.
flexi-launcher yes</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_tricks">Tricks</h2>
<div class="sectionbody">
<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
configuring and generally getting the most out of Megatest.</p></div>
<div class="sect2">
<h3 id="_limiting_your_running_jobs">Limiting your running jobs</h3>

<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
<div class="paragraph"><p>In your testconfig:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[test_meta]
jobgroup group1</pre>
</div></div>
<div class="paragraph"><p>In your megatest.config:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[jobgroups]
group1 10
custdes 4</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_debugging_tricks">Debugging Tricks</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_examining_the_environment">Examining The Environment</h3>
1127
1128
1129
1130
1131
1132
1133

1134
1135

1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208

1209
1210
1211
1212
1213
1214
1215
1216
1217


1218
1219
1220
1221
1222
1223
1224
1225
1226
<pre>sudo lsof -i
sudo netstat -lptu
sudo netstat -tulpn</pre>
</div></div>
</div>
</div>
</div>

<h1 id="_reference">Reference</h1>
<div class="sect1">

<h2 id="_megatest_config_file_settings">Megatest Config File Settings</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_disk_space_checks">Disk Space Checks</h3>
<div class="paragraph"><p>Some parameters you can put in the [setup] section of megatest.config:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre># minimum space required in a run disk
minspace 10000000

# minimum space required in dbdir:
dbdir-space-required 100000

# script that takes path as parameter and returns number of bytes available:
free-space-script check-space.sh</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_trim_trailing_spaces">Trim trailing spaces</h3>
<div class="listingblock">
<div class="content monospaced">
<pre>[configf:settings trim-trailing-spaces yes]</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_job_submission_control">Job Submission Control</h3>
<div class="sect3">
<h4 id="_submit_jobs_to_host_types_based_on_test_name_2">Submit jobs to Host Types based on Test Name</h4>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless
# there is no host-type match.
flexi-launcher yes</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_host_types">host-types</h4>
<div class="paragraph"><p>List of host types and the commandline to run a job on that host type.</p></div>
<div class="listingblock">
<div class="title">host-type &#8658; launch command</div>
<div class="content monospaced">
<pre>general nbfake</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_launchers">launchers</h4>
<div class="listingblock">
<div class="title">test/itempath &#8658; host-type</div>
<div class="content monospaced">
<pre>runfirst/sum% remote</pre>
</div></div>
</div>
</div>
<div class="sect2">
<h3 id="_miscellaneous_setup_items">Miscellaneous Setup Items</h3>
<div class="paragraph"><p>Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.</p></div>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[setup]
reruns 5</pre>
</div></div>

<div class="sect3">
<h4 id="_run_time_limit">Run time limit</h4>
<div class="listingblock">
<div class="content monospaced">
<pre>[setup]
# this will automatically kill the test if it runs for more than 1h 2m and 3s
runtimelim 1h 2m 3s</pre>
</div></div>
</div>


<div class="sect3">
<h4 id="_tests_browser_view">Tests browser view</h4>
<div class="paragraph"><p>The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests.</p></div>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Dot (graphviz) based tree
</p>
</li>







>
|
|
>
|
|
<
|













|
|





|
|
|
|


















|
|







|
|






<
|
|







>
|
|







>
>
|
|







1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331

1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391

1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
<pre>sudo lsof -i
sudo netstat -lptu
sudo netstat -tulpn</pre>
</div></div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_reference">Reference</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_megatest_config_file_settings">Megatest Config File Settings</h3>
<div class="sect3">

<h4 id="_disk_space_checks">Disk Space Checks</h4>
<div class="paragraph"><p>Some parameters you can put in the [setup] section of megatest.config:</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre># minimum space required in a run disk
minspace 10000000

# minimum space required in dbdir:
dbdir-space-required 100000

# script that takes path as parameter and returns number of bytes available:
free-space-script check-space.sh</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_trim_trailing_spaces">Trim trailing spaces</h4>
<div class="listingblock">
<div class="content monospaced">
<pre>[configf:settings trim-trailing-spaces yes]</pre>
</div></div>
</div>
<div class="sect3">
<h4 id="_job_submission_control">Job Submission Control</h4>
<div class="sect4">
<h5 id="_submit_jobs_to_host_types_based_on_test_name_2">Submit jobs to Host Types based on Test Name</h5>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless
# there is no host-type match.
flexi-launcher yes</pre>
</div></div>
</div>
<div class="sect4">
<h5 id="_host_types">host-types</h5>
<div class="paragraph"><p>List of host types and the commandline to run a job on that host type.</p></div>
<div class="listingblock">
<div class="title">host-type &#8658; launch command</div>
<div class="content monospaced">
<pre>general nbfake</pre>
</div></div>
</div>
<div class="sect4">
<h5 id="_launchers">launchers</h5>
<div class="listingblock">
<div class="title">test/itempath &#8658; host-type</div>
<div class="content monospaced">
<pre>runfirst/sum% remote</pre>
</div></div>
</div>

<div class="sect4">
<h5 id="_miscellaneous_setup_items">Miscellaneous Setup Items</h5>
<div class="paragraph"><p>Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.</p></div>
<div class="listingblock">
<div class="title">In megatest.config</div>
<div class="content monospaced">
<pre>[setup]
reruns 5</pre>
</div></div>
</div>
<div class="sect4">
<h5 id="_run_time_limit">Run time limit</h5>
<div class="listingblock">
<div class="content monospaced">
<pre>[setup]
# this will automatically kill the test if it runs for more than 1h 2m and 3s
runtimelim 1h 2m 3s</pre>
</div></div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_tests_browser_view">Tests browser view</h3>
<div class="paragraph"><p>The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests.</p></div>
<div class="olist arabic"><ol class="arabic">
<li>
<p>
Dot (graphviz) based tree
</p>
</li>
1234
1235
1236
1237
1238
1239
1240










1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
well in that mode then use "nodot" to turn it off.</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[setup]
nodot</pre>
</div></div>
</div>










</div>
<div class="sect2">
<h3 id="_database_settings">Database settings</h3>
<table class="tableblock frame-topbot grid-all"
style="
width:70%;
">
<caption class="title">Table 1. Database config settings in [setup] section of megatest.config</caption>
<col style="width:14%;">
<col style="width:28%;">
<col style="width:28%;">
<col style="width:28%;">
<thead>
<tr>
<th class="tableblock halign-center valign-top" >Var                       </th>







>
>
>
>
>
>
>
>
>
>







|







1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
well in that mode then use "nodot" to turn it off.</p></div>
<div class="listingblock">
<div class="content monospaced">
<pre>[setup]
nodot</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_dashboard_settings">Dashboard settings</h3>
<div class="listingblock">
<div class="title">Runs tab buttons, font and size</div>
<div class="content monospaced">
<pre>[dashboard]
btn-height x14
btn-fontsz 10
cell-width 60</pre>
</div></div>
</div>
<div class="sect2">
<h3 id="_database_settings">Database settings</h3>
<table class="tableblock frame-topbot grid-all"
style="
width:70%;
">
<caption class="title">Table 2. Database config settings in [setup] section of megatest.config</caption>
<col style="width:14%;">
<col style="width:28%;">
<col style="width:28%;">
<col style="width:28%;">
<thead>
<tr>
<th class="tableblock halign-center valign-top" >Var                       </th>
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
<div class="content monospaced">
<pre>[triggers]
COMPLETED/ xterm -e bash -s --</pre>
</div></div>
<div class="admonitionblock">
<table><tr>
<td class="icon">
<img src="/usr/images/icons/note.png" alt="Note">
</td>
<td class="content">There is a trailing space after the --</td>
</tr></table>
</div>
</div>
<div class="sect2">
<h3 id="_override_the_toplevel_html_file">Override the Toplevel HTML File</h3>







|







1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
<div class="content monospaced">
<pre>[triggers]
COMPLETED/ xterm -e bash -s --</pre>
</div></div>
<div class="admonitionblock">
<table><tr>
<td class="icon">
<img src="/nfs/pdx/disks/ice.disk.002/icfadm/pkgs/asciidoc/8.6.7/images/icons/note.png" alt="Note">
</td>
<td class="content">There is a trailing space after the --</td>
</tr></table>
</div>
</div>
<div class="sect2">
<h3 id="_override_the_toplevel_html_file">Override the Toplevel HTML File</h3>
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
<h2 id="_programming_api">Programming API</h2>
<div class="sectionbody">
<div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
<table class="tableblock frame-topbot grid-all"
style="
width:70%;
">
<caption class="title">Table 2. API Keys Related Calls</caption>
<col style="width:14%;">
<col style="width:28%;">
<col style="width:28%;">
<col style="width:28%;">
<thead>
<tr>
<th class="tableblock halign-center valign-top" >API Call                        </th>







|







1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
<h2 id="_programming_api">Programming API</h2>
<div class="sectionbody">
<div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
<table class="tableblock frame-topbot grid-all"
style="
width:70%;
">
<caption class="title">Table 3. API Keys Related Calls</caption>
<col style="width:14%;">
<col style="width:28%;">
<col style="width:28%;">
<col style="width:28%;">
<thead>
<tr>
<th class="tableblock halign-center valign-top" >API Call                        </th>
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
<td class="tableblock halign-center valign-top" ><p class="tableblock">(rmt:get-keys run-id)</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">( key1 key2 &#8230; )</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
</tr>
</tbody>
</table>
<div class="sect2">
<h3 id="_megatest_internals">Megatest Internals</h3>
<div class="imageblock graphviz">
<div class="content">
<img src="server.png" alt="server.png">
</div>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_example_appendix">Appendix A: Example Appendix</h2>
<div class="sectionbody">
<div class="paragraph"><p>One or more optional appendixes go here at section level zero.</p></div>
<div class="sect2">
<h3 id="_appendix_sub_section">Appendix Sub-section</h3>
<div class="admonitionblock">
<table><tr>
<td class="icon">
<img src="/usr/images/icons/note.png" alt="Note">
</td>
<td class="content">Preface and appendix subsections start out of sequence at level
2 (level 1 is skipped).  This only applies to multi-part book
documents.</td>
</tr></table>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_example_bibliography">Example Bibliography</h2>
<div class="sectionbody">
<div class="paragraph"><p>The bibliography list is a style of AsciiDoc bulleted list.</p></div>
<div class="ulist bibliography"><ul>
<li>
<p>
<a id="taoup"></a>[taoup] Eric Steven Raymond. <em>The Art of Unix
  Programming</em>. Addison-Wesley. ISBN 0-13-142901-9.
</p>
</li>
<li>
<p>
<a id="walsh-muellner"></a>[walsh-muellner] Norman Walsh &amp; Leonard Muellner.
  <em>DocBook - The Definitive Guide</em>. O&#8217;Reilly &amp; Associates. 1999.
  ISBN 1-56592-580-7.
</p>
</li>
</ul></div>
</div>
</div>
<div class="sect1">
<h2 id="_example_glossary">Example Glossary</h2>
<div class="sectionbody">
<div class="paragraph"><p>Glossaries are optional. Glossaries entries are an example of a style
of AsciiDoc labeled lists.</p></div>
<div class="dlist glossary"><dl>
<dt>
A glossary term
</dt>
<dd>
<p>
  The corresponding (indented) definition.
</p>
</dd>
<dt>
A second glossary term
</dt>
<dd>
<p>
  The corresponding (indented) definition.
</p>
</dd>
</dl></div>
</div>
</div>
<div class="sect1">
<h2 id="_example_colophon">Example Colophon</h2>
<div class="sectionbody">
<div class="paragraph"><p>Text at the end of a book describing facts about its production.</p></div>
</div>
</div>
<div class="sect1">
<h2 id="_example_index">Example Index</h2>
<div class="sectionbody">
</div>
</div>
</div>
<div id="footnotes"><hr></div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated 2016-08-20 20:32:22 MST
</div>
</div>
</body>
</html>







<
<
<
<
<


<
<
<

|

<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


<
<
<
<












|




1945
1946
1947
1948
1949
1950
1951





1952
1953



1954
1955
1956

1957














1958




















1959






















1960
1961




1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
<td class="tableblock halign-center valign-top" ><p class="tableblock">(rmt:get-keys run-id)</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">( key1 key2 &#8230; )</p></td>
<td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td>
</tr>
</tbody>
</table>





</div>
</div>



<div class="sect1">
<h2 id="_megatest_internals">Megatest Internals</h2>
<div class="sectionbody">

<div class="imageblock graphviz">














<div class="content">




















<img src="server.png" alt="server.png">






















</div>
</div>




</div>
</div>
<div class="sect1">
<h2 id="_example_index">Example Index</h2>
<div class="sectionbody">
</div>
</div>
</div>
<div id="footnotes"><hr></div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated 2016-10-19 10:23:07 PDT
</div>
</div>
</body>
</html>

Modified docs/manual/megatest_manual.txt from [49b99544b8] to [c82fa9e963].

1
2
3
4
5
6
7
8
9

10
11
12
13
14

15
16
17
18
19
20
21
22
23
24
25
26

27
28
29
30
31
32
33
The Megatest Users Manual
=========================
Matt Welland <matt@kiatoa.com>
v1.0, April 2012
:doctype: book


[preface]
Preface

=======
This book is organised as three sub-books; getting started, writing tests and reference.

Why Megatest?
~~~~~~~~~~~~~


The Megatest project was started for two reasons, the first was an
immediate and pressing need for a generalized tool to manage a suite
of regression tests and the second was the fact that the author had
written or maintained several such tools at different companies over
the years and it seemed a good thing to have a single open source
tool, flexible enough to meet the needs of any team doing continuous
integrating and or running a complex suite of tests for release
qualification.

Megatest Design Philosophy
~~~~~~~~~~~~~~~~~~~~~~~~~~


Megatest is intended to provide the minimum needed resources to make
writing a suite of tests and tasks for implementing continuous build
for software, design engineering or process control (via owlfs for
example) without being specialized for any specific problem
space. Megatest in of itself does not know what constitutes a PASS or
FAIL of a test or task. In most cases megatest is best used in









>
|



<
>











<
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14

15
16
17
18
19
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34
The Megatest Users Manual
=========================
Matt Welland <matt@kiatoa.com>
v1.0, April 2012
:doctype: book


[preface]
Preface
-------

This book is organised as three sub-books; getting started, writing tests and reference.

Why Megatest?

-------------

The Megatest project was started for two reasons, the first was an
immediate and pressing need for a generalized tool to manage a suite
of regression tests and the second was the fact that the author had
written or maintained several such tools at different companies over
the years and it seemed a good thing to have a single open source
tool, flexible enough to meet the needs of any team doing continuous
integrating and or running a complex suite of tests for release
qualification.

Megatest Design Philosophy

--------------------------

Megatest is intended to provide the minimum needed resources to make
writing a suite of tests and tasks for implementing continuous build
for software, design engineering or process control (via owlfs for
example) without being specialized for any specific problem
space. Megatest in of itself does not know what constitutes a PASS or
FAIL of a test or task. In most cases megatest is best used in
49
50
51
52
53
54
55
56

57
58
59
60
61
62
63
64
65
66






67



68
69
70
71
72
73

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

130
131
132
133

 * Encapsulated - the tests run in self-contained directories and all inputs
   and outputs to the process can be found in the run areas.

 * Deployable - anyone on the team, at any site, at any time can run the flow

Megatest Architecture
~~~~~~~~~~~~~~~~~~~~~


All data to specify the tests and configure the system is stored in
plain text files. All system state is stored in an sqlite3
database. Tests are launched using the launching system available for
the distributed compute platform in use. A template script is provided
which can launch jobs on local and remote Linux hosts. Currently
megatest uses the network filesystem to call home to your master
sqlite3 database.

include::../plan.txt[]






include::getting_started.txt[]



include::writing_tests.txt[]
include::howto.txt[]
include::reference.txt[]

Megatest Internals
~~~~~~~~~~~~~~~~~~


["graphviz", "server.png"]
----------------------------------------------------------------------
include::server.dot[]
----------------------------------------------------------------------


[appendix]
Example Appendix
================
One or more optional appendixes go here at section level zero.

Appendix Sub-section
~~~~~~~~~~~~~~~~~~~
NOTE: Preface and appendix subsections start out of sequence at level
2 (level 1 is skipped).  This only applies to multi-part book
documents.



[bibliography]
Example Bibliography
====================
The bibliography list is a style of AsciiDoc bulleted list.

[bibliography]
- [[[taoup]]] Eric Steven Raymond. 'The Art of Unix
  Programming'. Addison-Wesley. ISBN 0-13-142901-9.
- [[[walsh-muellner]]] Norman Walsh & Leonard Muellner.
  'DocBook - The Definitive Guide'. O'Reilly & Associates. 1999.
  ISBN 1-56592-580-7.


[glossary]
Example Glossary
================
Glossaries are optional. Glossaries entries are an example of a style
of AsciiDoc labeled lists.

[glossary]
A glossary term::
  The corresponding (indented) definition.

A second glossary term::
  The corresponding (indented) definition.


[colophon]
Example Colophon
================
Text at the end of a book describing facts about its production.


[index]
Example Index
=============

////////////////////////////////////////////////////////////////
The index is normally left completely empty, it's contents are
generated automatically by the DocBook toolchain.
////////////////////////////////////////////////////////////////







<
>










>
>
>
>
>
>

>
>
>





<
>







|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<



<
>




50
51
52
53
54
55
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

135
136
137

138
139
140
141
142

 * Encapsulated - the tests run in self-contained directories and all inputs
   and outputs to the process can be found in the run areas.

 * Deployable - anyone on the team, at any site, at any time can run the flow

Megatest Architecture

---------------------

All data to specify the tests and configure the system is stored in
plain text files. All system state is stored in an sqlite3
database. Tests are launched using the launching system available for
the distributed compute platform in use. A template script is provided
which can launch jobs on local and remote Linux hosts. Currently
megatest uses the network filesystem to call home to your master
sqlite3 database.

include::../plan.txt[]
// to allow the getting_started.txt to be a stand-alone document use level
shifting, note that the preceding blank line is needed.
// :leveloffset: 2

include::installation.txt[]

include::getting_started.txt[]

:leveloffset: 0

include::writing_tests.txt[]
include::howto.txt[]
include::reference.txt[]

Megatest Internals

------------------

["graphviz", "server.png"]
----------------------------------------------------------------------
include::server.dot[]
----------------------------------------------------------------------


// [appendix]
// Example Appendix
// ================
// One or more optional appendixes go here at section level zero.
// 
// Appendix Sub-section
// ~~~~~~~~~~~~~~~~~~~
// NOTE: Preface and appendix subsections start out of sequence at level
// 2 (level 1 is skipped).  This only applies to multi-part book
// documents.
// 
// 
// 
// [bibliography]
// Example Bibliography
// ====================
// The bibliography list is a style of AsciiDoc bulleted list.
// 
// [bibliography]
// - [[[taoup]]] Eric Steven Raymond. 'The Art of Unix
//   Programming'. Addison-Wesley. ISBN 0-13-142901-9.
// - [[[walsh-muellner]]] Norman Walsh & Leonard Muellner.
//   'DocBook - The Definitive Guide'. O'Reilly & Associates. 1999.
//   ISBN 1-56592-580-7.
// 
// 
// [glossary]
// Example Glossary
// ================
// Glossaries are optional. Glossaries entries are an example of a style
// of AsciiDoc labeled lists.
// 
// [glossary]
// A glossary term::
//   The corresponding (indented) definition.
// 
// A second glossary term::
//   The corresponding (indented) definition.
// 
// 
// [colophon]
// Example Colophon
// ================
// Text at the end of a book describing facts about its production.


[index]
Example Index

-------------
////////////////////////////////////////////////////////////////
The index is normally left completely empty, it's contents are
generated automatically by the DocBook toolchain.
////////////////////////////////////////////////////////////////

Modified docs/manual/reference.txt from [262af5e4f5] to [206fb51b8f].

1
2
3

4
5
6

7
8
9

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

26
27
28
29
30
31
32

33
34
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63
64

65
66
67
68
69
70
71
72

73
74
75
76
77
78
79
80
81
82
83

84
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99
100
101
102
103
104
105











106
107
108
109
110
111
112

Reference
=========


Megatest Config File Settings
-----------------------------


Disk Space Checks
~~~~~~~~~~~~~~~~~


Some parameters you can put in the [setup] section of megatest.config:

-------------------
# minimum space required in a run disk 
minspace 10000000

# minimum space required in dbdir:
dbdir-space-required 100000

# script that takes path as parameter and returns number of bytes available:
free-space-script check-space.sh
-------------------

Trim trailing spaces
~~~~~~~~~~~~~~~~~~~~


------------------
[configf:settings trim-trailing-spaces yes]
------------------

Job Submission Control
~~~~~~~~~~~~~~~~~~~~~~


Submit jobs to Host Types based on Test Name
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


.In megatest.config
------------------------
[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless 
# there is no host-type match.
flexi-launcher yes
------------------------

host-types
^^^^^^^^^^


List of host types and the commandline to run a job on that host type.

.host-type => launch command
------------
general nbfake
------------

launchers

^^^^^^^^^
.test/itempath => host-type
------------
runfirst/sum% remote
------------

Miscellaneous Setup Items
~~~~~~~~~~~~~~~~~~~~~~~~~


Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.

.In megatest.config
------------------
[setup]
reruns 5
------------------

Run time limit
^^^^^^^^^^^^^^


-----------------
[setup]
# this will automatically kill the test if it runs for more than 1h 2m and 3s
runtimelim 1h 2m 3s
-----------------

Tests browser view
^^^^^^^^^^^^^^^^^^


The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests. 

. Dot (graphviz) based tree
. No dot, plain listing

The default is the graphviz based tree but if your tests don't view
well in that mode then use "nodot" to turn it off.

-----------------
[setup]
nodot
-----------------












Database settings
~~~~~~~~~~~~~~~~~

.Database config settings in [setup] section of megatest.config
[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header"]
|======================


<
>


<
>


<
>















<
>






<
>


<
>



















<
>









>
|






<
>










<
>








<
>













>
>
>
>
>
>
>
>
>
>
>







1
2

3
4
5

6
7
8

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

25
26
27
28
29
30
31

32
33
34

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

73
74
75
76
77
78
79
80
81
82
83

84
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

Reference

---------

Megatest Config File Settings

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Disk Space Checks

^^^^^^^^^^^^^^^^^

Some parameters you can put in the [setup] section of megatest.config:

-------------------
# minimum space required in a run disk 
minspace 10000000

# minimum space required in dbdir:
dbdir-space-required 100000

# script that takes path as parameter and returns number of bytes available:
free-space-script check-space.sh
-------------------

Trim trailing spaces

^^^^^^^^^^^^^^^^^^^^

------------------
[configf:settings trim-trailing-spaces yes]
------------------

Job Submission Control

^^^^^^^^^^^^^^^^^^^^^^

Submit jobs to Host Types based on Test Name

++++++++++++++++++++++++++++++++++++++++++++

.In megatest.config
------------------------
[host-types]
general   nbfake
remote    bsub

[launchers]
runfirst/sum% remote
% general

[jobtools]
launcher bsub
# if defined and not "no" flexi-launcher will bypass launcher unless 
# there is no host-type match.
flexi-launcher yes
------------------------

host-types

++++++++++

List of host types and the commandline to run a job on that host type.

.host-type => launch command
------------
general nbfake
------------

launchers
+++++++++

.test/itempath => host-type
------------
runfirst/sum% remote
------------

Miscellaneous Setup Items

+++++++++++++++++++++++++

Attempt to rerun tests in "STUCK/DEAD", "n/a", "ZERO_ITEMS" states.

.In megatest.config
------------------
[setup]
reruns 5
------------------

Run time limit

++++++++++++++

-----------------
[setup]
# this will automatically kill the test if it runs for more than 1h 2m and 3s
runtimelim 1h 2m 3s
-----------------

Tests browser view

~~~~~~~~~~~~~~~~~~

The tests browser (see the Run Control tab on the dashboard) has two views for displaying the tests. 

. Dot (graphviz) based tree
. No dot, plain listing

The default is the graphviz based tree but if your tests don't view
well in that mode then use "nodot" to turn it off.

-----------------
[setup]
nodot
-----------------

Dashboard settings
~~~~~~~~~~~~~~~~~~

.Runs tab buttons, font and size
------------------
[dashboard]
btn-height x14
btn-fontsz 10
cell-width 60
------------------

Database settings
~~~~~~~~~~~~~~~~~

.Database config settings in [setup] section of megatest.config
[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header"]
|======================

Modified docs/manual/server.png from [a508d3edd1] to [ae7d7ee58e].

cannot compute difference between binary files

Modified docs/manual/writing_tests.txt from [3f710469e5] to [c1e61ad2b4].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

Writing Tests
-------------
// =============

Creating a new Test
~~~~~~~~~~~~~~~~~~~

//-------------------

The following steps will add a test "yourtestname" to your testsuite. This assumes
starting from a directory where you already have a megatest.config and
runconfigs.config.

. Create a directory tests/yourtestname
. Create a file tests/yourtestname/testconfig




<




<
<







1
2
3

4
5
6
7


8
9
10
11
12
13
14

Writing Tests
-------------


Creating a new Test
~~~~~~~~~~~~~~~~~~~



The following steps will add a test "yourtestname" to your testsuite. This assumes
starting from a directory where you already have a megatest.config and
runconfigs.config.

. Create a directory tests/yourtestname
. Create a file tests/yourtestname/testconfig

Modified docs/megatest-training.odp from [8b1c9793d2] to [ba7ab2ab9e].

cannot compute difference between binary files

Modified docs/megatest-training.pdf from [84211704af] to [efa6551238].

cannot compute difference between binary files

Modified docs/plan.txt from [37ebd56f39] to [92bba79ce7].

1

2

3


4


5








6


7












8
9

10
11
12

13
14
15
16
17
18
19
Road Map

========




Note 1: This road-map is tentative and subject to change without notice.











Note 2: Starting over. Old plan is commented out.















Current Items
-------------


ww05 - migrate to inmem-db
~~~~~~~~~~~~~~~~~~~~~~~~~~


. Switch to inmem db with fast sync to on disk db's [DONE]
. Server polls tasks table for next action
.. Task table used for tracking runner process [DONE]
.. Task table used for jobs to run
.. Task table used for queueing runner actions (remove runs, cleanRunExecute, etc)


>
|
>

>
>
|
>
>

>
>
>
>
>
>
>
>
|
>
>

>
>
>
>
>
>
>
>
>
>
>
>

<
>


<
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

37
38
39

40
41
42
43
44
45
46
47
Road Map
--------

Note 1: This road-map is still evolving and subject to change without notice.

Architecture Refactor
~~~~~~~~~~~~~~~~~~~~~

Goals
^^^^^

. Reduce load on the file system. Sqlite3 files on network filesystem can be
  a burden.
. Reduce number of servers and frequency of start/stop. This is mostly an
  issue of clutter but also a reduction in "moving parts".
. Coalesce activities to a single home host where possible. Give the user
  feedback that they have started the dashboard on a host other than the
  home host.
. Reduce number of processes involved in managing running tests.

Changes Needed
^^^^^^^^^^^^^^

. ACID compliant db will be on /tmp and synced to megatest.db with a five
  second max delay.
. Read/writes to db for processes on homehost will go direct to /tmp
  megatest.db file.
. Read/wites fron non-homehost processes will go through one server. Bulk
  reads (e.g. for dashboard or list-runs) will be cached on the current host
  in /tmp and synced from the home megatest.db in the testsuite area.
. Db syncs rely on the target db file timestame minus some margin.
. Since bulk reads do not use the server we can switch to simple RPC for the
  network transport.
. Test running manager process extended to manage multiple running tests.

Current Items

~~~~~~~~~~~~~

ww05 - migrate to inmem-db

^^^^^^^^^^^^^^^^^^^^^^^^^^

. Switch to inmem db with fast sync to on disk db's [DONE]
. Server polls tasks table for next action
.. Task table used for tracking runner process [DONE]
.. Task table used for jobs to run
.. Task table used for queueing runner actions (remove runs, cleanRunExecute, etc)

Modified env.scm from [88e7c2b715] to [d8ef48f13e].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(declare (unit env))

(use sql-de-lite) ;; srfi-1 posix regex regex-case srfi-69 nanomsg srfi-18 call-with-environment-variables)

(define (env:open-db fname)
  (let* ((db-exists (file-exists? fname))
	 (db        (open-database fname)))
    (if (not db-exists)
	(begin
	  (exec (sql db "CREATE TABLE envvars (













|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(declare (unit env))

(use sql-de-lite) ;; srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables)

(define (env:open-db fname)
  (let* ((db-exists (file-exists? fname))
	 (db        (open-database fname)))
    (if (not db-exists)
	(begin
	  (exec (sql db "CREATE TABLE envvars (

Modified example/megatest.config from [1023424f91] to [ce9f9de360].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[fields]
CFG_TYPE This is the refdb to use.
RUN_TYPE Can be: full or quick

[setup]
# Adjust max_concurrent_jobs to limit parallel jobs
max_concurrent_jobs 50

# This is your link path, best to set it and then not change it
linktree #{getenv PWD}/linktree

# Job tools control how your jobs are launched
[jobtools]
launcher nbfake

# As you run more tests you may need to add additional disks 
# the names are arbitrary but must be unique
[disks]
disk0 #{getenv PWD}/runs

[include local.megatest.config]










|








|



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[fields]
CFG_TYPE This is the refdb to use.
RUN_TYPE Can be: full or quick

[setup]
# Adjust max_concurrent_jobs to limit parallel jobs
max_concurrent_jobs 50

# This is your link path, best to set it and then not change it
linktree #{getenv MT_RUN_AREA_HOME}/linktree

# Job tools control how your jobs are launched
[jobtools]
launcher nbfake

# As you run more tests you may need to add additional disks 
# the names are arbitrary but must be unique
[disks]
disk0 #{getenv MT_RUN_AREA_HOME}/runs

[include local.megatest.config]

Modified genexample.scm from [2688ae0606] to [fa6512266d].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17



18
19
20
21
22
23
24
25
26
27
28
29
30
31
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(declare (unit genexample))
(use posix)

(define genexample:example-logpro
#<<EOF
;; You should have at least one expect:required. This ensures that your process ran



(expect:required in "LogFileBody" > 0 "Put description here" #/put pattern here/)

;; You may need ignores to suppress false error or warning hits from the later expects
;; NOTE: Order is important here!
(expect:ignore   in "LogFileBody"  < 99 "Ignore the word error in comments" #/^\/\/.*error/)
(expect:warning  in "LogFileBody"  = 0 "Any warning" #/warn/)
(expect:error    in "LogFileBody"  = 0 "Any error"  (list #/ERROR/ #/error/)) ;; but disallow any other errors
EOF
)

(define genexample:example-script
#<<EOF
#!/usr/bin/env bash













|



|
>
>
>
|
|
|
|
|
|
|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(declare (unit genexample))
(use posix regex)

(define genexample:example-logpro
#<<EOF
  ;; You should have at least one expect:required. This ensures that your process ran
  ;; comment out the line below and replace "put pattern here" with a pattern that will
  ;; always be seen in your log file if the step runs successfully.
  ;;
  ;; (expect:required in "LogFileBody" > 0 "Put description here" #/put pattern here/)
  ;;
  ;; You may need ignores to suppress false error or warning hits from the later expects
  ;; NOTE: Order is important here!
  (expect:ignore   in "LogFileBody"  < 99 "Ignore the word error in comments" #/^\/\/.*error/)
  (expect:warning  in "LogFileBody"  = 0 "Any warning" #/warn/)
  (expect:error    in "LogFileBody"  = 0 "Any error"  (list #/ERROR/ #/error/)) ;; but disallow any other errors
EOF
)

(define genexample:example-script
#<<EOF
#!/usr/bin/env bash

40
41
42
43
44
45
46
47

48
49
50
51
52
53
54
55
56
57

58
59
60
61

62
63
64
65
66
67



68
69
70

71
72
73

74


75
76

77
78
79
80











81
82
83
84

85
86



87
88


89
90
91
92
93
94
95



96
97


98
99
100
101
102
103
104
105
106


107
108
109
110
111
112
113
114
115
116
117
118
119
120
121



122
123
124


125
126
127
128
129
130
131
132




133





134



135




136
137

138
139
140
141
142
143
144
	(path   #f)
	(firstd #f))
    (print "Note: don't worry too much about typos in this process, you will be able to edit
the generated files before starting your first runs")

    ;; create the needed area
    (print "==================\nWhere can I create your Megatest regresssion/continuous build area? Note, your \n"
	   "tests will not necessarily be run in this area, disk space needs are modest.")

    (display "Enter your megatest directory: ")
    (set! path (read-line))

    (if (not (directory? path))
	(begin
	  (print "The path " path " does not exist or is not a directory. Attempting to create it now")
	  (create-directory path #t)))

    ;; First check that megatest.config does not exist!
    (if (file-exists? (conc path "/megatest.config"))

	(begin
	  (print "WARNING: megatest.config already exists! Do you wish to clobber files in " path "?")
	  (display "Enter y/n: ")
	  (if (not (equal? "y" (read-line)))

	      (begin
		(print "INFO: Creation of megatest files in " path " aborted")
		(exit 1)))))

    ;; first prompt user for fields
    ;;



    (print "==================\nNext you must specify fields or keys for your megatest area. These will 
be used to organise your runs. One field should probably be \"RELEASE\". 
Other examples of useful fields might be \"PLATFORM\", \"TARGET_OS\" or if you are in the

semiconductor business perhaps things like \"TECHNOLOGY_NODE\", \"DESIGN_KIT\" or \"METAL_STACK\".

The all caps is a convention because the variables you choose will be available to your tests

as environment variables. You can edit these values later but it is generally a good idea to


settle on them and get them right early on. Your runs will be stored in directories specified by
your keys. Example, if you have keys OSFAMILY/VARIANT/OSVER/RELEASE you may get a test \"build\"

in a directory like this: linux/ubuntu/11.04/rev_1.2/build

Please enter your keys now, separated by spaces or slashes. Only alpha-numeric characters. 
Upper case recommended.")











    (display "Enter keys separated by spaces or slashes: ")
    (set! keystr (read-line))
    (set! keys   (apply append
			(map string-split (string-split keystr "/"))))

    
    ;; Now get the link tree location and a first disk



    (print "==================\nNow you need an initial place to store your runs. These are called \"disks\" and you
can add more at any time. To get going provide a writeable directory name. ")


    (display "Enter your test runs directory: ")
    (set! firstd (read-line))
    (if (not (directory? firstd))
	(begin
	  (print "WARNING: you have specified a path " firstd " that does not exist. Attempting to create it...\n")
	  (create-directory firstd #t)))




    (print "==================\nMegatest uses a tree of symlinks to provide a uniform structure for finding all the tests
you run over time. Please provide a path where we can create this link tree.")


    (display "Enter link tree directory: ")
    (set! lntree (read-line))
    (if (not (directory? lntree))
	(begin
	  (print "WARNING: you have specified a path " lntree "that does not exist. Attempting to create it...\n")
	  (create-directory lntree #t)))
    
    (with-output-to-file (conc path "/megatest.config")
      (lambda ()


	(print "[fields]")
	(map (lambda (k)(print k " TEXT")) keys)
	(print "")
	(print "[setup]")
	(print "# Adjust max_concurrent_jobs to limit how much you load your machines")
	(print "max_concurrent_jobs 50\n")
	(print "# This is your link path, you can move it but it is generally better to keep it stable")
	(print "linktree " lntree)
	(print "\n# Job tools are more advanced ways to control how your jobs are launched")
	(print "[jobtools]\nuseshell yes\nlauncher nbfind\n")
	(print "# You can override environment variables for all your tests here")
	(print "[env-override]\nEXAMPLE_VAR example value\n")
	(print "# As you run more tests you may need to add additional disks, the names are arbitrary but must be unique")
	(print "[disks]\ndisk0 " firstd)))




    (print "==================\nI'm now creating a runconfigs.config file for you with a default section.
You can use this file to set variables for your tests based on the \"target\" (the combination
of keys).")


    (with-output-to-file (conc path "/runconfigs.config")
      (lambda ()
	(print "# The variables in the default category will be seen in all runs\n[default]\nALLTESTS see this variable\n")
	
	(print "# Your variables here are grouped by targets [" (string-intersperse keys "/") "]")
	(let ((example-target (string-intersperse (map (lambda (k)(conc k "_val")) keys) "/")))
	  (print "[" example-target "]")
	  (print "ANOTHERVAR only defined if target is " example-target))))










    ;; Now create a test and logpro file



    (print "You now have the basic common files for your megatest setup. Next run \"megatest -gentest\"\n"




	   "to create a test.\n\nThanks for using Megatest. You can edit your config files and create tests\n"
	   "in the " path " directory")))



;;======================================================================
;; create skeleton files for a test
;;======================================================================

(define (genexample:mk-megatest-test testname)







|
>








|
|
>

|

|
>






>
>
>
|
|
|
>
|

|
>
|
>
>
|
|
>
|

|
|
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
>


>
>
>
|
|
>
>







>
>
>
|
|
>
>









>
>






|
|

|



|

>
>
>
|

|
>
>







|
>
>
>
>

>
>
>
>
>

>
>
>
|
>
>
>
>
|
|
>







43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
	(path   #f)
	(firstd #f))
    (print "Note: don't worry too much about typos in this process, you will be able to edit
the generated files before starting your first runs")

    ;; create the needed area
    (print "==================\nWhere can I create your Megatest regresssion/continuous build area? Note, your \n"
	   "tests will not necessarily be run in this area, disk space needs are modest. Current directory is:\n\n"
	   (current-directory) "\n")
    (display "Enter your megatest directory: ")
    (set! path (read-line))

    (if (not (directory? path))
	(begin
	  (print "The path " path " does not exist or is not a directory. Attempting to create it now")
	  (create-directory path #t)))

    ;; First check that the directory is empty!
    (if (and (file-exists? path)
	     (not (null? (glob (conc path "/*")))))
	(begin
	  (print "WARNING: directory " path " is not empty, are you sure you want to continue?")
	  (display "Enter y/n: ")
	  (if (equal? "y" (read-line))
	      (print "Using directory " path " for your Megatest area.")
	      (begin
		(print "INFO: Creation of megatest files in " path " aborted")
		(exit 1)))))

    ;; first prompt user for fields
    ;;
    (print
     "==================

Next you must specify fields or keys for your megatest area. These
will be used to organise your runs. One field should probably be
\"RELEASE\".  Other examples of useful fields might be \"PLATFORM\",
\"TARGET_OS\" or if you are in the semiconductor business perhaps
things like \"TECHNOLOGY_NODE\", \"DESIGN_KIT\" or \"METAL_STACK\".

The all caps is a convention because the variables you choose will be
available to your tests as environment variables. You can edit these
values later but it is generally a good idea to settle on them
early.

Your runs will be stored in directories specified by your
keys. Example, if you have keys OSFAMILY/VARIANT/OSVER/RELEASE you may
get a test \"build\" in a directory like this:
linux/ubuntu/11.04/rev_1.2/build

Please enter your keys now, separated by spaces or slashes. Only alpha-numeric characters, 
upper case recommended. Example: COMPILER_VER/RELEASE_NAME/QUAL_LEVEL
")
    (set! keys (let loop ((keystr ""))
		 (if (equal? keystr "q")
		     (begin
		       (print "Quiting ...")
		       (exit))
		     (let ((keylst (apply append
					  (map string-split (string-split keystr "/")))))
		       (if (or (null? keylst)
			       (not (null? (filter string-null? keylst))))
			   (begin
			     (display "Enter keys separated by spaces or slashes: ")
			     (loop (read-line)))
			   keylst)))))

    (print "You have choosen " (string-intersperse keys ", ") " for your keys.")
    
    ;; Now get the link tree location and a first disk
    (print
     "==================

Now you need an initial place to store your runs. These are called \"disks\" and you
can add more at any time. To get going provide a writeable directory name. 

")
    (display "Enter your test runs directory: ")
    (set! firstd (read-line))
    (if (not (directory? firstd))
	(begin
	  (print "WARNING: you have specified a path " firstd " that does not exist. Attempting to create it...\n")
	  (create-directory firstd #t)))

    (print
     "==================

Megatest uses a tree of symlinks to provide a uniform structure for finding all the tests
you run over time. Please provide a path where we can create this link tree.

")
    (display "Enter link tree directory: ")
    (set! lntree (read-line))
    (if (not (directory? lntree))
	(begin
	  (print "WARNING: you have specified a path " lntree "that does not exist. Attempting to create it...\n")
	  (create-directory lntree #t)))
    
    (with-output-to-file (conc path "/megatest.config")
      (lambda ()
	(print "# This area uses Megatest. Learn more at http://www.kiatoa.com/fossils/megatest.")
	(print "#\n")
	(print "[fields]")
	(map (lambda (k)(print k " TEXT")) keys)
	(print "")
	(print "[setup]")
	(print "# Adjust max_concurrent_jobs to limit how much you load your machines")
	(print "max_concurrent_jobs 50\n")
	(print "# This is your link path. Avoid moving it once set.")
	(print "linktree " (common:real-path lntree))
	(print "\n# Job tools are more advanced ways to control how your jobs are launched")
	(print "[jobtools]\nuseshell yes\nlauncher nbfake\nmaxload 1.5\n")
	(print "# You can override environment variables for all your tests here")
	(print "[env-override]\nEXAMPLE_VAR example value\n")
	(print "# As you run more tests you may need to add additional disks, the names are arbitrary but must be unique")
	(print "[disks]\ndisk0 " (common:real-path firstd))))

    (print
     "==================

I'm now creating a runconfigs.config file for you with a default section.
You can use this file to set variables for your tests based on the \"target\" (the combination
of keys).

")
    (with-output-to-file (conc path "/runconfigs.config")
      (lambda ()
	(print "# The variables in the default category will be seen in all runs\n[default]\nALLTESTS see this variable\n")
	
	(print "# Your variables here are grouped by targets [" (string-intersperse keys "/") "]")
	(let ((example-target (string-intersperse (map (lambda (k)(conc k "_val")) keys) "/")))
	  (print "[" example-target "]")
	  (print "ANOTHERVAR only defined if target is " example-target))
	(print "\n# It can be handy to include a file based on the users unix username.\n"
	       "# To prevent cluttering up the top level directory we'll put this file\n# in a directory called \"configs\".")
	(print "[include #{getenv USER}.config]")
	))

    (create-directory (conc path "/configs") #t)
    (with-output-to-file (conc path "/configs/" (current-user-name) ".config")
      (lambda ()
	(print "# Override settings in ../runconfigs.config for user " (current-user-name) " here.")))
    
    ;; Now create a test and logpro file
    (print
     "==================

You now have the basic common files for your megatest setup. Next run
\"megatest -gen-test\" to create a test.

Thank you for using Megatest. 

You can edit your config files and create tests in the " path " directory

")))


;;======================================================================
;; create skeleton files for a test
;;======================================================================

(define (genexample:mk-megatest-test testname)
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205


206
207
208
209
210
211
212
213
     ((file-exists? "../megatest.config")      (set! rel-path "../"))
     ((file-exists? "../../megatest.config")   (set! rel-path "../../"))
     ((file-exists? "../../../megatest.config")(set! rel-path "../../../"))) ;; good enough dang it.

    ;; Don't gather data or continue if a) megatest.config can't be found or b) testconfig already exists
    (if (not rel-path)
	(begin
	  (print "ERROR: I could not find megatest.config, please run -get-megatest-test in the top dir of your megatest area")
	  (exit 1)))

    (if (file-exists? (conc rel-path "tests/" testname "/testconfig"))
	(begin
	  (print "WARNING: You already have a testconfig in " rel-path "tests/" testname ", do you want to clobber your files?")
	  (display "Enter y/n: ")
	  (if (not (equal? "y" (read-line)))
	      (begin
		(print "INFO: user abort of creation of test " testname)
		(exit 1)))))

    (print "We are going to generate a skeleton set of files for your test " testname "\n"
	   " *** Note: do not worry too much about typos, you can edit the files created when you are done.")

    (print "\n\n==================\nPlease describe this test. The description will be visible in various dialogs and reports")
    (display "Enter one line description for this test: ")
    (set! description (read-line))

    (print "\n\n==================\nDoes this test, " testname ", require any other test be run prior to launch?")
    (display (conc "Enter space delimited list of tests which " testname " must wait for: "))
    (set! waiton (read-line))

    (print "\n\n==================\nDo you wish to prioritize the running of this test over other tests? If so")
    (print "enter a number greater than zero here")
    (display "Enter a priority of 0 (default) or higher: ")
    (set! priority (read-line))

    ;; Get the steps
    (print "\n\n==================\nNow to enter the one or more steps that make up your test, note; you can add more later")
    (print "Hint; use .sh extension on the script names and we'll create placeholder scripts."

    (let ((stepname   #f)
	  (scriptname #f))
      (let loop ((done #f))
	(display "Enter the name for this step (blank to stop): ")
	(set! stepname (read-line))
	(if (not (equal? stepname ""))
	    (begin
	      (display "Enter the script or progam to run: ")
	      (set! scriptname (read-line))
	      (set! steps (append steps (list (list stepname scriptname))))))
	(if (not (equal? stepname ""))


	    (loop #f))))

    ;; Get the items
    (print "\n\n==================\nNext we need to get the variables and values you wish to iterate this test over (blank for none)")
    (let ((varname #f)
	  (values  #f))
      (let loop ((done #f))
	(display "Enter the variable name: ")







|














|




|








|
|












>
>
|







216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
     ((file-exists? "../megatest.config")      (set! rel-path "../"))
     ((file-exists? "../../megatest.config")   (set! rel-path "../../"))
     ((file-exists? "../../../megatest.config")(set! rel-path "../../../"))) ;; good enough dang it.

    ;; Don't gather data or continue if a) megatest.config can't be found or b) testconfig already exists
    (if (not rel-path)
	(begin
	  (print "ERROR: I could not find megatest.config, please run -create-test in the top dir of your megatest area")
	  (exit 1)))

    (if (file-exists? (conc rel-path "tests/" testname "/testconfig"))
	(begin
	  (print "WARNING: You already have a testconfig in " rel-path "tests/" testname ", do you want to clobber your files?")
	  (display "Enter y/n: ")
	  (if (not (equal? "y" (read-line)))
	      (begin
		(print "INFO: user abort of creation of test " testname)
		(exit 1)))))

    (print "We are going to generate a skeleton set of files for your test " testname "\n"
	   " *** Note: do not worry too much about typos, you can edit the files created when you are done.")

    (print "\n==================\nPlease describe this test. The description will be visible in various dialogs and reports")
    (display "Enter one line description for this test: ")
    (set! description (read-line))

    (print "\n\n==================\nDoes this test, " testname ", require any other test be run prior to launch?")
    (display (conc "Enter space delimited list of tests which " testname " must wait for (default is no waiton): "))
    (set! waiton (read-line))

    (print "\n\n==================\nDo you wish to prioritize the running of this test over other tests? If so")
    (print "enter a number greater than zero here")
    (display "Enter a priority of 0 (default) or higher: ")
    (set! priority (read-line))

    ;; Get the steps
    (print "\n==================\nNow to enter the one or more steps that make up your test, note; you can add more later")
    (print "Hint; use .sh extension on the script names and we'll create a placeholder script."

    (let ((stepname   #f)
	  (scriptname #f))
      (let loop ((done #f))
	(display "Enter the name for this step (blank to stop): ")
	(set! stepname (read-line))
	(if (not (equal? stepname ""))
	    (begin
	      (display "Enter the script or progam to run: ")
	      (set! scriptname (read-line))
	      (set! steps (append steps (list (list stepname scriptname))))))
	(if (not (equal? stepname ""))
	    (begin
	      (print "Added step " stepname " to list of steps.\n")
	      (loop #f)))))

    ;; Get the items
    (print "\n\n==================\nNext we need to get the variables and values you wish to iterate this test over (blank for none)")
    (let ((varname #f)
	  (values  #f))
      (let loop ((done #f))
	(display "Enter the variable name: ")
229
230
231
232
233
234
235
236
237
238
239
240














241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
	  (create-directory testdir #t)
	  (with-output-to-file (conc testdir "/testconfig")
	    (lambda ()
	      (print "# Add additional steps here. Format is \"stepname script\"\n[ezsteps]")
	      (map (lambda (stp)(print (string-intersperse stp " "))) steps)
	      (print "")
	      (print "# Test requirements are specified here\n[requirements]")
	      (print "waiton " waiton)
	      (print "priority " priority "\n")
	      (print "# Iteration for your tests are controlled by the items section\n[items]")
	      (map print items)
	      (print "")














	      (print "# test_meta is a section for storing additional data on your test\n[test_meta]")
	      (print "author " (get-environment-variable "USER"))
	      (print "owner  " (get-environment-variable "USER"))
	      (print "description " description)
	      (print "tags tagone,tagtwo")
	      (print "reviewed never")))
	  ;; Now create shell scripts (if extension is .sh) and logpro files
	  (for-each (lambda (stp)
		      (let ((stepname (car stp))
			    (script   (cadr stp)))
			(with-output-to-file (conc testdir "/" stepname ".logpro")
			  (lambda ()
			    (print genexample:example-logpro)))
			(with-output-to-file (conc testdir "/" script ".sh")
			  (lambda ()
			    (print genexample:example-script)))
			(system (conc "chmod ug+r " (conc testdir "/" script ".sh")))))
		    steps))))))
	  ;; 







|
|
|


>
>
>
>
>
>
>
>
>
>
>
>
>
>










|
|
<
|
|
|
|

<
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328

329
330
331
332
333

	  (create-directory testdir #t)
	  (with-output-to-file (conc testdir "/testconfig")
	    (lambda ()
	      (print "# Add additional steps here. Format is \"stepname script\"\n[ezsteps]")
	      (map (lambda (stp)(print (string-intersperse stp " "))) steps)
	      (print "")
	      (print "# Test requirements are specified here\n[requirements]")
	      (print (if (string-null? waiton) "# " "") "waiton " waiton)
	      (print "priority " (if (string-null? priority) 0 priority) "\n")
	      (print "# Iteration for your test is controlled by the items section\n[items]")
	      (map print items)
	      (print "")
	      (print "# Alternatively you could use a [itemstable] section")
	      (print "# [itemstable]")
	      (print "# ITEMVAR1  a    b    c")
	      (print "# ITEMVAR2  d    e    f")
	      (print "#\n# would result in items: a/d  b/e   c/f\n#\n")
	      (print "# Logpro rules for each step can be captured here in the testconfig")
	      (print "# note: The ;; after the stepname and the leading whitespace are required")
	      (print "#\n[logpro]\n")
	      (for-each (lambda (step)
			  (let ((stepname   (car step))
				(scriptname (cadr step)))
			    (print stepname " ;; rules for checking output from running step " step " with command " scriptname)
			    (print genexample:example-logpro "\n")))
			steps)
	      (print "# test_meta is a section for storing additional data on your test\n[test_meta]")
	      (print "author " (get-environment-variable "USER"))
	      (print "owner  " (get-environment-variable "USER"))
	      (print "description " description)
	      (print "tags tagone,tagtwo")
	      (print "reviewed never")))
	  ;; Now create shell scripts (if extension is .sh) and logpro files
	  (for-each (lambda (stp)
		      (let ((stepname (car stp))
			    (script   (cadr stp)))
			(if (string-match ".*\\.sh$" script)
			    (begin

			      (with-output-to-file (conc testdir "/" script)
				(lambda ()
				  (print genexample:example-script)))
			      (system (conc "chmod ug+r,a+x " (conc testdir "/" script)))))))
		    steps))))))

Modified gutils.scm from [628c78d614] to [4ace6c42c8].

46
47
48
49
50
51
52










53
54
    ((LAUNCHED)         (list "101 123 142"  state))
    ((CHECK)            (list "255 100 50"   state))
    ((REMOTEHOSTSTART)  (list "50 130 195"   state))
    ((RUNNING)          (list "9 131 232"    state))
    ((KILLREQ)          (list "39 82 206"    state))
    ((KILLED)           (list "234 101 17"   state))
    ((NOT_STARTED)      (list "240 240 240"  state))










    (else               (list "192 192 192"  state))))








>
>
>
>
>
>
>
>
>
>


46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    ((LAUNCHED)         (list "101 123 142"  state))
    ((CHECK)            (list "255 100 50"   state))
    ((REMOTEHOSTSTART)  (list "50 130 195"   state))
    ((RUNNING)          (list "9 131 232"    state))
    ((KILLREQ)          (list "39 82 206"    state))
    ((KILLED)           (list "234 101 17"   state))
    ((NOT_STARTED)      (list "240 240 240"  state))
    ;; for xor mode below
    ;;
    ((CLEAN)
     (case (string->symbol status)
       ((CLEAN-FAIL CLEAN-CHECK CLEAN-ABORT)  (list "200 130 13" status)) ;; orange requested for these
       (else  (list "60  235 63" status))))
    ((DIRTY-BETTER)     (list "160  255 153" status))
    ((DIRTY-WORSE)      (list "165 42  42" status))
    ((BOTH-BAD)         (list "180 33 49" status))

    (else               (list "192 192 192"  state))))

Modified launch.scm from [3e54d60917] to [cdfd063ce7].

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

;;======================================================================
;; launch a task - this runs on the originating host, tests themselves
;;
;;======================================================================

(use regex regex-case base64 sqlite3 srfi-18 directory-utils posix-extras z3 call-with-environment-variables csv)
(use defstruct pathname-expand)

(import (prefix base64 base64:))
(import (prefix sqlite3 sqlite3:))

(declare (unit launch))
(declare (uses common))
(declare (uses configf))







|







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

;;======================================================================
;; launch a task - this runs on the originating host, tests themselves
;;
;;======================================================================

(use regex regex-case base64 sqlite3 srfi-18 directory-utils posix-extras z3 call-with-environment-variables csv)
(use typed-records pathname-expand)

(import (prefix base64 base64:))
(import (prefix sqlite3 sqlite3:))

(declare (unit launch))
(declare (uses common))
(declare (uses configf))
66
67
68
69
70
71
72

73


74
75
76
77
78
79
80
    (if (file-exists? cname)
	(let* ((dat  (read-config cname #f #f))
	       (csvr (db:logpro-dat->csv dat stepname))
	       (csvt (let-values (( (fmt-cell fmt-record fmt-csv) (make-format ",")))
				 (fmt-csv (map list->csv-record csvr))))
	       (status (configf:lookup dat "final" "exit-status"))
	       (msg     (configf:lookup dat "final" "message")))

	  (rmt:csv->test-data run-id test-id csvt)


	  (cond
	   ((equal? status "PASS") "PASS") ;; skip the message part if status is pass
	   (status (conc (configf:lookup dat "final" "exit-status") ": " (if msg msg "no message")))
	   (else #f)))
	#f)))

(define (launch:runstep ezstep run-id test-id exit-info m tal testconfig)







>
|
>
>







66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    (if (file-exists? cname)
	(let* ((dat  (read-config cname #f #f))
	       (csvr (db:logpro-dat->csv dat stepname))
	       (csvt (let-values (( (fmt-cell fmt-record fmt-csv) (make-format ",")))
				 (fmt-csv (map list->csv-record csvr))))
	       (status (configf:lookup dat "final" "exit-status"))
	       (msg     (configf:lookup dat "final" "message")))
          ;;(if csvt  ;; this if blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
              (rmt:csv->test-data run-id test-id csvt)
            ;;  (BB> "Error: run-id/test-id/stepname="run-id"/"test-id"/"stepname" => bad csvr="csvr)
            ;;  )
	  (cond
	   ((equal? status "PASS") "PASS") ;; skip the message part if status is pass
	   (status (conc (configf:lookup dat "final" "exit-status") ": " (if msg msg "no message")))
	   (else #f)))
	#f)))

(define (launch:runstep ezstep run-id test-id exit-info m tal testconfig)
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
	      (list  "MT_TEST_NAME" test-name)
	      (list  "MT_ITEM_INFO" (conc itemdat))
	      (list  "MT_ITEMPATH"  item-path)
	      (list  "MT_RUNNAME"   runname)
	      (list  "MT_MEGATEST"  megatest)
	      (list  "MT_TARGET"    target)
	      (list  "MT_LINKTREE"  (configf:lookup *configdat* "setup" "linktree"))
	      (list  "MT_TESTSUITENAME" (common:get-testsuite-name))))

	  (if mt-bindir-path (setenv "PATH" (conc (getenv "PATH") ":" mt-bindir-path)))
	  ;; (change-directory top-path)
	  ;; Can setup as client for server mode now
	  ;; (client:setup)

	  







|







551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
	      (list  "MT_TEST_NAME" test-name)
	      (list  "MT_ITEM_INFO" (conc itemdat))
	      (list  "MT_ITEMPATH"  item-path)
	      (list  "MT_RUNNAME"   runname)
	      (list  "MT_MEGATEST"  megatest)
	      (list  "MT_TARGET"    target)
	      (list  "MT_LINKTREE"  (configf:lookup *configdat* "setup" "linktree"))
	      (list  "MT_TESTSUITE_NAME" (common:get-testsuite-name))))

	  (if mt-bindir-path (setenv "PATH" (conc (getenv "PATH") ":" mt-bindir-path)))
	  ;; (change-directory top-path)
	  ;; Can setup as client for server mode now
	  ;; (client:setup)

	  
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
;;   returns:
;;     *toppath*
;;   side effects:
;;     sets; *configdat*    (megatest.config info)
;;           *runconfigdat* (runconfigs.config info)
;;           *configstatus* (status of the read data)
;;
(define (launch:setup-new #!key (force #f))
  (let* ((toppath  (or *toppath* (getenv "MT_RUN_AREA_HOME"))) ;; preserve toppath
	 (runname  (common:args-get-runname))
	 (target   (common:args-get-target))
	 (linktree (common:get-linktree))
	 (sections (if target (list "default" target) #f)) ;; for runconfigs
	 (mtconfig (or (args:get-arg "-config") "megatest.config")) ;; allow overriding megatest.config 
	 (rundir   (if (and runname target linktree)(conc linktree "/" target "/" runname) #f))







|







699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
;;   returns:
;;     *toppath*
;;   side effects:
;;     sets; *configdat*    (megatest.config info)
;;           *runconfigdat* (runconfigs.config info)
;;           *configstatus* (status of the read data)
;;
(define (launch:setup #!key (force #f))
  (let* ((toppath  (or *toppath* (getenv "MT_RUN_AREA_HOME"))) ;; preserve toppath
	 (runname  (common:args-get-runname))
	 (target   (common:args-get-target))
	 (linktree (common:get-linktree))
	 (sections (if target (list "default" target) #f)) ;; for runconfigs
	 (mtconfig (or (args:get-arg "-config") "megatest.config")) ;; allow overriding megatest.config 
	 (rundir   (if (and runname target linktree)(conc linktree "/" target "/" runname) #f))
819
820
821
822
823
824
825

826

827
828
829
830
831
832
833
834
835
836
837
838
839
	       (if (not (file-exists? tlink))
		   (create-symbolic-link linktree tlink)))))
	  (begin
	    (debug:print-error 0 *default-log-port* "linktree not defined in [setup] section of megatest.config")
	    )))
    (if (and *toppath*
	     (directory-exists? *toppath*))

	(setenv "MT_RUN_AREA_HOME" *toppath*)

	(begin
	  (debug:print-error 0 *default-log-port* "failed to find the top path to your Megatest area.")))
    *toppath*))

(define launch:setup launch:setup-new)

(define (get-best-disk confdat testconfig)
  (let* ((disks   (or (and testconfig (hash-table-ref/default testconfig "disks" #f))
		      (hash-table-ref/default confdat "disks" #f)))
	 (minspace (let ((m (configf:lookup confdat "setup" "minspace")))
		     (string->number (or m "10000")))))
    (if disks 
	(let ((res (common:get-disk-with-most-free-space disks minspace))) ;; min size of 1000, seems tad dumb







>
|
>




<
<







822
823
824
825
826
827
828
829
830
831
832
833
834
835


836
837
838
839
840
841
842
	       (if (not (file-exists? tlink))
		   (create-symbolic-link linktree tlink)))))
	  (begin
	    (debug:print-error 0 *default-log-port* "linktree not defined in [setup] section of megatest.config")
	    )))
    (if (and *toppath*
	     (directory-exists? *toppath*))
	(begin
	  (setenv "MT_RUN_AREA_HOME" *toppath*)
	  (setenv "MT_TESTSUITE_NAME" (common:get-testsuite-name)))
	(begin
	  (debug:print-error 0 *default-log-port* "failed to find the top path to your Megatest area.")))
    *toppath*))



(define (get-best-disk confdat testconfig)
  (let* ((disks   (or (and testconfig (hash-table-ref/default testconfig "disks" #f))
		      (hash-table-ref/default confdat "disks" #f)))
	 (minspace (let ((m (configf:lookup confdat "setup" "minspace")))
		     (string->number (or m "10000")))))
    (if disks 
	(let ((res (common:get-disk-with-most-free-space disks minspace))) ;; min size of 1000, seems tad dumb
1030
1031
1032
1033
1034
1035
1036








1037
1038
1039
1040
1041
1042
1043
;; 2. create run dir on disk, path name is meaningful
;; 3. create link from run dir to megatest runs area 
;; 4. remotely run the test on allocated host
;;    - could be ssh to host from hosts table (update regularly with load)
;;    - could be netbatch
;;      (launch-test db (cadr status) test-conf))
(define (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat params)








  (change-directory *toppath*)
  (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
   (list ;; (list "MT_TEST_RUN_DIR" work-area)
    (list "MT_RUN_AREA_HOME" *toppath*)
    (list "MT_TEST_NAME" test-name)
    ;; (list "MT_ITEM_INFO" (conc itemdat)) 
    (list "MT_RUNNAME"   runname)







>
>
>
>
>
>
>
>







1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
;; 2. create run dir on disk, path name is meaningful
;; 3. create link from run dir to megatest runs area 
;; 4. remotely run the test on allocated host
;;    - could be ssh to host from hosts table (update regularly with load)
;;    - could be netbatch
;;      (launch-test db (cadr status) test-conf))
(define (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat params)
  (let loop ((delta        (- (current-seconds) *last-launch*))
	     (launch-delay (string->number (or (configf:lookup *configdat* "setup" "launch-delay") "5"))))
    (if (> launch-delay delta)
	(begin
	  (debug:print-info 0 *default-log-port* "Delaying launch of " test-name " for " (- launch-delay delta) " seconds")
	  (thread-sleep! (- launch-delay delta))
	  (loop (- (current-seconds) *last-launch*) launch-delay))))
  (set! *last-launch* (current-seconds))
  (change-directory *toppath*)
  (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
   (list ;; (list "MT_TEST_RUN_DIR" work-area)
    (list "MT_RUN_AREA_HOME" *toppath*)
    (list "MT_TEST_NAME" test-name)
    ;; (list "MT_ITEM_INFO" (conc itemdat)) 
    (list "MT_RUNNAME"   runname)

Modified megatest-version.scm from [9743cff00a] to [d8346aabbc].

1
2
3
4
5
6
7
;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6105)






|

1
2
3
4
5
6
7
;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6207)

Modified megatest.scm from [36ef6b845c] to [3fb995e08e].

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  -import-megatest.db     : migrate a database from v1.55 series to v1.60 series
  -sync-to-megatest.db    : migrate data back to megatest.db
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -transport http|zmq     : use http or zmq for transport (default is http) 
  -daemonize              : fork into background and disconnect from stdin/out
  -log logfile            : send stdout and stderr to logfile
  -list-servers           : list the servers 
  -stop-server id         : stop server specified by id (see output of -list-servers), use
                            0 to kill all
  -repl                   : start a repl (useful for extending megatest)
  -load file.scm          : load and run file.scm







|







144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  -import-megatest.db     : migrate a database from v1.55 series to v1.60 series
  -sync-to-megatest.db    : migrate data back to megatest.db
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -transport http|rpc     : use http or rpc for transport (default is http) 
  -daemonize              : fork into background and disconnect from stdin/out
  -log logfile            : send stdout and stderr to logfile
  -list-servers           : list the servers 
  -stop-server id         : stop server specified by id (see output of -list-servers), use
                            0 to kill all
  -repl                   : start a repl (useful for extending megatest)
  -load file.scm          : load and run file.scm
167
168
169
170
171
172
173

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove


Spreadsheet generation
  -extract-ods fname.ods  : extract an open document spreadsheet from the database
  -pathmod path           : insert path, i.e. path/runame/itempath/logfile.html
                            will clear the field if no rundir/testname/itempath/logfile
                            if it contains forward slashes the path will be converted
                            to windows style
Getting started
  -gen-megatest-area       : create a skeleton megatest area. You will be prompted for paths
  -gen-megatest-test tname : create a skeleton megatest test. You will be prompted for info

Examples

# Get test path, use '.' to get a single path or a specific path/file pattern
megatest -test-files 'logs/*.log' -target ubuntu/n%/no% -runname w49% -testpatt test_mt%

Called as " (string-intersperse (argv) " ") "







>








|
|







167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove
  -generate-html          : create a simple html tree for browsing your runs

Spreadsheet generation
  -extract-ods fname.ods  : extract an open document spreadsheet from the database
  -pathmod path           : insert path, i.e. path/runame/itempath/logfile.html
                            will clear the field if no rundir/testname/itempath/logfile
                            if it contains forward slashes the path will be converted
                            to windows style
Getting started
  -create-megatest-area       : create a skeleton megatest area. You will be prompted for paths
  -create-test testname       : create a skeleton megatest test. You will be prompted for info

Examples

# Get test path, use '.' to get a single path or a specific path/file pattern
megatest -test-files 'logs/*.log' -target ubuntu/n%/no% -runname w49% -testpatt test_mt%

Called as " (string-intersperse (argv) " ") "
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264

265
266
267
268
269
270
271
			"-env2file"
			"-envcap"
			"-envdelta"
			"-setvars"
			"-set-state-status"
			"-set-run-status"
			"-debug" ;; for *verbosity* > 2
			"-gen-megatest-test"
			"-override-timeout"
			"-test-files"  ;; -test-paths is for listing all
			"-load"        ;; load and exectute a scheme file
			"-section"
			"-var"
			"-dumpmode"
			"-run-id"
			"-ping"
			"-refdb2dat"
			"-o"
			"-log"
			"-archive"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			) 
		 (list  "-h" "-help" "--help"

			"-version"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-show-keys"
		        "-test-status"
			"-set-values"







|


















>







240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
			"-env2file"
			"-envcap"
			"-envdelta"
			"-setvars"
			"-set-state-status"
			"-set-run-status"
			"-debug" ;; for *verbosity* > 2
			"-create-test"
			"-override-timeout"
			"-test-files"  ;; -test-paths is for listing all
			"-load"        ;; load and exectute a scheme file
			"-section"
			"-var"
			"-dumpmode"
			"-run-id"
			"-ping"
			"-refdb2dat"
			"-o"
			"-log"
			"-archive"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			) 
		 (list  "-h" "-help" "--help"
			"-manual"
			"-version"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-show-keys"
		        "-test-status"
			"-set-values"
281
282
283
284
285
286
287

288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			"-local"         ;; run some commands using local db access


			;; misc queries
			"-list-disks"
			"-list-targets"
			"-list-db-targets"
			"-show-runconfig"
			"-show-config"
			"-show-cmdinfo"
			"-get-run-status"

			;; queries
			"-test-paths" ;; get path(s) to a test, ordered by youngest first

			"-runall"    ;; run all tests, respects -testpatt, defaults to %
			"-run"       ;; alias for -runall
			"-remove-runs"
			"-rebuild-db"
			"-cleanup-db"
			"-rollup"
			"-update-meta"
			"-gen-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"








>




















|







283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			"-local"         ;; run some commands using local db access
                        "-generate-html"

			;; misc queries
			"-list-disks"
			"-list-targets"
			"-list-db-targets"
			"-show-runconfig"
			"-show-config"
			"-show-cmdinfo"
			"-get-run-status"

			;; queries
			"-test-paths" ;; get path(s) to a test, ordered by youngest first

			"-runall"    ;; run all tests, respects -testpatt, defaults to %
			"-run"       ;; alias for -runall
			"-remove-runs"
			"-rebuild-db"
			"-cleanup-db"
			"-rollup"
			"-update-meta"
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"

401
402
403
404
405
406
407











408
409
410
411
412
413
414

(if (or (args:get-arg "-h")
	(args:get-arg "-help")
	(args:get-arg "--help"))
    (begin
      (print help)
      (exit)))












(if (args:get-arg "-start-dir")
    (if (file-exists? (args:get-arg "-start-dir"))
	(change-directory (args:get-arg "-start-dir"))
	(begin
	  (debug:print-error 0 *default-log-port* "non-existant start dir " (args:get-arg "-start-dir") " specified, exiting.")
	  (exit 1))))







>
>
>
>
>
>
>
>
>
>
>







404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428

(if (or (args:get-arg "-h")
	(args:get-arg "-help")
	(args:get-arg "--help"))
    (begin
      (print help)
      (exit)))

(if (args:get-arg "-manual")
    (let* ((htmlviewercmd (or (configf:lookup *configdat* "setup" "htmlviewercmd")
			      (common:which '("firefox" "arora"))))
	   (install-home  (common:get-install-area))
	   (manual-html   (conc install-home "/share/docs/megatest_manual.html")))
      (if (and install-home
	       (file-exists? manual-html))
	  (system (conc "(" htmlviewercmd " " manual-html " ) &"))
	  (system (conc "(" htmlviewercmd " http://www.kiatoa.com/cgi-bin/fossils/megatest/doc/tip/docs/manual/megatest_manual.html ) &")))
      (exit)))

(if (args:get-arg "-start-dir")
    (if (file-exists? (args:get-arg "-start-dir"))
	(change-directory (args:get-arg "-start-dir"))
	(begin
	  (debug:print-error 0 *default-log-port* "non-existant start dir " (args:get-arg "-start-dir") " specified, exiting.")
	  (exit 1))))
722
723
724
725
726
727
728
729

730
731
732
733
734
735
736
737
738
739
740
741
742
743

744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766

767
768
769
770
771
772

773
774
775
776
777
778
779
780

(if (args:get-arg "-server")

    ;; Server? Start up here.
    ;;
    (let ((tl        (launch:setup))
	  (run-id    (and (args:get-arg "-run-id")
			  (string->number (args:get-arg "-run-id")))))

      (if run-id
	  (begin
	    (server:launch run-id)
	    (set! *didsomething* #t))
	  (debug:print-error 0 *default-log-port* "server requires run-id be specified with -run-id")))

    ;; Not a server? This section will decide how to communicate
    ;;
    ;;  Setup client for all expect listed here
    (if (null? (lset-intersection 
		equal?
		(hash-table-keys args:arg-hash)
		'("-list-servers"
		  "-stop-server"

		  "-show-cmdinfo"
		  "-list-runs"
		  "-ping")))
	(if (launch:setup)
	    (let ((run-id    (and (args:get-arg "-run-id")
				  (string->number (args:get-arg "-run-id")))))
	      ;; (set! *fdb*   (filedb:open-db (conc *toppath* "/db/paths.db")))
	      ;; if not list or kill then start a client (if appropriate)
	      (if (or (args-defined? "-h" "-version" "-gen-megatest-area" "-gen-megatest-test")
		      (eq? (length (hash-table-keys args:arg-hash)) 0))
		  (debug:print-info 1 *default-log-port* "Server connection not needed")
		  (begin
		    ;; (if run-id 
		    ;;     (client:launch run-id) 
		    ;;     (client:launch 0)      ;; without run-id we'll start a server for "0"
		    #t
		    ))))))

;; MAY STILL NEED THIS
;;		       (set! *megatest-db* (make-dbr:dbstruct path: *toppath* local: #t))))))))))

(if (or (args:get-arg "-list-servers")
	(args:get-arg "-stop-server"))

    (let ((tl (launch:setup)))
      (if tl 
	  (let* ((tdbdat  (tasks:open-db))
		 (servers (tasks:get-all-servers (db:delay-if-busy tdbdat)))
		 (fmtstr  "~5a~12a~8a~20a~24a~10a~10a~10a~10a\n")
		 (servers-to-kill '())

		 (killinfo   (args:get-arg "-stop-server"))
		 (khost-port (if killinfo (if (substring-index ":" killinfo)(string-split ":") #f) #f))
		 (sid        (if killinfo (if (substring-index ":" killinfo) #f (string->number killinfo)) #f)))
	    (format #t fmtstr "Id" "MTver" "Pid" "Host" "Interface:OutPort" "InPort" "LastBeat" "State" "Transport")
	    (format #t fmtstr "==" "=====" "===" "====" "=================" "======" "========" "=====" "=========")
	    (for-each 
	     (lambda (server)
	       (let* ((id         (vector-ref server 0))







|
>


|











>








|













|
>






>
|







736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798

(if (args:get-arg "-server")

    ;; Server? Start up here.
    ;;
    (let ((tl        (launch:setup))
	  (run-id    (and (args:get-arg "-run-id")
			  (string->number (args:get-arg "-run-id"))))
          (transport-type (string->symbol (or (args:get-arg "-transport") "http"))))
      (if run-id
	  (begin
	    (server:launch run-id transport-type)
	    (set! *didsomething* #t))
	  (debug:print-error 0 *default-log-port* "server requires run-id be specified with -run-id")))

    ;; Not a server? This section will decide how to communicate
    ;;
    ;;  Setup client for all expect listed here
    (if (null? (lset-intersection 
		equal?
		(hash-table-keys args:arg-hash)
		'("-list-servers"
		  "-stop-server"
                  "-kill-server"
		  "-show-cmdinfo"
		  "-list-runs"
		  "-ping")))
	(if (launch:setup)
	    (let ((run-id    (and (args:get-arg "-run-id")
				  (string->number (args:get-arg "-run-id")))))
	      ;; (set! *fdb*   (filedb:open-db (conc *toppath* "/db/paths.db")))
	      ;; if not list or kill then start a client (if appropriate)
	      (if (or (args-defined? "-h" "-version" "-create-megatest-area" "-create-test")
		      (eq? (length (hash-table-keys args:arg-hash)) 0))
		  (debug:print-info 1 *default-log-port* "Server connection not needed")
		  (begin
		    ;; (if run-id 
		    ;;     (client:launch run-id) 
		    ;;     (client:launch 0)      ;; without run-id we'll start a server for "0"
		    #t
		    ))))))

;; MAY STILL NEED THIS
;;		       (set! *megatest-db* (make-dbr:dbstruct path: *toppath* local: #t))))))))))

(if (or (args:get-arg "-list-servers")
	(args:get-arg "-stop-server")
        (args:get-arg "-kill-server"))
    (let ((tl (launch:setup)))
      (if tl 
	  (let* ((tdbdat  (tasks:open-db))
		 (servers (tasks:get-all-servers (db:delay-if-busy tdbdat)))
		 (fmtstr  "~5a~12a~8a~20a~24a~10a~10a~10a~10a\n")
		 (servers-to-kill '())
                 (kill-switch  (if (args:get-arg "-kill-server") "-9" ""))
                 (killinfo   (or (args:get-arg "-stop-server") (args:get-arg "-kill-server") ))
		 (khost-port (if killinfo (if (substring-index ":" killinfo)(string-split ":") #f) #f))
		 (sid        (if killinfo (if (substring-index ":" killinfo) #f (string->number killinfo)) #f)))
	    (format #t fmtstr "Id" "MTver" "Pid" "Host" "Interface:OutPort" "InPort" "LastBeat" "State" "Transport")
	    (format #t fmtstr "==" "=====" "===" "====" "=================" "======" "========" "=====" "=========")
	    (for-each 
	     (lambda (server)
	       (let* ((id         (vector-ref server 0))
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
		     (if (> last-update 20)        ;; Mark as dead if not updated in last 20 seconds
			 (tasks:server-deregister (db:delay-if-busy tdbdat) hostname pullport: pullport pid: pid)))
		 (format #t fmtstr id mt-ver pid hostname (conc interface ":" pullport) pubport last-update
			 (if status "alive" "dead") transport)
		 (if (or (equal? id sid)
			 (equal? sid 0)) ;; kill all/any
		     (begin
		       (debug:print-info 0 *default-log-port* "Attempting to stop server with pid " pid)
		       (tasks:kill-server status hostname pullport pid transport)))))
	     servers)
	    (debug:print-info 1 *default-log-port* "Done with listservers")
	    (set! *didsomething* #t)
	    (exit)) ;; must do, would have to add checks to many/all calls below
	  (exit))))

;;======================================================================







|
|







818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
		     (if (> last-update 20)        ;; Mark as dead if not updated in last 20 seconds
			 (tasks:server-deregister (db:delay-if-busy tdbdat) hostname pullport: pullport pid: pid)))
		 (format #t fmtstr id mt-ver pid hostname (conc interface ":" pullport) pubport last-update
			 (if status "alive" "dead") transport)
		 (if (or (equal? id sid)
			 (equal? sid 0)) ;; kill all/any
		     (begin
		       (debug:print-info 0 *default-log-port* "Attempting to kill "kill-switch" server with pid " pid)
		       (tasks:kill-server hostname pid kill-switch: kill-switch)))))
	     servers)
	    (debug:print-info 1 *default-log-port* "Done with listservers")
	    (set! *didsomething* #t)
	    (exit)) ;; must do, would have to add checks to many/all calls below
	  (exit))))

;;======================================================================
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
	(args:get-arg "-get-run-status"))
    (general-run-call
     "-set-run-status"
     "set run status"
     (lambda (target runname keys keyvals)
       (let* ((runsdat  (rmt:get-runs-by-patt keys runname 
					(common:args-get-target)
					#f #f #f))
	      (header   (vector-ref runsdat 0))
	      (rows     (vector-ref runsdat 1)))
	 (if (null? rows)
	     (begin
	       (debug:print-info 0 *default-log-port* "No matching run found.")
	       (exit 1))
	     (let* ((row      (car (vector-ref runsdat 1)))







|







999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
	(args:get-arg "-get-run-status"))
    (general-run-call
     "-set-run-status"
     "set run status"
     (lambda (target runname keys keyvals)
       (let* ((runsdat  (rmt:get-runs-by-patt keys runname 
					(common:args-get-target)
					#f #f #f #f))
	      (header   (vector-ref runsdat 0))
	      (rows     (vector-ref runsdat 1)))
	 (if (null? rows)
	     (begin
	       (debug:print-info 0 *default-log-port* "No matching run found.")
	       (exit 1))
	     (let* ((row      (car (vector-ref runsdat 1)))
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835

(if (args:get-arg "-gui")
    (begin
      (debug:print 0 *default-log-port* "Look at the dashboard for now")
      ;; (megatest-gui)
      (set! *didsomething* #t)))

(if (args:get-arg "-gen-megatest-area")
    (begin
      (genexample:mk-megatest.config)
      (set! *didsomething* #t)))

(if (args:get-arg "-gen-megatest-test")
    (let ((testname (args:get-arg "-gen-megatest-test")))
      (genexample:mk-megatest-test testname)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the database schema, clean up the db
;;======================================================================








|




|
|







1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853

(if (args:get-arg "-gui")
    (begin
      (debug:print 0 *default-log-port* "Look at the dashboard for now")
      ;; (megatest-gui)
      (set! *didsomething* #t)))

(if (args:get-arg "-create-megatest-area")
    (begin
      (genexample:mk-megatest.config)
      (set! *didsomething* #t)))

(if (args:get-arg "-create-test")
    (let ((testname (args:get-arg "-create-test")))
      (genexample:mk-megatest-test testname)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the database schema, clean up the db
;;======================================================================

1877
1878
1879
1880
1881
1882
1883

1884
1885
1886
1887
1888
1889
1890
      (set! *didsomething* #t)))

;;======================================================================
;; Start a repl
;;======================================================================

;; fakeout readline


(if (or (getenv "MT_RUNSCRIPT")
	(args:get-arg "-repl")
	(args:get-arg "-load"))
    (let* ((toppath (launch:setup))
	   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f)))
      (if dbstruct







>







1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
      (set! *didsomething* #t)))

;;======================================================================
;; Start a repl
;;======================================================================

;; fakeout readline
(include "readline-fix.scm")

(if (or (getenv "MT_RUNSCRIPT")
	(args:get-arg "-repl")
	(args:get-arg "-load"))
    (let* ((toppath (launch:setup))
	   (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f)))
      (if dbstruct
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
	      (set! *db* dbstruct)
	      (set! *client-non-blocking-mode* #t)
	      (import extras) ;; might not be needed
	      ;; (import csi)
	      (import readline)
	      (import apropos)
	      ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ...
	      (include "readline-fix.scm")

	      (if *use-new-readline*
		  (begin
		    (install-history-file (get-environment-variable "HOME") ".megatest_history") ;;  [homedir] [filename] [nlines])
		    (current-input-port (make-readline-port "megatest> ")))
		  (begin
		    (gnu-history-install-file-manager







<







1925
1926
1927
1928
1929
1930
1931

1932
1933
1934
1935
1936
1937
1938
	      (set! *db* dbstruct)
	      (set! *client-non-blocking-mode* #t)
	      (import extras) ;; might not be needed
	      ;; (import csi)
	      (import readline)
	      (import apropos)
	      ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ...


	      (if *use-new-readline*
		  (begin
		    (install-history-file (get-environment-variable "HOME") ".megatest_history") ;;  [homedir] [filename] [nlines])
		    (current-input-port (make-readline-port "megatest> ")))
		  (begin
		    (gnu-history-install-file-manager
1987
1988
1989
1990
1991
1992
1993







1994
1995
1996
1997
1998
1999
2000
(if (args:get-arg "-sync-to-megatest.db")
    (begin
      (db:multi-db-sync 
       #f ;; do all run-ids
       'new2old
       )
      (set! *didsomething* #t)))








;;======================================================================
;; Exit and clean up
;;======================================================================

(if *runremote* (close-all-connections!))








>
>
>
>
>
>
>







2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
(if (args:get-arg "-sync-to-megatest.db")
    (begin
      (db:multi-db-sync 
       #f ;; do all run-ids
       'new2old
       )
      (set! *didsomething* #t)))

(if (args:get-arg "-generate-html")
    (let* ((toppath (launch:setup)))
      (if (tests:create-html-tree #f)
          (debug:print-info 0 *default-log-port* "HTML output created in " toppath "/lt/runs-index.html")
          (debug:print 0 *default-log-port* "Failed to create HTML output in " toppath "/lt/runs-index.html"))
      (set! *didsomething* #t)))

;;======================================================================
;; Exit and clean up
;;======================================================================

(if *runremote* (close-all-connections!))

Deleted multi-dboard.scm version [604c83dc90].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use format numbers sql-de-lite srfi-1 posix regex regex-case srfi-69 nanomsg srfi-18 call-with-environment-variables)
(require-library iup)
(import (prefix iup iup:))
(use canvas-draw)

(declare (uses margs))
(declare (uses megatest-version))
(declare (uses gutils))
(declare (uses tree))
(declare (uses configf))
(declare (uses portlogger))
(declare (uses keys))
(declare (uses common))

(include "common_records.scm")
;; (include "db_records.scm")
;; (include "key_records.scm")

(define help (conc 
	      "Megatest Dashboard, documentation at http://www.kiatoa.com/fossils/megatest
  version " megatest-version "
  license GPL, Copyright (C) Matt Welland 2011

Usage: dashboard [options]
  -h                : this help
  -group groupname  : display this group of areas
  -test testid      : control test identified by testid
  -guimonitor       : control panel for runs

Misc
  -rows N         : set number of rows
"))

;; process args
(define remargs (args:get-args 
		 (argv)
		 (list  "-group" ;; display this group of areas
			"-debug"
			) 
		 (list  "-h"
			"-v"
			"-q"
			)
		 args:arg-hash
		 0))

(if (args:get-arg "-h")
    (begin
      (print help)
      (exit)))

;; (if (args:get-arg "-host")
;;     (begin
;;       (set! (common:get-remote remote) (string-split (args:get-arg "-host" ":")))
;;       (client:launch))
;;     (client:launch))

(define *runremote* #f)
(define *windows* (make-hash-table))
(define *changed-main* (make-hash-table)) ;; set path/... => #t
(define *changed-mutex* (make-mutex))     ;; use for all incoming change requests
(define *searchpatts*   (make-hash-table))

(debug:setup)

(define *tim* (iup:timer))
(define *ord* #f)

(iup:attribute-set! *tim* "TIME" 300)
(iup:attribute-set! *tim* "RUN" "YES")

(define (message-window msg)
  (iup:show
   (iup:dialog
    (iup:vbox 
     (iup:label msg #:margin "40x40")))))

(define (iuplistbox-fill-list lb items . default)
  (let ((i 1)
	(selected-item (if (null? default) #f (car default))))
    (iup:attribute-set! lb "VALUE" (if selected-item selected-item ""))
    (for-each (lambda (item)
		(iup:attribute-set! lb (number->string i) item)
		(if selected-item
		    (if (equal? selected-item item)
			(iup:attribute-set! lb "VALUE" item))) ;; (number->string i))))
		(set! i (+ i 1)))
	      items)
    i))

(define (pad-list l n)(append l (make-list (- n (length l)))))


(define (mkstr . x)
  (string-intersperse (map conc x) ","))

(define (update-search x val)
  (hash-table-set! *searchpatts* x val))


;;======================================================================
;; R E C O R D S
;;======================================================================

;; NOTE: Consider switching to defstruct.

;; data for an area (regression or testsuite)
;;
(define-record areadat
  name               ;; area name
  path               ;; mt run area home
  configdat          ;; megatest config
  denoise            ;; focal point for not putting out same messages over and over
  client-signature   ;; key for client-server conversation
  remote             ;; hash of all the client side connnections
  run-keys           ;; target keys for this area
  runs               ;; used in dashboard, hash of run-ids -> rundat
  read-only          ;; can I write to this area?
  monitordb          ;; db handle for monitor.db
  maindb             ;; db handle for main.db
  )

;; rundat, basic run data
;;
(define-record rundat
  id                 ;; the run-id
  target             ;; val1/val2 ... corrosponding to run-keys in areadat
  runname
  state              ;; state of the run, symbol 
  status             ;; status of the run, symbol
  event-time         ;; when the run was initiated
  tests              ;; hash of test-id -> testdat, QUESTION: separate by run-id?
  db                 ;; db handle
  )

;; testdat, basic test data
(define-record testdat
  run-id             ;; what run is this from
  id                 ;; test id
  testname           ;; test name
  itempath           ;; item path
  state              ;; test state, symbol
  status             ;; test status, symbol
  event-time         ;; when the test started
  duration           ;; how long the test took
  )

;; general data for the dboard application
;;
(define-record data
  cfgdat             ;; data from ~/.megatest/<group>.dat
  areas              ;; hash of areaname -> area-rec
  current-window-id  ;; 
  current-tab-id     ;; 
  update-needed      ;; flag to indicate that the tab pointed to by current tab id needs refreshing immediately
  tabs               ;; hash of tab-id -> areaname (??) should be of type "tab"
  )

;; all the components of an area display, all fits into a tab but
;; parts may be swapped in/out as needed
;;
(define-record tab
  tree
  matrix    ;; the spreadsheet 
  areadat   ;; the one-structure (one day dbstruct will be put in here)
  view-path ;; <target/path>/<runname>/...
  view-type ;; standard, etc.
  controls  ;; the controls
  data      ;; all the data kept in sync with db
  filters   ;; user filters, alist name -> filter record, eventually store these in ~/.megatest/<group>.dat?
  run-id    ;; the current run-id
  test-ids  ;; the current test id hash, run-id => test-id
  command   ;; the command from the entry field
  headers   ;; hash of header  -> colnum
  rows      ;; hash of rowname -> rownum
  )

(define-record filter
  target    ;; hash of widgets for the target
  runname   ;; the runname widget
  testpatt  ;; the testpatt widget
  )

;;======================================================================
;; D B
;;======================================================================

;; These are all using sql-de-lite and independent of area so cannot use stuff 
;; from db.scm

;; NB// run-id=#f => return dbdir only
;;
(define (areadb:dbfile-path areadat run-id)
  (let* ((cfgdat  (areadat-configdat areadat))
	 (dbdir   (or (configf:lookup cfgdat "setup" "dbdir")
		      (conc (configf:lookup cfgdat "setup" "linktree") "/.db")))
	 (fname   (if run-id
		      (case run-id
			((-1) "monitor.db")
			((0) "main.db")
			(else (conc run-id ".db")))
		      #f)))
    (handle-exceptions
     exn
     (begin
       (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir)
       (exit 1))
     (if (not (directory? dbdir))(create-directory dbdir #t)))
    (if fname
	(conc dbdir "/" fname)
	dbdir)))

;; -1 => monitor.db
;;  0 => main.db
;; >1 => <run-id>.db
;;
(define (areadb:open areadat run-id)
  (let* ((runs   (areadat-runs areadat))
	 (rundat (if (> run-id 0) ;; it is a run
		     (hash-table-ref/default runs run-id #f)
		     #f))
	 (db     (case run-id ;; if already opened, get the db and return it
		   ((-1) (areadat-monitordb areadat))
		   ((0)  (areadat-maindb    areadat))
		   (else (if rundat
			     (rundat-db rundat)
			     #f)))))
    (if db
	db ;; merely return the already opened db
	(let* ((dbfile (areadb:dbfile-path areadat run-id)) ;; not already opened, so open it
	       (db     (if (file-exists? dbfile)
			   (open-database dbfile)
			   (begin
			     (debug:print-error 0 *default-log-port* "I was asked to open " dbfile ", but file does not exist or is not readable.")
			     #f))))
	  (case run-id
	    ((-1)(areadat-monitordb-set! areadat db))
	    ((0) (areadat-maindb-set!    areadat db))
	    (else (rundat-db-set!        rundat  db)))
	  db))))

;; populate the areadat tests info, does NOT fill the tests data itself unless asked
;;
(define (areadb:populate-run-info areadat)
  (let* ((runs   (or (areadat-runs areadat) (make-hash-table)))
	 (keys   (areadat-run-keys areadat))
	 (maindb (areadb:open areadat 0)))
    (if maindb
	(query (for-each-row (lambda (row)
			       (let ((id  (list-ref row 0))
				     (dat (apply make-rundat (append row (list #f #f))))) ;; add placeholders for tests and db
				 (print row)
				 (hash-table-set! runs id dat))))
	       (sql maindb (conc "SELECT id,"
				 (string-intersperse keys "||'/'||")
				 ",runname,state,status,event_time FROM runs WHERE state != 'deleted';")))
	(debug:print-error 0 *default-log-port* "no main.db found at "  (areadb:dbfile-path areadat 0)))
    areadat))

;; given an areadat and target/runname patt fill up runs data
;;
;; ?????/

;; given a list of run-ids refresh/retrieve runs data into areadat
;;
(define (areadb:fill-tests areadat #!key (run-ids #f))
  (let* ((runs   (or (areadat-runs areadat) (make-hash-table))))
    (for-each
     (lambda (run-id)
       (let* ((rundat (hash-table-ref/default runs run-id #f))
	      (tests  (if (and rundat
			       (rundat-tests rundat)) ;; re-use existing hash table?
			  (rundat-tests rundat)
			  (let ((ht (make-hash-table)))
			    (rundat-tests-set! rundat ht)
			    ht)))
	      (rundb  (areadb:open areadat run-id)))
	 (query (for-each-row (lambda (row)
				(let* ((id         (list-ref row 0))
				       (testname   (list-ref row 1))
				       (itempath   (list-ref row 2))
				       (state      (list-ref row 3))
				       (status     (list-ref row 4))
				       (eventtim   (list-ref row 5))
				       (duration   (list-ref row 6)))
				  (hash-table-set! tests id
						   (make-testdat run-id id testname itempath state status eventtim duration)))))
		(sql rundb "SELECT id,testname,item_path,state,status,event_time,run_duration FROM tests WHERE state != 'DELETED';"))))
     (or run-ids (hash-table-keys runs)))
    areadat))
    

;; initialize and refresh data
;;		
(define (dboard:general-updater con port)
  (for-each
   (lambda (window-id)
     ;; (print "Processing for window-id " window-id)
     (let* ((window-dat     (hash-table-ref *windows* window-id))
	    (areas          (data-areas     window-dat))
	    ;; (keys           (areadat-run-keys area-dat))
	    (tabs           (data-tabs      window-dat))
	    (tab-ids        (hash-table-keys tabs))
	    (current-tab    (if (null? tab-ids)
				#f
				(hash-table-ref tabs (car tab-ids))))
	    (current-tree   (if (null? tab-ids) #f (tab-tree   current-tab)))
	    (current-node   (if (null? tab-ids) 0  (string->number (iup:attribute current-tree "VALUE"))))
	    (current-path   (if (eq? current-node 0)
				"Areas"
				(string-intersperse (tree:node->path current-tree current-node) "/")))
	    (current-matrix (if (null? tab-ids) #f (tab-matrix current-tab)))
	    (seen-nodes     (make-hash-table))
	    (path-changed   (if current-tab
				(equal? current-path (tab-view-path current-tab))
				#t)))
       ;; (debug:print-info 0 *default-log-port* "Current path: " current-path)
       ;; now for each area in the window gather the data
       (if path-changed
	   (begin
	     (debug:print-info 0 *default-log-port* "clearing matrix - path changed")
	     (dboard:clear-matrix current-tab)))
       (for-each
	(lambda (area-name)
	  ;; (print "Processing for area-name " area-name)
	  (let* ((area-dat  (hash-table-ref areas area-name))
		 (area-path (areadat-path   area-dat))
		 (runs      (areadat-runs   area-dat)))
	    (if (hash-table-ref/default *changed-main* area-path 'processed)
		(begin
		  (print "Processing " area-dat " for area-name " area-name)
		  (hash-table-set! *changed-main* area-path #f)
		  (areadb:populate-run-info area-dat)
		  (for-each 
		   (lambda (run-id)
		     (let* ((run     (hash-table-ref runs run-id))
			    (target  (rundat-target run))
			    (runname (rundat-runname run)))
		       (if current-tree
			   (let* ((partial-path (append (string-split target "/")(list runname)))
				  (full-path    (cons area-name partial-path)))
			     (if (not (hash-table-exists? seen-nodes full-path))
				 (begin
				   (print "INFO: Adding node " partial-path " to section " area-name)
				   (tree:add-node current-tree "Areas" full-path)
				   (areadb:fill-tests area-dat run-ids: (list run-id))))
				   (hash-table-set! seen-nodes full-path #t)))))
		   (hash-table-keys runs))))
	    (if (or (equal? "Areas" current-path)
		    (string-match (conc "^Areas/" area-name "(|\\/.*)$") current-path))
		(dboard:redraw-area area-name area-dat current-tab current-matrix current-path))))
	(hash-table-keys areas))))
   (hash-table-keys *windows*)))

;;======================================================================
;; D A S H B O A R D   D B 
;;======================================================================

;; All moved to common.scm		

;;======================================================================
;; T R E E 
;;======================================================================

;; <area> - <target - ... > - <runname> - <test> - <itempath - ...>

(define (dashboard:tree-browser data adat window-id)
  ;; (iup:split
  (let* ((tb      (iup:treebox
		   #:value 0
		   #:title "Areas"
		   #:expand "YES"
		   #:addexpanded "NO"
		   #:selection-cb
		   (lambda (obj id state)
		     ;; (print "obj: " obj ", id: " id ", state: " state)
		     (let* ((tree-path (tree:node->path obj id))
			    (area      (car tree-path))
			    (areadat-path (cdr tree-path)))
		       #f
		       ;; (test-id  (tree-path->test-id (cdr run-path))))
		       ;; (if test-id
		       ;;    (hash-table-set! (dboard:data-curr-test-ids *data*)
		       ;;		     window-id test-id))
		       ;; (print "path: " (tree:node->path obj id) " test-id: " test-id))))))
		       )))))
    ;; (iup:attribute-set! tb "VALUE" "0")
    ;; (iup:attribute-set! tb "NAME" "Runs")
    ;; (iup:attribute-set! tb "ADDEXPANDED" "NO")
    ;; (dboard:data-tests-tree-set! *data* tb)
    tb))

;;======================================================================
;; M A I N   M A T R I X
;;======================================================================

;; General displayer
;;
(define (dashboard:main-matrix data adat window-id)
  (let* (;; (tab-dat         (areadat-
	 (view-matrix     (iup:matrix
			   ;; (runs-for-targ (db:get-runs-by-patt *dbstruct-local* *keys* "%" target #f #f #f))
			   #:expand "YES"
			   ;; #:fittosize "YES"
			   #:resizematrix "YES"
			   #:scrollbar "YES"
			   #:numcol 100
			   #:numlin 100
			   #:numcol-visible 3
			   #:numlin-visible 20
			   #:click-cb (lambda (obj lin col status)
					(print "obj: " obj " lin: " lin " col: " col " status: " status " value: " (iup:attribute obj "VALUE"))))))
    
    ;; (iup:attribute-set! view-matrix "RESIZEMATRIX" "YES")
    (iup:attribute-set! view-matrix "WIDTH0" "100")
    ;; (dboard:data-runs-matrix-set! *data* runs-matrix)
    ;; (iup:hbox
    ;;  (iup:frame 
    ;;   #:title "Runs browser"
    ;;   (iup:vbox
    view-matrix))

;;======================================================================
;; A R E A S
;;======================================================================

(define (dashboard:init-area data area-name apath)
  (let* ((mtconf      (dboard:read-mtconf apath))
	 (area-dat    (let ((ad (make-areadat
				 area-name ;; area name
				 apath     ;; path to area
				 ;; 'http     ;; transport
				 mtconf    ;; megatest.config
				 (make-hash-table) ;; denoise hash
				 #f        ;; client-signature
				 #f        ;; remote connections
				 (keys:config-get-fields mtconf) ;; run keys
				 (make-hash-table) ;; run-id -> (hash of test-ids => dat)
				 (and (file-exists? apath)(file-write-access? apath)) ;; read-only
				 #f
				 #f
				 )))
			(hash-table-set! (data-areas data) area-name ad)
			ad)))
    area-dat))

;; given the keys for an area and a path from the tree browser
;; return the level: areas area runs run tests test
;;
(define (dboard:get-view-type keys current-path)
  (let* ((path-parts (string-split current-path "/"))
	 (path-len   (length path-parts)))
    (cond
     ((equal? current-path "Areas")     'areas)
     ((eq? path-len 2)                  'area)
     ((<= (+ (length keys) 2) path-len) 'runs)
     (else                              'run))))

(define (dboard:clear-matrix tab)
  (if tab
      (begin
	(iup:attribute-set! (tab-matrix tab) "CLEARVALUE" "ALL")
	(tab-headers-set! tab (make-hash-table))
	(tab-rows-set!    tab (make-hash-table)))))

;; full redraw of a given area
;;
(define (dboard:redraw-area area-name area-dat tab-dat current-matrix current-path)
  (let* ((keys      (areadat-run-keys area-dat))
	 (runs      (areadat-runs     area-dat))
	 (headers   (tab-headers   tab-dat))
	 (rows      (tab-rows      tab-dat))
	 (used-cols (hash-table-values headers))
	 (used-rows (hash-table-values rows))
	 (touched   (make-hash-table)) ;; (vector row col) ==> true, touched cell
	 (view-type (dboard:get-view-type keys current-path))
	 (changed   #f)
	 (state-statuses  (list "PASS" "FAIL" "WARN" "CHECK" "SKIP" "RUNNING" "LAUNCHED")))
    ;; (debug:print 0 *default-log-port* "current-matrix=" current-matrix)
    (case view-type
      ((areas) ;; find row for this area, if not found, create new entry
       (let* ((curr-rownum (hash-table-ref/default rows area-name #f))
	      (next-rownum (+ (apply max (cons 0 used-rows)) 1))
	      (rownum      (or curr-rownum next-rownum))
	      (coord       (conc rownum ":0")))
	 (if (not curr-rownum)(hash-table-set! rows area-name rownum))
	 (if (not (equal? (iup:attribute current-matrix coord) area-name))
	     (begin
	       (let loop ((hed  (car state-statuses))
			  (tal  (cdr state-statuses))
			  (count 1))
		 (if (not (equal? (iup:attribute current-matrix (conc "0:" count)) hed))
		     (iup:attribute-set! current-matrix (conc "0:" count) hed))
		 (iup:attribute-set! current-matrix (conc rownum ":" count) "0")
		 (if (not (null? tal))
		     (loop (car tal)(cdr tal)(+ count 1))))
	       (debug:print-info 0 *default-log-port* "view-type=" view-type ", rownum=" rownum ", curr-rownum=" curr-rownum ", next-rownum=" next-rownum ", coord=" coord ", area-name=" area-name)
	       (iup:attribute-set! current-matrix coord area-name)
	       (set! changed #t))))))
    (if changed (iup:attribute-set! current-matrix "REDRAW" "ALL"))))
	     

       
   ;; (dboard:clear-matrix current-matrix used-cols used-rows touched) ;; clear all
    
	
  
;;======================================================================
;; D A S H B O A R D
;;======================================================================

(define (dashboard:area-panel aname data window-id)
  (let* ((apath      (configf:lookup (data-cfgdat data) aname "path")) ;;  (hash-table-ref (dboard:data-cfgdat data) area-name))
	 ;;          (hash-table-ref (dboard:data-cfgdat data) aname))
	 (area-dat   (dashboard:init-area data aname apath))
	 (tb         (dashboard:tree-browser data area-dat window-id)) ;; (dboard:areas-tree-browser data)
	 (ad         (dashboard:main-matrix  data area-dat window-id))
	 (areas      (data-areas data))
	 (dboard-dat (make-tab
		      #f           ;; tree
		      #f           ;; matrix
		      area-dat     ;;
		      #f           ;; view path
		      'default     ;; view type
		      #f           ;; controls
		      (make-hash-table) ;; cached data? not sure how to use this yet :)
		      #f           ;; filters
		      #f           ;; the run-id
		      (make-hash-table) ;; run-id -> test-id, for current test id
		      ""
		      (make-hash-table) ;; headername -> colnum
		      (make-hash-table) ;; rowname    -> rownum
		      )))
    (hash-table-set! (data-areas data) aname area-dat) ;; dboard-dat)
    (hash-table-set! (data-tabs data) window-id dboard-dat)
    (tab-tree-set!   dboard-dat tb)
    (tab-matrix-set! dboard-dat ad)
    (iup:split
     #:value 200
     tb ad)))


;; Main Panel
;;
(define (dashboard:main-panel data window-id)
  (iup:dialog
   #:title "Megatest Control Panel"
;;   #:menu (dcommon:main-menu data)
   #:shrink "YES"
   (iup:vbox
    (let* ((area-names  (hash-table-keys (data-cfgdat data)))
	   (area-panels (map (lambda (aname)
			       (dashboard:area-panel aname data window-id))
			     area-names))
	   (tabtop      (apply iup:tabs  
			       #:tabchangepos-cb (lambda (obj curr prev)
						   (data-current-tab-id-set! data curr)
						   (data-update-needed-set!  data #t)
						   (print "Tab is: " curr ", prev was " prev))
			       area-panels))
	   (tabs     (data-tabs data)))
      (if (not (null? area-names))
	  (let loop ((index 0)
		     (hed   (car area-names))
		     (tal   (cdr area-names)))
	    ;; (hash-table-set! tabs index hed)
	    (debug:print 0 *default-log-port* "Adding area " hed " with index " index " to dashboard")
	    (iup:attribute-set! tabtop (conc "TABTITLE" index) hed)
	    (if (not (null? tal))
		(loop (+ index 1)(car tal)(cdr tal)))))
      tabtop))))


;;======================================================================
;; N A N O M S G   S E R V E R
;;======================================================================

(define (dboard:server-service soc port)
  (print "server starting")
  (let loop ((msg-in (nn-recv soc))
	     (count  0))
    (if (eq? 0 (modulo count 1000))
	(print "server received: " msg-in ", count=" count))
    (cond
     ;;
     ;; quit
     ;;
     ((equal? msg-in "quit")
      (nn-send soc "Ok, quitting"))
     ;;
     ;; ping
     ;;
     ((and (>= (string-length msg-in) 4)
	   (equal? (substring msg-in 0 4) "ping"))
      (nn-send soc (conc (current-process-id)))
      (loop (nn-recv soc)(+ count 1)))
     ;;
     ;; main changed
     ;;
     ((and (>= (string-length msg-in) 4)
	   (equal? (substring msg-in 0 4) "main"))
      (let ((parts (string-split msg-in " ")))
	(hash-table-set! *changed-main* (cadr parts) #t)
	(nn-send soc "got it!")))
     ;;
     ;; ??
     ;;
     (else
      (nn-send soc "hello " msg-in " you got to the else clause!")))
    (loop (nn-recv soc)(if (> count 20000000)
			   0
			   (+ count 1)))))

(define (dboard:one-time-ping-receive soc port)
  (let ((msg-in (nn-recv soc)))
    (if (and (>= (string-length msg-in) 4)
	     (equal? (substring msg-in 0 4) "ping"))
	(nn-send soc (conc (current-process-id))))))

(define (dboard:server-start given-port #!key (num-tries 200))
  (let* ((rep (nn-socket 'rep))
	 (port (or given-port  (portlogger:main "find")))
	 (con (conc "tcp://*:" port)))
    ;; register this connect here ....
    (nn-bind rep con)
    (thread-start! 
     (make-thread (lambda ()
		    (dboard:one-time-ping-receive rep port))
		  "one time receive thread"))
    (if (dboard:ping-self "localhost" port)
	(begin
	  (print "INFO: dashboard nanomsg server started on " port)
	  (values rep port))
	(begin
	  (print "WARNING: couldn't create server on port " port)
	  (portlogger:main "set" "failed")
	  (if (> num-tries 0)
	      (dboard:server-start #f (- num-tries 1))
	      (begin
		(print "ERROR: failed to start nanomsg server")
		(values #f #f)))))))

(define (dboard:server-close con port)
  (nn-close con)
  (portlogger:main "set" port "released"))

(define (dboard:ping-self host port #!key (return-socket #t))
  ;; send a random number along with pid and check that we get it back
  (let* ((req     (nn-socket 'req))
	 (key     "ping")
	 (success #f)
	 (keepwaiting #t)
	 (ping    (make-thread
		   (lambda ()
		     (print "ping: sending string \"" key "\", expecting " (current-process-id))
		     (nn-send req key)
		     (let ((result  (nn-recv req)))
		       (if (equal? (conc (current-process-id)) result)
			   (begin
			     (print "ping, success: received \"" result "\"")
			     (set! success #t))
			   (begin
			     (print "ping, failed: received key \"" result "\"")
			     (set! keepwaiting #f)
			     (set! success #f)))))
		   "ping"))
	 (timeout (make-thread (lambda ()
				 (let loop ((count 0))
				   (thread-sleep! 1)
				   (print "still waiting after " count " seconds...")
				   (if (and keepwaiting (< count 10))
				       (loop (+ count 1))))
				 (if keepwaiting
				     (begin
				       (print "timeout waiting for ping")
				       (thread-terminate! ping))))
			       "timeout")))
    (nn-connect req (conc "tcp://" host ":" port))
    (handle-exceptions
     exn
     (begin
       (print-call-chain)
       (print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
       (print "exn=" (condition->list exn))
       (print "ping failed to connect to " host ":" port))
     (thread-start! timeout)
     (thread-start! ping)
     (thread-join! ping)
     (if success (thread-terminate! timeout)))
    (if return-socket
	(if success req #f)
	(begin
	  (nn-close req)
	  success))))

;;======================================================================
;; C O N F I G U R A T I O N 
;;======================================================================

;; Get the configuration file for a group name, if the group name is "default" and it doesn't 
;; exist, create it and add the current path if it contains megatest.config
;;
(define (dboard:get-config group-name)
  (let* ((fname (conc (getenv "HOME") "/.megatest/" group-name ".dat")))
    (if (file-exists? fname)
	(read-config fname (make-hash-table) #t)
	(if (dboard:create-config fname)
	    (dboard:get-config group-name)
	    (make-hash-table)))))

(define (dboard:create-config fname)
  ;; (handle-exceptions
  ;;  exn
  ;;  
  ;;  #f ;; failed to create - just give up
   (let* ((dirname       (pathname-directory fname))
	  (file-name     (pathname-strip-directory fname))
	  (curr-mtcfgdat (find-config "megatest.config"
				      toppath: (or (get-environment-variable "MT_RUN_AREA_HOME")(current-directory))))
	  (curr-mtcfg    (if (and curr-mtcfgdat (not (null? curr-mtcfgdat)))(cadr curr-mtcfgdat) #f))
	  (curr-mtpath   (if curr-mtcfg (car curr-mtcfgdat) #f)))
     (if curr-mtpath
	 (begin
	   (debug:print-info 0 *default-log-port* "Creating config file " fname)
	   (if (not (file-exists? dirname))
	       (create-directory dirname #t))
	   (with-output-to-file fname
	     (lambda ()
	       (let ((aname (pathname-strip-directory curr-mtpath)))
		 (print "[" aname "]")
		 (print  "path " curr-mtpath))))
	   #t)
	 (begin
	   (debug:print-info 0 *default-log-port* "Need to create a config but no megatest.config found: " curr-mtcfgdat)
	   #f))))
;; )

(define (dboard:read-mtconf apath)
  (let* ((mtconffile  (conc apath "/megatest.config")))
    (call-with-environment-variables
     (list (cons "MT_RUN_AREA_HOME" apath))
     (lambda ()
       (read-config mtconffile (make-hash-table) #f)) ;; megatest.config
     )))
	 

;;======================================================================
;; G U I   S T U F F 
;;======================================================================

;;; main. Theoretically could have multiple windows (each with a group of tags, thus window-id
;;;
(define (dboard:make-window window-id)
  (let* (;; (window-id 0)
	 (groupn    (or (args:get-arg "-group") "default"))
	 (cfgdat    (dboard:get-config groupn))
	 ;; (cfgdat    (if (file-exists? cfname)(read-config cfname (make-hash-table) #t)(make-hash-table)))
	 (data      (make-data
		     cfgdat ;; this is the data from ~/.megatest for the selected group
		     (make-hash-table) ;; areaname -> area-rec
		     0                 ;; current window id
		     0                 ;; current tab id
		     #f                ;; redraw needed for current tab id
		     (make-hash-table) ;; tab-id -> areaname
		     )))
    (hash-table-set! *windows* window-id data)
    (iup:show (dashboard:main-panel data window-id))
    (iup:main-loop)))

;; ease debugging by loading ~/.dashboardrc
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc")))
  (if (file-exists? debugcontrolf)
      (load debugcontrolf)))

(define (main)
  (let-values 
      (((con port)(dboard:server-start #f)))
    (let ((portnum   (if (string? port)(string->number port) port)))
      ;; got here, monitor/dashboard was started
      (mddb:register-dashboard portnum)
      (thread-start! (make-thread (lambda ()(dboard:server-service con portnum)) "server service"))
      (thread-start! (make-thread (lambda ()
				    (let loop ()
				      (dboard:general-updater con portnum)
				      (thread-sleep! 1)
				      (loop))) "general updater"))
      (dboard:make-window 0)
      (mddb:unregister-dashboard (get-host-name) portnum)
      (dboard:server-close con port))))

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































Deleted nmsg-transport.scm version [b30844cb1a].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358

;; Copyright 2006-2012, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

(require-extension (srfi 18) extras tcp s11n)

(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
(import (prefix sqlite3 sqlite3:))

;; (use nanomsg)

(declare (unit nmsg-transport))

(declare (uses common))
(declare (uses db))
(declare (uses tests))
(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
(declare (uses server))

(include "common_records.scm")
(include "db_records.scm")

;; Transition to pub --> sub with pull <-- push
;;
;;   1. client sends request to server via push to the pull port
;;   2. server puts request in queue or processes immediately as appropriate
;;   3. server puts responses from completed requests into pub port 
;;
;; TODO
;;
;; Done Tested
;; [x]  [ ]    1. Add columns pullport pubport to servers table
;; [x]  [ ]    2. Add rm of monitor.db if older than 11/12/2012 
;; [x]  [ ]    3. Add create of pullport and pubport with finding of available ports
;; [x]  [ ]    4. Add client compose of request
;; [x]  [ ]        - name of client: testname/itempath-test_id-hostname 
;; [x]  [ ]        - name of request: callname, params
;; [x]  [ ]        - request key: f(clientname, callname, params)
;; [x]  [ ]    5. Add processing of subscription hits
;; [x]  [ ]        - done when get key 
;; [x]  [ ]        - return results
;; [x]  [ ]    6. Add timeout processing
;; [x]  [ ]        - after 60 seconds
;; [ ]  [ ]            i. check server alive, connect to new if necessary
;; [ ]  [ ]           ii. resend request
;; [ ]  [ ]    7. Turn self ping back on

(define (nmsg-transport:make-server-url hostport #!key (bindall #f))
  (if (not hostport)
      #f
      (conc "tcp://" (if bindall "*" (car hostport)) ":" (cadr hostport))))

(define *server-loop-heart-beat* (current-seconds))
(define *heartbeat-mutex* (make-mutex))

;;======================================================================
;; S E R V E R
;;======================================================================

(define (nmsg-transport:run dbstruct hostn run-id server-id #!key (retrynum 1000))
  (debug:print 2 *default-log-port* "Attempting to start the server ...")
  (let* ((start-port      (portlogger:open-run-close portlogger:find-port))
	 (server-thread   (make-thread (lambda ()
					 (nmsg-transport:try-start-server dbstruct run-id start-port server-id))
				       "server thread"))
	 (tdbdat          (tasks:open-db)))
    (thread-start! server-thread)
    (thread-sleep! 0.1)
    (if (nmsg-transport:ping hostn start-port timeout: 2 expected-key: (current-process-id))
	(let ((interface (if (equal? hostn "-")(get-host-name) hostn)))
	  (tasks:server-set-interface-port (db:delay-if-busy tdbdat) server-id interface start-port)
	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep")
	  (set! *server-info* (list hostn start-port)) ;; probably not needed anymore? currently used by keep-running
	  (thread-sleep! 3) ;; give some margin for queries to complete before switching from file based access to server based access
	  ;; (set! *inmemdb*  dbstruct)
	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running")
	  (thread-start! (make-thread
			  (lambda ()(nmsg-transport:keep-running server-id run-id))
			  "keep running"))
	  (thread-join! server-thread))
	(if (> retrynum 0)
	    (begin
	      (debug:print 0 *default-log-port* "WARNING: Failed to connect to server (self) on host " hostn ":" start-port ", trying again.")
	      (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id "failed to start, never received server alive signature")
	      (portlogger:open-run-close portlogger:set-failed start-port)
	      (nmsg-transport:run dbstruct hostn run-id server-id))
	    (begin
	      (debug:print-error 0 *default-log-port* "could not find an open port to start server on. Giving up")
	      (exit 1))))))

(define (nmsg-transport:try-start-server dbstruct run-id portnum server-id)
  (let ((repsoc (nn-socket 'rep)))
    (nn-bind repsoc (conc "tcp://*:" portnum))
    (let loop ((msg-in (nn-recv repsoc)))
      (let* ((dat    (db:string->obj msg-in transport: 'nmsg)))
	(debug:print 0 *default-log-port* "server, received: " dat)
	(let ((result (api:execute-requests dbstruct dat)))
	  (debug:print 0 *default-log-port* "server, sending: " result)
	  (nn-send repsoc (db:obj->string result  transport: 'nmsg)))
	(loop (nn-recv repsoc))))))

;; all routes though here end in exit ...
;;
(define (nmsg-transport:launch run-id)
  (let* ((tdbdat   (tasks:open-db))
	 (dbstruct (db:setup run-id))
	 (hostn    (or (args:get-arg "-server") "-")))
    (set! *run-id*   run-id)
    (set! *inmemdb* dbstruct)
    ;; with nbfake daemonize isn't really needed
    ;;
    ;; (if (args:get-arg "-daemonize")
    ;;     (begin
    ;;       (daemon:ize)
    ;;       (if *alt-log-file* ;; we should re-connect to this port, I think daemon:ize disrupts it
    ;;           (begin
    ;;     	(current-error-port *alt-log-file*)
    ;;     	(current-output-port *alt-log-file*)))))
    (if (server:check-if-running run-id)
	(begin
	  (debug:print-info 0 *default-log-port* "Server for run-id " run-id " already running")
	  (exit 0)))
    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
	       (remtries  4))
      (if (not server-id)
	  (if (> remtries 0)
	      (begin
		(thread-sleep! 2)
		(if (not (server:check-if-running run-id))
		    (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
			  (- remtries 1))
		    (begin
		      (debug:print-info 0 *default-log-port* "Another server took the slot, exiting")
		      (exit 0))))
	      (begin
		;; since we didn't get the server lock we are going to clean up and bail out
		(debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
		(tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " http-transport:launch")
		))
	  ;; locked in a server id, try to start up
	  (nmsg-transport:run dbstruct hostn run-id server-id))
      (set! *didsomething* #t)
      (exit))))

;;======================================================================
;; S E R V E R   U T I L I T I E S 
;;======================================================================

(define (nmsg-transport:mk-signature)
  (message-digest-string (md5-primitive) 
			 (with-output-to-string
			   (lambda ()
			     (write (list (current-directory)
					  (argv)))))))

;;======================================================================
;; C L I E N T S
;;======================================================================

;; ping the server at host:port
;;   return the open socket if successful (return-socket == #t)
;;   expect the key expected-key returned in payload
;;   send our-key or #f as payload
;;
(define (nmsg-transport:ping hostn port #!key (timeout 3)(return-socket #t)(expected-key #f)(our-key #f)(socket #f))
  ;; send a random number along with pid and check that we get it back
  (let* ((host    (if (or (not hostn)
			  (equal? hostn "-")) ;; use localhost
		      (get-host-name)
		      hostn))
	 (req     (or socket
		      (let ((soc (nn-socket 'req)))
			(nn-connect soc (conc "tcp://" host ":" port))
			soc)))
	 (success #t)
	 (dat     (vector "ping" our-key))
	 (result  (condition-case 
		   (nmsg-transport:client-api-send-receive-raw req dat timeout: timeout)
		   ((timeout)(set! success #f) #f)))
	 (key     (if success 
		      (vector-ref result 1)
		      #f)))
    (debug:print 0 *default-log-port* "success=" success ", key=" key ", expected-key=" expected-key ", equal? " (equal? key expected-key))
    (if (and success
	     (or (not expected-key) ;; just getting a reply is good enough then
		 (equal? key expected-key)))
	(if return-socket
	    req
	    (begin
	      (if (not socket)(nn-close req)) ;; don't want a side effect of closing socket if handed it
	      #t))
	(begin
	  (if (not socket)(nn-close req)) ;; failed to ping, close socket as side effect
	  #f))))

;; send data to server, wait max of timeout seconds for a response.
;; return #( success/fail result )
;;
;; for effiency it is easier to do the obj->string and string->obj here.
;;
(define (nmsg-transport:client-api-send-receive-raw socreq indat #!key (enable-send #t)(timeout 25))
  (let* ((success     #f)
	 (result      #f)
	 (keepwaiting #t)
	 (dat         (db:obj->string indat transport: 'nmsg))
	 (send-recv   (make-thread
		       (lambda ()
			 (nn-send socreq dat)
			 (let* ((res (nn-recv socreq)))
			   (set! success #t)
			   (set! result (db:string->obj res transport: 'nmsg))))
		       "send-recv"))
	 (timeout     (make-thread
		       (lambda ()
			 (let loop ((count 0))
			   (thread-sleep! 1)
			   (debug:print-info 1 *default-log-port* "send-receive-raw, still waiting after " count " seconds...")
			   (if (and keepwaiting (< count timeout)) ;; yes, this is very aproximate
			       (loop (+ count 1))))
			 (if keepwaiting
			     (begin
			       (print "timeout waiting for ping")
			       (thread-terminate! send-recv))))
		       "timeout")))
    ;; replace with condition-case?
    (handle-exceptions
     exn
     (set! result "timeout")
     (thread-start! timeout)
     (thread-start! send-recv)
     (thread-join! send-recv)
     (if success (thread-terminate! timeout)))
    ;; raise timeout error if timed out
    (if success
	(if (and (vector? result)
		 (vector-ref result 0)) ;; did it fail at the server?
	    result                ;; nope, all good
	    (begin
	      (debug:print-error 0 *default-log-port* "error occured at server, info=" (vector-ref result 2))
	      (debug:print 0 *default-log-port* " client call chain:")
	      (print-call-chain (current-error-port))
	      (debug:print 0 *default-log-port* " server call chain:")
	      (pp (vector-ref result 1) (current-error-port))
	      (signal (vector-ref result 0))))
	(signal (make-composite-condition
		 (make-property-condition 'timeout 'message "nmsg-transport:client-api-send-receive-raw timed out talking to server"))))))

;; run nmsg-transport:keep-running in a parallel thread to monitor that the db is being 
;; used and to shutdown after sometime if it is not.
;;
(define (nmsg-transport:keep-running server-id run-id)
  ;; if none running or if > 20 seconds since 
  ;; server last used then start shutdown
  ;; This thread waits for the server to come alive
  (let* ((server-info (let loop ()
                        (let ((sdat #f))
                          (mutex-lock! *heartbeat-mutex*)
                          (set! sdat *server-info*)
                          (mutex-unlock! *heartbeat-mutex*)
                          (if sdat 
			      (begin
				(debug:print-info 0 *default-log-port* "keep-running got sdat=" sdat)
				sdat)
                              (begin
                                (thread-sleep! 0.5)
                                (loop))))))
         (iface       (car server-info))
         (port        (cadr server-info))
         (last-access 0)
	 (tdbdat      (tasks:open-db))
	 (server-timeout (let ((tmo (configf:lookup  *configdat* "server" "timeout")))
			   (if (and (string? tmo)
				    (string->number tmo))
			       (* 60 60 (string->number tmo))
			       ;; (* 3 24 60 60) ;; default to three days
			       (* 60 1)         ;; default to one minute
			       ;; (* 60 60 25)      ;; default to 25 hours
			       ))))
    (print "Keep-running got server pid " server-id ", using iface " iface " and port " port)
    (let loop ((count 0))
      (thread-sleep! 4) ;; no need to do this very often
      ;; NB// sync currently does NOT return queue-length
      (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1)))
      ;; (print "Server running, count is " count)
        (if (< count 1) ;; 3x3 = 9 secs aprox
            (loop (+ count 1)))
        
        (mutex-lock! *heartbeat-mutex*)
        (set! last-access *last-db-access*)
        (mutex-unlock! *heartbeat-mutex*)
	(db:sync-touched *inmemdb* run-id force-sync: #t)
        (if (and *server-run*
	       (> (+ last-access server-timeout)
		  (current-seconds)))
            (begin
              (debug:print-info 0 *default-log-port* "Server continuing, seconds since last db access: " (- (current-seconds) last-access))
              (loop 0))
            (begin
              (debug:print-info 0 *default-log-port* "Starting to shutdown the server.")
              (set! *time-to-exit* #t)
	      (db:sync-touched *inmemdb* run-id force-sync: #t)
              (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running")
              (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting")
              (exit)
	      ))))))

;;======================================================================
;; C L I E N T S
;;======================================================================

(define (nmsg-transport:client-connect iface portnum)
  (let* ((reqsoc      (nmsg-transport:ping iface portnum return-socket: #t)))
    (vector iface portnum #f #f #f (current-seconds) reqsoc)))

;; returns result, there is no sucess/fail flag - handled via excpections
;;
(define (nmsg-transport:client-api-send-receive run-id connection-info cmd param #!key (remtries 5))
  ;; NB// In the html version of this routine there is a call to 
  ;;      tasks:kill-server-run-id when there is an exception
  (mutex-lock! *http-mutex*)
  (let* ((packet  (vector cmd param))
	 (reqsoc  (http-transport:server-dat-get-socket connection-info))
	 (res     (nmsg-transport:client-api-send-receive-raw reqsoc packet)))
;;	 (status  (vector-ref rawres 0))
;;	 (result  (vector-ref rawres 1)))
    (mutex-unlock! *http-mutex*)
    res)) ;; (vector status (if status (db:string->obj result transport: 'nmsg) result))))
	
;;======================================================================
;; J U N K 
;;======================================================================

;; DO NOT USE
;;
(define (nmsg-transport:client-signal-handler signum)
  (handle-exceptions
   exn
   (debug:print 0 *default-log-port* " ... exiting ...")
   (let ((th1 (make-thread (lambda ()
			     (if (not *received-response*)
				 (receive-message* *runremote*))) ;; flush out last call if applicable
			   "eat response"))
	 (th2 (make-thread (lambda ()
			     (debug:print-error 0 *default-log-port* "Received ^C, attempting clean exit. Please be patient and wait a few seconds before hitting ^C again.")
			     (thread-sleep! 3) ;; give the flush three seconds to do it's stuff
			     (debug:print 0 *default-log-port* "       Done.")
			     (exit 4))
			   "exit on ^C timer")))
     (thread-start! th2)
     (thread-start! th1)
     (thread-join! th2))))

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<












































































































































































































































































































































































































































































































































































































































































































































Modified rmt.scm from [a85b27b9ab] to [51e718f694].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use json format)

(declare (unit rmt))
(declare (uses api))
(declare (uses tdb))
(declare (uses http-transport))
(declare (uses nmsg-transport))

;;
;; THESE ARE ALL CALLED ON THE CLIENT SIDE!!!
;;

;; ;; For debugging add the following to ~/.megatestrc
;;
;; (require-library trace)











|





|
<







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

19
20
21
22
23
24
25
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.
;;======================================================================

(use json format) ;; RADT => purpose of json format??

(declare (unit rmt))
(declare (uses api))
(declare (uses tdb))
(declare (uses http-transport))
;;(declare (uses nmsg-transport))

;;
;; THESE ARE ALL CALLED ON THE CLIENT SIDE!!!
;;

;; ;; For debugging add the following to ~/.megatestrc
;;
;; (require-library trace)
69
70
71
72
73
74
75



76
77
78
79
80
81
82
83
84
85
86

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

117
118
119
120
121
122
123
124
	;; NB// can cache the answer for server running for 10 seconds ...
	;;  ;; (and (not (rmt:write-frequency-over-limit? cmd run-id))
	(if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)
	    (client:setup run-id)
	    #f))))

(define *send-receive-mutex* (make-mutex)) ;; should have separate mutex per run-id



(define (rmt:send-receive cmd rid params #!key (attemptnum 1)) ;; start attemptnum at 1 so the modulo below works as expected
  ;; clean out old connections
  ;; (mutex-lock! *db-multi-sync-mutex*)
  (let ((expire-time (- (current-seconds) (server:get-timeout) 10))) ;; don't forget the 10 second margin
    (for-each 
     (lambda (run-id)
       (let ((connection (hash-table-ref/default *runremote* run-id #f)))
         (if (and (vector? connection)
        	  (< (http-transport:server-dat-get-last-access connection) expire-time))
             (begin
               (debug:print-info 0 *default-log-port* "Discarding connection to server for run-id " run-id ", too long between accesses")

               ;; SHOULD CLOSE THE CONNECTION HERE
	       (case *transport-type*
		 ((nmsg)(nn-close (http-transport:server-dat-get-socket 
				   (hash-table-ref *runremote* run-id)))))
               (hash-table-delete! *runremote* run-id)))))
     (hash-table-keys *runremote*)))
  ;; (mutex-unlock! *db-multi-sync-mutex*)
  ;; (mutex-lock! *send-receive-mutex*)
  (let* ((run-id          (if rid rid 0))
	 (connection-info (rmt:get-connection-info run-id)))
    ;; the nmsg method does the encoding under the hood (the http method should be changed to do this also)
    (if connection-info
	;; use the server if have connection info
	(let* ((dat     (case *transport-type*
			  ((http)(condition-case
				  (http-transport:client-api-send-receive run-id connection-info cmd params)
				  ((commfail)(vector #f "communications fail"))
				  ((exn)(vector #f "other fail"))))
			  ((nmsg)(condition-case
				  (nmsg-transport:client-api-send-receive run-id connection-info cmd params)
				  ((timeout)(vector #f "timeout talking to server"))))
			  (else  (exit))))
	       (success (if (vector? dat) (vector-ref dat 0) #f))
	       (res     (if (vector? dat) (vector-ref dat 1) #f)))
	  (if (vector? connection-info)(http-transport:server-dat-update-last-access connection-info))
	  (if success
	      (begin
		;; (mutex-unlock! *send-receive-mutex*)
		(case *transport-type* 
		  ((http) res) ;; (db:string->obj res))

		  ((nmsg) res))) ;; (vector-ref res 1)))
	      (begin ;; let ((new-connection-info (client:setup run-id)))
		(debug:print 0 *default-log-port* "WARNING: Communication failed, trying call to rmt:send-receive again.")
		;; (case *transport-type*
		;;   ((nmsg)(nn-close (http-transport:server-dat-get-socket connection-info))))
		(hash-table-delete! *runremote* run-id) ;; don't keep using the same connection
		;; NOTE: killing server causes this process to block forever. No idea why. Dec 2. 
		;; (if (eq? (modulo attemptnum 5) 0)







>
>
>











>
|
|
|
|














|
|
|









>
|







68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
	;; NB// can cache the answer for server running for 10 seconds ...
	;;  ;; (and (not (rmt:write-frequency-over-limit? cmd run-id))
	(if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)
	    (client:setup run-id)
	    #f))))

(define *send-receive-mutex* (make-mutex)) ;; should have separate mutex per run-id

;; RA => e.g. usage (rmt:send-receive 'get-var #f (list varname))
;;
(define (rmt:send-receive cmd rid params #!key (attemptnum 1)) ;; start attemptnum at 1 so the modulo below works as expected
  ;; clean out old connections
  ;; (mutex-lock! *db-multi-sync-mutex*)
  (let ((expire-time (- (current-seconds) (server:get-timeout) 10))) ;; don't forget the 10 second margin
    (for-each 
     (lambda (run-id)
       (let ((connection (hash-table-ref/default *runremote* run-id #f)))
         (if (and (vector? connection)
        	  (< (http-transport:server-dat-get-last-access connection) expire-time))
             (begin
               (debug:print-info 0 *default-log-port* "Discarding connection to server for run-id " run-id ", too long between accesses")
               ;; bb- disabling nanomsg
               ;; SHOULD CLOSE THE CONNECTION HERE 
	       ;; (case *transport-type*
	       ;;   ((nmsg)(nn-close (http-transport:server-dat-get-socket 
	       ;;  		   (hash-table-ref *runremote* run-id)))))
               (hash-table-delete! *runremote* run-id)))))
     (hash-table-keys *runremote*)))
  ;; (mutex-unlock! *db-multi-sync-mutex*)
  ;; (mutex-lock! *send-receive-mutex*)
  (let* ((run-id          (if rid rid 0))
	 (connection-info (rmt:get-connection-info run-id)))
    ;; the nmsg method does the encoding under the hood (the http method should be changed to do this also)
    (if connection-info
	;; use the server if have connection info
	(let* ((dat     (case *transport-type*
			  ((http)(condition-case
				  (http-transport:client-api-send-receive run-id connection-info cmd params)
				  ((commfail)(vector #f "communications fail"))
				  ((exn)(vector #f "other fail"))))
			  ;; ((nmsg)(condition-case
			  ;;         (nmsg-transport:client-api-send-receive run-id connection-info cmd params)
			  ;;         ((timeout)(vector #f "timeout talking to server"))))
			  (else  (exit))))
	       (success (if (vector? dat) (vector-ref dat 0) #f))
	       (res     (if (vector? dat) (vector-ref dat 1) #f)))
	  (if (vector? connection-info)(http-transport:server-dat-update-last-access connection-info))
	  (if success
	      (begin
		;; (mutex-unlock! *send-receive-mutex*)
		(case *transport-type* 
		  ((http) res) ;; (db:string->obj res))
		  ;; ((nmsg) res)
                  )) ;; (vector-ref res 1)))
	      (begin ;; let ((new-connection-info (client:setup run-id)))
		(debug:print 0 *default-log-port* "WARNING: Communication failed, trying call to rmt:send-receive again.")
		;; (case *transport-type*
		;;   ((nmsg)(nn-close (http-transport:server-dat-get-socket connection-info))))
		(hash-table-delete! *runremote* run-id) ;; don't keep using the same connection
		;; NOTE: killing server causes this process to block forever. No idea why. Dec 2. 
		;; (if (eq? (modulo attemptnum 5) 0)
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
			 (if (null? tal)
			     (if (> tot 10)
				 (cons newmax-cmd currmax)
				 (cons 'none 0))
			     (loop (car tal)(cdr tal) newmax-cmd currmax)))))))
    (mutex-unlock! *db-stats-mutex*)
    res))
	  
(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
  (let* ((dbstruct-local (if *dbstruct-db*
			     *dbstruct-db*
			     (let* ((dbdir (db:dbfile-path #f)) ;;  (conc    (configf:lookup *configdat* "setup" "linktree") "/.db"))
				    (db (make-dbr:dbstruct path:  dbdir local: #t)))
			       (set! *dbstruct-db* db)
			       db)))
	 (db-file-path   (db:dbfile-path 0))
	 ;; (read-only      (not (file-read-access? db-file-path)))
	 (start          (current-milliseconds))
	 (resdat         (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)))
	 (success        (vector-ref resdat 0))
	 (res            (vector-ref resdat 1))
	 (duration       (- (current-milliseconds) start)))







|

|
<
<
<
<
<







223
224
225
226
227
228
229
230
231
232





233
234
235
236
237
238
239
			 (if (null? tal)
			     (if (> tot 10)
				 (cons newmax-cmd currmax)
				 (cons 'none 0))
			     (loop (car tal)(cdr tal) newmax-cmd currmax)))))))
    (mutex-unlock! *db-stats-mutex*)
    res))

(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
  (let* ((dbstruct-local (db:open-local-db-handle))





	 (db-file-path   (db:dbfile-path 0))
	 ;; (read-only      (not (file-read-access? db-file-path)))
	 (start          (current-milliseconds))
	 (resdat         (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)))
	 (success        (vector-ref resdat 0))
	 (res            (vector-ref resdat 1))
	 (duration       (- (current-milliseconds) start)))
311
312
313
314
315
316
317
318

319
320
321
322
323
324
325

;; This login does no retries under the hood - it acts a bit like a ping.
;; Deprecated for nmsg-transport.
;;
(define (rmt:login-no-auto-client-setup connection-info run-id)
  (case *transport-type*
    ((http)(rmt:send-receive-no-auto-client-setup connection-info 'login run-id (list *toppath* megatest-version run-id *my-client-signature*)))
    ((nmsg)(nmsg-transport:client-api-send-receive run-id connection-info 'login (list *toppath* megatest-version run-id *my-client-signature*)))))


;; hand off a call to one of the db:queries statements
;; added run-id to make looking up the correct db possible 
;;
(define (rmt:general-call stmtname run-id . params)
  (rmt:send-receive 'general-call run-id (append (list stmtname run-id) params)))








|
>







310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325

;; This login does no retries under the hood - it acts a bit like a ping.
;; Deprecated for nmsg-transport.
;;
(define (rmt:login-no-auto-client-setup connection-info run-id)
  (case *transport-type*
    ((http)(rmt:send-receive-no-auto-client-setup connection-info 'login run-id (list *toppath* megatest-version run-id *my-client-signature*)))
    ;;((nmsg)(nmsg-transport:client-api-send-receive run-id connection-info 'login (list *toppath* megatest-version run-id *my-client-signature*)))
    ))

;; hand off a call to one of the db:queries statements
;; added run-id to make looking up the correct db possible 
;;
(define (rmt:general-call stmtname run-id . params)
  (rmt:send-receive 'general-call run-id (append (list stmtname run-id) params)))

340
341
342
343
344
345
346

347


348



349

350


351
352
353
354
355
356
357

;; These require run-id because the values come from the run!
;;
(define (rmt:get-key-val-pairs run-id)
  (rmt:send-receive 'get-key-val-pairs run-id (list run-id)))

(define (rmt:get-keys)

  (rmt:send-receive 'get-keys #f '()))






(define (rmt:get-key-vals run-id)

  (rmt:send-receive 'get-key-vals #f (list run-id)))



(define (rmt:get-targets)
  (rmt:send-receive 'get-targets #f '()))

(define (rmt:get-target run-id)
  (rmt:send-receive 'get-target run-id (list run-id)))








>
|
>
>

>
>
>

>
|
>
>







340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366

;; These require run-id because the values come from the run!
;;
(define (rmt:get-key-val-pairs run-id)
  (rmt:send-receive 'get-key-val-pairs run-id (list run-id)))

(define (rmt:get-keys)
  (if *db-keys* *db-keys* 
     (let ((res (rmt:send-receive 'get-keys #f '())))
       (set! *db-keys* res)
       res)))

;; we don't reuse run-id's (except possibly *after* a db cleanup) so it is safe
;; to cache the resuls in a hash
;;
(define (rmt:get-key-vals run-id)
  (or (hash-table-ref/default *keyvals* run-id #f)
      (let ((res (rmt:send-receive 'get-key-vals #f (list run-id))))
        (hash-table-set! *keyvals* run-id res)
        res)))

(define (rmt:get-targets)
  (rmt:send-receive 'get-targets #f '()))

(define (rmt:get-target run-id)
  (rmt:send-receive 'get-target run-id (list run-id)))

Modified rpc-transport.scm from [7e38f4f2de] to [62a65daa58].

36
37
38
39
40
41
42


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
   (apply (eval (string->symbol procstr)) params)))

;; all routes though here end in exit ...
;;
;; start_server? 
;;
(define (rpc-transport:launch run-id)


  (set! *run-id*   run-id)
  (if (args:get-arg "-daemonize")
      (daemon:ize))
  (if (server:check-if-running run-id)
      (begin
	(debug:print 0 *default-log-port* "INFO: Server for run-id " run-id " already running")
	(exit 0)))
  (let loop ((server-id (open-run-close tasks:server-lock-slot tasks:open-db run-id))
	     (remtries  4))
    (if (not server-id)
	(if (> remtries 0)
	    (begin
	      (thread-sleep! 2)
	      (loop (open-run-close tasks:server-lock-slot tasks:open-db run-id)
		    (- remtries 1)))
	    (begin
	      ;; since we didn't get the server lock we are going to clean up and bail out
	      (debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
	      (open-run-close tasks:server-delete-records-for-this-pid tasks:open-db " rpc-transport:launch")))
	(begin
	  (rpc-transport:run (if (args:get-arg "-server")(args:get-arg "-server") "-") run-id server-id)
	  (exit)))))

(define (rpc-transport:run hostn run-id server-id)
  (debug:print 2 *default-log-port* "Attempting to start the rpc server ...")
   ;; (trace rpc:publish-procedure!)

  (rpc:publish-procedure! 'server:login server:login)
  (rpc:publish-procedure! 'testing (lambda () "Just testing"))







>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|







36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
   (apply (eval (string->symbol procstr)) params)))

;; all routes though here end in exit ...
;;
;; start_server? 
;;
(define (rpc-transport:launch run-id)
  (let* ((tdbdat (tasks:open-db)))
    (BB> "rpc-transport:launch fired for run-id="run-id)
    (set! *run-id*   run-id)
    (if (args:get-arg "-daemonize")
        (daemon:ize))
    (if (server:check-if-running run-id)
        (begin
          (debug:print 0 *default-log-port* "INFO: Server for run-id " run-id " already running")
          (exit 0)))
    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
               (remtries  4))
      (if (not server-id)
          (if (> remtries 0)
              (begin
                (thread-sleep! 2)
                (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
                      (- remtries 1)))
              (begin
                ;; since we didn't get the server lock we are going to clean up and bail out
                (debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
                (tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " rpc-transport:launch")))
          (begin
            (rpc-transport:run (if (args:get-arg "-server")(args:get-arg "-server") "-") run-id server-id)
            (exit))))))

(define (rpc-transport:run hostn run-id server-id)
  (debug:print 2 *default-log-port* "Attempting to start the rpc server ...")
   ;; (trace rpc:publish-procedure!)

  (rpc:publish-procedure! 'server:login server:login)
  (rpc:publish-procedure! 'testing (lambda () "Just testing"))

Modified runs.scm from [de4f2b1394] to [7de1bce1de].

721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
					(debug:print-error 0 *default-log-port* "prereqs-not-met is not a list! " prereqs-not-met)
					'())))
	 (non-completed           (filter (lambda (x)             ;; remove hed from not completed list, duh, of course it is not completed!
					    (not (equal? x hed)))
					  (runs:calc-not-completed prereqs-not-met)))
	 (loop-list               (list hed tal reg reruns))
	 ;; configure the load runner
	 (numcpus                 (common:get-num-cpus))
	 (maxload                 (string->number (or (configf:lookup *configdat* "jobtools" "maxload") "3")))
	 (waitdelay               (string->number (or (configf:lookup *configdat* "jobtools" "waitdelay") "60"))))
    (debug:print-info 4 *default-log-port* "have-resources: " have-resources " prereqs-not-met: (" 
		      (string-intersperse 
		       (map (lambda (t)
			      (if (vector? t)
				  (conc (db:test-get-state t) "/" (db:test-get-status t))







|







721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
					(debug:print-error 0 *default-log-port* "prereqs-not-met is not a list! " prereqs-not-met)
					'())))
	 (non-completed           (filter (lambda (x)             ;; remove hed from not completed list, duh, of course it is not completed!
					    (not (equal? x hed)))
					  (runs:calc-not-completed prereqs-not-met)))
	 (loop-list               (list hed tal reg reruns))
	 ;; configure the load runner
	 (numcpus                 (common:get-num-cpus #f))
	 (maxload                 (string->number (or (configf:lookup *configdat* "jobtools" "maxload") "3")))
	 (waitdelay               (string->number (or (configf:lookup *configdat* "jobtools" "waitdelay") "60"))))
    (debug:print-info 4 *default-log-port* "have-resources: " have-resources " prereqs-not-met: (" 
		      (string-intersperse 
		       (map (lambda (t)
			      (if (vector? t)
				  (conc (db:test-get-state t) "/" (db:test-get-status t))
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
	  (debug:print-info 4 *default-log-port* "Exiting loop with...\n  hed=" hed "\n  tal=" tal "\n  reruns=" reruns))
	 )))
    ;; now *if* -run-wait we wait for all tests to be done
    ;; Now wait for any RUNNING tests to complete (if in run-wait mode)
    (thread-sleep! 5) ;; I think there is a race condition here. Let states/statuses settle
    (let wait-loop ((num-running      (rmt:get-count-tests-running-for-run-id run-id))
		    (prev-num-running 0))
      ;; (debug:print 0 *default-log-port* "num-running=" num-running ", prev-num-running=" prev-num-running)
      (if (and (or (args:get-arg "-run-wait")
		   (equal? (configf:lookup *configdat* "setup" "run-wait") "yes"))
	       (> num-running 0))
	  (begin
	    ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes
	    ;; (debug:print 0 *default-log-port* "Got here eh! num-running=" num-running " (> num-running 0) " (> num-running 0))
	    (if (> (current-seconds)(+ last-time-incomplete 900))







|







1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
	  (debug:print-info 4 *default-log-port* "Exiting loop with...\n  hed=" hed "\n  tal=" tal "\n  reruns=" reruns))
	 )))
    ;; now *if* -run-wait we wait for all tests to be done
    ;; Now wait for any RUNNING tests to complete (if in run-wait mode)
    (thread-sleep! 5) ;; I think there is a race condition here. Let states/statuses settle
    (let wait-loop ((num-running      (rmt:get-count-tests-running-for-run-id run-id))
		    (prev-num-running 0))
      ;; (BB> "num-running=" num-running ", prev-num-running=" prev-num-running)
      (if (and (or (args:get-arg "-run-wait")
		   (equal? (configf:lookup *configdat* "setup" "run-wait") "yes"))
	       (> num-running 0))
	  (begin
	    ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes
	    ;; (debug:print 0 *default-log-port* "Got here eh! num-running=" num-running " (> num-running 0) " (> num-running 0))
	    (if (> (current-seconds)(+ last-time-incomplete 900))
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
		  ((and skip-check
			(configf:lookup test-conf "skip" "rundelay"))
		   ;; run-ids = #f means *all* runs
		   (let* ((numseconds      (common:hms-string->seconds (configf:lookup test-conf "skip" "rundelay")))
			  (running-tests   (rmt:get-tests-for-runs-mindata #f full-test-name '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED") '() #f))
			  (completed-tests (rmt:get-tests-for-runs-mindata #f full-test-name '("COMPLETED" "INCOMPLETE") '("PASS" "FAIL" "ABORT") #f)) ;; ironically INCOMPLETE is same as COMPLETED in this contex
			  (last-run-times  (map db:mintest-get-event_time completed-tests))
			  (time-since-last (- (current-seconds) (if (null? last-run-times) 0 (apply max last-run-times)))))
		     (if (or (not (null? running-tests)) ;; have to skip if test is running
			     (> numseconds time-since-last))
			 (set! skip-test (conc "Skipping due to previous test run less than " (configf:lookup test-conf "skip" "rundelay") " ago"))))))
		 
		 (if skip-test
		     (begin
		       (mt:test-set-state-status-by-id run-id test-id "COMPLETED" "SKIP" skip-test)







|







1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
		  ((and skip-check
			(configf:lookup test-conf "skip" "rundelay"))
		   ;; run-ids = #f means *all* runs
		   (let* ((numseconds      (common:hms-string->seconds (configf:lookup test-conf "skip" "rundelay")))
			  (running-tests   (rmt:get-tests-for-runs-mindata #f full-test-name '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED") '() #f))
			  (completed-tests (rmt:get-tests-for-runs-mindata #f full-test-name '("COMPLETED" "INCOMPLETE") '("PASS" "FAIL" "ABORT") #f)) ;; ironically INCOMPLETE is same as COMPLETED in this contex
			  (last-run-times  (map db:mintest-get-event_time completed-tests))
			  (time-since-last (- (current-seconds) (if (null? last-run-times) 0 (common:max last-run-times)))))
		     (if (or (not (null? running-tests)) ;; have to skip if test is running
			     (> numseconds time-since-last))
			 (set! skip-test (conc "Skipping due to previous test run less than " (configf:lookup test-conf "skip" "rundelay") " ago"))))))
		 
		 (if skip-test
		     (begin
		       (mt:test-set-state-status-by-id run-id test-id "COMPLETED" "SKIP" skip-test)

Modified server.scm from [1952897710] to [19061b35b0].

19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

(declare (uses common))
(declare (uses db))
(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
(declare (uses synchash))
(declare (uses http-transport))
(declare (uses rpc-transport))
(declare (uses nmsg-transport))
(declare (uses launch))
(declare (uses daemon))

(include "common_records.scm")
(include "db_records.scm")

(define (server:make-server-url hostport)







|







19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

(declare (uses common))
(declare (uses db))
(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
(declare (uses synchash))
(declare (uses http-transport))
(declare (uses rpc-transport))
;;(declare (uses nmsg-transport))
(declare (uses launch))
(declare (uses daemon))

(include "common_records.scm")
(include "db_records.scm")

(define (server:make-server-url hostport)
45
46
47
48
49
50
51
52

53
54
55
56
57
58
59
60
61
62
63
64
;; Call this to start the actual server
;;

;; all routes though here end in exit ...
;;
;; start_server
;;
(define (server:launch run-id)

  (case *transport-type*
    ((http)(http-transport:launch run-id))
    ((nmsg)(nmsg-transport:launch run-id))
    ((rpc)  (rpc-transport:launch run-id))
    (else (debug:print-error 0 *default-log-port* "unknown server type " *transport-type*))))
;;       (else   (debug:print-error 0 *default-log-port* "No known transport set, transport=" transport ", using rpc")
;; 	      (rpc-transport:launch run-id)))))

;;======================================================================
;; S E R V E R   U T I L I T I E S 
;;======================================================================








|
>
|

|

|







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
;; Call this to start the actual server
;;

;; all routes though here end in exit ...
;;
;; start_server
;;
(define (server:launch run-id transport-type)
  (BB> "server:launch fired for run-id="run-id" transport-type="transport-type)
  (case transport-type
    ((http)(http-transport:launch run-id))
    ;;((nmsg)(nmsg-transport:launch run-id))
    ((rpc)  (rpc-transport:launch run-id))
    (else (debug:print-error 0 *default-log-port* "unknown server type " transport-type))))
;;       (else   (debug:print-error 0 *default-log-port* "No known transport set, transport=" transport ", using rpc")
;; 	      (rpc-transport:launch run-id)))))

;;======================================================================
;; S E R V E R   U T I L I T I E S 
;;======================================================================

99
100
101
102
103
104
105

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

121
122


123
124
125
126
127
128
129
    (else 
     (debug:print-error 0 *default-log-port* "unrecognised transport type: " *transport-type*)
     result)))

;; Given a run id start a server process    ### NOTE ### > file 2>&1 
;; if the run-id is zero and the target-host is set 
;; try running on that host

;;
(define  (server:run run-id)
  (let* ((curr-host   (get-host-name))
	 (curr-ip     (server:get-best-guess-address curr-host))
	 (target-host (configf:lookup *configdat* "server" "homehost" ))
	 (testsuite   (common:get-testsuite-name))
	 (logfile     (conc *toppath* "/logs/" run-id ".log"))
	 (cmdln (conc (common:get-megatest-exe)
		      " -server " (or target-host "-") " -run-id " run-id (if (equal? (configf:lookup *configdat* "server" "daemonize") "yes")
									      (conc " -daemonize -log " logfile)
									      "")
		      " -m testsuite:" testsuite))) ;; (conc " >> " logfile " 2>&1 &")))))
    (debug:print 0 *default-log-port* "INFO: Starting server (" cmdln ") as none running ...")
    (push-directory *toppath*)
    (if (not (directory-exists? "logs"))(create-directory "logs"))

    ;; Rotate logs, logic: 
    ;;                 if > 500k and older than 1 week, remove previous compressed log and compress this log


    (directory-fold 
     (lambda (file rem)
       (if (and (string-match "^.*.log" file)
		(> (file-size (conc "logs/" file)) 200000))
	   (let ((gzfile (conc "logs/" file ".gz")))
	     (if (file-exists? gzfile)
		 (begin







>















>

|
>
>







100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    (else 
     (debug:print-error 0 *default-log-port* "unrecognised transport type: " *transport-type*)
     result)))

;; Given a run id start a server process    ### NOTE ### > file 2>&1 
;; if the run-id is zero and the target-host is set 
;; try running on that host
;;   incidental: rotate logs in logs/ dir.
;;
(define  (server:run run-id)
  (let* ((curr-host   (get-host-name))
	 (curr-ip     (server:get-best-guess-address curr-host))
	 (target-host (configf:lookup *configdat* "server" "homehost" ))
	 (testsuite   (common:get-testsuite-name))
	 (logfile     (conc *toppath* "/logs/" run-id ".log"))
	 (cmdln (conc (common:get-megatest-exe)
		      " -server " (or target-host "-") " -run-id " run-id (if (equal? (configf:lookup *configdat* "server" "daemonize") "yes")
									      (conc " -daemonize -log " logfile)
									      "")
		      " -m testsuite:" testsuite))) ;; (conc " >> " logfile " 2>&1 &")))))
    (debug:print 0 *default-log-port* "INFO: Starting server (" cmdln ") as none running ...")
    (push-directory *toppath*)
    (if (not (directory-exists? "logs"))(create-directory "logs"))
    
    ;; Rotate logs, logic: 
    ;;                 if > 500k and older than 1 week:
    ;;                     remove previous compressed log and compress this log
    ;;
    (directory-fold 
     (lambda (file rem)
       (if (and (string-match "^.*.log" file)
		(> (file-size (conc "logs/" file)) 200000))
	   (let ((gzfile (conc "logs/" file ".gz")))
	     (if (file-exists? gzfile)
		 (begin
139
140
141
142
143
144
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
	     ;; look at target host, is it host.domain.tld or ip address and does it 
	     ;; match current ip or hostname
	     (not (string-match (conc "("curr-host "|" curr-host"\\..*)") target-host))
	     (not (equal? curr-ip target-host)))
	(begin
	  (debug:print-info 0 *default-log-port* "Starting server on " target-host ", logfile is " logfile)
	  (setenv "TARGETHOST" target-host)))

    (setenv "TARGETHOST_LOGF" logfile)
    (common:wait-for-normalized-load 4 " delaying server start due to load") ;; do not try starting servers on an already overloaded machine, just wait forever
    (system (conc "nbfake " cmdln))
    (unsetenv "TARGETHOST_LOGF")
    (if (get-environment-variable "TARGETHOST")(unsetenv "TARGETHOST"))
    ;; (system cmdln)
    (pop-directory)))

(define (server:get-client-signature)
  (if *my-client-signature* *my-client-signature*
      (let ((sig (server:mk-signature)))
	(set! *my-client-signature* sig)
	*my-client-signature*)))

;; kind start up of servers, wait 40 seconds before allowing another server for a given
;; run-id to be launched







>

|






|







144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
	     ;; look at target host, is it host.domain.tld or ip address and does it 
	     ;; match current ip or hostname
	     (not (string-match (conc "("curr-host "|" curr-host"\\..*)") target-host))
	     (not (equal? curr-ip target-host)))
	(begin
	  (debug:print-info 0 *default-log-port* "Starting server on " target-host ", logfile is " logfile)
	  (setenv "TARGETHOST" target-host)))
    
    (setenv "TARGETHOST_LOGF" logfile)
    (common:wait-for-normalized-load 4 " delaying server start due to load" remote-host: (get-environment-variable "TARGETHOST")) ;; do not try starting servers on an already overloaded machine, just wait forever
    (system (conc "nbfake " cmdln))
    (unsetenv "TARGETHOST_LOGF")
    (if (get-environment-variable "TARGETHOST")(unsetenv "TARGETHOST"))
    ;; (system cmdln)
    (pop-directory)))

(define (server:get-client-signature) ;; BB> why is this proc named "get-"?  it returns nothing -- set! has not return value.
  (if *my-client-signature* *my-client-signature*
      (let ((sig (server:mk-signature)))
	(set! *my-client-signature* sig)
	*my-client-signature*)))

;; kind start up of servers, wait 40 seconds before allowing another server for a given
;; run-id to be launched
184
185
186
187
188
189
190
191
192
193

194
195
196
197
198
199
200
	;;
	;; client:start returns #t if login was successful.
	;;
	(let ((res (case *transport-type*
		     ((http)(server:ping-server run-id 
						(tasks:hostinfo-get-interface server)
						(tasks:hostinfo-get-port      server)))
		     ((nmsg)(nmsg-transport:ping (tasks:hostinfo-get-interface server)
						 (tasks:hostinfo-get-port      server)
						 timeout: 2)))))

	  ;; if the server didn't respond we must remove the record
	  (if res
	      #t
	      (begin
		(debug:print-info 0 *default-log-port* "server at " server " not responding, removing record")
		(tasks:server-force-clean-running-records-for-run-id (db:delay-if-busy tdbdat) run-id 
				" server:check-if-running")







|
|
|
>







190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
	;;
	;; client:start returns #t if login was successful.
	;;
	(let ((res (case *transport-type*
		     ((http)(server:ping-server run-id 
						(tasks:hostinfo-get-interface server)
						(tasks:hostinfo-get-port      server)))
		     ;; ((nmsg)(nmsg-transport:ping (tasks:hostinfo-get-interface server)
		     ;;    			 (tasks:hostinfo-get-port      server)
		     ;;    			 timeout: 2))
                     )))
	  ;; if the server didn't respond we must remove the record
	  (if res
	      #t
	      (begin
		(debug:print-info 0 *default-log-port* "server at " server " not responding, removing record")
		(tasks:server-force-clean-running-records-for-run-id (db:delay-if-busy tdbdat) run-id 
				" server:check-if-running")

Modified tasks.scm from [7aab5e9e48] to [a06114a2ac].

227
228
229
230
231
232
233








234
235
236
237
238
239
240
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE state = 'running' AND run_id=?;"
		   (conc "defunct" tag) run-id))

(define (tasks:server-force-clean-run-record mdb run-id iface port tag)
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE state = 'running' AND run_id=? AND interface=? AND port=?;"
		   (conc "defunct" tag) run-id iface port))









(define (tasks:server-delete-records-for-this-pid mdb tag)
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE hostname=? AND pid=?;"
		   (conc "defunct" tag) (get-host-name) (current-process-id)))

(define (tasks:server-delete-record mdb server-id tag) 
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE id=?;"
		   (conc "defunct" tag) server-id)







>
>
>
>
>
>
>
>







227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE state = 'running' AND run_id=?;"
		   (conc "defunct" tag) run-id))

(define (tasks:server-force-clean-run-record mdb run-id iface port tag)
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE state = 'running' AND run_id=? AND interface=? AND port=?;"
		   (conc "defunct" tag) run-id iface port))


;; BB> adding missing func for --list-servers
(define (tasks:server-deregister mdb hostname #!key (pullport #f) (pid #f) (action #f)) ;;pullport pid: pid action: 'delete))
  (if (eq? action 'delete)
      (sqlite3:execute mdb "DELETE FROM servers WHERE pid=? AND port=? AND hostname=?;" pid pullport hostname)
      (sqlite3:execute mdb "UPDATE servers SET state='defunct', heartbeat=strftime('%s','now') WHERE hostname=? AND pid=?;"
                       hostname pid)))

(define (tasks:server-delete-records-for-this-pid mdb tag)
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE hostname=? AND pid=?;"
		   (conc "defunct" tag) (get-host-name) (current-process-id)))

(define (tasks:server-delete-record mdb server-id tag) 
  (sqlite3:execute mdb "UPDATE servers SET state=?,heartbeat=strftime('%s','now') WHERE id=?;"
		   (conc "defunct" tag) server-id)
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
     "SELECT id,pid,hostname,interface,port,pubport,start_time,priority,state,mt_version,strftime('%s','now')-heartbeat AS last_update,transport,run_id 
        FROM servers WHERE run_id=? AND state NOT LIKE 'defunct%' ORDER BY start_time DESC;"
     run-id)
    (reverse res)))

;; no elegance here ...
;;
(define (tasks:kill-server hostname pid)
  (debug:print-info 0 *default-log-port* "Attempting to kill server process " pid " on host " hostname)
  (setenv "TARGETHOST" hostname)
  (setenv "TARGETHOST_LOGF" "server-kills.log")
  (system (conc "nbfake kill " pid))
  (unsetenv "TARGETHOST_LOGF")
  (unsetenv "TARGETHOST"))
 
;; look up a server by run-id and send it a kill, also delete the record for that server
;;
(define (tasks:kill-server-run-id run-id #!key (tag "default"))
  (let* ((tdbdat  (tasks:open-db))







|



|







429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
     "SELECT id,pid,hostname,interface,port,pubport,start_time,priority,state,mt_version,strftime('%s','now')-heartbeat AS last_update,transport,run_id 
        FROM servers WHERE run_id=? AND state NOT LIKE 'defunct%' ORDER BY start_time DESC;"
     run-id)
    (reverse res)))

;; no elegance here ...
;;
(define (tasks:kill-server hostname pid #!key (kill-switch ""))
  (debug:print-info 0 *default-log-port* "Attempting to kill server process " pid " on host " hostname)
  (setenv "TARGETHOST" hostname)
  (setenv "TARGETHOST_LOGF" "server-kills.log")
  (system (conc "nbfake kill "kill-switch" "pid))
  (unsetenv "TARGETHOST_LOGF")
  (unsetenv "TARGETHOST"))
 
;; look up a server by run-id and send it a kill, also delete the record for that server
;;
(define (tasks:kill-server-run-id run-id #!key (tag "default"))
  (let* ((tdbdat  (tasks:open-db))

Modified tdb.scm from [03ea2dc1b8] to [85b17f8d7b].

207
208
209
210
211
212
213

214

215
216
217
218
219
220
221
222
223
224
225

226

227
228
229
230
231
232
233

;; NOTE: Run this local with #f for db !!!
(define (tdb:load-test-data run-id test-id)
  (let loop ((lin (read-line)))
    (if (not (eof-object? lin))
	(begin
	  (debug:print 4 *default-log-port* lin)

	  (rmt:csv->test-data run-id test-id lin)

	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))

;; NOTE: Run this local with #f for db !!!
(define (tdb:load-logpro-data run-id test-id)
  (let loop ((lin (read-line)))
    (if (not (eof-object? lin))
	(begin
	  (debug:print 4 *default-log-port* lin)

	  (rmt:csv->test-data run-id test-id lin)

	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))

(define (tdb:get-prev-tol-for-test tdb test-id category variable)
  ;; Finish me?







>
|
>











>
|
>







207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237

;; NOTE: Run this local with #f for db !!!
(define (tdb:load-test-data run-id test-id)
  (let loop ((lin (read-line)))
    (if (not (eof-object? lin))
	(begin
	  (debug:print 4 *default-log-port* lin)
          ;;(when lin ;; this when blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
          (rmt:csv->test-data run-id test-id lin)
          ;;)
	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))

;; NOTE: Run this local with #f for db !!!
(define (tdb:load-logpro-data run-id test-id)
  (let loop ((lin (read-line)))
    (if (not (eof-object? lin))
	(begin
	  (debug:print 4 *default-log-port* lin)
          ;;(when lin  ;; this when blocked stack dump caused by .dat file from logpro being 0-byte.  fixed by upgrading logpro
          (rmt:csv->test-data run-id test-id lin)
          ;;)
	  (loop (read-line)))))
  ;; roll up the current results.
  ;; FIXME: Add the status too 
  (rmt:test-data-rollup run-id test-id #f))

(define (tdb:get-prev-tol-for-test tdb test-id category variable)
  ;; Finish me?

Modified tests.scm from [b7b139f82b] to [8d5f3a1ead].

565
566
567
568
569
570
571





































































































































































































572
573
574
575
576
577
578
	;;(rmt:update-run-stats 
	;; run-id
	;; (hash-table-map
	;;  state-status-counts
	;;  (lambda (key val)
	;;	(append key (list val)))))
	))))






































































































































































































;; CHECK - WAS THIS ADDED OR REMOVED? MANUAL MERGE WITH API STUFF!!!
;;
;; get a pretty table to summarize steps
;;
;; (define (dcommon:process-steps-table steps);; db test-id #!key (work-area #f))
(define (tests:process-steps-table steps);; db test-id #!key (work-area #f))







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
	;;(rmt:update-run-stats 
	;; run-id
	;; (hash-table-map
	;;  state-status-counts
	;;  (lambda (key val)
	;;	(append key (list val)))))
	))))

(define tests:css-jscript-block
#<<EOF
<style type="text/css">
ul.LinkedList { display: block; }
/* ul.LinkedList ul { display: none; } */
.HandCursorStyle { cursor: pointer; cursor: hand; }  /* For IE */
  </style>

  <script type="text/JavaScript">
    // Add this to the onload event of the BODY element
    function addEvents() {
      activateTree(document.getElementById("LinkedList1"));
    }

    // This function traverses the list and add links 
    // to nested list items
    function activateTree(oList) {
      // Collapse the tree
      for (var i=0; i < oList.getElementsByTagName("ul").length; i++) {
        oList.getElementsByTagName("ul")[i].style.display="none";            
      }                                                                  
      // Add the click-event handler to the list items
      if (oList.addEventListener) {
        oList.addEventListener("click", toggleBranch, false);
      } else if (oList.attachEvent) { // For IE
        oList.attachEvent("onclick", toggleBranch);
      }
      // Make the nested items look like links
      addLinksToBranches(oList);
    }

    // This is the click-event handler
    function toggleBranch(event) {
      var oBranch, cSubBranches;
      if (event.target) {
        oBranch = event.target;
      } else if (event.srcElement) { // For IE
        oBranch = event.srcElement;
      }
      cSubBranches = oBranch.getElementsByTagName("ul");
      if (cSubBranches.length > 0) {
        if (cSubBranches[0].style.display == "block") {
          cSubBranches[0].style.display = "none";
        } else {
          cSubBranches[0].style.display = "block";
        }
      }
    }

    // This function makes nested list items look like links
    function addLinksToBranches(oList) {
      var cBranches = oList.getElementsByTagName("li");
      var i, n, cSubBranches;
      if (cBranches.length > 0) {
        for (i=0, n = cBranches.length; i < n; i++) {
          cSubBranches = cBranches[i].getElementsByTagName("ul");
          if (cSubBranches.length > 0) {
            addLinksToBranches(cSubBranches[0]);
            cBranches[i].className = "HandCursorStyle";
            cBranches[i].style.color = "blue";
            cSubBranches[0].style.color = "black";
            cSubBranches[0].style.cursor = "auto";
          }
        }
      }
    }
  </script>
EOF
)

(define (tests:run-record->test-path run numkeys)
   (append (take (vector->list run) numkeys)
	   (list (vector-ref run (+ 1 numkeys)))))

;; (tests:create-html-tree "test-index.html")
;;
(define (tests:create-html-tree outf)
  (let* ((lockfile  (conc outf ".lock"))
	 (runs-to-process '()))
    (if (common:simple-file-lock lockfile)
	(let* ((linktree  (common:get-linktree))
	       (oup       (open-output-file (or outf (conc linktree "/runs-index.html"))))
	       (area-name (common:get-testsuite-name))
	       (keys      (rmt:get-keys))
	       (numkeys   (length keys))
	       (runsdat   (rmt:get-runs "%" #f #f (map (lambda (x)(list x "%")) keys)))
	       (header    (vector-ref runsdat 0))
	       (runs      (vector-ref runsdat 1))
	       (runtreedat (map (lambda (x)
				  (tests:run-record->test-path x numkeys))
				runs))
	       (runs-htree (common:list->htree runtreedat)))
	  (set! runs-to-process runs)
	  (s:output-new
	   oup
	   (s:html tests:css-jscript-block
		   (s:title "Summary for " area-name)
		   (s:body 'onload "addEvents();"
			   (s:h1 "Summary for " area-name)
			   ;; top list
			   (s:ul 'id "LinkedList1" 'class "LinkedList"
				 (s:li
				  "Runs"
				  (common:htree->html runs-htree
						      '()
						      (lambda (x p)
							(let* ((targ-path (string-intersperse p "/"))
                                                               (full-path (conc linktree "/" targ-path))
                                                               (run-name  (car (reverse p))))
                                                          (if (and (file-exists? full-path)
                                                                   (directory?   full-path)
                                                                   (file-write-access? full-path))
                                                              (s:a run-name 'href (conc targ-path "/run-summary.html"))
                                                              (begin
                                                                (debug:print 0 *default-log-port* "INFO: Can't create " targ-path "/run-summary.html")
                                                                (conc run-name " (Not able to create summary at " targ-path ")")))))))))))
          (close-output-port oup)
	  (common:simple-file-release-lock lockfile)
	  (for-each
	   (lambda (run)
	     (let* ((test-subpath (tests:run-record->test-path run numkeys))
		    (run-id       (db:get-value-by-header run header "id"))
                    (run-dir      (tests:run-record->test-path run numkeys))
		    (test-dats    (rmt:get-tests-for-run
				   run-id
                                   "%/"       ;; testnamepatt
				   '()        ;; states
				   '()        ;; statuses
				   #f         ;; offset
				   #f         ;; num-to-get
				   #f         ;; hide/not-hide
				   #f         ;; sort-by
				   #f         ;; sort-order
				   #f         ;; 'shortlist                           ;; qrytype
                                   0         ;; last update
				   #f))
                    (tests-tree-dat (map (lambda (test-dat)
                                         ;; (tests:run-record->test-path x numkeys))
                                         (let* ((test-name  (db:test-get-testname test-dat))
                                                (item-path  (db:test-get-item-path test-dat))
                                                (full-name  (db:test-make-full-name test-name item-path))
                                                (path-parts (string-split full-name)))
                                           path-parts))
                                       test-dats))
                    (tests-htree (common:list->htree tests-tree-dat))
                    (html-dir    (conc linktree "/" (string-intersperse run-dir "/")))
                    (html-path   (conc html-dir "/run-summary.html"))
                    (oup         (if (and (file-exists? html-dir)
                                          (directory?   html-dir)
                                          (file-write-access? html-dir))
                                     (open-output-file  html-path)
                                     #f)))
               ;; (print "run-dir: " run-dir ", tests-tree-dat: " tests-tree-dat)
               (if oup
                   (begin
                     (s:output-new
                      oup
                      (s:html tests:css-jscript-block
                              (s:title "Summary for " area-name)
                              (s:body 'onload "addEvents();"
                                      (s:h1 "Summary for " (string-intersperse run-dir "/"))
                                      ;; top list
                                      (s:ul 'id "LinkedList1" 'class "LinkedList"
                                            (s:li
                                             "Tests"
                                             (common:htree->html tests-htree
                                                                 '()
                                                                 (lambda (x p)
                                                                   (let* ((targ-path (string-intersperse p "/"))
                                                                          (test-name (car p))
                                                                          (item-path ;; (if (> (length p) 2) ;; test-name + run-name
                                                                           (string-intersperse p "/"))
                                                                          (full-targ (conc html-dir "/" targ-path))
                                                                          (std-file  (conc full-targ "/test-summary.html"))
                                                                          (alt-file  (conc full-targ "/megatest-rollup-" test-name ".html"))
                                                                          (html-file (if (file-exists? alt-file)
                                                                                         alt-file
                                                                                         std-file))
                                                                          (run-name  (car (reverse p))))
                                                                     (if (and (not (file-exists? full-targ))
                                                                              (directory? full-targ)
                                                                              (file-write-access? full-targ))
                                                                         (tests:summarize-test 
                                                                          run-id 
                                                                          (rmt:get-test-id run-id test-name item-path)))
                                                                     (if (file-exists? full-targ)
                                                                         (s:a run-name 'href html-file)
                                                                         (begin
                                                                           (debug:print 0 *default-log-port* "ERROR: can't access " full-targ)
                                                                           (conc "No summary for " run-name)))))
                                                                 ))))))
                     (close-output-port oup)))))
           runs)
          #t)
	#f)))


;; CHECK - WAS THIS ADDED OR REMOVED? MANUAL MERGE WITH API STUFF!!!
;;
;; get a pretty table to summarize steps
;;
;; (define (dcommon:process-steps-table steps);; db test-id #!key (work-area #f))
(define (tests:process-steps-table steps);; db test-id #!key (work-area #f))
664
665
666
667
668
669
670
671

672
673
674
675
676
677
678
			   (if (eq? time-a time-b)
			       (string<? (conc (vector-ref a 2))
					 (conc (vector-ref b 2)))
			       #f))
		       (string<? (conc time-a)(conc time-b)))))))))


;; summarize test

(define (tests:summarize-test run-id test-id)
  (let* ((test-dat  (rmt:get-test-info-by-id run-id test-id))
	 (steps-dat (rmt:get-steps-for-test run-id test-id))
	 (test-name (db:test-get-testname test-dat))
	 (item-path (db:test-get-item-path test-dat))
	 (full-name (db:test-make-full-name test-name item-path))
	 (oup       (open-output-file (conc (db:test-get-rundir test-dat) "/test-summary.html")))







|
>







861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
			   (if (eq? time-a time-b)
			       (string<? (conc (vector-ref a 2))
					 (conc (vector-ref b 2)))
			       #f))
		       (string<? (conc time-a)(conc time-b)))))))))


;; summarize test in to a file test-summary.html in the test directory
;;
(define (tests:summarize-test run-id test-id)
  (let* ((test-dat  (rmt:get-test-info-by-id run-id test-id))
	 (steps-dat (rmt:get-steps-for-test run-id test-id))
	 (test-name (db:test-get-testname test-dat))
	 (item-path (db:test-get-item-path test-dat))
	 (full-name (db:test-make-full-name test-name item-path))
	 (oup       (open-output-file (conc (db:test-get-rundir test-dat) "/test-summary.html")))

Modified tests/Makefile from [cf65e1af4f] to [ca46bf23f9].

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

test0 : cleanprep
	cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG)

test1 : cleanprep

test2 : fullprep
	cd fullrun;$(MEGATEST) -preclean -runtests ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none :runname $(RUNNAME) -debug $(DEBUG) $(LOGGING)
	cd fullrun;megatest -preclean -runtests % -target ubuntu/nfs/none :runname $(RUNNAME)_01 -testpatt %/,%/ai -debug $(DEBUG)
	cd fullrun;megatest -preclean -runtests %/,%/ai -target ubuntu/nfs/none :runname $(RUNAME)_02 -debug $(DEBUG)
	cd fullrun;megatest -preclean -runtests runfirst/%,%/ai -target ubuntu/nfs/none :runname $(RUNNAME)_02 -debug $(DEBUG)
	cd fullrun;megatest -runtests %/,%/winter -target ubuntu/nfs/none :runname $(RUNNAME)_03  -debug $(DEBUG)
	sleep 40;cd fullrun;megatest -target ubuntu/nfs/none :runname $(RUNNAME) -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -debug $(DEBUG) $(LOGGING)


test3 : fullprep test3a test3b

test3a :
	@echo Run runfirst and any waitons.
	cd fullrun;$(MEGATEST) -preclean -runtests runfirst -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b 

test3b :
	@echo Run all_toplevel and all waitons
	cd fullrun;$(MEGATEST) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_c

test4 : cleanprep
	@echo "WARNING: No longer running fullprep, test converage may be lessened"
	cd fullrun;time $(MEGATEST) -debug $(DEBUG) -run-wait -runtests % -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)

test4a : cleanprep
	cd fullrun;time $(MEGATEST) -debug $(DEBUG) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)

# NOTE: Only one instance can be a server
test5 : cleanprep
	rm -f fullrun/a*.log fullrun/logs/*
	@echo "WARNING: No longer running fullprep, test converage may be lessened"
	cd fullrun;sleep 0;$(MEGATEST) -preclean -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log &
	cd fullrun;sleep 0;$(MEGATEST) -preclean -runtests % -target ubuntu/nfs/sleep1 :runname $(RUNNAME)_ae -debug $(DEBUG) $(LOGGING) > ae.log 2> ae.log &	







|
|
|
|
|
|






|



|



|


|







46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

test0 : cleanprep
	cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG)

test1 : cleanprep

test2 : fullprep
	cd fullrun;$(MEGATEST) -preclean -runtests ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none -runname $(RUNNAME) -debug $(DEBUG) $(LOGGING)
	cd fullrun;megatest -preclean -runtests % -target ubuntu/nfs/none -runname $(RUNNAME)_01 -testpatt %/,%/ai -debug $(DEBUG)
	cd fullrun;megatest -preclean -runtests %/,%/ai -target ubuntu/nfs/none -runname $(RUNAME)_02 -debug $(DEBUG)
	cd fullrun;megatest -preclean -runtests runfirst/%,%/ai -target ubuntu/nfs/none -runname $(RUNNAME)_02 -debug $(DEBUG)
	cd fullrun;megatest -runtests %/,%/winter -target ubuntu/nfs/none -runname $(RUNNAME)_03  -debug $(DEBUG)
	sleep 40;cd fullrun;megatest -target ubuntu/nfs/none -runname $(RUNNAME) -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -debug $(DEBUG) $(LOGGING)


test3 : fullprep test3a test3b

test3a :
	@echo Run runfirst and any waitons.
	cd fullrun;$(MEGATEST) -preclean -runtests runfirst -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b 

test3b :
	@echo Run all_toplevel and all waitons
	cd fullrun;$(MEGATEST) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_c

test4 : cleanprep
	@echo "WARNING: No longer running fullprep, test converage may be lessened"
	cd fullrun;time $(MEGATEST) -debug $(DEBUG) -run-wait -run -testpatt % -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)

test4a : cleanprep
	cd fullrun;time $(MEGATEST) -debug $(DEBUG) -preclean -run -testpatt all_toplevel -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)

# NOTE: Only one instance can be a server
test5 : cleanprep
	rm -f fullrun/a*.log fullrun/logs/*
	@echo "WARNING: No longer running fullprep, test converage may be lessened"
	cd fullrun;sleep 0;$(MEGATEST) -preclean -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log &
	cd fullrun;sleep 0;$(MEGATEST) -preclean -runtests % -target ubuntu/nfs/sleep1 :runname $(RUNNAME)_ae -debug $(DEBUG) $(LOGGING) > ae.log 2> ae.log &	

Modified tests/fullrun/megatest.config from [73b1295a6b] to [8446f6ae84].

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36


37
38
39
40
41
42
43
44
# the path points to where megatest.db exists
[refareas]
area1 /tmp/oldarea/megatest

[include ./configs/mt_include_1.config]

[dashboard]
pre-command  xterm -geometry 180x20 -e "
post-command |& tee results.log ;echo Press any key to continue;bash -c 'read -n 1 -s'" &
testsort -event_time

[misc]
home #{shell readlink -f $MT_RUN_AREA_HOME}
parent #{shell readlink -f $MT_RUN_AREA_HOME/..}
testsuite #{shell basename $MT_RUN_AREA_HOME}

[tests-paths]
1 #{get misc parent}/simplerun/tests

[setup]

# turn off faststart, put monitor.db in MT_RUN_AREA_HOME/db
# and set the dbdir to /var/tmp/$USER/mt_db to enable keeping
# the raw db in /var/tmp/$USER
#
faststart  no
monitordir #{getenv MT_RUN_AREA_HOME}/db


dbdir      #{getenv MT_RUN_AREA_HOME}/db

# sync more aggressively to megatest-db
megatest-db yes

# Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding
# this may save a few milliseconds on launching tests
# launchwait no







|
|


















>
>
|







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# the path points to where megatest.db exists
[refareas]
area1 /tmp/oldarea/megatest

[include ./configs/mt_include_1.config]

[dashboard]
# pre-command  xterm -geometry 180x20 -e "
# post-command |& tee results.log ;echo Press any key to continue;bash -c 'read -n 1 -s'" &
testsort -event_time

[misc]
home #{shell readlink -f $MT_RUN_AREA_HOME}
parent #{shell readlink -f $MT_RUN_AREA_HOME/..}
testsuite #{shell basename $MT_RUN_AREA_HOME}

[tests-paths]
1 #{get misc parent}/simplerun/tests

[setup]

# turn off faststart, put monitor.db in MT_RUN_AREA_HOME/db
# and set the dbdir to /var/tmp/$USER/mt_db to enable keeping
# the raw db in /var/tmp/$USER
#
faststart  no
monitordir #{getenv MT_RUN_AREA_HOME}/db
dbdirdefn  /tmp/#{getenv USER}/#{getenv MT_TESTSUITE_NAME}/db
dbdirmkdir #{scheme (create-directory "#{get setup dbdirdefn}" #t)}
dbdir      #{get setup dbdirdefn}

# sync more aggressively to megatest-db
megatest-db yes

# Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding
# this may save a few milliseconds on launching tests
# launchwait no

Modified utils/Makefile.git.installall from [d3a2bd23c6] to [307e7d57f5].

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#======================================================================
# N  A N O M S G
#======================================================================

# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

nanomsg-0.6-beta.tar.gz :
	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
	tar xf nanomsg-0.6-beta.tar.gz

$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc








|
|

|
|

|
|

|
|

|







209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#======================================================================
# N  A N O M S G
#======================================================================

# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

# nanomsg-0.6-beta.tar.gz :
# 	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
# 	tar xf nanomsg-0.6-beta.tar.gz

# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
# 	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
# 	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc

Modified utils/Makefile.installall from [981091d91c] to [aefb91939e].

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
$(CHICKEN_EGG_DIR)/sqlite3.so : $(PREFIX)/bin/sqlite3
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) sqlite3

#======================================================================
# N  A N O M S G
#======================================================================

# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

nanomsg-0.6-beta.tar.gz :
	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
	tar xf nanomsg-0.6-beta.tar.gz

$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc








|
|

|
|

|
|

|
|

|
|

|







193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
$(CHICKEN_EGG_DIR)/sqlite3.so : $(PREFIX)/bin/sqlite3
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) sqlite3

#======================================================================
# N  A N O M S G
#======================================================================

# # https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# # https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

# nanomsg-0.6-beta.tar.gz :
# 	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
# 	tar xf nanomsg-0.6-beta.tar.gz

# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
# 	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
# 	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc

Modified utils/Makefile.latest.installall from [149911e2cc] to [e858ad0d21].

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
$(CHICKEN_EGG_DIR)/sqlite3.so : $(PREFIX)/bin/sqlite3
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) sqlite3

#======================================================================
# N  A N O M S G
#======================================================================

# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

nanomsg-0.6-beta.tar.gz :
	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
	tar xf nanomsg-0.6-beta.tar.gz

$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc








|
|

|
|

|
|

|
|

|
|

|







193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
$(CHICKEN_EGG_DIR)/sqlite3.so : $(PREFIX)/bin/sqlite3
	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) sqlite3

#======================================================================
# N  A N O M S G
#======================================================================

# # https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz
# # https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz

# nanomsg-0.6-beta.tar.gz :
# 	wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz

# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz
# 	tar xf nanomsg-0.6-beta.tar.gz

# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING
# 	cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install

# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
# 	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg

# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install  -D no-library-checks nanomsg

#======================================================================
# M A T T S   U T I L S
#======================================================================

# opensrc

Modified utils/installall.sh from [504497f8ec] to [89ae2af8a7].

10
11
12
13
14
15
16

17
18
19

20


21
22
23
24
25
26



































27
28
29
30
31
32
33
#  This program is distributed WITHOUT ANY WARRANTY; without even the
#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
#  PURPOSE.

echo You may need to do the following first:
echo sudo apt-get install libreadline-dev
echo sudo apt-get install libwebkitgtk-dev 

echo sudo apt-get install libssl-dev
echo sudo apt-get install libmotif3 -OR- set KTYPE=26g4
echo KTYPE can be 26, 26g4, or 32

echo  


echo KTYPE=$KTYPE
echo You are using PREFIX=$PREFIX
echo You are using proxy="$proxy"
echo 
echo "Set additional_libpath to help find gtk or other libraries, don't forget a leading :"




































# NOTES:
#
# Centos with security setup may need to do commands such as following as root:
#
# NB// fix the paths first
#
# for a in /localdisk/chicken/4.8.0/lib/*.so;do chcon -t textrel_shlib_t $a; done 







>


|
>
|
>
>
|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#  This program is distributed WITHOUT ANY WARRANTY; without even the
#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
#  PURPOSE.

echo You may need to do the following first:
echo sudo apt-get install libreadline-dev
echo sudo apt-get install libwebkitgtk-dev 
echo sudo apt-get install libpangox-1.0-0 zlib1g-dev libfreetype6-dev cmake
echo sudo apt-get install libssl-dev
echo sudo apt-get install libmotif3 -OR- set KTYPE=26g4
echo
echo Set OPTION to std, currently OPTION=$OPTION
echo
echo Additionally, if you want mysql-client, you will need to make sure
echo mysql_config is in your path
echo
echo You are using PREFIX=$PREFIX
echo You are using proxy="$proxy"
echo 
echo "Set additional_libpath to help find gtk or other libraries, don't forget a leading :"

SYSTEM_TYPE=$(lsb_release -irs |tr ' ' '_' |tr '\n' '-')$(uname -i)-$OPTION

# Set up variables
#
case $SYSTEM_TYPE in
Ubuntu-16.04-x86_64-std)
	KTYPE=32
	CDVER=5.10
	IUPVER=3.17
	IMVER=3.11
	;;
Ubuntu-16.04-i686-std)
	KTYPE=32
	CDVER=5.10
	IUPVER=3.17
	IMVER=3.11
	;;
SUSE_LINUX_11-x86_64-std)
  KTYPE=26g4 
	CDVER=5.10
	IUPVER=3.17
	IMVER=3.11
  ;;
CentOS_5.11-x86_64-std)
  KTYPE=24g3 
  CDVER=5.4.1
  IUPVER=3.5
  IMVER=3.6.3
  ;; 
esac
			   
echo KTYPE=$KTYPE			  
echo CDVER=$CDVER
echo IUPVER=$IUPVER
echo IMVER=$IMVER	
# NOTES:
#
# Centos with security setup may need to do commands such as following as root:
#
# NB// fix the paths first
#
# for a in /localdisk/chicken/4.8.0/lib/*.so;do chcon -t textrel_shlib_t $a; done 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

88
89





90
91
92
93
94
95
96
97
98
99
100
101































102
103
104
105

106
107
108
109
110
111
112
113
114




115
116
117

118
119
120
121
122
123
124
125
126
127
128
129
130
131

132
133


134
135

136
137
138



139





140

141
142
143
144
145
146
147
148
149
150
151
152
153





154
155
156
157
158
159
160
161
162
163
164
165
166

167
168

169
170

171
172
173
174

175
176
177
178




179
180
181
182
183
184
185
186
187
188
189

190
191
192






















193















194
195
196





197


198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246


else
  export http_proxy=http://$proxy
  export PROX="-proxy $proxy"
fi

if [[ $KTYPE == "" ]]; then
  echo 'Using KTYPE=26'
  export KTYPE=26
else
  echo Using KTYPE=$KTYPE
fi

# Put all the downloaded tar files in tgz
mkdir -p tgz

# http://code.call-cc.org/releases/4.8.0/chicken-4.8.0.5.tar.gz
export CHICKEN_VERSION=4.8.0.5
export CHICKEN_BASEVER=4.8.0
chicken_targz=chicken-${CHICKEN_VERSION}.tar.gz
if ! [[ -e tgz/$chicken_targz ]]; then 
    wget http://code.call-cc.org/releases/${CHICKEN_BASEVER}/${chicken_targz}
    mv $chicken_targz tgz
fi 

BUILDHOME=$PWD
DEPLOYTARG=$BUILDHOME/deploy

if [[ $PREFIX == "" ]]; then
   PREFIX=$PWD/inst
fi

export PATH=$PREFIX/bin:$PATH
export LIBPATH=$PREFIX/lib$ADDITIONAL_LIBPATH
export LD_LIBRARY_PATH=$LIBPATH
export CHICKEN_INSTALL=$PREFIX/bin/chicken-install

echo "export PATH=$PREFIX/bin:\$PATH" > setup-chicken4x.sh
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> setup-chicken4x.sh






echo PATH=$PATH
echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH

if ! [[ -e $PREFIX/bin/csi ]]; then
    tar xfvz tgz/$chicken_targz
    cd chicken-${CHICKEN_VERSION}
    # make PLATFORM=linux PREFIX=$PREFIX spotless
    make PLATFORM=linux PREFIX=$PREFIX
    make PLATFORM=linux PREFIX=$PREFIX install
    cd $BUILDHOME
fi
































# Some eggs are quoted since they are reserved to Bash
# for f in matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5; do
# $CHICKEN_INSTALL $PROX -keep-installed matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client spiffy-request-vars md5 message-digest http-client spiffy-directory-listing

$CHICKEN_INSTALL $PROX -keep-installed matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client spiffy-request-vars s md5 message-digest piffy-directory-listing ssax sxml-serializer sxml-modifications logpro
#   if ! [[ -e $PREFIX/lib/chicken/6/$f.so ]];then
#     $CHICKEN_INSTALL $PROX $f
#     # $CHICKEN_INSTALL -deploy -prefix $DEPLOYTARG $PROX $f
#   else
#     echo Skipping install of egg $f as it is already installed
#   fi
# done





cd $BUILDHOME

for a in `ls */*.meta|cut -f1 -d/` ; do 

    echo $a
    (cd $a;$CHICKEN_INSTALL)
done

export LIBPATH=$PREFIX/lib$ADDITIONAL_LIBPATH
export LD_LIBRARY_PATH=$LIBPATH

export SQLITE3_VERSION=3071401
echo Install sqlite3
sqlite3_tgz=sqlite-autoconf-$SQLITE3_VERSION.tar.gz
if ! [[ -e tgz/$sqlite3_tgz ]]; then
    wget http://www.sqlite.org/$sqlite3_tgz
    mv $sqlite3_tgz tgz
fi


if ! [[ -e $PREFIX/bin/sqlite3 ]] ; then


    if [[ -e tgz/sqlite-autoconf-$SQLITE3_VERSION.tar.gz ]]; then
	tar xfz tgz/sqlite-autoconf-$SQLITE3_VERSION.tar.gz 

	(cd sqlite-autoconf-$SQLITE3_VERSION;./configure --prefix=$PREFIX;make;make install)
	# CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib" $CHICKEN_INSTALL -prefix $DEPLOYTARG -deploy $PROX sqlite3
	CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib" $CHICKEN_INSTALL $PROX sqlite3



    fi





fi


# $CHICKEN_INSTALL $PROX sqlite3

# IUP versions
if [[ x$USEOLDIUP == "x" ]];then
  CDVER=5.7
  IUPVER=3.8
  IMVER=3.8
else
  CDVER=5.7
  IUPVER=3.8
  IMVER=3.8
fi






if [[ `uname -a | grep x86_64` == "" ]]; then 
    export ARCHSIZE=''
else
    export ARCHSIZE=64_
fi
    # export files="cd-5.4.1_Linux${KTYPE}_lib.tar.gz im-3.6.3_Linux${KTYPE}_lib.tar.gz iup-3.5_Linux${KTYPE}_lib.tar.gz"
if [[ x$USEOLDIUP == "x" ]];then
   export files="cd-${CDVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz im-${IMVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz iup-${IUPVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz"
else
   echo WARNING: Using old IUP libraries
   export files="cd-5.4.1_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz im-3.6.3_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz iup-3.5_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz"
fi


mkdir -p $PREFIX/iuplib

for a in `echo $files` ; do
    if ! [[ -e tgz/$a ]] ; then

	wget http://www.kiatoa.com/matt/iup/$a
        mv $a tgz/$a
    fi
    echo Untarring tgz/$a into $BUILDHOME/lib

    (cd $PREFIX/lib;tar xfvz $BUILDHOME/tgz/$a;mv include/* ../include)
    # (cd $DEPLOYTARG;tar xfvz $BUILDHOME/$a)
done





# ffcall obtained from:
# cvs -z3 -d:pserver:anonymous@cvs.savannah.gnu.org:/sources/libffcall co ffcall 

if ! [[ -e tgz/ffcall.tar.gz ]] ; then
    wget http://www.kiatoa.com/matt/iup/ffcall.tar.gz 
    mv ffcall.tar.gz tgz
fi

tar xfvz tgz/ffcall.tar.gz

cd ffcall

./configure --prefix=$PREFIX --enable-shared
make
make install







































cd $BUILDHOME
export CSCLIBS=`echo $LD_LIBRARY_PATH | sed 's/:/ -L/g'`





CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web iup


# CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web -deploy -prefix $DEPLOYTARG iup
# iup:1.0.2 
CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks canvas-draw
# CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -deploy -prefix $DEPLOYTARG canvas-draw

# NB// Removed bunch of zmq compiling tricks. Look at older versions of this file if you need to recreate...

cd $BUILDHOME  

# git clone https://bitbucket.org/DerGuteMoritz/zmq/commits/branch/3.2 zmq-3.2
# cd zmq-3.2
# chicken-install
#
# cd $BUILDHOME

## WEBKIT=WebKit-r131972
## if  ! [[ -e ${WEBKIT}.tar.bz2 ]] ; then
##    #    http://builds.nightly.webkit.org/files/trunk/src/WebKit-r131972.tar.bz2
##    wget http://builds.nightly.webkit.org/files/trunk/src/${WEBKIT}.tar.bz2
## fi
## 
## if [[ x$only_it_worked == $I_wish ]] ;then
##    if [[ -e ${WEBKIT}.tar.bz2 ]] ; then
##       tar xfj ${WEBKIT}.tar.bz2
##       cd $WEBKIT
##       ./autogen.sh
##       ./configure --prefix=$PREFIX
##       make
##       make install
##    fi
## fi
## 
## cd $BUILHOME

# export CD_REL=d704525ebe1c6d08
# if ! [[ -e  Canvas_Draw-$CD_REL.zip ]]; then
#     wget http://www.kiatoa.com/matt/iup/Canvas_Draw-$CD_REL.zip
# fi
# 
# unzip -o Canvas_Draw-$CD_REL.zip
# 
# cd "Canvas Draw-$CD_REL/chicken"
# CSC_OPTIONS="-I$PREFIX/include -L$LIBPATH" $CHICKEN_INSTALL $PROX -D no-library-checks

echo You may need to add $LD_LIBRARY_PATH to your LD_LIBRARY_PATH variable, a setup-chicken4x.sh 
echo file can be found in the current directory which should work for setting up to run chicken4x

echo Testing iup
$PREFIX/bin/csi -b -eval '(use iup)(print "Success")'









|








|
|














|


>
|
|
>
>
>
>
>





|






>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




>
|
<
|
|
|
|
|
|
|
>
>
>
>
|

<
>
|
|
<
|
<
<
|
<
|
<
<
<
|
|
>

|
>
>
|
|
>
|
|
|
>
>
>
|
>
>
>
>
>
|
>


|
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>








|


|

>


>


>
|
|


>
|


|
>
>
>
>


|
|
|
<
<
|
|
|
|
>
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



>
>
>
>
>
|
>
>


|


<
<


<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<





>
>
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

184
185
186
187
188
189
190
191
192
193
194
195
196

197
198
199

200


201

202



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282


283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344


345
346



































347
348
349
350
351
352
353
else
  export http_proxy=http://$proxy
  export PROX="-proxy $proxy"
fi

if [[ $KTYPE == "" ]]; then
  echo 'Using KTYPE=26'
  export KTYPE=26g4
else
  echo Using KTYPE=$KTYPE
fi

# Put all the downloaded tar files in tgz
mkdir -p tgz

# http://code.call-cc.org/releases/4.8.0/chicken-4.8.0.5.tar.gz
export CHICKEN_VERSION=4.11.0
export CHICKEN_BASEVER=4.11.0
chicken_targz=chicken-${CHICKEN_VERSION}.tar.gz
if ! [[ -e tgz/$chicken_targz ]]; then 
    wget http://code.call-cc.org/releases/${CHICKEN_BASEVER}/${chicken_targz}
    mv $chicken_targz tgz
fi 

BUILDHOME=$PWD
DEPLOYTARG=$BUILDHOME/deploy

if [[ $PREFIX == "" ]]; then
   PREFIX=$PWD/inst
fi

export PATH=$PREFIX/bin:$PATH
export LIBPATH=$PREFIX/lib:$PREFIX/lib64:$ADDITIONAL_LIBPATH
export LD_LIBRARY_PATH=$LIBPATH
export CHICKEN_INSTALL=$PREFIX/bin/chicken-install
mkdir -p $PREFIX
echo "export PATH=$PREFIX/bin:\$PATH" > $PREFIX/setup-chicken4x.sh
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\$LD_LIBRARY_PATH" >> $PREFIX/setup-chicken4x.sh
echo "export CHICKEN_DOC_PAGER=cat" >> $PREFIX/setup-chicken4x.sh

echo "setenv PATH $PREFIX/bin:\$PATH" > $PREFIX/setup-chicken4x.csh
echo "setenv LD_LIBRARY_PATH $LD_LIBRARY_PATH:\$LD_LIBRARY_PATH" >> $PREFIX/setup-chicken4x.csh
echo "setenv CHICKEN_DOC_PAGER cat" >> $PREFIX/setup-chicken4x.csh

echo PATH=$PATH
echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH

if ! [[ -e $PREFIX/bin/csi ]]; then
    tar xfz tgz/$chicken_targz
    cd chicken-${CHICKEN_VERSION}
    # make PLATFORM=linux PREFIX=$PREFIX spotless
    make PLATFORM=linux PREFIX=$PREFIX
    make PLATFORM=linux PREFIX=$PREFIX install
    cd $BUILDHOME
fi
cd $BUILDHOME
#wget --no-check-certificate https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz 
#mv 1.0.0 1.0.0.tar.gz
# if ! [[ -e $PREFIX/lib64/libnanomsg.so.1.0.0 ]]; then
#         wget --no-check-certificate https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz 
#         mv 1.0.0 1.0.0.tar.gz
# 	tar xf 1.0.0.tar.gz 
# 	cd nanomsg-1.0.0
# 	./configure --prefix=$PREFIX
# 	make
# 	make install
# fi
# cd $BUILDHOME

export SQLITE3_VERSION=3090200
if ! [[ -e $PREFIX/bin/sqlite3 ]]; then
	echo Install sqlite3
	sqlite3_tgz=sqlite-autoconf-$SQLITE3_VERSION.tar.gz
	if ! [[ -e tgz/$sqlite3_tgz ]]; then
	    wget http://www.sqlite.org/2015/$sqlite3_tgz
	    mv $sqlite3_tgz tgz
	fi

	if ! [[ -e $PREFIX/bin/sqlite3 ]] ; then
	    if [[ -e tgz/sqlite-autoconf-$SQLITE3_VERSION.tar.gz ]]; then
		tar xfz tgz/sqlite-autoconf-$SQLITE3_VERSION.tar.gz 
		(cd sqlite-autoconf-$SQLITE3_VERSION;./configure --prefix=$PREFIX;make;make install)
	    fi
	fi
fi
cd $BUILDHOME

# Some eggs are quoted since they are reserved to Bash
# for f in matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5; do
# $CHICKEN_INSTALL $PROX -keep-installed matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client spiffy-request-vars md5 message-digest http-client spiffy-directory-listing
for egg in matchable readline apropos base64 regex-literals format "regex-case" "test" \
	coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo \

	tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client \
	spiffy-request-vars s md5 message-digest spiffy-directory-listing ssax sxml-serializer \
	sxml-modifications logpro z3 call-with-environment-variables \
	pathname-expand typed-records simple-exceptions numbers crypt parley srfi-42 \
	alist-lib ansi-escape-sequences args basic-sequences bindings chicken-doc chicken-doc-cmd \
	cock condition-utils debug define-record-and-printer easyffi easyffi-base \
	expand-full ezxdisp filepath foof-loop ini-file irc lalr lazy-seq \
	locale locale-builtin locale-categories locale-components locale-current locale-posix \
	locale-timezone loops low-level-macros procedural-macros refdb rfc3339 scsh-process \
	sexp-diff sha1 shell slice srfi-101 srfi-19 srfi-19-core srfi-19-date srfi-19-io \
	srfi-19-period srfi-19-support srfi-19-time srfi-19-timezone srfi-29 srfi-37 srfi-78 syslog \
	udp uuid uuid-lib zlib


do
	echo "Installing $egg"
	$CHICKEN_INSTALL $PROX -keep-installed $egg

	#$CHICKEN_INSTALL $PROX $egg


	if [ $? -ne 0 ]; then

		echo "$egg failed to install"



		exit 1
	fi
done

if [[ -e `which mysql_config` ]]; then
  $CHICKEN_INSTALL $PROX -keep-installed mysql-client
fi

for egg in "sqlite3" sql-de-lite # nanomsg
do
	echo "Installing $egg"
	CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib -L$PREFIX/lib64"  $CHICKEN_INSTALL $PROX -keep-installed $egg
	#CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib -L$PREFIX/lib64"  $CHICKEN_INSTALL $PROX $egg
	if [ $? -ne 0 ]; then
		echo "$egg failed to install"
		exit 1
	fi
done
cd $BUILDHOME
cd `$PREFIX/bin/csi -p '(chicken-home)'`
curl http://3e8.org/pub/chicken-doc/chicken-doc-repo.tgz | tar zx
cd $BUILDHOME



# $CHICKEN_INSTALL $PROX sqlite3
cd $BUILDHOME
# # IUP versions
# if [[ x$USEOLDIUP == "x" ]];then
#   CDVER=5.10
#   IUPVER=3.17
#   IMVER=3.11
# else
#   CDVER=5.10
#   IUPVER=3.17
#   IMVER=3.11
# fi
# if [[ x$KTYPE == "x24g3" ]];then
#   CDVER=5.4.1
#   IUPVER=3.5
#   IMVER=3.6.3
# fi

if [[ `uname -a | grep x86_64` == "" ]]; then 
    export ARCHSIZE=''
else
    export ARCHSIZE=64_
fi
    # export files="cd-5.4.1_Linux${KTYPE}_lib.tar.gz im-3.6.3_Linux${KTYPE}_lib.tar.gz iup-3.5_Linux${KTYPE}_lib.tar.gz"
if [[ x$USEOLDIUP == "x" ]];then
   export files="cd/cd-${CDVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz im/im-${IMVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz iup/iup-${IUPVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz"
else
   echo WARNING: Using old IUP libraries
   export files="cd/cd-5.4.1_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz im/im-3.6.3_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz iup/iup-3.5_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz"
fi
echo $files

mkdir -p $PREFIX/iuplib
mkdir -p iup/
for a in `echo $files` ; do
    if ! [[ -e tgz/$a ]] ; then
        echo wget -c -O tgz/$a http://www.kiatoa.com/matt/chicken-build/$a
	wget -c http://www.kiatoa.com/matt/chicken-build/$a
        mv `echo $a | cut -d'/' -f2` tgz/
    fi
    echo Untarring tgz/$a into $BUILDHOME/lib
    tar -xzf tgz/`echo $a | cut -d'/' -f2` -C iup/
    #(cd $PREFIX/lib;tar xfvz $BUILDHOME/tgz/$a;mv include/* ../include)
    # (cd $DEPLOYTARG;tar xfvz $BUILDHOME/$a)
done
cp iup/include/* $PREFIX/include/
cp iup/*.so $PREFIX/lib/
cp iup/*.a $PREFIX/lib/
cp iup/ftgl/lib/*/* $PREFIX/lib/
cd $BUILDHOME
# ffcall obtained from:
# cvs -z3 -d:pserver:anonymous@cvs.savannah.gnu.org:/sources/libffcall co ffcall 
#exit
if ! [[ -e $PREFIX/include/callback.h ]] ; then
	#fossil clone http://www.kiatoa.com/fossils/ffcall ffcall.fossil


	wget -c -O ffcall.tar.gz 'http://www.kiatoa.com/fossils/ffcall/tarball?name=ffcall&uuid=trunk'
	tar -xzf ffcall.tar.gz
	#mkdir -p ffcall
	cd ffcall
	#fossil open ../ffcall.fossil
	./configure --prefix=$PREFIX --enable-shared
	make CC="gcc -fPIC"
	make install
fi
cd $BUILDHOME
#wget -c -O opensrc.tar.gz 'http://www.kiatoa.com/fossils/opensrc/tarball?name=opensrc&uuid=trunk'
# Not working due to login problems.
if ! [[ -e $PREFIX/bin/hs ]] ; then
	#fossil clone http://www.kiatoa.com/fossils/opensrc opensrc.fossil
	#mkdir -p opensrc
	wget -c -O opensrc.tar.gz 'http://www.kiatoa.com/fossils/opensrc/tarball?name=opensrc&uuid=trunk'
	tar -xzf opensrc.tar.gz
	cd opensrc
	#fossil open ../opensrc.fossil
	cd histstore
	$PREFIX/bin/csc histstore.scm -o hs 
	cp -f hs $PREFIX/bin/hs 
	cd ../mutils
	$PREFIX/bin/chicken-install
	cd ../dbi 
	$PREFIX/bin/chicken-install
	cd ../margs
	$PREFIX/bin/chicken-install
fi
cd $BUILDHOME

if ! [[ -e $PREFIX/bin/stmlrun ]] ; then
	#fossil clone http://www.kiatoa.com/fossils/stml stml.fossil
	wget -c -O stml.tar.gz 'http://www.kiatoa.com/fossils/stml/tarball?name=stml&uuid=trunk'
	tar -xzf stml.tar.gz
	cd stml
	#fossil open ../stml.fossil
	cp install.cfg.template install.cfg
	echo "TARGDIR=$PREFIX/bin" > install.cfg
	echo "LOGDIR=/tmp/stmlrun" >> install.cfg
	echo "SQLITE3=$PREFIX/bin/sqlite3" >> install.cfg
	cp requirements.scm.template requirements.scm
	which csc
	make clean
	CSCOPTS="-C -fPIC" make
fi

cd $BUILDHOME
export CSCLIBS=`echo $LD_LIBRARY_PATH | sed 's/:/ -L/g'`
IUPEGGVER='iup'
if [[ $IUPVER == "3.5" ]]; then
  IUPEGGVER='iup:1.2.1'
fi

#CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web iup
CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web $IUPEGGVER

# CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web -deploy -prefix $DEPLOYTARG iup
# iup:1.0.2 
CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib" $CHICKEN_INSTALL $PROX -D no-library-checks canvas-draw
# CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -deploy -prefix $DEPLOYTARG canvas-draw



cd $BUILDHOME  




































echo You may need to add $LD_LIBRARY_PATH to your LD_LIBRARY_PATH variable, a setup-chicken4x.sh 
echo file can be found in the current directory which should work for setting up to run chicken4x

echo Testing iup
$PREFIX/bin/csi -b -eval '(use iup)(print "Success")'


Added utils/viewscreen version [dee289c6f4].







































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/bin/bash

if ! type screen &> /dev/null;then
  xterm -geometry 180x20 -e "$*;echo Press any key to continue;bash -c 'read -n 1 -s'" &
  exit
fi

if [[ $(screen -list | egrep 'Attached|Detached'|awk '{print $1}') == "" ]];then
    # echo "No screen found for displaying to. Run \"screen\" in an xterm"
    # exit 1
    xterm -e screen -e^ff &
    sleep 1
    screen -X hardstatus off
    screen -X hardstatus alwayslastline
    screen -X hardstatus string '%{= kG}[ %{G}%H %{g}][%= %{= kw}%?%-Lw%?%{r}(%{W}%n*%f%t%?(%u)%?%{r})%{w}%?%+Lw%?%?%= %{g}][%{B} %m-%d %{W} %c %{g}]'
fi

cmd="cd $PWD;$*"
screen -X screen bash -c "$cmd;echo \"Press any key to continue, ctrl-f <space> to see other windows\";bash -c 'read -n 1 -s'"

Modified vg.scm from [4b3f71521e] to [2067ad836c].

366
367
368
369
370
371
372


373
374
375
376
377






378
379
380
381
382
383
384
(define (vg:rgb->number r g b #!key (a 0))
  (bitwise-ior
    (arithmetic-shift a 24)
    (arithmetic-shift r 16)
    (arithmetic-shift g 8)
    b))



(define (vg:generate-color)
  (vg:rgb->number (random 255)
                  (random 255)
                  (random 255)))
  ;;(vg:rgb->number 0 0 0))







(define (vg:iup-color->number iup-color)
  (apply vg:rgb->number (map string->number (string-split iup-color))))

;;======================================================================
;; graphing
;;======================================================================







>
>




|
>
>
>
>
>
>







366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
(define (vg:rgb->number r g b #!key (a 0))
  (bitwise-ior
    (arithmetic-shift a 24)
    (arithmetic-shift r 16)
    (arithmetic-shift g 8)
    b))

;; Obsolete function
;;
(define (vg:generate-color)
  (vg:rgb->number (random 255)
                  (random 255)
                  (random 255)))

;; Need to return a string of random iup-color for graph
;;
(define (vg:generate-color-rgb)
  (conc (number->string (random 255)) " "
        (number->string (random 255)) " "
        (number->string (random 255))))

(define (vg:iup-color->number iup-color)
  (apply vg:rgb->number (map string->number (string-split iup-color))))

;;======================================================================
;; graphing
;;======================================================================