Changes In Branch v1.60
Through [9673fdc495]
Excluding Merge-Ins
This is equivalent to a diff from
b541c8f3a1
to 9673fdc495
Modified Makefile
from [660e49132a]
to [9b7ed373de].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
-
+
-
+
+
+
|
# make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)'
PREFIX=$(PWD)
CSCOPTS=
INSTALL=install
SRCFILES = common.scm items.scm launch.scm \
ods.scm runconfig.scm server.scm configf.scm \
db.scm keys.scm margs.scm megatest-version.scm \
process.scm runs.scm tasks.scm tests.scm genexample.scm \
fs-transport.scm http-transport.scm \
http-transport.scm nmsg-transport.scm filedb.scm \
client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm \
tree.scm ezsteps.scm lock-queue.scm sdb.scm
tree.scm ezsteps.scm lock-queue.scm sdb.scm \
rmt.scm api.scm tdb.scm rpc-transport.scm \
portlogger.scm archive.scm
# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \
json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \
spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3
|
︙ | | |
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
-
+
-
-
-
-
+
-
-
+
+
+
-
+
+
+
-
-
-
-
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
-
-
+
+
+
+
|
CKPATH=$(shell dirname $(shell dirname $(CSIPATH)))
# ARCHSTR=$(shell uname -m)_$(shell uname -r)
# BASH_MACHTYPE=$(shell bash -c "echo \$$MACHTYPE")
# ARCHSTR=$(BASH_MACHTYPE)_$(shell lsb_release -sr)
ARCHSTR=$(shell lsb_release -sr)
# ARCHSTR=$(shell bash -c "echo \$$MACHTYPE")
all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard txtdb
all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard
refdb : txtdb/txtdb.scm
csc -I txtdb txtdb/txtdb.scm -o refdb
mtest: $(OFILES) megatest.o
mtest: $(OFILES) megatest.o readline-fix.scm
csc $(CSCOPTS) $(OFILES) megatest.o -o mtest
dboard : $(OFILES) $(GOFILES) dashboard.scm
csc $(OFILES) dashboard.scm $(GOFILES) -o dboard
# newdboard : newdashboard.scm $(OFILES) $(GOFILES)
# csc $(OFILES) $(GOFILES) newdashboard.scm -o newdboard
ndboard : newdashboard.scm $(OFILES) $(GOFILES)
csc $(OFILES) $(GOFILES) newdashboard.scm -o ndboard
#
# $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm
# csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl
# Special dependencies for the includes
tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o megatest.o : db_records.scm
tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o \
archive.o megatest.o : db_records.scm
tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o : run_records.scm
db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm
tests.o tasks.o dashboard-tasks.o : task_records.scm
runs.o : test_records.scm
megatest.o : megatest-fossil-hash.scm
client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm zmq-transport.scm : common_records.scm rpc-transport.scm
# Temporary while transitioning to new routine
# runs.o : run-tests-queue-classic.scm run-tests-queue-new.scm
megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm
echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new
if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi
$(OFILES) $(GOFILES) : common_records.scm
%.o : %.scm
csc $(CSCOPTS) -c $<
$(PREFIX)/bin/.$(ARCHSTR)/mtest : mtest
@echo Installing to PREFIX=$(PREFIX)
$(INSTALL) mtest $(PREFIX)/bin/.$(ARCHSTR)/mtest
utils/mk_wrapper $(PREFIX) mtest $(PREFIX)/bin/megatest
chmod a+x $(PREFIX)/bin/megatest
# $(PREFIX)/bin/newdboard : newdboard
# $(INSTALL) newdboard $(PREFIX)/bin/newdboard
# utils/mk_wrapper $(PREFIX) newdboard $(PREFIX)/bin/newdashboard
# chmod a+x $(PREFIX)/bin/newdashboard
$(PREFIX)/bin/.$(ARCHSTR)/ndboard : ndboard
$(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard
$(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard
utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard
chmod a+x $(PREFIX)/bin/newdashboard
$(HELPERS) : utils/mt_*
# $(HELPERS) : utils/%
# $(INSTALL) $< $@
# chmod a+x $@
$(PREFIX)/bin/mt_laststep : utils/mt_laststep
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/mt_runstep : utils/mt_runstep
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/mt_ezstep : utils/mt_ezstep
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/mt_xterm : utils/mt_xterm
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/nbfake : utils/nbfake
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/nbfind : utils/nbfind
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/loadrunner : utils/loadrunner
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/refdb : refdb
$(INSTALL) $< $@
chmod a+x $@
# $(PREFIX)/bin/refdb : refdb
# $(INSTALL) $< $@
# chmod a+x $@
deploytarg/nbfake : utils/nbfake
$(INSTALL) $< $@
chmod a+x $@
deploytarg/nbfind : utils/nbfind
$(INSTALL) $< $@
chmod a+x $@
# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES)
utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
chmod a+x $(PREFIX)/bin/dashboard
$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/refdb $(PREFIX)/bin/mt_xterm
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
$(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/mt_xterm \
$(PREFIX)/bin/newdashboard
$(PREFIX)/bin/.$(ARCHSTR) :
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)
test: tests/tests.scm
cd tests;csi -I .. -b -n tests.scm
|
︙ | | |
161
162
163
164
165
166
167
168
169
170
171
172
173
|
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
|
csc -deploy $(CSCOPTS) $(OFILES) megatest.scm -o deploytarg
mv deploytarg/deploytarg deploytarg/mtest
deploytarg/dboard : $(OFILES) $(GOFILES) dashboard.scm deploytarg/apropos.so
csc -deploy $(OFILES) $(GOFILES) dashboard.scm -o deploytarg
mv deploytarg/deploytarg deploytarg/dboard
DATASHAREO=configf.o common.o process.o
datashare-testing/datashare : datashare.scm $(DATASHAREO)
csc datashare.scm $(DATASHAREO) -o datashare-testing/datashare
# DATASHAREO=configf.o common.o process.o tree.o dcommon.o margs.o launch.o gutils.o db.o synchash.o server.o \
# megatest-version.o tdb.o ods.o mt.o keys.o
datashare-testing/sd : datashare.scm $(OFILES)
csc datashare.scm $(OFILES) -o datashare-testing/sd
datashare : datashare-testing/datashare
./datashare-testing/datashare
sd : datashare-testing/sd
mkdir -p /tmp/$(USER)/datashare/disk1 /tmp/$(USER)/basepath
xterm : sd
(export BASEPATH=/tmp/$(USER)/basepath ; export PATH="$(PWD)/datashare-testing:$(PATH)" ; xterm &)
# "(define (toplevel-command . a) #f)"
readline-fix.scm :
if egrep 'version.*3.0' $(shell dirname $(shell dirname $(shell which csi)))/lib/chicken/7/readline.setup-info;then \
echo "(use-legacy-bindings)" > readline-fix.scm; \
else \
echo "" > readline-fix.scm;\
fi
|
Modified NOTES
from [f5959ca38f]
to [8d1d854887].
1
2
3
4
5
6
7
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
======================================================================
Try writing to in-memory db and every 2-5 seconds syncing to megatest.db
======================================================================
First, how much time will it take to write back the changes:
1. Get the run table
(define (get-all db)(let ((res '()))(for-each-row (lambda (a . b)(set! res (cons (apply vector a b) res))) db "SELECT * FROM tests;") res))
(define tdata (let ((start (current-milliseconds))(res (get-all *db*)))(print (- (current-milliseconds) start))res))
Result ranges from 34ms to 89ms but mostly around 40ms for 623 records on moosefs
Projecting to 15000 records:
Slow 2 seconds to read all
Median 1 second to read all
This seems like it would work with an update period of 2-5 seconds
TODO
----
1. open-db opens in-memory db and megatest.db, put handles in *memdb* and *db*, *memdb* is < run-id dbh >
2. Server is part of runtests
a. server start cycle - adapt to per run-id
i. states; starting, started, stopping, stopped
b. turn off write coalesing
3. Calls to -runtests, -remove-runs etc.
a. Might talk to running server if run specific
b. Can talk to megatest.db but not a generally good idea
c. Can start a runserver
4. Dashboard is fine except for writes?
======================================================================
Routines to convert for runs.scm
cdb:remote-run db:register-run
cdb:delete-tests-in-state *runremote*
cdb:get-test-info-by-id *runremote*
cdb:remote-run db:delete-old-deleted-test-records
cdb:remote-run db:delete-run
cdb:remote-run db:delete-test-records
cdb:remote-run db:delete-tests-for-run
cdb:remote-run db:find-and-mark-incomplete
cdb:remote-run db:get-count-tests-running
cdb:remote-run db:get-count-tests-running-in-jobgroup
cdb:remote-run db:get-keys
cdb:remote-run db:get-run-info
cdb:remote-run db:get-run-key-val
cdb:remote-run db:get-run-name-from-id
cdb:remote-run db:get-steps-for-test
cdb:remote-run db:get-test-id-cached
cdb:remote-run db:get-tests-for-runs-mindata
cdb:remote-run db:lock/unlock-run
cdb:remote-run db:set-sync
cdb:remote-run db:set-tests-state-status
cdb:remote-run db:set-var
cdb:remote-run db:testmeta-add-record
cdb:remote-run db:testmeta-get-record
cdb:remote-run db:testmeta-update-field
cdb:remote-run db:update-run-event_time
cdb:remote-run instead
cdb:remote-run server:start
cdb:remote-run test:get-matching-previous-test-run-records
cdb:tests-register-test *runremote*
(define (runs:shrink-can-run-more-tests-count) ;; the db is a dummy var so we can use cdb:remote-run
======================================================================
[87cbe68f31]
[be405e8e2e]
# FROM andyjpg on #chicken
(let ((original-exit (exit-handler)))
|
︙ | | |
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
Modified TODO
from [61ddd55e7d]
to [249cc9a526].
1
2
3
4
|
1
2
3
4
5
6
7
8
9
10
11
12
|
+
+
-
-
-
+
+
+
+
+
+
+
+
+
|
TODO
====
1. Confirm that branch transaction-for-sequential-writes content was added to trunk/development
2. Add a host chooser for ssh to launch-tests
3. Try making static executable
Migration to inmem db plus per run db
-------------------------------------
. Re-work the dbstruct data structure?
.. Move main.db to global?
.. [ run-id.db inmemdb last-mod last-read last-sync inuse ]
. Re-work all queries to use run-id to dereference server
. Open main.db directly in calls to -runtests etc. No need to talk remote?
|
Added all-exceptions.ods version [9f8aefbaf1].
cannot compute difference between binary files
Added api.scm version [8963db51e7].