Changes In Branch v1.60
Through [afb9cc1df1]
Excluding Merge-Ins
This is equivalent to a diff from
e89be5432c
to afb9cc1df1
Modified Makefile
from [660e49132a]
to [64fd867d54].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
-
+
-
+
+
|
# make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)'
PREFIX=$(PWD)
CSCOPTS=
INSTALL=install
SRCFILES = common.scm items.scm launch.scm \
ods.scm runconfig.scm server.scm configf.scm \
db.scm keys.scm margs.scm megatest-version.scm \
process.scm runs.scm tasks.scm tests.scm genexample.scm \
fs-transport.scm http-transport.scm \
http-transport.scm filedb.scm \
client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm \
tree.scm ezsteps.scm lock-queue.scm sdb.scm
tree.scm ezsteps.scm lock-queue.scm sdb.scm \
rmt.scm api.scm tdb.scm portlogger.scm
# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \
json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \
spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3
|
︙ | | |
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
-
-
+
+
+
+
-
-
-
-
+
+
+
+
+
+
|
mtest: $(OFILES) megatest.o
csc $(CSCOPTS) $(OFILES) megatest.o -o mtest
dboard : $(OFILES) $(GOFILES) dashboard.scm
csc $(OFILES) dashboard.scm $(GOFILES) -o dboard
# newdboard : newdashboard.scm $(OFILES) $(GOFILES)
# csc $(OFILES) $(GOFILES) newdashboard.scm -o newdboard
ndboard : newdashboard.scm $(OFILES) $(GOFILES)
csc $(OFILES) $(GOFILES) newdashboard.scm -o ndboard
#
# $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm
# csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl
# Special dependencies for the includes
tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o megatest.o : db_records.scm
tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o : run_records.scm
db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm
tests.o tasks.o dashboard-tasks.o : task_records.scm
runs.o : test_records.scm
megatest.o : megatest-fossil-hash.scm
client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm zmq-transport.scm : common_records.scm
# Temporary while transitioning to new routine
# runs.o : run-tests-queue-classic.scm run-tests-queue-new.scm
megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm
echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new
if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi
$(OFILES) $(GOFILES) : common_records.scm
%.o : %.scm
csc $(CSCOPTS) -c $<
$(PREFIX)/bin/.$(ARCHSTR)/mtest : mtest
@echo Installing to PREFIX=$(PREFIX)
$(INSTALL) mtest $(PREFIX)/bin/.$(ARCHSTR)/mtest
utils/mk_wrapper $(PREFIX) mtest $(PREFIX)/bin/megatest
chmod a+x $(PREFIX)/bin/megatest
# $(PREFIX)/bin/newdboard : newdboard
# $(INSTALL) newdboard $(PREFIX)/bin/newdboard
# utils/mk_wrapper $(PREFIX) newdboard $(PREFIX)/bin/newdashboard
# chmod a+x $(PREFIX)/bin/newdashboard
$(PREFIX)/bin/.$(ARCHSTR)/ndboard : ndboard
$(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard
$(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard
utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard
chmod a+x $(PREFIX)/bin/newdashboard
$(HELPERS) : utils/mt_*
$(INSTALL) $< $@
chmod a+x $@
$(PREFIX)/bin/mt_xterm : utils/mt_xterm
$(INSTALL) $< $@
|
︙ | | |
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
-
-
+
+
+
+
|
# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES)
utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
chmod a+x $(PREFIX)/bin/dashboard
$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/refdb $(PREFIX)/bin/mt_xterm
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
$(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/refdb $(PREFIX)/bin/mt_xterm \
$(PREFIX)/bin/newdashboard
$(PREFIX)/bin/.$(ARCHSTR) :
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)
test: tests/tests.scm
cd tests;csi -I .. -b -n tests.scm
|
︙ | | |
161
162
163
164
165
166
167
168
169
170
171
172
173
|
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
|
csc -deploy $(CSCOPTS) $(OFILES) megatest.scm -o deploytarg
mv deploytarg/deploytarg deploytarg/mtest
deploytarg/dboard : $(OFILES) $(GOFILES) dashboard.scm deploytarg/apropos.so
csc -deploy $(OFILES) $(GOFILES) dashboard.scm -o deploytarg
mv deploytarg/deploytarg deploytarg/dboard
DATASHAREO=configf.o common.o process.o
datashare-testing/datashare : datashare.scm $(DATASHAREO)
csc datashare.scm $(DATASHAREO) -o datashare-testing/datashare
# DATASHAREO=configf.o common.o process.o tree.o dcommon.o margs.o launch.o gutils.o db.o synchash.o server.o \
# megatest-version.o tdb.o ods.o mt.o keys.o
datashare-testing/sd : datashare.scm $(OFILES)
csc datashare.scm $(OFILES) -o datashare-testing/sd
datashare : datashare-testing/datashare
./datashare-testing/datashare
sd : datashare-testing/sd
mkdir -p /tmp/$(USER)/datashare/disk1 /tmp/$(USER)/basepath
xterm : sd
(export BASEPATH=/tmp/$(USER)/basepath ; export PATH="$(PWD)/datashare-testing:$(PATH)" ; xterm &)
|
Modified NOTES
from [f5959ca38f]
to [8d1d854887].
1
2
3
4
5
6
7
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
======================================================================
Try writing to in-memory db and every 2-5 seconds syncing to megatest.db
======================================================================
First, how much time will it take to write back the changes:
1. Get the run table
(define (get-all db)(let ((res '()))(for-each-row (lambda (a . b)(set! res (cons (apply vector a b) res))) db "SELECT * FROM tests;") res))
(define tdata (let ((start (current-milliseconds))(res (get-all *db*)))(print (- (current-milliseconds) start))res))
Result ranges from 34ms to 89ms but mostly around 40ms for 623 records on moosefs
Projecting to 15000 records:
Slow 2 seconds to read all
Median 1 second to read all
This seems like it would work with an update period of 2-5 seconds
TODO
----
1. open-db opens in-memory db and megatest.db, put handles in *memdb* and *db*, *memdb* is < run-id dbh >
2. Server is part of runtests
a. server start cycle - adapt to per run-id
i. states; starting, started, stopping, stopped
b. turn off write coalesing
3. Calls to -runtests, -remove-runs etc.
a. Might talk to running server if run specific
b. Can talk to megatest.db but not a generally good idea
c. Can start a runserver
4. Dashboard is fine except for writes?
======================================================================
Routines to convert for runs.scm
cdb:remote-run db:register-run
cdb:delete-tests-in-state *runremote*
cdb:get-test-info-by-id *runremote*
cdb:remote-run db:delete-old-deleted-test-records
cdb:remote-run db:delete-run
cdb:remote-run db:delete-test-records
cdb:remote-run db:delete-tests-for-run
cdb:remote-run db:find-and-mark-incomplete
cdb:remote-run db:get-count-tests-running
cdb:remote-run db:get-count-tests-running-in-jobgroup
cdb:remote-run db:get-keys
cdb:remote-run db:get-run-info
cdb:remote-run db:get-run-key-val
cdb:remote-run db:get-run-name-from-id
cdb:remote-run db:get-steps-for-test
cdb:remote-run db:get-test-id-cached
cdb:remote-run db:get-tests-for-runs-mindata
cdb:remote-run db:lock/unlock-run
cdb:remote-run db:set-sync
cdb:remote-run db:set-tests-state-status
cdb:remote-run db:set-var
cdb:remote-run db:testmeta-add-record
cdb:remote-run db:testmeta-get-record
cdb:remote-run db:testmeta-update-field
cdb:remote-run db:update-run-event_time
cdb:remote-run instead
cdb:remote-run server:start
cdb:remote-run test:get-matching-previous-test-run-records
cdb:tests-register-test *runremote*
(define (runs:shrink-can-run-more-tests-count) ;; the db is a dummy var so we can use cdb:remote-run
======================================================================
[87cbe68f31]
[be405e8e2e]
# FROM andyjpg on #chicken
(let ((original-exit (exit-handler)))
|
︙ | | |
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
Modified TODO
from [61ddd55e7d]
to [249cc9a526].
1
2
3
4
|
1
2
3
4
5
6
7
8
9
10
11
12
|
+
+
-
-
-
+
+
+
+
+
+
+
+
+
|
TODO
====
1. Confirm that branch transaction-for-sequential-writes content was added to trunk/development
2. Add a host chooser for ssh to launch-tests
3. Try making static executable
Migration to inmem db plus per run db
-------------------------------------
. Re-work the dbstruct data structure?
.. Move main.db to global?
.. [ run-id.db inmemdb last-mod last-read last-sync inuse ]
. Re-work all queries to use run-id to dereference server
. Open main.db directly in calls to -runtests etc. No need to talk remote?
|
Added api.scm version [a688529701].