Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -1,14 +1,16 @@ # make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)' +# rm .o ; make install CSCOPTS='-profile' ; ... ; chicken-profile | less + PREFIX=$(PWD) CSCOPTS= INSTALL=install SRCFILES = common.scm items.scm launch.scm \ ods.scm runconfig.scm server.scm configf.scm \ db.scm keys.scm margs.scm megatest-version.scm \ process.scm runs.scm tasks.scm tests.scm genexample.scm \ - http-transport.scm nmsg-transport.scm filedb.scm \ + http-transport.scm filedb.scm \ client.scm synchash.scm daemon.scm mt.scm \ ezsteps.scm lock-queue.scm sdb.scm \ rmt.scm api.scm tdb.scm rpc-transport.scm \ portlogger.scm archive.scm env.scm @@ -33,10 +35,12 @@ # ARCHSTR=$(shell uname -m)_$(shell uname -r) # BASH_MACHTYPE=$(shell bash -c "echo \$$MACHTYPE") # ARCHSTR=$(BASH_MACHTYPE)_$(shell lsb_release -sr) ARCHSTR=$(shell lsb_release -sr) # ARCHSTR=$(shell bash -c "echo \$$MACHTYPE") + +PNGFILES = $(shell cd docs/manual;ls *png) all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtest: $(OFILES) readline-fix.scm megatest.o csc $(CSCOPTS) $(OFILES) megatest.o -o mtest @@ -45,12 +49,20 @@ csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o dboard ndboard : newdashboard.scm $(OFILES) $(GOFILES) csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard -multi-dboard : multi-dboard.scm $(OFILES) $(GOFILES) - csc $(CSCOPTS) $(OFILES) $(GOFILES) multi-dboard.scm -o multi-dboard +# install documentation to $(PREFIX)/docs +# DOES NOT REBUILD DOCS +# +$(PREFIX)/share/docs/megatest_manual.html : docs/manual/megatest_manual.html + mkdir -p $(PREFIX)/share/docs + $(INSTALL) docs/manual/megatest_manual.html $(PREFIX)/share/docs/megatest_manual.html + for png in $(PNGFILES);do $(INSTALL) docs/manual/$$png $(PREFIX)/share/docs/$$png;done + +#multi-dboard : multi-dboard.scm $(OFILES) $(GOFILES) +# csc $(CSCOPTS) $(OFILES) $(GOFILES) multi-dboard.scm -o multi-dboard # # $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm # csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl @@ -60,14 +72,14 @@ tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o : run_records.scm db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm tests.o tasks.o dashboard-tasks.o : task_records.scm runs.o : test_records.scm megatest.o : megatest-fossil-hash.scm -client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm rpc-transport.scm +rmt.scm client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm rpc-transport.scm common_records.scm : altdb.scm vg.o dashboard.o : vg_records.scm - +dcommon.o : run_records.scm # Temporary while transitioning to new routine # runs.o : run-tests-queue-classic.scm run-tests-queue-new.scm megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new @@ -89,16 +101,16 @@ $(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard chmod a+x $(PREFIX)/bin/newdashboard -$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard - $(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard +#$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard +# $(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard -$(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard utils/mk_wrapper - utils/mk_wrapper $(PREFIX) mdboard $(PREFIX)/bin/mdboard - chmod a+x $(PREFIX)/bin/mdboard +# $(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard utils/mk_wrapper +# utils/mk_wrapper $(PREFIX) mdboard $(PREFIX)/bin/mdboard +# chmod a+x $(PREFIX)/bin/mdboard # $(HELPERS) : utils/% # $(INSTALL) $< $@ # chmod a+x $@ @@ -119,10 +131,14 @@ chmod a+x $@ $(PREFIX)/bin/nbfake : utils/nbfake $(INSTALL) $< $@ chmod a+x $@ + +$(PREFIX)/bin/viewscreen : utils/viewscreen + $(INSTALL) $< $@ + chmod a+x $@ $(PREFIX)/bin/nbfind : utils/nbfind $(INSTALL) $< $@ chmod a+x $@ @@ -136,25 +152,28 @@ deploytarg/nbfake : utils/nbfake $(INSTALL) $< $@ chmod a+x $@ -deploytarg/nbfind : utils/nbfind +deploytarg/viewscreen : utils/viewscreen $(INSTALL) $< $@ chmod a+x $@ +deploytarg/nbfind : utils/nbfind + $(INSTALL) $< $@ + chmod a+x $@ # install dashboard as dboard so wrapper script can be called dashboard $(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard chmod a+x $(PREFIX)/bin/dashboard $(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \ $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ - $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/mt_xterm \ - $(PREFIX)/bin/newdashboard $(PREFIX)/bin/mdboard + $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ + $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/.$(ARCHSTR) : mkdir -p $(PREFIX)/bin/.$(ARCHSTR) test: tests/tests.scm @@ -192,11 +211,11 @@ # chicken-install -prefix deploytarg -deploy $$i;done # deploytarg/libsqlite3.so : # CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3 -deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/nbfind deploytarg/apropos.so +deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/viewsceen deploytarg/nbfind deploytarg/apropos.so # deploytarg/libiupcd.so : $(CKPATH)/lib/libiupcd.so # for i in iup im cd av call sqlite; do \ # cp $(CKPATH)/lib/lib$$i* deploytarg/ ; \ # done @@ -257,7 +276,7 @@ fi if csi -ne '(use postgresql)';then \ echo "(use postgresql)(hash-table-set! *available-db* 'postgresql #t)" >> altdb.scm;\ fi -portlogger-example : portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o nmsg-transport.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o - csc $(CSCOPTS) portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o nmsg-transport.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o +portlogger-example : portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o + csc $(CSCOPTS) portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o ADDED Makefile.deploy Index: Makefile.deploy ================================================================== --- /dev/null +++ Makefile.deploy @@ -0,0 +1,341 @@ +# make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)' +PREFIX=$(PWD) +CSCOPTS= -deploy +INSTALL=install +SRCFILES = common.scm items.scm launch.scm \ + ods.scm runconfig.scm server.scm configf.scm \ + db.scm keys.scm margs.scm megatest-version.scm \ + process.scm runs.scm tasks.scm tests.scm genexample.scm \ + http-transport.scm filedb.scm \ + client.scm synchash.scm daemon.scm mt.scm \ + ezsteps.scm lock-queue.scm sdb.scm \ + rmt.scm api.scm tdb.scm rpc-transport.scm \ + portlogger.scm archive.scm env.scm + +# Eggs to install (straightforward ones) +EGGS=crypt matchable readline apropos base64 regex-literals format regex-case test coops trace csv \ +dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \ +json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \ +spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3 + +GUISRCF = dashboard-tests.scm dashboard-guimonitor.scm gutils.scm dcommon.scm tree.scm vg.scm + +OFILES = $(SRCFILES:%.scm=%.o) +GOFILES = $(GUISRCF:%.scm=%.o) + +ADTLSCR=mt_laststep mt_runstep mt_ezstep +HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR)) +DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR)) +MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}') + +CSIPATH=$(shell which csi) +CKPATH=$(shell dirname $(shell dirname $(CSIPATH))) +# ARCHSTR=$(shell uname -m)_$(shell uname -r) +# BASH_MACHTYPE=$(shell bash -c "echo \$$MACHTYPE") +# ARCHSTR=$(BASH_MACHTYPE)_$(shell lsb_release -sr) +ARCHSTR=$(shell lsb_release -sr) +# ARCHSTR=$(shell bash -c "echo \$$MACHTYPE") + +PNGFILES = $(shell cd docs/manual;ls *png) + +ARCHSIZE=64_ +IMVER=3.11 +IUPVER=3.17 +KTYPE=26g4 +CDVER=5.10 + +all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard eggs sqlite matt iup + +mtest: $(OFILES) readline-fix.scm megatest.o + mkdir -p $(PREFIX)/deploy + csc $(CSCOPTS) $(OFILES) megatest.o -o $(PREFIX)/deploy/mtest + +eggs: $(PREFIX)/deploy/mtest/fmt.so + +$(PREFIX)/deploy/mtest/fmt.so: + chicken-install -deploy -p $(PREFIX)/deploy/mtest base64 format regex-case simple-exceptions typed-records apropos directory-utils md5 spiffy http-client spiffy-request-vars spiffy-directory-listing posix-extras call-with-environment-variables csv typed-records pathname-expand json crypt dot-locking csv-xml z3 sql-de-lite hostinfo rpc directory-utils md5 spiffy http-client spiffy-request-vars spiffy-directory-listing posix-extras call-with-environment-variables rpc fmt + +sqlite: $(PREFIX)/deploy/mtest/sqlite3.so + +$(PREFIX)/deploy/mtest/sqlite3.so: + wget http://www.sqlite.org/2015/sqlite-autoconf-3090200.tar.gz + tar xfz sqlite-autoconf-3090200.tar.gz + cd sqlite-autoconf-3090200 + cd sqlite-autoconf-3090200 && ./configure --prefix=`realpath $(PREFIX)/deploy/mtest` + cd sqlite-autoconf-3090200 && make + cd sqlite-autoconf-3090200 && make install + CSC_OPTIONS='-I$(PREFIX)/deploy/mtest/include -L$(PREFIX)/deploy/mtest/' chicken-install -deploy -p $(PREFIX)/deploy/mtest sqlite3 + +matt: $(PREFIX)/deploy/mtest/stml.so + +$(PREFIX)/deploy/mtest/stml.so: + wget -c -O stml.tar.gz 'http://www.kiatoa.com/fossils/stml/tarball?name=stml&uuid=trunk' + tar -xzf stml.tar.gz + cd stml && cp install.cfg.template install.cfg + cd stml && echo "TARGDIR=`realpath $(PREFIX)/deploy/mtest`" > install.cfg + cd stml && echo "LOGDIR=/tmp/stmlrun" >> install.cfg + cd stml && echo "SQLITE3=$PREFIX/bin/sqlite3" >> install.cfg + cd stml && cp requirements.scm.template requirements.scm + cd stml && make clean + -cd stml && CSCOPTS="-C -fPIC" make + cd stml && chicken-install -deploy -p $(PREFIX)/deploy/mtest + wget -c -O opensrc.tar.gz 'http://www.kiatoa.com/fossils/opensrc/tarball?name=opensrc&uuid=trunk' + tar -xzf opensrc.tar.gz + cd opensrc/mutils && chicken-install -deploy -p $(PREFIX)/deploy/mtest + cd opensrc/dbi && chicken-install -deploy -p $(PREFIX)/deploy/mtest + cd opensrc/margs && chicken-install -deploy -p $(PREFIX)/deploy/mtest + +iup: $(PREFIX)/deploy/mtest/iup.so + +$(PREFIX)/deploy/mtest/iup.so: + wget -c http://www.kiatoa.com/matt/chicken-build/cd/cd-${CDVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz + wget -c http://www.kiatoa.com/matt/chicken-build/im/im-${IMVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz + wget -c http://www.kiatoa.com/matt/chicken-build/iup/iup-${IUPVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz + tar -xzvf cd-${CDVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz -C $(PREFIX)/deploy/mtest/ + tar -xzvf im-${IMVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz -C $(PREFIX)/deploy/mtest/ + tar -xzvf iup-${IUPVER}_Linux${KTYPE}_${ARCHSIZE}lib.tar.gz -C $(PREFIX)/deploy/mtest/ + cp $(PREFIX)/deploy/mtest/ftgl/lib/*/* $(PREFIX)/deploy/mtest/ + wget -c -O ffcall.tar.gz 'http://www.kiatoa.com/fossils/ffcall/tarball?name=ffcall&uuid=trunk' + tar -xzf ffcall.tar.gz + cd ffcall && ./configure --prefix=`realpath $(PREFIX)/deploy/mtest/` --enable-shared + cd ffcall && make CC="gcc -fPIC" + cd ffcall && make install + CSC_OPTIONS="-I$(PREFIX)/deploy/mtest/include -L$(PREFIX)/deploy/mtest" chicken-install -deploy -p $(PREFIX)/deploy/mtest -D no-library-checks -feature disable-iup-web iup + CSC_OPTIONS="-I$(PREFIX)/deploy/mtest/include -L$(PREFIX)/deploy/mtest" chicken-install -deploy -p $(PREFIX)/deploy/mtest -D no-library-checks canvas-draw + +dboard: $(OFILES) $(GOFILES) dashboard.scm + csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o $(PREFIX)/deploy/mtest/dboard2 + cp $(PREFIX)/deploy/mtest/dboard2/dboard2 $(PREFIX)/deploy/mtest/dboard + +ndboard : newdashboard.scm $(OFILES) $(GOFILES) + csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o $(PREFIX)/deploy/mtest/newdboard + +# install documentation to $(PREFIX)/docs +# DOES NOT REBUILD DOCS +# +$(PREFIX)/share/docs/megatest_manual.html : docs/manual/megatest_manual.html + mkdir -p $(PREFIX)/share/docs + $(INSTALL) docs/manual/megatest_manual.html $(PREFIX)/share/docs/megatest_manual.html + for png in $(PNGFILES);do $(INSTALL) docs/manual/$$png $(PREFIX)/share/docs/$$png;done + +#multi-dboard : multi-dboard.scm $(OFILES) $(GOFILES) +# csc $(CSCOPTS) $(OFILES) $(GOFILES) multi-dboard.scm -o multi-dboard + +# +# $(PREFIX)/bin/revtagfsl : utils/revtagfsl.scm +# csc utils/revtagfsl.scm -o $(PREFIX)/bin/revtagfsl + +# Special dependencies for the includes +tests.o db.o launch.o runs.o dashboard-tests.o dashboard-guimonitor.o dashboard-main.o monitor.o dashboard.o \ +archive.o megatest.o : db_records.scm +tests.o runs.o dashboard.o dashboard-tests.o dashboard-main.o : run_records.scm +db.o ezsteps.o keys.o launch.o megatest.o monitor.o runs-for-ref.o runs.o tests.o : key_records.scm +tests.o tasks.o dashboard-tasks.o : task_records.scm +runs.o : test_records.scm +megatest.o : megatest-fossil-hash.scm +client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm rpc-transport.scm +common_records.scm : altdb.scm +vg.o dashboard.o : vg_records.scm +dcommon.o : run_records.scm +# Temporary while transitioning to new routine +# runs.o : run-tests-queue-classic.scm run-tests-queue-new.scm + +megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm + echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new + if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi + +$(OFILES) $(GOFILES) : common_records.scm + +%.o : %.scm + csc $(CSCOPTS) -c $< + +$(PREFIX)/bin/.$(ARCHSTR)/mtest : mtest utils/mk_wrapper + @echo Installing to PREFIX=$(PREFIX) + $(INSTALL) mtest $(PREFIX)/bin/.$(ARCHSTR)/mtest + utils/mk_wrapper $(PREFIX) mtest $(PREFIX)/bin/megatest + chmod a+x $(PREFIX)/bin/megatest + +$(PREFIX)/bin/.$(ARCHSTR)/ndboard : ndboard + $(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/ndboard + +$(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper + utils/mk_wrapper $(PREFIX) ndboard $(PREFIX)/bin/newdashboard + chmod a+x $(PREFIX)/bin/newdashboard + +#$(PREFIX)/bin/.$(ARCHSTR)/mdboard : multi-dboard +# $(INSTALL) multi-dboard $(PREFIX)/bin/.$(ARCHSTR)/mdboard + +# $(PREFIX)/bin/mdboard : $(PREFIX)/bin/.$(ARCHSTR)/mdboard utils/mk_wrapper +# utils/mk_wrapper $(PREFIX) mdboard $(PREFIX)/bin/mdboard +# chmod a+x $(PREFIX)/bin/mdboard + +# $(HELPERS) : utils/% +# $(INSTALL) $< $@ +# chmod a+x $@ + +$(PREFIX)/bin/mt_laststep : utils/mt_laststep + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/mt_runstep : utils/mt_runstep + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/mt_ezstep : utils/mt_ezstep + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/mt_xterm : utils/mt_xterm + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/nbfake : utils/nbfake + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/viewscreen : utils/viewscreen + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/nbfind : utils/nbfind + $(INSTALL) $< $@ + chmod a+x $@ + +$(PREFIX)/bin/loadrunner : utils/loadrunner + $(INSTALL) $< $@ + chmod a+x $@ + +# $(PREFIX)/bin/refdb : refdb +# $(INSTALL) $< $@ +# chmod a+x $@ + +deploytarg/nbfake : utils/nbfake + $(INSTALL) $< $@ + chmod a+x $@ + +deploytarg/viewscreen : utils/viewscreen + $(INSTALL) $< $@ + chmod a+x $@ + +deploytarg/nbfind : utils/nbfind + $(INSTALL) $< $@ + chmod a+x $@ + +# install dashboard as dboard so wrapper script can be called dashboard +$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper + utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard + chmod a+x $(PREFIX)/bin/dashboard + $(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard + +install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \ + $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ + $(PREFIX)/bin/nbfind $(PREFIX)/bin/loadrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ + $(PREFIX)/share/docs/megatest_manual.html + +$(PREFIX)/bin/.$(ARCHSTR) : + mkdir -p $(PREFIX)/bin/.$(ARCHSTR) + +test: tests/tests.scm + cd tests;csi -I .. -b -n tests.scm + +ext-tests/.fslckout : $(MTQA_FOSSIL) + mkdir -p ext-tests + cd ext-tests;fossil open --nested $(MTQA_FOSSIL) + +$(MTQA_FOSSIL) : + fossil clone https://www.kiatoa.com/fossils/megatest_qa $(MTQA_FOSSIL) + +clean : + rm -f $(OFILES) $(GOFILES) megatest dboard dboard.o megatest.o dashboard.o megatest-fossil-hash.* altdb.scm + +#====================================================================== +# Make the records files +#====================================================================== + +# vg_records.scm : records.sh +# ./records.sh + +#====================================================================== +# Deploy section (not complete yet) +#====================================================================== + +$(DEPLOYHELPERS) : utils/mt_* + $(INSTALL) $< $@ + chmod a+X $@ + +deploytarg/apropos.so : Makefile + chicken-install -p deploytarg -deploy -keep-installed $(EGGS) + +# for i in apropos base64 canvas-draw csv-xml directory-utils dot-locking extras fmt format hostinfo http-client intarweb json md5 message-digest posix posix-extras readline regex regex-case s11n spiffy spiffy-request-vars sqlite3 srfi-1 srfi-18 srfi-69 tcp test uri-common check-errors synch matchable sql-null tcp-server rpc blob-utils string-utils variable-item defstruct uri-generic sendfile opensll openssl lookup-table list-utils stack; do \ +# chicken-install -prefix deploytarg -deploy $$i;done + +# deploytarg/libsqlite3.so : +# CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3 + +deploy : deploytarg/mtest deploytarg/dboard $(DEPLOYHELPERS) deploytarg/nbfake deploytarg/viewsceen deploytarg/nbfind deploytarg/apropos.so + +# deploytarg/libiupcd.so : $(CKPATH)/lib/libiupcd.so +# for i in iup im cd av call sqlite; do \ +# cp $(CKPATH)/lib/lib$$i* deploytarg/ ; \ +# done +# cp $(CKPATH)/include/*.h deploytarg + +# puts deployed megatest in directory "megatest" +deploytarg/mtest : $(OFILES) megatest.o deploytarg/apropos.so + csc -deploy $(CSCOPTS) $(OFILES) megatest.scm -o deploytarg + mv deploytarg/deploytarg deploytarg/mtest + +deploytarg/dboard : $(OFILES) $(GOFILES) dashboard.scm deploytarg/apropos.so + csc -deploy $(OFILES) $(GOFILES) dashboard.scm -o deploytarg + mv deploytarg/deploytarg deploytarg/dboard + +# DATASHAREO=configf.o common.o process.o tree.o dcommon.o margs.o launch.o gutils.o db.o synchash.o server.o \ +# megatest-version.o tdb.o ods.o mt.o keys.o +datashare-testing/sd : datashare.scm $(OFILES) + csc $(CSCOPTS) datashare.scm $(OFILES) -o datashare-testing/sd + +datashare-testing/sdat: sharedat.scm $(OFILES) + csc $(CSCOPTS) sharedat.scm $(OFILES) -o datashare-testing/sdat + +sd : datashare-testing/sd + mkdir -p /tmp/$(USER)/datashare/disk1 /tmp/$(USER)/basepath + +xterm : sd + (export BASEPATH=/tmp/$(USER)/basepath ; export PATH="$(PWD)/datashare-testing:$(PATH)" ; xterm &) + +datashare-testing/spublish : spublish.scm $(OFILES) + csc $(CSCOPTS) spublish.scm $(OFILES) -o datashare-testing/spublish + +datashare-testing/sretrieve : sretrieve.scm megatest-version.o margs.o configf.o process.o + csc $(CSCOPTS) sretrieve.scm megatest-version.o margs.o configf.o process.o -o datashare-testing/sretrieve + +sretrieve/sretrieve : datashare-testing/sretrieve + csc $(CSCOPTS) -deploy -deployed sretrieve.scm megatest-version.o margs.o configf.o process.o + chicken-install -keep-installed $(PROXY) -deploy -prefix sretrieve defstruct srfi-18 format sql-de-lite \ + srfi-1 posix regex regex-case srfi-69 + +# base64 dot-locking \ +# csv-xml z3 + +# "(define (toplevel-command . a) #f)" +# if egrep 'version.*3.0' $(shell dirname $(shell dirname $(shell which csi)))/lib/chicken/7/readline.setup-info;then \ + +readline-fix.scm : + if [[ $(shell chicken-status | grep readline | awk '{print $4}' | cut -d. -f1) -gt 3 ]];then \ + echo "(define *use-new-readline* #f)" > readline-fix.scm; \ + else \ + echo "(define *use-new-readline* #t)" > readline-fix.scm;\ + fi + +altdb.scm : + echo ";; optional alternate db setup" > altdb.scm + echo "(define *available-db* (make-hash-table))" >> altdb.scm + if csi -ne '(use mysql-client)';then \ + echo "(use mysql-client)(hash-table-set! *available-db* 'mysql #t)" >> altdb.scm; \ + fi + if csi -ne '(use postgresql)';then \ + echo "(use postgresql)(hash-table-set! *available-db* 'postgresql #t)" >> altdb.scm;\ + fi + +portlogger-example : portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o + csc $(CSCOPTS) portlogger-example.scm api.o archive.o client.o common.o configf.o daemon.o dashboard-tests.o db.o dcommon.o ezsteps.o filedb.o genexample.o gutils.o http-transport.o items.o keys.o launch.o lock-queue.o margs.o megatest-version.o mt.o ods.o portlogger.o process.o rmt.o rpc-transport.o runconfig.o runs.o sdb.o server.o synchash.o tasks.o tdb.o tests.o tree.o Index: NOTES ================================================================== --- NOTES +++ NOTES @@ -1,13 +1,19 @@ +===================================================================== +NOTES from looking at branch v1.62-rpc +===================================================================== + +*last-db-access* or *db-last-access* ==> which is it to be? +seen in singletest: ERROR: Unrecognised arguments: :first_err This is the first error ====================================================================== New way of launching needed to accomodate different target hosttypes for items ====================================================================== [flavors] -general ssh #{getbgesthost general} +general ssh #{getbesthost general} nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo [hosts] general cubian xena Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -17,10 +17,11 @@ ;; allow these queries through without starting a server ;; (define api:read-only-queries '(get-key-val-pairs get-keys + get-key-vals test-toplevel-num-items get-test-info-by-id test-get-rundir-from-test-id get-count-tests-running-for-testname get-count-tests-running @@ -34,11 +35,14 @@ test-get-paths-matching-keynames-target-new get-prereqs-not-met get-count-tests-running-for-run-id get-run-info get-run-status - register-run + get-run-stats + get-targets + get-target + ;; register-run get-tests-for-run get-test-id get-tests-for-runs-mindata get-run-name-from-id get-runs @@ -49,10 +53,11 @@ get-runs-by-patt get-steps-data get-steps-for-test read-test-data login + tasks-get-last testmeta-get-record have-incompletes? synchash-get )) @@ -107,11 +112,11 @@ (define (api:execute-requests dbstruct dat) (handle-exceptions exn (let ((call-chain (get-call-chain))) (print-call-chain (current-error-port)) - (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn)) + (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn)) (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens (if (not (vector? dat)) ;; it is an error to not receive a vector (vector #f #f "remote must be called with a vector") (vector ;; return a vector + the returned data structure #t Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -12,14 +12,14 @@ ;; C L I E N T S ;;====================================================================== (require-extension (srfi 18) extras tcp s11n) -(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest) +(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest) ;; (use zmq) -(import (prefix sqlite3 sqlite3:)) +(use (prefix sqlite3 sqlite3:)) (use spiffy uri-common intarweb http-client spiffy-request-vars uri-common intarweb directory-utils) (declare (unit client)) @@ -46,17 +46,23 @@ (define (client:connect iface port) (case (server:get-transport) ((rpc) (rpc:client-connect iface port)) ((http) (http:client-connect iface port)) ((zmq) (zmq:client-connect iface port)) - (else (rpc:client-connect iface port)))) + (else + (debug:print 0 *default-log-port* "ERROR: transport " (remote-transport *runremote*) " not supported (5)") + (exit)))) (define (client:setup run-id #!key (remaining-tries 10) (failed-connects 0)) (case (server:get-transport) - ((rpc) (rpc-transport:client-setup run-id)) ;;(client:setup-rpc run-id)) - ((http)(client:setup-http run-id)) - (else (rpc-transport:client-setup run-id)))) ;; (client:setup-rpc run-id)))) + ((rpc) (let ((res (client:setup-rpc run-id remaining-tries: remaining-tries))) + (remote-conndat-set! *runremote* res) + res)) + ((http)(client:setup-http run-id remaining-tries: remaining-tries failed-connects: failed-connects)) + (else + (debug:print 0 *default-log-port* "ERROR: transport " (remote-transport *runremote*) " not supported (6)") + (exit)))) ;; (client:setup-rpc run-id)))) ;; (define (client:login-no-auto-setup server-info run-id) ;; (case (server:get-transport) ;; ((rpc) (rpc:login-no-auto-client-setup server-info run-id)) ;; ((http) (rmt:login-no-auto-client-setup server-info run-id)) @@ -152,10 +158,30 @@ ;; ;; client:setup ;; ;; lookup_server, need to remove *runremote* stuff ;; + +(define (client:setup-rpc run-id #!key (remaining-tries 10) (failed-connects 0)) + (debug:print-info 2 *default-log-port* "client:setup-rpc remaining-tries=" remaining-tries) + (let* ((server-dat (tasks:get-server (db:delay-if-busy (tasks:open-db)) run-id)) + (num-available (tasks:num-in-available-state (db:delay-if-busy (tasks:open-db)) run-id))) + (cond + ((<= remaining-tries 0) + (debug:print-error 0 *default-log-port* "failed to start or connect to server for run-id " run-id) + (exit 1)) + (server-dat + (debug:print-info 4 *default-log-port* "client:setup-rpc server-dat=" server-dat ", remaining-tries=" remaining-tries) + + (rpc-transport:client-setup run-id server-dat remaining-tries: remaining-tries)) + (else + (if (< num-available 2) + (server:try-running run-id)) + (thread-sleep! (+ 2 (random (- 20 remaining-tries)))) ;; give server a little time to start up, randomize a little to avoid start storms. + (client:setup run-id remaining-tries: (- remaining-tries 1)))))) + + (define (client:setup-http run-id #!key (remaining-tries 10) (failed-connects 0)) (debug:print-info 2 *default-log-port* "client:setup remaining-tries=" remaining-tries) (let* ((tdbdat (tasks:open-db))) (if (<= remaining-tries 0) (begin @@ -165,30 +191,23 @@ (debug:print-info 4 *default-log-port* "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries) (if server-dat (let* ((iface (tasks:hostinfo-get-interface server-dat)) (hostname (tasks:hostinfo-get-hostname server-dat)) (port (tasks:hostinfo-get-port server-dat)) - (start-res (case *transport-type* - ((http)(http-transport:client-connect iface port)) - ((nmsg)(nmsg-transport:client-connect hostname port)))) - (ping-res (case *transport-type* - ((http)(rmt:login-no-auto-client-setup start-res run-id)) - ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id))) - (if logininfo - (car (vector-ref logininfo 1)) - #f)))))) + (start-res (http-transport:client-connect iface port)) + (ping-res (rmt:login-no-auto-client-setup start-res))) (if (and start-res ping-res) (begin - (hash-table-set! *runremote* run-id start-res) + (remote-conndat-set! *runremote* start-res) ;; (hash-table-set! *runremote* run-id start-res) (debug:print-info 2 *default-log-port* "connected to " (http-transport:server-dat-make-url start-res)) start-res) (begin ;; login failed but have a server record, clean out the record and try again (debug:print-info 0 *default-log-port* "client:setup, login failed, will attempt to start server ... start-res=" start-res ", run-id=" run-id ", server-dat=" server-dat) (case *transport-type* ((http)(http-transport:close-connections run-id))) - (hash-table-delete! *runremote* run-id) + (remote-conndat-set! *runremote* #f) ;; (hash-table-delete! *runremote* run-id) (tasks:kill-server-run-id run-id) (tasks:server-force-clean-run-record (db:delay-if-busy tdbdat) run-id (tasks:hostinfo-get-interface server-dat) (tasks:hostinfo-get-port server-dat) @@ -206,11 +225,11 @@ (if (< num-available 2) (server:try-running run-id)) (thread-sleep! (+ 5 (random (- 20 remaining-tries)))) ;; give server a little time to start up, randomize a little to avoid start storms. (client:setup run-id remaining-tries: (- remaining-tries 1))))))))) -;; keep this as a function to ease future +;; keep this as a function to ease future ;; this is unused, not porting for rpc -BB (define (client:start run-id server-info) (http-transport:client-connect (tasks:hostinfo-get-interface server-info) (tasks:hostinfo-get-port server-info))) ;; ;; client:signal-handler Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -7,11 +7,11 @@ ;; This program is distributed WITHOUT ANY WARRANTY; without even the ;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ;; PURPOSE. ;;====================================================================== -(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 nanomsg sql-de-lite hostinfo) +(use srfi-1 posix regex-case base64 format dot-locking csv-xml z3 sql-de-lite hostinfo md5 message-digest typed-records directory-utils) (require-extension regex posix) (require-extension (srfi 18) extras tcp rpc) (import (prefix sqlite3 sqlite3:)) @@ -18,10 +18,12 @@ (import (prefix base64 base64:)) (declare (unit common)) (include "common_records.scm") +(include "thunk-utils.scm") + ;; (require-library margs) ;; (include "margs.scm") ;; (define old-exit exit) @@ -42,82 +44,140 @@ (define home (getenv "HOME")) (define user (getenv "USER")) ;; GLOBAL GLETCHES + +;; CONTEXTS +(defstruct cxt + (taskdb #f) + (cmutex (make-mutex))) +(define *contexts* (make-hash-table)) +(define *context-mutex* (make-mutex)) + +;; safe method for accessing a context given a toppath +;; +(define (common:with-cxt toppath proc) + (mutex-lock! *context-mutex*) + (let ((cxt (hash-table-ref/default *contexts* toppath #f))) + (if (not cxt) + (set! cxt (let ((x (make-cxt)))(hash-table-set! *contexts* toppath x) x))) + (let ((cxt-mutex (cxt-mutex cxt))) + (mutex-unlock! *context-mutex*) + (mutex-lock! cxt-mutex) + ;; here we guard proc with exception handler so + ;; no matter how proc succeeds or fails, + ;; the cxt-mutex will be unlocked afterward. + (let* ((EXCEPTION-SYMBOL (gensym)) ;; use a generated symbol + (guarded-proc ;; to avoid collision + (lambda args + (let* ((res (condition-case + (apply proc args) + [x () (cons EXCEPTION-SYMBOL x)]))) + (mutex-unlock! cxt-mutex) + (if (and (pair? res) (eq? (car res) EXCEPTION)) + (abort (cdr res)) + res))))) + (guarded-proc cxt))))) + (define *db-keys* #f) (define *configinfo* #f) ;; raw results from setup, includes toppath and table from megatest.config (define *runconfigdat* #f) ;; run configs data (define *configdat* #f) ;; megatest.config data (define *configstatus* #f) ;; status of data; 'fulldata : all processing done, #f : no data yet, 'partialdata : partial read done (define *toppath* #f) (define *already-seen-runconfig-info* #f) -(define *waiting-queue* (make-hash-table)) (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar -(define *write-frequency* (make-hash-table)) ;; run-id => (vector (current-seconds) 0)) (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) +(define *time-zero* (current-seconds)) ;; for the watchdog ;; DATABASE -(define *dbstruct-db* #f) +(define *dbstruct-db* #f) ;; used to cache the dbstruct in db:setup. Goal is to remove this. +;; db stats (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) -(define *db-sync-mutex* (make-mutex)) -(define *db-multi-sync-mutex* (make-mutex)) -(define *db-local-sync* (make-hash-table)) ;; used to record last touch of db -(define *megatest-db* #f) -(define *last-db-access* (current-seconds)) ;; update when db is accessed via server +;; db access +(define *db-last-access* (current-seconds)) ;; last db access, used in server (define *db-write-access* #t) -(define *inmemdb* #f) +;; db sync +(define *db-last-write* 0) ;; used to record last touch of db +(define *db-last-sync* 0) ;; last time the sync to megatest.db happened +(define *db-sync-in-progress* #f) ;; if there is a sync in progress do not try to start another +(define *db-multi-sync-mutex* (make-mutex)) ;; protect access to *db-sync-in-progress*, *db-last-sync* and *db-last-write* +;; task db (define *task-db* #f) ;; (vector db path-to-db) (define *db-access-allowed* #t) ;; flag to allow access (define *db-access-mutex* (make-mutex)) +(define *db-cache-path* #f) ;; SERVER (define *my-client-signature* #f) -(define *transport-type* 'http) -(define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg -(define *runremote* (make-hash-table)) ;; if set up for server communication this will hold +(define *transport-type* #f) ;; override with [server] transport http|rpc|nmsg + +(define *DEFAULT-TRANSPORT* "http") +(define (common:set-transport-type) + (set! *transport-type* + (string->symbol + (or + (args:get-arg "-transport") + (configf:lookup *configdat* "server" "transport") + *DEFAULT-TRANSPORT*))) + *transport-type*) + +(define *runremote* #f) ;; if set up for server communication this will hold (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) -(define *client-non-blocking-mode* #f) (define *server-id* #f) (define *server-info* #f) (define *time-to-exit* #f) -(define *received-response* #f) -(define *default-numtries* 10) (define *server-run* #t) (define *run-id* #f) (define *server-kind-run* (make-hash-table)) +(define *home-host* #f) +(define *total-non-write-delay* 0) +(define *heartbeat-mutex* (make-mutex)) +;; client +(define *rmt-mutex* (make-mutex)) ;; remote access calls mutex + +;; RPC transport +(define *rpc:listener* #f) + +;; KEY info (define *target* (make-hash-table)) ;; cache the target here; target is keyval1/keyval2/.../keyvalN (define *keys* (make-hash-table)) ;; cache the keys here (define *keyvals* (make-hash-table)) (define *toptest-paths* (make-hash-table)) ;; cache toptest path settings here (define *test-paths* (make-hash-table)) ;; cache test-id to test run paths here (define *test-ids* (make-hash-table)) ;; cache run-id, testname, and item-path => test-id (define *test-info* (make-hash-table)) ;; cache the test info records, update the state, status, run_duration etc. from testdat.db (define *run-info-cache* (make-hash-table)) ;; run info is stable, no need to reget +(define *launch-setup-mutex* (make-mutex)) ;; need to be able to call launch:setup often so mutex it and re-call the real deal only if *toppath* not set +(define *homehost-mutex* (make-mutex)) -;; Awful. Please FIXME +;; cache environment vars for each run here (define *env-vars-by-run-id* (make-hash-table)) -(define *current-run-name* #f) ;; Testconfig and runconfig caches. (define *testconfigs* (make-hash-table)) ;; test-name => testconfig (define *runconfigs* (make-hash-table)) ;; target => runconfig ;; This is a cache of pre-reqs met, don't re-calc in cases where called with same params less than ;; five seconds ago (define *pre-reqs-met-cache* (make-hash-table)) +;; cache of verbosity given string +;; +(define *verbosity-cache* (make-hash-table)) + (define (common:clear-caches) (set! *target* (make-hash-table)) (set! *keys* (make-hash-table)) (set! *keyvals* (make-hash-table)) (set! *toptest-paths* (make-hash-table)) @@ -130,10 +190,12 @@ ;; Generic string database (define sdb:qry #f) ;; (make-sdb:qry)) ;; 'init #f) ;; Generic path database (define *fdb* #f) + +(define *last-launch* (current-seconds)) ;; use for throttling the launch rate. Would be better to use the db and last time of a test in LAUNCHED state. ;;====================================================================== ;; V E R S I O N ;;====================================================================== @@ -160,46 +222,74 @@ (common:version-signature)))) ;; Move me elsewhere ... ;; RADT => Why do we meed the version check here, this is called only if version misma ;; -(define (common:cleanup-db) +(define (common:cleanup-db dbstruct) (db:multi-db-sync - #f ;; do all run-ids + dbstruct ;; 'new2old 'killservers 'dejunk ;; 'adj-testids ;; 'old2new - 'new2old) + 'new2old + 'schema) (if (common:version-changed?) (common:set-last-run-version))) + +;; Rotate logs, logic: +;; if > 500k and older than 1 week: +;; remove previous compressed log and compress this log +;; WARNING: This proc operates assuming that it is in the directory above the +;; logs directory you wish to log-rotate. +;; +(define (common:rotate-logs) + (if (not (directory-exists? "logs"))(create-directory "logs")) + (directory-fold + (lambda (file rem) + (if (and (string-match "^.*.log" file) + (> (file-size (conc "logs/" file)) 200000)) + (let ((gzfile (conc "logs/" file ".gz"))) + (if (file-exists? gzfile) + (begin + (debug:print-info 0 *default-log-port* "removing " gzfile) + (delete-file gzfile))) + (debug:print-info 0 *default-log-port* "compressing " file) + (system (conc "gzip logs/" file))))) + '() + "logs")) ;; Force a megatest cleanup-db if version is changed and skip-version-check not specified ;; (define (common:exit-on-version-changed) (if (common:version-changed?) - (let ((mtconf (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.config"))) - (debug:print 0 *default-log-port* - "WARNING: Version mismatch!\n" - " expected: " (common:version-signature) "\n" - " got: " (common:get-last-run-version)) - (if (and (file-exists? mtconf) - (eq? (current-user-id)(file-owner mtconf))) ;; safe to run -cleanup-db - (begin - (debug:print 0 *default-log-port* " I see you are the owner of megatest.config, attempting to cleanup and reset to new version") - (handle-exceptions - exn - (begin - (debug:print 0 *default-log-port* "Failed to switch versions.") - (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn)) - (print-call-chain (current-error-port)) - (exit 1)) - (common:cleanup-db))) - (begin - (debug:print 0 *default-log-port* " to switch versions you can run: \"megatest -cleanup-db\"") - (exit 1)))))) + (if (common:on-homehost?) + (let ((mtconf (conc (get-environment-variable "MT_RUN_AREA_HOME") "/megatest.config")) + (dbstruct (db:setup))) + (debug:print 0 *default-log-port* + "WARNING: Version mismatch!\n" + " expected: " (common:version-signature) "\n" + " got: " (common:get-last-run-version)) + (if (and (file-exists? mtconf) + (eq? (current-user-id)(file-owner mtconf))) ;; safe to run -cleanup-db + (begin + (debug:print 0 *default-log-port* " I see you are the owner of megatest.config, attempting to cleanup and reset to new version") + (handle-exceptions + exn + (begin + (debug:print 0 *default-log-port* "Failed to switch versions.") + (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn)) + (print-call-chain (current-error-port)) + (exit 1)) + (common:cleanup-db dbstruct))) + (begin + (debug:print 0 *default-log-port* " to switch versions you can run: \"megatest -cleanup-db\"") + (exit 1)))) + (begin + (debug:print 0 *default-log-port* "ERROR: cannot migrate version unless on homehost. Exiting.") + (exit 1))))) ;;====================================================================== ;; S P A R S E A R R A Y S ;;====================================================================== @@ -317,36 +407,110 @@ ;;====================================================================== ;; S T A T E S A N D S T A T U S E S ;;====================================================================== (define *common:std-states* - '((0 "COMPLETED") - (1 "NOT_STARTED") - (2 "RUNNING") - (3 "REMOTEHOSTSTART") - (4 "LAUNCHED") - (5 "KILLED") - (6 "KILLREQ") - (7 "STUCK") - (8 "ARCHIVED"))) + '((0 "ARCHIVED") + (1 "STUCK") + (2 "KILLREQ") + (3 "KILLED") + (4 "NOT_STARTED") + (5 "COMPLETED") + (6 "LAUNCHED") + (7 "REMOTEHOSTSTART") + (8 "RUNNING") + )) (define *common:std-statuses* - '((0 "PASS") - (1 "WARN") - (2 "FAIL") + '(;; (0 "DELETED") + (1 "n/a") + (2 "PASS") (3 "CHECK") - (4 "n/a") - (5 "WAIVED") - (6 "SKIP") - (7 "DELETED") - (8 "STUCK/DEAD") + (4 "SKIP") + (5 "WARN") + (6 "WAIVED") + (7 "STUCK/DEAD") + (8 "FAIL") (9 "ABORT"))) -;; These are stopping conditions that prevent a test from being run -(define *common:cant-run-states-sym* - '(COMPLETED KILLED WAIVED UNKNOWN INCOMPLETE ABORT ARCHIVED)) +(define *common:ended-states* ;; states which indicate the test is stopped and will not proceed + '("COMPLETED" "ARCHIVED" "KILLED" "KILLREQ" "STUCK" "INCOMPLETE")) +(define *common:badly-ended-states* ;; these roll up as CHECK, i.e. results need to be checked + '("KILLED" "KILLREQ" "STUCK" "INCOMPLETE" "DEAD")) + +(define *common:running-states* ;; test is either running or can be run + '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED")) + +(define *common:cant-run-states* ;; These are stopping conditions that prevent a test from being run + '("COMPLETED" "KILLED" "UNKNOWN" "INCOMPLETE" "ARCHIVED")) + +(define *common:not-started-ok-statuses* ;; if not one of these statuses when in not_started state treat as dead + '("n/a" "na" "PASS" "FAIL" "WARN" "CHECK" "WAIVED" "DEAD" "SKIP")) + +(define (common:special-sort items order comp) + (let ((items-order (map reverse order)) + (acomp (or comp >))) + (sort items + (lambda (a b) + (let ((a-num (cadr (or (assoc a items-order) '(0 0)))) + (b-num (cadr (or (assoc b items-order) '(0 0))))) + (acomp a-num b-num)))))) + +;; ;; given a toplevel with currstate, currstatus apply state and status +;; ;; => (newstate . newstatus) +;; (define (common:apply-state-status currstate currstatus state status) +;; (let* ((cstate (string->symbol (string-downcase currstate))) +;; (cstatus (string->symbol (string-downcase currstatus))) +;; (sstate (string->symbol (string-downcase state))) +;; (sstatus (string->symbol (string-downcase status))) +;; (nstate #f) +;; (nstatus #f)) +;; (set! nstate +;; (case cstate +;; ((completed not_started killed killreq stuck archived) +;; (case sstate ;; completed -> sstate +;; ((completed killed killreq stuck archived) completed) +;; ((running remotehoststart launched) running) +;; (else unknown-error-1))) +;; ((running remotehoststart launched) +;; (case sstate +;; ((completed killed killreq stuck archived) #f) ;; need to look at all items +;; ((running remotehoststart launched) running) +;; (else unknown-error-2))) +;; (else unknown-error-3))) +;; (set! nstatus +;; (case sstatus +;; ((pass) +;; (case nstate +;; ((pass n/a deleted) pass) +;; ((warn) warn) +;; ((fail) fail) +;; ((check) check) +;; ((waived) waived) +;; ((skip) skip) +;; ((stuck/dead) stuck) +;; ((abort) abort) +;; (else unknown-error-4))) +;; ((warn) +;; (case nstate +;; ((pass warn n/a skip deleted) warn) +;; ((fail) fail) +;; ((check) check) +;; ((waived) waived) +;; ((stuck/dead) stuck) +;; (else unknown-error-5))) +;; ((fail) +;; (case nstate +;; ((pass warn fail check n/a waived skip deleted stuck/dead stuck) fail) +;; ((abort) abort) +;; (else unknown-error-6))) +;; (else unknown-error-7))) +;; (cons +;; (if nstate (symbol->string nstate) nstate) +;; (if nstatus (symbol->string nstatus) nstatus)))) + ;;====================================================================== ;; D E B U G G I N G S T U F F ;;====================================================================== (define *verbosity* 1) @@ -360,57 +524,129 @@ (let ((res (assoc key lst))) (if res (cadr res)(if (null? default) #f (car default))))) (define (common:get-testsuite-name) (or (configf:lookup *configdat* "setup" "testsuite" ) - (pathname-file *toppath*))) + (if *toppath* + (pathname-file *toppath*) + (pathname-file (current-directory))))) + +(define (common:get-db-tmp-area) + (if *db-cache-path* + *db-cache-path* + (let ((dbpath (create-directory (conc "/tmp/" (current-user-name) + "/megatest_localdb/" + (common:get-testsuite-name) "/" + (string-translate *toppath* "/" ".")) #t))) + (set! *db-cache-path* dbpath) + dbpath))) + +(define (common:get-area-path-signature) + (message-digest-string (md5-primitive) *toppath*)) ;;====================================================================== ;; E X I T H A N D L I N G ;;====================================================================== -(define (common:legacy-sync-recommended) - (or (args:get-arg "-runtests") - (args:get-arg "-server") - ;; (args:get-arg "-set-run-status") - (args:get-arg "-remove-runs") - ;; (args:get-arg "-get-run-status") - )) - -(define (common:legacy-sync-required) - (configf:lookup *configdat* "setup" "megatest-db")) +(define (common:run-sync?) + (let ((ohh (common:on-homehost?)) + (srv (args:get-arg "-server"))) + ;; (debug:print-info 0 *default-log-port* "common:run-sync? ohh=" ohh ", srv=" srv) + (and (common:on-homehost?) + (args:get-arg "-server")))) + +;;;; run-ids +;; if #f use *db-local-sync* : or 'local-sync-flags +;; if #t use timestamps : or 'timestamps +(define (common:sync-to-megatest.db dbstruct) + (let ((start-time (current-seconds)) + (res (db:multi-db-sync dbstruct 'new2old))) + (let ((sync-time (- (current-seconds) start-time))) + (debug:print-info 3 *default-log-port* "Sync of newdb to olddb completed in " sync-time " seconds") + (if (common:low-noise-print 30 "sync new to old") + (debug:print-info 0 *default-log-port* "Sync of newdb to olddb completed in " sync-time " seconds"))) + res)) + +;; currently the primary job of the watchdog is to run the sync back to megatest.db from the db in /tmp +;; if we are on the homehost and we are a server (by definition we are on the homehost if we are a server) +;; +(define (common:watchdog) + (thread-sleep! 0.05) ;; delay for startup + (let ((legacy-sync (common:run-sync?)) + (debug-mode (debug:debug-mode 1)) + (last-time (current-seconds))) + (debug:print-info 0 *default-log-port* "watchdog starting. legacy-sync is " legacy-sync) + (if legacy-sync + (let ((dbstruct (db:setup))) + (debug:print-info 0 *default-log-port* "Server running, periodic sync started.") + (let loop () + ;; sync for filesystem local db writes + ;; + (mutex-lock! *db-multi-sync-mutex*) + (let* ((need-sync (>= *db-last-write* *db-last-sync*)) ;; no sync since last write + (sync-in-progress *db-sync-in-progress*) + (should-sync (> (- (current-seconds) *db-last-sync*) 5)) ;; sync every five seconds minimum + (will-sync (and (or need-sync should-sync) + (not sync-in-progress))) + (start-time (current-seconds))) + ;; (debug:print-info 0 *default-log-port* "need-sync: " need-sync " sync-in-progress: " sync-in-progress " should-sync: " should-sync " will-sync: " will-sync) + (if will-sync (set! *db-sync-in-progress* #t)) + (mutex-unlock! *db-multi-sync-mutex*) + (if will-sync + (let ((res (common:sync-to-megatest.db dbstruct))) ;; did we sync any data? If so need to set the db touched flag to keep the server alive + (if (> res 0) ;; some records were transferred, keep the db alive + (begin + (mutex-lock! *heartbeat-mutex*) + (set! *db-last-access* (current-seconds)) + (mutex-unlock! *heartbeat-mutex*) + (debug:print-info 0 *default-log-port* "sync called, " res " records transferred.")) + (debug:print-info 2 *default-log-port* "sync called but zero records transferred")))) + (if will-sync + (begin + (mutex-lock! *db-multi-sync-mutex*) + (set! *db-sync-in-progress* #f) + (set! *db-last-sync* start-time) + (mutex-unlock! *db-multi-sync-mutex*))) + (if (and debug-mode + (> (- start-time last-time) 60)) + (begin + (set! last-time start-time) + (debug:print-info 4 *default-log-port* "timestamp -> " (seconds->time-string (current-seconds)) ", time since start -> " (seconds->hr-min-sec (- (current-seconds) *time-zero*)))))) + + ;; keep going unless time to exit + ;; + (if (not *time-to-exit*) + (let delay-loop ((count 0)) + (if (and (not *time-to-exit*) + (< count 4)) ;; was 11, changing to 4. + (begin + (thread-sleep! 1) + (delay-loop (+ count 1)))) + (loop))) + (if (common:low-noise-print 30) + (debug:print-info 0 *default-log-port* "Exiting watchdog timer, *time-to-exit* = " *time-to-exit*))))))) (define (std-exit-procedure) + (let ((no-hurry (if *time-to-exit* ;; hurry up #f (begin (set! *time-to-exit* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") (if (and no-hurry (debug:debug-mode 18)) (rmt:print-db-stats)) (let ((th1 (make-thread (lambda () ;; thread for cleaning up, give it five seconds - (let ((run-ids (hash-table-keys *db-local-sync*))) - (if (and (not (null? run-ids)) - (or (common:legacy-sync-recommended) - (configf:lookup *configdat* "setup" "megatest-db"))) - (if no-hurry (db:multi-db-sync run-ids 'new2old)))) - (if *dbstruct-db* (db:close-all *dbstruct-db*)) - (if *inmemdb* (db:close-all *inmemdb*)) - (if (and *megatest-db* - (sqlite3:database? *megatest-db*)) - (begin - (sqlite3:interrupt! *megatest-db*) - (sqlite3:finalize! *megatest-db* #t) - (set! *megatest-db* #f))) + (if *dbstruct-db* (db:close-all *dbstruct-db*)) ;; one second allocated (if *task-db* (let ((db (cdr *task-db*))) (if (sqlite3:database? db) (begin (sqlite3:interrupt! db) (sqlite3:finalize! db #t) - (vector-set! *task-db* 0 #f))))) + ;; (vector-set! *task-db* 0 #f) + (set! *task-db* #f))))) (close-output-port *default-log-port*) (set! *default-log-port* (current-error-port))) "Cleanup db exit thread")) (th2 (make-thread (lambda () (debug:print 4 *default-log-port* "Attempting clean exit. Please be patient and wait a few seconds...") (if no-hurry @@ -417,10 +653,19 @@ (thread-sleep! 5) ;; give the clean up few seconds to do it's stuff (thread-sleep! 2)) (debug:print 4 *default-log-port* " ... done") ) "clean exit"))) + + ;; let's try to clean up open sockets + (if *runremote* + (case (remote-transport *runremote*) + ((http) #t) + ((rpc) (rpc:close-all-connections!)) + (else + (debug:print-info 0 *default-log-port* "Transport "(remote-transport *runremote*)" not supported")))) + (thread-start! th1) (thread-start! th2) (thread-join! th1)))) (define (std-signal-handler signum) @@ -492,10 +737,57 @@ (define (common:get-disks #!key (configf #f)) (hash-table-ref/default (or configf (read-config "megatest.config" #f #t)) "disks" '("none" ""))) +;; return first command that exists, else #f +;; +(define (common:which cmds) + (if (null? cmds) + #f + (let loop ((hed (car cmds)) + (tal (cdr cmds))) + (let ((res (with-input-from-pipe (conc "which " hed) read-line))) + (if (and (string? res) + (file-exists? res)) + res + (if (null? tal) + #f + (loop (car tal)(cdr tal)))))))) + +(define (common:get-install-area) + (let ((exe-path (car (argv)))) + (if (file-exists? exe-path) + (handle-exceptions + exn + #f + (pathname-directory + (pathname-directory + (pathname-directory exe-path)))) + #f))) + +;; return first path that can be created or already exists and is writable +;; +(define (common:get-create-writeable-dir dirs) + (if (null? dirs) + #f + (let loop ((hed (car dirs)) + (tal (cdr dirs))) + (let ((res (or (and (directory? hed) + (file-write-access? hed) + hed) + (handle-exceptions + exn + #f + (create-directory hed #t))))) + (if (and (string? res) + (directory? res)) + res + (if (null? tal) + #f + (loop (car tal)(cdr tal)))))))) + ;;====================================================================== ;; T A R G E T S , S T A T E , S T A T U S , ;; R U N N A M E A N D T E S T P A T T ;;====================================================================== @@ -556,10 +848,60 @@ (if target (begin (debug:print-error 0 *default-log-port* "Invalid target, spaces or blanks not allowed \"" target "\", target should be: " (string-intersperse keys "/") ", have " tlist " for elements") #f) #f)))) + +;; logic for getting homehost. Returns (host . at-home) +;; IF *toppath* is not set, wait up to five seconds trying every two seconds +;; (this is to accomodate the watchdog) +;; +(define (common:get-homehost #!key (trynum 5)) + ;; called often especially at start up. use mutex to eliminate collisions + (mutex-lock! *homehost-mutex*) + (cond + (*home-host* + (mutex-unlock! *homehost-mutex*) + *home-host*) + ((not *toppath*) + (mutex-unlock! *homehost-mutex*) + (launch:setup) ;; safely mutexed now + (if (> trynum 0) + (begin + (thread-sleep! 2) + (common:get-homehost trynum: (- trynum 1))) + #f)) + (else + (let* ((currhost (get-host-name)) + (bestadrs (server:get-best-guess-address currhost)) + ;; first look in config, then look in file .homehost, create it if not found + (homehost (or (configf:lookup *configdat* "server" "homehost" ) + (let ((hhf (conc *toppath* "/.homehost"))) + (if (file-exists? hhf) + (with-input-from-file hhf read-line) + (if (file-write-access? *toppath*) + (begin + (with-output-to-file hhf + (lambda () + (print bestadrs))) + (begin + (mutex-unlock! *homehost-mutex*) + (car (common:get-homehost)))) + #f))))) + (at-home (or (equal? homehost currhost) + (equal? homehost bestadrs)))) + (set! *home-host* (cons homehost at-home)) + (mutex-unlock! *homehost-mutex*) + *home-host*)))) + +;; am I on the homehost? +;; +(define (common:on-homehost?) + (let ((hh (common:get-homehost))) + (if hh + (cdr hh) + #f))) ;;====================================================================== ;; M I S C L I S T S ;;====================================================================== @@ -579,10 +921,11 @@ (if (null? tala) ;; we are done talb (loop (car tala) (cdr tala) (car talb) + (cdr talb))) #f))))) ;; Needed for long lists to be sorted where (apply max ... ) dies ;; @@ -594,10 +937,76 @@ (loop (max hed max-val) (car tal) (cdr tal)) (max hed max-val)))) +;; get min or max, use > for max and < for min, this works around the limits on apply +;; +(define (common:min-max comp lst) + (if (null? lst) + #f ;; better than an exception for my needs + (fold (lambda (a b) + (if (comp a b) a b)) + (car lst) + lst))) + +;; path list to hash-table tree +;; ((a b c)(a b d)(e b c)) => ((a (b (d) (c))) (e (b (c)))) +;; +(define (common:list->htree lst) + (let ((resh (make-hash-table))) + (for-each + (lambda (inlst) + (let loop ((ht resh) + (hed (car inlst)) + (tal (cdr inlst))) + (if (hash-table-ref/default ht hed #f) + (if (not (null? tal)) + (loop (hash-table-ref ht hed) + (car tal) + (cdr tal))) + (begin + (hash-table-set! ht hed (make-hash-table)) + (loop ht hed tal))))) + lst) + resh)) + +;; hash-table tree to html list tree +;; +;; tipfunc takes two parameters: y the tip value and path the path to that point +;; +(define (common:htree->html ht path tipfunc) + (let ((datlist (sort (hash-table->alist ht) + (lambda (a b) + (string< (car a)(car b)))))) + (if (null? datlist) + (tipfunc #f path) ;; really shouldn't get here + (s:ul + (map (lambda (x) + (let* ((levelname (car x)) + (y (cdr x)) + (newpath (append path (list levelname))) + (leaf (or (not (hash-table? y)) + (null? (hash-table-keys y))))) + (if leaf + (s:li (tipfunc y newpath)) + (s:li + (list + levelname + (common:htree->html y newpath tipfunc)))))) + datlist))))) + +;; hash-table tree to alist tree +;; +(define (common:htree->atree ht) + (map (lambda (x) + (cons (car x) + (let ((y (cdr x))) + (if (hash-table? y) + (common:htree->atree y) + y)))) + (hash-table->alist ht))) ;;====================================================================== ;; M U N G E D A T A I N T O N I C E F O R M S ;;====================================================================== @@ -673,12 +1082,12 @@ (with-input-from-pipe (conc "/bin/readlink -f " path) (lambda () (read-line))))) -(define (get-cpu-load) - (car (common:get-cpu-load))) +(define (get-cpu-load #!key (remote-host #f)) + (car (common:get-cpu-load remote-host))) ;; (let* ((load-res (process:cmd-run->list "uptime")) ;; (load-rx (regexp "load average:\\s+(\\d+)")) ;; (cpu-load #f)) ;; (for-each (lambda (l) ;; (let ((match (string-search load-rx l))) @@ -689,16 +1098,76 @@ ;; (car load-res)) ;; cpu-load)) ;; get cpu load by reading from /proc/loadavg, return all three values ;; -(define (common:get-cpu-load) - (with-input-from-file "/proc/loadavg" - (lambda ()(list (read)(read)(read))))) +(define (common:get-cpu-load remote-host) + (if remote-host + (map (lambda (res) + (if (eof-object? res) 9e99 res)) + (with-input-from-pipe + (conc "ssh " remote-host " cat /proc/loadavg") + (lambda ()(list (read)(read)(read))))) + (with-input-from-file "/proc/loadavg" + (lambda ()(list (read)(read)(read)))))) + +;; get normalized cpu load by reading from /proc/loadavg and /proc/cpuinfo return all three values and the number of real cpus and the number of threads +;; returns list (normalized-proc-load normalized-core-load 1m 5m 15m ncores nthreads) +;; +(define (common:get-normalized-cpu-load remote-host) + (let ((data (if remote-host + (with-input-from-pipe + (conc "ssh " remote-host " cat /proc/loadavg;cat /proc/cpuinfo;echo end") + read-lines) + (append + (with-input-from-file "/proc/loadavg" + read-lines) + (with-input-from-file "/proc/cpuinfo" + read-lines) + (list "end")))) + (load-rx (regexp "^([\\d\\.]+)\\s+([\\d\\.]+)\\s+([\\d\\.]+)\\s+.*$")) + (proc-rx (regexp "^processor\\s+:\\s+(\\d+)\\s*$")) + (core-rx (regexp "^core id\\s+:\\s+(\\d+)\\s*$")) + (phys-rx (regexp "^physical id\\s+:\\s+(\\d+)\\s*$")) + (max-num (lambda (p n)(max (string->number p) n)))) + ;; (print "data=" data) + (if (null? data) ;; something went wrong + #f + (let loop ((hed (car data)) + (tal (cdr data)) + (loads #f) + (proc-num 0) ;; processor includes threads + (phys-num 0) ;; physical chip on motherboard + (core-num 0)) ;; core + ;; (print hed ", " loads ", " proc-num ", " phys-num ", " core-num) + (if (null? tal) ;; have all our data, calculate normalized load and return result + (let* ((act-proc (+ proc-num 1)) + (act-phys (+ phys-num 1)) + (act-core (+ core-num 1)) + (adj-proc-load (/ (car loads) act-proc)) + (adj-core-load (/ (car loads) act-core))) + (append (list (cons 'adj-proc-load adj-proc-load) + (cons 'adj-core-load adj-core-load)) + (list (cons '1m-load (car loads)) + (cons '5m-load (cadr loads)) + (cons '15m-load (caddr loads))) + (list (cons 'proc act-proc) + (cons 'core act-core) + (cons 'phys act-phys)))) + (regex-case + hed + (load-rx ( x l1 l5 l15 ) (loop (car tal)(cdr tal)(map string->number (list l1 l5 l15)) proc-num phys-num core-num)) + (proc-rx ( x p ) (loop (car tal)(cdr tal) loads (max-num p proc-num) phys-num core-num)) + (phys-rx ( x p ) (loop (car tal)(cdr tal) loads proc-num (max-num p phys-num) core-num)) + (core-rx ( x c ) (loop (car tal)(cdr tal) loads proc-num phys-num (max-num c core-num))) + (else + (begin + ;; (print "NO MATCH: " hed) + (loop (car tal)(cdr tal) loads proc-num phys-num core-num))))))))) -(define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000) (msg #f)) - (let* ((loadavg (common:get-cpu-load)) +(define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000) (msg #f)(remote-host #f)) + (let* ((loadavg (common:get-cpu-load remote-host)) (first (car loadavg)) (next (cadr loadavg)) (adjload (* maxload numcpus)) (loadjmp (- first next))) (cond @@ -711,26 +1180,30 @@ (> count 0)) (debug:print-info 0 *default-log-port* "waiting " waitdelay " seconds due to load jump " loadjmp " > numcpus " numcpus (if msg msg "")) (thread-sleep! waitdelay) (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1)))))) -(define (common:get-num-cpus) - (with-input-from-file "/proc/cpuinfo" - (lambda () - (let loop ((numcpu 0) - (inl (read-line))) - (if (eof-object? inl) - numcpu - (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl) - (+ numcpu 1) - numcpu) - (read-line))))))) +(define (common:get-num-cpus remote-host) + (let ((proc (lambda () + (let loop ((numcpu 0) + (inl (read-line))) + (if (eof-object? inl) + numcpu + (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl) + (+ numcpu 1) + numcpu) + (read-line))))))) + (if remote-host + (with-input-from-pipe + (conc "ssh " remote-host " cat /proc/cpuinfo") + proc) + (with-input-from-file "/proc/cpuinfo" proc)))) ;; wait for normalized cpu load to drop below maxload ;; -(define (common:wait-for-normalized-load maxload #!key (msg #f)) - (let ((num-cpus (common:get-num-cpus))) +(define (common:wait-for-normalized-load maxload #!key (msg #f)(remote-host #f)) + (let ((num-cpus (common:get-num-cpus remote-host))) (common:wait-for-cpuload maxload num-cpus 15 msg: msg))) (define (get-uname . params) (let* ((uname-res (process:cmd-run->list (conc "uname " (if (null? params) "-a" (car params))))) (uname #f)) @@ -790,30 +1263,36 @@ (if (number? newval) (set! freespc newval)))))) (car df-results)) freespc)) -;; check space in dbdir -;; returns: ok/not dbspace required-space -;; -(define (common:check-db-dir-space) - (let* ((dbdir (db:get-dbdir)) - (dbspace (if (directory? dbdir) - (get-df dbdir) - 0)) - (required (string->number - (or (configf:lookup *configdat* "setup" "dbdir-space-required") - "100000")))) +(define (common:check-space-in-dir dirpath required) + (let* ((dbspace (if (directory? dirpath) + (get-df dirpath) + 0))) (list (> dbspace required) dbspace required - dbdir))) + dirpath))) +;; check space in dbdir and in megatest dir +;; returns: ok/not dbspace required-space +;; +(define (common:check-db-dir-space) + (let* ((required (string->number + (or (configf:lookup *configdat* "setup" "dbdir-space-required") + "100000"))) + (dbdir (common:get-db-tmp-area)) ;; (db:get-dbdir)) + (tdbspace (common:check-space-in-dir dbdir required)) + (mdbspace (common:check-space-in-dir *toppath* required))) + (sort (list tdbspace mdbspace) (lambda (a b) + (< (cadr a)(cadr b)))))) + ;; check available space in dbdir, exit if insufficient ;; (define (common:check-db-dir-and-exit-if-insufficient) - (let* ((spacedat (common:check-db-dir-space)) + (let* ((spacedat (car (common:check-db-dir-space))) ;; look only at worst for now (is-ok (car spacedat)) (dbspace (cadr spacedat)) (required (caddr spacedat)) (dbdir (cadddr spacedat))) (if (not is-ok) @@ -933,16 +1412,20 @@ vars (lambda (var val) (setenv var val))) vars)) -(define (common:run-a-command cmd) - (let ((fullcmd (conc (dtests:get-pre-command) - cmd - (dtests:get-post-command)))) +(define (common:run-a-command cmd #!key (with-vars #f)) + (let* ((pre-cmd (dtests:get-pre-command)) + (post-cmd (dtests:get-post-command)) + (fullcmd (if (or pre-cmd post-cmd) + (conc pre-cmd cmd post-cmd) + (conc "viewscreen " cmd)))) (debug:print-info 02 *default-log-port* "Running command: " fullcmd) - (common:without-vars fullcmd "MT_.*"))) + (if with-vars + (common:without-vars cmd) + (common:without-vars fullcmd "MT_.*")))) ;;====================================================================== ;; T I M E A N D D A T E ;;====================================================================== @@ -1072,10 +1555,18 @@ ;; ((RUNNING) "9 131 232") ;; ((KILLREQ) "39 82 206") ;; ((KILLED) "234 101 17") ;; ((NOT_STARTED) "240 240 240") ;; (else "192 192 192"))) + +(define (common:iup-color->rgb-hex instr) + (string-intersperse + (map (lambda (x) + (number->string x 16)) + (map string->number + (string-split instr))) + "/")) (define (common:get-color-from-status status) (cond ((equal? status "PASS") "green") ((equal? status "FAIL") "red") @@ -1101,22 +1592,10 @@ (string-intersperse (map number->string (u8vector->list (if res res (hostname->ip hostname)))) "."))) -(define (common:open-nm-req addr) - (let* ((req (nn-socket 'req)) - (res (nn-connect req addr))) - req)) - -;; (with-output-to-string (lambda ()(serialize obj))) -(define (common:nm-send-receive soc msg) - (nn-send soc msg) - (nn-recv soc)) - -(define (common:close-nm-req soc) - (nn-close soc)) (define (common:send-dboard-main-changed) (let* ((dashboard-ips (mddb:get-dashboards))) (for-each (lambda (ipadr) @@ -1126,91 +1605,11 @@ (if (not res) ;; couldn't reach that dashboard - remove it from db (print "ERROR: couldn't reach dashboard " ipadr)) res)) dashboard-ips))) -(define (common:nm-send-receive-timeout req msg) - (let* ((key "ping") - (success #f) - (keepwaiting #t) - (result #f) - (sendrec (make-thread - (lambda () - (nn-send req msg) - (set! result (nn-recv req)) - (set! success #t)) - "send-receive")) - (timeout (make-thread (lambda () - (let loop ((count 0)) - (thread-sleep! 1) - (print "still waiting after count seconds...") - (if (and keepwaiting (< count 10)) - (loop (+ count 1)))) - (if keepwaiting - (begin - (print "timeout waiting for reply") - (thread-terminate! sendrec)))) - "timeout"))) - (handle-exceptions - exn - (begin - (print-call-chain) - (print 0 " message: " ((condition-property-accessor 'exn 'message) exn)) - (print "exn=" (condition->list exn))) - (thread-start! timeout) - (thread-start! sendrec) - (thread-join! sendrec) - (if success (thread-terminate! timeout))) - result)) - -(define (common:ping-nm req) - ;; send a random number and check that we get it back - (let* ((key "ping") - (success #f) - (keepwaiting #t) - (ping (make-thread - (lambda () - (print "ping: sending string \"" key "\", expecting " (current-process-id)) - (nn-send req key) - (let ((result (nn-recv req))) - (if (equal? (conc (current-process-id)) result) - (begin - (print "ping, success: received \"" result "\"") - (set! success #t)) - (begin - (print "ping, failed: received key \"" result "\"") - (set! keepwaiting #f) - (set! success #f))))) - "ping")) - (timeout (make-thread (lambda () - (let loop ((count 0)) - (thread-sleep! 1) - (print "still waiting after count seconds...") - (if (and keepwaiting (< count 10)) - (loop (+ count 1)))) - (if keepwaiting - (begin - (print "timeout waiting for ping") - (thread-terminate! ping)))) - "timeout"))) - (handle-exceptions - exn - (begin - (print-call-chain) - (print 0 " message: " ((condition-property-accessor 'exn 'message) exn)) - (print "exn=" (condition->list exn)) - (print "ping failed to connect to tcp://" hostport)) - (thread-start! timeout) - (thread-start! ping) - (thread-join! ping) - (if success (thread-terminate! timeout))) - (if return-socket - (if success req #f) - (begin - (nn-close req) - success)))) - + ;;====================================================================== ;; D A S H B O A R D D B ;;====================================================================== (define (mddb:open-db) Index: common_records.scm ================================================================== --- common_records.scm +++ common_records.scm @@ -44,23 +44,39 @@ (print ((condition-property-accessor 'exn 'message) exn)) (print "Callback error in " procname) (print "Full condition info:\n" (condition->list exn))))) (proc))) +;; Need a mutex protected way to get and set values +;; or use (define-simple-syntax ?? +;; +(define-inline (with-mutex mtx accessor record . val) + (mutex-lock! mtx) + (let ((res (apply accessor record val))) + (mutex-unlock! mtx) + res)) + +;; this was cached based on results from profiling but it turned out the profiling +;; somehow went wrong - perhaps too many processes writing to it. Leaving the caching +;; in for now but can probably take it out later. +;; (define (debug:calc-verbosity vstr) - (cond - ((number? vstr) vstr) - ((not (string? vstr)) 1) - ;; ((string-match "^\\s*$" vstr) 1) - (vstr (let ((debugvals (filter number? (map string->number (string-split vstr ","))))) - (cond - ((> (length debugvals) 1) debugvals) - ((> (length debugvals) 0)(car debugvals)) - (else 1)))) - ((args:get-arg "-v") 2) - ((args:get-arg "-q") 0) - (else 1))) + (or (hash-table-ref/default *verbosity-cache* vstr #f) + (let ((res (cond + ((number? vstr) vstr) + ((not (string? vstr)) 1) + ;; ((string-match "^\\s*$" vstr) 1) + (vstr (let ((debugvals (filter number? (map string->number (string-split vstr ","))))) + (cond + ((> (length debugvals) 1) debugvals) + ((> (length debugvals) 0)(car debugvals)) + (else 1)))) + ((args:get-arg "-v") 2) + ((args:get-arg "-q") 0) + (else 1)))) + (hash-table-set! *verbosity-cache* vstr res) + res))) ;; check verbosity, #t is ok (define (debug:check-verbosity verbosity vstr) (if (not (or (number? verbosity) (list? verbosity))) Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -17,12 +17,12 @@ (require-library iup) (import (prefix iup iup:)) (use canvas-draw) -(use sqlite3 srfi-1 posix regex regex-case srfi-69) -(import (prefix sqlite3 sqlite3:)) +(use srfi-1 posix regex regex-case srfi-69) +(use (prefix sqlite3 sqlite3:)) (declare (unit dashboard-tests)) (declare (uses common)) (declare (uses db)) (declare (uses gutils)) @@ -41,15 +41,15 @@ (define *dashboard-comment-share-slot* #f) (define (dtests:get-pre-command #!key (default-override #f)) (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "pre-command"))) - (or cfg-ovrd default-override "xterm -geometry 180x20 -e \""))) + (or cfg-ovrd default-override "viewscreen "))) ;; "xterm -geometry 180x20 -e \""))) (define (dtests:get-post-command #!key (default-override #f)) (let ((cfg-ovrd (configf:lookup *configdat* "dashboard" "post-command"))) - (or cfg-ovrd default-override ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &"))) + (or cfg-ovrd default-override ""))) ;; ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &"))) (define (test-info-panel testdat store-label widgets) (iup:frame #:title "Test Info" ; #:expand "YES" @@ -158,11 +158,11 @@ ;;====================================================================== ;; Run info panel ;;====================================================================== (define (run-info-panel db keydat testdat runname) (let* ((run-id (db:test-get-run_id testdat)) - (rundat (db:get-run-info db run-id)) + (rundat (rmt:get-run-info run-id)) (header (db:get-header rundat)) (event_time (db:get-value-by-header (db:get-rows rundat) (db:get-header rundat) "event_time"))) (iup:frame @@ -286,11 +286,12 @@ (iup:label "STATE:" #:size "30x") (let* ((btns (map (lambda (state) (let ((btn (iup:button state #:expand "HORIZONTAL" #:size "50x" #:font "Courier New, -10" #:action (lambda (x) - (rmt:test-set-state-status-by-id run-id test-id state #f #f) + ;; (rmt:test-set-state-status-by-id run-id test-id state #f #f) + (rmt:roll-up-pass-fail-counts run-id test-id #f state #f #f) ;; test-name passed in as test-id is respected (db:test-set-state! testdat state))))) btn)) (map cadr *common:std-states*)))) ;; (list "COMPLETED" "NOT_STARTED" "RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "KILLED" "KILLREQ")))) (vector-set! *state-status* 0 (lambda (state color) @@ -319,11 +320,12 @@ (iup:attribute-set! wtxtbox "VALUE" c) (if (not *dashboard-comment-share-slot*) (set! *dashboard-comment-share-slot* wtxtbox))) )))) (begin - (rmt:test-set-state-status-by-id run-id test-id #f status #f) + ;; (rmt:test-set-state-status-by-id run-id test-id #f status #f) + (rmt:roll-up-pass-fail-counts run-id test-id #f #f status #f) ;; test-name passed in as test-id is respected (db:test-set-status! testdat status)))))))) btn)) (map cadr *common:std-statuses*)))) ;; (list "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP")))) (vector-set! *state-status* 1 (lambda (status color) @@ -415,13 +417,13 @@ ;;====================================================================== ;; ;;====================================================================== (define (dashboard-tests:examine-test run-id test-id) ;; run-id run-key origtest) - (let* ((db-path (db:dbfile-path run-id)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/db/" run-id ".db")) - (dbstruct (make-dbr:dbstruct path: (db:dbfile-path #f) ;; (configf:lookup *configdat* "setup" "linktree") - local: #t)) + (let* ((db-path (db:dbfile-path)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/db/" run-id ".db")) + (dbstruct #f) ;; NOT ACTUALLY USED (db:setup)) ;; (make-dbr:dbstruct path: (db:dbfile-path #f) ;; (configf:lookup *configdat* "setup" "linktree") + ;; local: #t)) (testdat (rmt:get-test-info-by-id run-id test-id)) ;; (db:get-test-info-by-id dbstruct run-id test-id)) (db-mod-time 0) ;; (file-modification-time db-path)) (last-update 0) ;; (current-seconds)) (request-update #t)) (if (not testdat) @@ -513,11 +515,11 @@ ;; NOTE: BUG HIDER, try to eliminate this exception handler (handle-exceptions exn (debug:print-info 0 *default-log-port* "test db access issue in examine test for run-id " run-id ", test-id " test-id ": " ((condition-property-accessor 'exn 'message) exn)) (rmt:get-test-info-by-id run-id test-id ))))) - ;; (debug:print-info 0 *default-log-port* "need-update= " need-update " curr-mod-time = " curr-mod-time) + ;; (print "INFO: need-update= " need-update " curr-mod-time = " curr-mod-time) (cond ((and need-update newtestdat) (set! testdat newtestdat) (set! teststeps (tests:get-compressed-steps run-id test-id)) (set! logfile (conc (db:test-get-rundir testdat) "/" (db:test-get-final_logf testdat))) @@ -628,15 +630,13 @@ " -run -preclean -testpatt " (conc testname "/" (if (equal? item-path "") "%" item-path)) " -clean-cache" ))) - (common:without-vars - (conc (dtests:get-pre-command) - cmd - (dtests:get-post-command)) - "MT_.*")))) + (thread-start! (make-thread (lambda () + (common:run-a-command cmd)) + "clean-run-execute"))))) (remove-test (lambda (x) (iup:attribute-set! command-text-box "VALUE" (conc "megatest -remove-runs -target " keystring " -runname " runname " -testpatt " (conc testname "/" (if (equal? item-path "") Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -53,10 +53,11 @@ Usage: dashboard [options] -h : this help -test run-id,test-id : control test identified by testid -skip-version-check : skip the version check + -use-db-cache : access database via cache Misc -rows R : set number of rows -cols C : set number of columns ")) @@ -81,25 +82,42 @@ "-use-server" "-guimonitor" "-main" "-v" "-q" - "-use-local" + "-use-db-cache" "-skip-version-check" + "-repl" ) args:arg-hash 0)) + +(if (not (null? remargs)) + (begin + (print "Unrecognised arguments: " (string-intersperse remargs " ")) + (exit))) (if (args:get-arg "-h") (begin (print help) (exit))) +;; TODO: Move this inside (main) +;; (if (not (launch:setup)) (begin (print "Failed to find megatest.config, exiting") (exit 1))) + +;; create a watch dog to move changes from lt/.db/*.db to megatest.db +;; +(if (file-write-access? (conc *toppath* "/megatest.db")) + (thread-start! (make-thread common:watchdog "Watchdog thread")) + (if (not (args:get-arg "-use-db-cache")) + (begin + (debug:print-info 0 *default-log-port* "Forcing db-cache mode due to read-only access to megatest.db") + (hash-table-set! args:arg-hash "-use-db-cache" #t)))) ;; data common to all tabs goes here ;; (defstruct dboard:commondat ((curr-tab-num 0) : number) @@ -109,11 +127,10 @@ updaters updating uidat ;; needs to move to tabdat at some time hide-not-hide-tabs ) - (define (dboard:commondat-make) (make-dboard:commondat curr-tab-num: 0 tabdats: (make-hash-table) @@ -139,10 +156,11 @@ (dboard:commondat-tabdats commondat) tabnum tabdat)) ;; gets and calls updater list based on curr-tab-num +;; (define (dboard:common-run-curr-updaters commondat #!key (tab-num #f)) (if (dboard:common-get-tabdat commondat tab-num: tab-num) ;; only update if there is a tabdat (let* ((tnum (or tab-num (dboard:commondat-curr-tab-num commondat))) (updaters (hash-table-ref/default (dboard:commondat-updaters commondat) tnum @@ -206,13 +224,22 @@ (originx #f) (originy #f) ((layout-update-ok #t) : boolean) ((compact-layout #t) : boolean) + ;; Run times layout + ;; (graph-button-box #f) ;; RA => Think it is not referenced anywhere + (graph-matrix #f) + ((graph-matrix-table (make-hash-table)) : hash-table) ;; graph-dats referenced thru graph name info + ((graph-cell-table (make-hash-table)) : hash-table) ;; graph-dats referenced thru matrix cell info + ((graph-matrix-row 1) : number) + ((graph-matrix-col 1) : number) + ;; Controls used to launch runs etc. ((command "") : string) ;; for run control this is the command being built up - (command-tb #f) + (command-tb #f) ;; widget for the type of command; run, remove-runs etc. + (test-patterns-textbox #f) ;; text box widget for editing a list of test patterns (key-listboxes #f) (key-lbs #f) run-name ;; from run name setting widget states ;; states for -state s1,s2 ... statuses ;; statuses for -status s1,s2 ... @@ -231,29 +258,30 @@ ((status-ignore-hash (make-hash-table)) : hash-table) ;; hash of STATUS => #t/#f (target #f) (test-patts #f) ;; db info to file the .db files for the area + (access-mode (db:get-access-mode)) ;; use cached db or not (dbdir #f) (dbfpath #f) (dbkeys #f) - ((last-db-update 0) : number) ;; last db file timestamp + ((last-db-update (make-hash-table)) : hash-table) ;; last db file timestamp (monitor-db-path #f) ;; where to find monitor.db ro ;; is the database read-only? ;; tests data ((num-tests 10) : number) ;; total number of tests to show (used in the old runs display) ;; runs tree ((path-run-ids (make-hash-table)) : hash-table) ;; path (target / runname) => id (runs-tree #f) + ((runs-tree-ht (make-hash-table)) : hash-table) ;; track which targets added to tree (merge functionality with path-run-ids?) ;; tab data ((view-changed #t) : boolean) ((xadj 0) : number) ;; x slider number (if using canvas) ((yadj 0) : number) ;; y slider number (if using canvas) - ;; runs-summary tab state ((runs-summary-modes '((one-run . "Show One Run") (xor-two-runs . "XOR Two Runs") (xor-two-runs-hide-clean . "XOR; Hide Clean")) ) : list) ((runs-summary-mode-buttons '()) : list) ((runs-summary-mode 'one-run) : symbol) ((runs-summary-mode-change-callbacks '()) : list) @@ -267,11 +295,11 @@ (define (dboard:tabdat-target-string vec) (let ((targ (dboard:tabdat-target vec))) (if (list? targ)(string-intersperse targ "/") "no-target-specified"))) (define (dboard:tabdat-test-patts-use vec) - (let ((val (dboard:tabdat-test-patts vec)))(if val val ""))) + (let ((val (dboard:tabdat-test-patts vec)))(if val val ""))) ;;RADT => What is the if for? ;; additional setters for dboard:data (define (dboard:tabdat-test-patts-set!-use vec val) (dboard:tabdat-test-patts-set! vec (if (equal? val "") #f val))) @@ -280,21 +308,29 @@ (dboard:setup-tabdat dat) (dboard:setup-num-rows dat) dat)) (define (dboard:setup-tabdat tabdat) - (dboard:tabdat-dbdir-set! tabdat (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) - (dboard:tabdat-dbfpath-set! tabdat (db:dbfile-path 0)) + (dboard:tabdat-dbdir-set! tabdat (db:dbfile-path)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) + (dboard:tabdat-dbfpath-set! tabdat (db:dbfile-path)) (dboard:tabdat-monitor-db-path-set! tabdat (conc (dboard:tabdat-dbdir tabdat) "/monitor.db")) ;; HACK ALERT: this is a hack, please fix. (dboard:tabdat-ro-set! tabdat (not (file-read-access? (dboard:tabdat-dbfpath tabdat)))) - (dboard:tabdat-keys-set! tabdat (rmt:get-keys)) + (dboard:tabdat-keys-set! tabdat (db:dispatch-query (db:get-access-mode) rmt:get-keys db:get-keys)) (dboard:tabdat-dbkeys-set! tabdat (append (dboard:tabdat-keys tabdat) (list "runname"))) (dboard:tabdat-tot-runs-set! tabdat (rmt:get-num-runs "%")) ) + +;; RADT => Matrix defstruct addition +(defstruct dboard:graph-dat + ((id #f) : string) + ((color #f) : vector) + ((flag #t) : boolean) + ((cell #f) : number) + ) ;; data for runs, tests etc. was used in run summary? ;; (defstruct dboard:runsdat ;; new system @@ -332,55 +368,21 @@ run: run tests: (or tests (make-hash-table)) key-vals: key-vals )) -(define (dboard:rundat-copy-tests-to-by-name rundat) - (let ((src-ht (dboard:rundat-tests rundat)) - (trg-ht (dboard:rundat-tests-by-name rundat))) - (if (and (hash-table? src-ht)(hash-table? trg-ht)) - (begin - (hash-table-clear! trg-ht) - (for-each - (lambda (testdat) - (hash-table-set! trg-ht (test:test-get-fullname testdat) testdat)) - (hash-table-values src-ht))) - (debug:print 0 *default-log-port* "WARNING: src-ht " src-ht " trg-ht " trg-ht)))) - (defstruct dboard:testdat id ;; testid state ;; test state status ;; test status ) -(define (dboard:runsdat-get-col-num dat target runname force-set) - (let* ((runs-index (dboard:runsdat-runs-index dat)) - (col-name (conc target "/" runname)) - (res (hash-table-ref/default runs-index col-name #f))) - (if res - res - (if force-set - (let ((max-col-num (+ 1 (apply max -1 (hash-table-values runs-index))))) - (hash-table-set! runs-index col-name max-col-num) - max-col-num))))) - -(define (dboard:runsdat-get-row-num dat testname itempath force-set) - (let* ((tests-index (dboard:runsdat-runs-index dat)) - (row-name (conc testname "/" itempath)) - (res (hash-table-ref/default runs-index row-name #f))) - (if res - res - (if force-set - (let ((max-row-num (+ 1 (apply max -1 (hash-table-values tests-index))))) - (hash-table-set! runs-index row-name max-row-num) - max-row-num))))) - ;; default is to NOT set the cell if the column and row names are not pre-existing ;; (define (dboard:runsdat-set-test-cell dat target runname testname itempath test-id state status #!key (force-set #f)) - (let* ((col-num (dboard:runsdat-get-col-num dat target runname force-set)) - (row-num (dboard:runsdat-get-row-num dat testname itempath force-set))) + (let* ((col-num (dcommon:runsdat-get-col-num dat target runname force-set)) + (row-num (dcommon:runsdat-get-row-num dat testname itempath force-set))) (if (and row-num col-num) (let ((tdat (dboard:testdat id: test-id state: state status: status))) @@ -488,21 +490,22 @@ ;; gets all the tests for run-id that match testnamepatt and key-vals, merges them ;; ;; NOTE: Yes, this is used ;; (define (dboard:get-tests-for-run-duplicate tabdat run-id run testnamepatt key-vals) - (let* ((num-to-get + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (num-to-get (let ((num-tests-from-config (configf:lookup *configdat* "setup" "num-tests-to-get"))) (if num-tests-from-config (begin (BB> "override num-tests 100 -> "num-tests-from-config) (string->number num-tests-from-config)) 100))) (states (hash-table-keys (dboard:tabdat-state-ignore-hash tabdat))) (statuses (hash-table-keys (dboard:tabdat-status-ignore-hash tabdat))) - (do-not-use-db-file-timestamps (configf:lookup *configdat* "setup" "do-not-use-db-file-timestamps")) ;; this still hosts runs-summary-tab - (do-not-use-query-timestamps (configf:lookup *configdat* "setup" "do-not-use-query-timestamps")) ;; this no longer troubles runs-summary-tab + (do-not-use-db-file-timestamps #t) ;; (configf:lookup *configdat* "setup" "do-not-use-db-file-timestamps")) ;; this still hosts runs-summary-tab + (do-not-use-query-timestamps #t) ;; (configf:lookup *configdat* "setup" "do-not-use-query-timestamps")) ;; this no longer troubles runs-summary-tab (sort-info (get-curr-sort)) (sort-by (vector-ref sort-info 1)) (sort-order (vector-ref sort-info 2)) (bubble-type (if (member sort-order '(testname)) 'testname @@ -525,29 +528,30 @@ (db-pth (conc db-dir "/" run-id ".db"))) (dboard:rundat-db-path-set! run-dat db-pth) db-pth))) (tmptests (if (or do-not-use-db-file-timestamps (>= (common:lazy-modification-time db-path) last-update)) - (rmt:get-tests-for-run run-id testnamepatt states statuses ;; run-id testpatt states statuses - (dboard:rundat-run-data-offset run-dat) - num-to-get - (dboard:tabdat-hide-not-hide tabdat) ;; no-in - sort-by ;; sort-by - sort-order ;; sort-order - #f ;; 'shortlist ;; qrytype - (if (dboard:tabdat-filters-changed tabdat) - 0 - last-update) ;; last-update - *dashboard-mode*) ;; use dashboard mode + (db:dispatch-query access-mode rmt:get-tests-for-run db:get-tests-for-run + run-id testnamepatt states statuses ;; run-id testpatt states statuses + (dboard:rundat-run-data-offset run-dat) + num-to-get + (dboard:tabdat-hide-not-hide tabdat) ;; no-in + sort-by ;; sort-by + sort-order ;; sort-order + #f ;; 'shortlist ;; qrytype + (if (dboard:tabdat-filters-changed tabdat) + 0 + last-update) ;; last-update + *dashboard-mode*) ;; use dashboard mode '())) (use-new (dboard:tabdat-hide-not-hide tabdat)) (tests-ht (if (dboard:tabdat-filters-changed tabdat) (let ((ht (make-hash-table))) (dboard:rundat-tests-set! run-dat ht) ht) - (dboard:rundat-tests run-dat))) - (start-time (current-seconds))) + (dboard:rundat-tests run-dat)))) + ;;(start-time (current-seconds))) ;; to limit the amount of data transferred each cycle use limit of num-to-get and offset (dboard:rundat-run-data-offset-set! run-dat (if (< (length tmptests) num-to-get) @@ -601,15 +605,18 @@ ;; ;; create a virtual table of all the tests ;; keypatts: ( (KEY1 "abc%def")(KEY2 "%") ) ;; (define (update-rundat tabdat runnamepatt numruns testnamepatt keypatts) - (let* ((keys (rmt:get-keys)) - (last-runs-update (dboard:tabdat-last-runs-update tabdat)) - (allruns (rmt:get-runs runnamepatt numruns (dboard:tabdat-start-run-offset tabdat) keypatts)) + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (keys (db:dispatch-query access-mode rmt:get-keys db:get-keys)) + (last-runs-update (- (dboard:tabdat-last-runs-update tabdat) 2)) + (allruns (db:dispatch-query access-mode rmt:get-runs db:get-runs + runnamepatt numruns (dboard:tabdat-start-run-offset tabdat) keypatts)) ;;(allruns-tree (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f)) - (allruns-tree (rmt:get-runs-by-patt keys "%" #f #f #f #f last-runs-update));;'("id" "runname") + (allruns-tree (db:dispatch-query access-mode rmt:get-runs-by-patt db:get-runs-by-patt + keys "%" #f #f #f #f last-runs-update));;'("id" "runname") (header (db:get-header allruns)) (runs (db:get-rows allruns)) ;; RA => Filtered as per runpatt selected (runs-tree (db:get-rows allruns-tree)) ;; RA => Returns complete list of runs (start-time (current-seconds)) (runs-hash (let ((ht (make-hash-table))) @@ -671,12 +678,88 @@ (loop run tal new-res newmaxtests) ;; not done getting data for this run (loop (car tal)(cdr tal) new-res newmaxtests))))))) (dboard:tabdat-filters-changed-set! tabdat #f) (dboard:update-tree tabdat runs-hash header tb))) - - +;; this calls dboard:get-tests-for-run-duplicate for each run +;; +;; create a virtual table of all the tests +;; keypatts: ( (KEY1 "abc%def")(KEY2 "%") ) +;; +(define (dboard:update-rundat tabdat runnamepatt numruns testnamepatt keypatts) + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (keys (dboard:tabdat-keys tabdat)) ;; (db:dispatch-query access-mode rmt:get-keys db:get-keys))) + (last-runs-update (- (dboard:tabdat-last-runs-update tabdat) 2)) + (allruns (db:dispatch-query access-mode rmt:get-runs db:get-runs + runnamepatt numruns (dboard:tabdat-start-run-offset tabdat) keypatts)) + ;;(allruns-tree (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f)) + (allruns-tree (db:dispatch-query access-mode rmt:get-runs-by-patt db:get-runs-by-patt + keys "%" #f #f #f #f 0)) ;; last-runs-update));;'("id" "runname") + (header (db:get-header allruns)) + (runs (db:get-rows allruns)) ;; RA => Filtered as per runpatt selected + (runs-tree (db:get-rows allruns-tree)) ;; RA => Returns complete list of runs + (start-time (current-seconds)) + (runs-hash (let ((ht (make-hash-table))) + (for-each (lambda (run) + (hash-table-set! ht (db:get-value-by-header run header "id") run)) + runs-tree) ;; (vector-ref runs-dat 1)) + ht)) + (tb (dboard:tabdat-runs-tree tabdat))) + (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2)) + (dboard:tabdat-header-set! tabdat header) + ;; + ;; trim runs to only those that are changing often here + ;; + (if (null? runs) + (begin + (dboard:tabdat-allruns-set! tabdat '()) + (dboard:tabdat-all-test-names-set! tabdat '()) + (dboard:tabdat-item-test-names-set! tabdat '()) + (hash-table-clear! (dboard:tabdat-allruns-by-id tabdat))) + (let loop ((run (car runs)) + (tal (cdr runs)) + (res '()) + (maxtests 0)) + (let* ((run-id (db:get-value-by-header run header "id")) + (run-struct (hash-table-ref/default (dboard:tabdat-allruns-by-id tabdat) run-id #f)) + ;; (last-update (if run-struct (dboard:rundat-last-update run-struct) 0)) + (key-vals (rmt:get-key-vals run-id)) + (tests-ht (dboard:get-tests-for-run-duplicate tabdat run-id run testnamepatt key-vals)) + ;; GET RID OF dboard:get-tests-dat - it is superceded by dboard:get-tests-for-run-duplicate + ;; dboard:get-tests-for-run-duplicate - returns a hash table + ;; (dboard:get-tests-dat tabdat run-id last-update)) + (all-test-ids (hash-table-keys tests-ht)) + (num-tests (length all-test-ids))) + ;; (print "run-struct: " run-struct) + ;; NOTE: bubble-up also sets the global (dboard:tabdat-item-test-names tabdat) + ;; (tests (bubble-up tmptests priority: bubble-type)) + ;; NOTE: 11/01/2013 This routine is *NOT* getting called excessively. + ;; (debug:print 0 *default-log-port* "Getting data for run " run-id " with key-vals=" key-vals) + ;; Not sure this is needed? + (let* ((newmaxtests (max num-tests maxtests)) + ;; (last-update (- (current-seconds) 10)) + (run-struct (or run-struct + (dboard:rundat-make-init + run: run + tests: tests-ht + key-vals: key-vals))) + (new-res (if (null? all-test-ids) res (cons run-struct res))) + (elapsed-time (- (current-seconds) start-time))) + (if (null? all-test-ids) + (hash-table-delete! (dboard:tabdat-allruns-by-id tabdat) run-id) + (hash-table-set! (dboard:tabdat-allruns-by-id tabdat) run-id run-struct)) + (if (or (null? tal) + (> elapsed-time 2)) ;; stop loading data after 5 seconds, on the next call more data *should* be loaded since get-tests-for-run uses last update + (begin + (if (> elapsed-time 2)(print "NOTE: updates are taking a long time, " elapsed-time "s elapsed.")) + (dboard:tabdat-allruns-set! tabdat new-res) + maxtests) + (if (> (dboard:rundat-run-data-offset run-struct) 0) + (loop run tal new-res newmaxtests) ;; not done getting data for this run + (loop (car tal)(cdr tal) new-res newmaxtests))))))) + (dboard:tabdat-filters-changed-set! tabdat #f) + (dboard:update-tree tabdat runs-hash header tb))) (define *collapsed* (make-hash-table)) (define (toggle-hide lnum uidat) ; fulltestname) (let* ((btn (vector-ref (dboard:uidat-get-lftcol uidat) lnum)) @@ -833,11 +916,11 @@ (for-each (lambda (rundat) (if rundat (let* ((testdats (dboard:rundat-tests rundat)) (testnames (map test:test-get-fullname (hash-table-values testdats)))) - (dboard:rundat-copy-tests-to-by-name rundat) + (dcommon:rundat-copy-tests-to-by-name rundat) ;; for the normalized list of testnames (union of all runs) (if (not (and (dboard:tabdat-hide-empty-runs tabdat) (null? testnames))) (for-each (lambda (testname) (hash-table-set! all-test-names testname #t)) @@ -927,11 +1010,10 @@ (curr-title (vector-ref buttondat 2))) ;; (iup:attribute button "TITLE"))) (if (not (equal? curr-color color)) (iup:attribute-set! button "BGCOLOR" color)) (if (not (equal? curr-title buttontxt)) (iup:attribute-set! button "TITLE" buttontxt)) - ;;(print "RA => testdat " testdat " teststate " teststate " teststatus " teststatus " buttondat " buttondat " curr-color " curr-color " curr-title " curr-title "buttontxt" buttontxt " title " curr-title ) (vector-set! buttondat 0 run-id) (vector-set! buttondat 1 color) (vector-set! buttondat 2 buttontxt) (vector-set! buttondat 3 testdat) (vector-set! buttondat 4 run-key))) @@ -961,13 +1043,14 @@ (define (update-search commondat tabdat x val) (hash-table-set! (dboard:tabdat-searchpatts tabdat) x val) (dboard:tabdat-filters-changed-set! tabdat #t) (set-bg-on-filter commondat tabdat)) +;; force ALL updates to zero (effectively) +;; (define (mark-for-update tabdat) - ;; (dboard:tabdat-filters-changed-set! tabdat #t) - (dboard:tabdat-last-db-update-set! tabdat 0)) + (dboard:tabdat-last-db-update-set! tabdat (make-hash-table))) ;;====================================================================== ;; R U N C O N T R O L ;;====================================================================== @@ -1098,11 +1181,14 @@ ;; (define (dashboard:update-run-command tabdat) (let* ((cmd-tb (dboard:tabdat-command-tb tabdat)) (cmd (dboard:tabdat-command tabdat)) (test-patt (let ((tp (dboard:tabdat-test-patts tabdat))) - (if (equal? tp "") "%" tp))) + (if (or (not tp) + (equal? tp "")) + "%" + tp))) (states (dboard:tabdat-states tabdat)) (statuses (dboard:tabdat-statuses tabdat)) (target (let ((targ-list (dboard:tabdat-target tabdat))) (if targ-list (string-intersperse targ-list "/") "no-target-selected"))) (run-name (dboard:tabdat-run-name tabdat)) @@ -1185,10 +1271,11 @@ ;; (define (dashboard:update-tree-selector tabdat #!key (action-proc #f)) (let* ((tb (dboard:tabdat-runs-tree tabdat)) (runconf-targs (common:get-runconfig-targets)) (db-target-dat (rmt:get-targets)) + (runs-tree-ht (dboard:tabdat-runs-tree-ht tabdat)) (header (vector-ref db-target-dat 0)) (db-targets (vector-ref db-target-dat 1)) (munge-target (lambda (x) ;; create a target vector from a string. Pad with na if needed. (take (append (string-split x "/") (make-list (length header) "na")) @@ -1200,13 +1287,20 @@ (map munge-target runconf-targs) ))) (for-each (lambda (target) - (tree:add-node tb "Runs" target)) ;; (append key-vals (list run-name)) + (if (not (hash-table-ref/default runs-tree-ht target #f)) + ;; (let ((existing (tree:find-node tb target))) + ;; (if (not existing) + (begin + (tree:add-node tb "Runs" target) ;; (append key-vals (list run-name)) + (hash-table-set! runs-tree-ht target #t)))) all-targets))) +;; Run controls panel +;; (define (dashboard:run-controls commondat tabdat #!key (tab-num #f)) (let* ((targets (make-hash-table)) (test-records (make-hash-table)) (all-tests-registry (tests:get-all)) ;; (tests:get-valid-tests *toppath* '())) (test-names (hash-table-keys all-tests-registry)) @@ -1247,11 +1341,12 @@ (dcommon:command-tests-tasks-canvas tabdat test-records sorted-testnames tests-draw-state)))) (tb (dboard:tabdat-runs-tree tabdat))) (dboard:commondat-add-updater commondat (lambda () - (dashboard:update-tree-selector tabdat)) + (if (dashboard:database-changed? commondat tabdat context-key: 'run-control) + (dashboard:update-tree-selector tabdat))) tab-num: tab-num) result))) ;;(iup:frame ;; #:title "Logs" ;; To be replaced with tabs @@ -1259,24 +1354,38 @@ ;; #:multiline "YES"))) ;; (dboard:tabdat-logs-textbox-set! tabdat logs-tb) ;; logs-tb)) (define (dboard:runs-tree-browser commondat tabdat) - (let* ((tb + (let* ((txtbox (iup:textbox #:action (lambda (val a b) + (debug:catch-and-dump + (lambda () + (if b (dboard:tabdat-target-set! tabdat (string-split b "/"))) + (dashboard:update-run-command tabdat)) + "command-testname-selector tb action")) + #:value (dboard:test-patt->lines + (dboard:tabdat-test-patts-use tabdat)) + #:expand "HORIZONTAL" + ;; #:size "10x30" + )) + (tb (iup:treebox #:value 0 #:name "Runs" #:expand "YES" #:addexpanded "NO" + #:size "10x" #:selection-cb (lambda (obj id state) (debug:catch-and-dump (lambda () (let* ((run-path (tree:node->path obj id)) (run-id (tree-path->run-id tabdat (cdr run-path)))) ;; (dboard:tabdat-view-changed-set! tabdat #t) ;; ?? done below when run-id is a number - (dboard:tabdat-target-set! tabdat (cdr run-path)) ;; (print "run-path: " run-path) + (dboard:tabdat-target-set! tabdat (cdr run-path)) ;; (print "run-path: " run-path) + (iup:attribute-set! txtbox "VALUE" (string-intersperse (cdr run-path) "/")) + (dashboard:update-run-command tabdat) (dboard:tabdat-layout-update-ok-set! tabdat #f) (if (number? run-id) (begin ;; capture last two in tabdat. (dboard:tabdat-prev-run-id-set! @@ -1287,11 +1396,11 @@ (debug:print-error 5 *default-log-port* "tree-path->run-id returned non-number " run-id)))) "treebox")) ;; (print "path: " (tree:node->path obj id) " run-id: " run-id) ))) (dboard:tabdat-runs-tree-set! tabdat tb) - tb)) + (iup:vbox tb txtbox))) ;;====================================================================== ;; R U N C O N T R O L S ;;====================================================================== ;; @@ -1352,10 +1461,13 @@ ) "text-list-toggle-box")))) (dcommon:command-runname-selector commondat tabdat tab-num: tab-num) (dcommon:command-testname-selector commondat tabdat update-keyvals)) (iup:vbox + (iup:split + #:orientation "HORIZONTAL" + #:value 800 (let* ((cnv-obj (iup:canvas ;; #:size "500x400" #:expand "YES" #:scrollbar "YES" #:posx "0.5" @@ -1392,11 +1504,61 @@ (if (> step 0) (* scalex 0.02) (* scalex -0.02)))))) "wheel-cb")) ))) - cnv-obj))))) + cnv-obj) + (let* ((hb1 (iup:hbox)) + (graph-cell-table (dboard:tabdat-graph-cell-table tabdat)) + (changed #f) + (graph-matrix (iup:matrix + #:alignment1 "ALEFT" + #:expand "YES" ;; "HORIZONTAL" + #:scrollbar "YES" + #:numcol 10 + #:numlin 20 + #:numcol-visible (min 8) + #:numlin-visible 1 + #:click-cb + (lambda (obj row col status) + (let* + ((graph-cell (conc row ":" col)) + (graph-dat (hash-table-ref/default graph-cell-table graph-cell #f)) + (graph-flag (dboard:graph-dat-flag graph-dat))) + (if graph-flag + (dboard:graph-dat-flag-set! graph-dat #f) + (dboard:graph-dat-flag-set! graph-dat #t)) + (if (not (dboard:tabdat-running-layout tabdat)) + (begin + (dashboard:run-times-tab-run-data-updater commondat tabdat tab-num) + (dboard:tabdat-last-data-update-set! tabdat (current-seconds)) + (thread-start! (make-thread + (lambda () + (dboard:tabdat-running-layout-set! tabdat #t) + (dashboard:run-times-tab-layout-updater commondat tabdat tab-num) + (dboard:tabdat-running-layout-set! tabdat #f)) + "run-times-tab-layout-updater")))) + ;;(dboard:tabdat-view-changed-set! tabdat #t) + ))))) + (dboard:tabdat-graph-matrix-set! tabdat graph-matrix) + (iup:attribute-set! graph-matrix "WIDTH0" 0) + (iup:attribute-set! graph-matrix "HEIGHT0" 0) + graph-matrix)) + (iup:hbox + (iup:vbox + (iup:button "Show All" #:action (lambda (obj) + (for-each (lambda (graph-cell) + (let* ((graph-dat (hash-table-ref (dboard:tabdat-graph-cell-table tabdat) graph-cell))) + (dboard:graph-dat-flag-set! graph-dat #t))) + (hash-table-keys (dboard:tabdat-graph-cell-table tabdat)))))) + (iup:hbox + (iup:button "Hide All" #:action (lambda (obj) + (for-each (lambda (graph-cell) + (let* ((graph-dat (hash-table-ref (dboard:tabdat-graph-cell-table tabdat) graph-cell))) + (dboard:graph-dat-flag-set! graph-dat #f))) + (hash-table-keys (dboard:tabdat-graph-cell-table tabdat))))))) + )))) ;;====================================================================== ;; R U N ;;====================================================================== ;; @@ -1406,11 +1568,13 @@ (if (not (null? path)) (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) path #f) #f)) (define (dboard:get-tests-dat tabdat run-id last-update) - (let* ((tdat (if run-id (rmt:get-tests-for-run run-id + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (tdat (if run-id (db:dispatch-query access-mode rmt:get-tests-for-run db:get-tests-for-run + run-id (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "test-name" "%/%") (hash-table-keys (dboard:tabdat-state-ignore-hash tabdat)) ;; '() (hash-table-keys (dboard:tabdat-status-ignore-hash tabdat)) ;; '() #f #f ;; offset limit (dboard:tabdat-hide-not-hide tabdat) ;; not-in @@ -1437,41 +1601,44 @@ (if (and res (> (length res) 1)) (cadr res) #f))) (define (dboard:update-tree tabdat runs-hash runs-header tb) - (let* ((run-ids (sort (filter number? (hash-table-keys runs-hash)) + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (run-ids (sort (filter number? (hash-table-keys runs-hash)) (lambda (a b) (let* ((record-a (hash-table-ref runs-hash a)) (record-b (hash-table-ref runs-hash b)) (time-a (db:get-value-by-header record-a runs-header "event_time")) (time-b (db:get-value-by-header record-b runs-header "event_time"))) (< time-a time-b))))) (changed #f) (last-runs-update (dboard:tabdat-last-runs-update tabdat)) - (runs-dat (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update))) + (runs-dat (db:dispatch-query access-mode rmt:get-runs-by-patt db:get-runs-by-patt + (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update))) (dboard:tabdat-last-runs-update-set! tabdat (- (current-seconds) 2)) (for-each (lambda (run-id) (let* ((run-record (hash-table-ref/default runs-hash run-id #f)) (key-vals (map (lambda (key)(db:get-value-by-header run-record runs-header key)) (dboard:tabdat-keys tabdat))) (run-name (db:get-value-by-header run-record runs-header "runname")) (col-name (conc (string-intersperse key-vals "\n") "\n" run-name)) - (run-path (append key-vals (list run-name))) - (existing (tree:find-node tb run-path))) + (run-path (append key-vals (list run-name)))) (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f)) - (begin - (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path) - ;; (iup:attribute-set! (dboard:tabdat-runs-matrix tabdat) - ;; (conc rownum ":" colnum) col-name) - ;; (hash-table-set! runid-to-col run-id (list colnum run-record)) - ;; Here we update the tests treebox and tree keys - (tree:add-node tb "Runs" run-path ;; (append key-vals (list run-name)) - userdata: (conc "run-id: " run-id)) - (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id) - ;; (set! colnum (+ colnum 1)) - )))) + ;; (let ((existing (tree:find-node tb run-path))) + ;; (if (not existing) + (begin + (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path) + ;; (iup:attribute-set! (dboard:tabdat-runs-matrix tabdat) + ;; (conc rownum ":" colnum) col-name) + ;; (hash-table-set! runid-to-col run-id (list colnum run-record)) + ;; Here we update the tests treebox and tree keys + (tree:add-node tb "Runs" run-path) ;; (append key-vals (list run-name)) + ;; userdata: (conc "run-id: " run-id)))) + (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id) + ;; (set! colnum (+ colnum 1)) + )))) run-ids))) (define (dashboard:tests-ht->tests-dat tests-ht) (reverse (sort @@ -1514,12 +1681,14 @@ hide-clean: hide-clean) #f))) (define (dashboard:get-runs-hash tabdat) - (let* ((last-runs-update 0);;(dboard:tabdat-last-runs-update tabdat)) - (runs-dat (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (last-runs-update 0);;(dboard:tabdat-last-runs-update tabdat)) + (runs-dat (db:dispatch-query access-mode rmt:get-runs-by-patt db:get-runs-by-patt + (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) (runs-header (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records (runs (vector-ref runs-dat 1)) (run-id (dboard:tabdat-curr-run-id tabdat)) (runs-hash (let ((ht (make-hash-table))) (for-each (lambda (run) @@ -1527,14 +1696,17 @@ runs) ht))) runs-hash)) (define (dashboard:runs-summary-updater commondat tabdat tb cell-lookup run-matrix) - (dashboard:do-update-rundat tabdat) + ;; (if (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-rundat) + (dashboard:do-update-rundat tabdat) ;; ) (dboard:runs-summary-control-panel-updater tabdat) (let* ((last-runs-update (dboard:tabdat-last-runs-update tabdat)) - (runs-dat (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) + (runs-dat (db:dispatch-query (dboard:tabdat-access-mode tabdat) + rmt:get-runs-by-patt db:get-runs-by-patt + (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) (runs-header (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records (runs (vector-ref runs-dat 1)) (run-id (dboard:tabdat-curr-run-id tabdat)) (runs-hash (dashboard:get-runs-hash tabdat)) ;; (runs-hash (let ((ht (make-hash-table))) @@ -1541,11 +1713,12 @@ ;; (for-each (lambda (run) ;; (hash-table-set! ht (db:get-value-by-header run runs-header "id") run)) ;; runs) ;; ht)) ) - (dboard:update-tree tabdat runs-hash runs-header tb) + (if (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-tree) + (dboard:update-tree tabdat runs-hash runs-header tb)) (if run-id (let* ((matrix-content (case (dboard:tabdat-runs-summary-mode tabdat) ((one-run) (dashboard:run-id->tests-mindat run-id tabdat runs-hash)) ((xor-two-runs) (dashboard:runs-summary-xor-matrix-content tabdat runs-hash)) @@ -1560,20 +1733,16 @@ (max-visible (max (- (dboard:tabdat-num-tests tabdat) 15) 3)) ;; (dboard:tabdat-num-tests tabdat) is proportional to the size of the window (numrows 1) (numcols 1) (changed #f) ) - - - - (dboard:tabdat-filters-changed-set! tabdat #f) (let loop ((pass-num 0) (changed #f)) ;; Update the runs tree - (dboard:update-tree tabdat runs-hash runs-header tb) + ;; (dboard:update-tree tabdat runs-hash runs-header tb) (if (eq? pass-num 1) (begin ;; big reset (iup:attribute-set! run-matrix "CLEARVALUE" "ALL") ;; NOTE: Was CONTENTS (iup:attribute-set! run-matrix "CLEARATTRIB" "CONTENTS") @@ -1618,10 +1787,11 @@ (hash-table-set! cell-lookup key test-id) (if (not (equal? (iup:attribute run-matrix key) (cadr value))) (begin (set! changed #t) (iup:attribute-set! run-matrix key (cadr value)) + ;; (print "RA=> value" (car value)) (iup:attribute-set! run-matrix (conc "BGCOLOR" key) (car value)))))) matrix-content) ;; Col labels - do after setting Cell contents so they are accounted for in the size calc. @@ -1921,11 +2091,11 @@ )) "runs-summary-click-callback")))) (runs-summary-updater (lambda () (mutex-lock! update-mutex) - (if (or (dashboard:database-changed? commondat tabdat) + (if (or (dashboard:database-changed? commondat tabdat context-key: 'runs-summary-updater) (dboard:tabdat-view-changed tabdat)) (debug:catch-and-dump (lambda () ;; check that run-matrix is initialized before calling the updater (if run-matrix (dashboard:runs-summary-updater commondat tabdat tb cell-lookup run-matrix))) @@ -1964,11 +2134,10 @@ (mark-for-update tabdat) (update-search commondat tabdat "test-name" val)) "make-controls"))) (iup:hbox (iup:button "Quit" #:action (lambda (obj) - ;; (if (dboard:tabdat-dblocal tabdat) (db:close-all (dboard:tabdat-dblocal tabdat))) (exit)) #:expand "NO" #:size "40x15") (iup:button "Refresh" #:action (lambda (obj) (mark-for-update tabdat)) #:expand "NO" #:size "40x15") @@ -2170,11 +2339,11 @@ (lambda (obj) (common:run-a-command (conc "megatest -remove-runs -target " target " -runname " runname " -testpatt % ")))) - (iup:menu-item ;; RADT => itemize this run lists before merging with v1.61 + (iup:menu-item "Kill Complete Run" #:action (lambda (obj) (common:run-a-command (conc "megatest -set-state-status KILLREQ,n/a -target " target @@ -2495,37 +2664,31 @@ (iup:attribute-set! *tim* "TIME" 300) (iup:attribute-set! *tim* "RUN" "YES") (define *last-recalc-ended-time* 0) -(define (dashboard:been-changed tabdat) - (> (file-modification-time (dboard:tabdat-dbfpath tabdat)) (dboard:tabdat-last-db-update tabdat))) - -(define (dashboard:set-db-update-time tabdat) - (dboard:tabdat-last-db-update-set! tabdat (file-modification-time (dboard:tabdat-dbfpath tabdat)))) - (define (dashboard:recalc modtime please-update-buttons last-db-update-time) (or please-update-buttons - (and (> (current-milliseconds)(+ *last-recalc-ended-time* 150)) - (> modtime last-db-update-time) + (and ;; (> (current-milliseconds)(+ *last-recalc-ended-time* 150)) ;; can't use this - it needs to be tab specific + (> modtime (- last-db-update-time 3)) ;; add three seconds of margin (> (current-seconds)(+ last-db-update-time 1))))) ;; (define *monitor-db-path* #f) (define *last-monitor-update-time* 0) ;; Force creation of the db in case it isn't already there. (tasks:open-db) -(define (dashboard:get-youngest-run-db-mod-time tabdat) +(define (dashboard:get-youngest-run-db-mod-time dbdir) (handle-exceptions exn (begin (debug:print 0 *default-log-port* "WARNING: error in accessing databases in get-youngest-run-db-mod-time: " ((condition-property-accessor 'exn 'message) exn)) (current-seconds)) ;; something went wrong - just print an error and return current-seconds - (apply max (map (lambda (filen) - (file-modification-time filen)) - (glob (conc (dboard:tabdat-dbdir tabdat) "/*.db")))))) + (common:max (map (lambda (filen) + (file-modification-time filen)) + (glob (conc dbdir "/*.db*")))))) (define (dashboard:monitor-changed? commondat tabdat) (let* ((run-update-time (current-seconds)) (monitor-db-path (dboard:tabdat-monitor-db-path tabdat)) (monitor-modtime (if (and monitor-db-path (file-exists? monitor-db-path)) @@ -2537,16 +2700,31 @@ (begin (set! *last-monitor-update-time* run-update-time) ;; monitor-modtime) #t) #f))) -(define (dashboard:database-changed? commondat tabdat) +(define (dboard:get-last-db-update tabdat context) + (hash-table-ref/default (dboard:tabdat-last-db-update tabdat) context 0)) + +(define (dboard:set-last-db-update! tabdat context newtime) + (hash-table-set! (dboard:tabdat-last-db-update tabdat) context newtime)) + +;; DOES NOT WORK RELIABLY WITH /tmp WAL mode files. Timestamps only change when the db +;; is closed (I think). If db dir starts with /tmp always return true +;; +(define (dashboard:database-changed? commondat tabdat #!key (context-key 'default)) (let* ((run-update-time (current-seconds)) - (modtime (dashboard:get-youngest-run-db-mod-time tabdat)) ;; NOTE: ensure this is tabdat!! - (recalc (dashboard:recalc modtime (dboard:commondat-please-update commondat) (dboard:tabdat-last-db-update tabdat)))) - (dboard:commondat-please-update-set! commondat #f) - recalc)) + (dbdir (dboard:tabdat-dbdir tabdat)) + (modtime (dashboard:get-youngest-run-db-mod-time dbdir)) + (recalc (dashboard:recalc modtime + (dboard:commondat-please-update commondat) + (dboard:get-last-db-update tabdat context-key)))) + ;; (dboard:tabdat-last-db-update tabdat)))) + (if recalc + (dboard:set-last-db-update! tabdat context-key run-update-time)) + (dboard:commondat-please-update-set! commondat #f) + recalc)) ;; point inside line ;; (define-inline (dashboard:px-between px lx1 lx2) (and (< lx1 px)(> lx2 px))) @@ -2585,20 +2763,10 @@ (cons (cons x1 x2) (hash-table-ref/default rowhash (+ i rownum) '()))) (if (< i num-rows) (loop (+ i 1))))) -;; get min or max, use > for max and < for min, this works around the limits on apply -;; -(define (dboard:min-max comp lst) - (if (null? lst) - #f ;; better than an exception for my needs - (fold (lambda (a b) - (if (comp a b) a b)) - (car lst) - lst))) - ;; sort a list of test-ids by the event _time using a hash table of id => testdat ;; (define-inline (dboard:sort-testsdat-by-event-time test-ids tests-ht) (sort test-ids (lambda (a b) @@ -2648,12 +2816,15 @@ (db:test-get-event_time (hash-table-ref testsdat (car b)))))))))) ;; run times tab data updater ;; (define (dashboard:run-times-tab-run-data-updater commondat tabdat tab-num) - (let* ((last-runs-update (dboard:tabdat-last-runs-update tabdat)) - (runs-dat (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) + (let* ((access-mode (dboard:tabdat-access-mode tabdat)) + (last-runs-update (dboard:tabdat-last-runs-update tabdat)) + (runs-dat (db:dispatch-query access-mode + rmt:get-runs-by-patt db:get-runs-by-patt + (dboard:tabdat-keys tabdat) "%" #f #f #f #f last-runs-update)) (runs-header (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records (runs-hash (let ((ht (make-hash-table))) (for-each (lambda (run) (hash-table-set! ht (db:get-value-by-header run runs-header "id") run)) (vector-ref runs-dat 1)) @@ -2678,18 +2849,18 @@ (let* ((run-record (hash-table-ref/default runs-hash run-id #f)) (key-vals (map (lambda (key)(db:get-value-by-header run-record runs-header key)) (dboard:tabdat-keys tabdat))) (run-name (db:get-value-by-header run-record runs-header "runname")) (col-name (conc (string-intersperse key-vals "\n") "\n" run-name)) - (run-path (append key-vals (list run-name))) - (existing (tree:find-node tb run-path))) + (run-path (append key-vals (list run-name)))) + ;; (existing (tree:find-node tb run-path))) (if (not (hash-table-ref/default (dboard:tabdat-path-run-ids tabdat) run-path #f)) (begin (hash-table-set! (dboard:tabdat-run-keys tabdat) run-id run-path) ;; Here we update the tests treebox and tree keys - (tree:add-node tb "Runs" run-path ;; (append key-vals (list run-name)) - userdata: (conc "run-id: " run-id)) + (tree:add-node tb "Runs" run-path) ;; (append key-vals (list run-name)) + ;; userdata: (conc "run-id: " run-id)) (hash-table-set! (dboard:tabdat-path-run-ids tabdat) run-path run-id) ;; (set! colnum (+ colnum 1)) )))) run-ids)) ;; (print "Updating rundat") @@ -2701,13 +2872,15 @@ (take (append (or (dboard:tabdat-target tabdat);; (string-split (dboard: "/") '("%" "%")) (make-list num-keys "%")) num-keys) )) - (runpatt (if (dboard:tabdat-target tabdat) - (last (dboard:tabdat-target tabdat)) - "%")) + (runpatt (if (and (dboard:tabdat-target tabdat) + (list? (dboard:tabdat-target tabdat)) + (not (null? (dboard:tabdat-target tabdat)))) + (last (dboard:tabdat-target tabdat)) + "%")) (testpatt (or (dboard:tabdat-test-patts tabdat) "%")) (filtrstr (conc targpatt "/" runpatt "/" testpatt))) ;; (print "targpatt: " targpatt " runpatt: " runpatt " testpatt: " testpatt) (if (not (equal? (dboard:tabdat-last-filter-str tabdat) filtrstr)) @@ -2815,11 +2988,11 @@ (apply vector tstart (cdr zeropt)) (hash-table-ref/default res-ht fieldname '()))))))) fields) res-ht) #f))))) - + ;; graph data ;; tsc=timescale, tfn=function; time->x ;; (define (dboard:graph commondat tabdat tabnum llx lly ulx uly tstart tend tsc tfn compname cmargin) (let* ((dwg (dboard:tabdat-drawing tabdat)) @@ -2827,11 +3000,15 @@ (cnv (dboard:tabdat-cnv tabdat)) (dur (- tstart tend)) ;; time duration (cmp (vg:get-component dwg "runslib" compname)) (cfg (configf:get-section *configdat* "graph")) (stdcolor (vg:rgb->number 120 130 140)) - (delta-y (- uly lly))) + (delta-y (- uly lly)) + (graph-matrix-table (dboard:tabdat-graph-matrix-table tabdat)) + (graph-cell-table (dboard:tabdat-graph-cell-table tabdat)) + (graph-matrix (dboard:tabdat-graph-matrix tabdat)) + (changed #f)) (vg:add-obj-to-comp cmp (vg:make-rect-obj llx lly ulx uly)) (vg:add-obj-to-comp cmp @@ -2856,68 +3033,98 @@ (lambda (cf) (let* ((alldat (dboard:graph-read-data (cadr cf) tstart tend))) (if alldat (for-each (lambda (fieldn) - (let* ((dat (hash-table-ref alldat fieldn)) - (vals (map (lambda (x)(vector-ref x 2)) dat))) - (if (not (null? vals)) - (let* ((maxval (apply max vals)) - (minval (min 0 (apply min vals))) - (yoff (- minval lly)) ;; minval)) - (deltaval (- maxval minval)) - (yscale (/ delta-y (if (zero? deltaval) 1 deltaval))) - (yfunc (lambda (y)(+ lly (* yscale (- y minval))))) ;; (lambda (y)(* (+ y yoff) yscale)))) - (graph-color (vg:generate-color))) - ;; (print (car cf) "; maxval: " maxval " minval: " minval " deltaval: " deltaval " yscale: " yscale) - (vg:add-obj-to-comp - cmp - (vg:make-text-obj (- llx 10)(yfunc maxval) (conc maxval))) - (vg:add-obj-to-comp - cmp - (vg:make-text-obj (- llx 10)(yfunc minval) (conc minval))) - (fold - (lambda (next prev) ;; #(time ? val) #(time ? val) - (if prev - (let* ((yval (vector-ref prev 2)) - (yval-next (vector-ref next 2)) - (last-tval (tfn (vector-ref prev 0))) - (last-yval (yfunc yval)) ;; (+ lly (* yscale (vector-ref prev 2)))) - (next-yval (yfunc yval-next)) - (curr-tval (tfn (vector-ref next 0)))) - (if (>= curr-tval last-tval) - (begin - (vg:add-obj-to-comp - cmp - ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale)) - (vg:make-line-obj last-tval last-yval curr-tval last-yval - line-color: graph-color)) - (vg:add-obj-to-comp - cmp - ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale)) - (vg:make-line-obj curr-tval last-yval curr-tval next-yval - line-color: graph-color))) - (print "ERROR: curr-tval is not > last-tval; curr-tval " curr-tval ", last-tval " last-tval)))) - next) - ;; for init create vector tstart,0 - #f ;; (vector tstart minval minval) - dat) - - ;; (for-each - ;; (lambda (dpt) - ;; (let* ((tval (vector-ref dpt 0)) - ;; (yval (vector-ref dpt 2)) - ;; (stval (tfn tval)) - ;; (syval (yfunc yval))) - ;; (vg:add-obj-to-comp - ;; cmp - ;; (vg:make-rect-obj (- stval 2) lly (+ stval 2)(+ lly (* yval yscale)) - ;; fill-color: stdcolor)))) - ;; dat) - )))) ;; for each data point in the series + (let*-values (((dat) (hash-table-ref alldat fieldn)) + ((vals minval maxval) (if (null? dat) + (values '() #f #f) + (let loop ((hed (car dat)) + (tal (cdr dat)) + (res '()) + (min (vector-ref (car dat) 2)) + (max (vector-ref (car dat) 2))) + (let* ((val (vector-ref hed 2)) + (newmin (if (< val min) val min)) + (newmax (if (> val max) val max)) + (newres (cons val res))) + (if (null? tal) + (values (reverse res) newmin newmax) + (loop (car tal)(cdr tal) newres newmin newmax))))))) + (if (not (hash-table-exists? graph-matrix-table fieldn)) + (begin + (let* ((graph-color-rgb (vg:generate-color-rgb)) + (graph-color (vg:iup-color->number graph-color-rgb)) + (graph-matrix-col (dboard:tabdat-graph-matrix-col tabdat)) + (graph-matrix-row (dboard:tabdat-graph-matrix-row tabdat)) + (graph-cell (conc graph-matrix-row ":" graph-matrix-col)) + (graph-dat (make-dboard:graph-dat + id: fieldn + color: graph-color + flag: #t + cell: graph-cell + ))) + (hash-table-set! graph-matrix-table fieldn graph-dat) + (hash-table-set! graph-cell-table graph-cell graph-dat) + ;; (print "Graph data " graph-matrix-row " " graph-matrix-col " " fieldn " " graph-color " " graph-color-rgb " ") + ;; (print "Graph data " graph-matrix-row " " graph-matrix-col " " fieldn " " graph-color " " graph-color-rgb " ") + (set! changed #t) + (iup:attribute-set! graph-matrix (conc graph-matrix-row ":" graph-matrix-col) fieldn) + (iup:attribute-set! graph-matrix (conc "BGCOLOR" (conc graph-matrix-row ":" graph-matrix-col)) graph-color-rgb) + (if (> graph-matrix-col 10) + (begin + (dboard:tabdat-graph-matrix-col-set! tabdat 1) + (dboard:tabdat-graph-matrix-row-set! tabdat (+ graph-matrix-row 1))) + (dboard:tabdat-graph-matrix-col-set! tabdat (+ graph-matrix-col 1))) + ))) + (if (not (null? vals)) + (let* (;; (maxval (apply max vals)) + ;; (minval (min 0 (apply min vals))) + (yoff (- minval lly)) ;; minval)) + (deltaval (- maxval minval)) + (yscale (/ delta-y (if (zero? deltaval) 1 deltaval))) + (yfunc (lambda (y)(+ lly (* yscale (- y minval))))) ;; (lambda (y)(* (+ y yoff) yscale)))) + (graph-dat (hash-table-ref graph-matrix-table fieldn)) + (graph-color (dboard:graph-dat-color graph-dat)) + (graph-flag (dboard:graph-dat-flag graph-dat))) + (if graph-flag + (begin + (vg:add-obj-to-comp + cmp + (vg:make-text-obj (- llx 10)(yfunc maxval) (conc maxval))) + (vg:add-obj-to-comp + cmp + (vg:make-text-obj (- llx 10)(yfunc minval) (conc minval))) + (fold + (lambda (next prev) ;; #(time ? val) #(time ? val) + (if prev + (let* ((yval (vector-ref prev 2)) + (yval-next (vector-ref next 2)) + (last-tval (tfn (vector-ref prev 0))) + (last-yval (yfunc yval)) ;; (+ lly (* yscale (vector-ref prev 2)))) + (next-yval (yfunc yval-next)) + (curr-tval (tfn (vector-ref next 0)))) + (if (>= curr-tval last-tval) + (begin + (vg:add-obj-to-comp + cmp + ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale)) + (vg:make-line-obj last-tval last-yval curr-tval last-yval + line-color: graph-color)) + (vg:add-obj-to-comp + cmp + ;;(vg:make-rect-obj last-tval lly curr-tval last-yval ;; (- stval 2) lly (+ stval 2)(+ lly (* yval yscale)) + (vg:make-line-obj curr-tval last-yval curr-tval next-yval + line-color: graph-color))) + (print "ERROR: curr-tval is not > last-tval; curr-tval " curr-tval ", last-tval " last-tval)))) + next) + #f ;; (vector tstart minval minval) + dat) + )))))) ;; for each data point in the series (hash-table-keys alldat))))) - cfg))) + cfg) + (if changed (iup:attribute-set! graph-matrix "REDRAW" "ALL")))) ;; run times tab ;; (define (dashboard:run-times-tab-layout-updater commondat tabdat tab-num) ;; each test is an object in the run component @@ -2937,11 +3144,12 @@ (compact-layout (dboard:tabdat-compact-layout tabdat)) (row-height (if compact-layout 2 10)) (graph-height 120) (run-to-run-margin 25)) (dboard:tabdat-layout-update-ok-set! tabdat #t) - (if (canvas? cnv) + (if (and (canvas? cnv) + (not (null? allruns))) ;; allruns can go null when browsing the runs tree (let*-values (((sizex sizey sizexmm sizeymm) (canvas-size cnv)) ((originx originy) (canvas-origin cnv)) ((calc-y) (lambda (rownum) (- (/ sizey 2) (* rownum row-height)))) @@ -2982,12 +3190,12 @@ (all-tids (hash-table-keys tests-ht)) ;; (apply append hierdat)) ;; was testsdat (testsdat (hash-table-values tests-ht)) (runcomp (vg:comp-new));; new component for this run (rows-used (make-hash-table)) ;; keep track of what parts of the rows are used here row1 = (obj1 obj2 ...) ;; (row-height 4) - (run-start (dboard:min-max < (map db:test-get-event_time testsdat))) - (run-end (let ((re (dboard:min-max > (map (lambda (t)(+ (db:test-get-event_time t)(db:test-get-run_duration t))) testsdat)))) + (run-start (common:min-max < (map db:test-get-event_time testsdat))) + (run-end (let ((re (common:min-max > (map (lambda (t)(+ (db:test-get-event_time t)(db:test-get-run_duration t))) testsdat)))) (max re (+ 1 run-start)))) ;; use run-start+1 if run-start == run-end so delta is not zero (timeoffset (- run-start)) ;; (+ fixed-originx canvas-margin) run-start)) (run-duration (- run-end run-start)) (timescale (/ (- sizex (* 2 canvas-margin)) (if (> run-duration 0) @@ -3143,204 +3351,123 @@ (escapeloop #t) ;; (dboard:tabdat-layout-update-ok tabdat) ))))))))) ;; new-run-start-row ))) (debug:print 2 *default-log-port* "no tabdat for run-times-tab-updater")))) -(define (tabdat-values tabdat) - (let ((allruns (dboard:tabdat-allruns tabdat)) - (allruns-by-id (dboard:tabdat-allruns-by-id tabdat)) - (done-runs (dboard:tabdat-done-runs tabdat)) - (not-done-runs (dboard:tabdat-not-done-runs tabdat)) - (header (dboard:tabdat-header tabdat)) - (keys (dboard:tabdat-keys tabdat)) - (numruns (dboard:tabdat-numruns tabdat)) - (tot-runs (dboard:tabdat-tot-runs tabdat)) - (last-data-update (dboard:tabdat-last-data-update tabdat)) - (runs-mutex (dboard:tabdat-runs-mutex tabdat)) - (run-update-times (dboard:tabdat-run-update-times tabdat)) - (last-test-dat (dboard:tabdat-last-test-dat tabdat)) - (run-db-paths (dboard:tabdat-run-db-paths tabdat)) - (buttondat (dboard:tabdat-buttondat tabdat)) - (item-test-names (dboard:tabdat-item-test-names tabdat)) - (run-keys (dboard:tabdat-run-keys tabdat)) - (start-run-offset (dboard:tabdat-start-run-offset tabdat)) - (start-test-offset (dboard:tabdat-start-test-offset tabdat)) - (runs-btn-height (dboard:tabdat-runs-btn-height tabdat)) - (all-test-names (dboard:tabdat-all-test-names tabdat)) - (cnv (dboard:tabdat-cnv tabdat)) - (command (dboard:tabdat-command tabdat)) - (run-name (dboard:tabdat-run-name tabdat)) - (states (dboard:tabdat-states tabdat)) - (statuses (dboard:tabdat-statuses tabdat)) - (curr-run-id (dboard:tabdat-curr-run-id tabdat)) - (curr-test-ids (dboard:tabdat-curr-test-ids tabdat)) - (state-ignore-hash (dboard:tabdat-state-ignore-hash tabdat)) - (test-patts (dboard:tabdat-test-patts tabdat)) - (target (dboard:tabdat-target tabdat)) - (dbdir (dboard:tabdat-dbdir tabdat)) - (monitor-db-path (dboard:tabdat-monitor-db-path tabdat)) - (path-run-ids (dboard:tabdat-path-run-ids tabdat))) - (print "allruns is : " allruns) - (print "allruns-by-id is : " allruns-by-id) - (print "done-runs is : " done-runs) - (print "not-done-runs is : " not-done-runs) - (print "header is : " header ) - (print "keys is : " keys) - (print "numruns is : " numruns) - (print "tot-runs is : " tot-runs) - (print "last-data-update is : " last-data-update) - (print "runs-mutex is : " runs-mutex) - (print "run-update-times is : " run-update-times) - (print "last-test-dat is : " last-test-dat) - (print "run-db-paths is : " run-db-paths) - (print "buttondat is : " buttondat) - (print "item-test-names is : " item-test-names) - (print "run-keys is : " run-keys) - (print "start-run-offset is : " start-run-offset) - (print "start-test-offset is : " start-test-offset) - (print "runs-btn-height is : " runs-btn-height) - (print "all-test-names is : " all-test-names) - (print "cnv is : " cnv) - (print "command is : " command) - (print "run-name is : " run-name) - (print "states is : " states) - (print "statuses is : " statuses) - (print "curr-run-id is : " curr-run-id) - (print "curr-test-ids is : " curr-test-ids) - (print "state-ignore-hash is : " state-ignore-hash) - (print "test-patts is : " test-patts) - (print "target is : " target) - (print "dbdir is : " dbdir) - (print "monitor-db-path is : " monitor-db-path) - (print "path-run-ids is : " path-run-ids))) - +;; handy trick for printing a record +;; +;; (pp (dboard:tabdat->alist tabdat)) +;; +;; removing the tabdat-values proc +;; +;; (define (tabdat-values tabdat) + +;; runs update-rundat using the various filters from the gui +;; (define (dashboard:do-update-rundat tabdat) - (update-rundat + (dboard:update-rundat tabdat (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "runname" "%") (dboard:tabdat-numruns tabdat) (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "test-name" "%/%") - ;; (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) "item-name" "%") + ;; generate key patterns from the target stored in tabdat (let* ((dbkeys (dboard:tabdat-dbkeys tabdat))) - ;; (print "dbkeys: " dbkeys) (let ((fres (if (dboard:tabdat-target tabdat) (let ((ptparts (append (dboard:tabdat-target tabdat)(make-list (length dbkeys) "%")))) (map (lambda (k v)(list k v)) dbkeys ptparts)) (let ((res '())) - ;; (print "target: " (dboard:tabdat-target tabdat)) (for-each (lambda (key) (if (not (equal? key "runname")) (let ((val (hash-table-ref/default (dboard:tabdat-searchpatts tabdat) key #f))) (if val (set! res (cons (list key val) res)))))) dbkeys) res)))) - ;; (debug:print 0 *default-log-port* "fres: " fres) fres)))) (define (dashboard:runs-tab-updater commondat tab-num) (debug:catch-and-dump (lambda () (let* ((tabdat (dboard:common-get-tabdat commondat tab-num: tab-num)) (dbkeys (dboard:tabdat-dbkeys tabdat))) ;;(print "RA => calling runs-tab-updater with commondat " commondat " tab-num " tab-num) ;;(tabdat-values tabdat) ;;RA added + ;; (pp (dboard:tabdat->alist tabdat)) + ;; (if (dashboard:database-changed? commondat tabdat context-key: 'runs-rundat) (dashboard:do-update-rundat tabdat) (let ((uidat (dboard:commondat-uidat commondat))) - ;;(print "RA => Calling update-buttons with tabdat : " tabdat " uidat " uidat) + ;;(print "RA => Calling update-buttons with tabdat : " tabdat " uidat " uidat) (update-buttons tabdat uidat (dboard:tabdat-numruns tabdat) (dboard:tabdat-num-tests tabdat))) )) "dashboard:runs-tab-updater")) -;; ((2) -;; (dashboard:update-run-summary-tab)) -;; ((3) -;; (dashboard:update-new-view-tab)) -;; (else -;; (dboard:common-run-curr-updater commondat))) -;; (set! *last-recalc-ended-time* (current-milliseconds)))))))) - ;;====================================================================== ;; The heavy lifting starts here ;;====================================================================== +(define (main) + (let ((mtdb-path (conc *toppath* "/megatest.db"))) ;; + (if (and (file-exists? mtdb-path) + (file-write-access? mtdb-path)) + (if (not (args:get-arg "-skip-version-check")) + (common:exit-on-version-changed))) + (let* ((commondat (dboard:commondat-make))) + ;; Move this stuff to db.scm? I'm not sure that is the right thing to do... + (cond + ((args:get-arg "-test") ;; run-id,test-id + (let* ((dat (let ((d (map string->number (string-split (args:get-arg "-test") ",")))) + (if (> (length d) 1) + d + (list #f #f)))) + (run-id (car dat)) + (test-id (cadr dat))) + (if (and (number? run-id) + (number? test-id) + (>= test-id 0)) + (dashboard-tests:examine-test run-id test-id) + (begin + (debug:print 3 *default-log-port* "INFO: tried to open test with invalid run-id,test-id. " (args:get-arg "-test")) + (exit 1))))) + ;; ((args:get-arg "-guimonitor") + ;; (gui-monitor (dboard:tabdat-dblocal tabdat))) + (else + (dboard:commondat-uidat-set! commondat (make-dashboard-buttons commondat)) + (dboard:commondat-curr-tab-num-set! commondat 0) + (dboard:commondat-add-updater + commondat + (lambda () + (dashboard:runs-tab-updater commondat 1)) + tab-num: 1) + (iup:callback-set! *tim* + "ACTION_CB" + (lambda (time-obj) + (let ((update-is-running #f)) + (mutex-lock! (dboard:commondat-update-mutex commondat)) + (set! update-is-running (dboard:commondat-updating commondat)) + (if (not update-is-running) + (dboard:commondat-updating-set! commondat #t)) + (mutex-unlock! (dboard:commondat-update-mutex commondat)) + (if (not update-is-running) ;; we know that the update was not running and we now have a lock on doing an update + (begin + (dboard:common-run-curr-updaters commondat) ;; (dashboard:run-update commondat) + (mutex-lock! (dboard:commondat-update-mutex commondat)) + (dboard:commondat-updating-set! commondat #f) + (mutex-unlock! (dboard:commondat-update-mutex commondat))) + )) + 1)))) + + (let ((th1 (make-thread (lambda () + (thread-sleep! 1) + (dboard:common-run-curr-updaters commondat 0) ;; force update of summary tab + ) "update buttons once")) + (th2 (make-thread iup:main-loop "Main loop"))) + (thread-start! th2) + (thread-join! th2))))) + ;; ease debugging by loading ~/.dashboardrc (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc"))) (if (file-exists? debugcontrolf) (load debugcontrolf))) -(define (main) - (if (not (args:get-arg "-skip-version-check")) - (let ((th1 (make-thread common:exit-on-version-changed))) - (thread-start! th1) - (if (> megatest-version (common:get-last-run-version-number)) - (debug:print-info 0 *default-log-port* "Version bump detected, blocking until db sync complete") - (thread-join! th1)))) - (let* ((commondat (dboard:commondat-make))) - ;; Move this stuff to db.scm? I'm not sure that is the right thing to do... - (cond - ((args:get-arg "-test") ;; run-id,test-id - (let* ((dat (let ((d (map string->number (string-split (args:get-arg "-test") ",")))) ;; RADT couldn't find string->number, though it works - (if (> (length d) 1) - d - (list #f #f)))) - (run-id (car dat)) - (test-id (cadr dat))) - (if (and (number? run-id) - (number? test-id) - (>= test-id 0)) - (dashboard-tests:examine-test run-id test-id) - (begin - (debug:print 3 *default-log-port* "INFO: tried to open test with invalid run-id,test-id. " (args:get-arg "-test")) - (exit 1))))) - ;; ((args:get-arg "-guimonitor") - ;; (gui-monitor (dboard:tabdat-dblocal tabdat))) - (else - (dboard:commondat-uidat-set! commondat (make-dashboard-buttons commondat)) ;; (dboard:tabdat-dblocal data) - ;; (dboard:tabdat-numruns tabdat) - ;; (dboard:tabdat-num-tests tabdat) - ;; (dboard:tabdat-dbkeys tabdat) - ;; runs-sum-dat new-view-dat)) - ;; legacy setup of updaters for summary tab and runs tab - ;; summary tab - ;; (dboard:commondat-add-updater - ;; commondat - ;; (lambda () - ;; (dashboard:summary-tab-updater commondat 0)) - ;; tab-num: 0) - ;; runs tab - (dboard:commondat-curr-tab-num-set! commondat 0) - (dboard:commondat-add-updater - commondat - (lambda () - (dashboard:runs-tab-updater commondat 1)) - tab-num: 1) - (iup:callback-set! *tim* - "ACTION_CB" - (lambda (time-obj) - (let ((update-is-running #f)) - (mutex-lock! (dboard:commondat-update-mutex commondat)) - (set! update-is-running (dboard:commondat-updating commondat)) - (if (not update-is-running) - (dboard:commondat-updating-set! commondat #t)) - (mutex-unlock! (dboard:commondat-update-mutex commondat)) - (if (not update-is-running) ;; we know that the update was not running and we now have a lock on doing an update - (begin - (dboard:common-run-curr-updaters commondat) ;; (dashboard:run-update commondat) - (mutex-lock! (dboard:commondat-update-mutex commondat)) - (dboard:commondat-updating-set! commondat #f) - (mutex-unlock! (dboard:commondat-update-mutex commondat))) - )) - 1)))) - - (let ((th1 (make-thread (lambda () - (thread-sleep! 1) - (dboard:common-run-curr-updaters commondat 0) ;; force update of summary tab - ;; (dboard:commondat-please-update-set! commondat #t) ;; MRW: ww36.3 - why was please update set true here? Removing it for now. - ;; (dashboard:run-update commondat) - ) "update buttons once")) - (th2 (make-thread iup:main-loop "Main loop"))) - ;; (thread-start! th1) - (thread-start! th2) - (thread-join! th2)))) - -(main) +(if (args:get-arg "-repl") + (repl) + (main)) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -38,24 +38,29 @@ ;;====================================================================== ;; R E C O R D S ;;====================================================================== +;; each db entry is a pair ( db . dbfilepath ) +;; I propose this record evolves into the area record +;; (defstruct dbr:dbstruct - main - strdb - ((path #f) : string) - ((local #f) : boolean) - rundb - inmem - mtime - rtime - stime - inuse - refdb - ((locdbs (make-hash-table)) : hash-table) - olddb) + (tmpdb #f) + (mtdb #f) + (refndb #f) + (homehost #f) ;; not used yet + (on-homehost #f) ;; not used yet + ) ;; goal is to converge on one struct for an area but for now it is too confusing + + +;; record for keeping state,status and count for doing roll-ups in +;; iterated tests +;; +(defstruct dbr:counts + (state #f) + (status #f) + (count 0)) ;;====================================================================== ;; SQLITE3 HELPERS ;;====================================================================== @@ -64,11 +69,11 @@ ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn) (print "err-status: " err-status) (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn)) (print-call-chain (current-error-port)))) -;; convert to -inline +;; convert to -inline ;; (define (db:first-result-default db stmt default . params) (handle-exceptions exn (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn))) @@ -86,23 +91,15 @@ ;; if #f => get main db ;; if db already open - return inmem ;; if db not open, open inmem, rundb and sync then return inmem ;; inuse gets set automatically for rundb's ;; -(define (db:get-db dbstruct run-id) - (if (sqlite3:database? dbstruct) ;; pass sqlite3 databases on through - dbstruct - (begin - (let ((dbdat (if (or (not run-id) - (eq? run-id 0)) - (db:open-main dbstruct) - (db:open-rundb dbstruct run-id) - ))) - dbdat)))) - -;; legacy handling of structure for managing db's. Refactor this into dbr:? -;; +(define (db:get-db dbstruct . blah) ;; run-id) + (or (dbr:dbstruct-tmpdb dbstruct) + (db:open-db dbstruct))) + +;; ;; legacy handling of structure for managing db's. Refactor this into dbr:? (define (db:dbdat-get-db dbdat) (if (pair? dbdat) (car dbdat) dbdat)) @@ -114,35 +111,38 @@ ;; mod-read: ;; 'mod modified data ;; 'read read data ;; Locks the mutex and depending on 'mod or 'read passed, sets the last timestamp in dbstruct ;; -(define (db:done-with dbstruct run-id mod-read) - (if (not (sqlite3:database? dbstruct)) - (begin - (mutex-lock! *rundb-mutex*) - (if (eq? mod-read 'mod) - (dbr:dbstruct-mtime-set! dbstruct (current-milliseconds)) - (dbr:dbstruct-rtime-set! dbstruct (current-milliseconds))) - (dbr:dbstruct-inuse-set! dbstruct #f) - (mutex-unlock! *rundb-mutex*)))) - -;; (db:with-db dbstruct run-id sqlite3:exec "select blah from blaz;") +;; (define (db:done-with dbstruct run-id mod-read) +;; (if (not (sqlite3:database? dbstruct)) +;; (begin +;; (mutex-lock! *rundb-mutex*) +;; (if (eq? mod-read 'mod) +;; (dbr:dbstruct-mtime-set! dbstruct (current-milliseconds)) +;; (dbr:dbstruct-rtime-set! dbstruct (current-milliseconds))) +;; (dbr:dbstruct-inuse-set! dbstruct #f) +;; (mutex-unlock! *rundb-mutex*)))) + +;; (db:with-db dbstruct run-id sqlite3:exec "select blah fgrom blaz;") ;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no ;; (define (db:with-db dbstruct run-id r/w proc . params) (let* ((dbdat (if (dbr:dbstruct? dbstruct) (db:get-db dbstruct run-id) - dbstruct)) ;; cheat, allow for passing in a dbdat - (db (db:dbdat-get-db dbdat))) + (begin + (print-call-chain) + (print "db:with-db called with dbdat instead of dbstruct, FIXME!!") + dbstruct))) ;; cheat, allow for passing in a dbdat + (db (db:dbdat-get-db dbdat))) (handle-exceptions exn (begin (debug:print-error 0 *default-log-port* "sqlite3 issue in db:with-db, dbstruct=" dbstruct ", run-id=" run-id ", proc=" proc ", params=" params " error: " ((condition-property-accessor 'exn 'message) exn)) (print-call-chain (current-error-port))) (let ((res (apply proc db params))) - (if (vector? dbstruct)(db:done-with dbstruct run-id r/w)) + ;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w)) res)))) ;;====================================================================== ;; K E E P F I L E D B I N dbstruct ;;====================================================================== @@ -170,165 +170,151 @@ ;; NB// #f => return dbdir only ;; (was planned to be; zeroth db with name=main.db) ;; ;; If run-id is #f return to create and retrieve the path where the db will live. ;; -(define (db:dbfile-path run-id) - (let* ((dbdir (db:get-dbdir)) - (fname (if run-id - (if (eq? run-id 0) "main.db" (conc run-id ".db")) - #f))) +(define (db:dbfile-path . junk) ;; run-id) + (let* ((dbdir (common:get-db-tmp-area))) ;; (db:get-dbdir)) +;; (fname (if run-id +;; (if (eq? run-id 0) "main.db" (conc run-id ".db")) +;; #f))) (handle-exceptions exn (begin (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir) (exit 1)) (if (not (directory? dbdir))(create-directory dbdir #t))) - (if fname - (conc dbdir "/" fname) - dbdir))) + dbdir)) ;; (if fname +;; (conc dbdir "/" fname) +;; dbdir))) ;; Returns the database location as specified in config file ;; -(define (db:get-dbdir) - (or (configf:lookup *configdat* "setup" "dbdir") - (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))) +;; (define db:get-dbdir common:get-db-tmp-area) +;; (or (configf:lookup *configdat* "setup" "dbdir") +;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))) (define (db:set-sync db) (let ((syncprag (configf:lookup *configdat* "setup" "sychronous"))) - (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 1) ";")))) + (sqlite3:execute db (conc "PRAGMA synchronous = " (or syncprag 0) ";")))) ;; open an sql database inside a file lock ;; returns: db existed-prior-to-opening ;; RA => Returns a db handler; sets the lock if opened in writable mode ;; (define (db:lock-create-open fname initproc) - ;; (if (file-exists? fname) - ;; (let ((db (sqlite3:open-database fname))) - ;; (sqlite3:set-busy-handler! db (make-busy-timeout 136000)) - ;; (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") - ;; db) - (let* ((parent-dir (pathname-directory fname)) + (let* ((parent-dir (or (pathname-directory fname)(current-directory))) ;; no parent? go local (dir-writable (file-write-access? parent-dir)) (file-exists (file-exists? fname)) (file-write (if file-exists (file-write-access? fname) dir-writable ))) (if file-write ;; dir-writable (let (;; (lock (obtain-dot-lock fname 1 5 10)) (db (sqlite3:open-database fname))) (sqlite3:set-busy-handler! db (make-busy-timeout 136000)) - (db:set-sync db) ;; (sqlite3:execute db "PRAGMA synchronous = 0;") - (if (not file-exists)(initproc db)) + ;; (db:set-sync db) + (sqlite3:execute db "PRAGMA synchronous = 0;") + (if (not file-exists) + (begin + (if (string-match "^/tmp/.*" fname) ;; this is a file in /tmp + (sqlite3:execute db "PRAGMA journal_mode=WAL;") + (print "Creating " fname " in NON-WAL mode.")) + (initproc db))) ;; (release-dot-lock fname) db) (begin (debug:print 2 *default-log-port* "WARNING: opening db in non-writable dir " fname) (sqlite3:open-database fname))))) ;; ) -;; This routine creates the db. It is only called if the db is not already opened -;; -(define (db:open-rundb dbstruct run-id #!key (attemptnum 0)(do-not-open #f)) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) - (let* ((local (dbr:dbstruct-local dbstruct)) - (rdb (if local - (dbr:dbstruct-localdb dbstruct run-id) - (dbr:dbstruct-inmem dbstruct)))) ;; (dbr:dbstruct-runrec dbstruct run-id 'inmem))) - (if (or rdb - do-not-open) - rdb - (begin - (mutex-lock! *rundb-mutex*) - (let* ((dbpath (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db")) - (dbexists (file-exists? dbpath)) - (inmem (if local #f (db:open-inmem-db))) - (refdb (if local #f (db:open-inmem-db))) - (db (db:lock-create-open dbpath ;; this is the database physically on disk - (lambda (db) - (handle-exceptions - exn - (begin - ;; (release-dot-lock dbpath) - (if (> attemptnum 2) - (debug:print-error 0 *default-log-port* "tried twice, cannot create/initialize db for run-id " run-id ", at path " dbpath) - (db:open-rundb dbstruct run-id attemptnum (+ attemptnum 1)))) - (db:initialize-run-id-db db) - (sqlite3:execute - db - "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');" - (* run-id 30000) ;; allow for up to 30k tests per run - run-id) - ;; do a dummy query to test that the table exists and the db is truly readable - (sqlite3:execute db "SELECT * FROM tests WHERE id=?;" (* run-id 30000)) - )))) ;; add strings db to rundb, not in use yet - ;; )) ;; (sqlite3:open-database dbpath)) - (olddb (if *megatest-db* - *megatest-db* - (let ((db (db:open-megatest-db))) - (set! *megatest-db* db) - db))) - (write-access (file-write-access? dbpath)) - ;; (handler (make-busy-timeout 136000)) - ) - (if (and dbexists (not write-access)) - (set! *db-write-access* #f)) ;; only unset so other db's also can use this control - (dbr:dbstruct-rundb-set! dbstruct (cons db dbpath)) - (dbr:dbstruct-inuse-set! dbstruct #t) - (dbr:dbstruct-olddb-set! dbstruct olddb) - ;; (dbr:dbstruct-run-id-set! dbstruct run-id) - (mutex-unlock! *rundb-mutex*) - (if local - (begin - (dbr:dbstruct-localdb-set! dbstruct run-id db) ;; (dbr:dbstruct-inmem-set! dbstruct db) ;; direct access ... - db) - (begin - (dbr:dbstruct-inmem-set! dbstruct inmem) - ;; dec 14, 2014 - keep deleted records available. hunch is that they are needed for id placeholders - ;; (sqlite3:execute db "DELETE FROM tests WHERE state='DELETED';") ;; they just slow us down in this context - (db:sync-tables db:sync-tests-only db inmem) - (db:delay-if-busy refdb) ;; dbpath: (db:dbdat-get-path refdb)) ;; What does delaying here achieve? - (dbr:dbstruct-refdb-set! dbstruct refdb) - (db:sync-tables db:sync-tests-only inmem refdb) ;; use inmem as the reference, don't read again from db - ;; sync once more to deal with delays? - ;; (db:sync-tables db:sync-tests-only db inmem) - ;; (db:sync-tables db:sync-tests-only inmem refdb) - inmem))))))) - -;; This routine creates the db if not already present. It is only called if the db is not already ls opened -;; -(define (db:open-main dbstruct) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) - (let ((mdb (dbr:dbstruct-main dbstruct))) ;; RA => Returns the first reference in dbstruct - (if mdb - mdb - (begin - (mutex-lock! *rundb-mutex*) - (let* ((dbpath (db:dbfile-path 0)) - (dbexists (file-exists? dbpath)) - (db (db:lock-create-open dbpath db:initialize-main-db)) - (olddb (db:open-megatest-db)) - (write-access (file-write-access? dbpath)) - (dbdat (cons db dbpath))) - (if (and dbexists (not write-access)) - (set! *db-write-access* #f)) - (dbr:dbstruct-main-set! dbstruct dbdat) - (dbr:dbstruct-olddb-set! dbstruct olddb) ;; olddb is already a (cons db path) - (mutex-unlock! *rundb-mutex*) - (if (and (not dbexists) - *db-write-access*) ;; did not have a prior db and do have write access - (db:multi-db-sync #f 'old2new)) ;; migrate data from megatest.db automatically - dbdat))))) +;; ;; This routine creates the db. It is only called if the db is not already opened +;; ;; +;; (define (db:open-rundb dbstruct run-id #!key (attemptnum 0)(do-not-open #f)) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) +;; (let* ((dbfile (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db")) +;; (dbexists (file-exists? dbfile)) +;; (db (db:lock-create-open dbfile (lambda (db) +;; (handle-exceptions +;; exn +;; (begin +;; ;; (release-dot-lock dbpath) +;; (if (> attemptnum 2) +;; (debug:print-error 0 *default-log-port* "tried twice, cannot create/initialize db for run-id " run-id ", at path " dbpath) +;; (db:open-rundb dbstruct run-id attemptnum (+ attemptnum 1)))) +;; (db:initialize-run-id-db db) +;; (sqlite3:execute +;; db +;; "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');" +;; (* run-id 30000) ;; allow for up to 30k tests per run +;; run-id) +;; ;; do a dummy query to test that the table exists and the db is truly readable +;; (sqlite3:execute db "SELECT * FROM tests WHERE id=?;" (* run-id 30000)) +;; )))) ;; add strings db to rundb, not in use yet +;; (olddb (if *megatest-db* +;; *megatest-db* +;; (let ((db (db:open-megatest-db))) +;; (set! *megatest-db* db) +;; db))) +;; (write-access (file-write-access? dbfile))) +;; (if (and dbexists (not write-access)) +;; (set! *db-write-access* #f)) ;; only unset so other db's also can use this control +;; (dbr:dbstruct-rundb-set! dbstruct (cons db dbfile)) +;; (dbr:dbstruct-inuse-set! dbstruct #t) +;; (dbr:dbstruct-olddb-set! dbstruct olddb) +;; ;;; (mutex-unlock! *rundb-mutex*) ;;; why did we need a mutex on opening db's? +;; (db:sync-tables db:sync-tests-only *megatest-db* db) +;; db)) + +;; This routine creates the db if not already present. It is only called if the db is not already opened +;; +(define (db:open-db dbstruct #!key (areapath #f)) + (let ((tmpdb (dbr:dbstruct-tmpdb dbstruct))) ;; RA => Returns the first reference in dbstruct + (if tmpdb + tmpdb + ;; (mutex-lock! *rundb-mutex*) + (let* ((dbpath (db:dbfile-path)) ;; 0)) + (dbexists (file-exists? dbpath)) + (dbfexists (file-exists? (conc dbpath "/megatest.db"))) + (tmpdb (db:open-megatest-db path: dbpath)) ;; lock-create-open dbpath db:initialize-main-db)) + (mtdb (db:open-megatest-db)) + (refndb (db:open-megatest-db path: dbpath name: "megatest_ref.db")) + (write-access (file-write-access? dbpath))) + (if (and dbexists (not write-access)) + (set! *db-write-access* #f)) + (dbr:dbstruct-mtdb-set! dbstruct mtdb) + (dbr:dbstruct-tmpdb-set! dbstruct tmpdb) ;; olddb is already a (cons db path) + (dbr:dbstruct-refndb-set! dbstruct refndb) + ;; (mutex-unlock! *rundb-mutex*) + (if (and (not dbfexists) + write-access) ;; *db-write-access*) ;; did not have a prior db and do have write access + (begin + (debug:print 0 *default-log-port* "filling db " (db:dbdat-get-path tmpdb) " with data from " (db:dbdat-get-path mtdb)) + (db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb refndb tmpdb)) + (debug:print 0 *default-log-port* " db, " (db:dbdat-get-path tmpdb) " already exists, not propogating data from " (db:dbdat-get-path mtdb))) + ;; (db:multi-db-sync dbstruct 'old2new)) ;; migrate data from megatest.db automatically + tmpdb)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; -(define (db:setup run-id #!key (local #f)) - (let* ((dbdir (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) - (dbstruct (make-dbr:dbstruct path: dbdir local: local))) - dbstruct)) - -;; Open the classic megatest.db file in toppath -;; -(define (db:open-megatest-db) - (let* ((dbpath (conc *toppath* "/megatest.db")) +;; called in http-transport and replicated in rmt.scm for *local* access. +;; +(define (db:setup #!key (areapath #f)) + (or *dbstruct-db* + (if (common:on-homehost?) + (let* ((dbstruct (make-dbr:dbstruct))) + (db:open-db dbstruct areapath: areapath) + (set! *dbstruct-db* dbstruct) + dbstruct) + (begin + (debug:print 0 *default-log-port* "ERROR: attempt to open database when not on homehost. Exiting. Homehost: " (common:get-homehost)) + (exit 1))))) + +;; Open the classic megatest.db file (defaults to open in toppath) +;; +;; NOTE: returns a dbdat not a dbstruct! +;; +(define (db:open-megatest-db #!key (path #f)(name #f)) + (let* ((dbpath (conc (or path *toppath*) "/" (or name "megatest.db"))) (dbexists (file-exists? dbpath)) (db (db:lock-create-open dbpath (lambda (db) (db:initialize-main-db db) (db:initialize-run-id-db db)))) @@ -338,94 +324,47 @@ (cons db dbpath))) ;; sync run to disk if touched ;; (define (db:sync-touched dbstruct run-id #!key (force-sync #f)) - (let ((mtime (dbr:dbstruct-mtime dbstruct)) - (stime (dbr:dbstruct-stime dbstruct)) - (rundb (dbr:dbstruct-rundb dbstruct)) - (inmem (dbr:dbstruct-inmem dbstruct)) - (maindb (dbr:dbstruct-main dbstruct)) - (refdb (dbr:dbstruct-refdb dbstruct)) - (olddb (dbr:dbstruct-olddb dbstruct)) - ;; (runid (dbr:dbstruct-run-id dbstruct)) - ) + (let ((tmpdb (dbr:dbstruct-tmpdb dbstruct)) + (mtdb (dbr:dbstruct-mtdb dbstruct)) + (refndb (dbr:dbstruct-refndb dbstruct)) + (start-t (current-seconds))) (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id) - ;; (mutex-lock! *http-mutex*) - (if (eq? run-id 0) - ;; runid equal to 0 is main.db - (if maindb - (if (or (not (number? mtime)) - (not (number? stime)) - (> mtime stime) - force-sync) - (begin - (db:delay-if-busy maindb) - (db:delay-if-busy olddb) - (let ((num-synced (db:sync-tables (db:sync-main-list maindb) maindb olddb))) - (dbr:dbstruct-stime-set! dbstruct (current-milliseconds)) - num-synced) - 0)) - (begin - ;; this can occur when using local access (i.e. not in a server) - ;; need a flag to turn it off. - ;; - (debug:print 3 *default-log-port* "WARNING: call to sync main.db to megatest.db but main not initialized") - 0)) - ;; any other runid is a run - (if (or (not (number? mtime)) - (not (number? stime)) - (> mtime stime) - force-sync) - (begin - (db:delay-if-busy rundb) - (db:delay-if-busy olddb) - (dbr:dbstruct-stime-set! dbstruct (current-milliseconds)) - (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb olddb))) - ;; (mutex-unlock! *http-mutex*) - num-synced) - (begin - ;; (mutex-unlock! *http-mutex*) - 0)))))) - -(define (db:close-main dbstruct) - (let ((maindb (dbr:dbstruct-main dbstruct))) - (if maindb - (begin - (sqlite3:finalize! (db:dbdat-get-db maindb)) - (dbr:dbstruct-main-set! dbstruct #f))))) - -(define (db:close-run-db dbstruct run-id) - (let ((rdb (db:open-rundb dbstruct run-id do-not-open: #t))) - (if (and rdb - (sqlite3:database? rdb)) - (begin - (sqlite3:finalize! rdb) - (dbr:dbstruct-localdb-set! dbstruct run-id #f) - (dbr:dbstruct-inmem-set! dbstruct #f))))) + (mutex-lock! *db-multi-sync-mutex*) + (let ((update_info (cons (if force-sync 0 *db-last-sync*) "last_update"))) + (mutex-unlock! *db-multi-sync-mutex*) + (db:sync-tables (db:sync-all-tables-list dbstruct) update_info tmpdb refndb mtdb)) + (mutex-lock! *db-multi-sync-mutex*) + (set! *db-last-sync* start-t) + (mutex-unlock! *db-multi-sync-mutex*))) ;; close all opened run-id dbs (define (db:close-all dbstruct) - ;; finalize main.db - (db:sync-touched dbstruct 0 force-sync: #t) - ;;(common:db-block-further-queries) - ;; (mutex-lock! *db-sync-mutex*) ;; with this perhaps it isn't necessary to use the block-further-queries mechanism? - - (db:close-main dbstruct) - - (let ((locdbs (dbr:dbstruct-locdbs dbstruct))) - (if (hash-table? locdbs) - (for-each (lambda (run-id) - (db:close-run-db dbstruct run-id)) - (hash-table-keys locdbs))))) - -(define (db:open-inmem-db) - (let* ((db (sqlite3:open-database ":memory:")) - (handler (make-busy-timeout 3600))) - (sqlite3:set-busy-handler! db handler) - (db:initialize-run-id-db db) - (cons db #f))) + (if (dbr:dbstruct? dbstruct) + (begin + ;; (db:sync-touched dbstruct 0 force-sync: #t) ;; NO. Do not do this here. Instead we rely on a server to be started when there are writes, even if the server itself is not going to be used as a server. + (let ((tdb (db:dbdat-get-db (dbr:dbstruct-tmpdb dbstruct))) + (mdb (db:dbdat-get-db (dbr:dbstruct-mtdb dbstruct))) + (rdb (db:dbdat-get-db (dbr:dbstruct-refndb dbstruct)))) + (if tdb (sqlite3:finalize! tdb)) + (if mdb (sqlite3:finalize! mdb)) + (if rdb (sqlite3:finalize! rdb)))))) + +;; (let ((locdbs (dbr:dbstruct-locdbs dbstruct))) +;; (if (hash-table? locdbs) +;; (for-each (lambda (run-id) +;; (db:close-run-db dbstruct run-id)) +;; (hash-table-keys locdbs))))) + +;; (define (db:open-inmem-db) +;; (let* ((db (sqlite3:open-database ":memory:")) +;; (handler (make-busy-timeout 3600))) +;; (sqlite3:set-busy-handler! db handler) +;; (db:initialize-run-id-db db) +;; (cons db #f))) ;; just tests, test_steps and test_data tables (define db:sync-tests-only (list ;; (list "strs" @@ -475,12 +414,12 @@ '("status" #f) '("type" #f)))) ;; needs db to get keys, this is for syncing all tables ;; -(define (db:sync-main-list db) - (let ((keys (db:get-keys db))) +(define (db:sync-main-list dbstruct) + (let ((keys (db:get-keys dbstruct))) (list (list "keys" '("id" #f) '("fieldname" #f) '("fieldtype" #f)) @@ -499,10 +438,14 @@ '("iterated" #f) '("avg_runtime" #f) '("avg_disk" #f) '("tags" #f) '("jobgroup" #f))))) + +(define (db:sync-all-tables-list dbstruct) + (append (db:sync-main-list dbstruct) + db:sync-tests-only)) ;; use bunch of Unix commands to try to break the lock and recreate the db ;; (define (db:move-and-recreate-db dbdat) (let* ((dbpath (db:dbdat-get-path dbdat)) @@ -577,16 +520,18 @@ #t)))))) ;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) ) ;; db's are dbdat's ;; -(define (db:sync-tables tbls fromdb todb . slave-dbs) - (mutex-lock! *db-sync-mutex*) +;; if last-update specified ("field-name" . time-in-seconds) +;; then sync only records where field-name >= time-in-seconds +;; IFF field-name exists +;; +(define (db:sync-tables tbls last-update fromdb todb . slave-dbs) (handle-exceptions exn (begin - (mutex-unlock! *db-sync-mutex*) (debug:print 0 *default-log-port* "EXCEPTION: database probably overloaded or unreadable in db:sync-tables.") (print-call-chain (current-error-port)) (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn)) (print "exn=" (condition->list exn)) (debug:print 0 *default-log-port* " status: " ((condition-property-accessor 'sqlite3 'status) exn)) @@ -599,19 +544,11 @@ (debug:print-error 0 *default-log-port* "Failed to rebuild " dbpath ", exiting now.") (exit))))) (cons todb slave-dbs)) 0) -;; (if *server-run* ;; we are inside a server, throw a sync-failed error -;; (signal (make-composite-condition -;; (make-property-condition 'sync-failed 'message "db:sync-tables failed in a server context."))) -;; 0)) ;; return zero for num synced - - ;; (set! *time-to-exit* #t) ;; let watch dog know that it is time to die. - ;; (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "shutting-down") - ;; (portlogger:open-run-close portlogger:set-port port "released") - ;; (exit 1))) + ;; this is the work to be done (cond ((not fromdb) (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with fromdb missing") -1) ((not todb) (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with todb missing") -2) ((not (sqlite3:database? (db:dbdat-get-db fromdb))) (debug:print-error 0 *default-log-port* "db:sync-tables called with fromdb not a database " fromdb) -3) @@ -625,15 +562,26 @@ (tot-count 0)) (for-each ;; table (lambda (tabledat) (let* ((tablename (car tabledat)) (fields (cdr tabledat)) + (use-last-update (if last-update + (if (pair? last-update) + (member (car last-update) ;; last-update field name + (map car fields)) + (begin + (debug:print 0 *default-log-port* "ERROR: parameter last-update for db:sync-tables must be a pair, received: " last-update) ;; found in fields + #f)) + #f)) (num-fields (length fields)) (field->num (make-hash-table)) (num->field (apply vector (map car fields))) (full-sel (conc "SELECT " (string-intersperse (map car fields) ",") - " FROM " tablename ";")) + " FROM " tablename (if use-last-update ;; apply last-update criteria + (conc " " (car last-update) ">=" (cdr last-update)) + "") + ";")) (full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) " " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );")) (fromdat '()) (fromdats '()) (totrecords 0) @@ -706,195 +654,307 @@ fromdats) (sqlite3:finalize! stmth))) (append (list todb) slave-dbs)))) tbls) (let* ((runtime (- (current-milliseconds) start-time)) - (should-print (common:low-noise-print 120 "db sync" (> runtime 500)))) ;; low and high sync times treated as separate. + (should-print (or (debug:debug-mode 12) + (common:low-noise-print 120 "db sync" (> runtime 500))))) ;; low and high sync times treated as separate. (if should-print (debug:print 3 *default-log-port* "INFO: db sync, total run time " runtime " ms")) (for-each (lambda (dat) (let ((tblname (car dat)) (count (cdr dat))) (set! tot-count (+ tot-count count)) (if (> count 0) (if should-print (debug:print 0 *default-log-port* (format #f " ~10a ~5a" tblname count)))))) (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b)))))) - tot-count))) - (mutex-unlock! *db-sync-mutex*))) + tot-count))))) + +(define (db:patch-schema-rundb frundb) + ;; + ;; remove this some time after September 2016 (added in version v1.6031 + ;; + (for-each + (lambda (table-name) + (handle-exceptions + exn + (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn)) + (debug:print 0 *default-log-port* "Column last_update already added to " table-name " table") + (db:general-sqlite-error-dump exn "alter table " table-name " ..." #f "none")) + (sqlite3:execute + frundb + (conc "ALTER TABLE " table-name " ADD COLUMN last_update INTEGER DEFAULT 0"))) + (sqlite3:execute + frundb + (conc "DROP TRIGGER IF EXISTS update_" table-name "_trigger;")) + (sqlite3:execute + frundb + (conc "CREATE TRIGGER IF NOT EXISTS update_" table-name "_trigger AFTER UPDATE ON " table-name " + FOR EACH ROW + BEGIN + UPDATE " table-name " SET last_update=(strftime('%s','now')) + WHERE id=old.id; + END;")) + ) + '("tests" "test_steps" "test_data"))) + +(define (db:patch-schema-maindb maindb) + ;; + ;; remove all these some time after september 2016 (added in v1.6031 + ;; + (handle-exceptions + exn + (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn)) + (debug:print 0 *default-log-port* "Column last_update already added to runs table") + (db:general-sqlite-error-dump exn "alter table runs ..." #f "none")) + (sqlite3:execute + maindb + "ALTER TABLE runs ADD COLUMN last_update INTEGER DEFAULT 0")) + ;; these schema changes don't need exception handling + (sqlite3:execute + maindb + "CREATE TRIGGER IF NOT EXISTS update_runs_trigger AFTER UPDATE ON runs + FOR EACH ROW + BEGIN + UPDATE runs SET last_update=(strftime('%s','now')) + WHERE id=old.id; + END;") + (sqlite3:execute maindb "CREATE TABLE IF NOT EXISTS run_stats ( + id INTEGER PRIMARY KEY, + run_id INTEGER, + state TEXT, + status TEXT, + count INTEGER, + last_update INTEGER DEFAULT (strftime('%s','now')))") + (sqlite3:execute maindb "CREATE TRIGGER IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats + FOR EACH ROW + BEGIN + UPDATE run_stats SET last_update=(strftime('%s','now')) + WHERE id=old.id; + END;")) + +(define *global-db-store* (make-hash-table)) + +(define (db:get-access-mode) + (if (args:get-arg "-use-db-cache") 'cached 'rmt)) + +;; Add db direct +;; +(define (db:dispatch-query access-mode rmt-cmd db-cmd . params) + (if (eq? access-mode 'cached) + (print "not doing cached calls right now")) +;; (apply db:call-with-cached-db db-cmd params) + (apply rmt-cmd params)) +;;) + +;; return the target db handle so it can be used +;; +(define (db:cache-for-read-only source target #!key (use-last-update #f)) + (if (and (hash-table-ref/default *global-db-store* target #f) + (>= (file-modification-time target)(file-modification-time source))) + (hash-table-ref *global-db-store* target) + (let* ((toppath (launch:setup)) + (targ-db-last-mod (if (file-exists? target) + (file-modification-time target) + 0)) + (cache-db (or (hash-table-ref/default *global-db-store* target #f) + (db:open-megatest-db path: target))) + (source-db (db:open-megatest-db path: source)) + (curr-time (current-seconds)) + (res '()) + (last-update (if use-last-update (cons "last_update" targ-db-last-mod) #f))) + (db:sync-tables (db:sync-main-list source-db) last-update source-db cache-db) + (db:sync-tables db:sync-tests-only last-update source-db cache-db) + (hash-table-set! *global-db-store* target cache-db) + cache-db))) + +;; call a proc with a cached db +;; +(define (db:call-with-cached-db proc . params) + ;; first cache the db in /tmp + (let* ((cname-part (conc "megatest_cache/" (common:get-testsuite-name))) + (fname (conc (common:get-area-path-signature) ".db")) + (cache-dir (common:get-create-writeable-dir + (list (conc "/tmp/" (current-user-name) "/" cname-part) + (conc "/tmp/" (current-user-name) "-" cname-part) + (conc "/tmp/" (current-user-name) "_" cname-part)))) + (megatest-db (conc *toppath* "/megatest.db"))) + ;; (debug:print-info 0 *default-log-port* "Using cache dir " cache-dir) + (if (not cache-dir) + (begin + (debug:print 0 *default-log-port* "ERROR: Failed to find an area to write the cache db") + (exit 1)) + (let* ((th1 (make-thread + (lambda () + (if (and (file-exists? megatest-db) + (file-write-access? megatest-db)) + (begin + (common:sync-to-megatest.db 'timestamps) ;; internally mutexes on *db-local-sync* + (debug:print-info 2 *default-log-port* "Done syncing to megatest.db")))) + "call-with-cached-db sync-to-megatest.db")) + (cache-db (db:cache-for-read-only + megatest-db + (conc cache-dir "/" fname) + use-last-update: #t))) + (thread-start! th1) + (apply proc cache-db params) + )))) ;; options: ;; ;; 'killservers - kills all servers ;; 'dejunk - removes junk records ;; 'adj-testids - move test-ids into correct ranges ;; 'old2new - sync megatest.db records to .db/{main,1,2 ...}.db ;; 'new2old - sync .db/{main,1,2,3 ...}.db to megatest.db ;; 'closeall - close all opened dbs +;; 'schema - attempt to apply schema changes ;; ;; run-ids: '(1 2 3 ...) or #f (for all) ;; -(define (db:multi-db-sync run-ids . options) - (let* ((toppath (launch:setup)) - (dbstruct (if toppath (make-dbr:dbstruct path: toppath) #f)) - (mtdb (if toppath (db:open-megatest-db))) - (allow-cleanup (if run-ids #f #t)) - (run-ids (if run-ids - run-ids - (if toppath (begin - (db:delay-if-busy mtdb) - (db:get-all-run-ids mtdb))))) - (tdbdat (tasks:open-db)) - (servers (tasks:get-all-servers (db:delay-if-busy tdbdat)))) - - ;; kill servers - (if (member 'killservers options) - (for-each - (lambda (server) - (tasks:server-delete-record (db:delay-if-busy tdbdat) (vector-ref server 0) "dbmigration") - (tasks:kill-server (vector-ref server 2)(vector-ref server 1))) - servers)) - - ;; clear out junk records - ;; - (if (member 'dejunk options) - (begin - (db:delay-if-busy mtdb) - (db:clean-up mtdb))) - - ;; adjust test-ids to fit into proper range - ;; - (if (member 'adj-testids options) - (begin - (db:delay-if-busy mtdb) - (db:prep-megatest.db-for-migration mtdb))) - - ;; sync runs, test_meta etc. - ;; - (if (member 'old2new options) - (begin - (db:sync-tables (db:sync-main-list mtdb) mtdb (db:get-db dbstruct #f)) - (for-each - (lambda (run-id) - (db:delay-if-busy mtdb) - (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id)) - (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))) - (debug:print 0 *default-log-port* "INFO: Propagating " (length testrecs) " records for run-id=" run-id " to run specific db") - (db:replace-test-records dbstruct run-id testrecs) - (sqlite3:finalize! (db:dbdat-get-db (dbr:dbstruct-rundb dbstruct))))) - run-ids))) - - ;; now ensure all newdb data are synced to megatest.db - ;; do not use the run-ids list passed in to the function - ;; - (if (member 'new2old options) - (let* ((maindb (make-dbr:dbstruct path: toppath local: #t)) - (src-run-ids (if run-ids run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0))))) - (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <)) - (count 1) - (total (length all-run-ids)) - (dead-runs '())) - (for-each - (lambda (run-id) - (debug:print 0 *default-log-port* "Processing run " (if (eq? run-id 0) " main.db " run-id) ", " count " of " total) - (set! count (+ count 1)) - (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)) - (frundb (db:dbdat-get-db (db:get-db fromdb run-id)))) - ;; (db:delay-if-busy frundb) - ;; (db:delay-if-busy mtdb) - ;; (db:clean-up frundb) - (if (eq? run-id 0) - (let ((maindb (db:dbdat-get-db (db:get-db fromdb #f)))) - (db:sync-tables (db:sync-main-list dbstruct) (db:get-db fromdb #f) mtdb) - (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f))) - ;; - ;; Feb 18, 2016: add field last_update to runs table - ;; - ;; remove all these some time after september 2016 (added in v1.6031 - ;; - (handle-exceptions - exn - (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn)) - (debug:print 0 *default-log-port* "Column last_update already added to runs table") - (db:general-sqlite-error-dump exn "alter table runs ..." run-id "none")) - (sqlite3:execute - maindb - "ALTER TABLE runs ADD COLUMN last_update INTEGER DEFAULT 0")) - ;; these schema changes don't need exception handling - (sqlite3:execute - maindb - "CREATE TRIGGER IF NOT EXISTS update_runs_trigger AFTER UPDATE ON runs - FOR EACH ROW - BEGIN - UPDATE runs SET last_update=(strftime('%s','now')) - WHERE id=old.id; - END;") - (sqlite3:execute maindb "CREATE TABLE IF NOT EXISTS run_stats ( - id INTEGER PRIMARY KEY, - run_id INTEGER, - state TEXT, - status TEXT, - count INTEGER, - last_update INTEGER DEFAULT (strftime('%s','now')))") - (sqlite3:execute maindb "CREATE TRIGGER IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats - FOR EACH ROW - BEGIN - UPDATE run_stats SET last_update=(strftime('%s','now')) - WHERE id=old.id; - END;") - ) - (begin - ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db - (db:sync-tables db:sync-tests-only (db:get-db fromdb run-id) mtdb) - (db:clean-up-rundb (db:get-db fromdb run-id)) - ;; - ;; Feb 18, 2016: add field last_update to tests, test_steps and test_data - ;; - ;; remove this some time after September 2016 (added in version v1.6031 - ;; - (for-each - (lambda (table-name) - (handle-exceptions - exn - (if (string-match ".*duplicate.*" ((condition-property-accessor 'exn 'message) exn)) - (debug:print 0 *default-log-port* "Column last_update already added to " table-name " table") - (db:general-sqlite-error-dump exn "alter table " table-name " ..." #f "none")) - (sqlite3:execute - frundb - (conc "ALTER TABLE " table-name " ADD COLUMN last_update INTEGER DEFAULT 0"))) - (sqlite3:execute - frundb - (conc "DROP TRIGGER IF EXISTS update_" table-name "_trigger;")) - (sqlite3:execute - frundb - (conc "CREATE TRIGGER IF NOT EXISTS update_" table-name "_trigger AFTER UPDATE ON " table-name " - FOR EACH ROW - BEGIN - UPDATE " table-name " SET last_update=(strftime('%s','now')) - WHERE id=old.id; - END;")) - ) - '("tests" "test_steps" "test_data")))))) - all-run-ids) - ;; removed deleted runs - (let ((dbdir (tasks:get-task-db-path))) - (for-each (lambda (run-id) - (let ((fullname (conc dbdir "/" run-id ".db"))) - (if (file-exists? fullname) - (begin - (debug:print 0 *default-log-port* "Removing database file for deleted run " fullname) - (delete-file fullname))))) - dead-runs)))) - - ;; (db:close-all dbstruct) - ;; (sqlite3:finalize! mdb) - )) +(define (db:multi-db-sync dbstruct . options) + (if (not (launch:setup)) + (debug:print 0 *default-log-port* "ERROR: not able to setup up for megatest.") + (let* ((mtdb (dbr:dbstruct-mtdb dbstruct)) + (tmpdb (dbr:dbstruct-tmpdb dbstruct)) + (refndb (dbr:dbstruct-refndb dbstruct)) + (allow-cleanup #t) ;; (if run-ids #f #t)) + (tdbdat (tasks:open-db)) + (servers (tasks:get-all-servers (db:delay-if-busy tdbdat))) + (data-synced 0)) ;; count of changed records (I hope) + + ;; kill servers + (if (member 'killservers options) + (for-each + (lambda (server) + (tasks:server-delete-record (db:delay-if-busy tdbdat) (vector-ref server 0) "dbmigration") + (tasks:kill-server (vector-ref server 2)(vector-ref server 1))) + servers)) + + ;; clear out junk records + ;; + (if (member 'dejunk options) + (begin + (db:delay-if-busy mtdb) ;; ok to delay on mtdb + (db:clean-up mtdb) + (db:clean-up tmpdb) + (db:clean-up refndb))) + + ;; adjust test-ids to fit into proper range + ;; + ;; (if (member 'adj-testids options) + ;; (begin + ;; (db:delay-if-busy mtdb) + ;; (db:prep-megatest.db-for-migration mtdb))) + + ;; sync runs, test_meta etc. + ;; + (if (member 'old2new options) + ;; (begin + (db:sync-tables (db:sync-all-tables-list dbstruct) #f mtdb tmpdb refndb)) + ;; (db:sync-main-list mtdb) mtdb (db:get-db dbstruct #f)) +;; (for-each +;; (lambda (run-id) +;; (db:delay-if-busy mtdb) +;; (let ((testrecs (db:get-all-tests-info-by-run-id mtdb run-id))) +;; ;; (dbstruct (if toppath (make-dbr:dbstruct path: toppath local: #t) #f))) +;; (debug:print 0 *default-log-port* "INFO: Propagating " (length testrecs) " records for run-id=" run-id " to run specific db") +;; (db:replace-test-records dbstruct run-id testrecs) +;; (sqlite3:finalize! (db:dbdat-get-db (dbr:dbstruct-rundb dbstruct))))) +;; run-ids))) + + ;; now ensure all newdb data are synced to megatest.db + ;; do not use the run-ids list passed in to the function + ;; + (if (member 'new2old options) + (set! data-synced + (+ (db:sync-tables (db:sync-all-tables-list dbstruct) #f tmpdb refndb mtdb) + data-synced))) + + + (if (member 'fixschema options) + (begin + (db:patch-schema-maindb (db:dbdat-get-db mtdb)) + (db:patch-schema-maindb (db:dbdat-get-db tmpdb)) + (db:patch-schema-maindb (db:dbdat-get-db refndb)) + (db:patch-schema-rundb (db:dbdat-get-db mtdb)) + (db:patch-schema-rundb (db:dbdat-get-db tmpdb)) + (db:patch-schema-rundb (db:dbdat-get-db refndb)))) + + ;; (let* ((maindb (make-dbr:dbstruct path: toppath local: #t)) + ;; (src-run-ids (if run-ids run-ids (db:get-all-run-ids (db:dbdat-get-db (db:get-db maindb 0))))) + ;; (all-run-ids (sort (delete-duplicates (cons 0 src-run-ids)) <)) + ;; (count 1) + ;; (total (length all-run-ids)) + ;; (dead-runs '())) + ;; ;; first fix schema if needed + ;; (map + ;; (lambda (th) + ;; (thread-join! th)) + ;; (map + ;; (lambda (run-id) + ;; (thread-start! + ;; (make-thread + ;; (lambda () + ;; (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)) +;; (if (member 'schema options) + ;; (if (eq? run-id 0) + ;; (let ((maindb (db:dbdat-get-db (db:get-db fromdb #f)))) + ;; (db:patch-schema-maindb run-id maindb)) + ;; (db:patch-schema-rundb run-id frundb))) + ;; (set! count (+ count 1)) + ;; (debug:print 0 *default-log-port* "Finished patching schema for " (if (eq? run-id 0) " main.db " (conc run-id ".db")) ", " count " of " total))))) + ;; all-run-ids)) + ;; ;; Then sync and fix db's + ;; (set! count 0) + ;; (process-fork + ;; (lambda () + ;; (map + ;; (lambda (th) + ;; (thread-join! th)) + ;; (map + ;; (lambda (run-id) + ;; (thread-start! + ;; (make-thread + ;; (lambda () + ;; (let* ((fromdb (if toppath (make-dbr:dbstruct path: toppath local: #t) #f)) + ;; (frundb (db:dbdat-get-db (db:get-db fromdb run-id)))) + ;; (if (eq? run-id 0) + ;; (let ((maindb (db:dbdat-get-db (db:get-db fromdb #f)))) +;; (db:sync-tables (db:sync-main-list dbstruct) #f (db:get-db fromdb #f) mtdb) + ;; (set! dead-runs (db:clean-up-maindb (db:get-db fromdb #f)))) + ;; (begin + ;; ;; NB// must sync first to ensure deleted tests get marked as such in megatest.db +;; (db:sync-tables db:sync-tests-only #f (db:get-db fromdb run-id) mtdb) + ;; (db:clean-up-rundb (db:get-db fromdb run-id))))) + ;; (set! count (+ count 1)) + ;; (debug:print 0 *default-log-port* "Finished clean up of " + ;; (if (eq? run-id 0) + ;; " main.db " (conc run-id ".db")) ", " count " of " total))))) + ;; all-run-ids)))) + + ;; removed deleted runs +;; (let ((dbdir (tasks:get-task-db-path))) +;; (for-each (lambda (run-id) +;; (let ((fullname (conc dbdir "/" run-id ".db"))) +;; (if (file-exists? fullname) +;; (begin +;; (debug:print 0 *default-log-port* "Removing database file for deleted run " fullname) +;; (delete-file fullname))))) +;; dead-runs)))) +;; + ;; (db:close-all dbstruct) + ;; (sqlite3:finalize! mdb) + data-synced))) ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) (debug:print-info 11 *default-log-port* "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params) + (print "I don't work anymore. open-run-close-no-exception-handling needs fixing or removing...") + (exit) (if (or *db-write-access* - (not (member proc *db:all-write-procs*))) + (not #t)) ;; was: (member proc * db:all-write-procs *))) (let* ((db (cond ((pair? idb) (db:dbdat-get-db idb)) ((sqlite3:database? idb) idb) ((not idb) (debug:print-error 0 *default-log-port* "cannot open-run-close with #f anymore")) ((procedure? idb) (idb)) @@ -1272,11 +1332,11 @@ ;;====================================================================== ;; L O G G I N G D B ;;====================================================================== -(define (open-logging-db) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) +(define (open-logging-db) (let* ((dbpath (conc (if *toppath* (conc *toppath* "/") "") "logging.db")) ;; fname) (dbexists (file-exists? dbpath)) (db (sqlite3:open-database dbpath)) (handler (make-busy-timeout (if (args:get-arg "-override-timeout") (string->number (args:get-arg "-override-timeout")) @@ -1328,11 +1388,11 @@ ;; ;; HOWEVER: this code in run:test seems to work fine ;; (> (- (current-seconds)(+ (db:test-get-event_time testdat) ;; (db:test-get-run_duration testdat))) ;; 600) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (test-id run-dir uname testname item-path) (if (and (equal? uname "n/a") (equal? item-path "")) ;; this is a toplevel test ;; what to do with toplevel? call rollup? @@ -1344,11 +1404,11 @@ "SELECT id,rundir,uname,testname,item_path FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time) > (run_duration + ?) AND state IN ('RUNNING','REMOTEHOSTSTART');" run-id deadtime) ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config ;; - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (test-id run-dir uname testname item-path) (if (and (equal? uname "n/a") (equal? item-path "")) ;; this is a toplevel test ;; what to do with toplevel? call rollup? @@ -1362,10 +1422,15 @@ (if (and (null? incompleted) (null? oldlaunched) (null? toplevels)) #f #t))) + +;; given a launch delay (minimum time from last launch) return amount of time to wait +;; +;; (define (db:launch-delay-left dbstruct run-id launch-delay) + ;; select end_time-now from ;; (select testname,item_path,event_time+run_duration as ;; end_time,strftime('%s','now') as now from tests where state in ;; ('RUNNING','REMOTEHOSTSTART','LAUNCED')); @@ -1387,11 +1452,11 @@ ;; ;; HOWEVER: this code in run:test seems to work fine ;; (> (- (current-seconds)(+ (db:test-get-event_time testdat) ;; (db:test-get-run_duration testdat))) ;; 600) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (test-id run-dir uname testname item-path) (if (and (equal? uname "n/a") (equal? item-path "")) ;; this is a toplevel test ;; what to do with toplevel? call rollup? @@ -1403,11 +1468,11 @@ "SELECT id,rundir,uname,testname,item_path FROM tests WHERE run_id=? AND (strftime('%s','now') - event_time) > (run_duration + ?) AND state IN ('RUNNING','REMOTEHOSTSTART');" run-id deadtime) ;; in LAUNCHED for more than one day. Could be long due to job queues TODO/BUG: Need override for this in config ;; - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (test-id run-dir uname testname item-path) (if (and (equal? uname "n/a") (equal? item-path "")) ;; this is a toplevel test ;; what to do with toplevel? call rollup? @@ -1419,11 +1484,11 @@ (debug:print-info 18 *default-log-port* "Found " (length oldlaunched) " old LAUNCHED items, " (length toplevels) " old LAUNCHED toplevel tests and " (length incompleted) " tests marked RUNNING but apparently dead.") ;; These are defunct tests, do not do all the overhead of set-state-status. Force them to INCOMPLETE. ;; - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (let* (;; (min-incompleted (filter (lambda (x) ;; (let* ((testpath (cadr x)) ;; (tdatpath (conc testpath "/testdat.db")) ;; (dbexists (file-exists? tdatpath))) ;; (or (not dbexists) ;; if no file then something wrong - mark as incomplete @@ -1434,24 +1499,27 @@ (if (> (length all-ids) 0) (begin (debug:print 0 *default-log-port* "WARNING: Marking test(s); " (string-intersperse (map conc all-ids) ", ") " as INCOMPLETE") (sqlite3:execute db - (conc "UPDATE tests SET state='INCOMPLETE' WHERE id IN (" + (conc "UPDATE tests SET state='INCOMPLETE' WHERE run_id=? AND id IN (" (string-intersperse (map conc all-ids) ",") - ");"))))) + ");") + run-id)))) ;; Now do rollups for the toplevel tests ;; - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (for-each (lambda (toptest) (let ((test-name (list-ref toptest 3))) ;; (run-id (list-ref toptest 5))) (db:top-test-set-per-pf-counts dbstruct run-id test-name))) toplevels))) +;; BUG: Probably broken - does not explicitly use run-id in the query +;; (define (db:top-test-set-per-pf-counts dbstruct run-id test-name) (db:general-call (db:get-db dbstruct run-id) 'top-test-set-per-pf-counts (list test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name test-name))) ;; Clean out old junk and vacuum the database @@ -1482,11 +1550,11 @@ ;; delete all runs that are state='deleted' "DELETE FROM runs WHERE state='deleted';" ;; delete empty runs "DELETE FROM runs WHERE id NOT IN (SELECT DISTINCT r.id FROM runs AS r INNER JOIN tests AS t ON t.run_id=r.id);" )))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:with-transaction db (lambda () (sqlite3:for-each-row (lambda (tot) (debug:print-info 0 *default-log-port* "Records count before clean: " tot)) @@ -1496,11 +1564,11 @@ (debug:print-info 0 *default-log-port* "Records count after clean: " tot)) count-stmt))) (map sqlite3:finalize! statements) (sqlite3:finalize! count-stmt) ;; (db:find-and-mark-incomplete db) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:execute db "VACUUM;"))) ;; Clean out old junk and vacuum the database ;; ;; Ultimately do something like this: @@ -1523,11 +1591,11 @@ ;; delete all tests that belong to runs that are 'deleted' ;; (conc "DELETE FROM tests WHERE run_id NOT IN (" (string-intersperse (map conc valid-runs) ",") ");") ;; delete all tests that are 'DELETED' "DELETE FROM tests WHERE state='DELETED';" )))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:with-transaction db (lambda () (sqlite3:for-each-row (lambda (tot) (debug:print-info 0 *default-log-port* "Records count before clean: " tot)) @@ -1537,11 +1605,11 @@ (debug:print-info 0 *default-log-port* "Records count after clean: " tot)) count-stmt))) (map sqlite3:finalize! statements) (sqlite3:finalize! count-stmt) ;; (db:find-and-mark-incomplete db) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:execute db "VACUUM;"))) ;; Clean out old junk and vacuum the database ;; ;; Ultimately do something like this: @@ -1570,11 +1638,11 @@ (sqlite3:for-each-row (lambda (run-id) (set! dead-runs (cons run-id dead-runs))) db "SELECT id FROM runs WHERE state='deleted';") - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:with-transaction db (lambda () (sqlite3:for-each-row (lambda (tot) (debug:print-info 0 *default-log-port* "Records count before clean: " tot)) @@ -1584,11 +1652,11 @@ (debug:print-info 0 *default-log-port* "Records count after clean: " tot)) count-stmt))) (map sqlite3:finalize! statements) (sqlite3:finalize! count-stmt) ;; (db:find-and-mark-incomplete db) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:execute db "VACUUM;") dead-runs)) ;;====================================================================== ;; M E T A G E T A N D S E T V A R S @@ -1742,23 +1810,23 @@ (key=?str (string-intersperse (map (lambda (k)(conc k "=?")) keys) " AND "))) (debug:print 3 *default-log-port* "keys: " keys " allvals: " allvals " keyvals: " keyvals " key=?str is " key=?str) (debug:print 2 *default-log-port* "NOTE: using target " (string-intersperse (map cadr keyvals) "/") " for this run") (if (and runname (null? (filter (lambda (x)(not x)) keyvals))) ;; there must be a better way to "apply and" (let ((res #f)) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (apply sqlite3:execute db (conc "INSERT OR IGNORE INTO runs (runname,state,status,owner,event_time" comma keystr ") VALUES (?,?,?,?,strftime('%s','now')" comma valslots ");") allvals) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (apply sqlite3:for-each-row (lambda (id) (set! res id)) db (let ((qry (conc "SELECT id FROM runs WHERE (runname=? " andstr key=?str ");"))) ;(debug:print 4 *default-log-port* "qry: " qry) qry) qryvals) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:execute db "UPDATE runs SET state=?,status=?,event_time=strftime('%s','now') WHERE id=? AND state='deleted';" state status res) res) (begin (debug:print-error 0 *default-log-port* "Called without all necessary keys") #f)))) @@ -1807,11 +1875,11 @@ (vector header res))) ;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db) ;; (define (db:get-changed-run-ids since-time) - (let* ((dbdir (db:dbfile-path #f)) ;; (configf:lookup *configdat* "setup" "dbdir")) + (let* ((dbdir (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir")) (alldbs (glob (conc dbdir "/[0-9]*.db"))) (changed (filter (lambda (dbfile) (> (file-modification-time dbfile) since-time)) alldbs))) (delete-duplicates @@ -1946,11 +2014,11 @@ (totals (make-hash-table)) (curr (make-hash-table)) (res '()) (runs-info '())) ;; First get all the runname/run-ids - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (run-id runname) (set! runs-info (cons (list run-id runname) runs-info))) db "SELECT id,runname FROM runs WHERE state != 'deleted' ORDER BY event_time DESC;") ;; If you change this to the more logical ASC please adjust calls to db:get-run-stats @@ -1994,11 +2062,10 @@ ;; (define (db:get-runs-by-patt dbstruct keys runnamepatt targpatt offset limit fields last-update) ;; test-name) (let* ((tmp (runs:get-std-run-fields keys (or fields '("id" "runname" "state" "status" "owner" "event_time")))) (keystr (car tmp)) (header (cadr tmp)) - (res '()) (key-patt "") (runwildtype (if (substring-index "%" runnamepatt) "like" "glob")) (qry-str #f) (keyvals (if targpatt (keys:target->keyval keys targpatt) '()))) (for-each (lambda (keyval) @@ -2019,19 +2086,21 @@ " ORDER BY event_time " (if limit (conc " LIMIT " limit) "") (if offset (conc " OFFSET " offset) "") ";")) (debug:print-info 4 *default-log-port* "runs:get-runs-by-patt qry=" qry-str " " runnamepatt) - (db:with-db dbstruct #f #f ;; reads db, does not write to it. - (lambda (db) - (sqlite3:for-each-row - (lambda (a . r) - (set! res (cons (list->vector (cons a r)) res))) - db - qry-str - runnamepatt))) - (vector header res))) + (vector header + (reverse + (db:with-db dbstruct #f #f ;; reads db, does not write to it. + (lambda (db) + (sqlite3:fold-row + (lambda (res . r) + (cons (list->vector r) res)) + '() + db + qry-str + runnamepatt))))))) ;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo)) (define (db:get-run-info dbstruct run-id) ;;(if (hash-table-ref/default *run-info-cache* run-id #f) ;; (hash-table-ref *run-info-cache* run-id) @@ -2042,11 +2111,11 @@ (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (header (append keys remfields)) (keystr (conc (keys->keystr keys) "," (string-intersperse remfields ",")))) (debug:print-info 11 *default-log-port* "db:get-run-info run-id: " run-id " header: " header " keystr: " keystr) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (a . x) (set! res (apply vector a x))) db (conc "SELECT " keystr " FROM runs WHERE id=? AND state != 'deleted';") @@ -2070,16 +2139,19 @@ ;; First set any related tests to DELETED (let* ((rdbdat (db:get-db dbstruct run-id)) (rdb (db:dbdat-get-db rdbdat)) (dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat))) - (db:delay-if-busy rdbdat) - (sqlite3:execute rdb "UPDATE tests SET state='DELETED',comment='';") - (sqlite3:execute rdb "DELETE FROM test_steps;") - (sqlite3:execute rdb "DELETE FROM test_data;") - (db:delay-if-busy dbdat) - (sqlite3:execute db "UPDATE runs SET state='deleted',comment='' WHERE id=?;" run-id))) + ;; (db:delay-if-busy rdbdat) + (sqlite3:with-transaction + db + (lambda () + (sqlite3:execute rdb "DELETE FROM test_steps WHERE test_id IN (SELECT id FROM tests WHERE run_id=?);" run-id) + (sqlite3:execute rdb "DELETE FROM test_data WHERE test_id IN (SELECT id FROM tests WHERE run_id=?);" run-id) + (sqlite3:execute rdb "UPDATE tests SET state='DELETED',comment='' WHERE run_id=?;" run-id) + ;; (db:delay-if-busy dbdat) + (sqlite3:execute db "UPDATE runs SET state='deleted',comment='' WHERE id=?;" run-id))))) (define (db:update-run-event_time dbstruct run-id) (db:with-db dbstruct #f @@ -2103,11 +2175,11 @@ (debug:print-info 1 *default-log-port* "" newlockval " run number " run-id))))) (define (db:set-run-status dbstruct run-id status msg) (let* ((dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (if msg (sqlite3:execute db "UPDATE runs SET status=?,comment=? WHERE id=?;" status msg run-id) (sqlite3:execute db "UPDATE runs SET status=? WHERE id=?;" status run-id)))) (define (db:get-run-status dbstruct run-id) @@ -2137,11 +2209,11 @@ (dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat))) (for-each (lambda (key) (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;"))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (key-val) (set! res (cons (list key key-val) res))) db qry run-id))) keys) @@ -2154,11 +2226,11 @@ (dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat))) (for-each (lambda (key) (let ((qry (conc "SELECT " key " FROM runs WHERE id=?;"))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (key-val) (set! res (cons key-val res))) db qry run-id))) keys) @@ -2363,22 +2435,25 @@ (db:general-call dbdat 'delete-test-step-records (list test-id)) ;; (db:delay-if-busy) (db:general-call dbdat 'delete-test-data-records (list test-id)) (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id))) +;; (define (db:delete-old-deleted-test-records dbstruct) - (let ((run-ids (db:get-all-run-ids dbstruct)) + (let (;; (run-ids (db:get-all-run-ids dbstruct)) (targtime (- (current-seconds)(* 30 24 60 60)))) ;; one month in the past - (for-each - (lambda (run-id) - (db:with-db - dbstruct - run-id - #t - (lambda (db) - (sqlite3:execute db "DELETE FROM tests WHERE state='DELETED' AND event_time=) (if (>= value expected) "pass" "fail")) ((<=) (if (<= value expected) "pass" "fail")) (else (conc "ERROR: bad tol comparator " tol)))))) (debug:print 4 *default-log-port* "AFTER2: category: " category " variable: " variable " value: " value ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:execute db "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);" test-id category variable value expected tol units (if comment comment "") status type))) csvlist))) ;; This routine moved from tdb.scm, tdb:read-test-data @@ -2948,11 +3023,11 @@ ;; (define (db:read-test-data dbstruct run-id test-id categorypatt) (let* ((dbdat (db:get-db dbstruct run-id)) (db (db:dbdat-get-db dbdat)) (res '())) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (id test_id category variable value expected tol units comment status type) (set! res (cons (vector id test_id category variable value expected tol units comment status type) res))) db "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt) @@ -3025,14 +3100,14 @@ (string-substitute (regexp "=") "_" (base64:base64-encode (z3:encode-buffer (with-output-to-string - (lambda ()(serialize obj))))) + (lambda ()(serialize obj))))) ;; BB: serialize - this is what causes problems between different builds of megatest communicating. serialize is sensitive to binary image of mtest. #t)) ((zmq nmsg)(with-output-to-string (lambda ()(serialize obj)))) - (else obj))) + (else obj))) ;; rpc (define (db:string->obj msg #!key (transport 'http)) (case transport ;; ((fs) msg) ((http fs) @@ -3042,52 +3117,146 @@ (base64:base64-decode (string-substitute (regexp "_") "=" msg #t))) (lambda ()(deserialize))) (begin - (debug:print-error 0 *default-log-port* "reception failed. Received " msg " but cannot translate it.") + (debug:print-error 0 *default-log-port* "reception failed. Received \"" msg "\" but cannot translate it.") + (print-call-chain (current-error-port)) msg))) ;; crude reply for when things go awry ((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize)))) - (else msg))) + (else msg))) ;; rpc + +;; This is to be the big daddy call (define (db:test-set-status-state dbstruct run-id test-id status state msg) (let ((dbdat (db:get-db dbstruct run-id))) (if (member state '("LAUNCHED" "REMOTEHOSTSTART")) (db:general-call dbdat 'set-test-start-time (list test-id))) - (if msg - (db:general-call dbdat 'state-status-msg (list state status msg test-id)) - (db:general-call dbdat 'state-status (list state status test-id))) - (mt:process-triggers run-id test-id state status))) + ;; (if msg + ;; (db:general-call dbdat 'state-status-msg (list state status msg test-id)) + ;; (db:general-call dbdat 'state-status (list state status test-id))) + (db:set-state-status-and-roll-up-items dbstruct run-id test-id #f state status msg) + ;; process the test_data table + (if (and test-id state status (equal? status "AUTO")) + (db:test-data-rollup dbstruct run-id test-id status)) + (mt:process-triggers run-id test-id state status))) + +;; state is the priority rollup of all states +;; status is the priority rollup of all completed statesfu +;; +;; if test-name is an integer work off that instead of test-name test-path +;; +(define (db:set-state-status-and-roll-up-items dbstruct run-id test-name item-path state status comment) + ;; establish info on incoming test followed by info on top level test + (let* ((db (db:dbdat-get-db (dbr:dbstruct-tmpdb dbstruct))) + (testdat (if (number? test-name) + (db:get-test-info-by-id dbstruct run-id test-name) ;; test-name is actually a test-id + (db:get-test-info dbstruct run-id test-name item-path))) + (test-id (db:test-get-id testdat)) + (test-name (if (number? test-name) + (db:test-get-testname testdat) + test-name)) + (item-path (db:test-get-item-path testdat)) + (tl-testdat (db:get-test-info dbstruct run-id test-name "")) + (tl-test-id (db:test-get-id tl-testdat))) + (sqlite3:with-transaction + db + (lambda () + (db:test-set-state-status-by-id dbstruct run-id test-id state status comment) + (if (not (equal? item-path "")) ;; only roll up IF incoming test is an item + (let* ((state-status-counts (db:get-all-state-status-counts-for-test db run-id test-name item-path)) ;; item-path is used to exclude current state/status of THIS test + (running (length (filter (lambda (x) + (member (dbr:counts-state x) *common:running-states*)) + state-status-counts))) + (bad-not-started (length (filter (lambda (x) + (and (equal? (dbr:counts-state x) "NOT_STARTED") + (not (member (dbr:counts-status x) + *common:not-started-ok-statuses*)))) + state-status-counts))) + (all-curr-states (common:special-sort ;; worst -> best (sort of) + (delete-duplicates + (cons state (map dbr:counts-state state-status-counts))) + *common:std-states* >)) + (all-curr-statuses (common:special-sort ;; worst -> best + (delete-duplicates + (cons status (map dbr:counts-status state-status-counts))) + *common:std-statuses* >)) + (newstate (if (> running 0) + "RUNNING" + (if (> bad-not-started 0) + "COMPLETED" + (car all-curr-states)))) + (newstatus (if (> bad-not-started 0) + "CHECK" + (car all-curr-statuses)))) + ;; (print "Setting toplevel to: " newstate "/" newstatus) + (db:test-set-state-status-by-id dbstruct run-id tl-test-id newstate newstatus #f))))))) + +(define db:roll-up-pass-fail-counts db:set-state-status-and-roll-up-items) ;; call with state = #f to roll up with out accounting for state/status of this item ;; -(define (db:roll-up-pass-fail-counts dbstruct run-id test-name item-path state status) - (if (not (equal? item-path "")) - (let ((dbdat (db:get-db dbstruct run-id))) - ;; (db (db:dbdat-get-db dbdat))) - (db:general-call dbdat 'update-pass-fail-counts (list test-name test-name test-name)) - (db:top-test-set-per-pf-counts dbstruct run-id test-name)))) - -;; (case (string->symbol status) -;; ((RUNNING) (db:general-call dbdat 'top-test-set-running (list test-name))) -;; ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name))) -;; ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name)))) - -;; (if (or (not state) -;; (not (equal? item-path ""))) -;; ;; just do a rollup -;; (begin -;; (db:top-test-set-per-pf-counts dbdat run-id test-name) -;; #f) -;; (begin -;; ;; NOTE: No else clause needed for this case -;; (case (string->symbol status) -;; ((RUNNING) (db:general-call dbdat 'top-test-set-running (list test-name))) -;; ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name))) -;; ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name)))) -;; #f) -;; ))) +;; (define (db:roll-up-pass-fail-counts dbstruct run-id test-name item-path state status) +;; (if (not (equal? item-path "")) ;; if asked to do this for a specific item then do an incremental update +;; (let* ((dbdat (db:get-db dbstruct run-id)) +;; (toptestdat (db:get-test-info dbstruct run-id test-name item-path)) +;; (currtopstate (db:test-get-state toptestdat)) +;; (currtopstatus (db:test-get-status toptestdat)) +;; (nextss (common:apply-state-status currtopstate currtopstatus state status)) +;; (newtopstate (car nextss)) ;; #f or a symbol +;; (newtopstatus (cdr nextss))) ;; #f or a symbol +;; (if (not newtopstate) ;; need to calculate it +;; +;; ;; We rely on the toplevel to track status as state varies. I.e. preserve an ABORT +;; +;; +;; ;; (db (db:dbdat-get-db dbdat))) +;; (db:general-call dbdat 'update-pass-fail-counts (list test-name test-name test-name)) +;; (db:top-test-set-per-pf-counts dbstruct run-id test-name)))) +;; +;; ;; (case (string->symbol status) +;; ;; ((RUNNING) (db:general-call dbdat 'top-test-set-running (list test-name))) +;; ;; ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name))) +;; ;; ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name)))) +;; +;; ;; (if (or (not state) +;; ;; (not (equal? item-path ""))) +;; ;; ;; just do a rollup +;; ;; (begin +;; ;; (db:top-test-set-per-pf-counts dbdat run-id test-name) +;; ;; #f) +;; ;; (begin +;; ;; ;; NOTE: No else clause needed for this case +;; ;; (case (string->symbol status) +;; ;; ((RUNNING) (db:general-call dbdat 'top-test-set-running (list test-name))) +;; ;; ((LAUNCHED) (db:general-call dbdat 'top-test-set (list "LAUNCHED" test-name))) +;; ;; ((ABORT INCOMPLETE) (db:general-call dbdat 'top-test-set (list status test-name)))) +;; ;; #f) +;; ;; ))) + +(define (db:get-all-state-status-counts-for-test db run-id test-name item-path) + (sqlite3:map-row + (lambda (state status count) + (make-dbr:counts state: state status: status count: count)) + db + "SELECT state,status,count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND item_path !=? GROUP BY state,status;" + run-id test-name item-path)) + + +(define (db:get-all-item-states db run-id test-name) + (sqlite3:map-row + (lambda (a) a) + db + "SELECT DISTINCT state FROM tests WHERE item_path != '' AND state != 'DELETED' AND run_id=? AND testname=?" + run-id test-name)) + +(define (db:get-all-item-statuses db run-id test-name) + (sqlite3:map-row + (lambda (a) a) + db + "SELECT DISTINCT status FROM tests WHERE item_path != '' AND state != 'DELETED' AND state='COMPLETED' AND run_id=? AND testname=?" + run-id test-name)) (define (db:test-get-logfile-info dbstruct run-id test-name) (db:with-db dbstruct run-id @@ -3102,12 +3271,12 @@ (set! res (list path final_logf)) (if (directory? path) (debug:print 2 *default-log-port* "Found path: " path) (debug:print 2 *default-log-port* "No such path: " path))) ;; ) db - "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='';" - test-name) + "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='' AND run_id=?;" + test-name run-id) res)))) ;;====================================================================== ;; A G R E G A T E D T R A N S A C T I O N D B W R I T E S ;;====================================================================== @@ -3136,25 +3305,25 @@ ELSE status END WHERE id=?;") ;; DONE '(test-set-log "UPDATE tests SET final_logf=? WHERE id=?;") ;; DONE ;; '(test-set-rundir-by-test-id "UPDATE tests SET rundir=? WHERE id=?") ;; DONE ;; '(test-set-rundir "UPDATE tests SET rundir=? AND testname=? AND item_path=?;") ;; DONE - '(test-set-rundir-shortdir "UPDATE tests SET rundir=?,shortdir=? WHERE testname=? AND item_path=?;") + '(test-set-rundir-shortdir "UPDATE tests SET rundir=?,shortdir=? WHERE testname=? AND item_path=? AND run_id=?;") ;; BROKEN!!! NEEDS run-id '(delete-tests-in-state ;; "DELETE FROM tests WHERE state=?;") ;; DONE "UPDATE tests SET state='DELETED' WHERE state=?") '(tests:test-set-toplog "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path='';") '(update-cpuload-diskfree "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;") ;; DONE '(update-uname-host "UPDATE tests SET uname=?,host=? WHERE id=?;") ;; DONE + '(update-test-rundat "INSERT INTO test_rundat (test_id,update_time,cpuload,diskfree,diskusage,run_duration) VALUES (?,?,?,?,?,?);") '(update-test-state "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") '(update-test-status "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") ;; stuff for roll-up-pass-fail-counts '(update-pass-fail-counts "UPDATE tests SET fail_count=(SELECT count(id) FROM tests WHERE testname=? AND item_path != '' AND status IN ('FAIL','CHECK','INCOMPLETE','ABORT')), pass_count=(SELECT count(id) FROM tests WHERE testname=? AND item_path != '' AND status IN ('PASS','WARN','WAIVED')) - WHERE testname=? AND item_path='';") ;; DONE - '(top-test-set "UPDATE tests SET state=? WHERE testname=? AND item_path='';") ;; DONE - '(top-test-set-running "UPDATE tests SET state='RUNNING' WHERE testname=? AND item_path='';") ;; DONE + WHERE testname=? AND item_path='' AND run_id=?;") ;; DONE ;; BROKEN!!! NEEDS run-id + '(top-test-set-running "UPDATE tests SET state='RUNNING' WHERE testname=? AND item_path='' AND run_id=?;") ;; DONE ;; BROKEN!!! NEEDS run-id ;; Might be the following top-test-set-per-pf-counts query could be better based off of something like this: ;; ;; select state,status,count(state) from tests where run_id=59 AND testname='runfirst' group by state,status; @@ -3240,11 +3409,11 @@ AND item_path != '' AND state = 'COMPLETED' AND status = 'PASS') > 0 THEN 'PASS' WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' ELSE 'UNKNOWN' END - WHERE testname=? AND item_path='';") ;; DONE + WHERE testname=? AND item_path='';") ;; DONE ;; BROKEN!!! NEEDS run-id ;; STEPS '(delete-test-step-records "UPDATE test_steps SET status='DELETED' WHERE test_id=?;") '(delete-test-data-records "UPDATE test_data SET status='DELETED' WHERE test_id=?;") ;; using status since no state field )) @@ -3262,16 +3431,16 @@ sync set-verbosity killserver )) -(define (db:login dbstruct calling-path calling-version run-id client-signature) +(define (db:login dbstruct calling-path calling-version client-signature) (cond ((not (equal? calling-path *toppath*)) (list #f "Login failed due to mismatch paths: " calling-path ", " *toppath*)) - ((not (equal? *run-id* run-id)) - (list #f "Login failed due to mismatch run-id: " run-id ", " *run-id*)) + ;; ((not (equal? *run-id* run-id)) + ;; (list #f "Login failed due to mismatch run-id: " run-id ", " *run-id*)) ((not (equal? megatest-version calling-version)) (list #f "Login failed due to mismatch megatest version: " calling-version ", " megatest-version)) (else (hash-table-set! *logged-in-clients* client-signature (current-seconds)) '(#t "successful login")))) @@ -3280,11 +3449,11 @@ (let ((query (let ((q (alist-ref (if (string? stmtname) (string->symbol stmtname) stmtname) db:queries))) (if q (car q) #f)))) - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (apply sqlite3:execute (db:dbdat-get-db dbdat) query params) #t)) ;; get a summary of state and status counts to calculate a rollup ;; @@ -3332,17 +3501,17 @@ ;; Run this remotely!! ;; (define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path) (let* ((dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat)) - (keys (db:get-keys db)) + (keys (db:get-keys dbstruct)) (selstr (string-intersperse keys ",")) (qrystr (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")) (keyvals #f) (tests-hash (make-hash-table))) ;; first look up the key values from the run selected by run-id - (db:delay-if-busy dbdat) + ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row (lambda (a . b) (set! keyvals (cons a b))) db (conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id) @@ -3381,11 +3550,11 @@ ;; Function recursively checks if .journal exists; if yes means db busy; call itself after delayed interval ;; return the sqlite3 db handle if possible ;; (define (db:delay-if-busy dbdat #!key (count 6)) - (if (not (configf:lookup *configdat* "server" "delay-on-busy")) ;;RADT => two conditions in a if block?? also understand what config looked up + (if (not (configf:lookup *configdat* "server" "delay-on-busy")) (and dbdat (db:dbdat-get-db dbdat)) (if dbdat (let* ((dbpath (db:dbdat-get-path dbdat)) (db (db:dbdat-get-db dbdat)) ;; we'll return this so (db:delay--if-busy can be called inline (dbfj (conc dbpath "-journal"))) @@ -3392,11 +3561,11 @@ (if (handle-exceptions exn (begin (debug:print-info 0 *default-log-port* "WARNING: failed to test for existance of " dbfj) (thread-sleep! 1) - (db:delay-if-busy count (- count 1))) + (db:delay-if-busy count (- count 1))) (file-exists? dbfj)) (case count ((6) (thread-sleep! 0.2) (db:delay-if-busy count: 5)) @@ -3416,11 +3585,11 @@ (thread-sleep! 6.4) (db:delay-if-busy count: 0)) (else (debug:print-info 0 *default-log-port* "delaying db access due to high database load.") (thread-sleep! 12.8)))) - db) ;; RADT => why does it need to return db, not #t + db) "bogus result from db:delay-if-busy"))) (define (db:test-get-records-for-index-file dbstruct run-id test-name) (let ((res '())) (db:with-db Index: dcommon.scm ================================================================== --- dcommon.scm +++ dcommon.scm @@ -24,10 +24,11 @@ (declare (uses synchash)) (include "common_records.scm") (include "db_records.scm") (include "key_records.scm") +(include "run_records.scm") ;; yes, this is non-ideal (define dashboard:update-summary-tab #f) (define dashboard:update-servers-table #f) @@ -215,11 +216,11 @@ (hash-table-set! (dboard:tabdat-path-test-ids data) test-path test-id) (if (not rownum) (let ((rownums (hash-table-values testname-to-row))) (set! rownum (if (null? rownums) 1 - (+ 1 (apply max rownums)))) + (+ 1 (common:max rownums)))) (hash-table-set! testname-to-row fullname rownum) ;; create the label (set! changed (dcommon:modifiy-if-different (dboard:tabdat-runs-matrix data) (conc rownum ":" 0) @@ -259,10 +260,45 @@ (if changed (iup:attribute-set! (dboard:tabdat-runs-matrix data) "REDRAW" "ALL")) ;; (debug:print 2 *default-log-port* "run-changes: " run-changes) ;; (debug:print 2 *default-log-port* "test-changes: " test-changes) (list run-changes all-test-changes))) + +(define (dcommon:runsdat-get-col-num dat target runname force-set) + (let* ((runs-index (dboard:runsdat-runs-index dat)) + (col-name (conc target "/" runname)) + (res (hash-table-ref/default runs-index col-name #f))) + (if res + res + (if force-set + (let ((max-col-num (+ 1 (common:max (cons-1 (hash-table-values runs-index)))))) + (hash-table-set! runs-index col-name max-col-num) + max-col-num))))) + +(define (dcommon:runsdat-get-row-num dat testname itempath force-set) + (let* ((tests-index (dboard:runsdat-runs-index dat)) + (row-name (conc testname "/" itempath)) + (res (hash-table-ref/default runs-index row-name #f))) + (if res + res + (if force-set + (let ((max-row-num (+ 1 (common:max (cons -1 (hash-table-values tests-index)))))) + (hash-table-set! runs-index row-name max-row-num) + max-row-num))))) + +(define (dcommon:rundat-copy-tests-to-by-name rundat) + (let ((src-ht (dboard:rundat-tests rundat)) + (trg-ht (dboard:rundat-tests-by-name rundat))) + (if (and (hash-table? src-ht)(hash-table? trg-ht)) + (begin + (hash-table-clear! trg-ht) + (for-each + (lambda (testdat) + (hash-table-set! trg-ht (test:test-get-fullname testdat) testdat)) + (hash-table-values src-ht))) + (debug:print 0 *default-log-port* "WARNING: src-ht " src-ht " trg-ht " trg-ht)))) + ;;====================================================================== ;; TESTS DATA ;;====================================================================== @@ -507,18 +543,18 @@ (define (dcommon:run-stats commondat tabdat #!key (tab-num #f)) (let* ((stats-matrix (iup:matrix expand: "YES")) (changed #f) (stats-updater (lambda () - (if (dashboard:database-changed? commondat tabdat) + (if (dashboard:database-changed? commondat tabdat context-key: 'run-stats) (let* ((run-stats (rmt:get-run-stats)) (indices (common:sparse-list-generate-index run-stats)) ;; proc: set-cell)) (row-indices (car indices)) (col-indices (cadr indices)) - (max-row (if (null? row-indices) 1 (apply max (map cadr row-indices)))) + (max-row (if (null? row-indices) 1 (common:max (map cadr row-indices)))) (max-col (if (null? col-indices) 1 - (apply max (map cadr col-indices)))) + (common:max (map cadr col-indices)))) (max-visible (max (- (dboard:tabdat-num-tests tabdat) 15) 3)) (max-col-vis (if (> max-col 10) 10 max-col)) (numrows 1) (numcols 1)) (iup:attribute-set! stats-matrix "CLEARVALUE" "CONTENTS") @@ -560,12 +596,15 @@ (if (not (equal? (iup:attribute stats-matrix key) value)) (begin (set! changed #t) (iup:attribute-set! stats-matrix key value))))) run-stats) - (if changed (iup:attribute-set! stats-matrix "REDRAW" "ALL"))))))) - (stats-updater) + (if changed (iup:attribute-set! stats-matrix "REDRAW" "ALL"))) + )))) + ;; (dboard:commondat-please-update-set! commondat #t) ;; force redraw on first pass + ;; (mark-for-update tabdat) + ;; (stats-updater) (dboard:commondat-add-updater commondat stats-updater tab-num: tab-num) ;; (set! dashboard:update-summary-tab updater) (iup:attribute-set! stats-matrix "WIDTHDEF" "40") (iup:vbox ;; (iup:label "Run statistics" #:expand "HORIZONTAL") @@ -999,21 +1038,21 @@ ))) (dboard:tabdat-command-tb-set! data tb) tb) (iup:button "Execute" #:size "50x" #:action (lambda (obj) - (let ((cmd (conc "xterm -geometry 180x20 -e \"" - (iup:attribute (dboard:tabdat-command-tb data) "VALUE") - ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &"))) - (system cmd))))))) + ;; (let ((cmd (conc ;; "xterm -geometry 180x20 -e \"" + (common:run-a-command (iup:attribute (dboard:tabdat-command-tb data) "VALUE"))))))) + ;; ";echo Press any key to continue;bash -c 'read -n 1 -s'\" &"))) + ;; (system cmd))))))) (define (dcommon:command-action-selector commondat tabdat #!key (tab-num #f)) (iup:frame #:title "Set the action to take" (iup:hbox ;; (iup:label "Command to run" #:expand "HORIZONTAL" #:size "70x" #:alignment "LEFT:ACENTER") - (let* ((cmds-list '("run" "remove-runs" "set-state-status" "lock-runs" "unlock-runs")) + (let* ((cmds-list '("run" "remove-runs")) ;; "set-state-status" "lock-runs" "unlock-runs")) (lb (iup:listbox #:expand "HORIZONTAL" #:dropdown "YES" #:action (lambda (obj val index lbstate) ;; (print obj " " val " " index " " lbstate) (dboard:tabdat-command-set! tabdat val) @@ -1046,24 +1085,25 @@ (iup:attribute-set! tb "VALUE" val) (dboard:tabdat-run-name-set! tabdat val) (dashboard:update-run-command tabdat)))) "command-runname-selector lb action")))) (refresh-runs-list (lambda () - (if (dashboard:database-changed? commondat tabdat) - (let* ((target (dboard:tabdat-target-string tabdat)) - (runs-for-targ (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" target #f #f #f 0)) + (if (dashboard:database-changed? commondat tabdat context-key: 'runname-selector-runs-list) + (let* (;; (target (dboard:tabdat-target-string tabdat)) + (runs-for-targ (rmt:get-runs-by-patt (dboard:tabdat-keys tabdat) "%" #f #f #f #f 0)) (runs-header (vector-ref runs-for-targ 0)) (runs-dat (vector-ref runs-for-targ 1)) (run-names (cons default-run-name (map (lambda (x) (db:get-value-by-header x runs-header "runname")) runs-dat)))) + ;; (print "DEBUGINFO: run-names=" run-names) ;; (iup:attribute-set! lb "REMOVEITEM" "ALL") (iuplistbox-fill-list lb run-names selected-item: default-run-name)))))) ;; (dboard:tabdat-updater-for-runs-set! tabdat refresh-runs-list) (dboard:commondat-add-updater commondat refresh-runs-list tab-num: tab-num) - (refresh-runs-list) + ;; (refresh-runs-list) (dboard:tabdat-run-name-set! tabdat default-run-name) (iup:hbox tb lb)))) @@ -1081,13 +1121,14 @@ (dashboard:update-run-command tabdat)) "command-testname-selector tb action")) #:value (dboard:test-patt->lines (dboard:tabdat-test-patts-use tabdat)) #:expand "YES" - #:size "10x30" + #:size "x30" ;; was 10x30 #:multiline "YES"))) (set! test-patterns-textbox tb) + (dboard:tabdat-test-patterns-textbox-set! tabdat tb) tb)) ;; (iup:frame ;; #:title "Target" ;; ;; Target selectors ;; (apply iup:hbox @@ -1112,11 +1153,11 @@ (map cadr *common:std-statuses*) ;; '("PASS" "FAIL" "n/a" "CHECK" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD") (lambda (all) (dboard:tabdat-statuses-set! tabdat all) (dashboard:update-run-command tabdat))))))) -(define (dcommon:command-tests-tasks-canvas data test-records sorted-testnames tests-draw-state) +(define (dcommon:command-tests-tasks-canvas tabdat test-records sorted-testnames tests-draw-state) (iup:frame #:title "Tests and Tasks" (let* ((updater #f) (last-xadj 0) (last-yadj 0) @@ -1158,11 +1199,12 @@ (selected-tests (hash-table-ref tests-draw-state 'selected-tests)) (scalef (hash-table-ref tests-draw-state 'scalef)) (sizey (hash-table-ref tests-draw-state 'sizey)) (xoffset (dcommon:get-xoffset tests-draw-state #f #f)) (yoffset (dcommon:get-yoffset tests-draw-state #f #f)) - (new-y (- sizey y))) + (new-y (- sizey y)) + (test-patterns-textbox (dboard:tabdat-test-patterns-textbox tabdat))) ;; (print "xoffset=" xoffset ", yoffset=" yoffset) ;; (print "\tx\ty\tllx\tlly\turx\tury") (for-each (lambda (test-name) (let* ((rec-coords (hash-table-ref tests-info test-name)) (llx (dcommon:x->canvas (list-ref rec-coords 0) scalef xoffset)) @@ -1174,21 +1216,25 @@ (if (and (eq? pressed 1) (>= x llx) (>= new-y lly) (<= x urx) (<= new-y ury)) - (let ((patterns (string-split (iup:attribute test-patterns-textbox "VALUE")))) + (let* ((box-patterns (string-split (iup:attribute test-patterns-textbox "VALUE"))) + (test-patts (string-split (or (dboard:tabdat-test-patts tabdat) + "") + ",")) + (patterns (delete-duplicates (append box-patterns test-patts)))) (let* ((selected (not (member test-name patterns))) (newpatt-list (if selected (cons test-name patterns) (delete test-name patterns))) (newpatt (string-intersperse newpatt-list "\n"))) + (iup:attribute-set! test-patterns-textbox "VALUE" newpatt) (iup:attribute-set! obj "REDRAW" "ALL") (hash-table-set! selected-tests test-name selected) - (iup:attribute-set! test-patterns-textbox "VALUE" newpatt) - (dboard:tabdat-test-patts-set!-use data (dboard:lines->test-patt newpatt)) - (dashboard:update-run-command data) + (dboard:tabdat-test-patts-set!-use tabdat (dboard:lines->test-patt newpatt)) + (dashboard:update-run-command tabdat) (if updater (updater last-xadj last-yadj))))))) (hash-table-keys tests-info))))))) canvas-obj))) ;;====================================================================== DELETED debugger.scm Index: debugger.scm ================================================================== --- debugger.scm +++ /dev/null @@ -1,73 +0,0 @@ -(use iup) - -(define *debugger-control* #f) -(define *debugger-rownum* 0) -(define *debugger-matrix* #f) -(define *debugger* #f) - -(define (debugger) - (if (not *debugger*) - (set! *debugger* - (thread-start! - (make-thread - (lambda () - (show - (dialog - (let ((pause #f) - (mtrx (matrix - #:expand "YES" - #:numlin 30 - #:numcol 3 - #:numlin-visible 20 - #:numcol-visible 2 - #:alignment1 "ALEFT" - ))) - (set! pause (button "Pause" - #:action (lambda (obj) - (set! *debugger-control* (not *debugger-control*)) - (attribute-set! pause "BGCOLOR" (if *debugger-control* - "200 0 0" - "0 0 200"))))) - (set! *debugger-matrix* mtrx) - (attribute-set! mtrx "WIDTH1" "300") - (vbox - mtrx - (hbox - pause))))) - (main-loop))))))) - -(define (debugger-start #!key (start 2)) - (set! *debugger-rownum* start)) - -(define (debugger-trace-var varname varval) - (let ((oldval (attribute *debugger-matrix* (conc *debugger-rownum* ":1"))) - (newval (conc varval))) - (if (not (equal? oldval newval)) - (begin - ;; (print "DEBUG: " varname " = " newval) - (attribute-set! *debugger-matrix* (conc *debugger-rownum* ":0") varname) - (attribute-set! *debugger-matrix* (conc *debugger-rownum* ":1") (conc varval)) - ;; (attribute-set! *debugger-matrix* "FITTOTEXT" "C1") - )) - (set! *debugger-rownum* (+ *debugger-rownum* 1)))) - - -(define (debugger-pauser) - (debugger) - (attribute-set! *debugger-matrix* "REDRAW" "ALL") - (let loop () - (if *debugger-control* - (begin - (print "PAUSED!") - (thread-sleep! 1) - (loop)) - ;;(thread-sleep! 0.01) - ))) - -;; ;; lets use the debugger eh? -;; (debugger-start) -;; (debugger-trace-var "can-run-more" can-run-more) -;; (debugger-trace-var "hed" hed) -;; (debugger-trace-var "prereqs-not-met" (runs:pretty-string prereqs-not-met)) -;; (debugger-pauser) - ADDED defunct/multi-dboard.scm Index: defunct/multi-dboard.scm ================================================================== --- /dev/null +++ defunct/multi-dboard.scm @@ -0,0 +1,801 @@ +;;====================================================================== +;; Copyright 2006-2013, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + +(use format numbers sql-de-lite srfi-1 posix regex regex-case srfi-69 srfi-18 call-with-environment-variables) +(require-library iup) +(import (prefix iup iup:)) +(use canvas-draw) + +(declare (uses margs)) +(declare (uses megatest-version)) +(declare (uses gutils)) +(declare (uses tree)) +(declare (uses configf)) +(declare (uses portlogger)) +(declare (uses keys)) +(declare (uses common)) + +(include "common_records.scm") +;; (include "db_records.scm") +;; (include "key_records.scm") + +(define help (conc + "Megatest Dashboard, documentation at http://www.kiatoa.com/fossils/megatest + version " megatest-version " + license GPL, Copyright (C) Matt Welland 2011 + +Usage: dashboard [options] + -h : this help + -group groupname : display this group of areas + -test testid : control test identified by testid + -guimonitor : control panel for runs + +Misc + -rows N : set number of rows +")) + +;; process args +(define remargs (args:get-args + (argv) + (list "-group" ;; display this group of areas + "-debug" + ) + (list "-h" + "-v" + "-q" + ) + args:arg-hash + 0)) + +(if (args:get-arg "-h") + (begin + (print help) + (exit))) + +;; (if (args:get-arg "-host") +;; (begin +;; (set! (common:get-remote remote) (string-split (args:get-arg "-host" ":"))) +;; (client:launch)) +;; (client:launch)) + +(define *runremote* #f) +(define *windows* (make-hash-table)) +(define *changed-main* (make-hash-table)) ;; set path/... => #t +(define *changed-mutex* (make-mutex)) ;; use for all incoming change requests +(define *searchpatts* (make-hash-table)) + +(debug:setup) + +(define *tim* (iup:timer)) +(define *ord* #f) + +(iup:attribute-set! *tim* "TIME" 300) +(iup:attribute-set! *tim* "RUN" "YES") + +(define (message-window msg) + (iup:show + (iup:dialog + (iup:vbox + (iup:label msg #:margin "40x40"))))) + +(define (iuplistbox-fill-list lb items . default) + (let ((i 1) + (selected-item (if (null? default) #f (car default)))) + (iup:attribute-set! lb "VALUE" (if selected-item selected-item "")) + (for-each (lambda (item) + (iup:attribute-set! lb (number->string i) item) + (if selected-item + (if (equal? selected-item item) + (iup:attribute-set! lb "VALUE" item))) ;; (number->string i)))) + (set! i (+ i 1))) + items) + i)) + +(define (pad-list l n)(append l (make-list (- n (length l))))) + + +(define (mkstr . x) + (string-intersperse (map conc x) ",")) + +(define (update-search x val) + (hash-table-set! *searchpatts* x val)) + + +;;====================================================================== +;; R E C O R D S +;;====================================================================== + +;; NOTE: Consider switching to defstruct. + +;; data for an area (regression or testsuite) +;; +(define-record areadat + name ;; area name + path ;; mt run area home + configdat ;; megatest config + denoise ;; focal point for not putting out same messages over and over + client-signature ;; key for client-server conversation + remote ;; hash of all the client side connnections + run-keys ;; target keys for this area + runs ;; used in dashboard, hash of run-ids -> rundat + read-only ;; can I write to this area? + monitordb ;; db handle for monitor.db + maindb ;; db handle for main.db + ) + +;; rundat, basic run data +;; +(define-record rundat + id ;; the run-id + target ;; val1/val2 ... corrosponding to run-keys in areadat + runname + state ;; state of the run, symbol + status ;; status of the run, symbol + event-time ;; when the run was initiated + tests ;; hash of test-id -> testdat, QUESTION: separate by run-id? + db ;; db handle + ) + +;; testdat, basic test data +(define-record testdat + run-id ;; what run is this from + id ;; test id + testname ;; test name + itempath ;; item path + state ;; test state, symbol + status ;; test status, symbol + event-time ;; when the test started + duration ;; how long the test took + ) + +;; general data for the dboard application +;; +(define-record data + cfgdat ;; data from ~/.megatest/.dat + areas ;; hash of areaname -> area-rec + current-window-id ;; + current-tab-id ;; + update-needed ;; flag to indicate that the tab pointed to by current tab id needs refreshing immediately + tabs ;; hash of tab-id -> areaname (??) should be of type "tab" + ) + +;; all the components of an area display, all fits into a tab but +;; parts may be swapped in/out as needed +;; +(define-record tab + tree + matrix ;; the spreadsheet + areadat ;; the one-structure (one day dbstruct will be put in here) + view-path ;; //... + view-type ;; standard, etc. + controls ;; the controls + data ;; all the data kept in sync with db + filters ;; user filters, alist name -> filter record, eventually store these in ~/.megatest/.dat? + run-id ;; the current run-id + test-ids ;; the current test id hash, run-id => test-id + command ;; the command from the entry field + headers ;; hash of header -> colnum + rows ;; hash of rowname -> rownum + ) + +(define-record filter + target ;; hash of widgets for the target + runname ;; the runname widget + testpatt ;; the testpatt widget + ) + +;;====================================================================== +;; D B +;;====================================================================== + +;; These are all using sql-de-lite and independent of area so cannot use stuff +;; from db.scm + +;; NB// run-id=#f => return dbdir only +;; +(define (areadb:dbfile-path areadat run-id) + (let* ((cfgdat (areadat-configdat areadat)) + (dbdir (or (configf:lookup cfgdat "setup" "dbdir") + (conc (configf:lookup cfgdat "setup" "linktree") "/.db"))) + (fname (if run-id + (case run-id + ((-1) "monitor.db") + ((0) "main.db") + (else (conc run-id ".db"))) + #f))) + (handle-exceptions + exn + (begin + (debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir) + (exit 1)) + (if (not (directory? dbdir))(create-directory dbdir #t))) + (if fname + (conc dbdir "/" fname) + dbdir))) + +;; -1 => monitor.db +;; 0 => main.db +;; >1 => .db +;; +(define (areadb:open areadat run-id) + (let* ((runs (areadat-runs areadat)) + (rundat (if (> run-id 0) ;; it is a run + (hash-table-ref/default runs run-id #f) + #f)) + (db (case run-id ;; if already opened, get the db and return it + ((-1) (areadat-monitordb areadat)) + ((0) (areadat-maindb areadat)) + (else (if rundat + (rundat-db rundat) + #f))))) + (if db + db ;; merely return the already opened db + (let* ((dbfile (areadb:dbfile-path areadat run-id)) ;; not already opened, so open it + (db (if (file-exists? dbfile) + (open-database dbfile) + (begin + (debug:print-error 0 *default-log-port* "I was asked to open " dbfile ", but file does not exist or is not readable.") + #f)))) + (case run-id + ((-1)(areadat-monitordb-set! areadat db)) + ((0) (areadat-maindb-set! areadat db)) + (else (rundat-db-set! rundat db))) + db)))) + +;; populate the areadat tests info, does NOT fill the tests data itself unless asked +;; +(define (areadb:populate-run-info areadat) + (let* ((runs (or (areadat-runs areadat) (make-hash-table))) + (keys (areadat-run-keys areadat)) + (maindb (areadb:open areadat 0))) + (if maindb + (query (for-each-row (lambda (row) + (let ((id (list-ref row 0)) + (dat (apply make-rundat (append row (list #f #f))))) ;; add placeholders for tests and db + (print row) + (hash-table-set! runs id dat)))) + (sql maindb (conc "SELECT id," + (string-intersperse keys "||'/'||") + ",runname,state,status,event_time FROM runs WHERE state != 'deleted';"))) + (debug:print-error 0 *default-log-port* "no main.db found at " (areadb:dbfile-path areadat 0))) + areadat)) + +;; given an areadat and target/runname patt fill up runs data +;; +;; ?????/ + +;; given a list of run-ids refresh/retrieve runs data into areadat +;; +(define (areadb:fill-tests areadat #!key (run-ids #f)) + (let* ((runs (or (areadat-runs areadat) (make-hash-table)))) + (for-each + (lambda (run-id) + (let* ((rundat (hash-table-ref/default runs run-id #f)) + (tests (if (and rundat + (rundat-tests rundat)) ;; re-use existing hash table? + (rundat-tests rundat) + (let ((ht (make-hash-table))) + (rundat-tests-set! rundat ht) + ht))) + (rundb (areadb:open areadat run-id))) + (query (for-each-row (lambda (row) + (let* ((id (list-ref row 0)) + (testname (list-ref row 1)) + (itempath (list-ref row 2)) + (state (list-ref row 3)) + (status (list-ref row 4)) + (eventtim (list-ref row 5)) + (duration (list-ref row 6))) + (hash-table-set! tests id + (make-testdat run-id id testname itempath state status eventtim duration))))) + (sql rundb "SELECT id,testname,item_path,state,status,event_time,run_duration FROM tests WHERE state != 'DELETED';")))) + (or run-ids (hash-table-keys runs))) + areadat)) + + +;; initialize and refresh data +;; +(define (dboard:general-updater con port) + (for-each + (lambda (window-id) + ;; (print "Processing for window-id " window-id) + (let* ((window-dat (hash-table-ref *windows* window-id)) + (areas (data-areas window-dat)) + ;; (keys (areadat-run-keys area-dat)) + (tabs (data-tabs window-dat)) + (tab-ids (hash-table-keys tabs)) + (current-tab (if (null? tab-ids) + #f + (hash-table-ref tabs (car tab-ids)))) + (current-tree (if (null? tab-ids) #f (tab-tree current-tab))) + (current-node (if (null? tab-ids) 0 (string->number (iup:attribute current-tree "VALUE")))) + (current-path (if (eq? current-node 0) + "Areas" + (string-intersperse (tree:node->path current-tree current-node) "/"))) + (current-matrix (if (null? tab-ids) #f (tab-matrix current-tab))) + (seen-nodes (make-hash-table)) + (path-changed (if current-tab + (equal? current-path (tab-view-path current-tab)) + #t))) + ;; (debug:print-info 0 *default-log-port* "Current path: " current-path) + ;; now for each area in the window gather the data + (if path-changed + (begin + (debug:print-info 0 *default-log-port* "clearing matrix - path changed") + (dboard:clear-matrix current-tab))) + (for-each + (lambda (area-name) + ;; (print "Processing for area-name " area-name) + (let* ((area-dat (hash-table-ref areas area-name)) + (area-path (areadat-path area-dat)) + (runs (areadat-runs area-dat))) + (if (hash-table-ref/default *changed-main* area-path 'processed) + (begin + (print "Processing " area-dat " for area-name " area-name) + (hash-table-set! *changed-main* area-path #f) + (areadb:populate-run-info area-dat) + (for-each + (lambda (run-id) + (let* ((run (hash-table-ref runs run-id)) + (target (rundat-target run)) + (runname (rundat-runname run))) + (if current-tree + (let* ((partial-path (append (string-split target "/")(list runname))) + (full-path (cons area-name partial-path))) + (if (not (hash-table-exists? seen-nodes full-path)) + (begin + (print "INFO: Adding node " partial-path " to section " area-name) + (tree:add-node current-tree "Areas" full-path) + (areadb:fill-tests area-dat run-ids: (list run-id)))) + (hash-table-set! seen-nodes full-path #t))))) + (hash-table-keys runs)))) + (if (or (equal? "Areas" current-path) + (string-match (conc "^Areas/" area-name "(|\\/.*)$") current-path)) + (dboard:redraw-area area-name area-dat current-tab current-matrix current-path)))) + (hash-table-keys areas)))) + (hash-table-keys *windows*))) + +;;====================================================================== +;; D A S H B O A R D D B +;;====================================================================== + +;; All moved to common.scm + +;;====================================================================== +;; T R E E +;;====================================================================== + +;; - - - - + +(define (dashboard:tree-browser data adat window-id) + ;; (iup:split + (let* ((tb (iup:treebox + #:value 0 + #:title "Areas" + #:expand "YES" + #:addexpanded "NO" + #:selection-cb + (lambda (obj id state) + ;; (print "obj: " obj ", id: " id ", state: " state) + (let* ((tree-path (tree:node->path obj id)) + (area (car tree-path)) + (areadat-path (cdr tree-path))) + #f + ;; (test-id (tree-path->test-id (cdr run-path)))) + ;; (if test-id + ;; (hash-table-set! (dboard:data-curr-test-ids *data*) + ;; window-id test-id)) + ;; (print "path: " (tree:node->path obj id) " test-id: " test-id)))))) + ))))) + ;; (iup:attribute-set! tb "VALUE" "0") + ;; (iup:attribute-set! tb "NAME" "Runs") + ;; (iup:attribute-set! tb "ADDEXPANDED" "NO") + ;; (dboard:data-tests-tree-set! *data* tb) + tb)) + +;;====================================================================== +;; M A I N M A T R I X +;;====================================================================== + +;; General displayer +;; +(define (dashboard:main-matrix data adat window-id) + (let* (;; (tab-dat (areadat- + (view-matrix (iup:matrix + ;; (runs-for-targ (db:get-runs-by-patt *dbstruct-local* *keys* "%" target #f #f #f)) + #:expand "YES" + ;; #:fittosize "YES" + #:resizematrix "YES" + #:scrollbar "YES" + #:numcol 100 + #:numlin 100 + #:numcol-visible 3 + #:numlin-visible 20 + #:click-cb (lambda (obj lin col status) + (print "obj: " obj " lin: " lin " col: " col " status: " status " value: " (iup:attribute obj "VALUE")))))) + + ;; (iup:attribute-set! view-matrix "RESIZEMATRIX" "YES") + (iup:attribute-set! view-matrix "WIDTH0" "100") + ;; (dboard:data-runs-matrix-set! *data* runs-matrix) + ;; (iup:hbox + ;; (iup:frame + ;; #:title "Runs browser" + ;; (iup:vbox + view-matrix)) + +;;====================================================================== +;; A R E A S +;;====================================================================== + +(define (dashboard:init-area data area-name apath) + (let* ((mtconf (dboard:read-mtconf apath)) + (area-dat (let ((ad (make-areadat + area-name ;; area name + apath ;; path to area + ;; 'http ;; transport + mtconf ;; megatest.config + (make-hash-table) ;; denoise hash + #f ;; client-signature + #f ;; remote connections + (keys:config-get-fields mtconf) ;; run keys + (make-hash-table) ;; run-id -> (hash of test-ids => dat) + (and (file-exists? apath)(file-write-access? apath)) ;; read-only + #f + #f + ))) + (hash-table-set! (data-areas data) area-name ad) + ad))) + area-dat)) + +;; given the keys for an area and a path from the tree browser +;; return the level: areas area runs run tests test +;; +(define (dboard:get-view-type keys current-path) + (let* ((path-parts (string-split current-path "/")) + (path-len (length path-parts))) + (cond + ((equal? current-path "Areas") 'areas) + ((eq? path-len 2) 'area) + ((<= (+ (length keys) 2) path-len) 'runs) + (else 'run)))) + +(define (dboard:clear-matrix tab) + (if tab + (begin + (iup:attribute-set! (tab-matrix tab) "CLEARVALUE" "ALL") + (tab-headers-set! tab (make-hash-table)) + (tab-rows-set! tab (make-hash-table))))) + +;; full redraw of a given area +;; +(define (dboard:redraw-area area-name area-dat tab-dat current-matrix current-path) + (let* ((keys (areadat-run-keys area-dat)) + (runs (areadat-runs area-dat)) + (headers (tab-headers tab-dat)) + (rows (tab-rows tab-dat)) + (used-cols (hash-table-values headers)) + (used-rows (hash-table-values rows)) + (touched (make-hash-table)) ;; (vector row col) ==> true, touched cell + (view-type (dboard:get-view-type keys current-path)) + (changed #f) + (state-statuses (list "PASS" "FAIL" "WARN" "CHECK" "SKIP" "RUNNING" "LAUNCHED"))) + ;; (debug:print 0 *default-log-port* "current-matrix=" current-matrix) + (case view-type + ((areas) ;; find row for this area, if not found, create new entry + (let* ((curr-rownum (hash-table-ref/default rows area-name #f)) + (next-rownum (+ (apply max (cons 0 used-rows)) 1)) + (rownum (or curr-rownum next-rownum)) + (coord (conc rownum ":0"))) + (if (not curr-rownum)(hash-table-set! rows area-name rownum)) + (if (not (equal? (iup:attribute current-matrix coord) area-name)) + (begin + (let loop ((hed (car state-statuses)) + (tal (cdr state-statuses)) + (count 1)) + (if (not (equal? (iup:attribute current-matrix (conc "0:" count)) hed)) + (iup:attribute-set! current-matrix (conc "0:" count) hed)) + (iup:attribute-set! current-matrix (conc rownum ":" count) "0") + (if (not (null? tal)) + (loop (car tal)(cdr tal)(+ count 1)))) + (debug:print-info 0 *default-log-port* "view-type=" view-type ", rownum=" rownum ", curr-rownum=" curr-rownum ", next-rownum=" next-rownum ", coord=" coord ", area-name=" area-name) + (iup:attribute-set! current-matrix coord area-name) + (set! changed #t)))))) + (if changed (iup:attribute-set! current-matrix "REDRAW" "ALL")))) + + + + ;; (dboard:clear-matrix current-matrix used-cols used-rows touched) ;; clear all + + + +;;====================================================================== +;; D A S H B O A R D +;;====================================================================== + +(define (dashboard:area-panel aname data window-id) + (let* ((apath (configf:lookup (data-cfgdat data) aname "path")) ;; (hash-table-ref (dboard:data-cfgdat data) area-name)) + ;; (hash-table-ref (dboard:data-cfgdat data) aname)) + (area-dat (dashboard:init-area data aname apath)) + (tb (dashboard:tree-browser data area-dat window-id)) ;; (dboard:areas-tree-browser data) + (ad (dashboard:main-matrix data area-dat window-id)) + (areas (data-areas data)) + (dboard-dat (make-tab + #f ;; tree + #f ;; matrix + area-dat ;; + #f ;; view path + 'default ;; view type + #f ;; controls + (make-hash-table) ;; cached data? not sure how to use this yet :) + #f ;; filters + #f ;; the run-id + (make-hash-table) ;; run-id -> test-id, for current test id + "" + (make-hash-table) ;; headername -> colnum + (make-hash-table) ;; rowname -> rownum + ))) + (hash-table-set! (data-areas data) aname area-dat) ;; dboard-dat) + (hash-table-set! (data-tabs data) window-id dboard-dat) + (tab-tree-set! dboard-dat tb) + (tab-matrix-set! dboard-dat ad) + (iup:split + #:value 200 + tb ad))) + + +;; Main Panel +;; +(define (dashboard:main-panel data window-id) + (iup:dialog + #:title "Megatest Control Panel" +;; #:menu (dcommon:main-menu data) + #:shrink "YES" + (iup:vbox + (let* ((area-names (hash-table-keys (data-cfgdat data))) + (area-panels (map (lambda (aname) + (dashboard:area-panel aname data window-id)) + area-names)) + (tabtop (apply iup:tabs + #:tabchangepos-cb (lambda (obj curr prev) + (data-current-tab-id-set! data curr) + (data-update-needed-set! data #t) + (print "Tab is: " curr ", prev was " prev)) + area-panels)) + (tabs (data-tabs data))) + (if (not (null? area-names)) + (let loop ((index 0) + (hed (car area-names)) + (tal (cdr area-names))) + ;; (hash-table-set! tabs index hed) + (debug:print 0 *default-log-port* "Adding area " hed " with index " index " to dashboard") + (iup:attribute-set! tabtop (conc "TABTITLE" index) hed) + (if (not (null? tal)) + (loop (+ index 1)(car tal)(cdr tal))))) + tabtop)))) + + +;;====================================================================== +;; N A N O M S G S E R V E R +;;====================================================================== + +(define (dboard:server-service soc port) + (print "server starting") + (let loop ((msg-in (nn-recv soc)) + (count 0)) + (if (eq? 0 (modulo count 1000)) + (print "server received: " msg-in ", count=" count)) + (cond + ;; + ;; quit + ;; + ((equal? msg-in "quit") + (nn-send soc "Ok, quitting")) + ;; + ;; ping + ;; + ((and (>= (string-length msg-in) 4) + (equal? (substring msg-in 0 4) "ping")) + (nn-send soc (conc (current-process-id))) + (loop (nn-recv soc)(+ count 1))) + ;; + ;; main changed + ;; + ((and (>= (string-length msg-in) 4) + (equal? (substring msg-in 0 4) "main")) + (let ((parts (string-split msg-in " "))) + (hash-table-set! *changed-main* (cadr parts) #t) + (nn-send soc "got it!"))) + ;; + ;; ?? + ;; + (else + (nn-send soc "hello " msg-in " you got to the else clause!"))) + (loop (nn-recv soc)(if (> count 20000000) + 0 + (+ count 1))))) + +(define (dboard:one-time-ping-receive soc port) + (let ((msg-in (nn-recv soc))) + (if (and (>= (string-length msg-in) 4) + (equal? (substring msg-in 0 4) "ping")) + (nn-send soc (conc (current-process-id)))))) + +(define (dboard:server-start given-port #!key (num-tries 200)) + (let* ((rep (nn-socket 'rep)) + (port (or given-port (portlogger:main "find"))) + (con (conc "tcp://*:" port))) + ;; register this connect here .... + (nn-bind rep con) + (thread-start! + (make-thread (lambda () + (dboard:one-time-ping-receive rep port)) + "one time receive thread")) + (if (dboard:ping-self "localhost" port) + (begin + (print "INFO: dashboard nanomsg server started on " port) + (values rep port)) + (begin + (print "WARNING: couldn't create server on port " port) + (portlogger:main "set" "failed") + (if (> num-tries 0) + (dboard:server-start #f (- num-tries 1)) + (begin + (print "ERROR: failed to start nanomsg server") + (values #f #f))))))) + +(define (dboard:server-close con port) + (nn-close con) + (portlogger:main "set" port "released")) + +(define (dboard:ping-self host port #!key (return-socket #t)) + ;; send a random number along with pid and check that we get it back + (let* ((req (nn-socket 'req)) + (key "ping") + (success #f) + (keepwaiting #t) + (ping (make-thread + (lambda () + (print "ping: sending string \"" key "\", expecting " (current-process-id)) + (nn-send req key) + (let ((result (nn-recv req))) + (if (equal? (conc (current-process-id)) result) + (begin + (print "ping, success: received \"" result "\"") + (set! success #t)) + (begin + (print "ping, failed: received key \"" result "\"") + (set! keepwaiting #f) + (set! success #f))))) + "ping")) + (timeout (make-thread (lambda () + (let loop ((count 0)) + (thread-sleep! 1) + (print "still waiting after " count " seconds...") + (if (and keepwaiting (< count 10)) + (loop (+ count 1)))) + (if keepwaiting + (begin + (print "timeout waiting for ping") + (thread-terminate! ping)))) + "timeout"))) + (nn-connect req (conc "tcp://" host ":" port)) + (handle-exceptions + exn + (begin + (print-call-chain) + (print 0 " message: " ((condition-property-accessor 'exn 'message) exn)) + (print "exn=" (condition->list exn)) + (print "ping failed to connect to " host ":" port)) + (thread-start! timeout) + (thread-start! ping) + (thread-join! ping) + (if success (thread-terminate! timeout))) + (if return-socket + (if success req #f) + (begin + (nn-close req) + success)))) + +;;====================================================================== +;; C O N F I G U R A T I O N +;;====================================================================== + +;; Get the configuration file for a group name, if the group name is "default" and it doesn't +;; exist, create it and add the current path if it contains megatest.config +;; +(define (dboard:get-config group-name) + (let* ((fname (conc (getenv "HOME") "/.megatest/" group-name ".dat"))) + (if (file-exists? fname) + (read-config fname (make-hash-table) #t) + (if (dboard:create-config fname) + (dboard:get-config group-name) + (make-hash-table))))) + +(define (dboard:create-config fname) + ;; (handle-exceptions + ;; exn + ;; + ;; #f ;; failed to create - just give up + (let* ((dirname (pathname-directory fname)) + (file-name (pathname-strip-directory fname)) + (curr-mtcfgdat (find-config "megatest.config" + toppath: (or (get-environment-variable "MT_RUN_AREA_HOME")(current-directory)))) + (curr-mtcfg (if (and curr-mtcfgdat (not (null? curr-mtcfgdat)))(cadr curr-mtcfgdat) #f)) + (curr-mtpath (if curr-mtcfg (car curr-mtcfgdat) #f))) + (if curr-mtpath + (begin + (debug:print-info 0 *default-log-port* "Creating config file " fname) + (if (not (file-exists? dirname)) + (create-directory dirname #t)) + (with-output-to-file fname + (lambda () + (let ((aname (pathname-strip-directory curr-mtpath))) + (print "[" aname "]") + (print "path " curr-mtpath)))) + #t) + (begin + (debug:print-info 0 *default-log-port* "Need to create a config but no megatest.config found: " curr-mtcfgdat) + #f)))) +;; ) + +(define (dboard:read-mtconf apath) + (let* ((mtconffile (conc apath "/megatest.config"))) + (call-with-environment-variables + (list (cons "MT_RUN_AREA_HOME" apath)) + (lambda () + (read-config mtconffile (make-hash-table) #f)) ;; megatest.config + ))) + + +;;====================================================================== +;; G U I S T U F F +;;====================================================================== + +;;; main. Theoretically could have multiple windows (each with a group of tags, thus window-id +;;; +(define (dboard:make-window window-id) + (let* (;; (window-id 0) + (groupn (or (args:get-arg "-group") "default")) + (cfgdat (dboard:get-config groupn)) + ;; (cfgdat (if (file-exists? cfname)(read-config cfname (make-hash-table) #t)(make-hash-table))) + (data (make-data + cfgdat ;; this is the data from ~/.megatest for the selected group + (make-hash-table) ;; areaname -> area-rec + 0 ;; current window id + 0 ;; current tab id + #f ;; redraw needed for current tab id + (make-hash-table) ;; tab-id -> areaname + ))) + (hash-table-set! *windows* window-id data) + (iup:show (dashboard:main-panel data window-id)) + (iup:main-loop))) + +;; ease debugging by loading ~/.dashboardrc +(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc"))) + (if (file-exists? debugcontrolf) + (load debugcontrolf))) + +(define (main) + (let-values + (((con port)(dboard:server-start #f))) + (let ((portnum (if (string? port)(string->number port) port))) + ;; got here, monitor/dashboard was started + (mddb:register-dashboard portnum) + (thread-start! (make-thread (lambda ()(dboard:server-service con portnum)) "server service")) + (thread-start! (make-thread (lambda () + (let loop () + (dboard:general-updater con portnum) + (thread-sleep! 1) + (loop))) "general updater")) + (dboard:make-window 0) + (mddb:unregister-dashboard (get-host-name) portnum) + (dboard:server-close con port)))) + ADDED defunct/nmsg-transport.scm Index: defunct/nmsg-transport.scm ================================================================== --- /dev/null +++ defunct/nmsg-transport.scm @@ -0,0 +1,358 @@ + +;; Copyright 2006-2012, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. + +(require-extension (srfi 18) extras tcp s11n) + +(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest) +(import (prefix sqlite3 sqlite3:)) + +;; (use nanomsg) + +(declare (unit nmsg-transport)) + +(declare (uses common)) +(declare (uses db)) +(declare (uses tests)) +(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. +(declare (uses server)) + +(include "common_records.scm") +(include "db_records.scm") + +;; Transition to pub --> sub with pull <-- push +;; +;; 1. client sends request to server via push to the pull port +;; 2. server puts request in queue or processes immediately as appropriate +;; 3. server puts responses from completed requests into pub port +;; +;; TODO +;; +;; Done Tested +;; [x] [ ] 1. Add columns pullport pubport to servers table +;; [x] [ ] 2. Add rm of monitor.db if older than 11/12/2012 +;; [x] [ ] 3. Add create of pullport and pubport with finding of available ports +;; [x] [ ] 4. Add client compose of request +;; [x] [ ] - name of client: testname/itempath-test_id-hostname +;; [x] [ ] - name of request: callname, params +;; [x] [ ] - request key: f(clientname, callname, params) +;; [x] [ ] 5. Add processing of subscription hits +;; [x] [ ] - done when get key +;; [x] [ ] - return results +;; [x] [ ] 6. Add timeout processing +;; [x] [ ] - after 60 seconds +;; [ ] [ ] i. check server alive, connect to new if necessary +;; [ ] [ ] ii. resend request +;; [ ] [ ] 7. Turn self ping back on + +(define (nmsg-transport:make-server-url hostport #!key (bindall #f)) + (if (not hostport) + #f + (conc "tcp://" (if bindall "*" (car hostport)) ":" (cadr hostport)))) + +(define *server-loop-heart-beat* (current-seconds)) +(define *heartbeat-mutex* (make-mutex)) + +;;====================================================================== +;; S E R V E R +;;====================================================================== + +(define (nmsg-transport:run dbstruct hostn run-id server-id #!key (retrynum 1000)) + (debug:print 2 *default-log-port* "Attempting to start the server ...") + (let* ((start-port (portlogger:open-run-close portlogger:find-port)) + (server-thread (make-thread (lambda () + (nmsg-transport:try-start-server dbstruct run-id start-port server-id)) + "server thread")) + (tdbdat (tasks:open-db))) + (thread-start! server-thread) + (thread-sleep! 0.1) + (if (nmsg-transport:ping hostn start-port timeout: 2 expected-key: (current-process-id)) + (let ((interface (if (equal? hostn "-")(get-host-name) hostn))) + (tasks:server-set-interface-port (db:delay-if-busy tdbdat) server-id interface start-port) + (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep") + (set! *server-info* (list hostn start-port)) ;; probably not needed anymore? currently used by keep-running + (thread-sleep! 3) ;; give some margin for queries to complete before switching from file based access to server based access + ;; (set! *inmemdb* dbstruct) + (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running") + (thread-start! (make-thread + (lambda ()(nmsg-transport:keep-running server-id run-id)) + "keep running")) + (thread-join! server-thread)) + (if (> retrynum 0) + (begin + (debug:print 0 *default-log-port* "WARNING: Failed to connect to server (self) on host " hostn ":" start-port ", trying again.") + (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id "failed to start, never received server alive signature") + (portlogger:open-run-close portlogger:set-failed start-port) + (nmsg-transport:run dbstruct hostn run-id server-id)) + (begin + (debug:print-error 0 *default-log-port* "could not find an open port to start server on. Giving up") + (exit 1)))))) + +(define (nmsg-transport:try-start-server dbstruct run-id portnum server-id) + (let ((repsoc (nn-socket 'rep))) + (nn-bind repsoc (conc "tcp://*:" portnum)) + (let loop ((msg-in (nn-recv repsoc))) + (let* ((dat (db:string->obj msg-in transport: 'nmsg))) + (debug:print 0 *default-log-port* "server, received: " dat) + (let ((result (api:execute-requests dbstruct dat))) + (debug:print 0 *default-log-port* "server, sending: " result) + (nn-send repsoc (db:obj->string result transport: 'nmsg))) + (loop (nn-recv repsoc)))))) + +;; all routes though here end in exit ... +;; +(define (nmsg-transport:launch run-id) + (let* ((tdbdat (tasks:open-db)) + (dbstruct (db:setup run-id)) + (hostn (or (args:get-arg "-server") "-"))) + (set! *run-id* run-id) + (set! *inmemdb* dbstruct) + ;; with nbfake daemonize isn't really needed + ;; + ;; (if (args:get-arg "-daemonize") + ;; (begin + ;; (daemon:ize) + ;; (if *alt-log-file* ;; we should re-connect to this port, I think daemon:ize disrupts it + ;; (begin + ;; (current-error-port *alt-log-file*) + ;; (current-output-port *alt-log-file*))))) + (if (server:check-if-running run-id) + (begin + (debug:print-info 0 *default-log-port* "Server for run-id " run-id " already running") + (exit 0))) + (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)) + (remtries 4)) + (if (not server-id) + (if (> remtries 0) + (begin + (thread-sleep! 2) + (if (not (server:check-if-running run-id)) + (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id) + (- remtries 1)) + (begin + (debug:print-info 0 *default-log-port* "Another server took the slot, exiting") + (exit 0)))) + (begin + ;; since we didn't get the server lock we are going to clean up and bail out + (debug:print-info 2 *default-log-port* "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue") + (tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " http-transport:launch") + )) + ;; locked in a server id, try to start up + (nmsg-transport:run dbstruct hostn run-id server-id)) + (set! *didsomething* #t) + (exit)))) + +;;====================================================================== +;; S E R V E R U T I L I T I E S +;;====================================================================== + +(define (nmsg-transport:mk-signature) + (message-digest-string (md5-primitive) + (with-output-to-string + (lambda () + (write (list (current-directory) + (argv))))))) + +;;====================================================================== +;; C L I E N T S +;;====================================================================== + +;; ping the server at host:port +;; return the open socket if successful (return-socket == #t) +;; expect the key expected-key returned in payload +;; send our-key or #f as payload +;; +(define (nmsg-transport:ping hostn port #!key (timeout 3)(return-socket #t)(expected-key #f)(our-key #f)(socket #f)) + ;; send a random number along with pid and check that we get it back + (let* ((host (if (or (not hostn) + (equal? hostn "-")) ;; use localhost + (get-host-name) + hostn)) + (req (or socket + (let ((soc (nn-socket 'req))) + (nn-connect soc (conc "tcp://" host ":" port)) + soc))) + (success #t) + (dat (vector "ping" our-key)) + (result (condition-case + (nmsg-transport:client-api-send-receive-raw req dat timeout: timeout) + ((timeout)(set! success #f) #f))) + (key (if success + (vector-ref result 1) + #f))) + (debug:print 0 *default-log-port* "success=" success ", key=" key ", expected-key=" expected-key ", equal? " (equal? key expected-key)) + (if (and success + (or (not expected-key) ;; just getting a reply is good enough then + (equal? key expected-key))) + (if return-socket + req + (begin + (if (not socket)(nn-close req)) ;; don't want a side effect of closing socket if handed it + #t)) + (begin + (if (not socket)(nn-close req)) ;; failed to ping, close socket as side effect + #f)))) + +;; send data to server, wait max of timeout seconds for a response. +;; return #( success/fail result ) +;; +;; for effiency it is easier to do the obj->string and string->obj here. +;; +(define (nmsg-transport:client-api-send-receive-raw socreq indat #!key (enable-send #t)(timeout 25)) + (let* ((success #f) + (result #f) + (keepwaiting #t) + (dat (db:obj->string indat transport: 'nmsg)) + (send-recv (make-thread + (lambda () + (nn-send socreq dat) + (let* ((res (nn-recv socreq))) + (set! success #t) + (set! result (db:string->obj res transport: 'nmsg)))) + "send-recv")) + (timeout (make-thread + (lambda () + (let loop ((count 0)) + (thread-sleep! 1) + (debug:print-info 1 *default-log-port* "send-receive-raw, still waiting after " count " seconds...") + (if (and keepwaiting (< count timeout)) ;; yes, this is very aproximate + (loop (+ count 1)))) + (if keepwaiting + (begin + (print "timeout waiting for ping") + (thread-terminate! send-recv)))) + "timeout"))) + ;; replace with condition-case? + (handle-exceptions + exn + (set! result "timeout") + (thread-start! timeout) + (thread-start! send-recv) + (thread-join! send-recv) + (if success (thread-terminate! timeout))) + ;; raise timeout error if timed out + (if success + (if (and (vector? result) + (vector-ref result 0)) ;; did it fail at the server? + result ;; nope, all good + (begin + (debug:print-error 0 *default-log-port* "error occured at server, info=" (vector-ref result 2)) + (debug:print 0 *default-log-port* " client call chain:") + (print-call-chain (current-error-port)) + (debug:print 0 *default-log-port* " server call chain:") + (pp (vector-ref result 1) (current-error-port)) + (signal (vector-ref result 0)))) + (signal (make-composite-condition + (make-property-condition 'timeout 'message "nmsg-transport:client-api-send-receive-raw timed out talking to server")))))) + +;; run nmsg-transport:keep-running in a parallel thread to monitor that the db is being +;; used and to shutdown after sometime if it is not. +;; +(define (nmsg-transport:keep-running server-id run-id) + ;; if none running or if > 20 seconds since + ;; server last used then start shutdown + ;; This thread waits for the server to come alive + (let* ((server-info (let loop () + (let ((sdat #f)) + (mutex-lock! *heartbeat-mutex*) + (set! sdat *server-info*) + (mutex-unlock! *heartbeat-mutex*) + (if sdat + (begin + (debug:print-info 0 *default-log-port* "keep-running got sdat=" sdat) + sdat) + (begin + (thread-sleep! 0.5) + (loop)))))) + (iface (car server-info)) + (port (cadr server-info)) + (last-access 0) + (tdbdat (tasks:open-db)) + (server-timeout (let ((tmo (configf:lookup *configdat* "server" "timeout"))) + (if (and (string? tmo) + (string->number tmo)) + (* 60 60 (string->number tmo)) + ;; (* 3 24 60 60) ;; default to three days + (* 60 1) ;; default to one minute + ;; (* 60 60 25) ;; default to 25 hours + )))) + (print "Keep-running got server pid " server-id ", using iface " iface " and port " port) + (let loop ((count 0)) + (thread-sleep! 4) ;; no need to do this very often + ;; NB// sync currently does NOT return queue-length + (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1))) + ;; (print "Server running, count is " count) + (if (< count 1) ;; 3x3 = 9 secs aprox + (loop (+ count 1))) + + (mutex-lock! *heartbeat-mutex*) + (set! last-access *last-db-access*) + (mutex-unlock! *heartbeat-mutex*) + (db:sync-touched *inmemdb* run-id force-sync: #t) + (if (and *server-run* + (> (+ last-access server-timeout) + (current-seconds))) + (begin + (debug:print-info 0 *default-log-port* "Server continuing, seconds since last db access: " (- (current-seconds) last-access)) + (loop 0)) + (begin + (debug:print-info 0 *default-log-port* "Starting to shutdown the server.") + (set! *time-to-exit* #t) + (db:sync-touched *inmemdb* run-id force-sync: #t) + (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running") + (debug:print-info 0 *default-log-port* "Server shutdown complete. Exiting") + (exit) + )))))) + +;;====================================================================== +;; C L I E N T S +;;====================================================================== + +(define (nmsg-transport:client-connect iface portnum) + (let* ((reqsoc (nmsg-transport:ping iface portnum return-socket: #t))) + (vector iface portnum #f #f #f (current-seconds) reqsoc))) + +;; returns result, there is no sucess/fail flag - handled via excpections +;; +(define (nmsg-transport:client-api-send-receive run-id connection-info cmd param #!key (remtries 5)) + ;; NB// In the html version of this routine there is a call to + ;; tasks:kill-server-run-id when there is an exception + (mutex-lock! *http-mutex*) + (let* ((packet (vector cmd param)) + (reqsoc (http-transport:server-dat-get-socket connection-info)) + (res (nmsg-transport:client-api-send-receive-raw reqsoc packet))) +;; (status (vector-ref rawres 0)) +;; (result (vector-ref rawres 1))) + (mutex-unlock! *http-mutex*) + res)) ;; (vector status (if status (db:string->obj result transport: 'nmsg) result)))) + +;;====================================================================== +;; J U N K +;;====================================================================== + +;; DO NOT USE +;; +(define (nmsg-transport:client-signal-handler signum) + (handle-exceptions + exn + (debug:print 0 *default-log-port* " ... exiting ...") + (let ((th1 (make-thread (lambda () + (if (not *received-response*) + (receive-message* *runremote*))) ;; flush out last call if applicable + "eat response")) + (th2 (make-thread (lambda () + (debug:print-error 0 *default-log-port* "Received ^C, attempting clean exit. Please be patient and wait a few seconds before hitting ^C again.") + (thread-sleep! 3) ;; give the flush three seconds to do it's stuff + (debug:print 0 *default-log-port* " Done.") + (exit 4)) + "exit on ^C timer"))) + (thread-start! th2) + (thread-start! th1) + (thread-join! th2)))) + Index: docs/api.html ================================================================== --- docs/api.html +++ docs/api.html @@ -825,10 +825,11 @@

1.2. Get List of Runs

URL: <base>/runs

Method: GET

Filter Params: target, testpatt, offset, limit

+

Megatest Cmd: megatest -start-dir <path to megatest area> -list-runs % -target % -dumpmode json -fields runs:runname,id+tests:state,status:id

Response:

[ { @@ -864,10 +865,11 @@

1.3. Trigger a new Run

URL: <base>/runs

Method: POST

+

Megatest Cmd: megatest -runtests % -target <target> :runname <run_name> -run

Request Params:

{"target": "target_value", "runname" : "runname", "test_pattern": "optional test pattern"}

@@ -901,10 +903,11 @@

1.4. Get perticular Run

URL: <base>/runs/:id

Method: GET

Filter Params: testpatt

+

Megatest Cmd: megatest -start-dir <path to megatest area> -list-runs <runname> -target % -dumpmode json -fields runs:runname,id+tests:state,status:id

Response:

[ { @@ -952,10 +955,11 @@

1.6. Get List of tests within a run

URL: <base>/runs/:id/tests

Method: GET

+

Megatest Cmd: megatest -start-dir <path to megatest area> -list-runs <runname> -target % -dumpmode json -fields runs:runname,id+tests:state,status:id

Response:

[ "tests" : @@ -979,10 +983,11 @@

1.8. Get perticular test that belongs to a Runs

URL: <base>/runs/:id/tests/:id

Method: GET

+

Megatest Cmd: megatest -start-dir <path to megatest area> -list-runs <runname> -target % -testpattern <pattern> -dumpmode json -fields runs:runname,id+tests:state,status:id

Response:

{"id": "4", "name":"test1", "item_path": "", "shortdir": "/temp/foo/bar/target2/runname2/test1", "final_logf": "megatest-rollup-test1.html", "status": "PASS"}

@@ -1010,10 +1015,10 @@
Index: docs/api.txt ================================================================== --- docs/api.txt +++ docs/api.txt @@ -40,10 +40,12 @@ Method: GET Filter Params: target, testpatt, offset, limit +Megatest Cmd: megatest -start-dir -list-runs % -target % -dumpmode json -fields runs:runname,id+tests:state,status:id + Response: ================== [ @@ -84,10 +86,12 @@ URL: /runs Method: POST +Megatest Cmd: megatest -runtests % -target :runname -run + Request Params: ================== {"[blue]#target#": "target_value", "[blue]#runname#" : "runname", "[blue]#test_pattern#": "optional test pattern"} ================== @@ -127,10 +131,13 @@ URL: /runs/:id Method: GET Filter Params: testpatt + +Megatest Cmd: megatest -start-dir -list-runs -target % -dumpmode json -fields runs:runname,id+tests:state,status:id + Response: ================== [ @@ -188,10 +195,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ URL: /runs/:id/tests Method: GET + +Megatest Cmd: megatest -start-dir -list-runs -target % -dumpmode json -fields runs:runname,id+tests:state,status:id + Response: ================== [ "[red]#tests#" : @@ -222,10 +232,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ URL: /runs/:id/tests/:id Method: GET + +Megatest Cmd: megatest -start-dir -list-runs -target % -testpattern -dumpmode json -fields runs:runname,id+tests:state,status:id + Response: ================== {"[blue]#id#": "4", "[blue]#name#":"test1", "[blue]#item_path#": "", "[blue]#shortdir#": "/temp/foo/bar/target2/runname2/test1", "[blue]#final_logf#": "megatest-rollup-test1.html", "[blue]#status#": "PASS"} Index: docs/dashboard.png ================================================================== --- docs/dashboard.png +++ docs/dashboard.png cannot compute difference between binary files ADDED docs/inprogress/graph-draw-arch.fig Index: docs/inprogress/graph-draw-arch.fig ================================================================== --- /dev/null +++ docs/inprogress/graph-draw-arch.fig @@ -0,0 +1,52 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +6 5700 3075 8400 3675 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 5700 3075 8400 3075 8400 3675 5700 3675 5700 3075 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9 + 5700 3525 5925 3525 5925 3225 6750 3225 6750 3450 7350 3450 + 7350 3600 8325 3600 8250 3525 +-6 +6 7425 6825 10125 7425 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7425 6825 10125 6825 10125 7425 7425 7425 7425 6825 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9 + 7425 7275 7650 7275 7650 6975 8475 6975 8475 7200 9075 7200 + 9075 7350 10050 7350 9975 7275 +-6 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 3000 4650 3000 3225 600 3225 600 4650 3000 4650 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 2550 5100 2550 3675 150 3675 150 5100 2550 5100 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 3000 3825 5550 3450 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 5475 2400 8475 2400 8475 4650 5475 4650 5475 2400 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 7275 4725 8175 6375 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 1 + 8175 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6225 6300 11025 6300 11025 9000 6225 9000 6225 6300 +2 4 2 1 0 7 50 -1 -1 3.000 0 0 7 0 0 5 + 8850 5850 8850 900 75 900 75 5850 8850 5850 +2 4 0 1 0 7 50 -1 -1 3.000 0 0 7 0 0 5 + 4875 5550 4875 4500 3450 4500 3450 5550 4875 5550 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 4500 4500 5475 4200 +4 0 0 50 -1 0 12 0.0000 4 195 915 750 3525 graph data\001 +4 0 0 50 -1 0 12 0.0000 4 195 525 5550 2700 layout\001 +4 0 0 50 -1 0 12 0.0000 4 195 1800 6375 6525 display on dashboard\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 3525 4875 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 6150 675 1425 Very slow! Threaded running of procedure: runtimes-tab-layout-updater\001 +4 0 0 50 -1 0 12 0.0000 4 195 2865 8325 6225 fast!runtimes-tab-canvas-updater\001 ADDED docs/inprogress/megatest-architecture-2.fig Index: docs/inprogress/megatest-architecture-2.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest-architecture-2.fig @@ -0,0 +1,28 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 750 975 5850 975 5850 7425 750 7425 750 975 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6000 975 9975 975 9975 7425 6000 7425 6000 975 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 900 1500 5250 1500 5250 2475 900 2475 900 1500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 900 2625 5250 2625 5250 3675 900 3675 900 2625 +2 3 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 8 + 5325 1500 5325 5850 900 5850 900 7275 5700 7275 5700 1425 + 5250 1425 5325 1500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 900 3750 5250 3750 5250 4725 900 4725 900 3750 +4 0 0 50 -1 0 12 0.0000 4 165 1170 975 1275 megatest.exe\001 +4 0 0 50 -1 0 12 0.0000 4 150 1275 6150 1275 dashboard.exe\001 +4 0 0 50 -1 0 12 0.0000 4 195 900 1050 1725 run engine\001 +4 0 0 50 -1 0 12 0.0000 4 150 780 975 2850 database\001 +4 0 0 50 -1 0 12 0.0000 4 195 2025 1050 6075 data transport - use http\001 +4 0 0 50 -1 0 12 0.0000 4 195 2325 975 3900 test or launch management\001 ADDED docs/inprogress/megatest-architecture-proposed-2.fig Index: docs/inprogress/megatest-architecture-proposed-2.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest-architecture-proposed-2.fig @@ -0,0 +1,490 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +6 600 1350 1575 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1125 1500 450 150 1125 1500 1575 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1124 2177 450 150 1124 2177 1574 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 675 1575 675 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1575 1500 1575 2175 +-6 +6 1875 825 2850 1875 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2400 975 450 150 2400 975 2850 1125 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2399 1652 450 150 2399 1652 2849 1802 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1950 1050 1950 1650 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 2850 975 2850 1650 +-6 +6 3225 450 4200 1500 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3750 600 450 150 3750 600 4200 750 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3749 1277 450 150 3749 1277 4199 1427 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3300 675 3300 1275 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4200 600 4200 1275 +-6 +6 3075 2925 4050 3975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 3075 450 150 3600 3075 4050 3225 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3599 3752 450 150 3599 3752 4049 3902 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3150 3150 3150 3750 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4050 3075 4050 3750 +-6 +6 7275 4050 12825 9675 +6 8175 4125 8400 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4125 8400 4125 8400 4350 8175 4350 8175 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4350 8400 4350 8400 4575 8175 4575 8175 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4575 8400 4575 8400 4800 8175 4800 8175 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4800 8400 4800 8400 5025 8175 5025 8175 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5025 8400 5025 8400 5250 8175 5250 8175 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5250 8400 5250 8400 5475 8175 5475 8175 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5475 8400 5475 8400 5700 8175 5700 8175 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5700 8400 5700 8400 5925 8175 5925 8175 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5925 8400 5925 8400 6150 8175 6150 8175 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6150 8400 6150 8400 6375 8175 6375 8175 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6375 8400 6375 8400 6600 8175 6600 8175 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6600 8400 6600 8400 6825 8175 6825 8175 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6825 8400 6825 8400 7050 8175 7050 8175 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7050 8400 7050 8400 7275 8175 7275 8175 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7275 8400 7275 8400 7500 8175 7500 8175 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7500 8400 7500 8400 7725 8175 7725 8175 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7725 8400 7725 8400 7950 8175 7950 8175 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7950 8400 7950 8400 8175 8175 8175 8175 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8175 8400 8175 8400 8400 8175 8400 8175 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8400 8400 8400 8400 8625 8175 8625 8175 8400 +-6 +6 8475 4125 8700 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4125 8700 4125 8700 4350 8475 4350 8475 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4350 8700 4350 8700 4575 8475 4575 8475 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4575 8700 4575 8700 4800 8475 4800 8475 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4800 8700 4800 8700 5025 8475 5025 8475 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5025 8700 5025 8700 5250 8475 5250 8475 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5250 8700 5250 8700 5475 8475 5475 8475 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5475 8700 5475 8700 5700 8475 5700 8475 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5700 8700 5700 8700 5925 8475 5925 8475 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5925 8700 5925 8700 6150 8475 6150 8475 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6150 8700 6150 8700 6375 8475 6375 8475 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6375 8700 6375 8700 6600 8475 6600 8475 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6600 8700 6600 8700 6825 8475 6825 8475 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6825 8700 6825 8700 7050 8475 7050 8475 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7050 8700 7050 8700 7275 8475 7275 8475 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7275 8700 7275 8700 7500 8475 7500 8475 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7500 8700 7500 8700 7725 8475 7725 8475 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7725 8700 7725 8700 7950 8475 7950 8475 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7950 8700 7950 8700 8175 8475 8175 8475 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8175 8700 8175 8700 8400 8475 8400 8475 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8400 8700 8400 8700 8625 8475 8625 8475 8400 +-6 +6 8775 4125 9000 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4125 9000 4125 9000 4350 8775 4350 8775 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4350 9000 4350 9000 4575 8775 4575 8775 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4575 9000 4575 9000 4800 8775 4800 8775 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4800 9000 4800 9000 5025 8775 5025 8775 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5025 9000 5025 9000 5250 8775 5250 8775 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5250 9000 5250 9000 5475 8775 5475 8775 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5475 9000 5475 9000 5700 8775 5700 8775 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5700 9000 5700 9000 5925 8775 5925 8775 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5925 9000 5925 9000 6150 8775 6150 8775 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6150 9000 6150 9000 6375 8775 6375 8775 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6375 9000 6375 9000 6600 8775 6600 8775 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6600 9000 6600 9000 6825 8775 6825 8775 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6825 9000 6825 9000 7050 8775 7050 8775 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7050 9000 7050 9000 7275 8775 7275 8775 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7275 9000 7275 9000 7500 8775 7500 8775 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7500 9000 7500 9000 7725 8775 7725 8775 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7725 9000 7725 9000 7950 8775 7950 8775 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7950 9000 7950 9000 8175 8775 8175 8775 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8175 9000 8175 9000 8400 8775 8400 8775 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8400 9000 8400 9000 8625 8775 8625 8775 8400 +-6 +6 9075 4125 9300 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4125 9300 4125 9300 4350 9075 4350 9075 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4350 9300 4350 9300 4575 9075 4575 9075 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4575 9300 4575 9300 4800 9075 4800 9075 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4800 9300 4800 9300 5025 9075 5025 9075 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5025 9300 5025 9300 5250 9075 5250 9075 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5250 9300 5250 9300 5475 9075 5475 9075 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5475 9300 5475 9300 5700 9075 5700 9075 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5700 9300 5700 9300 5925 9075 5925 9075 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5925 9300 5925 9300 6150 9075 6150 9075 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6150 9300 6150 9300 6375 9075 6375 9075 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6375 9300 6375 9300 6600 9075 6600 9075 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6600 9300 6600 9300 6825 9075 6825 9075 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6825 9300 6825 9300 7050 9075 7050 9075 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7050 9300 7050 9300 7275 9075 7275 9075 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7275 9300 7275 9300 7500 9075 7500 9075 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7500 9300 7500 9300 7725 9075 7725 9075 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7725 9300 7725 9300 7950 9075 7950 9075 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7950 9300 7950 9300 8175 9075 8175 9075 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8175 9300 8175 9300 8400 9075 8400 9075 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8400 9300 8400 9300 8625 9075 8625 9075 8400 +-6 +6 9375 4125 9600 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4125 9600 4125 9600 4350 9375 4350 9375 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4350 9600 4350 9600 4575 9375 4575 9375 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4575 9600 4575 9600 4800 9375 4800 9375 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4800 9600 4800 9600 5025 9375 5025 9375 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5025 9600 5025 9600 5250 9375 5250 9375 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5250 9600 5250 9600 5475 9375 5475 9375 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5475 9600 5475 9600 5700 9375 5700 9375 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5700 9600 5700 9600 5925 9375 5925 9375 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5925 9600 5925 9600 6150 9375 6150 9375 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6150 9600 6150 9600 6375 9375 6375 9375 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6375 9600 6375 9600 6600 9375 6600 9375 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6600 9600 6600 9600 6825 9375 6825 9375 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6825 9600 6825 9600 7050 9375 7050 9375 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7050 9600 7050 9600 7275 9375 7275 9375 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7275 9600 7275 9600 7500 9375 7500 9375 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7500 9600 7500 9600 7725 9375 7725 9375 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7725 9600 7725 9600 7950 9375 7950 9375 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7950 9600 7950 9600 8175 9375 8175 9375 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8175 9600 8175 9600 8400 9375 8400 9375 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8400 9600 8400 9600 8625 9375 8625 9375 8400 +-6 +# Dimension line: 1-1/16 in +6 7875 9375 9150 9675 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7875 9525 9150 9525 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 8085 9375 8085 9675 8939 9675 8939 9375 8085 9375 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7875 9375 7875 9675 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 9150 9375 9150 9675 +4 1 0 48 -1 0 12 -0.0000 4 180 735 8512 9585 1-1/16 in\001 +-6 +# Dimension line: 1-11/16 in +6 7425 4125 7725 6150 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7575 4125 7575 6150 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 7425 5617 7725 5617 7725 4657 7425 4657 7425 5617 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 6150 7725 6150 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 4125 7725 4125 +4 1 0 48 -1 0 12 1.5708 4 180 840 7635 5137 1-11/16 in\001 +-6 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7800 4050 12825 4050 12825 8925 7800 8925 7800 4050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6225 12450 6225 12450 8325 9225 8325 9225 6225 +2 2 0 1 4 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6150 9675 6150 9675 8400 9225 8400 9225 6150 +4 0 0 50 -1 0 12 0.0000 4 150 150 8475 9300 X\001 +4 0 0 50 -1 0 12 0.0000 4 150 135 7275 6975 Y\001 +-6 +6 14100 150 19950 6075 +6 14850 1350 15825 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 1500 450 150 15375 1500 15825 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 2177 450 150 15374 2177 15824 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 1575 14925 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 1500 15825 2175 +-6 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 4050 17025 3450 15750 3450 15750 4050 17025 4050 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 3375 15525 2400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 5325 17175 5325 17175 5850 15825 5850 15825 5325 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 4050 16350 5325 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 4800 18900 4800 18900 5325 17550 5325 17550 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 3900 19725 3900 19725 4425 18375 4425 18375 3900 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 4050 17850 4800 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 3750 18375 4125 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 3900 18075 2625 15900 1875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 150 19950 150 19950 6075 14100 6075 14100 150 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 3675 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 3825 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 4125 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 1200 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 1020 14325 525 basic model\001 +-6 +6 14850 7425 15825 8475 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 7575 450 150 15375 7575 15825 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 8252 450 150 15374 8252 15824 8402 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 7650 14925 8250 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 7575 15825 8250 +-6 +6 17775 6675 18750 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18300 6825 450 150 18300 6825 18750 6975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18299 7502 450 150 18299 7502 18749 7652 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 17850 6900 17850 7500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 18750 6825 18750 7500 +-6 +6 4875 6075 5850 7125 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5400 6225 450 150 5400 6225 5850 6375 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5399 6902 450 150 5399 6902 5849 7052 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4950 6300 4950 6900 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 5850 6225 5850 6900 +-6 +6 5400 7425 7350 8925 +6 5475 7650 6450 8700 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 6000 7800 450 150 6000 7800 6450 7950 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5999 8477 450 150 5999 8477 6449 8627 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 5550 7875 5550 8475 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 6450 7800 6450 8475 +-6 +4 0 0 50 -1 0 12 0.0000 4 195 1905 5400 8850 pointers to the servers\001 +4 0 0 50 -1 0 12 0.0000 4 150 930 5550 7575 monitor.db\001 +-6 +6 6150 2700 7500 3225 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6150 2700 7500 2700 7500 3225 6150 3225 6150 2700 +4 0 0 50 -1 0 12 0.0000 4 180 870 6225 2925 run2/test1\001 +-6 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1725 5025 1275 2475 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5550 4500 5550 225 225 225 225 4500 5550 4500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1875 7725 1875 5775 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 1425 7725 2775 7725 2775 8250 1425 8250 1425 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 3375 7725 4725 7725 4725 8250 3375 8250 3375 7725 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 3675 7725 2175 5775 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 2775 5700 2775 5100 1500 5100 1500 5700 2775 5700 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 6600 3300 2925 5025 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 10125 17025 9525 15750 9525 15750 10125 17025 10125 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 9450 15525 8475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 11400 17175 11400 17175 11925 15825 11925 15825 11400 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 10125 16350 11400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 10875 18900 10875 18900 11400 17550 11400 17550 10875 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 9975 19725 9975 19725 10500 18375 10500 18375 9975 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 10125 17850 10875 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 9825 18375 10200 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 9975 18075 8700 15900 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 6225 19950 6225 19950 12150 14100 12150 14100 6225 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16575 9375 17850 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2100 10425 6150 10425 6150 14400 2100 14400 2100 10425 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2325 10875 5925 10875 5925 13800 2325 13800 2325 10875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2400 10950 3975 10950 3975 11625 2400 11625 2400 10950 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5325 12675 5325 12075 4050 12075 4050 12675 5325 12675 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 3975 11250 4575 12075 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 2175 5025 3075 3750 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 4800 6375 2850 5550 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 3600 2475 7425 6525 +4 0 0 50 -1 0 12 0.0000 4 195 990 1800 2625 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 150 690 3150 4125 main.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 990 4200 3600 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 195 330 1950 6825 http\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 1575 7950 run1/test1\001 +4 0 0 50 -1 0 12 0.0000 4 150 720 1650 5400 server-1\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 2175 2025 2.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 750 2550 1.db\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 3450 7950 run1/test2\001 +4 0 0 50 -1 0 12 0.0000 4 150 1110 9675 3750 Dashboardm\001 +4 0 0 50 -1 0 12 1.5708 4 150 390 8325 3975 run1\001 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 9750 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 9900 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 10200 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 7275 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 1305 17700 7875 mysql/postgres\001 +4 0 0 50 -1 0 12 0.0000 4 195 1875 14325 6600 mysql/postgres model\001 +4 0 0 50 -1 0 12 0.0000 4 195 4065 600 9300 Current state, no bypass - (if we switch to rpc?)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1125 2175 10650 prev try RPC\001 +4 0 0 50 -1 0 12 0.0000 4 165 1095 2475 11100 rmt:get-tests\001 +4 0 0 50 -1 0 12 0.0000 4 195 2205 450 525 /tmp//??? /.db/*.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 4800 7350 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 1785 600 8775 Possible Future state\001 +4 0 0 50 -1 0 12 0.0000 4 150 1110 8025 450 CHANGES:\001 +4 0 0 50 -1 0 12 0.0000 4 195 2145 8025 705 1. http -> rcp or nanomsg\001 +4 0 0 50 -1 0 12 0.0000 4 195 3330 8025 960 2. cache db moves from inmem to /tmp\001 ADDED docs/inprogress/megatest-architecture-proposed.fig Index: docs/inprogress/megatest-architecture-proposed.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest-architecture-proposed.fig @@ -0,0 +1,488 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +6 600 1350 1575 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1125 1500 450 150 1125 1500 1575 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1124 2177 450 150 1124 2177 1574 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 675 1575 675 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1575 1500 1575 2175 +-6 +6 1875 825 2850 1875 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2400 975 450 150 2400 975 2850 1125 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2399 1652 450 150 2399 1652 2849 1802 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1950 1050 1950 1650 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 2850 975 2850 1650 +-6 +6 3225 450 4200 1500 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3750 600 450 150 3750 600 4200 750 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3749 1277 450 150 3749 1277 4199 1427 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3300 675 3300 1275 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4200 600 4200 1275 +-6 +6 3075 2925 4050 3975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 3075 450 150 3600 3075 4050 3225 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3599 3752 450 150 3599 3752 4049 3902 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3150 3150 3150 3750 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4050 3075 4050 3750 +-6 +6 7275 4050 12825 9675 +6 8175 4125 8400 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4125 8400 4125 8400 4350 8175 4350 8175 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4350 8400 4350 8400 4575 8175 4575 8175 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4575 8400 4575 8400 4800 8175 4800 8175 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4800 8400 4800 8400 5025 8175 5025 8175 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5025 8400 5025 8400 5250 8175 5250 8175 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5250 8400 5250 8400 5475 8175 5475 8175 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5475 8400 5475 8400 5700 8175 5700 8175 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5700 8400 5700 8400 5925 8175 5925 8175 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5925 8400 5925 8400 6150 8175 6150 8175 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6150 8400 6150 8400 6375 8175 6375 8175 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6375 8400 6375 8400 6600 8175 6600 8175 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6600 8400 6600 8400 6825 8175 6825 8175 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6825 8400 6825 8400 7050 8175 7050 8175 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7050 8400 7050 8400 7275 8175 7275 8175 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7275 8400 7275 8400 7500 8175 7500 8175 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7500 8400 7500 8400 7725 8175 7725 8175 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7725 8400 7725 8400 7950 8175 7950 8175 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7950 8400 7950 8400 8175 8175 8175 8175 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8175 8400 8175 8400 8400 8175 8400 8175 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8400 8400 8400 8400 8625 8175 8625 8175 8400 +-6 +6 8475 4125 8700 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4125 8700 4125 8700 4350 8475 4350 8475 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4350 8700 4350 8700 4575 8475 4575 8475 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4575 8700 4575 8700 4800 8475 4800 8475 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4800 8700 4800 8700 5025 8475 5025 8475 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5025 8700 5025 8700 5250 8475 5250 8475 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5250 8700 5250 8700 5475 8475 5475 8475 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5475 8700 5475 8700 5700 8475 5700 8475 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5700 8700 5700 8700 5925 8475 5925 8475 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5925 8700 5925 8700 6150 8475 6150 8475 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6150 8700 6150 8700 6375 8475 6375 8475 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6375 8700 6375 8700 6600 8475 6600 8475 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6600 8700 6600 8700 6825 8475 6825 8475 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6825 8700 6825 8700 7050 8475 7050 8475 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7050 8700 7050 8700 7275 8475 7275 8475 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7275 8700 7275 8700 7500 8475 7500 8475 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7500 8700 7500 8700 7725 8475 7725 8475 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7725 8700 7725 8700 7950 8475 7950 8475 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7950 8700 7950 8700 8175 8475 8175 8475 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8175 8700 8175 8700 8400 8475 8400 8475 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8400 8700 8400 8700 8625 8475 8625 8475 8400 +-6 +6 8775 4125 9000 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4125 9000 4125 9000 4350 8775 4350 8775 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4350 9000 4350 9000 4575 8775 4575 8775 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4575 9000 4575 9000 4800 8775 4800 8775 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4800 9000 4800 9000 5025 8775 5025 8775 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5025 9000 5025 9000 5250 8775 5250 8775 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5250 9000 5250 9000 5475 8775 5475 8775 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5475 9000 5475 9000 5700 8775 5700 8775 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5700 9000 5700 9000 5925 8775 5925 8775 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5925 9000 5925 9000 6150 8775 6150 8775 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6150 9000 6150 9000 6375 8775 6375 8775 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6375 9000 6375 9000 6600 8775 6600 8775 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6600 9000 6600 9000 6825 8775 6825 8775 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6825 9000 6825 9000 7050 8775 7050 8775 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7050 9000 7050 9000 7275 8775 7275 8775 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7275 9000 7275 9000 7500 8775 7500 8775 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7500 9000 7500 9000 7725 8775 7725 8775 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7725 9000 7725 9000 7950 8775 7950 8775 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7950 9000 7950 9000 8175 8775 8175 8775 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8175 9000 8175 9000 8400 8775 8400 8775 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8400 9000 8400 9000 8625 8775 8625 8775 8400 +-6 +6 9075 4125 9300 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4125 9300 4125 9300 4350 9075 4350 9075 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4350 9300 4350 9300 4575 9075 4575 9075 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4575 9300 4575 9300 4800 9075 4800 9075 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4800 9300 4800 9300 5025 9075 5025 9075 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5025 9300 5025 9300 5250 9075 5250 9075 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5250 9300 5250 9300 5475 9075 5475 9075 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5475 9300 5475 9300 5700 9075 5700 9075 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5700 9300 5700 9300 5925 9075 5925 9075 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5925 9300 5925 9300 6150 9075 6150 9075 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6150 9300 6150 9300 6375 9075 6375 9075 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6375 9300 6375 9300 6600 9075 6600 9075 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6600 9300 6600 9300 6825 9075 6825 9075 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6825 9300 6825 9300 7050 9075 7050 9075 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7050 9300 7050 9300 7275 9075 7275 9075 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7275 9300 7275 9300 7500 9075 7500 9075 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7500 9300 7500 9300 7725 9075 7725 9075 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7725 9300 7725 9300 7950 9075 7950 9075 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7950 9300 7950 9300 8175 9075 8175 9075 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8175 9300 8175 9300 8400 9075 8400 9075 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8400 9300 8400 9300 8625 9075 8625 9075 8400 +-6 +6 9375 4125 9600 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4125 9600 4125 9600 4350 9375 4350 9375 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4350 9600 4350 9600 4575 9375 4575 9375 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4575 9600 4575 9600 4800 9375 4800 9375 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4800 9600 4800 9600 5025 9375 5025 9375 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5025 9600 5025 9600 5250 9375 5250 9375 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5250 9600 5250 9600 5475 9375 5475 9375 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5475 9600 5475 9600 5700 9375 5700 9375 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5700 9600 5700 9600 5925 9375 5925 9375 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5925 9600 5925 9600 6150 9375 6150 9375 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6150 9600 6150 9600 6375 9375 6375 9375 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6375 9600 6375 9600 6600 9375 6600 9375 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6600 9600 6600 9600 6825 9375 6825 9375 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6825 9600 6825 9600 7050 9375 7050 9375 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7050 9600 7050 9600 7275 9375 7275 9375 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7275 9600 7275 9600 7500 9375 7500 9375 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7500 9600 7500 9600 7725 9375 7725 9375 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7725 9600 7725 9600 7950 9375 7950 9375 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7950 9600 7950 9600 8175 9375 8175 9375 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8175 9600 8175 9600 8400 9375 8400 9375 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8400 9600 8400 9600 8625 9375 8625 9375 8400 +-6 +# Dimension line: 1-1/16 in +6 7875 9375 9150 9675 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7875 9525 9150 9525 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 8085 9375 8085 9675 8939 9675 8939 9375 8085 9375 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7875 9375 7875 9675 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 9150 9375 9150 9675 +4 1 0 48 -1 0 12 -0.0000 4 180 735 8512 9585 1-1/16 in\001 +-6 +# Dimension line: 1-11/16 in +6 7425 4125 7725 6150 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7575 4125 7575 6150 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 7425 5617 7725 5617 7725 4657 7425 4657 7425 5617 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 6150 7725 6150 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 4125 7725 4125 +4 1 0 48 -1 0 12 1.5708 4 180 840 7635 5137 1-11/16 in\001 +-6 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7800 4050 12825 4050 12825 8925 7800 8925 7800 4050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6225 12450 6225 12450 8325 9225 8325 9225 6225 +2 2 0 1 4 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6150 9675 6150 9675 8400 9225 8400 9225 6150 +4 0 0 50 -1 0 12 0.0000 4 150 150 8475 9300 X\001 +4 0 0 50 -1 0 12 0.0000 4 150 135 7275 6975 Y\001 +-6 +6 14100 150 19950 6075 +6 14850 1350 15825 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 1500 450 150 15375 1500 15825 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 2177 450 150 15374 2177 15824 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 1575 14925 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 1500 15825 2175 +-6 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 4050 17025 3450 15750 3450 15750 4050 17025 4050 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 3375 15525 2400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 5325 17175 5325 17175 5850 15825 5850 15825 5325 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 4050 16350 5325 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 4800 18900 4800 18900 5325 17550 5325 17550 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 3900 19725 3900 19725 4425 18375 4425 18375 3900 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 4050 17850 4800 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 3750 18375 4125 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 3900 18075 2625 15900 1875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 150 19950 150 19950 6075 14100 6075 14100 150 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 3675 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 3825 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 4125 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 1200 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 1020 14325 525 basic model\001 +-6 +6 14850 7425 15825 8475 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 7575 450 150 15375 7575 15825 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 8252 450 150 15374 8252 15824 8402 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 7650 14925 8250 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 7575 15825 8250 +-6 +6 17775 6675 18750 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18300 6825 450 150 18300 6825 18750 6975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18299 7502 450 150 18299 7502 18749 7652 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 17850 6900 17850 7500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 18750 6825 18750 7500 +-6 +6 4875 6075 5850 7125 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5400 6225 450 150 5400 6225 5850 6375 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5399 6902 450 150 5399 6902 5849 7052 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4950 6300 4950 6900 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 5850 6225 5850 6900 +-6 +6 5400 7425 7350 8925 +6 5475 7650 6450 8700 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 6000 7800 450 150 6000 7800 6450 7950 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 5999 8477 450 150 5999 8477 6449 8627 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 5550 7875 5550 8475 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 6450 7800 6450 8475 +-6 +4 0 0 50 -1 0 12 0.0000 4 195 1905 5400 8850 pointers to the servers\001 +4 0 0 50 -1 0 12 0.0000 4 150 930 5550 7575 monitor.db\001 +-6 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1725 5025 1275 2475 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5550 4500 5550 225 225 225 225 4500 5550 4500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1875 7725 1875 5775 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 2775 5400 7125 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 1425 7725 2775 7725 2775 8250 1425 8250 1425 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 3375 7725 4725 7725 4725 8250 3375 8250 3375 7725 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 3675 7725 2175 5775 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 2775 5700 2775 5100 1500 5100 1500 5700 2775 5700 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 6600 3300 2925 5025 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 10125 17025 9525 15750 9525 15750 10125 17025 10125 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 9450 15525 8475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 11400 17175 11400 17175 11925 15825 11925 15825 11400 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 10125 16350 11400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 10875 18900 10875 18900 11400 17550 11400 17550 10875 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 9975 19725 9975 19725 10500 18375 10500 18375 9975 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 10125 17850 10875 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 9825 18375 10200 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 9975 18075 8700 15900 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 6225 19950 6225 19950 12150 14100 12150 14100 6225 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16575 9375 17850 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2100 10425 6150 10425 6150 14400 2100 14400 2100 10425 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2325 10875 5925 10875 5925 13800 2325 13800 2325 10875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2400 10950 3975 10950 3975 11625 2400 11625 2400 10950 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5325 12675 5325 12075 4050 12075 4050 12675 5325 12675 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 3975 11250 4575 12075 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 2175 5025 3075 3750 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6150 2700 7500 2700 7500 3225 6150 3225 6150 2700 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 4800 6375 2850 5550 +4 0 0 50 -1 0 12 0.0000 4 195 990 1800 2625 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 150 690 3150 4125 main.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 990 4200 3600 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 195 330 1950 6825 http\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 1575 7950 run1/test1\001 +4 0 0 50 -1 0 12 0.0000 4 150 720 1650 5400 server-1\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 2175 2025 2.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 750 2550 1.db\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 3450 7950 run1/test2\001 +4 0 0 50 -1 0 12 0.0000 4 150 1110 9675 3750 Dashboardm\001 +4 0 0 50 -1 0 12 1.5708 4 150 390 8325 3975 run1\001 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 9750 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 9900 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 10200 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 7275 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 1305 17700 7875 mysql/postgres\001 +4 0 0 50 -1 0 12 0.0000 4 195 1875 14325 6600 mysql/postgres model\001 +4 0 0 50 -1 0 12 0.0000 4 195 4065 600 9300 Current state, no bypass - (if we switch to rpc?)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1125 2175 10650 prev try RPC\001 +4 0 0 50 -1 0 12 0.0000 4 165 1095 2475 11100 rmt:get-tests\001 +4 0 0 50 -1 0 12 0.0000 4 195 2205 450 525 /tmp//??? /.db/*.db\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 6225 2925 run2/test1\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 4800 7350 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 1785 600 8775 Possible Future state\001 +4 0 0 50 -1 0 12 0.0000 4 150 1110 8025 450 CHANGES:\001 +4 0 0 50 -1 0 12 0.0000 4 195 2145 8025 705 1. http -> rcp or nanomsg\001 +4 0 0 50 -1 0 12 0.0000 4 195 3330 8025 960 2. cache db moves from inmem to /tmp\001 ADDED docs/inprogress/megatest-architecture.fig Index: docs/inprogress/megatest-architecture.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest-architecture.fig @@ -0,0 +1,528 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +6 600 1350 1575 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1125 1500 450 150 1125 1500 1575 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1124 2177 450 150 1124 2177 1574 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 675 1575 675 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1575 1500 1575 2175 +-6 +6 1875 825 2850 1875 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2400 975 450 150 2400 975 2850 1125 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 2399 1652 450 150 2399 1652 2849 1802 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1950 1050 1950 1650 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 2850 975 2850 1650 +-6 +6 3225 450 4200 1500 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3750 600 450 150 3750 600 4200 750 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3749 1277 450 150 3749 1277 4199 1427 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3300 675 3300 1275 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4200 600 4200 1275 +-6 +6 3075 2925 4050 3975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 3075 450 150 3600 3075 4050 3225 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3599 3752 450 150 3599 3752 4049 3902 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3150 3150 3150 3750 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 4050 3075 4050 3750 +-6 +6 7275 4050 12825 9675 +6 8175 4125 8400 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4125 8400 4125 8400 4350 8175 4350 8175 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4350 8400 4350 8400 4575 8175 4575 8175 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4575 8400 4575 8400 4800 8175 4800 8175 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 4800 8400 4800 8400 5025 8175 5025 8175 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5025 8400 5025 8400 5250 8175 5250 8175 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5250 8400 5250 8400 5475 8175 5475 8175 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5475 8400 5475 8400 5700 8175 5700 8175 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5700 8400 5700 8400 5925 8175 5925 8175 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 5925 8400 5925 8400 6150 8175 6150 8175 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6150 8400 6150 8400 6375 8175 6375 8175 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6375 8400 6375 8400 6600 8175 6600 8175 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6600 8400 6600 8400 6825 8175 6825 8175 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 6825 8400 6825 8400 7050 8175 7050 8175 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7050 8400 7050 8400 7275 8175 7275 8175 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7275 8400 7275 8400 7500 8175 7500 8175 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7500 8400 7500 8400 7725 8175 7725 8175 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7725 8400 7725 8400 7950 8175 7950 8175 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 7950 8400 7950 8400 8175 8175 8175 8175 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8175 8400 8175 8400 8400 8175 8400 8175 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8175 8400 8400 8400 8400 8625 8175 8625 8175 8400 +-6 +6 8475 4125 8700 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4125 8700 4125 8700 4350 8475 4350 8475 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4350 8700 4350 8700 4575 8475 4575 8475 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4575 8700 4575 8700 4800 8475 4800 8475 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 4800 8700 4800 8700 5025 8475 5025 8475 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5025 8700 5025 8700 5250 8475 5250 8475 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5250 8700 5250 8700 5475 8475 5475 8475 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5475 8700 5475 8700 5700 8475 5700 8475 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5700 8700 5700 8700 5925 8475 5925 8475 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 5925 8700 5925 8700 6150 8475 6150 8475 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6150 8700 6150 8700 6375 8475 6375 8475 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6375 8700 6375 8700 6600 8475 6600 8475 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6600 8700 6600 8700 6825 8475 6825 8475 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 6825 8700 6825 8700 7050 8475 7050 8475 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7050 8700 7050 8700 7275 8475 7275 8475 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7275 8700 7275 8700 7500 8475 7500 8475 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7500 8700 7500 8700 7725 8475 7725 8475 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7725 8700 7725 8700 7950 8475 7950 8475 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 7950 8700 7950 8700 8175 8475 8175 8475 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8175 8700 8175 8700 8400 8475 8400 8475 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8475 8400 8700 8400 8700 8625 8475 8625 8475 8400 +-6 +6 8775 4125 9000 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4125 9000 4125 9000 4350 8775 4350 8775 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4350 9000 4350 9000 4575 8775 4575 8775 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4575 9000 4575 9000 4800 8775 4800 8775 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 4800 9000 4800 9000 5025 8775 5025 8775 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5025 9000 5025 9000 5250 8775 5250 8775 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5250 9000 5250 9000 5475 8775 5475 8775 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5475 9000 5475 9000 5700 8775 5700 8775 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5700 9000 5700 9000 5925 8775 5925 8775 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 5925 9000 5925 9000 6150 8775 6150 8775 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6150 9000 6150 9000 6375 8775 6375 8775 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6375 9000 6375 9000 6600 8775 6600 8775 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6600 9000 6600 9000 6825 8775 6825 8775 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 6825 9000 6825 9000 7050 8775 7050 8775 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7050 9000 7050 9000 7275 8775 7275 8775 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7275 9000 7275 9000 7500 8775 7500 8775 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7500 9000 7500 9000 7725 8775 7725 8775 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7725 9000 7725 9000 7950 8775 7950 8775 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 7950 9000 7950 9000 8175 8775 8175 8775 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8175 9000 8175 9000 8400 8775 8400 8775 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8775 8400 9000 8400 9000 8625 8775 8625 8775 8400 +-6 +6 9075 4125 9300 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4125 9300 4125 9300 4350 9075 4350 9075 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4350 9300 4350 9300 4575 9075 4575 9075 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4575 9300 4575 9300 4800 9075 4800 9075 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 4800 9300 4800 9300 5025 9075 5025 9075 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5025 9300 5025 9300 5250 9075 5250 9075 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5250 9300 5250 9300 5475 9075 5475 9075 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5475 9300 5475 9300 5700 9075 5700 9075 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5700 9300 5700 9300 5925 9075 5925 9075 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 5925 9300 5925 9300 6150 9075 6150 9075 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6150 9300 6150 9300 6375 9075 6375 9075 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6375 9300 6375 9300 6600 9075 6600 9075 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6600 9300 6600 9300 6825 9075 6825 9075 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 6825 9300 6825 9300 7050 9075 7050 9075 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7050 9300 7050 9300 7275 9075 7275 9075 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7275 9300 7275 9300 7500 9075 7500 9075 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7500 9300 7500 9300 7725 9075 7725 9075 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7725 9300 7725 9300 7950 9075 7950 9075 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 7950 9300 7950 9300 8175 9075 8175 9075 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8175 9300 8175 9300 8400 9075 8400 9075 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9075 8400 9300 8400 9300 8625 9075 8625 9075 8400 +-6 +6 9375 4125 9600 8625 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4125 9600 4125 9600 4350 9375 4350 9375 4125 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4350 9600 4350 9600 4575 9375 4575 9375 4350 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4575 9600 4575 9600 4800 9375 4800 9375 4575 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 4800 9600 4800 9600 5025 9375 5025 9375 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5025 9600 5025 9600 5250 9375 5250 9375 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5250 9600 5250 9600 5475 9375 5475 9375 5250 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5475 9600 5475 9600 5700 9375 5700 9375 5475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5700 9600 5700 9600 5925 9375 5925 9375 5700 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 5925 9600 5925 9600 6150 9375 6150 9375 5925 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6150 9600 6150 9600 6375 9375 6375 9375 6150 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6375 9600 6375 9600 6600 9375 6600 9375 6375 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6600 9600 6600 9600 6825 9375 6825 9375 6600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 6825 9600 6825 9600 7050 9375 7050 9375 6825 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7050 9600 7050 9600 7275 9375 7275 9375 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7275 9600 7275 9600 7500 9375 7500 9375 7275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7500 9600 7500 9600 7725 9375 7725 9375 7500 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7725 9600 7725 9600 7950 9375 7950 9375 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 7950 9600 7950 9600 8175 9375 8175 9375 7950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8175 9600 8175 9600 8400 9375 8400 9375 8175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9375 8400 9600 8400 9600 8625 9375 8625 9375 8400 +-6 +# Dimension line: 1-1/16 in +6 7875 9375 9150 9675 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7875 9525 9150 9525 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 8085 9375 8085 9675 8939 9675 8939 9375 8085 9375 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7875 9375 7875 9675 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 9150 9375 9150 9675 +4 1 0 48 -1 0 12 -0.0000 4 180 735 8512 9585 1-1/16 in\001 +-6 +# Dimension line: 1-11/16 in +6 7425 4125 7725 6150 +# main dimension line +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 1 1 2 + 1 1 1.00 60.00 120.00 + 1 1 1.00 60.00 120.00 + 7575 4125 7575 6150 +# text box +2 3 0 1 4 7 49 -1 20 0.000 0 0 -1 0 0 5 + 7425 5617 7725 5617 7725 4657 7425 4657 7425 5617 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 6150 7725 6150 +# tick +2 1 0 1 0 7 50 -1 -1 4.000 0 0 -1 0 0 2 + 7425 4125 7725 4125 +4 1 0 48 -1 0 12 1.5708 4 180 840 7635 5137 1-11/16 in\001 +-6 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7800 4050 12825 4050 12825 8925 7800 8925 7800 4050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6225 12450 6225 12450 8325 9225 8325 9225 6225 +2 2 0 1 4 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9225 6150 9675 6150 9675 8400 9225 8400 9225 6150 +4 0 0 50 -1 0 12 0.0000 4 150 150 8475 9300 X\001 +4 0 0 50 -1 0 12 0.0000 4 150 135 7275 6975 Y\001 +-6 +6 975 5100 1500 5700 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1258 5186 242 86 1258 5186 1500 5271 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 1257 5573 242 86 1257 5573 1499 5658 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1015 5229 1015 5571 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 1500 5186 1500 5571 +-6 +6 3000 6075 3525 6675 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3283 6161 242 86 3283 6161 3525 6246 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3282 6548 242 86 3282 6548 3524 6633 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3040 6204 3040 6546 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 3525 6161 3525 6546 +-6 +6 7575 2625 8100 3225 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 7858 2711 242 86 7858 2711 8100 2796 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 7857 3098 242 86 7857 3098 8099 3183 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 7615 2754 7615 3096 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 8100 2711 8100 3096 +-6 +6 9525 450 10500 1500 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 10050 600 450 150 10050 600 10500 750 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 10049 1277 450 150 10049 1277 10499 1427 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 9600 675 9600 1275 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 10500 600 10500 1275 +-6 +6 14100 150 19950 6075 +6 14850 1350 15825 2400 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 1500 450 150 15375 1500 15825 1650 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 2177 450 150 15374 2177 15824 2327 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 1575 14925 2175 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 1500 15825 2175 +-6 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 4050 17025 3450 15750 3450 15750 4050 17025 4050 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 3375 15525 2400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 5325 17175 5325 17175 5850 15825 5850 15825 5325 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 4050 16350 5325 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 4800 18900 4800 18900 5325 17550 5325 17550 4800 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 3900 19725 3900 19725 4425 18375 4425 18375 3900 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 4050 17850 4800 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 3750 18375 4125 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 3900 18075 2625 15900 1875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 150 19950 150 19950 6075 14100 6075 14100 150 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 3675 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 3825 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 4125 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 1200 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 1020 14325 525 basic model\001 +-6 +6 14850 7425 15825 8475 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15375 7575 450 150 15375 7575 15825 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 15374 8252 450 150 15374 8252 15824 8402 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 14925 7650 14925 8250 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 15825 7575 15825 8250 +-6 +6 17775 6675 18750 7725 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18300 6825 450 150 18300 6825 18750 6975 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 18299 7502 450 150 18299 7502 18749 7652 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 17850 6900 17850 7500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 18750 6825 18750 7500 +-6 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1725 5025 1275 2475 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5550 4500 5550 225 225 225 225 4500 5550 4500 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1875 7725 1875 5775 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 2775 5400 7125 5700 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 2400 7725 3900 6675 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 4275 6075 3825 4200 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 7125 5850 4800 6300 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 1425 7725 2775 7725 2775 8250 1425 8250 1425 7725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 3375 7725 4725 7725 4725 8250 3375 8250 3375 7725 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 3675 7725 2175 5775 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 3975 7725 3975 6750 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7125 825 8475 825 8475 1350 7125 1350 7125 825 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 2775 5700 2775 5100 1500 5100 1500 5700 2775 5700 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 4800 6675 4800 6075 3525 6075 3525 6675 4800 6675 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 7575 3225 7575 2625 6300 2625 6300 3225 7575 3225 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 7650 1425 6975 2625 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 6300 2925 2850 1725 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 6975 3300 7725 3900 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 4 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 7200 1350 5925 2550 5925 4875 4650 6000 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 4 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 1425 8025 825 6750 825 2850 900 2700 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 17025 10125 17025 9525 15750 9525 15750 10125 17025 10125 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16050 9450 15525 8475 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 15825 11400 17175 11400 17175 11925 15825 11925 15825 11400 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16350 10125 16350 11400 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 17550 10875 18900 10875 18900 11400 17550 11400 17550 10875 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 18375 9975 19725 9975 19725 10500 18375 10500 18375 9975 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16725 10125 17850 10875 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 17025 9825 18375 10200 +2 1 0 1 14 7 50 -1 -1 3.000 0 0 -1 1 1 3 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 18975 9975 18075 8700 15900 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 14100 6225 19950 6225 19950 12150 14100 12150 14100 6225 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 1 2 + 0 0 1.00 60.00 120.00 + 0 0 1.00 60.00 120.00 + 16575 9375 17850 7950 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2100 10425 6150 10425 6150 14400 2100 14400 2100 10425 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2325 10875 5925 10875 5925 13800 2325 13800 2325 10875 +2 2 0 1 0 7 50 -1 -1 3.000 0 0 -1 0 0 5 + 2400 10950 3975 10950 3975 11625 2400 11625 2400 10950 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 5325 12675 5325 12075 4050 12075 4050 12675 5325 12675 +2 1 0 1 0 7 50 -1 -1 3.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 3975 11250 4575 12075 +4 0 0 50 -1 0 12 0.0000 4 195 990 1800 2625 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 150 690 3150 4125 main.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 990 4200 3600 last_update\001 +4 0 0 50 -1 0 12 0.0000 4 180 1500 450 525 link tree /.db/*.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 330 1950 6825 http\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 1575 7950 run1/test1\001 +4 0 0 50 -1 0 12 0.0000 4 150 720 1650 5400 server-1\001 +4 0 0 50 -1 0 12 0.0000 4 150 1035 3675 6375 server-main\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 2175 2025 2.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 375 750 2550 1.db\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 3450 7950 run1/test2\001 +4 0 0 50 -1 0 12 0.0000 4 150 1110 9675 3750 Dashboardm\001 +4 0 0 50 -1 0 12 0.0000 4 180 870 7275 1050 run2/test1\001 +4 0 0 50 -1 0 12 0.0000 4 150 720 6375 2850 server-2\001 +4 0 0 50 -1 0 12 1.5708 4 150 390 8325 3975 run1\001 +4 0 0 50 -1 0 12 0.0000 4 195 1905 9450 1650 pointers to the servers\001 +4 0 0 50 -1 0 12 0.0000 4 150 930 9600 375 monitor.db\001 +4 0 0 50 -1 0 12 0.0000 4 150 900 15825 9750 rpc-server\001 +4 0 0 50 -1 0 12 0.0000 4 165 270 17475 9900 tcp\001 +4 0 0 50 -1 0 12 0.0000 4 120 315 18525 10200 test\001 +4 0 0 50 -1 0 12 0.0000 4 195 1065 14850 7275 megatest.db\001 +4 0 0 50 -1 0 12 0.0000 4 195 1305 17700 7875 mysql/postgres\001 +4 0 0 50 -1 0 12 0.0000 4 195 1875 14325 6600 mysql/postgres model\001 +4 0 0 50 -1 0 12 0.0000 4 150 1140 600 8775 Current state\001 +4 0 0 50 -1 0 12 0.0000 4 195 4065 600 9300 Current state, no bypass - (if we switch to rpc?)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1125 2175 10650 prev try RPC\001 +4 0 0 50 -1 0 12 0.0000 4 165 1095 2475 11100 rmt:get-tests\001 ADDED docs/inprogress/megatest-query-view.fig Index: docs/inprogress/megatest-query-view.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest-query-view.fig @@ -0,0 +1,59 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 900 675 4350 675 4350 1650 900 1650 900 675 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 4350 1200 6975 1725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6975 1350 10725 1350 10725 3075 6975 3075 6975 1350 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 7125 1800 8925 2025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 8850 1875 10125 1875 10125 2550 8850 2550 8850 1875 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 9825 2550 10125 6000 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 11400 7350 11400 6000 9000 6000 9000 7350 11400 7350 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 9750 6000 9375 2550 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 8775 2250 7125 2025 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 6975 1800 4350 1275 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 975 3600 4350 3600 4350 4575 975 4575 975 3600 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 5 + 0 0 1.00 60.00 120.00 + 1050 1650 1050 2025 4575 2025 4575 1050 4350 1050 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 4350 3825 6975 2700 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 0 0 1.00 60.00 120.00 + 9000 2625 4350 4200 +4 0 0 50 -1 0 12 0.0000 4 150 390 1050 975 Test\001 +4 0 0 50 -1 0 12 0.0000 4 150 585 7125 1650 Server\001 +4 0 0 50 -1 0 12 0.0000 4 195 1020 4800 1125 http request\001 +4 0 0 50 -1 0 12 0.0000 4 195 750 9075 2100 db query\001 +4 0 0 50 -1 0 12 0.0000 4 150 345 9525 6375 disk\001 +4 0 0 50 -1 0 12 0.0000 4 195 1905 1725 1200 (rmt:tests-get-info ....)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1905 1800 4125 (rmt:tests-get-info ....)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1110 5100 3300 send-request\001 +4 0 0 50 -1 0 12 0.0000 4 150 2145 5475 3900 call-back with result data\001 +4 0 0 50 -1 0 12 0.0000 4 150 390 1125 3750 Test\001 +4 0 0 50 -1 0 12 0.0000 4 195 675 7350 2550 api.scm\001 +4 0 0 50 -1 0 12 0.0000 4 195 990 7350 2805 recieve-req\001 +4 0 0 50 -1 0 12 0.0000 4 150 1515 7350 3060 send-res-callback\001 ADDED docs/inprogress/megatest_qa.fig Index: docs/inprogress/megatest_qa.fig ================================================================== --- /dev/null +++ docs/inprogress/megatest_qa.fig @@ -0,0 +1,38 @@ +#FIG 3.2 Produced by xfig version 3.2.5-alpha5 +Landscape +Center +Inches +Letter +100.00 +Single +-2 +1200 2 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 6000 300 6000 9675 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 525 675 4500 675 4500 2550 525 2550 525 675 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 900 1125 2325 1125 2325 1575 900 1575 900 1125 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2 + 225 150 225 7050 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 525 3150 3750 3150 3750 4275 525 4275 525 3150 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 12750 5025 12750 750 6450 750 6450 5025 12750 5025 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9300 1725 10800 1725 10800 2100 9300 2100 9300 1725 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 9300 2175 10800 2175 10800 2550 9300 2550 9300 2175 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 6750 1275 12600 1275 12600 2925 6750 2925 6750 1275 +2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5 + 12450 2700 12450 1650 6975 1650 6975 2700 12450 2700 +4 0 0 50 -1 0 12 0.0000 4 120 405 675 900 tests\001 +4 0 0 50 -1 0 12 0.0000 4 195 900 975 1425 nested_mt\001 +4 0 0 50 -1 0 12 0.0000 4 195 2790 675 3375 nested_mt (a full megatest suite)\001 +4 0 0 50 -1 0 12 0.0000 4 165 1110 375 5100 megatest_qa\001 +4 0 0 50 -1 0 12 0.0000 4 150 420 525 300 code\001 +4 0 0 50 -1 0 12 0.0000 4 150 1005 6750 375 Actual runs\001 +4 0 0 50 -1 0 12 0.0000 4 165 1710 6675 1050 outer megatest runs\001 +4 0 0 50 -1 0 12 0.0000 4 195 1785 6900 1500 test (e.g. nested_mt)\001 +4 0 0 50 -1 0 12 0.0000 4 195 1665 7125 1875 nested_mt testsuite\001 Index: docs/manual/Makefile ================================================================== --- docs/manual/Makefile +++ docs/manual/Makefile @@ -13,13 +13,16 @@ # asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 design_spec.txt # all : server.ps megatest_manual.html client.ps complex-itemmap.png -megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt *png +megatest_manual.html : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt installation.txt *png asciidoc -b html5 -a icons -a iconsdir=$(DISPATH)/images/icons -a toc2 megatest_manual.txt # dos2unix megatest_manual.html + +megatest.pdf : megatest_manual.txt getting_started.txt writing_tests.txt reference.txt ../plan.txt howto.txt *png + a2x -a toc -f pdf megatest_manual.txt server.ps : server.dot dot -Tps server.dot > server.ps client.ps : client.dot Index: docs/manual/getting_started.txt ================================================================== --- docs/manual/getting_started.txt +++ docs/manual/getting_started.txt @@ -1,81 +1,99 @@ Getting Started -=============== +--------------- [partintro] .Getting started with Megatest -- -How to install Megatest and set it up for running your regressions and continuous integration process. +Creating a testsuite or flow and your first test or task. -- -Installation ------------- - -Dependencies -~~~~~~~~~~~~ - -Chicken scheme and a number of "eggs" are required for building -Megatest. See the script installall.sch in the utils directory of the -distribution for a mostly automated way to install everything needed -for building Megatest on Linux. - -footnote:[An example footnote.] -indexterm:[Example index entry] - -// -// -// And now for something completely different: ((monkeys)), lions and -// tigers (Bengal and Siberian) using the alternative syntax index -// entries. -// (((Big cats,Lions))) -// (((Big cats,Tigers,Bengal Tiger))) -// (((Big cats,Tigers,Siberian Tiger))) -// Note that multi-entry terms generate separate index entries. -// -// Here are a couple of image examples: an image:images/smallnew.png[] -// example inline image followed by an example block image: -// -// .Tiger block image -// image::images/tiger.png[Tiger image] -// -// Followed by an example table: -// -// .An example table -// [width="60%",options="header"] -// |============================================== -// | Option | Description -// | -a 'USER GROUP' | Add 'USER' to 'GROUP'. -// | -R 'GROUP' | Disables access to 'GROUP'. -// |============================================== -// -// .An example example -// =============================================== -// Lorum ipum... -// =============================================== -// -// [[X1]] -// Sub-section with Anchor -// ~~~~~~~~~~~~~~~~~~~~~~~ -// Sub-section at level 2. -// -// Chapter Sub-section -// ^^^^^^^^^^^^^^^^^^^ -// Sub-section at level 3. -// -// Chapter Sub-section -// +++++++++++++++++++ -// Sub-section at level 4. -// -// This is the maximum sub-section depth supported by the distributed -// AsciiDoc configuration. -// footnote:[A second example footnote.] -// -// -// The Second Chapter -// ------------------ -// An example link to anchor at start of the <>. -// indexterm:[Second example index entry] -// -// An example link to a bibliography entry <>. -// -// +After installing Megatest you can create a flow or testsuite and add some +tests using the helpers. Here is a quickstart sequence to get you up and +running your first automated testsuite. + +Creating a Megatest Area +~~~~~~~~~~~~~~~~~~~~~~~~ + +Choose Target Keys +^^^^^^^^^^^^^^^^^^ + +First choose your "target" keys. These are used to organise your runs in a +way that is meaningful to your project. If you are unsure about what to use +for keys just use a single generic key such as "RUNTYPE". These keys will be +used to hand values to your tests via environment variables so ensure they +are unique. Prefixing them with something such as PROJKEYS_ is a good +strategy. + +Examples of keys: + +.Example keys +[width="60%",options="header"] +|============================================== +| Option | Description +| RELEASE/ITERATION | This example is used by Megatest for its internal QA. +| ARCH/OS/RELEASE | For a software project targeting multiple platforms +| UCTRLR/NODETYPE | Microcontroller project with different controllers +running same software +|============================================== + +Create Area Config Files +^^^^^^^^^^^^^^^^^^^^^^^^ + +You will need to choose locations for your runs (the data generated every +time you run the testsuite) and link tree. For getting started answer the +prompts with "runs" and "links". We use the Unix editor "vi" in the examples +below but you can use any plain text editor. + +.Using the helper to create a Megatest area +------------------ +megatest -create-megatest-area + +# optional: verify that the settings are ok +vi megatest.config +vi runconfigs.config +------------------ + +Creating a Test +~~~~~~~~~~~~~~~ + +Choose the test name for your first test and run the helper. You can edit +the files after the initial creation. You will need to enter names and +scripts for the steps to be run and then edit the +tests//testconfig file and modify the logpro rules to properly +process the log output from your steps. For your first test just hit enter +for the "waiton", "priority" and iteration variable prompts. + +Hint: for geting started make your logpro rules very liberal. expect:error +patterns should match nothing and comment out expect:required rules. + +.Using the helper to create a Megatest test +--------------- +megatest -create-test myfirsttest + +# then edit the generated config +vi tests/myfirsttest/testconfig +--------------- + +Running your test +~~~~~~~~~~~~~~~~~ + +First choose a target and runname. If you have a two-place target such as +RELEASE/ITERATION a target would look like v1.0/aff3 where v1.0 is the +RELEASE and aff3 is the ITERATION. For a run name just use something like +run1. + +.Running all tests (testpatt of "%" matches all tests) +--------------- +megatest -run -target v1.0/aff3 -runname run1 -testpatt % -log run1.log +--------------- + +Viewing the results +~~~~~~~~~~~~~~~~~~~ + +Start the dashboard and browse your run in the "Runs" tab. + +.Starting dashboard +---------------- +dashboard -rows 24 +---------------- Index: docs/manual/howto.txt ================================================================== --- docs/manual/howto.txt +++ docs/manual/howto.txt @@ -1,14 +1,14 @@ How To Do Things -================ +---------------- Process Runs ------------- +~~~~~~~~~~~~ Remove Runs -~~~~~~~~~~~ +^^^^^^^^^^^ From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that comes up push the clean test button. The command field will be prefilled with a template command for removing that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests. @@ -21,30 +21,30 @@ ---------------- megatest -remove-runs -target %/%/% -runname % -testpatt % -v ---------------- Archive Runs -~~~~~~~~~~~~ +^^^^^^^^^^^^ Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run durations, time stamps etc.) are all preserved in the megatest database. For setup information see the Archiving topic in the reference section of this manual. To Archive -^^^^^^^^^^ +++++++++++ Hint: use the test control panel to create a template command by pushing the "Archive Tests" button. .Archive a full run ---------------- megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt % ---------------- To Restore -^^^^^^^^^^ +++++++++++ .Retrieve a single test ---------------- megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/% ---------------- @@ -78,17 +78,17 @@ # match. flexi-launcher yes ------------------------ Tricks -====== +------ This section is a compendium of a various useful tricks for debugging, configuring and generally getting the most out of Megatest. Limiting your running jobs --------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously. In your testconfig: @@ -102,13 +102,10 @@ --------------- [jobgroups] group1 10 custdes 4 --------------- - - - Debugging Tricks ---------------- Examining The Environment ADDED docs/manual/installation.txt Index: docs/manual/installation.txt ================================================================== --- /dev/null +++ docs/manual/installation.txt @@ -0,0 +1,10 @@ +Installation +------------ + +Dependencies +~~~~~~~~~~~~ + +Chicken scheme and a number of "eggs" are required for building +Megatest. See the script installall.sh in the utils directory of the +source distribution for an automated way to install everything +needed for building Megatest on Linux. Index: docs/manual/megatest_manual.html ================================================================== --- docs/manual/megatest_manual.html +++ docs/manual/megatest_manual.html @@ -1,10 +1,10 @@ - + The Megatest Users Manual + + +EOF +) + +(define (tests:run-record->test-path run numkeys) + (append (take (vector->list run) numkeys) + (list (vector-ref run (+ 1 numkeys))))) + +;; (tests:create-html-tree "test-index.html") +;; +(define (tests:create-html-tree outf) + (let* ((lockfile (conc outf ".lock")) + (runs-to-process '())) + (if (common:simple-file-lock lockfile) + (let* ((linktree (common:get-linktree)) + (oup (open-output-file (or outf (conc linktree "/runs-index.html")))) + (area-name (common:get-testsuite-name)) + (keys (rmt:get-keys)) + (numkeys (length keys)) + (runsdat (rmt:get-runs "%" #f #f (map (lambda (x)(list x "%")) keys))) + (header (vector-ref runsdat 0)) + (runs (vector-ref runsdat 1)) + (runtreedat (map (lambda (x) + (tests:run-record->test-path x numkeys)) + runs)) + (runs-htree (common:list->htree runtreedat))) + (set! runs-to-process runs) + (s:output-new + oup + (s:html tests:css-jscript-block + (s:title "Summary for " area-name) + (s:body 'onload "addEvents();" + (s:h1 "Summary for " area-name) + ;; top list + (s:ul 'id "LinkedList1" 'class "LinkedList" + (s:li + "Runs" + (common:htree->html runs-htree + '() + (lambda (x p) + (let* ((targ-path (string-intersperse p "/")) + (full-path (conc linktree "/" targ-path)) + (run-name (car (reverse p)))) + (if (and (file-exists? full-path) + (directory? full-path) + (file-write-access? full-path)) + (s:a run-name 'href (conc targ-path "/run-summary.html")) + (begin + (debug:print 0 *default-log-port* "INFO: Can't create " targ-path "/run-summary.html") + (conc run-name " (Not able to create summary at " targ-path ")"))))))))))) + (close-output-port oup) + (common:simple-file-release-lock lockfile) + (for-each + (lambda (run) + (let* ((test-subpath (tests:run-record->test-path run numkeys)) + (run-id (db:get-value-by-header run header "id")) + (run-dir (tests:run-record->test-path run numkeys)) + (test-dats (rmt:get-tests-for-run + run-id + "%/" ;; testnamepatt + '() ;; states + '() ;; statuses + #f ;; offset + #f ;; num-to-get + #f ;; hide/not-hide + #f ;; sort-by + #f ;; sort-order + #f ;; 'shortlist ;; qrytype + 0 ;; last update + #f)) + (tests-tree-dat (map (lambda (test-dat) + ;; (tests:run-record->test-path x numkeys)) + (let* ((test-name (db:test-get-testname test-dat)) + (item-path (db:test-get-item-path test-dat)) + (full-name (db:test-make-full-name test-name item-path)) + (path-parts (string-split full-name))) + path-parts)) + test-dats)) + (tests-htree (common:list->htree tests-tree-dat)) + (html-dir (conc linktree "/" (string-intersperse run-dir "/"))) + (html-path (conc html-dir "/run-summary.html")) + (oup (if (and (file-exists? html-dir) + (directory? html-dir) + (file-write-access? html-dir)) + (open-output-file html-path) + #f))) + ;; (print "run-dir: " run-dir ", tests-tree-dat: " tests-tree-dat) + (if oup + (begin + (s:output-new + oup + (s:html tests:css-jscript-block + (s:title "Summary for " area-name) + (s:body 'onload "addEvents();" + (s:h1 "Summary for " (string-intersperse run-dir "/")) + ;; top list + (s:ul 'id "LinkedList1" 'class "LinkedList" + (s:li + "Tests" + (common:htree->html tests-htree + '() + (lambda (x p) + (let* ((targ-path (string-intersperse p "/")) + (test-name (car p)) + (item-path ;; (if (> (length p) 2) ;; test-name + run-name + (string-intersperse p "/")) + (full-targ (conc html-dir "/" targ-path)) + (std-file (conc full-targ "/test-summary.html")) + (alt-file (conc full-targ "/megatest-rollup-" test-name ".html")) + (html-file (if (file-exists? alt-file) + alt-file + std-file)) + (run-name (car (reverse p)))) + (if (and (not (file-exists? full-targ)) + (directory? full-targ) + (file-write-access? full-targ)) + (tests:summarize-test + run-id + (rmt:get-test-id run-id test-name item-path))) + (if (file-exists? full-targ) + (s:a run-name 'href html-file) + (begin + (debug:print 0 *default-log-port* "ERROR: can't access " full-targ) + (conc "No summary for " run-name))))) + )))))) + (close-output-port oup))))) + runs) + #t) + #f))) + ;; CHECK - WAS THIS ADDED OR REMOVED? MANUAL MERGE WITH API STUFF!!! ;; ;; get a pretty table to summarize steps ;; @@ -666,11 +864,12 @@ (conc (vector-ref b 2))) #f)) (stringnumber priority))) - (if tmp tmp (begin (debug:print-error 0 *default-log-port* "bad priority value " priority ", using 0") 0))) - 0))) - (all-tests (hash-table-keys test-records)) - (all-waited-on (let loop ((hed (car all-tests)) - (tal (cdr all-tests)) - (res '())) - (let* ((trec (hash-table-ref test-records hed)) - (waitons (or (tests:testqueue-get-waitons trec) '()))) - (if (null? tal) - (append res waitons) - (loop (car tal)(cdr tal)(append res waitons)))))) - (sort-fn1 - (lambda (a b) - (let* ((a-record (hash-table-ref test-records a)) - (b-record (hash-table-ref test-records b)) - (a-waitons (or (tests:testqueue-get-waitons a-record) '())) - (b-waitons (or (tests:testqueue-get-waitons b-record) '())) - (a-config (tests:testqueue-get-testconfig a-record)) - (b-config (tests:testqueue-get-testconfig b-record)) - (a-raw-pri (config-lookup a-config "requirements" "priority")) - (b-raw-pri (config-lookup b-config "requirements" "priority")) - (a-priority (mungepriority a-raw-pri)) - (b-priority (mungepriority b-raw-pri))) - (tests:testqueue-set-priority! a-record a-priority) - (tests:testqueue-set-priority! b-record b-priority) - ;; (debug:print 0 *default-log-port* "a=" a ", b=" b ", a-waitons=" a-waitons ", b-waitons=" b-waitons) - (cond - ;; is - ((member a b-waitons) ;; is b waiting on a? - ;; (debug:print 0 *default-log-port* "case1") - #t) - ((member b a-waitons) ;; is a waiting on b? - ;; (debug:print 0 *default-log-port* "case2") - #f) - ((and (not (null? a-waitons)) ;; both have waitons - do not disturb - (not (null? b-waitons))) - ;; (debug:print 0 *default-log-port* "case2.1") - #t) - ((and (null? a-waitons) ;; no waitons for a but b has waitons - (not (null? b-waitons))) - ;; (debug:print 0 *default-log-port* "case3") - #f) - ((and (not (null? a-waitons)) ;; a has waitons but b does not - (null? b-waitons)) - ;; (debug:print 0 *default-log-port* "case4") - #t) - ((not (eq? a-priority b-priority)) ;; use - (> a-priority b-priority)) - (else - ;; (debug:print 0 *default-log-port* "case5") - (string>? a b)))))) - - (sort-fn2 - (lambda (a b) - (> (mungepriority (tests:testqueue-get-priority (hash-table-ref test-records a))) - (mungepriority (tests:testqueue-get-priority (hash-table-ref test-records b))))))) - ;; (let ((dot-res (tests:run-dot (tests:tests->dot test-records) "plain"))) - ;; (debug:print "dot-res=" dot-res)) - ;; (let ((data (map cdr (filter - ;; (lambda (x)(equal? "node" (car x))) - ;; (map string-split (tests:easy-dot test-records "plain")))))) - ;; (map car (sort data (lambda (a b) - ;; (> (string->number (caddr a))(string->number (caddr b))))))) - ;; )) - (sort all-tests sort-fn1))) ;; avoid dealing with deleted tests, look at the hash table + (if (eq? (hash-table-size test-records) 0) + '() + (let* ((mungepriority (lambda (priority) + (if priority + (let ((tmp (any->number priority))) + (if tmp tmp (begin (debug:print-error 0 *default-log-port* "bad priority value " priority ", using 0") 0))) + 0))) + (all-tests (hash-table-keys test-records)) + (all-waited-on (let loop ((hed (car all-tests)) + (tal (cdr all-tests)) + (res '())) + (let* ((trec (hash-table-ref test-records hed)) + (waitons (or (tests:testqueue-get-waitons trec) '()))) + (if (null? tal) + (append res waitons) + (loop (car tal)(cdr tal)(append res waitons)))))) + (sort-fn1 + (lambda (a b) + (let* ((a-record (hash-table-ref test-records a)) + (b-record (hash-table-ref test-records b)) + (a-waitons (or (tests:testqueue-get-waitons a-record) '())) + (b-waitons (or (tests:testqueue-get-waitons b-record) '())) + (a-config (tests:testqueue-get-testconfig a-record)) + (b-config (tests:testqueue-get-testconfig b-record)) + (a-raw-pri (config-lookup a-config "requirements" "priority")) + (b-raw-pri (config-lookup b-config "requirements" "priority")) + (a-priority (mungepriority a-raw-pri)) + (b-priority (mungepriority b-raw-pri))) + (tests:testqueue-set-priority! a-record a-priority) + (tests:testqueue-set-priority! b-record b-priority) + ;; (debug:print 0 *default-log-port* "a=" a ", b=" b ", a-waitons=" a-waitons ", b-waitons=" b-waitons) + (cond + ;; is + ((member a b-waitons) ;; is b waiting on a? + ;; (debug:print 0 *default-log-port* "case1") + #t) + ((member b a-waitons) ;; is a waiting on b? + ;; (debug:print 0 *default-log-port* "case2") + #f) + ((and (not (null? a-waitons)) ;; both have waitons - do not disturb + (not (null? b-waitons))) + ;; (debug:print 0 *default-log-port* "case2.1") + #t) + ((and (null? a-waitons) ;; no waitons for a but b has waitons + (not (null? b-waitons))) + ;; (debug:print 0 *default-log-port* "case3") + #f) + ((and (not (null? a-waitons)) ;; a has waitons but b does not + (null? b-waitons)) + ;; (debug:print 0 *default-log-port* "case4") + #t) + ((not (eq? a-priority b-priority)) ;; use + (> a-priority b-priority)) + (else + ;; (debug:print 0 *default-log-port* "case5") + (string>? a b)))))) + + (sort-fn2 + (lambda (a b) + (> (mungepriority (tests:testqueue-get-priority (hash-table-ref test-records a))) + (mungepriority (tests:testqueue-get-priority (hash-table-ref test-records b))))))) + ;; (let ((dot-res (tests:run-dot (tests:tests->dot test-records) "plain"))) + ;; (debug:print "dot-res=" dot-res)) + ;; (let ((data (map cdr (filter + ;; (lambda (x)(equal? "node" (car x))) + ;; (map string-split (tests:easy-dot test-records "plain")))))) + ;; (map car (sort data (lambda (a b) + ;; (> (string->number (caddr a))(string->number (caddr b))))))) + ;; )) + (sort all-tests sort-fn1)))) ;; avoid dealing with deleted tests, look at the hash table (define (tests:easy-dot test-records outtype) (let-values (((fd temp-path) (file-mkstemp (conc "/tmp/" (current-user-name) ".XXXXXX")))) (let ((all-testnames (hash-table-keys test-records)) (temp-port (open-output-file* fd))) @@ -1131,10 +1332,11 @@ "SELECT count(id) FROM test_rundat;") res)) 0) (define (tests:update-central-meta-info run-id test-id cpuload diskfree minutes uname hostname) + (rmt:general-call 'update-test-rundat run-id test-id (current-seconds) (or cpuload -1)(or diskfree -1) -1 (or minutes -1)) (if (and cpuload diskfree) (rmt:general-call 'update-cpuload-diskfree run-id cpuload diskfree test-id)) (if minutes (rmt:general-call 'update-run-duration run-id minutes test-id)) (if (and uname hostname) Index: tests/Makefile ================================================================== --- tests/Makefile +++ tests/Makefile @@ -48,34 +48,34 @@ cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG) test1 : cleanprep test2 : fullprep - cd fullrun;$(MEGATEST) -preclean -runtests ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none :runname $(RUNNAME) -debug $(DEBUG) $(LOGGING) - cd fullrun;megatest -preclean -runtests % -target ubuntu/nfs/none :runname $(RUNNAME)_01 -testpatt %/,%/ai -debug $(DEBUG) - cd fullrun;megatest -preclean -runtests %/,%/ai -target ubuntu/nfs/none :runname $(RUNAME)_02 -debug $(DEBUG) - cd fullrun;megatest -preclean -runtests runfirst/%,%/ai -target ubuntu/nfs/none :runname $(RUNNAME)_02 -debug $(DEBUG) - cd fullrun;megatest -runtests %/,%/winter -target ubuntu/nfs/none :runname $(RUNNAME)_03 -debug $(DEBUG) - sleep 40;cd fullrun;megatest -target ubuntu/nfs/none :runname $(RUNNAME) -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -debug $(DEBUG) $(LOGGING) + cd fullrun;$(MEGATEST) -preclean -runtests ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none -runname $(RUNNAME) -debug $(DEBUG) $(LOGGING) + cd fullrun;megatest -preclean -runtests % -target ubuntu/nfs/none -runname $(RUNNAME)_01 -testpatt %/,%/ai -debug $(DEBUG) + cd fullrun;megatest -preclean -runtests %/,%/ai -target ubuntu/nfs/none -runname $(RUNAME)_02 -debug $(DEBUG) + cd fullrun;megatest -preclean -runtests runfirst/%,%/ai -target ubuntu/nfs/none -runname $(RUNNAME)_02 -debug $(DEBUG) + cd fullrun;megatest -runtests %/,%/winter -target ubuntu/nfs/none -runname $(RUNNAME)_03 -debug $(DEBUG) + sleep 40;cd fullrun;megatest -target ubuntu/nfs/none -runname $(RUNNAME) -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -debug $(DEBUG) $(LOGGING) test3 : fullprep test3a test3b test3a : @echo Run runfirst and any waitons. - cd fullrun;$(MEGATEST) -preclean -runtests runfirst -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b + cd fullrun;$(MEGATEST) -preclean -runtests runfirst -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b test3b : @echo Run all_toplevel and all waitons - cd fullrun;$(MEGATEST) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_c + cd fullrun;$(MEGATEST) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_c test4 : cleanprep @echo "WARNING: No longer running fullprep, test converage may be lessened" - cd fullrun;time $(MEGATEST) -debug $(DEBUG) -run-wait -runtests % -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING) + cd fullrun;time $(MEGATEST) -debug $(DEBUG) -run-wait -run -testpatt % -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING) test4a : cleanprep - cd fullrun;time $(MEGATEST) -debug $(DEBUG) -preclean -runtests all_toplevel -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING) + cd fullrun;time $(MEGATEST) -debug $(DEBUG) -preclean -run -testpatt all_toplevel -reqtarg ubuntu/nfs/none -runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING) # NOTE: Only one instance can be a server test5 : cleanprep rm -f fullrun/a*.log fullrun/logs/* @echo "WARNING: No longer running fullprep, test converage may be lessened" Index: tests/fullrun/megatest.config ================================================================== --- tests/fullrun/megatest.config +++ tests/fullrun/megatest.config @@ -12,12 +12,12 @@ area1 /tmp/oldarea/megatest [include ./configs/mt_include_1.config] [dashboard] -pre-command xterm -geometry 180x20 -e " -post-command |& tee results.log ;echo Press any key to continue;bash -c 'read -n 1 -s'" & +# pre-command xterm -geometry 180x20 -e " +# post-command |& tee results.log ;echo Press any key to continue;bash -c 'read -n 1 -s'" & testsort -event_time [misc] home #{shell readlink -f $MT_RUN_AREA_HOME} parent #{shell readlink -f $MT_RUN_AREA_HOME/..} @@ -32,20 +32,26 @@ # and set the dbdir to /var/tmp/$USER/mt_db to enable keeping # the raw db in /var/tmp/$USER # faststart no monitordir #{getenv MT_RUN_AREA_HOME}/db -dbdir #{getenv MT_RUN_AREA_HOME}/db +dbdirdefn /tmp/#{getenv USER}/#{getenv MT_TESTSUITE_NAME}/db +dbdirmkdir #{scheme (create-directory "#{get setup dbdirdefn}" #t)} +dbdir #{get setup dbdirdefn} # sync more aggressively to megatest-db megatest-db yes # Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding # this may save a few milliseconds on launching tests # launchwait no waivercommentpatt ^WW\d+ [a-z].* incomplete-timeout 1 + +# wait 0.5 seconds between launching every process +# +launch-delay 0.5 # wait for runs to completely complete. yes, anything else is no run-wait yes # If set to "default" the old code is used. Otherwise defaults to 200 or uses @@ -151,11 +157,12 @@ # force use of server always # required yes # Use http instead of direct filesystem access -transport http +transport rpc +# transport http # transport fs # transport nmsg synchronous 0 @@ -243,11 +250,11 @@ ((sleeprunner) "sleeprunner") \ (else "nbfake"))} # launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log -# launcher #{shell if which bsub > /dev/null;then echo bsub -q priority -o openlava.log;else echo sleeprunner;fi} +# launcher #{ shell if which bsub > /dev/null;then echo bsub -q priority -o openlava.log;else echo sleeprunner;fi} # launcher nbfake [configf:settings trim-trailing-spaces yes] # Override the rollup for specific tests Index: tests/rununittest.sh ================================================================== --- tests/rununittest.sh +++ tests/rununittest.sh @@ -1,19 +1,21 @@ #!/bin/bash # Usage: rununittest.sh testname debuglevel # +banner $1 # put megatest on path from correct location mtbindir=$(readlink -f ../bin) export PATH="${mtbindir}:$PATH" # Clean setup # -dbdir=$(cd simplerun;megatest -show-config -section setup -var linktree)/.db -rm -f simplerun/megatest.db simplerun/monitor.db simplerun/db/monitor.db $dbdir/*.db +dbdir=$(echo /tmp/$USER/megatest_localdb/simplerun/.[a-zA-Z]*/) +echo "dbdir=$dbdir" +rm -f simplerun/megatest.db simplerun/monitor.db simplerun/db/monitor.db $dbdir rm -rf simplelinks/ simpleruns/ simplerun/db/ $dbdir mkdir -p simplelinks simpleruns (cd simplerun;cp ../../*_records.scm .;perl -pi.bak -e 's/define-inline/define/' *_records.scm) (cd simplerun;cp ../../altdb.scm .) Index: tests/unittests/basicserver.scm ================================================================== --- tests/unittests/basicserver.scm +++ tests/unittests/basicserver.scm @@ -10,19 +10,83 @@ (define run-id 1) (test "setup for run" #t (begin (launch:setup) (string? (getenv "MT_RUN_AREA_HOME")))) -;; NON Server tests go here - -(test #f #f (db:dbdat-get-path *db*)) -(test #f #f (db:get-run-name-from-id *db* run-id)) -;; (test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) - -;; (exit) - -;; Server tests go here +(test #f #t (and (server:kind-run *toppath*) #t)) + + +(define user (current-user-name)) +(define runname "mytestrun") +(define keys (rmt:get-keys)) +(define runinfo #f) +(define keyvals '(("SYSTEM" "abc")("RELEASE" "def"))) +(define header (list "SYSTEM" "RELEASE" "id" "runname" "state" "status" "owner" "event_time")) + +;; Setup +;; +;; (test #f #f (not (client:setup run-id))) +;; (test #f #f (not (hash-table-ref/default *runremote* run-id #f))) + +;; Login +;; +(test #f'(#t "successful login") + (rmt:login run-id)) + +;; Keys +;; +(test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) + +;; No data in db +;; +(test #f '() (rmt:get-all-run-ids)) +(test #f #f (rmt:get-run-name-from-id run-id)) +(test #f + (vector + header + (vector #f #f #f #f)) + (rmt:get-run-info run-id)) + +;; Insert data into db +;; +(test #f 1 (rmt:register-run keyvals runname "new" "n/a" user)) +;; (test #f #f (rmt:get-runs-by-patt keys runname)) +(test #f #t (rmt:general-call 'register-test run-id run-id "test-one" "")) +(define test-one-id #f) +(test #f 1 (let ((test-id (rmt:get-test-id run-id "test-one" ""))) + (set! test-one-id test-id) + test-id)) +(define test-one-rec #f) +(test #f "test-one" (let ((test-rec (rmt:get-test-info-by-id run-id test-one-id))) + (set! test-one-rec test-rec) + (vector-ref test-rec 2))) + +;; With data in db +;; +(print "Using runame=" runname) +(test #f '(1) (rmt:get-all-run-ids)) +(test #f runname (rmt:get-run-name-from-id run-id)) +(test #f + runname + (let ((run-info (rmt:get-run-info run-id))) + (db:get-value-by-header (db:get-rows run-info) + (db:get-header run-info) + "runname"))) + +;; test killing server +;; +(for-each + (lambda (run-id) + (test #f #t (and (tasks:kill-server-run-id run-id) #t)) + (test #f #f (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id))) + (list 0 1)) + +;; Tests to assess reading/writing while servers are starting/stopping +;; NO LONGER APPLICABLE + +;; Server tests go here +(define (server-tests-dont-run-right-now) (for-each (lambda (run-id) (test #f #f (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)) (server:kind-run run-id) (test "did server start within 20 seconds?" @@ -51,82 +115,14 @@ (begin (thread-sleep! 1.1) (loop (- remtries 1)(tasks:get-server (db:delay-if-busy (tasks:open-db)) run-id))) res))))) ) - (list 0 1)) - -(define user (current-user-name)) -(define runname "mytestrun") -(define keys (rmt:get-keys)) -(define runinfo #f) -(define keyvals '(("SYSTEM" "abc")("RELEASE" "def"))) -(define header (list "SYSTEM" "RELEASE" "id" "runname" "state" "status" "owner" "event_time")) - -;; Setup -;; -(test #f #f (not (client:setup run-id))) -(test #f #f (not (hash-table-ref/default *runremote* run-id #f))) - -;; Login -;; -(test #f'(#t "successful login") - (rmt:login-no-auto-client-setup (hash-table-ref/default *runremote* run-id #f) run-id)) -(test #f '(#t "successful login") - (rmt:login run-id)) - -;; Keys -;; -(test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) - -;; No data in db -;; -(test #f '() (rmt:get-all-run-ids)) -(test #f #f (rmt:get-run-name-from-id run-id)) -(test #f - (vector - header - (vector #f #f #f #f)) - (rmt:get-run-info run-id)) - -;; Insert data into db -;; -(test #f 1 (rmt:register-run keyvals runname "new" "n/a" user)) -;; (test #f #f (rmt:get-runs-by-patt keys runname)) -(test #f #t (rmt:general-call 'register-test run-id run-id "test-one" "")) -(define test-one-id #f) -(test #f 30001 (let ((test-id (rmt:get-test-id run-id "test-one" ""))) - (set! test-one-id test-id) - test-id)) -(define test-one-rec #f) -(test #f "test-one" (let ((test-rec (rmt:get-test-info-by-id run-id test-one-id))) - (set! test-one-rec test-rec) - (vector-ref test-rec 2))) - -;; With data in db -;; -(print "Using runame=" runname) -(test #f '(1) (rmt:get-all-run-ids)) -(test #f runname (rmt:get-run-name-from-id run-id)) -(test #f - runname - (let ((run-info (rmt:get-run-info run-id))) - (db:get-value-by-header (db:get-rows run-info) - (db:get-header run-info) - "runname"))) - -(for-each (lambda (run-id) -;; test killing server -;; -(tasks:kill-server-run-id run-id) - -(test #f #f (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)) -) -(list 0 1)) - -;; Tests to assess reading/writing while servers are starting/stopping -(define start-time (current-seconds)) + (list 0 1))) + +(define start-time (current-seconds)) +(define (reading-writing-while-server-starting-stopping-dont-run-now) (let loop ((test-state 'start)) (let* ((server-dats (tasks:get-server-records (db:delay-if-busy (tasks:open-db)) run-id)) (first-dat (if (not (null? server-dats)) (car server-dats) #f))) @@ -149,11 +145,11 @@ ((shutting-down) (loop test-state)) (else (print "Don't know what to do if get here")))) ((server-shutdown) (loop test-state))))) - +) ;;====================================================================== ;; END OF TESTS ;;====================================================================== Index: tests/unittests/runs.scm ================================================================== --- tests/unittests/runs.scm +++ tests/unittests/runs.scm @@ -1,6 +1,8 @@ (define keys (rmt:get-keys)) + +(test #f #t (and (server:kind-run *toppath*) #t)) (test "get all legal tests" (list "test1" "test2") (sort (hash-table-keys (tests:get-all)) string<=?)) (test "register-run" #t (number? (rmt:register-run @@ -9,12 +11,12 @@ "new" "n/a" "bob"))) (test #f #t (rmt:register-test 1 "nada" "")) -(test #f 30001 (rmt:get-test-id 1 "nada" "")) -(test #f "NOT_STARTED" (vector-ref (rmt:get-test-info-by-id 1 30001) 3)) ;; "nada" "") 3)) +(test #f 1 (rmt:get-test-id 1 "nada" "")) +(test #f "NOT_STARTED" (vector-ref (rmt:get-test-info-by-id 1 1) 3)) (test #f "FOO LIKE 'abc%def'" (db:patt->like "FOO" "abc%def")) (test #f "key2" (vector-ref (car (vector-ref (mt:get-runs-by-patt '("SYSTEM" "RELEASE") "%" "key1/key2") 1)) 1)) (test #f "SYSTEM,RELEASE,id,runname,state,status,owner,event_time" (car (runs:get-std-run-fields keys '("id" "runname" "state" "status" "owner" "event_time")))) @@ -49,11 +51,11 @@ ; (hash-table-set! args:arg-hash "-keepgoing" #t) (hash-table-set! args:arg-hash "-itempatt" "%") (hash-table-set! args:arg-hash "-testpatt" "%") (hash-table-set! args:arg-hash "-target" "ubuntu/r1.2") ;; SYSTEM/RELEASE (hash-table-set! args:arg-hash "-runname" "testrun") -(test "Setup for a run" #t (begin (launch:setup-for-run) #t)) +(test "Setup for a run" #t (string? (launch:setup))) (define *tdb* #f) (define keyvals #f) (test "target->keyval" #t (let ((kv (keys:target->keyval keys (args:get-arg "-target")))) (print "keyvals=" kv ", keys=" keys) @@ -151,11 +153,11 @@ (test "launch-test" #t (string? (file-exists? ;; (launch-test test-id run-id run-info keyvals runname test-conf test-name test-path itemdat params) - (launch-test 30001 1 rinfo keyvals "run1" tconfig "test1" test-path '() (make-hash-table))))) + (launch-test 1 1 rinfo keyvals "run1" tconfig "test1" test-path '() (make-hash-table))))) ;;====================================================================== ;; M O R E R E M O T E C A L L S ;;====================================================================== @@ -169,13 +171,18 @@ ;; T E S T I T E M M A P ;;====================================================================== (test #f "a/b/c" (db:multi-pattern-apply "d/e/f" "d a\ne b\nf c")) (test #f "blah/foo/bar/baz" (db:convert-test-itempath "blah/baz/bar/foo" "^([^/]+)/([^/]+)/([^/]+)$ \\3/\\2/\\1")) -(test #f #t (db:compare-itempaths "abc/def/123" "abc/ghi/123" "ghi def")) -(test #f #f (db:compare-itempaths "some/5" "item/5" ".*/")) -(test #f #t (db:compare-itempaths "some/5" "item/5" ".*/ some/")) +(define itemmaps (alist->hash-table + '(("test1" "ghi def") + ("test2" ".*/") + ("test3" ".*/ some/")))) + +(test #f #t (db:compare-itempaths "test1" "abc/def/123" "abc/ghi/123" itemmaps)) +(test #f #f (db:compare-itempaths "test2" "some/5" "item/5" ".*/" itemmaps)) +(test #f #t (db:compare-itempaths "test3" "some/5" "item/5" ".*/ some/" itemmaps)) (test #f '() (rmt:get-prereqs-not-met 1 '("rollup") "some/5" mode: '(toplevel) itemmap: ".*/" "/")) (test #f '() (rmt:get-prereqs-not-met 1 '("rollup") "some/5" mode: '(normal) itemmap: ".*/" "/")) (test #f '() (rmt:get-prereqs-not-met 1 '("rollup") "some/5" mode: '(itemmatch) itemmap: ".*/" "/")) (test #f '() (rmt:get-prereqs-not-met 1 '("rollup") "some/5" mode: '(itemwait) itemmap: ".*/" "/")) ADDED thunk-utils.scm Index: thunk-utils.scm ================================================================== --- /dev/null +++ thunk-utils.scm @@ -0,0 +1,121 @@ +(use srfi-18) + + +;; wrap a proc with a mutex so that two threads may not call proc simultaneously. +;; will catch exceptions to ensure mutex is unlocked even if exception is thrown. +;; will generate a unique mutex for proc unless one is specified with canned-mutex: option +;; +;; example 1: (define thread-safe-+ (make-synchronized-proc +)) +;; example 2: (define thread-safe-plus +;; (make-synchronized-proc +;; (lambda (x y) +;; (+ x y)))) + +(define (make-synchronized-proc proc + #!key (canned-mutex #f)) + (let* ((guard-mutex (if canned-mutex canned-mutex (make-mutex))) + (guarded-proc ;; we are guarding the thunk against exceptions. We will record whether result of evaluation is an exception or a regular result. + (lambda args + (mutex-lock! guard-mutex) + (let* ((EXCEPTION (gensym)) ;; using gensym to avoid potential collision with a proc that returns a pair having the first element be our flag. gensym guarantees the symbol is unique. + (res + (condition-case + (apply proc args) ;; this is what we are guarding the execution of + [x () (cons EXCEPTION x)] + ))) + (mutex-unlock! guard-mutex) + (cond + ((and (pair? res) (eq? (car res) EXCEPTION)) + (raise (cdr res))) + (else + res)))))) + guarded-proc)) + + +;; retry an operation (depends on srfi-18) +;; ================== +;; idea here is to avoid spending time on coding retrying something. Trying to be generic here. +;; +;; Exception handling: +;; ------------------- +;; if evaluating the thunk results in exception, it will be retried. +;; on last try, if final-failure-returns-actual is true, the exception will be re-thrown to caller. +;; +;; look at options below #!key to see how to configure behavior +;; +;; + +(define (retry-thunk + the-thunk + #!key ;;;; options below + (accept-result? (lambda (x) x)) ;; retry if predicate applied to thunk's result is false + (retries 4) ;; how many tries + (failure-value #f) ;; return this on final failure, unless following option is enabled: + (final-failure-returns-actual #f) ;; on failure, on the last try, just return the result, not failure-value + + (retry-delay 0.1) ;; delay between tries + (back-off-factor 1) ;; multiply retry-delay by this factor on retry + (random-delay 0.1) ;; add a random portion of this value to wait + + (chatty #f) ;; print status as we go, for debugging. + ) + + (when chatty (print) (print "Entered retry-thunk") (print "-=-=-=-=-=-")) + (let* ((guarded-thunk ;; we are guarding the thunk against exceptions. We will record whether result of evaluation is an exception or a regular result. + (lambda () + (let* ((EXCEPTION (gensym)) ;; using gensym to avoid potential collision + (res + (condition-case + (the-thunk) ;; this is what we are guarding the execution of + [x () (cons EXCEPTION x)] + ))) + (cond + ((and (pair? res) (eq? (car res) EXCEPTION)) + (if chatty + (print " - the-thunk threw exception >"(cdr res)"<")) + (cons 'exception (cdr res))) + (else + (if chatty + (print " - the-thunk returned result >"res"<")) + (cons 'regular-result res))))))) + + (let loop ((guarded-res (guarded-thunk)) + (retries-left retries) + (fail-wait retry-delay)) + (if chatty (print " ==========")) + (let* ((wait-time (+ fail-wait (+ (* fail-wait back-off-factor) + (* random-delay + (/ (random 1024) 1024) )))) + (res-type (car guarded-res)) + (res-value (cdr guarded-res))) + (cond + ((and (eq? res-type 'regular-result) (accept-result? res-value)) + (if chatty (print " + return result that satisfied accept-result? >"res-value"<")) + res-value) + + ((> retries-left 0) + (if chatty (print " - sleep "wait-time)) + (thread-sleep! wait-time) + (if chatty (print " + retry ["retries-left" tries left]")) + (loop (guarded-thunk) + (sub1 retries-left) + wait-time)) + + ((eq? res-type 'regular-result) + (if final-failure-returns-actual + (begin + (if chatty (print " + last try failed- return the result >"res-value"<")) + res-value) + (begin + (if chatty (print " + last try failed- return canned failure value >"failure-value"<")) + failure-value))) + + (else ;; no retries left; result was not accepted and res-type can only be 'exception + (if final-failure-returns-actual + (begin + (if chatty (print " + last try failed with exception- re-throw it >"res-value"<")) + (abort res-value)); re-raise the exception. TODO: find a way for call-history to show as though from entry to this function + (begin + (if chatty (print " + last try failed with exception- return canned failure value >"failure-value"<")) + failure-value)))))))) + Index: tree.scm ================================================================== --- tree.scm +++ tree.scm @@ -135,10 +135,10 @@ ;; (print "obj: " obj ", id: " id ", state: " state) (let* ((run-path (tree:node->path obj id)) (run-id (tree-path->run-id (cdr run-path)))) (if run-id (begin - (dboard:data-curr-run-id-set! *data* run-id) + (dboard:data-curr-run-id-set! data run-id) (dashboard:update-run-summary-tab))) ;; (print "path: " (tree:node->path obj id) " run-id: " run-id) )))) |# Index: utils/Makefile.git.installall ================================================================== --- utils/Makefile.git.installall +++ utils/Makefile.git.installall @@ -211,23 +211,23 @@ #====================================================================== # https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz # https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz -nanomsg-0.6-beta.tar.gz : - wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz - -nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz - tar xf nanomsg-0.6-beta.tar.gz - -$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING - cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install - -$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat - CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg - -# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg +# nanomsg-0.6-beta.tar.gz : +# wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz + +# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz +# tar xf nanomsg-0.6-beta.tar.gz + +# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING +# cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install + +# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat +# CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg + +# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg #====================================================================== # M A T T S U T I L S #====================================================================== Index: utils/Makefile.installall ================================================================== --- utils/Makefile.installall +++ utils/Makefile.installall @@ -195,26 +195,26 @@ #====================================================================== # N A N O M S G #====================================================================== -# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz -# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz - -nanomsg-0.6-beta.tar.gz : - wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz - -nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz - tar xf nanomsg-0.6-beta.tar.gz - -$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING - cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install - -$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat - CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg - -# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg +# # https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz +# # https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz + +# nanomsg-0.6-beta.tar.gz : +# wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz + +# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz +# tar xf nanomsg-0.6-beta.tar.gz + +# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING +# cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install + +# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat +# CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg + +# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg #====================================================================== # M A T T S U T I L S #====================================================================== Index: utils/Makefile.latest.installall ================================================================== --- utils/Makefile.latest.installall +++ utils/Makefile.latest.installall @@ -195,26 +195,26 @@ #====================================================================== # N A N O M S G #====================================================================== -# https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz -# https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz - -nanomsg-0.6-beta.tar.gz : - wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz - -nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz - tar xf nanomsg-0.6-beta.tar.gz - -$(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING - cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install - -$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat - CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg - -# LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg +# # https://github.com/nanomsg/nanomsg/releases/download/0.6-beta/nanomsg-0.6-beta.tar.gz +# # https://github.com/nanomsg/nanomsg/releases/download/0.8-beta/nanomsg-0.8-beta.tar.gz + +# nanomsg-0.6-beta.tar.gz : +# wget http://download.nanomsg.org/nanomsg-0.6-beta.tar.gz + +# nanomsg-0.6-beta/COPYING : nanomsg-0.6-beta.tar.gz +# tar xf nanomsg-0.6-beta.tar.gz + +# $(PREFIX)/bin/nanocat : nanomsg-0.6-beta/COPYING +# cd nanomsg-0.6-beta;./configure --prefix=$(PREFIX);make;make install + +# $(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat +# CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg + +# # LD_LIBRARY_PATH=/mfs/pkgs/chicken/4.10.0-amd64/lib CSC_OPTIONS="-I/mfs/pkgs/chicken/4.10.0-amd64/include -L/mfs/pkgs/chicken/4.10.0-amd64/lib -C \"-fPIC\"" /mfs/pkgs/chicken/4.10.0-amd64/bin/chicken-install -D no-library-checks nanomsg #====================================================================== # M A T T S U T I L S #====================================================================== ADDED utils/find-unused-globals.sh Index: utils/find-unused-globals.sh ================================================================== --- /dev/null +++ utils/find-unused-globals.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +echo "Finding unused globals:" + +for var in $(egrep '^\s*\(define\s+\*' *.scm|awk '{print $2}'|sort -u);do + if ! $(egrep -v '^\s*\(define' *scm| grep "$var" > /dev/null);then + echo "$var not used"; + fi; +done + +echo +echo "Finding globals without proper definition in common.scm:" + +for var in $(egrep -v '^\s*\(define' *.scm|\ + grep -P -v '^\s*;'|\ + grep -P '\*[a-zA-Z]+\S+\*'|\ + tr '*' '/' |\ + perl -p -e 's%.*(\/\S+\/).*%$1%'|\ + egrep '\/[a-zA-Z]+\S+\/'|\ + sort -u);do + newvar=$(echo $var | tr '/' '*') + # echo "VAR is $var, newvar is $newvar" + if ! $(grep -P '^\s*\(define\s+' common.scm|\ + grep -P -v '^\s*;'|\ + grep "$newvar" > /dev/null);then + echo "$newvar not defined in common.scm" + fi +done + Index: utils/installall.sh ================================================================== --- utils/installall.sh +++ utils/installall.sh @@ -10,14 +10,13 @@ # This program is distributed WITHOUT ANY WARRANTY; without even the # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR # PURPOSE. echo You may need to do the following first: -echo sudo apt-get install libreadline-dev -echo sudo apt-get install libwebkitgtk-dev +echo sudo apt-get install libreadline-dev libsqlite3-dev libwebkitgtk-dev echo sudo apt-get install libpangox-1.0-0 zlib1g-dev libfreetype6-dev cmake -echo sudo apt-get install libssl-dev +echo sudo apt-get install libssl-dev uuid-dev libglu1-mesa-dev echo sudo apt-get install libmotif3 -OR- set KTYPE=26g4 echo echo Set OPTION to std, currently OPTION=$OPTION echo echo Additionally, if you want mysql-client, you will need to make sure @@ -25,21 +24,37 @@ echo echo You are using PREFIX=$PREFIX echo You are using proxy="$proxy" echo echo "Set additional_libpath to help find gtk or other libraries, don't forget a leading :" + +if [[ "$OPTION"x == "x" ]];then + OPTION=std +fi SYSTEM_TYPE=$(lsb_release -irs |tr ' ' '_' |tr '\n' '-')$(uname -i)-$OPTION + +# default chicken version variables. Override in case statement as appropriate +CHICKEN_VERSION=4.10.0 +CHICKEN_BASEVER=4.10.0 # Set up variables # case $SYSTEM_TYPE in Ubuntu-16.04-x86_64-std) KTYPE=32 CDVER=5.10 IUPVER=3.17 IMVER=3.11 + ;; +Ubuntu-16.04-x86_64-new) + KTYPE=32 + CDVER=5.10 + IUPVER=3.17 + IMVER=3.11 + CHICKEN_VERSION=4.10.0 + CHICKEN_BASEVER=4.10.0 ;; Ubuntu-16.04-i686-std) KTYPE=32 CDVER=5.10 IUPVER=3.17 @@ -103,12 +118,10 @@ # Put all the downloaded tar files in tgz mkdir -p tgz # http://code.call-cc.org/releases/4.8.0/chicken-4.8.0.5.tar.gz -export CHICKEN_VERSION=4.11.0 -export CHICKEN_BASEVER=4.11.0 chicken_targz=chicken-${CHICKEN_VERSION}.tar.gz if ! [[ -e tgz/$chicken_targz ]]; then wget http://code.call-cc.org/releases/${CHICKEN_BASEVER}/${chicken_targz} mv $chicken_targz tgz fi @@ -145,20 +158,20 @@ cd $BUILDHOME fi cd $BUILDHOME #wget --no-check-certificate https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz #mv 1.0.0 1.0.0.tar.gz -if ! [[ -e $PREFIX/lib64/libnanomsg.so.1.0.0 ]]; then - wget --no-check-certificate https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz - mv 1.0.0 1.0.0.tar.gz - tar xf 1.0.0.tar.gz - cd nanomsg-1.0.0 - ./configure --prefix=$PREFIX - make - make install -fi -cd $BUILDHOME +# if ! [[ -e $PREFIX/lib64/libnanomsg.so.1.0.0 ]]; then +# wget --no-check-certificate https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz +# mv 1.0.0 1.0.0.tar.gz +# tar xf 1.0.0.tar.gz +# cd nanomsg-1.0.0 +# ./configure --prefix=$PREFIX +# make +# make install +# fi +# cd $BUILDHOME export SQLITE3_VERSION=3090200 if ! [[ -e $PREFIX/bin/sqlite3 ]]; then echo Install sqlite3 sqlite3_tgz=sqlite-autoconf-$SQLITE3_VERSION.tar.gz @@ -177,11 +190,11 @@ cd $BUILDHOME # Some eggs are quoted since they are reserved to Bash # for f in matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5; do # $CHICKEN_INSTALL $PROX -keep-installed matchable readline apropos base64 regex-literals format "regex-case" "test" coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client spiffy-request-vars md5 message-digest http-client spiffy-directory-listing -for egg in matchable readline apropos base64 regex-literals format "regex-case" "test" \ +for egg in matchable readline apropos dbi base64 regex-literals format "regex-case" "test" \ coops trace csv dot-locking posix-utils posix-extras directory-utils hostinfo \ tcp rpc csv-xml fmt json md5 awful http-client spiffy uri-common intarweb http-client \ spiffy-request-vars s md5 message-digest spiffy-directory-listing ssax sxml-serializer \ sxml-modifications logpro z3 call-with-environment-variables \ pathname-expand typed-records simple-exceptions numbers crypt parley srfi-42 \ @@ -206,11 +219,11 @@ if [[ -e `which mysql_config` ]]; then $CHICKEN_INSTALL $PROX -keep-installed mysql-client fi -for egg in "sqlite3" sql-de-lite nanomsg +for egg in "sqlite3" sql-de-lite # nanomsg do echo "Installing $egg" CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib -L$PREFIX/lib64" $CHICKEN_INSTALL $PROX -keep-installed $egg #CSC_OPTIONS="-I$PREFIX/include -L$PREFIX/lib -L$PREFIX/lib64" $CHICKEN_INSTALL $PROX $egg if [ $? -ne 0 ]; then @@ -302,12 +315,12 @@ cd histstore $PREFIX/bin/csc histstore.scm -o hs cp -f hs $PREFIX/bin/hs cd ../mutils $PREFIX/bin/chicken-install - cd ../dbi - $PREFIX/bin/chicken-install + # cd ../dbi + # $PREFIX/bin/chicken-install cd ../margs $PREFIX/bin/chicken-install fi cd $BUILDHOME ADDED utils/viewscreen Index: utils/viewscreen ================================================================== --- /dev/null +++ utils/viewscreen @@ -0,0 +1,19 @@ +#!/bin/bash + +if ! type screen &> /dev/null;then + xterm -geometry 180x20 -e "$*;echo Press any key to continue;bash -c 'read -n 1 -s'" & + exit +fi + +if [[ $(screen -list | egrep 'Attached|Detached'|awk '{print $1}') == "" ]];then + # echo "No screen found for displaying to. Run \"screen\" in an xterm" + # exit 1 + xterm -e screen -e^ff & + sleep 1 + screen -X hardstatus off + screen -X hardstatus alwayslastline + screen -X hardstatus string '%{= kG}[ %{G}%H %{g}][%= %{= kw}%?%-Lw%?%{r}(%{W}%n*%f%t%?(%u)%?%{r})%{w}%?%+Lw%?%?%= %{g}][%{B} %m-%d %{W} %c %{g}]' +fi + +cmd="cd $PWD;$*" +screen -X screen bash -c "$cmd;echo \"Press any key to continue, ctrl-f to see other windows\";bash -c 'read -n 1 -s'" & Index: vg.scm ================================================================== --- vg.scm +++ vg.scm @@ -368,15 +368,23 @@ (arithmetic-shift a 24) (arithmetic-shift r 16) (arithmetic-shift g 8) b)) +;; Obsolete function +;; (define (vg:generate-color) (vg:rgb->number (random 255) (random 255) (random 255))) - ;;(vg:rgb->number 0 0 0)) + +;; Need to return a string of random iup-color for graph +;; +(define (vg:generate-color-rgb) + (conc (number->string (random 255)) " " + (number->string (random 255)) " " + (number->string (random 255)))) (define (vg:iup-color->number iup-color) (apply vg:rgb->number (map string->number (string-split iup-color)))) ;;======================================================================