Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -4,11 +4,11 @@ INSTALL=install SRCFILES = common.scm items.scm launch.scm \ ods.scm runconfig.scm server.scm configf.scm \ db.scm keys.scm margs.scm megatest-version.scm \ process.scm runs.scm tasks.scm tests.scm genexample.scm \ - fs-transport.scm zmq-transport.scm http-transport.scm \ + fs-transport.scm http-transport.scm \ client.scm gutils.scm synchash.scm daemon.scm GUISRCF = dashboard-tests.scm dashboard-guimonitor.scm OFILES = $(SRCFILES:%.scm=%.o) @@ -109,11 +109,11 @@ chmod a+x $(PREFIX)/bin/dashboard install : bin $(PREFIX)/bin/mtest $(PREFIX)/bin/megatest $(PREFIX)/bin/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake $(PREFIX)/bin/nbfind $(PREFIX)/bin/newdboard deploytarg/apropos.so : Makefile - for i in apropos base64 canvas-draw csv-xml directory-utils dot-locking extras fmt format hostinfo http-client intarweb json md5 message-digest posix posix-extras readline regex regex-case s11n spiffy spiffy-request-vars sqlite3 srfi-1 srfi-18 srfi-69 tcp test uri-common zmq check-errors synch matchable sql-null tcp-server rpc blob-utils string-utils variable-item defstruct uri-generic sendfile opensll openssl lookup-table list-utils stack; do \ + for i in apropos base64 canvas-draw csv-xml directory-utils dot-locking extras fmt format hostinfo http-client intarweb json md5 message-digest posix posix-extras readline regex regex-case s11n spiffy spiffy-request-vars sqlite3 srfi-1 srfi-18 srfi-69 tcp test uri-common check-errors synch matchable sql-null tcp-server rpc blob-utils string-utils variable-item defstruct uri-generic sendfile opensll openssl lookup-table list-utils stack; do \ chicken-install -prefix deploytarg -deploy $$i;done deploytarg/libsqlite3.so : CSC_OPTIONS="-Ideploytarg -Ldeploytarg" $CHICKEN_INSTALL -prefix deploytarg -deploy sqlite3 Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -12,11 +12,13 @@ ;; C L I E N T S ;;====================================================================== (require-extension (srfi 18) extras tcp s11n) -(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest zmq) +(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest) +;; (use zmq) + (import (prefix sqlite3 sqlite3:)) (use spiffy uri-common intarweb http-client spiffy-request-vars) (declare (unit client)) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -19,11 +19,11 @@ (use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64) (import (prefix sqlite3 sqlite3:)) (import (prefix base64 base64:)) ;; Note, try to remove this dependency -(use zmq) +;; (use zmq) (declare (unit db)) (declare (uses common)) (declare (uses keys)) (declare (uses ods)) @@ -1139,15 +1139,15 @@ ;; NOTE: Can remove the regex and base64 encoding for zmq (define (db:obj->string obj) (case *transport-type* ((fs) obj) - ((http) + ((http) (string-substitute - (regexp "=") "_" - (base64:base64-encode (with-output-to-string (lambda ()(serialize obj)))) - #t)) + (regexp "=") "_" + (base64:base64-encode (with-output-to-string (lambda ()(serialize obj)))) + #t)) ((zmq)(with-output-to-string (lambda ()(serialize obj)))) (else obj))) (define (db:string->obj msg) (case *transport-type* @@ -1442,11 +1442,11 @@ (< (current-seconds) timeout)) (begin (thread-sleep! 0.01) (loop)))) (set! *number-of-writes* (+ *number-of-writes* 1)) - (set! *writes-total-delay* (+ *writes-total-delay* 1)) + (set! *writes-total-delay* (+ *writes-total-delay* (- (current-milliseconds) start-time))) got-it)) (define (db:process-queue-item db item) (let* ((stmt-key (cdb:packet-get-qtype item)) (qry-sig (cdb:packet-get-query-sig item)) @@ -1529,19 +1529,20 @@ "SELECT id,item_path,state,status,run_duration,final_logf,comment FROM tests WHERE run_id=? AND testname=? AND item_path != '';" run-id test-name) res)) ;; Rollup the pass/fail counts from itemized tests into fail_count and pass_count +;; NOTE: Is this duplicating (db:test-data-rollup db test-id status) ???? (define (db:roll-up-pass-fail-counts db run-id test-name item-path status) ;; (cdb:flush-queue *runremote*) (if (and (not (equal? item-path "")) (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP"))) (begin (sqlite3:execute db "UPDATE tests - SET fail_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status='FAIL'), + SET fail_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status IN ('FAIL','CHECK')), pass_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status IN ('PASS','WARN','WAIVED')) WHERE run_id=? AND testname=? AND item_path='';" run-id test-name run-id test-name run-id test-name) ;; (thread-sleep! 0.1) ;; give other processes a chance here, no, better to be done ASAP? (if (equal? status "RUNNING") ;; running takes priority over all other states, force the test state to RUNNING @@ -1553,16 +1554,20 @@ WHEN (SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND state in ('RUNNING','NOT_STARTED')) > 0 THEN 'RUNNING' ELSE 'COMPLETED' END, - status=CASE - WHEN fail_count > 0 THEN 'FAIL' - WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' - ELSE 'UNKNOWN' END + status=CASE + WHEN fail_count > 0 THEN 'FAIL' + WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' + WHEN (SELECT count(id) FROM tests + WHERE run_id=? AND testname=? + AND item_path != '' + AND status = 'SKIP') > 0 THEN 'SKIP' + ELSE 'UNKNOWN' END WHERE run_id=? AND testname=? AND item_path='';" - run-id test-name run-id test-name)) + run-id test-name run-id test-name run-id test-name)) #f) #f)) ;;====================================================================== ;; Tests meta data @@ -1699,11 +1704,15 @@ ;; Now rollup the counts to the central megatest.db (cdb:pass-fail-counts *runremote* test-id fail-count pass-count) ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" ;; fail-count pass-count test-id) - (cdb:flush-queue *runremote*) + + ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines + ;; next time you read this! + ;; + ;; (cdb:flush-queue *runremote*) ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set ;; if the test is not FAIL then set status based on the fail and pass counts. (cdb:test-rollup-test_data-pass-fail *runremote* test-id) ;; (sqlite3:execute Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -22,10 +22,11 @@ (declare (uses common)) (declare (uses db)) (declare (uses tests)) (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. (declare (uses server)) +(declare (uses daemon)) (include "common_records.scm") (include "db_records.scm") (define (http-transport:make-server-url hostport) @@ -43,10 +44,22 @@ ;; Call this to start the actual server ;; (define *db:process-queue-mutex* (make-mutex)) +(define (server:get-best-guess-address hostname) + (let ((res #f)) + (for-each + (lambda (adr) + (if (not (eq? (u8vector-ref adr 0) 127)) + (set! res adr))) + (vector->list (hostinfo-addresses (hostname->hostinfo hostname)))) + (string-intersperse + (map number->string + (u8vector->list + (if res res (hostname->ip hostname)))) "."))) + (define (http-transport:run hostn) (debug:print 2 "Attempting to start the server ...") (if (not *toppath*) (if (not (setup-for-run)) (begin @@ -56,11 +69,12 @@ ;; #f ;; (get-host-name) ;; hostn)) (db #f) ;; (open-db)) ;; we don't want the server to be opening and closing the db unnecesarily (hostname (get-host-name)) (ipaddrstr (let ((ipstr (if (string=? "-" hostn) - (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") + ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") + (server:get-best-guess-address hostname) #f))) (if ipstr ipstr hostn))) ;; hostname))) (start-port (if (and (args:get-arg "-port") (string->number (args:get-arg "-port"))) (string->number (args:get-arg "-port")) @@ -118,11 +132,11 @@ exn (begin (print-error-message exn) (if (< portnum 9000) (begin - (print "WARNING: failed to start on portnum: " portnum ", trying next port") + (debug:print 0 "WARNING: failed to start on portnum: " portnum ", trying next port") (thread-sleep! 0.1) ;; (open-run-close tasks:remove-server-records tasks:open-db) (open-run-close tasks:server-delete tasks:open-db ipaddrstr portnum) (http-transport:try-start-server ipaddrstr (+ portnum 1))) (print "ERROR: Tried and tried but could not start the server"))) @@ -133,11 +147,12 @@ tasks:open-db (current-process-id) ipaddrstr portnum 0 'live 'http) (print "INFO: Trying to start server on " ipaddrstr ":" portnum) ;; This starts the spiffy server - (start-server port: portnum) + ;; NEED WAY TO SET IP TO #f TO BIND ALL + (start-server bind-address: ipaddrstr port: portnum) (open-run-close tasks:server-delete tasks:open-db ipaddrstr portnum) (print "INFO: server has been stopped"))) ;;====================================================================== ;; S E R V E R U T I L I T I E S @@ -187,16 +202,17 @@ (serverdat (list iface port))) (set! login-res (client:login serverdat)) (if (and (not (null? login-res)) (car login-res)) (begin - (debug:print-info 2 "Logged in and connected to " iface ":" port) + (debug:print-info 0 "Logged in and connected to " iface ":" port) (set! *runremote* serverdat) serverdat) (begin - (debug:print-info 2 "Failed to login or connect to " iface ":" port) + (debug:print-info 0 "Failed to login or connect to " iface ":" port) (set! *runremote* #f) + (set! *transport-type* 'fs) #f)))) ;; run http-transport:keep-running in a parallel thread to monitor that the db is being ;; used and to shutdown after sometime if it is not. @@ -216,17 +232,18 @@ (loop)))))) (iface (car server-info)) (port (cadr server-info)) (last-access 0) (tdb (tasks:open-db)) - (spid (tasks:server-get-server-id tdb #f iface port #f)) + (spid ;;(open-run-close tasks:server-get-server-id tasks:open-db #f iface port #f)) + (tasks:server-get-server-id tdb #f iface port #f)) (server-timeout (let ((tmo (config-lookup *configdat* "server" "timeout"))) (if (and (string? tmo) (string->number tmo)) (* 60 60 (string->number tmo)) ;; default to three days - (* 3 24 60))))) + (* 3 24 60 60))))) (debug:print-info 2 "server-timeout: " server-timeout ", server pid: " spid " on " iface ":" port) (let loop ((count 0)) (thread-sleep! 4) ;; no need to do this very often ;; NB// sync currently does NOT return queue-length (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1))) @@ -237,18 +254,20 @@ ;; Check that iface and port have not changed (can happen if server port collides) (mutex-lock! *heartbeat-mutex*) (set! sdat *runremote*) (mutex-unlock! *heartbeat-mutex*) - (if (not (equal? sdat (list iface port))) + (if (or (not (equal? sdat (list iface port))) + (not spid)) (begin - (debug:print-info 1 "interface changed, refreshing iface and port info") + (debug:print-info 0 "interface changed, refreshing iface and port info") (set! iface (car sdat)) (set! port (cadr sdat)) (set! spid (tasks:server-get-server-id tdb #f iface port #f)))) ;; NOTE: Get rid of this mechanism! It really is not needed... + ;; (open-run-close tasks:server-update-heartbeat tasks:open-db spid) (tasks:server-update-heartbeat tdb spid) ;; (if ;; (or (> numrunning 0) ;; stay alive for two days after last access (mutex-lock! *heartbeat-mutex*) (set! last-access *last-db-access*) @@ -261,11 +280,11 @@ (loop 0)) (begin (debug:print-info 0 "Starting to shutdown the server.") ;; need to delete only *my* server entry (future use) (set! *time-to-exit* #t) - (tasks:server-deregister-self tdb (get-host-name)) + (open-run-close tasks:server-deregister-self tasks:open-db (get-host-name)) (thread-sleep! 1) (debug:print-info 0 "Max cached queries was " *max-cache-size*) (debug:print-info 0 "Number of cached writes " *number-of-writes*) (debug:print-info 0 "Average cached write time " (if (eq? *number-of-writes* 0) @@ -312,10 +331,27 @@ (set! *didsomething* #t) (thread-join! th2)) (debug:print 0 "ERROR: Failed to setup for megatest"))) (exit))) +;; (use trace) +;; (trace http-transport:keep-running +;; tasks:server-update-heartbeat +;; tasks:server-get-server-id) +;; tasks:get-best-server +;; http-transport:run +;; http-transport:launch +;; http-transport:try-start-server +;; http-transport:client-send-receive +;; http-transport:make-server-url +;; tasks:server-register +;; tasks:server-delete +;; start-server +;; hostname->ip +;; with-input-from-request +;; tasks:server-deregister-self) + (define (http-transport:server-signal-handler signum) (handle-exceptions exn (debug:print " ... exiting ...") (let ((th1 (make-thread (lambda () Index: megatest-version.scm ================================================================== --- megatest-version.scm +++ megatest-version.scm @@ -1,7 +1,7 @@ ;; Always use two digit decimal ;; 1.01, 1.02...1.10,1.11 ... 1.99,2.00.. (declare (unit megatest-version)) -(define megatest-version 1.5415) +(define megatest-version 1.5417) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -12,11 +12,11 @@ (use sqlite3 srfi-1 posix regex regex-case srfi-69 base64 format readline apropos json) ;; (srfi 18) extras) (import (prefix sqlite3 sqlite3:)) (import (prefix base64 base64:)) -(use zmq) +;; (use zmq) (declare (uses common)) (declare (uses megatest-version)) (declare (uses margs)) (declare (uses runs)) @@ -23,10 +23,11 @@ (declare (uses launch)) (declare (uses server)) (declare (uses client)) (declare (uses tests)) (declare (uses genexample)) +(declare (uses daemon)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm") @@ -127,11 +128,11 @@ -env2file fname : write the environment to fname.csh and fname.sh -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are overwritten by values set in config files. -server -|hostname : start the server (reduces contention on megatest.db), use - to automatically figure out hostname - -transport http|zmq : use http or zmq for transport (default is http) + -transport http|fs : use http or direct access for transport (default is http) -daemonize : fork into background and disconnect from stdin/out -list-servers : list the servers -stop-server id : stop server specified by id (see output of -list-servers) -repl : start a repl (useful for extending megatest) -load file.scm : load and run file.scm @@ -313,21 +314,29 @@ (hash-table-keys args:arg-hash) '("-runtests" "-list-runs" "-rollup" "-remove-runs" "-lock" "-unlock" "-update-meta" "-extract-ods")))) (if (setup-for-run) - (let ((servers (open-run-close tasks:get-best-server tasks:open-db))) + (let loop ((servers (open-run-close tasks:get-best-server tasks:open-db)) + (trycount 0)) (if (or (not servers) (null? servers)) (begin - (debug:print 0 "INFO: Starting server as none running ...") - ;; (server:launch (string->symbol (args:get-arg "-transport" "http")))) - (system (conc (car (argv)) " -server - -daemonize -transport " (args:get-arg "-transport" "http"))) - (thread-sleep! 3)) ;; give the server a few seconds to start - (debug:print 0 "INFO: Servers already running " servers) + (if (eq? trycount 0) ;; just do the server start once + (begin + (debug:print 0 "INFO: Starting server as none running ...") + ;; (server:launch (string->symbol (args:get-arg "-transport" "http")))) + ;; (process-run (car (argv)) (list "-server" "-" "-daemonize" "-transport" (args:get-arg "-transport" "http"))) + (process-fork (lambda () + (daemon:ize) + (server:launch (string->symbol (args:get-arg "-transport" "http"))))) + (thread-sleep! 3)) + (debug:print-info 0 "Waiting for server to start")) + (loop (open-run-close tasks:get-best-server tasks:open-db) + (+ trycount 1))) + (debug:print 0 "INFO: Server(s) running " servers) ))))) - (if (or (args:get-arg "-list-servers") (args:get-arg "-stop-server")) (let ((tl (setup-for-run))) (if tl @@ -370,12 +379,11 @@ (debug:print-info 0 "Attempting to stop server with pid " pid) (tasks:kill-server status hostname pullport pid transport))))) servers) (debug:print-info 1 "Done with listservers") (set! *didsomething* #t) - (exit) ;; must do, would have to add checks to many/all calls below - ) + (exit)) ;; must do, would have to add checks to many/all calls below (exit))) ;; if not list or kill then start a client (if appropriate) (if (or (args-defined? "-h" "-version" "-gen-megatest-area" "-gen-megatest-test") (eq? (length (hash-table-keys args:arg-hash)) 0)) (debug:print-info 1 "Server connection not needed") @@ -500,12 +508,13 @@ ;; (print "[" targetstr "]")))) (print targetstr)))) (if (not db-targets) (let* ((run-id (db:get-value-by-header run header "id")) (tests (cdb:remote-run db:get-tests-for-run #f run-id testpatt '() '()))) - (debug:print 1 "Run: " targetstr " status: " (db:get-value-by-header run header "state") - " run-id: " run-id ", number tests: " (length tests)) + (print "Run: " targetstr "/" (db:get-value-by-header run header "runname") + " status: " (db:get-value-by-header run header "state") + " run-id: " run-id ", number tests: " (length tests)) (for-each (lambda (test) (format #t " Test: ~25a State: ~15a Status: ~15a Runtime: ~5@as Time: ~22a Host: ~10a\n" (conc (db:test-get-testname test) Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -8,11 +8,12 @@ ;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ;; PURPOSE. (require-extension (srfi 18) extras tcp s11n) -(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest zmq) +(use srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest) +;; (use zmq) (use spiffy uri-common intarweb http-client spiffy-request-vars) (declare (unit server)) @@ -19,11 +20,11 @@ (declare (uses common)) (declare (uses db)) (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. (declare (uses synchash)) (declare (uses http-transport)) -(declare (uses zmq-transport)) +;; (declare (uses zmq-transport)) (declare (uses daemon)) (include "common_records.scm") (include "db_records.scm") Index: tasks.scm ================================================================== --- tasks.scm +++ tasks.scm @@ -26,11 +26,11 @@ (let* ((dbpath (conc *toppath* "/monitor.db")) (exists (file-exists? dbpath)) (mdb (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) (handler (make-busy-timeout 36000))) (sqlite3:set-busy-handler! mdb handler) - (sqlite3:execute mdb (conc "PRAGMA synchronous = 0;")) + (sqlite3:execute mdb (conc "PRAGMA synchronous = 1;")) (if (not exists) (begin (sqlite3:execute mdb "CREATE TABLE IF NOT EXISTS tasks_queue (id INTEGER PRIMARY KEY, action TEXT DEFAULT '', owner TEXT, @@ -141,10 +141,11 @@ "SELECT id FROM servers WHERE pid=-999;"))) (if hostname hostname iface)(if pid pid port)) res)) (define (tasks:server-update-heartbeat mdb server-id) + (debug:print-info 0 "Heart beat update of server id=" server-id) (sqlite3:execute mdb "UPDATE servers SET heartbeat=strftime('%s','now') WHERE id=?;" server-id)) ;; alive servers keep the heartbeat field upto date with seconds every 6 or so seconds (define (tasks:server-alive? mdb server-id #!key (iface #f)(hostname #f)(port #f)(pid #f)) (let* ((server-id (if server-id Index: tests/Makefile ================================================================== --- tests/Makefile +++ tests/Makefile @@ -49,16 +49,16 @@ test4 : fullprep cd fullrun;$(MEGATEST) -debug $(DEBUG) -runall -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING) # NOTE: Only one instance can be a server test5 : fullprep - cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log & - cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ab -debug $(DEBUG) $(LOGGING) > ab.log 2> ab.log & - cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ac -debug $(DEBUG) $(LOGGING) > ac.log 2> ac.log & - cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ad -debug $(DEBUG) $(LOGGING) > ad.log 2> ad.log & -# cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ae -debug $(DEBUG) $(LOGGING) > ae.log 2> ae.log & -# cd fullrun;sleep 10;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_af -debug $(DEBUG) $(LOGGING) > af.log 2> af.log & + cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log & + cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ab -debug $(DEBUG) $(LOGGING) > ab.log 2> ab.log & + cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ac -debug $(DEBUG) $(LOGGING) > ac.log 2> ac.log & + cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ad -debug $(DEBUG) $(LOGGING) > ad.log 2> ad.log & +# cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ae -debug $(DEBUG) $(LOGGING) > ae.log 2> ae.log & +# cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_af -debug $(DEBUG) $(LOGGING) > af.log 2> af.log & test6: fullprep cd fullrun;$(MEGATEST) -runtests runfirst -testpatt %/1 -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_itempatt -v cd fullrun;$(MEGATEST) -runtests runfirst -testpatt %blahha% -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_itempatt -debug 10 cd fullrun;$(MEGATEST) -rollup :runname newrun -target ubuntu/nfs/none -debug 10 ADDED tests/fdktestqa/testqa/tests/bigrun2/step1.sh Index: tests/fdktestqa/testqa/tests/bigrun2/step1.sh ================================================================== --- /dev/null +++ tests/fdktestqa/testqa/tests/bigrun2/step1.sh @@ -0,0 +1,7 @@ +#!/bin/sh +prev_test=`$MT_MEGATEST -test-paths -target $MT_TARGET :runname $MT_RUNNAME -testpatt bigrun/$NUMBER` +if [ -e $prev_test/testconfig ]; then + exit 0 +else + exit 1 +fi ADDED tests/fdktestqa/testqa/tests/bigrun2/testconfig Index: tests/fdktestqa/testqa/tests/bigrun2/testconfig ================================================================== --- /dev/null +++ tests/fdktestqa/testqa/tests/bigrun2/testconfig @@ -0,0 +1,22 @@ +# Add additional steps here. Format is "stepname script" +[ezsteps] +step1 step1.sh + +# Test requirements are specified here +[requirements] +waiton bigrun +priority 0 +mode itemmatch + + +# Iteration for your tests are controlled by the items section +[items] +NUMBER #{scheme (string-intersperse (map number->string (sort (let loop ((a 0)(res '()))(if (< a 120)(loop (+ a 1)(cons a res)) res)) >)) " ")} + +# test_meta is a section for storing additional data on your test +[test_meta] +author matt +owner matt +description An example test +tags tagone,tagtwo +reviewed never ADDED tests/fslsync/megatest.config Index: tests/fslsync/megatest.config ================================================================== --- /dev/null +++ tests/fslsync/megatest.config @@ -0,0 +1,20 @@ +[fields] +YEAR TEXT +WEEKNUM TEXT +DAY TEXT + +[setup] +# Adjust max_concurrent_jobs to limit how much you load your machines +max_concurrent_jobs 50 + +# This is your link path, you can move it but it is generally better to keep it stable +linktree #{shell readlink -f #{getenv MT_RUN_AREA_HOME}/fslsynclinks} + +# Job tools are more advanced ways to control how your jobs are launched +[jobtools] +useshell yes +launcher nbfind + +# As you run more tests you may need to add additional disks, the names are arbitrary but must be unique +[disks] +disk0 #{shell readlink -f #{getenv MT_RUN_AREA_HOME}/fslsyncruns} ADDED tests/fslsync/runconfigs.config Index: tests/fslsync/runconfigs.config ================================================================== --- /dev/null +++ tests/fslsync/runconfigs.config @@ -0,0 +1,5 @@ +[default] +WORKAREA /tmp/#{getenv USER}/fslsync +FSLSAREA /tmp/#{getenv USER}/fsls +AREANAMES code data +SITENAMES #{shell cat $MT_RUN_AREA_HOME/sites.dat} ADDED tests/fslsync/sites.dat.template Index: tests/fslsync/sites.dat.template ================================================================== --- /dev/null +++ tests/fslsync/sites.dat.template @@ -0,0 +1,1 @@ +site1 ADDED tests/fslsync/tests/setup/mkdirs.logpro Index: tests/fslsync/tests/setup/mkdirs.logpro ================================================================== --- /dev/null +++ tests/fslsync/tests/setup/mkdirs.logpro @@ -0,0 +1,8 @@ +;; You should have at least one expect:required. This ensures that your process ran +(expect:required in "LogFileBody" > 0 "done" #/done/) + +;; You may need ignores to suppress false error or warning hits from the later expects +;; NOTE: Order is important here! +(expect:ignore in "LogFileBody" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +(expect:warning in "LogFileBody" = 0 "Any warning" #/warn/) +(expect:error in "LogFileBody" = 0 "Any error" (list #/ERROR/ #/error/)) ;; but disallow any other errors ADDED tests/fslsync/tests/setup/mkdirs.sh Index: tests/fslsync/tests/setup/mkdirs.sh ================================================================== --- /dev/null +++ tests/fslsync/tests/setup/mkdirs.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Create needed directories both local and remote + +# Remote +ssh $SITENAME mkdir -vp $WORKAREA/$SITENAME/$AREANAME + +# Local +mkdir -vp $WORKAREA/$SITENAME/$AREANAME + +echo done ADDED tests/fslsync/tests/setup/seedcache.logpro Index: tests/fslsync/tests/setup/seedcache.logpro ================================================================== --- /dev/null +++ tests/fslsync/tests/setup/seedcache.logpro @@ -0,0 +1,8 @@ +;; You should have at least one expect:required. This ensures that your process ran +(expect:required in "LogFileBody" > 0 "done" #/done/) + +;; You may need ignores to suppress false error or warning hits from the later expects +;; NOTE: Order is important here! +(expect:ignore in "LogFileBody" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +(expect:warning in "LogFileBody" = 0 "Any warning" #/warn/) +(expect:error in "LogFileBody" = 0 "Any error" (list #/ERROR/ #/error/)) ;; but disallow any other errors ADDED tests/fslsync/tests/setup/seedcache.sh Index: tests/fslsync/tests/setup/seedcache.sh ================================================================== --- /dev/null +++ tests/fslsync/tests/setup/seedcache.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Copy any non-existant files to the cache before doing the rsync +# in the hopes of saving some time. + +echo done ADDED tests/fslsync/tests/setup/testconfig Index: tests/fslsync/tests/setup/testconfig ================================================================== --- /dev/null +++ tests/fslsync/tests/setup/testconfig @@ -0,0 +1,21 @@ +# Add additional steps here. Format is "stepname script" +[ezsteps] +mkdirs mkdirs.sh +seedcache seedcache.sh + +# Test requirements are specified here +[requirements] +priority 0 + +# Iteration for your tests are controlled by the items section +[items] +AREANAME #{getenv AREANAMES} +SITENAME #{getenv SITENAMES} + +# test_meta is a section for storing additional data on your test +[test_meta] +author matt +owner matt +description Setup needed directories and seed the caches +tags tagone,tagtwo +reviewed never ADDED tests/fslsync/tests/sync/fsync.logpro Index: tests/fslsync/tests/sync/fsync.logpro ================================================================== --- /dev/null +++ tests/fslsync/tests/sync/fsync.logpro @@ -0,0 +1,8 @@ +;; You should have at least one expect:required. This ensures that your process ran +(expect:required in "LogFileBody" > 0 "done" #/done/) + +;; You may need ignores to suppress false error or warning hits from the later expects +;; NOTE: Order is important here! +(expect:ignore in "LogFileBody" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +(expect:warning in "LogFileBody" = 0 "Any warning" #/warn/) +(expect:error in "LogFileBody" = 0 "Any error" (list #/ERROR/ #/error/)) ;; but disallow any other errors ADDED tests/fslsync/tests/sync/fsync.sh Index: tests/fslsync/tests/sync/fsync.sh ================================================================== --- /dev/null +++ tests/fslsync/tests/sync/fsync.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Get the list of fossils from the cache + +FILES=$(ls $FSLSAREA/$AREANAME|grep fossil) + +# Do the remote sync from CACHE to FOSSILS +ssh $SITENAME /bin/bash < 0 "done" #/done/) + +;; You may need ignores to suppress false error or warning hits from the later expects +;; NOTE: Order is important here! +(expect:ignore in "LogFileBody" < 99 "Ignore the word error in comments" #/^\/\/.*error/) +(expect:warning in "LogFileBody" = 0 "Any warning" #/warn/) +(expect:error in "LogFileBody" = 0 "Any error" (list #/ERROR/ #/error/)) ;; but disallow any other errors ADDED tests/fslsync/tests/sync/rsync.sh Index: tests/fslsync/tests/sync/rsync.sh ================================================================== --- /dev/null +++ tests/fslsync/tests/sync/rsync.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Sync to remote cache +rsync -avz $FSLSAREA/$AREANAME/ $SITENAME:$WORKAREA/$SITENAME/$AREANAME/ & +# Sync to local cache +rsync -avz $SITENAME:$FSLSAREA/$AREANAME/ $WORKAREA/$SITENAME/$AREANAME/ & + +# Wait until rsyncs complete +wait + +echo done ADDED tests/fslsync/tests/sync/testconfig Index: tests/fslsync/tests/sync/testconfig ================================================================== --- /dev/null +++ tests/fslsync/tests/sync/testconfig @@ -0,0 +1,22 @@ +# Add additional steps here. Format is "stepname script" +[ezsteps] +rsync rsync.sh +fsync fsync.sh + +# Test requirements are specified here +[requirements] +waiton setup +priority 0 + +# Iteration for your tests are controlled by the items section +[items] +AREANAME #{getenv AREANAMES} +SITENAME #{getenv SITENAMES} + +# test_meta is a section for storing additional data on your test +[test_meta] +author matt +owner matt +description Sync fossils to remote +tags tagone,tagtwo +reviewed never ADDED utils/example-launch-dispatcher.scm Index: utils/example-launch-dispatcher.scm ================================================================== --- /dev/null +++ utils/example-launch-dispatcher.scm @@ -0,0 +1,12 @@ + +(let ((target (assoc + ;; Put the variable name here, note: only *one* ' + ;; 'TARGET_OS + 'MANYITEMS + (read (open-input-string (get-environment-variable "MT_ITEM_INFO")))))) + (case (if target target 'var-undef) + ((suse) (system "echo suse-launcher.pl")) + ((redhat) (system "echo red-hat-launcher.pl")) + ((af) (system "echo Got af")) + ((var-undef) (system "echo Variable not in MT_ITEM_INFO list")) + (else (system "echo normal-launcher.pl")))) Index: utils/installall.sh ================================================================== --- utils/installall.sh +++ utils/installall.sh @@ -164,137 +164,137 @@ # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -feature disable-iup-web -deploy -prefix $DEPLOYTARG iup # iup:1.0.2 CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks canvas-draw # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -D no-library-checks -deploy -prefix $DEPLOYTARG canvas-draw -#====================================================================== -# Note uuid needed only for zmq 2.x series -#====================================================================== - -# http://download.zeromq.org/zeromq-3.2.1-rc2.tar.gz -# zpatchlev=-rc2 -# http://download.zeromq.org/zeromq-2.2.0.tar.gz - -if [[ -e /usr/lib/libzmq.so ]]; then - echo "Using system installed zmq library" - $CHICKEN_INSTALL zmq -else -ZEROMQ=zeromq-2.2.0 -# ZEROMQ=zeromq-3.2.2 - -# wget http://www.kernel.org/pub/linux/utils/util-linux/v2.22/util-linux-2.22.tar.gz -UTIL_LINUX=2.21 -# UTIL_LINUX=2.20.1 -if ! [[ -e util-linux-${UTIL_LINUX}.tar.gz ]] ; then - # wget http://www.kiatoa.com/matt/util-linux-2.20.1.tar.gz - wget http://www.kernel.org/pub/linux/utils/util-linux/v${UTIL_LINUX}/util-linux-${UTIL_LINUX}.tar.gz -fi - -if [[ -e util-linux-${UTIL_LINUX}.tar.gz ]] ; then - tar xfz util-linux-${UTIL_LINUX}.tar.gz - cd util-linux-${UTIL_LINUX} - mkdir -p build - cd build - if [[ $UTIL_LINUX = "2.22" ]] ; then - ../configure --prefix=$PREFIX \ ---enable-shared \ ---disable-use-tty-group \ ---disable-makeinstall-chown \ ---disable-makeinstall-setuid \ ---disable-libtool-lock \ ---disable-login \ ---disable-sulogin \ ---disable-su \ ---disable-schedutils \ ---disable-libmount \ ---disable-mount \ ---disable-losetup \ ---disable-fsck \ ---disable-partx \ ---disable-mountpoint \ ---disable-fallocate \ ---disable-unshare \ ---disable-eject \ ---disable-agetty \ ---disable-cramfs \ ---disable-switch_root \ ---disable-pivot_root \ ---disable-kill \ ---disable-libblkid \ ---disable-utmpdump \ ---disable-rename \ ---disable-chsh-only-listed \ ---disable-wall \ ---disable-pg-bell \ ---disable-require-password \ ---disable-libtool-lock \ ---disable-nls \ ---disable-dmesg \ ---without-ncurses - else - ../configure --prefix=$PREFIX \ - --enable-shared \ - --disable-mount \ - --disable-fsck \ - --disable-partx \ - --disable-largefile \ - --disable-tls \ - --disable-libmount \ - --disable-mountpoint \ - --disable-nls \ - --disable-rpath \ - --disable-agetty \ - --disable-cramfs \ - --disable-switch_root \ - --disable-pivot_root \ - --disable-fallocate \ - --disable-unshare \ - --disable-rename \ - --disable-schedutils \ - --disable-libblkid \ - --disable-wall CFLAGS='-fPIC' - -# --disable-makeinstall-chown \ -# --disable-makeinstall-setuid \ - -# --disable-chsh-only-listed -# --disable-pg-bell let pg not ring the bell on invalid keys -# --disable-require-password -# --disable-use-tty-group do not install wall and write setgid tty -# --disable-makeinstall-chown -# --disable-makeinstall-setuid - fi - - (cd libuuid;make install) - # make - # make install - cp $PREFIX/include/uuid/uuid.h $PREFIX/include/uuid.h -fi - - -cd $BUILDHOME - -if ! [[ -e ${ZEROMQ}${zpatchlev}.tar.gz ]] ; then - wget http://download.zeromq.org/${ZEROMQ}${zpatchlev}.tar.gz -fi - -if [[ -e ${ZEROMQ}${zpatchlev}.tar.gz ]] ; then - tar xfz ${ZEROMQ}.tar.gz - cd ${ZEROMQ} - ln -s $PREFIX/include/uuid src - # LDFLAGS=-L$PREFIX/lib ./configure --prefix=$PREFIX - - ./configure --enable-static --prefix=$PREFIX --with-uuid=$PREFIX LDFLAGS="-L$PREFIX/lib" CPPFLAGS="-fPIC -I$PREFIX/include" LIBS="-lgcc" - # --disable-shared CPPFLAGS="-fPIC - # LDFLAGS="-L/usr/lib64 -L$PREFIX/lib" ./configure --enable-static --prefix=$PREFIX - make - make install - CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX zmq - # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -deploy -prefix $DEPLOYTARG zmq -fi -fi # if zmq is in /usr/lib - +# disabled zmq # #====================================================================== +# disabled zmq # # Note uuid needed only for zmq 2.x series +# disabled zmq # #====================================================================== +# disabled zmq # +# disabled zmq # # http://download.zeromq.org/zeromq-3.2.1-rc2.tar.gz +# disabled zmq # # zpatchlev=-rc2 +# disabled zmq # # http://download.zeromq.org/zeromq-2.2.0.tar.gz +# disabled zmq # +# disabled zmq # if [[ -e /usr/lib/libzmq.so ]]; then +# disabled zmq # echo "Using system installed zmq library" +# disabled zmq # $CHICKEN_INSTALL zmq +# disabled zmq # else +# disabled zmq # ZEROMQ=zeromq-2.2.0 +# disabled zmq # # ZEROMQ=zeromq-3.2.2 +# disabled zmq # +# disabled zmq # # wget http://www.kernel.org/pub/linux/utils/util-linux/v2.22/util-linux-2.22.tar.gz +# disabled zmq # UTIL_LINUX=2.21 +# disabled zmq # # UTIL_LINUX=2.20.1 +# disabled zmq # if ! [[ -e util-linux-${UTIL_LINUX}.tar.gz ]] ; then +# disabled zmq # # wget http://www.kiatoa.com/matt/util-linux-2.20.1.tar.gz +# disabled zmq # wget http://www.kernel.org/pub/linux/utils/util-linux/v${UTIL_LINUX}/util-linux-${UTIL_LINUX}.tar.gz +# disabled zmq # fi +# disabled zmq # +# disabled zmq # if [[ -e util-linux-${UTIL_LINUX}.tar.gz ]] ; then +# disabled zmq # tar xfz util-linux-${UTIL_LINUX}.tar.gz +# disabled zmq # cd util-linux-${UTIL_LINUX} +# disabled zmq # mkdir -p build +# disabled zmq # cd build +# disabled zmq # if [[ $UTIL_LINUX = "2.22" ]] ; then +# disabled zmq # ../configure --prefix=$PREFIX \ +# disabled zmq # --enable-shared \ +# disabled zmq # --disable-use-tty-group \ +# disabled zmq # --disable-makeinstall-chown \ +# disabled zmq # --disable-makeinstall-setuid \ +# disabled zmq # --disable-libtool-lock \ +# disabled zmq # --disable-login \ +# disabled zmq # --disable-sulogin \ +# disabled zmq # --disable-su \ +# disabled zmq # --disable-schedutils \ +# disabled zmq # --disable-libmount \ +# disabled zmq # --disable-mount \ +# disabled zmq # --disable-losetup \ +# disabled zmq # --disable-fsck \ +# disabled zmq # --disable-partx \ +# disabled zmq # --disable-mountpoint \ +# disabled zmq # --disable-fallocate \ +# disabled zmq # --disable-unshare \ +# disabled zmq # --disable-eject \ +# disabled zmq # --disable-agetty \ +# disabled zmq # --disable-cramfs \ +# disabled zmq # --disable-switch_root \ +# disabled zmq # --disable-pivot_root \ +# disabled zmq # --disable-kill \ +# disabled zmq # --disable-libblkid \ +# disabled zmq # --disable-utmpdump \ +# disabled zmq # --disable-rename \ +# disabled zmq # --disable-chsh-only-listed \ +# disabled zmq # --disable-wall \ +# disabled zmq # --disable-pg-bell \ +# disabled zmq # --disable-require-password \ +# disabled zmq # --disable-libtool-lock \ +# disabled zmq # --disable-nls \ +# disabled zmq # --disable-dmesg \ +# disabled zmq # --without-ncurses +# disabled zmq # else +# disabled zmq # ../configure --prefix=$PREFIX \ +# disabled zmq # --enable-shared \ +# disabled zmq # --disable-mount \ +# disabled zmq # --disable-fsck \ +# disabled zmq # --disable-partx \ +# disabled zmq # --disable-largefile \ +# disabled zmq # --disable-tls \ +# disabled zmq # --disable-libmount \ +# disabled zmq # --disable-mountpoint \ +# disabled zmq # --disable-nls \ +# disabled zmq # --disable-rpath \ +# disabled zmq # --disable-agetty \ +# disabled zmq # --disable-cramfs \ +# disabled zmq # --disable-switch_root \ +# disabled zmq # --disable-pivot_root \ +# disabled zmq # --disable-fallocate \ +# disabled zmq # --disable-unshare \ +# disabled zmq # --disable-rename \ +# disabled zmq # --disable-schedutils \ +# disabled zmq # --disable-libblkid \ +# disabled zmq # --disable-wall CFLAGS='-fPIC' +# disabled zmq # +# disabled zmq # # --disable-makeinstall-chown \ +# disabled zmq # # --disable-makeinstall-setuid \ +# disabled zmq # +# disabled zmq # # --disable-chsh-only-listed +# disabled zmq # # --disable-pg-bell let pg not ring the bell on invalid keys +# disabled zmq # # --disable-require-password +# disabled zmq # # --disable-use-tty-group do not install wall and write setgid tty +# disabled zmq # # --disable-makeinstall-chown +# disabled zmq # # --disable-makeinstall-setuid +# disabled zmq # fi +# disabled zmq # +# disabled zmq # (cd libuuid;make install) +# disabled zmq # # make +# disabled zmq # # make install +# disabled zmq # cp $PREFIX/include/uuid/uuid.h $PREFIX/include/uuid.h +# disabled zmq # fi +# disabled zmq # +# disabled zmq # +# disabled zmq # cd $BUILDHOME +# disabled zmq # +# disabled zmq # if ! [[ -e ${ZEROMQ}${zpatchlev}.tar.gz ]] ; then +# disabled zmq # wget http://download.zeromq.org/${ZEROMQ}${zpatchlev}.tar.gz +# disabled zmq # fi +# disabled zmq # +# disabled zmq # if [[ -e ${ZEROMQ}${zpatchlev}.tar.gz ]] ; then +# disabled zmq # tar xfz ${ZEROMQ}.tar.gz +# disabled zmq # cd ${ZEROMQ} +# disabled zmq # ln -s $PREFIX/include/uuid src +# disabled zmq # # LDFLAGS=-L$PREFIX/lib ./configure --prefix=$PREFIX +# disabled zmq # +# disabled zmq # ./configure --enable-static --prefix=$PREFIX --with-uuid=$PREFIX LDFLAGS="-L$PREFIX/lib" CPPFLAGS="-fPIC -I$PREFIX/include" LIBS="-lgcc" +# disabled zmq # # --disable-shared CPPFLAGS="-fPIC +# disabled zmq # # LDFLAGS="-L/usr/lib64 -L$PREFIX/lib" ./configure --enable-static --prefix=$PREFIX +# disabled zmq # make +# disabled zmq # make install +# disabled zmq # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX zmq +# disabled zmq # # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" $CHICKEN_INSTALL $PROX -deploy -prefix $DEPLOYTARG zmq +# disabled zmq # fi +# disabled zmq # fi # if zmq is in /usr/lib +# disabled zmq # cd $BUILDHOME ## WEBKIT=WebKit-r131972 ## if ! [[ -e ${WEBKIT}.tar.bz2 ]] ; then ## # http://builds.nightly.webkit.org/files/trunk/src/WebKit-r131972.tar.bz2 Index: utils/mk_wrapper ================================================================== --- utils/mk_wrapper +++ utils/mk_wrapper @@ -3,11 +3,14 @@ prefix=$1 cmd=$2 echo "#!/bin/bash" if [ "$LD_LIBRARY_PATH" != "" ];then + echo "INFO: Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >&2 echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" +else + echo "INFO: LD_LIBRARY_PATH not set" >&2 fi fullcmd="$prefix/bin/$cmd" echo "$fullcmd \"\$@\""