Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -41,19 +41,19 @@ (define (client:logout serverdat area-dat) (let ((ok (and (socket? serverdat) (cdb:logout serverdat (megatest:area-path area-dat) (client:get-signature))))) ok)) -(define (client:connect iface port) - (case (server:get-transport) +(define (client:connect iface port area-dat) + (case (server:get-transport area-dat) ((rpc) (rpc:client-connect iface port)) ((http) (http:client-connect iface port)) ((zmq) (zmq:client-connect iface port)) (else (rpc:client-connect iface port)))) (define (client:setup run-id area-dat #!key (remaining-tries 10) (failed-connects 0)) - (case (server:get-transport) + (case (server:get-transport area-dat) ((rpc) (rpc-transport:client-setup run-id area-dat)) ((http)(client:setup-http run-id area-dat)) (else (rpc-transport:client-setup run-id area-dat)))) ;; (define (client:login-no-auto-setup server-info run-id) @@ -203,11 +203,11 @@ ))) (begin ;; no server registered (let ((num-available (tasks:num-in-available-state (db:dbdat-get-db tdbdat) run-id))) (debug:print-info 0 "client:setup, no server registered, remaining-tries=" remaining-tries " num-available=" num-available) (if (< num-available 2) - (server:try-running run-id)) + (server:try-running run-id area-dat)) (thread-sleep! (+ 5 (random (- 20 remaining-tries)))) ;; give server a little time to start up, randomize a little to avoid start storms. (client:setup run-id area-dat remaining-tries: (- remaining-tries 1))))))))) ;; keep this as a function to ease future (define (client:start run-id server-info) Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -279,12 +279,12 @@ (let* ((configdat (megatest:area-configdat area-dat)) (run-ids (hash-table-keys *db-local-sync*))) (if (and (not (null? run-ids)) (configf:lookup configdat "setup" "megatest-db")) (db:multi-db-sync run-ids 'new2old))) - (if *dbstruct-db* (db:close-all *dbstruct-db*)) - (if *inmemdb* (db:close-all *inmemdb*)) + (if *dbstruct-db* (db:close-all *dbstruct-db* area-dat)) + (if *inmemdb* (db:close-all *inmemdb* area-dat)) (if (and *megatest-db* (sqlite3:database? *megatest-db*)) (begin (sqlite3:interrupt! *megatest-db*) (sqlite3:finalize! *megatest-db* #t) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -279,11 +279,11 @@ (dbr:dbstruct-set-olddb! dbstruct olddb) ;; olddb is already a (cons db path) dbdat)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; -(define (db:setup run-id #!key (local #f)) +(define (db:setup run-id area-dat #!key (local #f)) (let* ((dbdir (db:dbfile-path #f area-dat)) ;; (conc (configf:lookup configdat "setup" "linktree") "/.db")) (dbstruct (make-dbr:dbstruct path: dbdir local: local))) dbstruct)) ;; Open the classic megatest.db file in toppath Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -446,14 +446,14 @@ (let ((new-server-id (tasks:server-am-i-the-server? (db:delay-if-busy tdbdat area-dat) run-id))) ;; try to ensure no double registering of servers (if (equal? new-server-id server-id) (begin (tasks:server-set-state! (db:delay-if-busy tdbdat area-dat) server-id "dbprep") (thread-sleep! 0.5) ;; give some margin for queries to complete before switching from file based access to server based access - (set! *inmemdb* (db:setup run-id)) + (set! *inmemdb* (db:setup run-id area-dat)) ;; force initialization ;; (db:get-db *inmemdb* #t) - (db:get-db *inmemdb* run-id) + (db:get-db *inmemdb* area-dat run-id) (tasks:server-set-state! (db:delay-if-busy tdbdat area-dat) server-id "running")) (begin ;; gotta exit nicely (tasks:server-set-state! (db:delay-if-busy tdbdat area-dat) server-id "collision") (http-transport:server-shutdown server-id port area-dat)))))) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -635,11 +635,11 @@ )) (if (args:get-arg "-ping") (let* ((run-id (string->number (args:get-arg "-run-id"))) (host:port (args:get-arg "-ping"))) - (server:ping run-id host:port))) + (server:ping run-id host:port *area-dat*))) ;;====================================================================== ;; Start the server - can be done in conjunction with -runall or -runtests (one day...) ;; we start the server if not running else start the client thread ;;====================================================================== Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -76,11 +76,11 @@ (let ((cinfo (common:get-remote remote run-id))) (if cinfo cinfo ;; NB// can cache the answer for server running for 10 seconds ... ;; ;; (and (not (rmt:write-frequency-over-limit? cmd run-id)) - (if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db area-dat)) run-id) + (if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db area-dat) area-dat) run-id) (client:setup run-id area-dat remote: remote) #f)))) (define (rmt:discard-old-connections area-dat) ;; clean out old connections @@ -165,16 +165,16 @@ (tasks:start-and-wait-for-server (db:delay-if-busy (tasks:open-db area-dat)) run-id 10) (thread-sleep! (random 5)) ;; give some time to settle and minimize collison? (rmt:send-receive cmd rid params area-dat attemptnum: (+ attemptnum 1))) (begin (server:kind-run run-id area-dat) - (rmt:open-qry-close-locally cmd run-id params area-dat)))) + (rmt:open-qry-close-locally cmd run-id area-dat params area-dat)))) (begin ;; (debug:print 0 "ERROR: Communication failed!") ;; (mutex-unlock! *send-receive-mutex*) ;; (exit) - (rmt:open-qry-close-locally cmd run-id params area-dat) + (rmt:open-qry-close-locally cmd run-id area-dat params area-dat) ))))) (define (rmt:update-db-stats run-id rawcmd params duration) (mutex-lock! *db-stats-mutex*) (handle-exceptions @@ -230,30 +230,30 @@ (cons 'none 0)) (loop (car tal)(cdr tal) newmax-cmd currmax))))))) (mutex-unlock! *db-stats-mutex*) res)) -(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5)) +(define (rmt:open-qry-close-locally cmd run-id area-dat params #!key (remretries 5)) (let* ((dbstruct-local (if *dbstruct-db* *dbstruct-db* - (let* ((dbdir (db:dbfile-path #f)) + (let* ((dbdir (db:dbfile-path #f area-dat)) (db (make-dbr:dbstruct path: dbdir local: #t))) (set! *dbstruct-db* db) db))) - (db-file-path (db:dbfile-path 0)) + (db-file-path (db:dbfile-path 0 area-dat)) ;; (read-only (not (file-read-access? db-file-path))) (start (current-milliseconds)) - (resdat (api:execute-requests dbstruct-local (vector (symbol->string cmd) params))) + (resdat (api:execute-requests dbstruct-local area-dat (vector (symbol->string cmd) params))) (success (vector-ref resdat 0)) (res (vector-ref resdat 1)) (duration (- (current-milliseconds) start))) (if (not success) (if (> remretries 0) (begin (debug:print 0 "ERROR: local query failed. Trying again.") (thread-sleep! (/ (random 5000) 1000)) ;; some random delay - (rmt:open-qry-close-locally cmd run-id params remretries: (- remretries 1))) + (rmt:open-qry-close-locally cmd run-id area-dat params remretries: (- remretries 1))) (begin (debug:print 0 "ERROR: too many retries in rmt:open-qry-close-locally, giving up") #f)) (begin (rmt:update-db-stats run-id cmd params duration) Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -94,11 +94,11 @@ (define (runs:set-megatest-env-vars run-id area-dat #!key (inkeys #f)(inrunname #f)(inkeyvals #f)) (let* ((configdat (megatest:area-configdat area-dat)) (toppath (megatest:area-path area-dat)) (target (or (common:args-get-target) (get-environment-variable "MT_TARGET"))) - (keys (if inkeys inkeys (rmt:get-keys))) + (keys (if inkeys inkeys (rmt:get-keys area-dat))) (keyvals (if inkeyvals inkeyvals (keys:target->keyval keys target))) (vals (hash-table-ref/default *env-vars-by-run-id* run-id #f)) (link-tree (configf:lookup configdat "setup" "linktree"))) ;; get the info from the db and put it in the cache (if link-tree @@ -119,11 +119,11 @@ (debug:print 2 "setenv " key " " val) (safe-setenv key val))) (if (not (get-environment-variable "MT_TARGET"))(setenv "MT_TARGET" target)) (alist->env-vars (hash-table-ref/default configdat "env-override" '())) ;; Lets use this as an opportunity to put MT_RUNNAME in the environment - (let ((runname (if inrunname inrunname (rmt:get-run-name-from-id run-id)))) + (let ((runname (if inrunname inrunname (rmt:get-run-name-from-id run-id area-dat)))) (if runname (setenv "MT_RUNNAME" runname) (debug:print 0 "ERROR: no value for runname for id " run-id))) (setenv "MT_RUN_AREA_HOME" toppath))) @@ -169,12 +169,12 @@ (if (runs:lownoise "waiting on tasks" 60) (debug:print-info 2 "waiting for tasks to complete, sleeping briefly ...")) 2);; obviously haven't had any work to do for a while (else 0))) (let* ((configdat (megatest:area-configdat area-dat)) - (num-running (rmt:get-count-tests-running run-id)) - (num-running-in-jobgroup (rmt:get-count-tests-running-in-jobgroup run-id jobgroup)) + (num-running (rmt:get-count-tests-running run-id area-dat)) + (num-running-in-jobgroup (rmt:get-count-tests-running-in-jobgroup run-id jobgroup area-dat)) (job-group-limit (let ((jobg-count (config-lookup configdat "jobgroups" jobgroup))) (if (string? jobg-count) (string->number jobg-count) jobg-count)))) (if (> (+ num-running num-running-in-jobgroup) 0) @@ -236,12 +236,12 @@ (rmt:tasks-set-state-given-param-key task-key "killed")) (print "Killed by signal " signum ". Exiting") (exit))) ;; register this run in monitor.db - (rmt:tasks-add "run-tests" user target runname test-patts task-key) ;; params) - (rmt:tasks-set-state-given-param-key task-key "running") + (rmt:tasks-add "run-tests" user target runname test-patts task-key area-dat) ;; params) + (rmt:tasks-set-state-given-param-key task-key "running" area-dat) (runs:set-megatest-env-vars run-id area-dat inkeys: keys inrunname: runname) ;; these may be needed by the launching process (if (file-exists? runconfigf) (setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals target) (debug:print 0 "WARNING: You do not have a run config file: " runconfigf)) @@ -274,11 +274,11 @@ ;; (rmt:general-call 'delete-tests-in-state run-id "NOT_STARTED") ;; Now convert FAIL and anything in allow-auto-rerun to NOT_STARTED ;; (for-each (lambda (state) - (rmt:set-tests-state-status run-id test-names state #f "NOT_STARTED" state)) + (rmt:set-tests-state-status run-id test-names state #f "NOT_STARTED" state area-dat)) (string-split (or (configf:lookup configdat "setup" "allow-auto-rerun") ""))))) ;; Ensure all tests are registered in the test_meta table (runs:update-all-test_meta #f) @@ -391,17 +391,17 @@ (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry area-dat)))) (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry area-dat))) "runs:run-tests-queue")) (th2 (make-thread (lambda () ;; (rmt:find-and-mark-incomplete-all-runs))))) CAN'T INTERRUPT IT ... - (let ((run-ids (rmt:get-all-run-ids))) + (let ((run-ids (rmt:get-all-run-ids area-dat))) (for-each (lambda (run-id) (if keep-going (handle-exceptions exn (debug:print 0 "error in calling find-and-mark-incomplete for run-id " run-id) - (rmt:find-and-mark-incomplete run-id #f)))) ;; ovr-deadtime))) + (rmt:find-and-mark-incomplete run-id #f area-dat)))) ;; ovr-deadtime))) run-ids))) "runs: mark-incompletes"))) (thread-start! th1) (thread-start! th2) (thread-join! th1) @@ -415,11 +415,11 @@ (if (not (hash-table-ref/default flags "-rerun" #f)) (hash-table-set! flags "-rerun" "STUCK/DEAD,n/a,ZERO_ITEMS")) (runs:run-tests target runname test-patts user flags area-dat run-count: (- run-count 1))))) (debug:print-info 0 "No tests to run"))) (debug:print-info 4 "All done by here") - (rmt:tasks-set-state-given-param-key task-key "done") + (rmt:tasks-set-state-given-param-key task-key "done" area-dat) ;; (sqlite3:finalize! tasks-db) )) ;; loop logic. These are used in runs:run-tests-queue to make it a bit more readable. @@ -463,11 +463,11 @@ (define runs:nothing-left-in-queue-count 0) (define (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records itemmap) (let* ((loop-list (list hed tal reg reruns)) - (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode itemmap: itemmap)) + (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode area-dat itemmap: itemmap)) ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap)) (fails (runs:calc-fails prereqs-not-met)) (prereq-fails (runs:calc-prereq-fail prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met)) (runnables (runs:calc-runnable prereqs-not-met))) @@ -522,11 +522,11 @@ (runs:set-megatest-env-vars run-id area-dat inrunname: runname) ;; these may be needed by the launching process (let ((items-list (items:get-items-from-config tconfig))) (if (list? items-list) (begin (if (null? items-list) - (let ((test-id (rmt:get-test-id run-id test-name ""))) + (let ((test-id (rmt:get-test-id run-id test-name "" area-dat))) (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "ZERO_ITEMS" "Failed to run due to failed prerequisites")))) (tests:testqueue-set-items! test-record items-list) (list hed tal reg reruns)) (begin (debug:print 0 "ERROR: The proc from reading the items table did not yield a list - please report this") @@ -560,11 +560,11 @@ (not (and (null? tal)(null? reg)))) (let ((trimmed-tal (mt:discard-blocked-tests run-id hed tal test-records)) (trimmed-reg (mt:discard-blocked-tests run-id hed reg test-records))) (debug:print 1 "WARNING: test " hed " has discarded prerequisites, removing it from the queue") - (let ((test-id (rmt:get-test-id run-id hed ""))) + (let ((test-id (rmt:get-test-id run-id hed "" area-dat))) (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to discarded prerequisites"))) (if (and (null? trimmed-tal) (null? trimmed-reg)) #f @@ -586,11 +586,11 @@ ;; num-retries code was here ;; we use this opportunity to move contents of reg to tal (list (car newtal)(append (cdr newtal) reg) '() reruns)) ;; an issue with prereqs not yet met? (begin (debug:print-info 1 "no fails in prerequisites for " hed " but nothing seen running in a while, dropping test " hed " from the run queue") - (let ((test-id (rmt:get-test-id run-id hed ""))) + (let ((test-id (rmt:get-test-id run-id hed "" area-dat))) (if test-id (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "TIMED_OUT" "Nothing seen running in a while."))) (list (runs:queue-next-hed tal reg reglen regfull) (runs:queue-next-tal tal reg reglen regfull) (runs:queue-next-reg tal reg reglen regfull) reruns)))) @@ -600,11 +600,11 @@ (not (null? prereq-fails))) (member 'normal testmode)) (debug:print-info 1 "test " hed " (mode=" testmode ") has failed prerequisite(s); " (string-intersperse (map (lambda (t)(conc (db:test-get-testname t) ":" (db:test-get-state t)"/"(db:test-get-status t))) fails) ", ") ", removing it from to-do list") - (let ((test-id (rmt:get-test-id run-id hed ""))) + (let ((test-id (rmt:get-test-id run-id hed "" area-dat))) (if test-id (if (not (null? prereq-fails)) (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites") (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL" "Failed to run due to failed prerequisites")))) (if (or (not (null? reg))(not (null? tal))) @@ -653,11 +653,11 @@ (have-resources (car run-limits-info)) (num-running (list-ref run-limits-info 1)) (num-running-in-jobgroup (list-ref run-limits-info 2)) (max-concurrent-jobs (list-ref run-limits-info 3)) (job-group-limit (list-ref run-limits-info 4)) - (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode itemmap: itemmap)) + (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode area-dat itemmap: itemmap)) ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap)) (fails (runs:calc-fails prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met)) (loop-list (list hed tal reg reruns)) ;; configure the load runner @@ -699,21 +699,21 @@ ;; ((not (hash-table-ref/default test-registry (db:test-make-full-name test-name item-path) #f)) (debug:print-info 4 "Pre-registering test " test-name "/" item-path " to create placeholder" ) ;; always do firm registration now in v1.60 and greater ;; (eq? *transport-type* 'fs) ;; no point in parallel registration if use fs (let register-loop ((numtries 15)) - (rmt:general-call 'register-test run-id run-id test-name item-path) + (rmt:general-call 'register-test run-id area-dat run-id test-name item-path) (if (rmt:get-test-id run-id test-name item-path) (hash-table-set! test-registry (db:test-make-full-name test-name item-path) 'done) (if (> numtries 0) (begin (thread-sleep! 0.5) (register-loop (- numtries 1))) (debug:print 0 "ERROR: failed to register test " (db:test-make-full-name test-name item-path))))) (if (not (eq? (hash-table-ref/default test-registry (db:test-make-full-name test-name "") #f) 'done)) (begin - (rmt:general-call 'register-test run-id run-id test-name "") + (rmt:general-call 'register-test run-id area-dat run-id test-name "") (if (rmt:get-test-id run-id test-name "") (hash-table-set! test-registry (db:test-make-full-name test-name "") 'done)))) (runs:shrink-can-run-more-tests-count) ;; DELAY TWEAKER (still needed?) (if (and (null? tal)(null? reg)) (list hed tal (append reg (list hed)) reruns) @@ -983,11 +983,11 @@ ;; Ensure all top level tests get registered. This way they show up as "NOT_STARTED" on the dashboard ;; and it is clear they *should* have run but did not. (if (not (hash-table-ref/default test-registry (db:test-make-full-name test-name "") #f)) (begin - (rmt:general-call 'register-test run-id run-id test-name "" area-dat) + (rmt:general-call 'register-test run-id area-dat run-id test-name "" area-dat) (hash-table-set! test-registry (db:test-make-full-name test-name "") 'done))) ;; Fast skip of tests that are already "COMPLETED" - NO! Cannot do that as the items may not have been expanded yet :( ;; (if (member (hash-table-ref/default test-registry tfullname #f) @@ -1266,11 +1266,11 @@ ;; (if (not test-id)(set! test-id (rmt:get-test-id run-id test-name item-path area-dat))) (if (not test-id) (begin (debug:print 2 "WARN: Test not pre-created? test-name=" test-name ", item-path=" item-path ", run-id=" run-id) - (rmt:general-call 'register-test run-id run-id test-name item-path area-dat) + (rmt:general-call 'register-test run-id area-dat run-id test-name item-path area-dat) (set! test-id (rmt:get-test-id run-id test-name item-path area-dat)))) (debug:print-info 4 "test-id=" test-id ", run-id=" run-id ", test-name=" test-name ", item-path=\"" item-path "\"") (set! testdat (rmt:get-test-info-by-id run-id test-id area-dat)) (if (not testdat) (begin Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -63,11 +63,11 @@ ;;====================================================================== ;; Get the transport (define (server:get-transport area-dat) (if (megatest:area-transport area-dat) - (megatest-area-transport area-dat) + (megatest:area-transport area-dat) (let ((ttype (string->symbol (or (args:get-arg "-transport") (configf:lookup (megatest:area-configdat area-dat) "server" "transport") "rpc")))) (megatest:area-transport-set! area-dat ttype)