Index: NOTES ================================================================== --- NOTES +++ NOTES @@ -30,10 +30,45 @@ a. Might talk to running server if run specific b. Can talk to megatest.db but not a generally good idea c. Can start a runserver 4. Dashboard is fine except for writes? +====================================================================== +Routines to convert for runs.scm + +cdb:remote-run db:register-run + +cdb:delete-tests-in-state *runremote* +cdb:get-test-info-by-id *runremote* +cdb:remote-run db:delete-old-deleted-test-records +cdb:remote-run db:delete-run +cdb:remote-run db:delete-test-records +cdb:remote-run db:delete-tests-for-run +cdb:remote-run db:find-and-mark-incomplete +cdb:remote-run db:get-count-tests-running +cdb:remote-run db:get-count-tests-running-in-jobgroup +cdb:remote-run db:get-keys +cdb:remote-run db:get-run-info +cdb:remote-run db:get-run-key-val +cdb:remote-run db:get-run-name-from-id +cdb:remote-run db:get-steps-for-test +cdb:remote-run db:get-test-id-cached +cdb:remote-run db:get-tests-for-runs-mindata +cdb:remote-run db:lock/unlock-run +cdb:remote-run db:set-sync +cdb:remote-run db:set-tests-state-status +cdb:remote-run db:set-var +cdb:remote-run db:testmeta-add-record +cdb:remote-run db:testmeta-get-record +cdb:remote-run db:testmeta-update-field +cdb:remote-run db:update-run-event_time +cdb:remote-run instead +cdb:remote-run server:start +cdb:remote-run test:get-matching-previous-test-run-records +cdb:tests-register-test *runremote* +(define (runs:shrink-can-run-more-tests-count) ;; the db is a dummy var so we can use cdb:remote-run + ====================================================================== [87cbe68f31] [be405e8e2e] Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -30,10 +30,15 @@ ((get-run-info) (let ((res (apply db:get-run-info db params))) (list (vector-ref res 0) (vector->list (vector-ref res 1))))) ((register-run) (apply db:register-run db params)) ((login) (apply db:login db params)) + ((general-call) (let ((stmtname (car params)) + (realparams (cdr params))) + (db:general-call db stmtname realparams))) + ((set-tests-state-status) (apply db:set-state-status db params)) + ((get-tests-for-run) (map vector->list (apply db:get-tests-for-run db params))) (else (list "ERROR" 0)))) ;; http-server send-response ;; api:process-request @@ -44,13 +49,17 @@ (define (api:process-request db $) ;; the $ is the request vars proc (let* ((cmd ($ 'cmd)) (paramsj ($ 'params)) (params (rmt:json-str->dat paramsj)) (res (api:execute-requests db cmd params))) + + ;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds + (db:sync-to *inmemdb* *db*) + (rmt:dat->json-str (if (or (string? res) (list? res) (number? res) (boolean? res)) res (list "ERROR" 1 cmd params res))))) Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -55,11 +55,11 @@ (define *time-to-exit* #f) (define *received-response* #f) (define *default-numtries* 10) (define *server-run* #t) (define *db-write-access* #t) - +(define *inmemdb* #f) (define *target* (make-hash-table)) ;; cache the target here; target is keyval1/keyval2/.../keyvalN (define *keys* (make-hash-table)) ;; cache the keys here (define *keyvals* (make-hash-table)) (define *toptest-paths* (make-hash-table)) ;; cache toptest path settings here Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -1065,13 +1065,11 @@ ;; states and statuses are lists, turn them into ("PASS","FAIL"...) and use NOT IN ;; i.e. these lists define what to NOT show. ;; states and statuses are required to be lists, empty is ok ;; not-in #t = above behaviour, #f = must match -(define (db:get-tests-for-run db run-id testpatt states statuses offset limit not-in sort-by sort-order - #!key - (qryvals #f)) +(define (db:get-tests-for-run db run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals) (let* ((qryvalstr (case qryvals ((shortlist) "id,run_id,testname,item_path,state,status") ((#f) "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment") (else qryvals))) (res '()) @@ -2014,10 +2012,19 @@ (thread-sleep! 0.01) (loop)))) (set! *number-of-writes* (+ *number-of-writes* 1)) (set! *writes-total-delay* (+ *writes-total-delay* (- (current-milliseconds) start-time))) got-it)) + +(define (db:general-call db stmtname params) + (let ((query (let ((q (alist-ref (if (string? stmtname) + (string->symbol stmtname) + stmtname) + db:queries))) + (if q (car q) #f)))) + (apply sqlite3:execute db query params) + #t)) (define (db:process-queue-item db item) (let* ((stmt-key (cdb:packet-get-qtype item)) (qry-sig (cdb:packet-get-query-sig item)) (return-address (cdb:packet-get-client-sig item)) @@ -2069,10 +2076,11 @@ (server:reply return-address qry-sig #t 1)) ;; (length data))) ((set-verbosity) (set! *verbosity* (car params)) (server:reply return-address qry-sig #t (list #t *verbosity*))) ((killserver) + (db:sync-to *inmemdb* *db*) (let ((hostname (car *runremote*)) (port (cadr *runremote*)) (pid (car params)) (th1 (make-thread (lambda ()(thread-sleep! 3)(debug:print 0 "Server exiting!")(exit 0)) "Server exit thread"))) (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!") Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -426,72 +426,70 @@ (* 60 60 (string->number tmo)) ;; default to three days (* 3 24 60 60))))) (debug:print-info 2 "server-timeout: " server-timeout ", server pid: " spid " on " iface ":" port) (let loop ((count 0)) - (thread-sleep! 4) ;; no need to do this very often - ;; NB// sync currently does NOT return queue-length - - ;; Use this opportunity to sync the inmemdb to db - (db:sync-to *inmemdb* *db*) - - (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1))) - ;; (print "Server running, count is " count) - (if (< count 1) ;; 3x3 = 9 secs aprox - (loop (+ count 1))) - - ;; Check that iface and port have not changed (can happen if server port collides) - (mutex-lock! *heartbeat-mutex*) - (set! sdat *runremote*) - (mutex-unlock! *heartbeat-mutex*) - - (if (or (not (equal? sdat (list iface port))) - (not spid)) - (begin - (debug:print-info 0 "interface changed, refreshing iface and port info") - (set! iface (car sdat)) - (set! port (cadr sdat)) - (set! spid (tasks:server-get-server-id tdb #f iface port #f)))) - - ;; NOTE: Get rid of this mechanism! It really is not needed... - ;; (open-run-close tasks:server-update-heartbeat tasks:open-db spid) - (tasks:server-update-heartbeat tdb spid) - - ;; (if ;; (or (> numrunning 0) ;; stay alive for two days after last access - (mutex-lock! *heartbeat-mutex*) - (set! last-access *last-db-access*) - (mutex-unlock! *heartbeat-mutex*) - ;; (debug:print 11 "last-access=" last-access ", server-timeout=" server-timeout) - (if (and *server-run* - (> (+ last-access server-timeout) - (current-seconds))) - (begin - (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access)) - (loop 0)) - (begin - (debug:print-info 0 "Starting to shutdown the server.") - ;; need to delete only *my* server entry (future use) - (set! *time-to-exit* #t) - (open-run-close tasks:server-deregister-self tasks:open-db (get-host-name)) - (thread-sleep! 1) - (debug:print-info 0 "Max cached queries was " *max-cache-size*) - (debug:print-info 0 "Number of cached writes " *number-of-writes*) - (debug:print-info 0 "Average cached write time " - (if (eq? *number-of-writes* 0) - "n/a (no writes)" - (/ *writes-total-delay* - *number-of-writes*)) - " ms") - (debug:print-info 0 "Number non-cached queries " *number-non-write-queries*) - (debug:print-info 0 "Average non-cached time " - (if (eq? *number-non-write-queries* 0) - "n/a (no queries)" - (/ *total-non-write-delay* - *number-non-write-queries*)) - " ms") - (debug:print-info 0 "Server shutdown complete. Exiting") - (exit))))))) + ;; Use this opportunity to sync the inmemdb to db + (if *inmemdb* (db:sync-to *inmemdb* *db*)) + + (thread-sleep! 4) ;; no need to do this very often + + (if (< count 1) ;; 3x3 = 9 secs aprox + (loop (+ count 1))) + + ;; Check that iface and port have not changed (can happen if server port collides) + (mutex-lock! *heartbeat-mutex*) + (set! sdat *runremote*) + (mutex-unlock! *heartbeat-mutex*) + + (if (or (not (equal? sdat (list iface port))) + (not spid)) + (begin + (debug:print-info 0 "interface changed, refreshing iface and port info") + (set! iface (car sdat)) + (set! port (cadr sdat)) + (set! spid (tasks:server-get-server-id tdb #f iface port #f)))) + + ;; NOTE: Get rid of this mechanism! It really is not needed... + ;; (open-run-close tasks:server-update-heartbeat tasks:open-db spid) + (tasks:server-update-heartbeat tdb spid) + + ;; (if ;; (or (> numrunning 0) ;; stay alive for two days after last access + (mutex-lock! *heartbeat-mutex*) + (set! last-access *last-db-access*) + (mutex-unlock! *heartbeat-mutex*) + ;; (debug:print 11 "last-access=" last-access ", server-timeout=" server-timeout) + (if (and *server-run* + (> (+ last-access server-timeout) + (current-seconds))) + (begin + (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access)) + (loop 0)) + (begin + (debug:print-info 0 "Starting to shutdown the server.") + ;; need to delete only *my* server entry (future use) + (set! *time-to-exit* #t) + (if *inmemdb* (db:sync-to *inmemdb* *db*)) + (open-run-close tasks:server-deregister-self tasks:open-db (get-host-name)) + (thread-sleep! 1) + (debug:print-info 0 "Max cached queries was " *max-cache-size*) + (debug:print-info 0 "Number of cached writes " *number-of-writes*) + (debug:print-info 0 "Average cached write time " + (if (eq? *number-of-writes* 0) + "n/a (no writes)" + (/ *writes-total-delay* + *number-of-writes*)) + " ms") + (debug:print-info 0 "Number non-cached queries " *number-non-write-queries*) + (debug:print-info 0 "Average non-cached time " + (if (eq? *number-non-write-queries* 0) + "n/a (no queries)" + (/ *total-non-write-delay* + *number-non-write-queries*)) + " ms") + (debug:print-info 0 "Server shutdown complete. Exiting") + (exit)))))) ;; all routes though here end in exit ... (define (http-transport:launch) (if (not *toppath*) (if (not (setup-for-run)) Index: mt.scm ================================================================== --- mt.scm +++ mt.scm @@ -65,20 +65,20 @@ ;;====================================================================== ;; T E S T S ;;====================================================================== (define (mt:get-tests-for-run run-id testpatt states status #!key (not-in #t) (sort-by 'event_time) (sort-order "ASC") (qryvals #f)) - (let loop ((testsdat (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status 0 500 not-in sort-by sort-order qryvals: qryvals)) + (let loop ((testsdat (rmt:get-tests-for-run run-id testpatt states status 0 500 not-in sort-by sort-order qryvals)) (res '()) (offset 0) (limit 500)) (let* ((full-list (append res testsdat)) (have-more (eq? (length testsdat) limit))) (if have-more (let ((new-offset (+ offset limit))) (debug:print-info 4 "More than " limit " tests, have " (length full-list) " tests so far.") - (loop (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals: qryvals) + (loop (rmt:get-tests-for-run run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals) full-list new-offset limit)) full-list)))) Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -66,10 +66,18 @@ ;; A D M I N ;;====================================================================== (define (rmt:login) (rmt:send-receive 'login (list *toppath* megatest-version *my-client-signature*))) + +;;====================================================================== +;; G E N E R A L C A L L +;;====================================================================== + +;; hand off a call to one of the db:queries statements +(define (rmt:general-call stmtname . params) + (rmt:send-receive 'general-call (append (list stmtname) params))) ;;====================================================================== ;; K E Y S ;;====================================================================== @@ -102,10 +110,17 @@ ;; WARNING: This currently bypasses the transaction wrapped writes system (define (rmt:test-set-state-status-by-id test-id newstate newstatus newcomment) (rmt:send-receive 'test-set-state-status-by-id (list test-id newstate newstatus newcomment))) + +(define (rmt:set-tests-state-status run-id testnames currstate currstatus newstate newstatus) + (rmt:send-receive 'set-tests-state-status (list run-id testnames currstate currstatus newstate newstatus))) + +(define (rmt:get-tests-for-run run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals) + (map list->vector (rmt:send-receive 'get-tests-for-run (list run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals)))) + ;;====================================================================== ;; R U N S ;;====================================================================== (define (rmt:get-run-info run-id) Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -200,11 +200,11 @@ ;; (define (runs:run-tests target runname test-patts user flags) ;; test-names (common:clear-caches) ;; clear all caches (let* ((keys (keys:config-get-fields *configdat*)) (keyvals (keys:target->keyval keys target)) - (run-id (cdb:remote-run db:register-run #f keyvals runname "new" "n/a" user)) ;; test-name))) + (run-id (rmt:register-run keyvals runname "new" "n/a" user)) ;; test-name))) (deferred '()) ;; delay running these since they have a waiton clause (runconfigf (conc *toppath* "/runconfigs.config")) (required-tests '()) (test-records (make-hash-table)) (all-tests-registry (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names @@ -211,11 +211,11 @@ (all-test-names (hash-table-keys all-tests-registry)) (test-names (tests:filter-test-names all-test-names test-patts))) ;; Update the synchronous setting in the db based on the default or what is set by the user ;; This is done once here on a call to run tests rather than on every call to open-db - (cdb:remote-run db:set-sync #f) + ;; (cdb:remote-run db:set-sync #f) (set-megatest-env-vars run-id inkeys: keys) ;; these may be needed by the launching process (if (file-exists? runconfigf) (setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals "pre-launch-env-vars") (debug:print 0 "WARNING: You do not have a run config file: " runconfigf)) @@ -232,12 +232,12 @@ (begin ;; have to delete test records where NOT_STARTED since they can cause -keepgoing to ;; get stuck due to becoming inaccessible from a failed test. I.e. if test B depends ;; on test A but test B reached the point on being registered as NOT_STARTED and test ;; A failed for some reason then on re-run using -keepgoing the run can never complete. - (cdb:delete-tests-in-state *runremote* run-id "NOT_STARTED") - (cdb:remote-run db:set-tests-state-status #f run-id test-names #f "FAIL" "NOT_STARTED" "FAIL"))) + (rmt:general-call 'delete-tests-in-state run-id "NOT_STARTED") + (rmt:set-tests-state-status run-id test-names #f "FAIL" "NOT_STARTED" "FAIL"))) ;; Ensure all tests are registered in the test_meta table (runs:update-all-test_meta #f) ;; now add non-directly referenced dependencies (i.e. waiton) @@ -750,11 +750,11 @@ ;; Do mark-and-find clean up of db before starting runing of quue ;; ;; (cdb:remote-run db:find-and-mark-incomplete #f) - (let ((run-info (cdb:remote-run db:get-run-info #f run-id)) + (let ((run-info (rmt:get-run-info run-id)) (tests-info (mt:get-tests-for-run run-id #f '() '())) ;; qryvals: "id,testname,item_path")) (sorted-test-names (tests:sort-by-priority-and-waiton test-records)) (test-registry (make-hash-table)) (registry-mutex (make-mutex)) (num-retries 0) @@ -1049,11 +1049,11 @@ ;; (if (not test-id)(set! test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path))) (if (not test-id) (begin (debug:print 2 "WARN: Test not pre-created? test-name=" test-name ", item-path=" item-path ", run-id=" run-id) - (cdb:tests-register-test *runremote* run-id test-name item-path) + (rmt:general-call 'tests-register-test run-id test-name item-path) (set! test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path)))) (debug:print-info 4 "test-id=" test-id ", run-id=" run-id ", test-name=" test-name ", item-path=\"" item-path "\"") (set! testdat (cdb:get-test-info-by-id *runremote* test-id)) (if (not testdat) (begin Index: tests/unittests/server.scm ================================================================== --- tests/unittests/server.scm +++ tests/unittests/server.scm @@ -1,9 +1,13 @@ ;;====================================================================== ;; S E R V E R ;;====================================================================== +;; Run like this: +;; +;; (cd ..;make && make install) && ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) + (set! *transport-type* 'http) (test "setup for run" #t (begin (setup-for-run) (string? (getenv "MT_RUN_AREA_HOME")))) @@ -29,18 +33,23 @@ (thread-sleep! 3) ;; need to wait for server to start. Yes, a better way is needed. (test "get-best-server" #t (begin (client:launch) (let ((dat (open-run-close tasks:get-best-server tasks:open-db))) (vector? dat)))) -;; (print "dat: " dat) -;; (set! *runremote* (list (vector-ref dat 1)(vector-ref dat 2) #f)) ;; host ip pullport pubport -;; (and (string? (car *runremote*)) -;; (number? (cadr *runremote*))))) + +(define *keys* (keys:config-get-fields *configdat*)) +(define *keyvals* (keys:target->keyval *keys* "a/b/c")) (test #f #t (string? (car *runremote*))) (test #f '(#t "successful login") (rmt:login)) ;; *runremote* *toppath* *my-client-signature*))) -(test #f #f (rmt:get-test-info-by-id 99)) +(test #f #f (rmt:get-test-info-by-id 99)) ;; get non-existant test +(test #f 1 (rmt:register-run *keyvals* "firstrun" "new" "n/a" (current-user-name))) +(test "get run info" "firstrun" (let ((rinfo (rmt:get-run-info 1))) + (vector-ref (vector-ref rinfo 1) 3))) +(test "get tests (no data)" '() (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f)) +(test "register test" #t (rmt:general-call 'register-test 1 "test1" "")) +(test "get tests (some data)" 1 (length (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f))) ;; ;; (set! *verbosity* 20) ;; (test #f *verbosity* (cadr (cdb:set-verbosity *runremote* *verbosity*))) ;; (test #f #f (cdb:roll-up-pass-fail-counts *runremote* 1 "test1" "" "PASS")) ;; ;; (set! *verbosity* 1) @@ -71,7 +80,7 @@ ;;====================================================================== ;; D B ;;====================================================================== -(test #f #f (cdb:kill-server *runremote* #f)) ;; *toppath* *my-client-signature* #f))) +(test #f '(#t "exit process started") (cdb:kill-server *runremote* #f)) ;; *toppath* *my-client-signature* #f)))