Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -6,11 +6,12 @@ ods.scm runconfig.scm server.scm configf.scm \ db.scm keys.scm margs.scm megatest-version.scm \ process.scm runs.scm tasks.scm tests.scm genexample.scm \ fs-transport.scm http-transport.scm \ client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm \ - tree.scm ezsteps.scm lock-queue.scm sdb.scm + tree.scm ezsteps.scm lock-queue.scm sdb.scm filedb.scm \ + rmt.scm api.scm tdb.scm GUISRCF = dashboard-tests.scm dashboard-guimonitor.scm OFILES = $(SRCFILES:%.scm=%.o) GOFILES = $(GUISRCF:%.scm=%.o) Index: NOTES ================================================================== --- NOTES +++ NOTES @@ -30,10 +30,45 @@ a. Might talk to running server if run specific b. Can talk to megatest.db but not a generally good idea c. Can start a runserver 4. Dashboard is fine except for writes? +====================================================================== +Routines to convert for runs.scm + +cdb:remote-run db:register-run + +cdb:delete-tests-in-state *runremote* +cdb:get-test-info-by-id *runremote* +cdb:remote-run db:delete-old-deleted-test-records +cdb:remote-run db:delete-run +cdb:remote-run db:delete-test-records +cdb:remote-run db:delete-tests-for-run +cdb:remote-run db:find-and-mark-incomplete +cdb:remote-run db:get-count-tests-running +cdb:remote-run db:get-count-tests-running-in-jobgroup +cdb:remote-run db:get-keys +cdb:remote-run db:get-run-info +cdb:remote-run db:get-run-key-val +cdb:remote-run db:get-run-name-from-id +cdb:remote-run db:get-steps-for-test +cdb:remote-run db:get-test-id-cached +cdb:remote-run db:get-tests-for-runs-mindata +cdb:remote-run db:lock/unlock-run +cdb:remote-run db:set-sync +cdb:remote-run db:set-tests-state-status +cdb:remote-run db:set-var +cdb:remote-run db:testmeta-add-record +cdb:remote-run db:testmeta-get-record +cdb:remote-run db:testmeta-update-field +cdb:remote-run db:update-run-event_time +cdb:remote-run instead +cdb:remote-run server:start +cdb:remote-run test:get-matching-previous-test-run-records +cdb:tests-register-test *runremote* +(define (runs:shrink-can-run-more-tests-count) ;; the db is a dummy var so we can use cdb:remote-run + ====================================================================== [87cbe68f31] [be405e8e2e] ADDED api.scm Index: api.scm ================================================================== --- /dev/null +++ api.scm @@ -0,0 +1,111 @@ +;;====================================================================== +;; Copyright 2006-2013, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + +(declare (unit api)) +(declare (uses rmt)) +(declare (uses db)) + +;; These are called by the server on recipt of /api calls + +(define (api:execute-requests db cmd params) + (case (string->symbol cmd) + ;; KEYS + ((get-key-val-pairs) (apply db:get-key-val-pairs db params)) + ((get-keys) (db:get-keys db)) + + ;; TESTS + ;; json doesn't do vectors, convert to list + ((get-test-info-by-id) (apply db:get-test-info-by-id db params)) + ((test-get-rundir-from-test-id) (apply db:test-get-rundir-from-test-id db params)) + ((test-set-state-status-by-id) (apply db:test-set-state-status-by-id db params)) + ((get-count-tests-running) (db:get-count-tests-running db)) + ((get-count-tests-running-in-jobgroup) (apply db:get-count-tests-running-in-jobgroup db params)) + ((delete-test-records) (apply db:delete-test-records db params)) + ((delete-old-deleted-test-records) (db:delete-old-deleted-test-records db)) + ((test-set-status-state) (apply db:test-set-status-state db params)) + ((get-previous-test-run-record) (apply db:get-previous-test-run-record db params)) + ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records db params)) + ((db:test-get-logfile-info) (apply db:test-get-logfile-info db params)) + ((test-get-records-for-index-file (apply db:test-get-records-for-index-file db params))) + ((get-testinfo-state-status) (apply db:get-testinfo-state-status db params)) + ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new db params)) + ((get-prereqs-not-met) (apply db:get-prereqs-not-met db params)) + ((roll-up-pass-fail-counts) (apply db:roll-up-pass-fail-counts db params)) + ((update-fail-pass-counts) (apply db:general-call db 'update-pass-fail-counts params)) + ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id db params)) + + ;; RUNS + ((get-run-info) (apply db:get-run-info db params)) + ((register-run) (apply db:register-run db params)) + ((set-tests-state-status) (apply db:set-tests-state-status db params)) + ((get-tests-for-run) (apply db:get-tests-for-run db params)) + ((get-test-id) (apply db:get-test-id-not-cached db params)) + ((get-tests-for-runs-mindata) (apply db:get-tests-for-runs-mindata db params)) + ((get-run-name-from-id) (apply db:get-run-name-from-id db params)) + ((delete-run) (apply db:delete-run db params)) + ((get-runs) (apply db:get-runs db params)) + ((get-runs-by-patt) (apply db:get-runs-by-patt db params)) + ((lock/unlock-run) (apply db:lock/unlock-run db params)) + ((update-run-event_time) (apply db:update-run-event_time db params)) + + ;; MISC + ((login) (apply db:login db params)) + ((general-call) (let ((stmtname (car params)) + (realparams (cdr params))) + (db:general-call db stmtname realparams))) + ((sync-inmem->db) (db:sync-back)) + ((kill-server) + (db:sync-to *inmemdb* *db*) + (let ((hostname (car *runremote*)) + (port (cadr *runremote*)) + (pid (if (null? params) #f (car params))) + (th1 (make-thread (lambda ()(thread-sleep! 3)(debug:print 0 "Server exiting!")(exit 0)) "Server exit thread"))) + (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!") + (debug:print-info 1 "current pid=" (current-process-id)) + (open-run-close tasks:server-deregister tasks:open-db + hostname + port: port) + (set! *server-run* #f) + (thread-sleep! 3) + (if pid + (process-signal pid signal/kill) + (thread-start! th1)) + '(#t "exit process started"))) + + ;; TESTMETA + ((testmeta-get-record) (apply db:testmeta-get-record db params)) + ((testmeta-add-record) (apply db:testmeta-add-record db params)) + ((testmeta-update-field) (apply db:testmeta-update-field db params)) + (else + (list "ERROR" 0)))) + +;; http-server send-response +;; api:process-request +;; db:* +;; +;; NB// Runs on the server as part of the server loop +;; +(define (api:process-request db $) ;; the $ is the request vars proc + (let* ((cmd ($ 'cmd)) + (paramsj ($ 'params)) + (params (db:string->obj paramsj)) ;; (rmt:json-str->dat paramsj)) + (res (api:execute-requests db cmd params))) + + ;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds + ;; (rmt:dat->json-str + ;; (if (or (string? res) + ;; (list? res) + ;; (number? res) + ;; (boolean? res)) + ;; res + ;; (list "ERROR, not string, list, number or boolean" 1 cmd params res))))) + (db:obj->string res))) + Index: client.scm ================================================================== --- client.scm +++ client.scm @@ -35,14 +35,10 @@ (if *my-client-signature* *my-client-signature* (let ((sig (conc (get-host-name) " " (current-process-id)))) (set! *my-client-signature* sig) *my-client-signature*))) -;; client:login serverdat -(define (client:login serverdat) - (cdb:login serverdat *toppath* (client:get-signature))) - ;; Not currently used! But, I think it *should* be used!!! (define (client:logout serverdat) (let ((ok (and (socket? serverdat) (cdb:logout serverdat *toppath* (client:get-signature))))) ok)) Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -27,11 +27,11 @@ (define getenv get-environment-variable) (define home (getenv "HOME")) (define user (getenv "USER")) -;; global gletches +;; GLOBAL GLETCHES (define *db-keys* #f) (define *configinfo* #f) (define *configdat* #f) (define *toppath* #f) (define *already-seen-runconfig-info* #f) @@ -55,11 +55,11 @@ (define *time-to-exit* #f) (define *received-response* #f) (define *default-numtries* 10) (define *server-run* #t) (define *db-write-access* #t) - +(define *inmemdb* #f) (define *target* (make-hash-table)) ;; cache the target here; target is keyval1/keyval2/.../keyvalN (define *keys* (make-hash-table)) ;; cache the keys here (define *keyvals* (make-hash-table)) (define *toptest-paths* (make-hash-table)) ;; cache toptest path settings here @@ -96,14 +96,29 @@ ;;====================================================================== ;; S T A T E S A N D S T A T U S E S ;;====================================================================== (define *common:std-states* - (list "COMPLETED" "NOT_STARTED" "RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "KILLED" "KILLREQ" "STUCK")) + '((0 "COMPLETED") + (1 "NOT_STARTED") + (2 "RUNNING") + (3 "REMOTEHOSTSTART") + (4 "LAUNCHED") + (5 "KILLED") + (6 "KILLREQ") + (7 "STUCK"))) (define *common:std-statuses* - (list "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD")) + '((0 "PASS") + (1 "WARN") + (2 "FAIL") + (3 "CHECK") + (4 "n/a") + (5 "WAIVED") + (6 "SKIP") + (7 "DELETED") + (8 "STUCK/DEAD"))) ;; These are stopping conditions that prevent a test from being run (define *common:cant-run-states-sym* '(COMPLETED KILLED WAIVED UNKNOWN INCOMPLETE)) Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -132,13 +132,13 @@ ;;====================================================================== ;; Run info panel ;;====================================================================== -(define (run-info-panel keydat testdat runname) +(define (run-info-panel db keydat testdat runname) (let* ((run-id (db:test-get-run_id testdat)) - (rundat (cdb:remote-run db:get-run-info #f run-id)) + (rundat (db:get-run-info db run-id)) (header (db:get-header rundat)) (event_time (db:get-value-by-header (db:get-row rundat) (db:get-header rundat) "event_time"))) (iup:frame @@ -213,33 +213,34 @@ (define *dashboard-test-db* #t) ;;====================================================================== ;; Set fields ;;====================================================================== -(define (set-fields-panel test-id testdat #!key (db #f)) +(define (set-fields-panel db test-id testdat #!key (db #f)) (let ((newcomment #f) (newstatus #f) (newstate #f)) (iup:frame #:title "Set fields" (iup:vbox (iup:hbox (iup:label "Comment:") (iup:textbox #:action (lambda (val a b) - (open-run-close db:test-set-state-status-by-id db test-id #f #f b) + (db:test-set-state-status-by-id db test-id #f #f b) + ;; IDEA: Just set a variable with the proc to call? (set! newcomment b)) #:value (db:test-get-comment testdat) #:expand "HORIZONTAL")) (apply iup:hbox (iup:label "STATE:" #:size "30x") (let* ((btns (map (lambda (state) (let ((btn (iup:button state #:expand "HORIZONTAL" #:size "50x" #:font "Courier New, -10" #:action (lambda (x) - (open-run-close db:test-set-state-status-by-id db test-id state #f #f) + (db:test-set-state-status-by-id db test-id state #f #f) (db:test-set-state! testdat state))))) btn)) - *common:std-states*))) ;; (list "COMPLETED" "NOT_STARTED" "RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "KILLED" "KILLREQ")))) + (map cadr *common:std-states*)))) ;; (list "COMPLETED" "NOT_STARTED" "RUNNING" "REMOTEHOSTSTART" "LAUNCHED" "KILLED" "KILLREQ")))) (vector-set! *state-status* 0 (lambda (state color) (for-each (lambda (btn) (let* ((name (iup:attribute btn "TITLE")) @@ -252,14 +253,19 @@ (iup:label "STATUS:" #:size "30x") (let* ((btns (map (lambda (status) (let ((btn (iup:button status #:expand "HORIZONTAL" #:size "50x" #:font "Courier New, -10" #:action (lambda (x) - (open-run-close db:test-set-state-status-by-id db test-id #f status #f) - (db:test-set-status! testdat status))))) + (let ((t (iup:attribute x "TITLE"))) + (if (equal? t "WAIVED") + (iup:show (dashboard-tests:waiver testdat (lambda (c) + (set! newcomment c)))) + (begin + (open-run-close db:test-set-state-status-by-id db test-id #f status #f) + (db:test-set-status! testdat status)))))))) btn)) - *common:std-statuses*))) ;; (list "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP")))) + (map cadr *common:std-statuses*)))) ;; (list "PASS" "WARN" "FAIL" "CHECK" "n/a" "WAIVED" "SKIP")))) (vector-set! *state-status* 1 (lambda (status color) (for-each (lambda (btn) (let* ((name (iup:attribute btn "TITLE")) @@ -300,41 +306,176 @@ ;; (iup:button "Refresh test data" ;; #:expand "HORIZONTAL" ;; #:action (lambda (obj) ;; (print "Refresh test data " stepname)) ))) + +(define (dashboard-tests:waiver testdat cmtcmd) + (let* ((wpatt (configf:lookup *configdat* "setup" "waivercommentpatt")) + (wregx (if (string? wpatt)(regexp wpatt) #f)) + (wmesg (iup:label (if wpatt (conc "Comment must match pattern " wpatt) ""))) + (comnt (iup:textbox #:action (lambda (val a b) + (if wpatt + (if (string-match wregx b) + (iup:attribute-set! wmesg "TITLE" (conc "Comment matches " wpatt)) + (iup:attribute-set! wmesg "TITLE" (conc "Comment does not match " wpatt)) + ))) + #:value (db:test-get-comment testdat) + #:expand "HORIZONTAL")) + (dlog #f)) + (set! dlog (iup:dialog ;; #:close_cb (lambda (a)(exit)) ; #:expand "YES" + #:title "SET WAIVER" + (iup:vbox ; #:expand "YES" + (iup:label (conc "Enter justification for waiving test " + (db:test-get-testname testdat) + (if (equal? (db:test-get-item-path testdat) "") + "" + (conc "/" (db:test-get-item-path testdat))))) + wmesg ;; the informational msg on whether it matches + comnt + (iup:hbox + (iup:button "Apply and Close " + #:expand "HORIZONTAL" + #:action (lambda (obj) + (let ((comment (iup:attribute comnt "VALUE")) + (test-id (db:test-get-id testdat))) + (if (or (not wpatt) + (string-match wregx comment)) + (begin + (open-run-close db:test-set-state-status-by-id #f test-id #f "WAIVED" comment) + (db:test-set-status! testdat "WAIVED") + (cmtcmd comment) + (iup:destroy! dlog)))))) + (iup:button "Cancel" + #:expand "HORIZONTAL" + #:action (lambda (obj) + (iup:destroy! dlog))))))) + dlog)) + +;; CHECK - WAS THIS ADDED OR REMOVED? MANUAL MERGE WITH API STUFF!!! +;; +;; get a pretty table to summarize steps +;; +(define (dashboard-tests:process-steps-table steps);; db test-id #!key (work-area #f)) +;; (let ((steps (db:get-steps-for-test db test-id work-area: work-area))) + ;; organise the steps for better readability + (let ((res (make-hash-table))) + (for-each + (lambda (step) + (debug:print 6 "step=" step) + (let ((record (hash-table-ref/default + res + (tdb:step-get-stepname step) + ;; stepname start end status Duration Logfile + (vector (tdb:step-get-stepname step) "" "" "" "" "")))) + (debug:print 6 "record(before) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)) + (case (string->symbol (tdb:step-get-state step)) + ((start)(vector-set! record 1 (tdb:step-get-event_time step)) + (vector-set! record 3 (if (equal? (vector-ref record 3) "") + (tdb:step-get-status step))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + ((end) + (vector-set! record 2 (any->number (tdb:step-get-event_time step))) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (let ((startt (any->number (vector-ref record 1))) + (endt (any->number (vector-ref record 2)))) + (debug:print 4 "record[1]=" (vector-ref record 1) + ", startt=" startt ", endt=" endt + ", get-status: " (tdb:step-get-status step)) + (if (and (number? startt)(number? endt)) + (seconds->hr-min-sec (- endt startt)) "-1"))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + (else + (vector-set! record 2 (tdb:step-get-state step)) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (tdb:step-get-event_time step)))) + (hash-table-set! res (tdb:step-get-stepname step) record) + (debug:print 6 "record(after) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)))) + ;; (else (vector-set! record 1 (tdb:step-get-event_time step))) + (sort steps (lambda (a b) + (cond + ((< (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t) + ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) + (< (tdb:step-get-id a) (tdb:step-get-id b))) + (else #f))))) + res)) + +(define (dashboard-tests:get-compressed-steps test-id #!key (work-area #f)) + (if (or (not work-area) + (file-exists? (conc work-area "/testdat.db"))) + (let* ((steps-data (tdb:get-steps-for-test test-id work-area)) + (comprsteps (dashboard-tests:process-steps-table steps-data))) ;; (open-run-close db:get-steps-table #f test-id work-area: work-area))) + (map (lambda (x) + ;; take advantage of the \n on time->string + (vector + (vector-ref x 0) + (let ((s (vector-ref x 1))) + (if (number? s)(seconds->time-string s) s)) + (let ((s (vector-ref x 2))) + (if (number? s)(seconds->time-string s) s)) + (vector-ref x 3) ;; status + (vector-ref x 4) + (vector-ref x 5))) ;; time delta + (sort (hash-table-values comprsteps) + (lambda (a b) + (let ((time-a (vector-ref a 1)) + (time-b (vector-ref b 1))) + (if (and (number? time-a)(number? time-b)) + (if (< time-a time-b) + #t + (if (eq? time-a time-b) + (string curr-mod-time db-mod-time) + (need-update (or (and (>= curr-mod-time db-mod-time) (> (current-milliseconds)(+ last-update 250))) ;; every half seconds if db touched (> (current-milliseconds)(+ last-update 10000)) ;; force update even 10 seconds request-update)) (newtestdat (if need-update ;; NOTE: BUG HIDER, try to eliminate this exception handler (handle-exceptions exn - (debug:print-info 0 "WARNING: test db access issue for test " test-id ": " ((condition-property-accessor 'exn 'message) exn)) - (make-db:test) - (let* ((newdat (open-run-close db:get-test-info-by-id db test-id )) - (tstdat (if newdat - (open-run-close tests:testdat-get-testinfo db test-id #f) - '()))) - (if (and newdat - (not (null? tstdat))) ;; (update-time cpuload diskfree run-duration) - (let* ((rec (car tstdat)) - (cpuload (vector-ref rec 1)) - (diskfree (vector-ref rec 2)) - (run-dur (vector-ref rec 3))) - (db:test-set-run_duration! newdat run-dur) - (db:test-set-diskfree! newdat diskfree) - (db:test-set-cpuload! newdat cpuload))) - ;; (debug:print 0 "newdat=" newdat) - newdat) - ) - #f))) - ;; (debug:print 0 "newtestdat=" newtestdat) + (debug:print-info 0 "test db access issue: " ((condition-property-accessor 'exn 'message) exn)) + (db:get-test-info-by-id db test-id ))))) + ;; (debug:print-info 0 "need-update= " need-update " curr-mod-time = " curr-mod-time) (cond ((and need-update newtestdat) (set! testdat newtestdat) - (set! teststeps (db:get-compressed-steps test-id work-area: rundir)) + (set! teststeps (dashboard-tests:get-compressed-steps test-id work-area: rundir)) (set! logfile (conc (db:test-get-rundir testdat) "/" (db:test-get-final_logf testdat))) (set! rundir (db:test-get-rundir testdat)) (set! testfullname (db:test-get-fullname testdat)) ;; (debug:print 0 "INFO: teststeps=" (intersperse teststeps "\n ")) @@ -515,11 +639,11 @@ (iup:dialog #:close_cb (lambda (a)(exit)) ; #:expand "YES" #:title testfullname (iup:vbox ; #:expand "YES" ;; The run and test info (iup:hbox ; #:expand "YES" - (run-info-panel keydat testdat runname) + (run-info-panel db keydat testdat runname) (test-info-panel testdat store-label widgets) (test-meta-panel testmeta store-meta)) (host-info-panel testdat store-label) ;; The controls (iup:frame #:title "Actions" @@ -533,11 +657,11 @@ (iup:button "Kill All Jobs" #:action kill-jobs #:size "80x") (iup:button "Close" #:action (lambda (x)(exit)) #:size "80x")) (apply iup:hbox (list command-text-box command-launch-button)))) - (set-fields-panel test-id testdat) + (set-fields-panel db test-id testdat) (let ((tabs (iup:tabs ;; Replace here with matrix (let ((steps-matrix (iup:matrix #:font "Courier New, -8" @@ -641,11 +765,11 @@ (db:test-data-get-tol x) (db:test-data-get-status x) (db:test-data-get-units x) (db:test-data-get-type x) (db:test-data-get-comment x))) - (open-run-close db:read-test-data db test-id "%"))) + (tdb:open-run-close-db-by-test-id-local test-id #f tdb:read-test-data test-id "%"))) "\n"))) (if (not (equal? currval newval)) (iup:attribute-set! test-data "VALUE" newval ))))) ;; "TITLE" newval))))) test-data)) ;;(dashboard:run-controls) Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -39,10 +39,11 @@ (declare (uses mt)) (include "common_records.scm") (include "db_records.scm") (include "run_records.scm") +(include "megatest-fossil-hash.scm") (define help (conc "Megatest Dashboard, documentation at http://www.kiatoa.com/fossils/megatest version " megatest-version " license GPL, Copyright (C) Matt Welland 2013 @@ -63,10 +64,11 @@ (list "-rows" "-run" "-test" "-debug" "-host" + "-transport" ) (list "-h" "-use-server" "-guimonitor" "-main" @@ -84,29 +86,29 @@ (if (not (setup-for-run)) (begin (print "Failed to find megatest.config, exiting") (exit 1))) -(define *db* #f) ;; (open-db)) - -(if (args:get-arg "-host") - (begin - (set! *runremote* (string-split (args:get-arg "-host" ":"))) - (client:launch)) - (if (not (args:get-arg "-use-server")) - (set! *transport-type* 'fs) ;; force fs access - (client:launch))) +(define *db* (open-db)) + +;; (if (args:get-arg "-host") +;; (begin +;; (set! *runremote* (string-split (args:get-arg "-host" ":"))) +;; (client:launch)) +;; (if (not (args:get-arg "-use-server")) +;; (set! *transport-type* 'fs) ;; force fs access +;; (client:launch))) ;; HACK ALERT: this is a hack, please fix. (define *read-only* (not (file-read-access? (conc *toppath* "/megatest.db")))) ;; (client:setup *db*) (define toplevel #f) (define dlg #f) (define max-test-num 0) -;; (define *keys* (open-run-close db:get-keys #f)) -(define *keys* (cdb:remote-run db:get-keys #f)) +(define *keys* (db:get-keys *db*)) +;; (define *keys* (cdb:remote-run db:get-keys #f)) ;; (define *keys* (db:get-keys *db*)) (define *dbkeys* (append *keys* (list "runname"))) (define *header* #f) @@ -116,11 +118,11 @@ (define *buttondat* (make-hash-table)) ;; (define *alltestnamelst* '()) (define *searchpatts* (make-hash-table)) (define *num-runs* 8) -(define *tot-run-count* (cdb:remote-run db:get-num-runs #f "%")) +(define *tot-run-count* (db:get-num-runs *db* "%")) ;; (define *tot-run-count* (db:get-num-runs *db* "%")) ;; Update management ;; (define *last-update* (current-seconds)) @@ -203,11 +205,11 @@ (null? (filter (lambda (x)(> x 3)) delta)))) ;; keypatts: ( (KEY1 "abc%def")(KEY2 "%") ) (define (update-rundat runnamepatt numruns testnamepatt keypatts) (let* ((referenced-run-ids '()) - (allruns (cdb:remote-run db:get-runs #f runnamepatt numruns ;; (+ numruns 1) ;; (/ numruns 2)) + (allruns (db:get-runs *db* runnamepatt numruns ;; (+ numruns 1) ;; (/ numruns 2)) *start-run-offset* keypatts)) (header (db:get-header allruns)) (runs (db:get-rows allruns)) (result '()) (maxtests 0) @@ -222,18 +224,19 @@ ;; ;; trim runs to only those that are changing often here ;; (for-each (lambda (run) (let* ((run-id (db:get-value-by-header run header "id")) - (tests (mt:get-tests-for-run run-id testnamepatt states statuses - not-in: *hide-not-hide* - sort-by: sort-by - sort-order: sort-order - qryvals: 'shortlist)) + (tests (db:get-tests-for-run *db* run-id testnamepatt states statuses + #f #f + *hide-not-hide* + sort-by + sort-order + 'shortlist)) ;; NOTE: bubble-up also sets the global *all-item-test-names* ;; (tests (bubble-up tmptests priority: bubble-type)) - (key-vals (cdb:remote-run db:get-key-vals #f run-id))) + (key-vals (db:get-key-vals *db* run-id))) ;; NOTE: 11/01/2013 This routine is *NOT* getting called excessively. ;; (debug:print 0 "Getting data for run " run-id " with key-vals=" key-vals) ;; Not sure this is needed? (set! referenced-run-ids (cons run-id referenced-run-ids)) (if (> (length tests) maxtests) @@ -556,11 +559,11 @@ (iup:attribute-set! lb "VALUE" newval) newval)))))) (define (dashboard:update-target-selector key-lbs #!key (action-proc #f)) (let* ((runconf-targs (common:get-runconfig-targets)) - (db-target-dat (open-run-close db:get-targets #f)) + (db-target-dat (db:get-targets *db*)) (header (vector-ref db-target-dat 0)) (db-targets (vector-ref db-target-dat 1)) (all-targets (append db-targets (map (lambda (x) (list->vector @@ -821,11 +824,11 @@ (iup:attribute-set! tb "VALUE" val) (dboard:data-set-run-name! *data* val) (dashboard:update-run-command)))) (refresh-runs-list (lambda () (let* ((target (dboard:data-get-target-string *data*)) - (runs-for-targ (mt:get-runs-by-patt *keys* "%" target)) + (runs-for-targ (db:get-runs-by-patt *db* *keys* "%" target #f #f)) (runs-header (vector-ref runs-for-targ 0)) (runs-dat (vector-ref runs-for-targ 1)) (run-names (cons default-run-name (map (lambda (x) (db:get-value-by-header x runs-header "runname")) @@ -870,19 +873,19 @@ ;; Text box for STATES (iup:frame #:title "States" (dashboard:text-list-toggle-box ;; Move these definitions to common and find the other useages and replace! - *common:std-states* ;; '("COMPLETED" "RUNNING" "STUCK" "INCOMPLETE" "LAUNCHED" "REMOTEHOSTSTART" "KILLED") + (map cadr *common:std-states*) ;; '("COMPLETED" "RUNNING" "STUCK" "INCOMPLETE" "LAUNCHED" "REMOTEHOSTSTART" "KILLED") (lambda (all) (dboard:data-set-states! *data* all) (dashboard:update-run-command)))) ;; Text box for STATES (iup:frame #:title "Statuses" (dashboard:text-list-toggle-box - *common:std-statuses* ;; '("PASS" "FAIL" "n/a" "CHECK" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD") + (map cadr *common:std-statuses*) ;; '("PASS" "FAIL" "n/a" "CHECK" "WAIVED" "SKIP" "DELETED" "STUCK/DEAD") (lambda (all) (dboard:data-set-statuses! *data* all) (dashboard:update-run-command)))))))) (iup:frame @@ -981,11 +984,11 @@ ;;====================================================================== ;; S U M M A R Y ;;====================================================================== ;; ;; General info about the run(s) and megatest area -(define (dashboard:summary) +(define (dashboard:summary db) (let ((rawconfig (read-config (conc *toppath* "/megatest.config") #f 'return-string))) (iup:vbox (iup:split ;; #:value 500 (iup:frame @@ -1006,11 +1009,11 @@ ;; (iup:frame ;; #:title "Disks Areas" (dcommon:section-matrix rawconfig "disks" "Disk area" "Path")))) (iup:frame #:title "Run statistics" - (dcommon:run-stats))))) + (dcommon:run-stats db))))) ;;====================================================================== ;; R U N ;;====================================================================== ;; @@ -1022,11 +1025,11 @@ #f)) (define dashboard:update-run-summary-tab #f) ;; (define (tests window-id) -(define (dashboard:one-run) +(define (dashboard:one-run db) (let* ((tb (iup:treebox #:value 0 #:name "Runs" #:expand "YES" #:addexpanded "NO" @@ -1042,19 +1045,21 @@ ;; (print "path: " (tree:node->path obj id) " run-id: " run-id) )))) (run-matrix (iup:matrix #:expand "YES")) (updater (lambda () - (let* ((runs-dat (mt:get-runs-by-patt *keys* "%" #f)) + (let* ((runs-dat (db:get-runs-by-patt db *keys* "%" #f #f #f)) (runs-header (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records (run-id (dboard:data-get-curr-run-id *data*)) - (tests-dat (let ((tdat (mt:get-tests-for-run run-id + (tests-dat (let ((tdat (db:get-tests-for-run db run-id (hash-table-ref/default *searchpatts* "test-name" "%/%") (hash-table-keys *state-ignore-hash*) ;; '() (hash-table-keys *status-ignore-hash*) ;; '() - not-in: *hide-not-hide* - qryvals: "id,testname,item_path,state,status"))) ;; get 'em all + #f #f + *hide-not-hide* + #f #f + "id,testname,item_path,state,status"))) ;; get 'em all (sort tdat (lambda (a b) (let* ((aval (vector-ref a 2)) (bval (vector-ref b 2)) (anum (string->number aval)) (bnum (string->number bval))) @@ -1167,11 +1172,11 @@ ;;====================================================================== ;; R U N S ;;====================================================================== -(define (make-dashboard-buttons nruns ntests keynames) +(define (make-dashboard-buttons db nruns ntests keynames) (let* ((nkeys (length keynames)) (runsvec (make-vector nruns)) (header (make-vector nruns)) (lftcol (make-vector ntests)) (keycol (make-vector ntests)) @@ -1241,21 +1246,21 @@ (mark-for-update) (if (eq? val 1) (hash-table-set! *status-ignore-hash* status #t) (hash-table-delete! *status-ignore-hash* status)) (set-bg-on-filter)))) - *common:std-statuses*)) ;; '("PASS" "FAIL" "WARN" "CHECK" "WAIVED" "STUCK/DEAD" "n/a" "SKIP"))) + (map cadr *common:std-statuses*))) ;; '("PASS" "FAIL" "WARN" "CHECK" "WAIVED" "STUCK/DEAD" "n/a" "SKIP"))) (apply iup:hbox (map (lambda (state) (iup:toggle state #:action (lambda (obj val) (mark-for-update) (if (eq? val 1) (hash-table-set! *state-ignore-hash* state #t) (hash-table-delete! *state-ignore-hash* state)) (set-bg-on-filter)))) - *common:std-states*)) ;; '("RUNNING" "COMPLETED" "INCOMPLETE" "LAUNCHED" "NOT_STARTED" "KILLED" "DELETED"))) + (map cadr *common:std-states*))) ;; '("RUNNING" "COMPLETED" "INCOMPLETE" "LAUNCHED" "NOT_STARTED" "KILLED" "DELETED"))) (iup:valuator #:valuechanged_cb (lambda (obj) (let ((val (inexact->exact (round (/ (string->number (iup:attribute obj "VALUE")) 10)))) (oldmax (string->number (iup:attribute obj "MAX"))) (maxruns *tot-run-count*)) (set! *start-run-offset* val) @@ -1378,13 +1383,13 @@ controls)) (tabs (iup:tabs #:tabchangepos-cb (lambda (obj curr prev) (set! *please-update-buttons* #t) (set! *current-tab-number* curr)) - (dashboard:summary) + (dashboard:summary db) runs-view - (dashboard:one-run) + (dashboard:one-run db) (dashboard:run-controls) ))) ;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c))) (iup:attribute-set! tabs "TABTITLE0" "Summary") (iup:attribute-set! tabs "TABTITLE1" "Runs") @@ -1423,11 +1428,11 @@ (or please-update-buttons (and (> (current-milliseconds)(+ *last-recalc-ended-time* 150)) (> modtime last-db-update-time) (> (current-seconds)(+ last-db-update-time 1))))) -(define *monitor-db-path* (conc *toppath* "/monitor.db")) +(define *monitor-db-path* (conc *toppath* "/db/monitor.db")) (define *last-monitor-update-time* 0) ;; Force creation of the db in case it isn't already there. (let ((db (tasks:open-db))) (sqlite3:finalize! db)) @@ -1484,11 +1489,11 @@ (if runid (begin (lambda (x) (on-exit (lambda () (if *db* (sqlite3:finalize! *db*)))) - (cdb:remote-run examine-run *db* runid))) + (examine-run *db* runid))) (begin (print "ERROR: runid is not a number " (args:get-arg "-run")) (exit 1))))) ((args:get-arg "-test") (let ((testid (string->number (args:get-arg "-test")))) @@ -1499,11 +1504,11 @@ (debug:print 3 "INFO: tried to open test with invalid test-id. " (args:get-arg "-test")) (exit 1))))) ((args:get-arg "-guimonitor") (gui-monitor *db*)) (else - (set! uidat (make-dashboard-buttons *num-runs* *num-tests* *dbkeys*)) + (set! uidat (make-dashboard-buttons *db* *num-runs* *num-tests* *dbkeys*)) (iup:callback-set! *tim* "ACTION_CB" (lambda (x) (let ((update-is-running #f)) (mutex-lock! *update-mutex*) @@ -1518,5 +1523,6 @@ (set! *update-is-running* #f) (mutex-unlock! *update-mutex*)))) 1)))) (iup:main-loop) +(sqlite3:finalize! *db*) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -28,10 +28,11 @@ (declare (uses keys)) (declare (uses ods)) (declare (uses fs-transport)) (declare (uses client)) (declare (uses mt)) +(declare (uses filedb)) (include "common_records.scm") (include "db_records.scm") (include "key_records.scm") (include "run_records.scm") @@ -42,10 +43,12 @@ (define *completed-writes* (make-hash-table)) (define *incoming-last-time* (current-seconds)) (define *incoming-mutex* (make-mutex)) (define *completed-mutex* (make-mutex)) (define *cache-on* #f) + + (define (db:set-sync db) (let* ((syncval (config-lookup *configdat* "setup" "synchronous")) (val (cond ;; 0 | OFF | 1 | NORMAL | 2 | FULL; ((not syncval) #f) @@ -69,12 +72,12 @@ (begin (debug:print 0 "ERROR: Attempted to open db when not in megatest area. Exiting.") (exit)))) (let* ((dbpath (conc *toppath* "/megatest.db")) ;; fname) (dbexists (file-exists? dbpath)) - (write-access (file-write-access? dbpath)) (db (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) + (write-access (file-write-access? dbpath)) (handler (make-busy-timeout (if (args:get-arg "-override-timeout") (string->number (args:get-arg "-override-timeout")) 136000)))) ;; 136000))) ;; 136000 = 2.2 minutes (if (and dbexists (not write-access)) @@ -85,10 +88,217 @@ (db:initialize db)) ;; Moving db:set-sync to a call in run.scm - it is a persistent value and only needs to be set once ;; (db:set-sync db) db)) +(define (open-in-mem-db) + (let* ((path (configf:lookup *configdat* "setup" "tmpdb")) + (fname (if path (conc path "/temp-megatest.db") #f)) + (exists (and path (file-exists? fname))) + (db (if path + (begin + (create-directory path #t) + (sqlite3:open-database fname)) + (sqlite3:open-database ":memory:"))) + (handler (make-busy-timeout 3600))) + (if (or (not path) + (not exists)) + (db:initialize db)) + (sqlite3:set-busy-handler! db handler) + db)) + +;; (define (db:sync-table tblname fields fromdb todb) + +(define (db:tbls db) + (let ((keys (db:get-keys db))) + (list + (list "metadat" '("var" #f) '("val" #f)) + (append (list "runs" + '("id" #f)) + (map (lambda (k)(list k #f)) + (append keys + (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count"))))))) + +;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) ) +(define (db:sync-tables tbls fromdb todb) + (let ((stmts (make-hash-table)) ;; table-field => stmt + (all-stmts '())) ;; ( ( stmt1 value1 ) ( stml2 value2 )) + (for-each ;; table + (lambda (tabledat) + (let* ((tablename (car tabledat)) + (fields (cdr tabledat)) + (num-fields (length fields)) + (field->num (make-hash-table)) + (num->field (apply vector (map car fields))) + (full-sel (conc "SELECT " (string-intersperse (map car fields) ",") + " FROM " tablename ";")) + (full-ins (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) " + " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );")) + (fromdat '()) + (todat (make-hash-table)) + (count 0)) + + ;; set up the field->num table + (for-each + (lambda (field) + (hash-table-set! field->num field count) + (set! count (+ count 1))) + fields) + + ;; read the source table + (sqlite3:for-each-row + (lambda (a . b) + (set! fromdat (cons (apply vector a b) fromdat))) + fromdb + full-sel) + + ;; read the target table + (sqlite3:for-each-row + (lambda (a . b) + (hash-table-set! todat a (apply vector a b))) + todb + full-sel) + + ;; first pass implementation, just insert all changed rows + (let ((stmth (sqlite3:prepare todb full-ins))) + (sqlite3:with-transaction + todb + (lambda () + (for-each ;; + (lambda (fromrow) + (let* ((a (vector-ref fromrow 0)) + (curr (hash-table-ref todat a)) + (same #t)) + (let loop ((i 0)) + (if (not (equal? (vector-ref fromrow i)(vector-ref curr i))) + (set! same #f)) + (if (and same + (< i (- num-fields 1))) + (loop (+ i 1)))) + (if (not same)(apply sqlite3:execute full-ins (vector->list fromrow))))) + fromdat))) + (sqlite3:finalize! stmth)))) + tbls))) + + +(define (db:sync-to fromdb todb) + ;; strategy + ;; 1. Get all run-ids + ;; 2. For each run-id + ;; a. Sync that run in a transaction + (let ((trecchgd 0) + (rrecchgd 0) + (tmrecchgd 0)) + + ;; First sync test_meta data + (let ((tmgetstmt (sqlite3:prepare todb "SELECT id,testname,author,owner,description,reviewed,iterated,avg_runtime,avg_disk,tags,jobgroup FROM test_meta WHERE id=?;")) + (tmputstmt (sqlite3:prepare todb "INSERT OR REPLACE INTO test_meta (id,testname,author,owner,description,reviewed,iterated,avg_runtime,avg_disk,tags,jobgroup) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")) + (tmdats (db:testmeta-get-all fromdb))) + ;; (debug:print 7 "Updating as many as " (length tdats) " records for run " run-id) + (for-each + (lambda (tmdat) ;; iterate over tests + (let ((testm-id (vector-ref tmdat 0))) + (sqlite3:with-transaction + todb + (lambda () + (let ((curr-tmdat #f)) + (sqlite3:for-each-row + (lambda (a . b) + (set! curr-tmdat (apply vector a b))) + tmgetstmt testm-id) + (if (not (equal? curr-tmdat tmdat)) ;; something changed + (begin + (debug:print 0 " test-id: " testm-id + "\ncurr-tdat: " curr-tmdat + "\n tdat: " tmdat) + (apply sqlite3:execute tmputstmt (vector->list tmdat)) + (set! tmrecchgd (+ tmrecchgd 1))))))))) + tmdats) + (sqlite3:finalize! tmgetstmt) + (sqlite3:finalize! tmputstmt)) + + ;; First sync tests data + (let ((run-ids (db:get-all-run-ids fromdb)) + (tgetstmt (sqlite3:prepare todb "SELECT id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment FROM tests WHERE id=?;")) + (tputstmt (sqlite3:prepare todb "INSERT OR REPLACE INTO tests (id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );"))) + (for-each + (lambda (run-id) + (let ((tdats (db:get-all-tests-info-by-run-id fromdb run-id))) + ;; (debug:print 7 "Updating as many as " (length tdats) " records for run " run-id) + (for-each + (lambda (tdat) ;; iterate over tests + (let ((test-id (vector-ref tdat 0))) + (sqlite3:with-transaction + todb + (lambda () + (let ((curr-tdat #f)) + (sqlite3:for-each-row + (lambda (a . b) + (set! curr-tdat (apply vector a b))) + tgetstmt + test-id) + (if (not (equal? curr-tdat tdat)) ;; something changed + (begin + (debug:print 0 " test-id: " test-id + "\ncurr-tdat: " curr-tdat + "\n tdat: " tdat) + (apply sqlite3:execute tputstmt (vector->list tdat)) + (set! trecchgd (+ trecchgd 1))))))))) + tdats))) + run-ids) + (sqlite3:finalize! tgetstmt) + (sqlite3:finalize! tputstmt)) + + ;; Next sync runs table + (let* ((rdats '()) + (keys (db:get-keys fromdb)) + (rstdfields (conc "id," (string-intersperse keys ",") ",runname,state,status,owner,event_time,comment,fail_count,pass_count")) + (rnumfields (length (string-split rstdfields ","))) + (runslots (string-intersperse (make-list rnumfields "?") ",")) + (rgetstmt (sqlite3:prepare todb (conc "SELECT " rstdfields " FROM runs WHERE id=?;"))) + (rputstmt (sqlite3:prepare todb (conc "INSERT OR REPLACE INTO runs (" rstdfields ") VALUES ( " runslots " );")))) + ;; first collect all the source run data + (sqlite3:for-each-row + (lambda (a . b) + (set! rdats (cons (apply vector a b) rdats))) + fromdb + (conc "SELECT " rstdfields " FROM runs;")) + (sqlite3:with-transaction + todb + (lambda () + (for-each + (lambda (rdat) + (let ((run-id (vector-ref rdat 0)) + (curr-rdat #f)) + ;; first get the current value of the equivalent row from the target + ;; read, then insert/overwrite if different + (sqlite3:for-each-row + (lambda (a . b) + (set! curr-rdat (apply vector a b))) + rgetstmt + run-id) + (if (not (equal? curr-rdat rdat)) + (begin + (debug:print 0 " run-id: " run-id + "\ncurr-rdat: " curr-rdat + "\n rdat: " rdat) + (set! rrecchgd (+ rrecchgd 1)) + (apply sqlite3:execute rputstmt (vector->list rdat)))))) + rdats))) + (sqlite3:finalize! rgetstmt) + (sqlite3:finalize! rputstmt)) + + (if (> rrecchgd 0) (debug:print 0 "synced " rrecchgd " changed records in runs table")) + (if (> trecchgd 0) (debug:print 0 "synced " trecchgd " changed records in tests table")) + (if (> tmrecchgd 0) (debug:print 0 "sync'd " tmrecchgd " changed records in test_meta table")) + (+ rrecchgd trecchgd tmrecchgd))) + +(define (db:sync-back) + (db:sync-to *inmemdb* *db*)) + ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) (debug:print-info 11 "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params) (if (or *db-write-access* (not (member proc *db:all-write-procs*))) @@ -165,43 +375,43 @@ (sqlite3:execute db "INSERT INTO keys (fieldname,fieldtype) VALUES (?,?);" key "TEXT")) keys) (sqlite3:execute db (conc "CREATE TABLE IF NOT EXISTS runs (id INTEGER PRIMARY KEY, " fieldstr (if havekeys "," "") - "runname TEXT," - "state TEXT DEFAULT ''," - "status TEXT DEFAULT ''," - "owner TEXT DEFAULT ''," - "event_time TIMESTAMP," - "comment TEXT DEFAULT ''," + "runname TEXT DEFAULT 'norun'," + "state TEXT DEFAULT ''," + "status TEXT DEFAULT ''," + "owner TEXT DEFAULT ''," + "event_time TIMESTAMP DEFAULT (strftime('%s','now'))," + "comment TEXT DEFAULT ''," "fail_count INTEGER DEFAULT 0," "pass_count INTEGER DEFAULT 0," "CONSTRAINT runsconstraint UNIQUE (runname" (if havekeys "," "") keystr "));")) (sqlite3:execute db (conc "CREATE INDEX runs_index ON runs (runname" (if havekeys "," "") keystr ");")) (sqlite3:execute db "CREATE TABLE IF NOT EXISTS tests (id INTEGER PRIMARY KEY, - run_id INTEGER, - testname TEXT, - host TEXT DEFAULT 'n/a', - cpuload REAL DEFAULT -1, - diskfree INTEGER DEFAULT -1, - uname TEXT DEFAULT 'n/a', - rundir TEXT DEFAULT 'n/a', - shortdir TEXT DEFAULT '', - item_path TEXT DEFAULT '', - state TEXT DEFAULT 'NOT_STARTED', - status TEXT DEFAULT 'FAIL', - attemptnum INTEGER DEFAULT 0, - final_logf TEXT DEFAULT 'logs/final.log', - logdat BLOB, - run_duration INTEGER DEFAULT 0, - comment TEXT DEFAULT '', - event_time TIMESTAMP, - fail_count INTEGER DEFAULT 0, - pass_count INTEGER DEFAULT 0, - archived INTEGER DEFAULT 0, -- 0=no, 1=in progress, 2=yes + run_id INTEGER DEFAULT -1, + testname TEXT DEFAULT 'noname', + host TEXT DEFAULT 'n/a', + cpuload REAL DEFAULT -1, + diskfree INTEGER DEFAULT -1, + uname TEXT DEFAULT 'n/a', + rundir TEXT DEFAULT 'n/a', + shortdir TEXT DEFAULT '', + item_path TEXT DEFAULT '', + state TEXT DEFAULT 'NOT_STARTED', + status TEXT DEFAULT 'FAIL', + attemptnum INTEGER DEFAULT 0, + final_logf TEXT DEFAULT 'logs/final.log', + logdat TEXT DEFAULT '', + run_duration INTEGER DEFAULT 0, + comment TEXT DEFAULT '', + event_time TIMESTAMP DEFAULT (strftime('%s','now')), + fail_count INTEGER DEFAULT 0, + pass_count INTEGER DEFAULT 0, + archived INTEGER DEFAULT 0, -- 0=no, 1=in progress, 2=yes CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path) );") (sqlite3:execute db "CREATE INDEX tests_index ON tests (run_id, testname, item_path);") (sqlite3:execute db "CREATE VIEW runs_tests AS SELECT * FROM runs INNER JOIN tests ON runs.id=tests.run_id;") (sqlite3:execute db "CREATE TABLE IF NOT EXISTS test_steps @@ -221,14 +431,14 @@ (sqlite3:execute db "CREATE TABLE IF NOT EXISTS test_meta (id INTEGER PRIMARY KEY, testname TEXT DEFAULT '', author TEXT DEFAULT '', owner TEXT DEFAULT '', description TEXT DEFAULT '', - reviewed TIMESTAMP, + reviewed TIMESTAMP DEFAULT (strftime('%s','now')), iterated TEXT DEFAULT '', - avg_runtime REAL, - avg_disk REAL, + avg_runtime REAL DEFAULT -1, + avg_disk REAL DEFAULT -1, tags TEXT DEFAULT '', jobgroup TEXT DEFAULT 'default', CONSTRAINT test_meta_constraint UNIQUE (testname));") (sqlite3:execute db "CREATE TABLE IF NOT EXISTS test_data (id INTEGER PRIMARY KEY, test_id INTEGER, @@ -245,110 +455,10 @@ ;; Must do this *after* running patch db !! No more. (db:set-var db "MEGATEST_VERSION" megatest-version) (debug:print-info 11 "db:initialize END") )) -;;====================================================================== -;; T E S T S P E C I F I C D B -;;====================================================================== - -;; Create the sqlite db for the individual test(s) -(define (open-test-db work-area) - (debug:print-info 11 "open-test-db " work-area) - (if (and work-area - (directory? work-area) - (file-read-access? work-area)) - (let* ((dbpath (conc work-area "/testdat.db")) - (tdb-writeable (file-write-access? dbpath)) - (dbexists (file-exists? dbpath)) - (handler (make-busy-timeout (if (args:get-arg "-override-timeout") - (string->number (args:get-arg "-override-timeout")) - 136000)))) - (handle-exceptions - exn - (begin - (debug:print 2 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test" - ((condition-property-accessor 'exn 'message) exn)) - (set! db (sqlite3:open-database ":memory:"))) ;; open an in-memory db to allow readonly access - (set! db (sqlite3:open-database dbpath))) - (if *db-write-access* (sqlite3:set-busy-handler! db handler)) - (if (not dbexists) - (begin - (sqlite3:execute db "PRAGMA synchronous = FULL;") - (debug:print-info 11 "Initialized test database " dbpath) - (db:testdb-initialize db))) - ;; (sqlite3:execute db "PRAGMA synchronous = 0;") - (debug:print-info 11 "open-test-db END (sucessful)" work-area) - ;; now let's test that everything is correct - (handle-exceptions - exn - (begin - (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test" - ((condition-property-accessor 'exn 'message) exn)) - #f) - ;; Is there a cheaper single line operation that will check for existance of a table - ;; and raise an exception ? - (sqlite3:execute db "SELECT id FROM test_data LIMIT 1;")) - db) - (begin - (debug:print-info 11 "open-test-db END (unsucessful)" work-area) - #f))) - -;; find and open the testdat.db file for an existing test -(define (db:open-test-db-by-test-id db test-id #!key (work-area #f)) - (let* ((test-path (if work-area - work-area - (cdb:remote-run db:test-get-rundir-from-test-id db test-id)))) - (debug:print 3 "TEST PATH: " test-path) - (open-test-db test-path))) - -(define (db:testdb-initialize db) - (debug:print 11 "db:testdb-initialize START") - (for-each - (lambda (sqlcmd) - (sqlite3:execute db sqlcmd)) - (list "CREATE TABLE IF NOT EXISTS test_rundat ( - id INTEGER PRIMARY KEY, - update_time TIMESTAMP, - cpuload INTEGER DEFAULT -1, - diskfree INTEGER DEFAULT -1, - diskusage INTGER DEFAULT -1, - run_duration INTEGER DEFAULT 0);" - "CREATE TABLE IF NOT EXISTS test_data ( - id INTEGER PRIMARY KEY, - test_id INTEGER, - category TEXT DEFAULT '', - variable TEXT, - value REAL, - expected REAL, - tol REAL, - units TEXT, - comment TEXT DEFAULT '', - status TEXT DEFAULT 'n/a', - type TEXT DEFAULT '', - CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable));" - "CREATE TABLE IF NOT EXISTS test_steps ( - id INTEGER PRIMARY KEY, - test_id INTEGER, - stepname TEXT, - state TEXT DEFAULT 'NOT_STARTED', - status TEXT DEFAULT 'n/a', - event_time TIMESTAMP, - comment TEXT DEFAULT '', - logfile TEXT DEFAULT '', - CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state));" - ;; test_meta can be used for handing commands to the test - ;; e.g. KILLREQ - ;; the ackstate is set to 1 once the command has been completed - "CREATE TABLE IF NOT EXISTS test_meta ( - id INTEGER PRIMARY KEY, - var TEXT, - val TEXT, - ackstate INTEGER DEFAULT 0, - CONSTRAINT metadat_constraint UNIQUE (var));")) - (debug:print 11 "db:testdb-initialize END")) - ;;====================================================================== ;; L O G G I N G D B ;;====================================================================== (define (open-logging-db) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) @@ -713,10 +823,18 @@ res) (begin (debug:print 0 "ERROR: Called without all necessary keys") #f)))) +(define (db:get-all-run-ids db) + (let ((res '())) + (sqlite3:for-each-row + (lambda (run-id) + (set! res (cons run-id res))) + db + "SELECT DISTINCT run_id FROM tests;") + res)) ;; replace header and keystr with a call to runs:get-std-run-fields ;; ;; keypatts: ( (KEY1 "abc%def")(KEY2 "%") ) ;; runpatts: patt1,patt2 ... @@ -836,11 +954,11 @@ (set! key-patt (conc key-patt " AND " key " " wildtype " '" patt "'")) (begin (debug:print 0 "ERROR: searching for runs with no pattern set for " fulkey) (exit 6))))) keyvals) - (set! qry-str (conc "SELECT " keystr " FROM runs WHERE state != 'deleted' AND runname " runwildtype " ? " key-patt " ORDER BY event_time" + (set! qry-str (conc "SELECT " keystr " FROM runs WHERE state != 'deleted' AND runname " runwildtype " ? " key-patt " ORDER BY event_time " (if limit (conc " LIMIT " limit) "") (if offset (conc " OFFSET " offset) "") ";")) (debug:print-info 4 "runs:get-runs-by-patt qry=" qry-str " " runnamepatt) (sqlite3:for-each-row @@ -853,11 +971,11 @@ ;; use (get-value-by-header (db:get-header runinfo)(db:get-row runinfo)) (define (db:get-run-info db run-id) ;;(if (hash-table-ref/default *run-info-cache* run-id #f) ;; (hash-table-ref *run-info-cache* run-id) - (let* ((res #f) + (let* ((res (vector #f #f #f #f)) (keys (db:get-keys db)) (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (header (append keys remfields)) (keystr (conc (keys->keystr keys) "," (string-intersperse remfields ",")))) @@ -872,32 +990,26 @@ (let ((finalres (vector header res))) ;; (hash-table-set! *run-info-cache* run-id finalres) finalres))) (define (db:set-comment-for-run db run-id comment) - (debug:print-info 11 "db:set-comment-for-run START run-id: " run-id " comment: " comment) - (sqlite3:execute db "UPDATE runs SET comment=? WHERE id=?;" comment run-id) - (debug:print-info 11 "db:set-comment-for-run END run-id: " run-id " comment: " comment)) + (sqlite3:execute db "UPDATE runs SET comment=? WHERE id=?;" comment run-id)) ;; does not (obviously!) removed dependent data. But why not!!? (define (db:delete-run db run-id) - (common:clear-caches) ;; don't trust caches after doing any deletion ;; First set any related tests to DELETED (let ((stmt1 (sqlite3:prepare db "UPDATE tests SET state='DELETED',comment='' WHERE run_id=?;")) (stmt2 (sqlite3:prepare db "UPDATE runs SET state='deleted',comment='' WHERE id=?;"))) (sqlite3:with-transaction db (lambda () (sqlite3:execute stmt1 run-id) (sqlite3:execute stmt2 run-id))) (sqlite3:finalize! stmt1) (sqlite3:finalize! stmt2))) -;; (sqlite3:execute db "DELETE FROM runs WHERE id=?;" run-id)) (define (db:update-run-event_time db run-id) - (debug:print-info 11 "db:update-run-event_time START run-id: " run-id) - (sqlite3:execute db "UPDATE runs SET event_time=strftime('%s','now') WHERE id=?;" run-id) - (debug:print-info 11 "db:update-run-event_time END run-id: " run-id)) + (sqlite3:execute db "UPDATE runs SET event_time=strftime('%s','now') WHERE id=?;" run-id)) (define (db:lock/unlock-run db run-id lock unlock user) (let ((newlockval (if lock "locked" (if unlock "unlocked" @@ -975,13 +1087,11 @@ ;; states and statuses are lists, turn them into ("PASS","FAIL"...) and use NOT IN ;; i.e. these lists define what to NOT show. ;; states and statuses are required to be lists, empty is ok ;; not-in #t = above behaviour, #f = must match -(define (db:get-tests-for-run db run-id testpatt states statuses offset limit not-in sort-by sort-order - #!key - (qryvals #f)) +(define (db:get-tests-for-run db run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals) (let* ((qryvalstr (case qryvals ((shortlist) "id,run_id,testname,item_path,state,status") ((#f) "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment") (else qryvals))) (res '()) @@ -1020,15 +1130,15 @@ ((rundir) " ORDER BY length(rundir) ") ((testname) (conc " ORDER BY testname " (if sort-order (conc sort-order ",") "") " item_path ")) ((statestatus) (conc " ORDER BY state " (if sort-order (conc sort-order ",") "") " status ")) ((event_time) " ORDER BY event_time ") (else (if (string? sort-by) - (conc " ORDER BY " sort-by) - ""))) - (if sort-order sort-order "") - (if limit (conc " LIMIT " limit) "") - (if offset (conc " OFFSET " offset) "") + (conc " ORDER BY " sort-by " ") + " "))) + (if sort-order sort-order " ") + (if limit (conc " LIMIT " limit) " ") + (if offset (conc " OFFSET " offset) " ") ";" ))) (debug:print-info 8 "db:get-tests-for-run qry=" qry) (sqlite3:for-each-row (lambda (a . b) ;; id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment) @@ -1054,19 +1164,21 @@ (vector-ref inrec 3) ;; item-path -1 "-" "-")) (define (db:get-tests-for-run-state-status db run-id testpatt) - (let ((res '()) - (tests-match-qry (tests:match->sqlqry testpatt))) + (let* ((res '()) + (tests-match-qry (tests:match->sqlqry testpatt)) + (qry (conc "SELECT id,testname,item_path,state,status FROM tests WHERE run_id=? " + (if tests-match-qry (conc " AND (" tests-match-qry ") ") "")))) + (debug:print-info 8 "db:get-tests-for-run qry=" qry) (sqlite3:for-each-row (lambda (id testname item-path state status) ;; id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment (set! res (cons (vector id run-id testname state status -1 "" -1 -1 "" "-" item-path -1 "-" "-") res))) db - (conc "SELECT id,testname,item_path,state,status FROM tests WHERE run_id=? " - (if tests-match-qry (conc " AND (" tests-match-qry ") ") "")) + qry run-id) res)) (define (db:get-testinfo-state-status db test-id) (let ((res #f)) @@ -1135,43 +1247,18 @@ db qry ) res)) -;; this one is a bit broken BUG FIXME -(define (db:delete-test-step-records db test-id #!key (work-area #f)) - ;; Breaking it into two queries for better file access interleaving - (let* ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))) - ;; test db's can go away - must check every time - (if (sqlite3:database? tdb) - (begin - (sqlite3:execute tdb "DELETE FROM test_steps;") - (sqlite3:execute tdb "DELETE FROM test_data;") - (sqlite3:finalize! tdb))))) - -;; -(define (db:delete-test-records db tdb test-id #!key (force #f)) - (common:clear-caches) - (if tdb - (begin - (sqlite3:execute tdb "DELETE FROM test_steps;") - (sqlite3:execute tdb "DELETE FROM test_data;"))) - ;; (sqlite3:execute db "DELETE FROM tests WHERE id=?;" test-id)) - (if db - (begin - (sqlite3:execute db "DELETE FROM test_steps WHERE test_id=?;" test-id) - (sqlite3:execute db "DELETE FROM test_data WHERE test_id=?;" test-id) - (if force - (sqlite3:execute db "DELETE FROM tests WHERE id=?;" test-id) - (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id))))) +(define (db:delete-test-records db test-id) + (tdb:delete-test-step-records db test-id) + (sqlite3:execute db "UPDATE tests SET state='DELETED',status='n/a',comment='' WHERE id=?;" test-id)) (define (db:delete-tests-for-run db run-id) - (common:clear-caches) (sqlite3:execute db "DELETE FROM tests WHERE run_id=?;" run-id)) (define (db:delete-old-deleted-test-records db) - (common:clear-caches) (let ((targtime (- (current-seconds)(* 30 24 60 60)))) ;; one month in the past (sqlite3:execute db "DELETE FROM tests WHERE state='DELETED' AND event_timesqlqry testpatt)) - (runsqry (sqlite3:prepare db (conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';"))) - (tstsqry (sqlite3:prepare db (conc "SELECT rundir FROM tests WHERE run_id=? AND " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")))) + (rqry (conc "SELECT id FROM runs WHERE " keystr " AND runname LIKE '" runname "';")) + (runsqry (sqlite3:prepare db rqry)) + (tqry (conc "SELECT rundir FROM tests WHERE run_id=? AND " testqry " AND state LIKE '" statepatt "' AND status LIKE '" statuspatt "' ORDER BY event_time ASC;")) + (tstsqry (sqlite3:prepare db tqry))) + (debug:print 8 "db:test-get-paths-matching-keynames-target-new\n rqry=" rqry "\n tqry=" tqry) (sqlite3:for-each-row (lambda (rid) (set! row-ids (cons rid row-ids))) runsqry) + (sqlite3:finalize! runsqry) (for-each (lambda (rid) (sqlite3:for-each-row (lambda (p) (set! res (cons p res))) tstsqry rid)) row-ids) (sqlite3:finalize! tstsqry) - (sqlite3:finalize! runsqry) res)) ;; look through tests from matching runs for a file (define (db:test-get-first-path-matching db keynames target fname) ;; [refpaths] is the section where references to other megatest databases are stored @@ -1528,23 +1547,23 @@ ;;====================================================================== ;; NOTE: Can remove the regex and base64 encoding for zmq (define (db:obj->string obj) (case *transport-type* - ((fs) obj) - ((http) + ;; ((fs) obj) + ((http fs) (string-substitute (regexp "=") "_" (base64:base64-encode (with-output-to-string (lambda ()(serialize obj)))) #t)) ((zmq)(with-output-to-string (lambda ()(serialize obj)))) (else obj))) (define (db:string->obj msg) (case *transport-type* - ((fs) msg) - ((http) + ;; ((fs) msg) + ((http fs) (if (string? msg) (with-input-from-string (base64:base64-decode (string-substitute (regexp "_") "=" msg #t)) @@ -1551,166 +1570,27 @@ (lambda ()(deserialize))) (vector #f #f #f))) ;; crude reply for when things go awry ((zmq)(with-input-from-string msg (lambda ()(deserialize)))) (else msg))) -(define (cdb:use-non-blocking-mode proc) - (set! *client-non-blocking-mode* #t) - (let ((res (proc))) - (set! *client-non-blocking-mode* #f) - res)) - -;; params = 'target cached remparams -;; -;; make-vector-record cdb packet client-sig qtype immediate query-sig params qtime -;; -;; cdb:client-call is the unified interface to all the transports. It dispatches the -;; query to a server routine (e.g. server:client-send-recieve) that -;; transports the data to the server where it is passed to db:process-queue-item -;; which either returns the data to the calling server routine or -;; directly calls the returning procedure (e.g. zmq). -;; -(define (cdb:client-call serverdat qtype immediate numretries . params) - (debug:print-info 11 "cdb:client-call serverdat=" serverdat ", qtype=" qtype ", immediate=" immediate ", numretries=" numretries ", params=" params) - (case *transport-type* - ((fs) - (let ((packet (vector "na" qtype immediate "na" params 0))) - (fs:process-queue-item packet))) - ((http) - (let* ((client-sig (client:get-signature)) - (query-sig (message-digest-string (md5-primitive) (conc qtype immediate params))) - (zdat (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds))))) ;; (with-output-to-string (lambda ()(serialize params)))) - (debug:print-info 11 "zdat=" zdat) - (let* ((res #f) - (rawdat (http-transport:client-send-receive serverdat zdat)) - (tmp #f)) - (debug:print-info 11 "Sent " zdat ", received " rawdat) - (if rawdat - (begin - (set! tmp (db:string->obj rawdat)) - (vector-ref tmp 2)) - (begin - (debug:print 0 "ERROR: Communication with the server failed. Exiting if possible") - (exit 1)))))) - ((zmq) - (handle-exceptions - exn - (begin - (debug:print-info 0 "cdb:client-call timeout or error. Trying again in 5 seconds") - (thread-sleep! 5) - (if (> numretries 0)(apply cdb:client-call serverdat qtype immediate (- numretries 1) params))) - (let* ((push-socket (vector-ref serverdat 0)) - (sub-socket (vector-ref serverdat 1)) - (client-sig (client:get-signature)) - (query-sig (message-digest-string (md5-primitive) (conc qtype immediate params))) - (zdat (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds)))) ;; (with-output-to-string (lambda ()(serialize params)))) - (res #f) - (send-receive (lambda () - (debug:print-info 11 "sending message") - (send-message push-socket zdat) - (debug:print-info 11 "message sent") - (let loop () - ;; get the sender info - ;; this should match (client:get-signature) - ;; we will need to process "all" messages here some day - (receive-message* sub-socket) - ;; now get the actual message - (let ((myres (db:string->obj (receive-message* sub-socket)))) - (if (equal? query-sig (vector-ref myres 1)) - (set! res (vector-ref myres 2)) - (loop))))))) - ;; (timeout (lambda () - ;; (let loop ((n numretries)) - ;; (thread-sleep! 15) - ;; (if (not res) - ;; (if (> numretries 0) - ;; (begin - ;; (debug:print 2 "WARNING: no reply to query " params ", trying resend") - ;; (debug:print-info 11 "re-sending message") - ;; (send-message push-socket zdat) - ;; (debug:print-info 11 "message re-sent") - ;; (loop (- n 1))) - ;; ;; (apply cdb:client-call *runremote* qtype immediate (- numretries 1) params)) - ;; (begin - ;; (debug:print 0 "ERROR: cdb:client-call timed out " params ", exiting.") - ;; (exit 5)))))))) - (debug:print-info 11 "Starting threads") - (let ((th1 (make-thread send-receive "send receive")) - ;; (th2 (make-thread timeout "timeout")) - ) - (thread-start! th1) - ;; (thread-start! th2) - (thread-join! th1) - (debug:print-info 11 "cdb:client-call returning res=" res) - res)))))) - -(define (cdb:set-verbosity serverdat val) - (cdb:client-call serverdat 'set-verbosity #f *default-numtries* val)) - -(define (cdb:login serverdat keyval signature) - (cdb:client-call serverdat 'login #t *default-numtries* keyval megatest-version signature)) - -(define (cdb:logout serverdat keyval signature) - (cdb:client-call serverdat 'logout #t *default-numtries* keyval signature)) - -(define (cdb:num-clients serverdat) - (cdb:client-call serverdat 'numclients #t *default-numtries*)) - -;; I think this would be more efficient if executed on client side FIXME??? -(define (cdb:test-set-status-state serverdat test-id status state msg) - (if (member state '("LAUNCHED" "REMOTEHOSTSTART")) - (cdb:client-call serverdat 'set-test-start-time #t *default-numtries* test-id)) - (if msg - (cdb:client-call serverdat 'state-status-msg #t *default-numtries* state status msg test-id) - (cdb:client-call serverdat 'state-status #t *default-numtries* state status test-id))) ;; run-id test-name item-path minutes cpuload diskfree tmpfree) - -(define (cdb:test-rollup-test_data-pass-fail serverdat test-id) - (cdb:client-call serverdat 'test_data-pf-rollup #t *default-numtries* test-id test-id test-id test-id)) - -(define (cdb:pass-fail-counts serverdat test-id fail-count pass-count) - (cdb:client-call serverdat 'pass-fail-counts #t *default-numtries* fail-count pass-count test-id)) - -(define (cdb:tests-register-test serverdat run-id test-name item-path) - (cdb:client-call serverdat 'register-test #t *default-numtries* run-id test-name item-path)) - -;; more transactioned calls, these for roll-up-pass-fail stuff -(define (cdb:update-pass-fail-counts serverdat run-id test-name) - (cdb:client-call serverdat 'update-fail-pass-counts #t *default-numtries* run-id test-name run-id test-name run-id test-name)) - -(define (cdb:top-test-set-running serverdat run-id test-name) - (cdb:client-call serverdat 'top-test-set-running #t *default-numtries* run-id test-name)) - -(define (cdb:top-test-set-per-pf-counts serverdat run-id test-name) - (cdb:client-call serverdat 'top-test-set-per-pf-counts #t *default-numtries* run-id test-name run-id test-name run-id test-name)) - -;;= - -(define (cdb:flush-queue serverdat) - (cdb:client-call serverdat 'flush #f *default-numtries*)) - -(define (cdb:kill-server serverdat pid) - (cdb:client-call serverdat 'killserver #t *default-numtries* pid)) - -(define (cdb:roll-up-pass-fail-counts serverdat run-id test-name item-path status) - (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:roll-up-pass-fail-counts #f run-id test-name item-path status)) - -(define (cdb:get-test-info serverdat run-id test-name item-path) - (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info #f run-id test-name item-path)) - -(define (cdb:get-test-info-by-id serverdat test-id) - (let ((test-dat (cdb:client-call serverdat 'immediate #f *default-numtries* open-run-close db:get-test-info-by-id #f test-id))) - (hash-table-set! *test-info* test-id (vector (current-seconds) test-dat)) ;; cached for use where up-to-date info is not needed - test-dat)) - -;; db should be db open proc or #f -(define (cdb:remote-run proc db . params) - (if (or *db-write-access* - (not (member proc *db:all-write-procs*))) - (apply cdb:client-call *runremote* 'immediate #f *default-numtries* open-run-close proc #f params) - (begin - (debug:print 0 "ERROR: Attempt to access read-only database") - #f))) +(define (db:test-set-status-state db test-id status state msg) + (if (member state '("LAUNCHED" "REMOTEHOSTSTART")) + (db:general-call db 'set-test-start-time (list test-id))) + (if msg + (db:general-call db 'state-status-msg (list state status msg test-id)) + (db:general-call db 'state-status (list state status test-id)))) + +(define (db:roll-up-pass-fail-counts db run-id test-name item-path status) + (if (and (not (equal? item-path "")) + (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP"))) + (begin + (db:general-call db 'update-pass-fail-counts (list run-id test-name run-id test-name run-id test-name)) + (if (equal? status "RUNNING") + (db:general-call db 'top-test-set-running (list run-id test-name)) + (db:general-call db 'top-test-set-per-pf-counts (list run-id test-name run-id test-name run-id test-name))) + #f) + #f)) (define (db:test-get-logfile-info db run-id test-name) (let ((res #f)) (sqlite3:for-each-row (lambda (path final_logf) @@ -1757,11 +1637,11 @@ '(update-run-duration "UPDATE tests SET run_duration=? WHERE id=?;") '(update-uname-host "UPDATE tests SET uname=?,host=? WHERE id=?;") '(update-test-state "UPDATE tests SET state=? WHERE state=? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") '(update-test-status "UPDATE tests SET status=? WHERE status like ? AND run_id=? AND testname=? AND NOT (item_path='' AND testname IN (SELECT DISTINCT testname FROM tests WHERE testname=? AND item_path != ''));") ;; stuff for roll-up-pass-fail-counts - '(update-fail-pass-counts "UPDATE tests + '(update-pass-fail-counts "UPDATE tests SET fail_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status IN ('FAIL','CHECK')), pass_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status IN ('PASS','WARN','WAIVED')) WHERE run_id=? AND testname=? AND item_path='';") '(top-test-set-running "UPDATE tests SET state='RUNNING' WHERE run_id=? AND testname=? AND item_path='';") '(top-test-set-per-pf-counts "UPDATE tests @@ -1791,197 +1671,130 @@ sync set-verbosity killserver )) -;; not used, intended to indicate to run in calling process -(define db:run-local-queries '()) ;; rollup-tests-pass-fail)) - -(define (db:process-cached-writes db) - (let ((queries (make-hash-table)) - (data #f)) - (mutex-lock! *incoming-mutex*) - ;; data is a list of query packets (length data) 0) - ;; Process if we have data - (begin - (debug:print-info 7 "Writing cached data " data) - - ;; Prepare the needed sql statements - ;; - (for-each (lambda (request-item) - (let ((stmt-key (vector-ref request-item 0)) - (query (vector-ref request-item 1))) - (hash-table-set! queries stmt-key (sqlite3:prepare db query)))) - data) - - ;; No outer loop needed. Single loop for write items only. Reads trigger flush of queue - ;; and then are executed. - (sqlite3:with-transaction - db - (lambda () - (for-each - (lambda (hed) - (let* ((params (vector-ref hed 2)) - (stmt-key (vector-ref hed 0)) - (stmt (hash-table-ref/default queries stmt-key #f))) - (if stmt - (apply sqlite3:execute stmt params) - (debug:print 0 "ERROR: Problem Executing " stmt-key " for " params)))) - data))) - - ;; let all the waiting calls know all is done - (mutex-lock! *completed-mutex*) - (for-each (lambda (item) - (let ((qry-sig (cdb:packet-get-client-sig item))) - (debug:print-info 7 "Registering query " qry-sig " as done") - (hash-table-set! *completed-writes* qry-sig #t))) - data) - (mutex-unlock! *completed-mutex*) - - ;; Finalize the statements. Should this be done inside the mutex above? - ;; I think sqlite3 mutexes will keep the data safe - (for-each (lambda (stmt-key) - (sqlite3:finalize! (hash-table-ref queries stmt-key))) - (hash-table-keys queries)) - - ;; Do a little record keeping - (let ((cache-size (length data))) - (if (> cache-size *max-cache-size*) - (set! *max-cache-size* cache-size))) - #t) - #f))) +(define (db:login db calling-path calling-version client-signature) + (if (and (equal? calling-path *toppath*) + (equal? megatest-version calling-version)) + (begin + (hash-table-set! *logged-in-clients* client-signature (current-seconds)) + '(#t "successful login")) ;; path matches - pass! Should vet the caller at this time ... + (list #f (conc "Login failed due to mismatch paths: " calling-path ", " *toppath*)))) + +(define (db:process-write db request-item) + (let ((stmt-key (vector-ref request-item 0)) + (query (vector-ref request-item 1)) + (params (vector-ref request-item 2)) + (queryh (sqlite3:prepare db query))) + (apply sqlite3:execute stmt params) + #f)) (define *db:process-queue-mutex* (make-mutex)) (define *number-of-writes* 0) (define *writes-total-delay* 0) (define *total-non-write-delay* 0) (define *number-non-write-queries* 0) -;; The queue is a list of vectors where the zeroth slot indicates the type of query to -;; apply and the second slot is the time of the query and the third entry is a list of -;; values to be applied +(define (db:general-call db stmtname params) + (let ((query (let ((q (alist-ref (if (string? stmtname) + (string->symbol stmtname) + stmtname) + db:queries))) + (if q (car q) #f)))) + (apply sqlite3:execute db query params) + #t)) + +;; get the previous record for when this test was run where all keys match but runname +;; returns #f if no such test found, returns a single test record if found +;; +;; Run this server-side +;; +(define (db:get-previous-test-run-record db run-id test-name item-path) + (let* ((keys (db:get-keys db)) + (selstr (string-intersperse keys ",")) + (qrystr (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")) + (keyvals #f)) + ;; first look up the key values from the run selected by run-id + (sqlite3:for-each-row + (lambda (a . b) + (set! keyvals (cons a b))) + db + (conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id) + (if (not keyvals) + #f + (let ((prev-run-ids '())) + (apply sqlite3:for-each-row + (lambda (id) + (set! prev-run-ids (cons id prev-run-ids))) + db + (conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append keyvals (list run-id))) + ;; for each run starting with the most recent look to see if there is a matching test + ;; if found then return that matching test record + (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids) + (if (null? prev-run-ids) #f + (let loop ((hed (car prev-run-ids)) + (tal (cdr prev-run-ids))) + (let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f))) + (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results) + (if (and (null? results) + (not (null? tal))) + (loop (car tal)(cdr tal)) + (if (null? results) #f + (car results)))))))))) + +;; get the previous records for when these tests were run where all keys match but runname +;; NB// Merge this with test:get-previous-test-run-records? This one looks for all matching tests +;; can use wildcards. Also can likely be factored in with get test paths? +;; +;; Run this remotely!! ;; -(define (db:queue-write-and-wait db qry-sig query params) - (let ((queue-len 0) - (res #f) - (got-it #f) - (qry-pkt (vector qry-sig query params)) - (start-time (current-milliseconds)) - (timeout (+ 10 (current-seconds)))) ;; set the time out to 10 secs in future - - ;; Put the item in the queue *incoming-writes* - (mutex-lock! *incoming-mutex*) - (set! *incoming-writes* (cons qry-pkt *incoming-writes*)) - (set! queue-len (length *incoming-writes*)) - (mutex-unlock! *incoming-mutex*) - - (debug:print-info 7 "Current write queue length is " queue-len) - - ;; poll for the write to complete, timeout after 10 seconds - ;; periodic flushing of the queue is taken care of by - ;; db:flush-queue - (let loop () - (thread-sleep! 0.001) - (mutex-lock! *completed-mutex*) - (if (hash-table-ref/default *completed-writes* qry-sig #f) - (begin - (hash-table-delete! *completed-writes* qry-sig) - (set! got-it #t))) - (mutex-unlock! *completed-mutex*) - (if (and (not got-it) - (< (current-seconds) timeout)) - (begin - (thread-sleep! 0.01) - (loop)))) - (set! *number-of-writes* (+ *number-of-writes* 1)) - (set! *writes-total-delay* (+ *writes-total-delay* (- (current-milliseconds) start-time))) - got-it)) - -(define (db:process-queue-item db item) - (let* ((stmt-key (cdb:packet-get-qtype item)) - (qry-sig (cdb:packet-get-query-sig item)) - (return-address (cdb:packet-get-client-sig item)) - (params (cdb:packet-get-params item)) - (query (let ((q (alist-ref stmt-key db:queries))) - (if q (car q) #f)))) - (debug:print-info 11 "Special queries/requests stmt-key=" stmt-key ", return-address=" return-address ", query=" query ", params=" params) - (if query - ;; hand queries off to the write queue - (let ((response (case *transport-type* - ((http) - (debug:print-info 7 "Queuing item " item " for wrapped write") - (db:queue-write-and-wait db qry-sig query params)) - (else - (apply sqlite3:execute db query params) - #t)))) - (debug:print-info 7 "Received " response " from wrapped write") - (server:reply return-address qry-sig response response)) - ;; otherwise if appropriate flush the queue (this is a read or complex query) - (begin - (cond - ((member stmt-key db:special-queries) - (let ((starttime (current-milliseconds))) - (debug:print-info 9 "Handling special statement " stmt-key) - (case stmt-key - ((immediate) - ;; This is a read or mixed read-write query, must clear the cache - (case *transport-type* - ((http) - (mutex-lock! *db:process-queue-mutex*) - (db:process-cached-writes db) - (mutex-unlock! *db:process-queue-mutex*))) - (let* ((proc (car params)) - (remparams (cdr params)) - ;; we are being handed a procedure so call it - ;; (debug:print-info 11 "Running (apply " proc " " remparams ")") - (result (server:reply return-address qry-sig #t (apply proc remparams)))) - (set! *total-non-write-delay* (+ *total-non-write-delay* (- (current-milliseconds) starttime))) - (set! *number-non-write-queries* (+ *number-non-write-queries* 1)) - result)) - ((login) - (if (< (length params) 3) ;; should get toppath, version and signature - (server:reply return-address qry-sig '(#f "login failed due to missing params")) ;; missing params - (let ((calling-path (car params)) - (calling-vers (cadr params)) - (client-key (caddr params))) - (if (and (equal? calling-path *toppath*) - (equal? megatest-version calling-vers)) - (begin - (hash-table-set! *logged-in-clients* client-key (current-seconds)) - (server:reply return-address qry-sig #t '(#t "successful login"))) ;; path matches - pass! Should vet the caller at this time ... - (server:reply return-address qry-sig #f (list #f (conc "Login failed due to mismatch paths: " calling-path ", " *toppath*))))))) - ((flush sync) - (server:reply return-address qry-sig #t 1)) ;; (length data))) - ((set-verbosity) - (set! *verbosity* (car params)) - (server:reply return-address qry-sig #t (list #t *verbosity*))) - ((killserver) - (let ((hostname (car *runremote*)) - (port (cadr *runremote*)) - (pid (car params))) - (debug:print 0 "WARNING: Server on " hostname ":" port " going down by user request!") - (debug:print-info 1 "current pid=" (current-process-id)) - (open-run-close tasks:server-deregister tasks:open-db - hostname - port: port) - (set! *server-run* #f) - (thread-sleep! 3) - (process-signal pid signal/kill) - (server:reply return-address qry-sig #t '(#t "exit process started")))) - (else ;; not a command, i.e. is a query - (debug:print 0 "ERROR: Unrecognised query/command " stmt-key) - (server:reply return-address qry-sig #f 'failed))))) - (else - (debug:print-info 11 "Executing " stmt-key " for " params) - (apply sqlite3:execute (hash-table-ref queries stmt-key) params) - (server:reply return-address qry-sig #t #t))))))) +(define (db:get-matching-previous-test-run-records db run-id test-name item-path) + (let* ((keys (db:get-keys db)) + (selstr (string-intersperse (map (lambda (x)(vector-ref x 0)) keys) ",")) + (qrystr (string-intersperse (map (lambda (x)(conc (vector-ref x 0) "=?")) keys) " AND ")) + (keyvals #f) + (tests-hash (make-hash-table))) + ;; first look up the key values from the run selected by run-id + (sqlite3:for-each-row + (lambda (a . b) + (set! keyvals (cons a b))) + db + (conc "SELECT " selstr " FROM runs WHERE id=? ORDER BY event_time DESC;") run-id) + (if (not keyvals) + '() + (let ((prev-run-ids '())) + (apply sqlite3:for-each-row + (lambda (id) + (set! prev-run-ids (cons id prev-run-ids))) + db + (conc "SELECT id FROM runs WHERE " qrystr " AND id != ?;") (append keyvals (list run-id))) + ;; collect all matching tests for the runs then + ;; extract the most recent test and return that. + (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals + ", previous run ids found: " prev-run-ids) + (if (null? prev-run-ids) '() ;; no previous runs? return null + (let loop ((hed (car prev-run-ids)) + (tal (cdr prev-run-ids))) + (let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f))) + (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name + ", item-path " item-path " results: " (intersperse results "\n")) + ;; Keep only the youngest of any test/item combination + (for-each + (lambda (testdat) + (let* ((full-testname (conc (db:test-get-testname testdat) "/" (db:test-get-item-path testdat))) + (stored-test (hash-table-ref/default tests-hash full-testname #f))) + (if (or (not stored-test) + (and stored-test + (> (db:test-get-event_time testdat)(db:test-get-event_time stored-test)))) + ;; this test is younger, store it in the hash + (hash-table-set! tests-hash full-testname testdat)))) + results) + (if (null? tal) + (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests + (loop (car tal)(cdr tal)))))))))) (define (db:test-get-records-for-index-file db run-id test-name) (let ((res '())) (sqlite3:for-each-row (lambda (id itempath state status run_duration logf comment) @@ -2011,327 +1824,18 @@ ;; update one of the testmeta fields (define (db:testmeta-update-field db testname field value) (sqlite3:execute db (conc "UPDATE test_meta SET " field "=? WHERE testname=?;") value testname)) -;;====================================================================== -;; T E S T D A T A -;;====================================================================== - -(define (db:csv->test-data db test-id csvdata #!key (work-area #f)) - (debug:print 4 "test-id " test-id ", csvdata: " csvdata) - (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))) - (if (sqlite3:database? tdb) - (let ((csvlist (csv->list (make-csv-reader - (open-input-string csvdata) - '((strip-leading-whitespace? #t) - (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata))) - (for-each - (lambda (csvrow) - (let* ((padded-row (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9)) - (category (list-ref padded-row 0)) - (variable (list-ref padded-row 1)) - (value (any->number-if-possible (list-ref padded-row 2))) - (expected (any->number-if-possible (list-ref padded-row 3))) - (tol (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number - (units (list-ref padded-row 5)) - (comment (list-ref padded-row 6)) - (status (let ((s (list-ref padded-row 7))) - (if (and (string? s)(or (string-match (regexp "^\\s*$") s) - (string-match (regexp "^n/a$") s))) - #f - s))) ;; if specified on the input then use, else calculate - (type (list-ref padded-row 8))) - ;; look up expected,tol,units from previous best fit test if they are all either #f or '' - (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value - ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type) - - (if (and (or (not expected)(equal? expected "")) - (or (not tol) (equal? expected "")) - (or (not units) (equal? expected ""))) - (let-values (((new-expected new-tol new-units)(db:get-prev-tol-for-test db test-id category variable))) - (set! expected new-expected) - (set! tol new-tol) - (set! units new-units))) - - (debug:print 4 "AFTER: category: " category " variable: " variable " value: " value - ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) - ;; calculate status if NOT specified - (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers - (if (number? tol) ;; if tol is a number then we do the standard comparison - (let* ((max-val (+ expected tol)) - (min-val (- expected tol)) - (result (and (>= value min-val)(<= value max-val)))) - (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result) - (set! status (if result "pass" "fail"))) - (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op. - (case (string->symbol tol) ;; tol should be >, <, >=, <= - ((>) (if (> value expected) "pass" "fail")) - ((<) (if (< value expected) "pass" "fail")) - ((>=) (if (>= value expected) "pass" "fail")) - ((<=) (if (<= value expected) "pass" "fail")) - (else (conc "ERROR: bad tol comparator " tol)))))) - (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value - ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) - (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);" - test-id category variable value expected tol units (if comment comment "") status type))) - csvlist) - (sqlite3:finalize! tdb))))) - -;; get a list of test_data records matching categorypatt -(define (db:read-test-data db test-id categorypatt #!key (work-area #f)) - (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))) - (if (sqlite3:database? tdb) - (let ((res '())) - (sqlite3:for-each-row - (lambda (id test_id category variable value expected tol units comment status type) - (set! res (cons (vector id test_id category variable value expected tol units comment status type) res))) - tdb - "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt) - (sqlite3:finalize! tdb) - (reverse res)) - '()))) - -;; NOTE: Run this local with #f for db !!! -(define (db:load-test-data db test-id #!key (work-area #f)) - (let loop ((lin (read-line))) - (if (not (eof-object? lin)) - (begin - (debug:print 4 lin) - (db:csv->test-data db test-id lin work-area: work-area) - (loop (read-line))))) - ;; roll up the current results. - ;; FIXME: Add the status to - (db:test-data-rollup db test-id #f work-area: work-area)) - -;; WARNING: Do NOT call this for the parent test on an iterated test -;; Roll up test_data pass/fail results -;; look at the test_data status field, -;; if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS. -;; if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored -(define (db:test-data-rollup db test-id status #!key (work-area #f)) - (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area)) - (fail-count 0) - (pass-count 0)) - (if (sqlite3:database? tdb) - (begin - (sqlite3:for-each-row - (lambda (fcount pcount) - (set! fail-count fcount) - (set! pass-count pcount)) - tdb - "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count, - (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;" - test-id test-id) - (sqlite3:finalize! tdb) - - ;; Now rollup the counts to the central megatest.db - (cdb:pass-fail-counts *runremote* test-id fail-count pass-count) - ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" - ;; fail-count pass-count test-id) - - ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines - ;; next time you read this! - ;; - ;; (cdb:flush-queue *runremote*) - ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set - - ;; if the test is not FAIL then set status based on the fail and pass counts. - (cdb:test-rollup-test_data-pass-fail *runremote* test-id) - ;; (sqlite3:execute - ;; db ;;; NOTE: Should this be WARN,FAIL? A WARN is not a FAIL????? BUG FIXME - ;; "UPDATE tests - ;; SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 - ;; THEN 'FAIL' - ;; WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND - ;; (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL') - ;; THEN 'PASS' - ;; ELSE status - ;; END WHERE id=?;" - ;; test-id test-id test-id test-id) - )))) - -(define (db:get-prev-tol-for-test db test-id category variable) - ;; Finish me? - (values #f #f #f)) - -;;====================================================================== -;; S T E P S -;;====================================================================== - -(define (db:step-get-time-as-string vec) - (seconds->time-string (db:step-get-event_time vec))) - -;; db-get-test-steps-for-run -(define (db:get-steps-for-test db test-id #!key (work-area #f)) - (let* ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area)) - (res '())) - (if (sqlite3:database? tdb) - (handle-exceptions - exn - (begin - (debug:print 0 "ERROR: error on access to testdat for test with id " test-id) - '()) - (begin - (sqlite3:for-each-row - (lambda (id test-id stepname state status event-time logfile) - (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res))) - tdb - "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC; - test-id) - (sqlite3:finalize! tdb) - (reverse res))) - '()))) - -;; get a pretty table to summarize steps -;; -(define (db:get-steps-table db test-id #!key (work-area #f)) - (let ((steps (db:get-steps-for-test db test-id work-area: work-area))) - ;; organise the steps for better readability - (let ((res (make-hash-table))) - (for-each - (lambda (step) - (debug:print 6 "step=" step) - (let ((record (hash-table-ref/default - res - (db:step-get-stepname step) - ;; stepname start end status Duration Logfile - (vector (db:step-get-stepname step) "" "" "" "" "")))) - (debug:print 6 "record(before) = " record - "\nid: " (db:step-get-id step) - "\nstepname: " (db:step-get-stepname step) - "\nstate: " (db:step-get-state step) - "\nstatus: " (db:step-get-status step) - "\ntime: " (db:step-get-event_time step)) - (case (string->symbol (db:step-get-state step)) - ((start)(vector-set! record 1 (db:step-get-event_time step)) - (vector-set! record 3 (if (equal? (vector-ref record 3) "") - (db:step-get-status step))) - (if (> (string-length (db:step-get-logfile step)) - 0) - (vector-set! record 5 (db:step-get-logfile step)))) - ((end) - (vector-set! record 2 (any->number (db:step-get-event_time step))) - (vector-set! record 3 (db:step-get-status step)) - (vector-set! record 4 (let ((startt (any->number (vector-ref record 1))) - (endt (any->number (vector-ref record 2)))) - (debug:print 4 "record[1]=" (vector-ref record 1) - ", startt=" startt ", endt=" endt - ", get-status: " (db:step-get-status step)) - (if (and (number? startt)(number? endt)) - (seconds->hr-min-sec (- endt startt)) "-1"))) - (if (> (string-length (db:step-get-logfile step)) - 0) - (vector-set! record 5 (db:step-get-logfile step)))) - (else - (vector-set! record 2 (db:step-get-state step)) - (vector-set! record 3 (db:step-get-status step)) - (vector-set! record 4 (db:step-get-event_time step)))) - (hash-table-set! res (db:step-get-stepname step) record) - (debug:print 6 "record(after) = " record - "\nid: " (db:step-get-id step) - "\nstepname: " (db:step-get-stepname step) - "\nstate: " (db:step-get-state step) - "\nstatus: " (db:step-get-status step) - "\ntime: " (db:step-get-event_time step)))) - ;; (else (vector-set! record 1 (db:step-get-event_time step))) - (sort steps (lambda (a b) - (cond - ((< (db:step-get-event_time a)(db:step-get-event_time b)) #t) - ((eq? (db:step-get-event_time a)(db:step-get-event_time b)) - (< (db:step-get-id a) (db:step-get-id b))) - (else #f))))) - res))) - -;; get a pretty table to summarize steps -;; -(define (db:get-steps-table-list db test-id #!key (work-area #f)) - (let ((steps (db:get-steps-for-test db test-id work-area: work-area))) - ;; organise the steps for better readability - (let ((res (make-hash-table))) - (for-each - (lambda (step) - (debug:print 6 "step=" step) - (let ((record (hash-table-ref/default - res - (db:step-get-stepname step) - ;; stepname start end status - (vector (db:step-get-stepname step) "" "" "" "" "")))) - (debug:print 6 "record(before) = " record - "\nid: " (db:step-get-id step) - "\nstepname: " (db:step-get-stepname step) - "\nstate: " (db:step-get-state step) - "\nstatus: " (db:step-get-status step) - "\ntime: " (db:step-get-event_time step)) - (case (string->symbol (db:step-get-state step)) - ((start)(vector-set! record 1 (db:step-get-event_time step)) - (vector-set! record 3 (if (equal? (vector-ref record 3) "") - (db:step-get-status step))) - (if (> (string-length (db:step-get-logfile step)) - 0) - (vector-set! record 5 (db:step-get-logfile step)))) - ((end) - (vector-set! record 2 (any->number (db:step-get-event_time step))) - (vector-set! record 3 (db:step-get-status step)) - (vector-set! record 4 (let ((startt (any->number (vector-ref record 1))) - (endt (any->number (vector-ref record 2)))) - (debug:print 4 "record[1]=" (vector-ref record 1) - ", startt=" startt ", endt=" endt - ", get-status: " (db:step-get-status step)) - (if (and (number? startt)(number? endt)) - (seconds->hr-min-sec (- endt startt)) "-1"))) - (if (> (string-length (db:step-get-logfile step)) - 0) - (vector-set! record 5 (db:step-get-logfile step)))) - (else - (vector-set! record 2 (db:step-get-state step)) - (vector-set! record 3 (db:step-get-status step)) - (vector-set! record 4 (db:step-get-event_time step)))) - (hash-table-set! res (db:step-get-stepname step) record) - (debug:print 6 "record(after) = " record - "\nid: " (db:step-get-id step) - "\nstepname: " (db:step-get-stepname step) - "\nstate: " (db:step-get-state step) - "\nstatus: " (db:step-get-status step) - "\ntime: " (db:step-get-event_time step)))) - ;; (else (vector-set! record 1 (db:step-get-event_time step))) - (sort steps (lambda (a b) - (cond - ((< (db:step-get-event_time a)(db:step-get-event_time b)) #t) - ((eq? (db:step-get-event_time a)(db:step-get-event_time b)) - (< (db:step-get-id a) (db:step-get-id b))) - (else #f))))) - res))) - -(define (db:get-compressed-steps test-id #!key (work-area #f)(tdb #f)) - (if (or (not work-area) - (file-exists? (conc work-area "/testdat.db"))) - (let* ((comprsteps (open-run-close db:get-steps-table tdb test-id work-area: work-area))) - (map (lambda (x) - ;; take advantage of the \n on time->string - (vector - (vector-ref x 0) - (let ((s (vector-ref x 1))) - (if (number? s)(seconds->time-string s) s)) - (let ((s (vector-ref x 2))) - (if (number? s)(seconds->time-string s) s)) - (vector-ref x 3) ;; status - (vector-ref x 4) - (vector-ref x 5))) ;; time delta - (sort (hash-table-values comprsteps) - (lambda (a b) - (let ((time-a (vector-ref a 1)) - (time-b (vector-ref b 1))) - (if (and (number? time-a)(number? time-b)) - (if (< time-a time-b) - #t - (if (eq? time-a time-b) - (stringtest-data - db:test-data-rollup - db:teststep-set-status! )) +;; (define *db:all-write-procs* +;; (list +;; db:set-var +;; db:del-var +;; db:register-run +;; db:set-comment-for-run +;; db:delete-run +;; db:update-run-event_time +;; db:lock/unlock-run +;; db:delete-test-step-records +;; db:delete-test-records +;; db:delete-tests-for-run +;; db:delete-old-deleted-test-records +;; db:set-tests-state-status +;; db:test-set-state-status-by-id +;; db:test-set-state-status-by-run-id-testname +;; db:testmeta-add-record +;; db:csv->test-data +;; )) Index: db_records.scm ================================================================== --- db_records.scm +++ db_records.scm @@ -97,38 +97,38 @@ ;; S T E P S ;;====================================================================== ;; Run steps ;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time (define (make-db:step)(make-vector 7)) -(define-inline (db:step-get-id vec) (vector-ref vec 0)) -(define-inline (db:step-get-test_id vec) (vector-ref vec 1)) -(define-inline (db:step-get-stepname vec) (vector-ref vec 2)) -(define-inline (db:step-get-state vec) (vector-ref vec 3)) -(define-inline (db:step-get-status vec) (vector-ref vec 4)) -(define-inline (db:step-get-event_time vec) (vector-ref vec 5)) -(define-inline (db:step-get-logfile vec) (vector-ref vec 6)) -(define-inline (db:step-set-id! vec val)(vector-set! vec 0 val)) -(define-inline (db:step-set-test_id! vec val)(vector-set! vec 1 val)) -(define-inline (db:step-set-stepname! vec val)(vector-set! vec 2 val)) -(define-inline (db:step-set-state! vec val)(vector-set! vec 3 val)) -(define-inline (db:step-set-status! vec val)(vector-set! vec 4 val)) -(define-inline (db:step-set-event_time! vec val)(vector-set! vec 5 val)) -(define-inline (db:step-set-logfile! vec val)(vector-set! vec 6 val)) +(define-inline (tdb:step-get-id vec) (vector-ref vec 0)) +(define-inline (tdb:step-get-test_id vec) (vector-ref vec 1)) +(define-inline (tdb:step-get-stepname vec) (vector-ref vec 2)) +(define-inline (tdb:step-get-state vec) (vector-ref vec 3)) +(define-inline (tdb:step-get-status vec) (vector-ref vec 4)) +(define-inline (tdb:step-get-event_time vec) (vector-ref vec 5)) +(define-inline (tdb:step-get-logfile vec) (vector-ref vec 6)) +(define-inline (tdb:step-set-id! vec val)(vector-set! vec 0 val)) +(define-inline (tdb:step-set-test_id! vec val)(vector-set! vec 1 val)) +(define-inline (tdb:step-set-stepname! vec val)(vector-set! vec 2 val)) +(define-inline (tdb:step-set-state! vec val)(vector-set! vec 3 val)) +(define-inline (tdb:step-set-status! vec val)(vector-set! vec 4 val)) +(define-inline (tdb:step-set-event_time! vec val)(vector-set! vec 5 val)) +(define-inline (tdb:step-set-logfile! vec val)(vector-set! vec 6 val)) ;; The steps table (define (make-db:steps-table)(make-vector 5)) -(define-inline (db:steps-table-get-stepname vec) (vector-ref vec 0)) -(define-inline (db:steps-table-get-start vec) (vector-ref vec 1)) -(define-inline (db:steps-table-get-end vec) (vector-ref vec 2)) -(define-inline (db:steps-table-get-status vec) (vector-ref vec 3)) -(define-inline (db:steps-table-get-runtime vec) (vector-ref vec 4)) -(define-inline (db:step-stable-set-stepname! vec val)(vector-set! vec 0 val)) -(define-inline (db:step-stable-set-start! vec val)(vector-set! vec 1 val)) -(define-inline (db:step-stable-set-end! vec val)(vector-set! vec 2 val)) -(define-inline (db:step-stable-set-status! vec val)(vector-set! vec 3 val)) -(define-inline (db:step-stable-set-runtime! vec val)(vector-set! vec 4 val)) +(define-inline (tdb:steps-table-get-stepname vec) (vector-ref vec 0)) +(define-inline (tdb:steps-table-get-start vec) (vector-ref vec 1)) +(define-inline (tdb:steps-table-get-end vec) (vector-ref vec 2)) +(define-inline (tdb:steps-table-get-status vec) (vector-ref vec 3)) +(define-inline (tdb:steps-table-get-runtime vec) (vector-ref vec 4)) +(define-inline (tdb:step-stable-set-stepname! vec val)(vector-set! vec 0 val)) +(define-inline (tdb:step-stable-set-start! vec val)(vector-set! vec 1 val)) +(define-inline (tdb:step-stable-set-end! vec val)(vector-set! vec 2 val)) +(define-inline (tdb:step-stable-set-status! vec val)(vector-set! vec 3 val)) +(define-inline (tdb:step-stable-set-runtime! vec val)(vector-set! vec 4 val)) ;; use this one for db-get-run-info (define-inline (db:get-row vec)(vector-ref vec 1)) ;; The data structure for handing off requests via wire ADDED dbwars/NOTES Index: dbwars/NOTES ================================================================== --- /dev/null +++ dbwars/NOTES @@ -0,0 +1,31 @@ +Before using prepare: + +matt@xena:/tmp/megatest/dbwars$ ./sqlite3-test insert +Adding 1047 test3 item/39 host0-0.3-200000-240-this one sucks eh? (added 51886 records so far) +Adding 1122 test5 item/52 host2-0.2-200000-120-this is a good one eh? (added 78889 records so far) +Adding 1050 test7 item/31 host1-0.1-100000-120-this is a good one eh? (added 110641 records so far) +create-tests ran register-test 144000 times in 41.0 seconds + +After using prepare: + +matt@xena:/tmp/megatest/dbwars$ csc sqlite3-test.scm && ./sqlite3-test insert +Adding 1082 test4 item/74 host1-0.3-100000-120-this is a good one eh? (added 61281 records so far) +Adding 1138 test7 item/43 host2-0.3-200000-120-this is a good one eh? (added 109001 records so far) +Adding 1023 test9 item/00 host0-0.2-100000-240-this one sucks eh? (added 143878 records so far) +create-tests ran register-test 144000 times in 38.0 seconds + +After moving the prepare outside the call (so it isn't done each time): + +matt@xena:/tmp/megatest/dbwars$ ./sqlite3-test insert +Adding 1042 test4 item/59 host0-0.3-200000-120-this is a good one eh? (added 63401 records so far) +Adding 1011 test6 item/40 host0-0.1-200000-120-this one sucks eh? (added 94906 records so far) +Adding 1076 test9 item/34 host1-0.2-200000-120-just eh, eh? (added 139035 records so far) +create-tests ran register-test 144000 times in 33.0 seconds + +Using sql-de-lite with very similar code: + +matt@xena:/tmp/megatest/dbwars$ ./sql-de-lite-test insert +Adding 1029 test4 item/53 host0-0.2-200000-240- (added 64252 records so far) +Adding 1134 test7 item/64 host2-0.3-100000-240-this is a good one eh? (added 105973 records so far) +create-tests ran register-test 144000 times in 31.0 seconds + ADDED dbwars/sql-de-lite-test.scm Index: dbwars/sql-de-lite-test.scm ================================================================== --- /dev/null +++ dbwars/sql-de-lite-test.scm @@ -0,0 +1,19 @@ + +(use sql-de-lite) +(include "test-common.scm") + +(define db (open-database "test.db")) + +(exec (sql db test-table-defn)) +(exec (sql db syncsetup)) + +(define (register-test stmth run-id testname host cpuload diskfree uname rundir shortdir item-path state status final-logf run-duration comment event-time) + (exec + stmth ;; (sql db test-insert) + run-id + testname host cpuload diskfree uname rundir shortdir item-path state status final-logf run-duration comment event-time)) + +(let ((stmth (sql db test-insert))) + (create-tests stmth)) + +(close-database db) ADDED dbwars/sqlite3-test.scm Index: dbwars/sqlite3-test.scm ================================================================== --- /dev/null +++ dbwars/sqlite3-test.scm @@ -0,0 +1,20 @@ + +(use sqlite3) +(include "test-common.scm") + +(define db (open-database "test.db")) + +(execute db test-table-defn) +(execute db syncsetup) + + +(define (register-test stmth run-id testname host cpuload diskfree uname rundir shortdir item-path state status final-logf run-duration comment event-time) + (execute stmth + run-id + testname host cpuload diskfree uname rundir shortdir item-path state status final-logf run-duration comment event-time)) + +(let ((stmth (prepare db test-insert))) + (create-tests stmth) + (finalize! stmth)) + +(finalize! db) ADDED dbwars/test-common.scm Index: dbwars/test-common.scm ================================================================== --- /dev/null +++ dbwars/test-common.scm @@ -0,0 +1,129 @@ +(use srfi-18 srfi-69 apropos) + +(define args (argv)) + +(if (not (eq? (length args) 2)) + (begin + (print "Usage: sqlitecompare [insert|update]") + (exit 0))) + +(define action (string->symbol (cadr args))) + +(system "rm -f test.db") + +(define test-table-defn + "CREATE TABLE IF NOT EXISTS tests + (id INTEGER PRIMARY KEY, + run_id INTEGER, + testname TEXT, + host TEXT DEFAULT 'n/a', + cpuload REAL DEFAULT -1, + diskfree INTEGER DEFAULT -1, + uname TEXT DEFAULT 'n/a', + rundir TEXT DEFAULT 'n/a', + shortdir TEXT DEFAULT '', + item_path TEXT DEFAULT '', + state TEXT DEFAULT 'NOT_STARTED', + status TEXT DEFAULT 'FAIL', + attemptnum INTEGER DEFAULT 0, + final_logf TEXT DEFAULT 'logs/final.log', + logdat BLOB, + run_duration INTEGER DEFAULT 0, + comment TEXT DEFAULT '', + event_time TIMESTAMP, + fail_count INTEGER DEFAULT 0, + pass_count INTEGER DEFAULT 0, + archived INTEGER DEFAULT 0, -- 0=no, 1=in progress, 2=yes + CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path) + );") + +(define test-insert "INSERT INTO tests (run_id,testname,host,cpuload,diskfree,uname,rundir,shortdir,item_path,state,status,final_logf,run_duration,comment,event_time) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );") +(define syncsetup "PRAGMA synchronous = OFF;") + +(define tests '("test0" "test1" "test2" "test3" "test4" "test5" "test6" "test7" "test8" "test9")) +(define items '()) +(for-each + (lambda (n) + (for-each + (lambda (m) + (set! items (cons (conc "item/" n m) items))) + '(0 1 2 3 4 5 6 7 8 9))) + '(0 1 2 3 4 5 6 7 8 9)) +(define hosts '("host0" "host1" "host2")) ;; "host3" "host4" "host5" "host6" "host7" "host8" "host9")) +(define cpuloads '(0.1 0.2 0.3)) ;; 0.4 0.5 0.6 0.7 0.8 0.9)) +(define diskfrees '(100000 200000)) ;; 300000 400000 500000 600000 700000 800000 900000)) +(define uname "Linux xena 3.5.0-40-generic #62~precise1-Ubuntu SMP Fri Aug 23 17:59:10 UTC 2013 i686 i686 i386 GNU/Linux") +(define basedir "/mfs/matt/data/megatest/runs/testing") +(define final-logf "finallog.html") +(define run-durations (list 120 240)) ;; 260)) +(define comments '("" "this is a good one eh?" "this one sucks eh?" "just eh, eh?")) + +(define run-ids (make-hash-table)) +(define max-run-id 1000) + +(define (test-factors->run-id host cpuload diskfree run-duration comment) + (let* ((factor (conc host "-" cpuload "-" diskfree "-" run-duration "-" comment)) + (run-id (hash-table-ref/default run-ids factor #f))) + (if run-id + (list run-id factor) + (let ((new-id (+ max-run-id 1))) + (set! max-run-id new-id) + (hash-table-set! run-ids factor new-id) + (list new-id factor))))) + + +(define (create-tests stmth) + (let ((num-created 0) + (last-print (current-seconds)) + (start-time (current-seconds))) + (for-each + (lambda (test) + (for-each + (lambda (item) + (for-each + (lambda (host) + (for-each + (lambda (cpuload) + (for-each + (lambda (diskfree) + (for-each + (lambda (run-duration) + (for-each + (lambda (comment) + (let* ((run-id-dat (test-factors->run-id host cpuload diskfree run-duration comment)) + (run-id (car run-id-dat)) + (factor (cadr run-id-dat)) + (curr-time (current-seconds))) + (if (> (- curr-time last-print) 10) + (begin + (print "Adding " run-id " " test " " item " " factor " (added " num-created " records so far)") + (set! last-print curr-time))) + (set! num-created (+ num-created 1)) + (register-test stmth ;; db + run-id + test ;; testname + host + cpuload + diskfree + uname + (conc basedir "/" test "/" item) ;; rundir + (conc test "/" item) ;; shortdir + item ;; item-path + "NOT_STARTED" ;; state + "NA" ;; status + final-logf + run-duration + comment + (current-seconds)))) + comments)) + run-durations)) + diskfrees)) + cpuloads)) + hosts)) + items)) + tests) + (print "create-tests ran register-test " num-created " times in " (- (current-seconds) start-time) " seconds"))) + + + Index: dcommon.scm ================================================================== --- dcommon.scm +++ dcommon.scm @@ -375,15 +375,15 @@ (iup:attribute-set! general-matrix "3:0" "Version") (iup:attribute-set! general-matrix "3:1" megatest-version) general-matrix)) -(define (dcommon:run-stats) +(define (dcommon:run-stats db) (let* ((stats-matrix (iup:matrix expand: "YES")) (changed #f) (updater (lambda () - (let* ((run-stats (mt:get-run-stats)) + (let* ((run-stats (db:get-run-stats db)) (indices (common:sparse-list-generate-index run-stats)) ;; proc: set-cell)) (row-indices (car indices)) (col-indices (cadr indices)) (max-row (if (null? row-indices) 1 (apply max (map cadr row-indices)))) (max-col (if (null? col-indices) 1 Index: docs/megatest-training.odp ================================================================== --- docs/megatest-training.odp +++ docs/megatest-training.odp cannot compute difference between binary files Index: docs/megatest-training.pdf ================================================================== --- docs/megatest-training.pdf +++ docs/megatest-training.pdf cannot compute difference between binary files Index: ezsteps.scm ================================================================== --- ezsteps.scm +++ ezsteps.scm @@ -98,11 +98,11 @@ (let ((exinfo (vector-ref exit-info 2)) (logfna (if logpro-used (conc stepname ".html") ""))) ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect) (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: test-run-dir)) (if logpro-used - (cdb:test-set-log! *runremote* test-id (conc stepname ".html"))) + (rmt:test-set-log! test-id (conc stepname ".html"))) ;; set the test final status (let* ((this-step-status (cond ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn) ((eq? (vector-ref exit-info 2) 0) 'pass) (else 'fail))) Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -74,34 +74,41 @@ (hostname (get-host-name)) (ipaddrstr (let ((ipstr (if (string=? "-" hostn) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") (server:get-best-guess-address hostname) #f))) - (if ipstr ipstr hostn))) ;; hostname))) + (if ipstr ipstr hostn))) ;; hostname))) (start-port (if (and (args:get-arg "-port") (string->number (args:get-arg "-port"))) (string->number (args:get-arg "-port")) (if (and (config-lookup *configdat* "server" "port") (string->number (config-lookup *configdat* "server" "port"))) (string->number (config-lookup *configdat* "server" "port")) (+ 5000 (random 1001))))) (link-tree-path (config-lookup *configdat* "setup" "linktree"))) - (set! *cache-on* #t) + (set! db *inmemdb*) (root-path (if link-tree-path link-tree-path (current-directory))) ;; WARNING: SECURITY HOLE. FIX ASAP! (handle-directory spiffy-directory-listing) ;; http-transport:handle-directory) ;; simple-directory-handler) ;; Setup the web server and a /ctrl interface ;; (vhost-map `(((* any) . ,(lambda (continue) ;; open the db on the first call - (if (not db)(set! db (open-db))) + ;; This is were we set up the database connections (let* (($ (request-vars source: 'both)) (dat ($ 'dat)) (res #f)) (cond + ((equal? (uri-path (request-uri (current-request))) + '(/ "api")) + (send-response body: (api:process-request db $) ;; the $ is the request vars proc + headers: '((content-type text/plain))) + (mutex-lock! *heartbeat-mutex*) + (set! *last-db-access* (current-seconds)) + (mutex-unlock! *heartbeat-mutex*)) ;; This is the /ctrl path where data is handed to the server and ;; responses ((equal? (uri-path (request-uri (current-request))) '(/ "ctrl")) (let* ((packet (db:string->obj dat)) @@ -175,10 +182,54 @@ ;; C L I E N T S ;;====================================================================== (define *http-mutex* (make-mutex)) +;; This next block all imported en-mass from the api branch +(define *http-requests-in-progress* 0) +(define *http-connections-next-cleanup* (current-seconds)) + +(define (http-transport:get-time-to-cleanup) + (let ((res #f)) + (mutex-lock! *http-mutex*) + (set! res (> (current-seconds) *http-connections-next-cleanup*)) + (mutex-unlock! *http-mutex*) + res)) + +(define (http-transport:inc-requests-count) + (mutex-lock! *http-mutex*) + (set! *http-requests-in-progress* (+ 1 *http-requests-in-progress*)) + ;; Use this opportunity to slow things down iff there are too many requests in flight + (if (> *http-requests-in-progress* 5) + (begin + (debug:print-info 0 "Whoa there buddy, ease up...") + (thread-sleep! 1))) + (mutex-unlock! *http-mutex*)) + +(define (http-transport:dec-requests-count proc) + (mutex-lock! *http-mutex*) + (proc) + (set! *http-requests-in-progress* (- *http-requests-in-progress* 1)) + (mutex-unlock! *http-mutex*)) + +(define (http-transport:dec-requests-count-and-close-all-connections) + (set! *http-requests-in-progress* (- *http-requests-in-progress* 1)) + (let loop ((etime (+ (current-seconds) 5))) ;; give up in five seconds + (if (> *http-requests-in-progress* 0) + (if (> etime (current-seconds)) + (begin + (thread-sleep! 0.05) + (loop etime)) + (debug:print 0 "ERROR: requests still in progress after 5 seconds of waiting. I'm going to pass on cleaning up http connections")) + (close-all-connections!))) + (set! *http-connections-next-cleanup* (+ (current-seconds) 10)) + (mutex-unlock! *http-mutex*)) + +(define (http-transport:inc-requests-and-prep-to-close-all-connections) + (mutex-lock! *http-mutex*) + (set! *http-requests-in-progress* (+ 1 *http-requests-in-progress*))) + ;; (system "megatest -list-servers | grep alive || megatest -server - -daemonize && sleep 4") ;; ;; ;; 1 Hello, world! Goodbye Dolly @@ -239,17 +290,98 @@ (let ((match (string-search (regexp "(.*)<.body>") res))) (debug:print-info 11 "match=" match) (let ((final (cadr match))) (debug:print-info 11 "final=" final) final))))))) + +;; Send "cmd" with json payload "params" to serverdat and receive result +;; +(define (http-transport:client-api-send-receive serverdat cmd params #!key (numretries 30)) + (let* ((fullurl (if (list? serverdat) + (cadddr serverdat) ;; this is the uri for /api + (begin + (debug:print 0 "FATAL ERROR: http-transport:client-send-receive called with no server info") + (exit 1)))) + (res #f)) + (handle-exceptions + exn + (begin + ;; TODO: Send this output to a log file so it isn't lost when running as daemon + (print "ERROR IN http-transport:client-send-receive " ((condition-property-accessor 'exn 'message) exn)) + (thread-sleep! 2) + (if (> numretries 0) + (http-transport:client-api-send-receive serverdat cmd params numretries: (- numretries 1)))) + (begin + (debug:print-info 11 "fullurl=" fullurl "\n") + ;; set up the http-client here + (max-retry-attempts 5) + ;; consider all requests indempotent + (retry-request? (lambda (request) + #t)) ;; (thread-sleep! (/ (if (> numretries 100) 100 numretries) 10)) + ;; (set! numretries (- numretries 1)) + ;; #t)) + ;; send the data and get the response + ;; extract the needed info from the http data and + ;; process and return it. + + ;; (with-input-from-request "http://localhost/echo-service" + ;; '((test . "value")) read-string) + + (let* ((send-recieve (lambda () + ;; (let ((dat #f) + ;; (cleanup (http-transport:get-time-to-cleanup))) + ;; (if cleanup + ;; (http-transport:inc-requests-and-prep-to-close-all-connections) + ;; (http-transport:inc-requests-count)) + ;; ;; Do the actual data transfer NB// KEPP THIS IN SYNC WITH http-transport:client-send-receive + (mutex-lock! *http-mutex*) + (set! res (with-input-from-request ;; was dat + fullurl + (list (cons 'key "thekey") + (cons 'cmd cmd) + (cons 'params params)) + read-string)) + ;; Shouldn't this be a call to the managed call-all-connections stuff above? + (close-all-connections!) + (mutex-unlock! *http-mutex*) + )) + ;; (if cleanup + ;; ;; mutex already set + ;; (begin + ;; (set! res dat) + ;; (http-transport:dec-requests-count-and-close-all-connections)) + ;; (http-transport:dec-requests-count + ;; (lambda () + ;; (set! res dat))))))) + (time-out (lambda () + (thread-sleep! 45) + (if (not res) + (begin + (debug:print 0 "WARNING: communication with the server timed out.") + (mutex-unlock! *http-mutex*) + (http-transport:client-api-send-receive serverdat cmd params numretries: (- numretries 1)) + (if (< numretries 3) ;; on last try just exit + (begin + (debug:print 0 "ERROR: communication with the server timed out. Giving up.") + (exit 1))))))) + (th1 (make-thread send-recieve "with-input-from-request")) + (th2 (make-thread time-out "time out"))) + (thread-start! th1) + (thread-start! th2) + (thread-join! th1) + (thread-terminate! th2) + (debug:print-info 11 "got res=" res) + res))))) (define (http-transport:client-connect iface port) (let* ((login-res #f) (uri-dat (make-request method: 'POST uri: (uri-reference (conc "http://" iface ":" port "/ctrl")))) - (serverdat (list iface port uri-dat))) - (set! login-res (client:login serverdat)) - (if (and (not (null? login-res)) + (uri-api-dat (make-request method: 'POST uri: (uri-reference (conc "http://" iface ":" port "/api")))) + (serverdat (list iface port uri-dat uri-api-dat))) + (set! *runremote* serverdat) ;; may or may not be good ... + (set! login-res (rmt:login)) + (if (and (list? login-res) (car login-res)) (begin (debug:print-info 2 "Logged in and connected to " iface ":" port) (set! *runremote* serverdat) serverdat) @@ -290,68 +422,70 @@ (* 60 60 (string->number tmo)) ;; default to three days (* 3 24 60 60))))) (debug:print-info 2 "server-timeout: " server-timeout ", server pid: " spid " on " iface ":" port) (let loop ((count 0)) - (thread-sleep! 4) ;; no need to do this very often - ;; NB// sync currently does NOT return queue-length - (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1))) - ;; (print "Server running, count is " count) - (if (< count 1) ;; 3x3 = 9 secs aprox - (loop (+ count 1))) - - ;; Check that iface and port have not changed (can happen if server port collides) - (mutex-lock! *heartbeat-mutex*) - (set! sdat *runremote*) - (mutex-unlock! *heartbeat-mutex*) - - (if (or (not (equal? sdat (list iface port))) - (not spid)) - (begin - (debug:print-info 0 "interface changed, refreshing iface and port info") - (set! iface (car sdat)) - (set! port (cadr sdat)) - (set! spid (tasks:server-get-server-id tdb #f iface port #f)))) - - ;; NOTE: Get rid of this mechanism! It really is not needed... - ;; (open-run-close tasks:server-update-heartbeat tasks:open-db spid) - (tasks:server-update-heartbeat tdb spid) - - ;; (if ;; (or (> numrunning 0) ;; stay alive for two days after last access - (mutex-lock! *heartbeat-mutex*) - (set! last-access *last-db-access*) - (mutex-unlock! *heartbeat-mutex*) - ;; (debug:print 11 "last-access=" last-access ", server-timeout=" server-timeout) - (if (and *server-run* - (> (+ last-access server-timeout) - (current-seconds))) - (begin - (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access)) - (loop 0)) - (begin - (debug:print-info 0 "Starting to shutdown the server.") - ;; need to delete only *my* server entry (future use) - (set! *time-to-exit* #t) - (open-run-close tasks:server-deregister-self tasks:open-db (get-host-name)) - (thread-sleep! 1) - (debug:print-info 0 "Max cached queries was " *max-cache-size*) - (debug:print-info 0 "Number of cached writes " *number-of-writes*) - (debug:print-info 0 "Average cached write time " - (if (eq? *number-of-writes* 0) - "n/a (no writes)" - (/ *writes-total-delay* - *number-of-writes*)) - " ms") - (debug:print-info 0 "Number non-cached queries " *number-non-write-queries*) - (debug:print-info 0 "Average non-cached time " - (if (eq? *number-non-write-queries* 0) - "n/a (no queries)" - (/ *total-non-write-delay* - *number-non-write-queries*)) - " ms") - (debug:print-info 0 "Server shutdown complete. Exiting") - (exit))))))) + ;; Use this opportunity to sync the inmemdb to db + (if *inmemdb* (db:sync-to *inmemdb* *db*)) + + (thread-sleep! 4) ;; no need to do this very often + + (if (< count 1) ;; 3x3 = 9 secs aprox + (loop (+ count 1))) + + ;; Check that iface and port have not changed (can happen if server port collides) + (mutex-lock! *heartbeat-mutex*) + (set! sdat *runremote*) + (mutex-unlock! *heartbeat-mutex*) + + (if (or (not (equal? sdat (list iface port))) + (not spid)) + (begin + (debug:print-info 0 "interface changed, refreshing iface and port info") + (set! iface (car sdat)) + (set! port (cadr sdat)) + (set! spid (tasks:server-get-server-id tdb #f iface port #f)))) + + ;; NOTE: Get rid of this mechanism! It really is not needed... + ;; (open-run-close tasks:server-update-heartbeat tasks:open-db spid) + (tasks:server-update-heartbeat tdb spid) + + ;; (if ;; (or (> numrunning 0) ;; stay alive for two days after last access + (mutex-lock! *heartbeat-mutex*) + (set! last-access *last-db-access*) + (mutex-unlock! *heartbeat-mutex*) + ;; (debug:print 11 "last-access=" last-access ", server-timeout=" server-timeout) + (if (and *server-run* + (> (+ last-access server-timeout) + (current-seconds))) + (begin + (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access)) + (loop 0)) + (begin + (debug:print-info 0 "Starting to shutdown the server.") + ;; need to delete only *my* server entry (future use) + (set! *time-to-exit* #t) + (if *inmemdb* (db:sync-to *inmemdb* *db*)) + (open-run-close tasks:server-deregister-self tasks:open-db (get-host-name)) + (thread-sleep! 1) + (debug:print-info 0 "Max cached queries was " *max-cache-size*) + (debug:print-info 0 "Number of cached writes " *number-of-writes*) + (debug:print-info 0 "Average cached write time " + (if (eq? *number-of-writes* 0) + "n/a (no writes)" + (/ *writes-total-delay* + *number-of-writes*)) + " ms") + (debug:print-info 0 "Number non-cached queries " *number-non-write-queries*) + (debug:print-info 0 "Average non-cached time " + (if (eq? *number-non-write-queries* 0) + "n/a (no queries)" + (/ *total-non-write-delay* + *number-non-write-queries*)) + " ms") + (debug:print-info 0 "Server shutdown complete. Exiting") + (exit)))))) ;; all routes though here end in exit ... (define (http-transport:launch) (if (not *toppath*) (if (not (setup-for-run)) @@ -370,15 +504,20 @@ (let* ((th2 (make-thread (lambda () (http-transport:run (if (args:get-arg "-server") (args:get-arg "-server") "-"))) "Server run")) - (th3 (make-thread http-transport:keep-running "Keep running")) - (th1 (make-thread server:write-queue-handler "write queue"))) + (th3 (make-thread http-transport:keep-running "Keep running"))) +;; (th1 (make-thread server:write-queue-handler "write queue"))) + (set! *cache-on* #t) + (set! *db* (open-db)) + (set! *inmemdb* (open-in-mem-db)) + (db:sync-to *db* *inmemdb*) + (thread-start! th2) (thread-start! th3) - (thread-start! th1) + ;; (thread-start! th1) (set! *didsomething* #t) (thread-join! th2)) (debug:print 0 "ERROR: Failed to setup for megatest"))) (exit))) Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -19,10 +19,11 @@ (declare (unit launch)) (declare (uses common)) (declare (uses configf)) (declare (uses db)) +(declare (uses tdb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") @@ -91,11 +92,11 @@ (debug:print 2 "Exectuing " test-name " (id: " test-id ") on " (get-host-name)) ;; Setup the *runremote* global var (if *runremote* (debug:print 2 "ERROR: I'm not expecting *runremote* to be set at this time")) ;; (set! *runremote* runremote) ;; (set! *transport-type* (string->symbol transport)) - (set! keys (cdb:remote-run db:get-keys #f)) + (set! keys (rmt:get-keys)) (set! keyvals (keys:target->keyval keys target)) ;; apply pre-overrides before other variables. The pre-override vars must not ;; clobbers things from the official sources such as megatest.config and runconfigs.config (if (string? set-vars) (let ((varpairs (string-split set-vars ","))) @@ -145,11 +146,11 @@ (alist->env-vars env-ovrd) (set-megatest-env-vars run-id inkeys: keys inkeyvals: keyvals) (set-item-env-vars itemdat) (save-environment-as-files "megatest") ;; open-run-close not needed for test-set-meta-info - (tests:set-full-meta-info #f test-id run-id 0 work-area) + (tests:set-full-meta-info test-id run-id 0 work-area) ;; (tests:test-set-status! test-id "REMOTEHOSTSTART" "n/a" (args:get-arg "-m") #f) (tests:test-force-state-status! test-id "REMOTEHOSTSTART" "n/a") (thread-sleep! 0.3) ;; NFS slowness has caused grief here @@ -235,11 +236,11 @@ ;; call the command using mt_ezstep (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd)) (debug:print 4 "script: " script) ;; DO NOT remote - (db:teststep-set-status! #f test-id stepname "start" "-" #f #f work-area: work-area) + (tdb:teststep-set-status! test-id stepname "start" "-" #f #f work-area: work-area) ;; now launch (let ((pid (process-run script))) (let processloop ((i 0)) (let-values (((pid-val exit-status exit-code)(process-wait pid #t))) (mutex-lock! m) @@ -253,13 +254,13 @@ (processloop (+ i 1)))) )) (let ((exinfo (vector-ref exit-info 2)) (logfna (if logpro-used (conc stepname ".html") ""))) ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect) - (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: work-area)) + (tdb:teststep-set-status! test-id stepname "end" exinfo #f logfna work-area: work-area)) (if logpro-used - (cdb:test-set-log! *runremote* test-id (conc stepname ".html"))) + (rmt:test-set-log! test-id (conc stepname ".html"))) ;; set the test final status (let* ((this-step-status (cond ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn) ((eq? (vector-ref exit-info 2) 0) 'pass) (else 'fail))) @@ -305,11 +306,11 @@ (round (- (current-seconds) start-seconds))))) (kill-tries 0)) - (tests:set-full-meta-info #f test-id run-id (calc-minutes) work-area) + (tests:set-full-meta-info test-id run-id (calc-minutes) work-area) (let loop ((minutes (calc-minutes))) (begin (set! kill-job? (or (test-get-kill-request test-id) ;; run-id test-name itemdat)) (and runtlim (let* ((run-seconds (- (current-seconds) start-seconds)) (time-exceeded (> run-seconds runtlim))) @@ -317,11 +318,11 @@ (begin (debug:print-info 0 "KILLING TEST DUE TO TIME LIMIT EXCEEDED! Runtime=" run-seconds " seconds, limit=" runtlim) #t) #f))))) ;; open-run-close not needed for test-set-meta-info - (tests:set-partial-meta-info #f test-id run-id minutes work-area) + (tests:set-partial-meta-info test-id run-id minutes work-area) (if kill-job? (begin (mutex-lock! m) ;; NOTE: The pid can change as different steps are run. Do we need handshaking between this ;; section and the runit section? Or add a loop that tries three times with a 1/4 second @@ -346,16 +347,14 @@ ;; (system (conc "kill -9 -" pid)))) (begin (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process") (tests:test-set-status! test-id "KILLED" "FAIL" (args:get-arg "-m") #f) - (sqlite3:finalize! tdb) (exit 1) ;; IS THIS NECESSARY OR WISE??? ))) (set! kill-tries (+ 1 kill-tries)) (mutex-unlock! m))) - ;; (sqlite3:finalize! db) (if keep-going (begin (thread-sleep! 3) ;; (+ 3 (random 6))) ;; add some jitter to the call home time to spread out the db accesses (if keep-going (loop (calc-minutes))))))) @@ -366,18 +365,15 @@ (thread-start! th1) (thread-start! th2) (thread-join! th2) (set! keep-going #f) (thread-join! th1) - ;; (thread-sleep! 1) - ;; (thread-terminate! th1) ;; Not sure if this is a good idea (thread-sleep! 1) ;; give thread th1 a chance to be done TODO: Verify this is needed. At 0.1 I was getting fail to stop, increased to total of 1.1 sec. - ;; (tests:update-central-meta-info test-id cpuload diskfree minutes #f #f) (mutex-lock! m) (let* ((item-path (item-list->path itemdat)) ;; only state and status needed - use lazy routine - (testinfo (cdb:remote-run db:get-testinfo-state-status #f test-id))) ;;;(cdb:get-test-info-by-id *runremote* test-id))) ;; )) ;; run-id test-name item-path))) + (testinfo (rmt:get-testinfo-state-status test-id))) ;;;(cdb:get-test-info-by-id *runremote* test-id))) ;; )) ;; run-id test-name item-path))) ;; Am I completed? (if (member (db:test-get-state testinfo) '("REMOTEHOSTSTART" "RUNNING")) ;; NOTE: It should *not* be REMOTEHOSTSTART but for reasons I don't yet understand it sometimes gets stuck in that state ;; (not (equal? (db:test-get-state testinfo) "COMPLETED")) (let ((new-state (if kill-job? "KILLED" "COMPLETED") ;; (if (eq? (vector-ref exit-info 2) 0) ;; exited with "good" status ;; "COMPLETED" ;; (db:test-get-state testinfo))) ;; else preseve the state as set within the test @@ -404,11 +400,11 @@ ;; (thread-sleep! 0.1) ;; give other processes an opportunity to access the db as rollup is lower priority ;; (cdb:roll-up-pass-fail-counts *runremote* run-id test-name item-path new-status))) )) ;; for automated creation of the rollup html file this is a good place... (if (not (equal? item-path "")) - (tests:summarize-items #f run-id test-id test-name #f))) ;; don't force - just update if no + (tests:summarize-items run-id test-id test-name #f))) ;; don't force - just update if no (mutex-unlock! m) (debug:print 2 "Output from running " fullrunscript ", pid " (vector-ref exit-info 0) " in work area " work-area ":\n====\n exit code " (vector-ref exit-info 2) "\n" "====\n") (if (not (vector-ref exit-info 1)) (exit 4))))))) @@ -427,11 +423,16 @@ given-toppath: (get-environment-variable "MT_RUN_AREA_HOME") pathenvvar: "MT_RUN_AREA_HOME")) (set! *configdat* (if (car *configinfo*)(car *configinfo*) #f)) (set! *toppath* (if (car *configinfo*)(cadr *configinfo*) #f)) (if *toppath* - (setenv "MT_RUN_AREA_HOME" *toppath*) ;; to be deprecated + (let ((dbdir (conc *toppath* "/db"))) + (handle-exceptions + exn + (debug:print 0 "ERROR: failed to create the " dbdir " area for your database files") + (if (not (directory-exists? dbdir))(create-directory dbdir))) + (setenv "MT_RUN_AREA_HOME" *toppath*)) (debug:print 0 "ERROR: failed to find the top path to your Megatest area.")))) *toppath*) (define (get-best-disk confdat) (let* ((disks (hash-table-ref/default confdat "disks" #f)) @@ -498,11 +499,11 @@ (lnkbase (conc linktree "/" target "/" runname)) (lnkpath (conc lnkbase "/" testname)) (lnkpathf (conc lnkpath (if not-iterated "" "/") item-path))) ;; Update the rundir path in the test record for all - (cdb:test-set-rundir-by-test-id *runremote* test-id lnkpathf) + (rmt:general-call 'test-set-rundir-by-test-id lnkpathf test-id) (debug:print 2 "INFO:\n lnkbase=" lnkbase "\n lnkpath=" lnkpath "\n toptest-path=" toptest-path "\n test-path=" test-path) (if (not (file-exists? linktree)) (begin (debug:print 0 "WARNING: linktree did not exist! Creating it now at " linktree) @@ -518,15 +519,15 @@ ;; again. ;; NB - This is not working right - some top tests are not getting the path set!!! (if (not (hash-table-ref/default *toptest-paths* testname #f)) - (let* ((testinfo (cdb:get-test-info-by-id *runremote* test-id)) ;; run-id testname item-path)) + (let* ((testinfo (rmt:get-test-info-by-id test-id)) ;; run-id testname item-path)) (curr-test-path (if testinfo (db:test-get-rundir testinfo) #f))) (hash-table-set! *toptest-paths* testname curr-test-path) ;; NB// Was this for the test or for the parent in an iterated test? - (cdb:test-set-rundir! *runremote* run-id testname "" lnkpath) ;; toptest-path) + (rmt:general-call 'test-set-rundir lnkpath run-id testname "") ;; toptest-path) (if (or (not curr-test-path) (not (directory-exists? toptest-path))) (begin (debug:print-info 2 "Creating " toptest-path " and link " lnkpath) (create-directory toptest-path #t) @@ -581,24 +582,15 @@ ;; If there is already a symlink delete it and recreate it. (handle-exceptions exn (begin - (debug:print 0 "ERROR: Failed to re-create link " lnktarget ((condition-property-accessor 'exn 'message) exn) ", exiting") + (debug:print 0 "ERROR: Failed to re-create link " linktarget ((condition-property-accessor 'exn 'message) exn) ", exiting") (exit)) (if (symbolic-link? lnktarget) (delete-file lnktarget)) (if (not (file-exists? lnktarget)) (create-symbolic-link test-path lnktarget))))) - ;; I suspect this section was deleting test directories under some - ;; wierd sitations? This doesn't make sense - reenabling the rm -f - ;; I honestly don't remember *why* this chunk was needed... - ;; (let ((testlink (conc lnkpath "/" testname))) - ;; (if (and (file-exists? testlink) - ;; (or (regular-file? testlink) - ;; (symbolic-link? testlink))) - ;; (system (conc "rm -f " testlink))) - ;; (system (conc "ln -sf " test-path " " testlink))) (if (directory? test-path) (begin (let* ((ovrcmd (let ((cmd (config-lookup *configdat* "setup" "testcopycmd"))) (if cmd ;; substitute the TEST_SRC_PATH and TEST_TARG_PATH @@ -667,11 +659,11 @@ (cmdparms #f) (fullcmd #f) ;; (define a (with-output-to-string (lambda ()(write x)))) (mt-bindir-path #f) (item-path (item-list->path itemdat)) ;; (test-id (cdb:remote-run db:get-test-id #f run-id test-name item-path)) - (testinfo (cdb:get-test-info-by-id *runremote* test-id)) + (testinfo (rmt:get-test-info-by-id test-id)) (mt_target (string-intersperse (map cadr keyvals) "/")) (debug-param (append (if (args:get-arg "-debug") (list "-debug" (args:get-arg "-debug")) '()) (if (args:get-arg "-logging")(list "-logging") '())))) (setenv "MT_ITEMPATH" item-path) (if hosts (set! hosts (string-split hosts))) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -25,10 +25,13 @@ (declare (uses client)) (declare (uses tests)) (declare (uses genexample)) (declare (uses daemon)) (declare (uses db)) +(declare (uses tdb)) +(declare (uses mt)) +(declare (uses api)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm") @@ -466,11 +469,11 @@ (print x)) targets) (set! *didsomething* #t))) (define (full-runconfigs-read) - (let* ((keys (cdb:remote-run db:get-keys #f)) + (let* ((keys (rmt:get-keys)) (target (if (args:get-arg "-reqtarg") (args:get-arg "-reqtarg") (if (args:get-arg "-target") (args:get-arg "-target") #f))) @@ -571,22 +574,24 @@ ;;====================================================================== ;; Query runs ;;====================================================================== +;; NOTE: list-runs and list-db-targets operate on local db!!! +;; (if (or (args:get-arg "-list-runs") (args:get-arg "-list-db-targets")) (if (setup-for-run) - (let* ((db #f) + (let* ((db (open-db)) (runpatt (args:get-arg "-list-runs")) (testpatt (if (args:get-arg "-testpatt") (args:get-arg "-testpatt") "%")) - (runsdat (cdb:remote-run db:get-runs #f runpatt #f #f '())) + (runsdat (db:get-runs db runpatt #f #f '())) (runs (db:get-rows runsdat)) (header (db:get-header runsdat)) - (keys (cdb:remote-run db:get-keys #f)) + (keys (db:get-keys db)) (db-targets (args:get-arg "-list-db-targets")) (seen (make-hash-table))) ;; Each run (for-each (lambda (run) @@ -599,11 +604,11 @@ (hash-table-set! seen targetstr #t) ;; (print "[" targetstr "]")))) (print targetstr)))) (if (not db-targets) (let* ((run-id (db:get-value-by-header run header "id")) - (tests (mt:get-tests-for-run run-id testpatt '() '()))) + (tests (db:get-tests-for-run db run-id testpatt '() '() #f #f #f 'testname 'asc #f))) (print "Run: " targetstr "/" (db:get-value-by-header run header "runname") " status: " (db:get-value-by-header run header "state") " run-id: " run-id ", number tests: " (length tests)) (for-each (lambda (test) @@ -617,29 +622,29 @@ (db:test-get-status test) (db:test-get-run_duration test) (db:test-get-event_time test) (db:test-get-host test)) (if (not (or (equal? (db:test-get-status test) "PASS") - (equal? (db:test-get-status test) "WARN") + (equal? (db:test-get-status test) "WARN") (equal? (db:test-get-state test) "NOT_STARTED"))) (begin - (print " cpuload: " (db:test-get-cpuload test) + (print " cpuload: " (db:test-get-cpuload test) "\n diskfree: " (db:test-get-diskfree test) "\n uname: " (db:test-get-uname test) "\n rundir: " (db:test-get-rundir test) ) ;; Each test ;; DO NOT remote run - (let ((steps (db:get-steps-for-test #f (db:test-get-id test)))) + (let ((steps (tdb:get-steps-for-test (db:test-get-id test)))) (for-each (lambda (step) (format #t " Step: ~20a State: ~10a Status: ~10a Time ~22a\n" - (db:step-get-stepname step) - (db:step-get-state step) - (db:step-get-status step) - (db:step-get-event_time step))) + (tdb:step-get-stepname step) + (tdb:step-get-state step) + (tdb:step-get-status step) + (tdb:step-get-event_time step))) steps))))) tests))))) runs) (set! *didsomething* #t)))) @@ -763,11 +768,11 @@ (exit 1))) (if (not (setup-for-run)) (begin (debug:print 0 "Failed to setup, giving up on -test-paths or -test-files, exiting") (exit 1))) - (let* ((keys (cdb:remote-run db:get-keys db)) + (let* ((keys (rmt:get-keys)) ;; db:test-get-paths must not be run remote (paths (db:test-get-paths-matching db keys target (args:get-arg "-test-files")))) (set! *didsomething* #t) (for-each (lambda (path) (print path)) @@ -799,11 +804,11 @@ (test-name (assoc/default 'test-name cmdinfo)) (runscript (assoc/default 'runscript cmdinfo)) (db-host (assoc/default 'db-host cmdinfo)) (run-id (assoc/default 'run-id cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) - (db #f) + (db (open-db)) (state (args:get-arg ":state")) (status (args:get-arg ":status")) (target (args:get-arg "-target"))) (change-directory testpath) ;; (set! *runremote* runremote) @@ -814,28 +819,30 @@ (exit 1))) (if (not (setup-for-run)) (begin (debug:print 0 "Failed to setup, giving up on -archive, exiting") (exit 1))) - (let* ((keys (cdb:remote-run db:get-keys db)) + (let* ((keys (db:get-keys db)) ;; DO NOT run remote (paths (db:test-get-paths-matching db keys target))) (set! *didsomething* #t) (for-each (lambda (path) (print path)) - paths))) + paths)) + (if (sqlite3:database? db)(sqlite3:finalize! db))) ;; else do a general-run-call (general-run-call "-test-paths" "Get paths to tests" (lambda (target runname keys keyvals) - (let* ((db #f) + (let* ((db (open-db)) ;; DO NOT run remote (paths (db:test-get-paths-matching db keys target))) (for-each (lambda (path) (print path)) - paths)))))) + paths) + (sqlite3:finalize! db)))))) ;;====================================================================== ;; Extract a spreadsheet from the runs database ;;====================================================================== @@ -842,17 +849,19 @@ (if (args:get-arg "-extract-ods") (general-run-call "-extract-ods" "Make ods spreadsheet" (lambda (target runname keys keyvals) - (let ((db #f) + (let ((db (open-db)) (outputfile (args:get-arg "-extract-ods")) (runspatt (args:get-arg ":runname")) (pathmod (args:get-arg "-pathmod"))) ;; (keyvalalist (keys->alist keys "%"))) (debug:print 2 "Extract ods, outputfile: " outputfile " runspatt: " runspatt " keyvals: " keyvals) - (cdb:remote-run db:extract-ods-file db outputfile keyvals (if runspatt runspatt "%") pathmod))))) + (db:extract-ods-file db outputfile keyvals (if runspatt runspatt "%") pathmod) + (sqlite3:finalize! db) + (set! *didsomething* #t))))) ;;====================================================================== ;; execute the test ;; - gets called on remote host ;; - receives info from the -execute param @@ -894,11 +903,11 @@ (begin (debug:print 0 "Failed to setup, exiting") (exit 1))) (if (and state status) ;; DO NOT remote run, makes calls to the testdat.db test db. - (db:teststep-set-status! db test-id step state status msg logfile work-area: work-area) + (tdb:teststep-set-status! test-id step state status msg logfile work-area: work-area) (begin (debug:print 0 "ERROR: You must specify :state and :status with every call to -step") (exit 6)))))) (if (args:get-arg "-step") @@ -953,17 +962,17 @@ ;; (client:setup) (if (args:get-arg "-load-test-data") ;; has sub commands that are rdb: ;; DO NOT put this one into either cdb:remote-run or open-run-close - (db:load-test-data db test-id work-area: work-area)) + (tdb:load-test-data test-id work-area: work-area)) (if (args:get-arg "-setlog") (let ((logfname (args:get-arg "-setlog"))) - (cdb:test-set-log! *runremote* test-id logfname))) + (rmt:test-set-log! test-id logfname))) (if (args:get-arg "-set-toplog") ;; DO NOT run remote - (tests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog"))) + (tests:test-set-toplog! run-id test-name (args:get-arg "-set-toplog"))) (if (args:get-arg "-summarize-items") ;; DO NOT run remote (tests:summarize-items db run-id test-id test-name #t)) ;; do force here (if (args:get-arg "-runstep") (if (null? remargs) @@ -985,11 +994,11 @@ (fullcmd (conc "(" (string-intersperse (cons cmd params) " ") ") " redir " " logfile))) ;; mark the start of the test ;; DO NOT run remote - (db:teststep-set-status! db test-id stepname "start" "n/a" (args:get-arg "-m") logfile work-area: work-area) + (tdb:teststep-set-status! test-id stepname "start" "n/a" (args:get-arg "-m") logfile work-area: work-area) ;; run the test step (debug:print-info 2 "Running \"" fullcmd "\" in directory \"" startingdir) (change-directory startingdir) (set! exitstat (system fullcmd)) ;; cmd params)) (set! *globalexitstatus* exitstat) @@ -1002,14 +1011,14 @@ (debug:print-info 2 "running \"" cmd "\"") (change-directory startingdir) (set! exitstat (system cmd)) (set! *globalexitstatus* exitstat) ;; no necessary (change-directory testpath) - (cdb:test-set-log! *runremote* test-id htmllogfile))) + (rmt:test-set-log! test-id htmllogfile))) (let ((msg (args:get-arg "-m"))) ;; DO NOT run remote - (db:teststep-set-status! db test-id stepname "end" exitstat msg logfile work-area: work-area)) + (tdb:teststep-set-status! test-id stepname "end" exitstat msg logfile work-area: work-area)) ))) (if (or (args:get-arg "-test-status") (args:get-arg "-set-values")) (let ((newstatus (cond ((number? status) (if (equal? status 0) "PASS" "FAIL")) @@ -1027,17 +1036,17 @@ (if (and (args:get-arg "-test-status") (or (not state) (not status))) (begin (debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help) - ;; (sqlite3:finalize! db) + (if (sqlite3:database? db)(sqlite3:finalize! db)) (exit 6))) (let* ((msg (args:get-arg "-m")) (numoth (length (hash-table-keys otherdata)))) ;; Convert to rpc inside the tests:test-set-status! call, not here (tests:test-set-status! test-id state newstatus msg otherdata work-area: work-area)))) - (if db (sqlite3:finalize! db)) + (if (sqlite3:database? db)(sqlite3:finalize! db)) (set! *didsomething* #t)))) ;;====================================================================== ;; Various helper commands can go below here ;;====================================================================== @@ -1050,11 +1059,11 @@ (begin (debug:print 0 "Failed to setup, exiting") (exit 1))) (set! keys (cdb:remote-run db:get-keys db)) (debug:print 1 "Keys: " (string-intersperse keys ", ")) - (if db (sqlite3:finalize! db)) + (if (sqlite3:database? db)(sqlite3:finalize! db)) (set! *didsomething* #t))) (if (args:get-arg "-gui") (begin (debug:print 0 "Look at the dashboard for now") Index: mt.scm ================================================================== --- mt.scm +++ mt.scm @@ -17,10 +17,11 @@ (declare (uses items)) (declare (uses runconfig)) (declare (uses tests)) (declare (uses server)) (declare (uses runs)) +(declare (uses rmt)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -39,11 +40,11 @@ ;; ;; Use: (db-get-value-by-header (db:get-header runinfo)(db:get-row runinfo)) ;; to extract info from the structure returned ;; (define (mt:get-runs-by-patt keys runnamepatt targpatt) - (let loop ((runsdat (cdb:remote-run db:get-runs-by-patt #f keys runnamepatt targpatt 0 500)) + (let loop ((runsdat (rmt:get-runs-by-patt keys runnamepatt targpatt 0 500)) (res '()) (offset 0) (limit 500)) ;; (print "runsdat: " runsdat) (let* ((header (vector-ref runsdat 0)) @@ -51,11 +52,11 @@ (full-list (append res runslst)) (have-more (eq? (length runslst) limit))) ;; (debug:print 0 "header: " header " runslst: " runslst " have-more: " have-more) (if have-more (let ((new-offset (+ offset limit)) - (next-batch (cdb:remote-run db:get-runs-by-patt #f keys runnamepatt targpatt offset limit))) + (next-batch (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit))) (debug:print-info 4 "More than " limit " runs, have " (length full-list) " runs so far.") (debug:print-info 0 "next-batch: " next-batch) (loop next-batch full-list new-offset @@ -65,28 +66,25 @@ ;;====================================================================== ;; T E S T S ;;====================================================================== (define (mt:get-tests-for-run run-id testpatt states status #!key (not-in #t) (sort-by 'event_time) (sort-order "ASC") (qryvals #f)) - (let loop ((testsdat (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status 0 500 not-in sort-by sort-order qryvals: qryvals)) + (let loop ((testsdat (rmt:get-tests-for-run run-id testpatt states status 0 500 not-in sort-by sort-order qryvals)) (res '()) (offset 0) (limit 500)) (let* ((full-list (append res testsdat)) (have-more (eq? (length testsdat) limit))) (if have-more (let ((new-offset (+ offset limit))) (debug:print-info 4 "More than " limit " tests, have " (length full-list) " tests so far.") - (loop (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals: qryvals) + (loop (rmt:get-tests-for-run run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals) full-list new-offset limit)) full-list)))) -(define (mt:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal)) - (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode)) - (define (mt:lazy-get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal)) (let* ((key (list run-id waitons ref-item-path mode)) (res (hash-table-ref/default *pre-reqs-met-cache* key #f)) (useres (let ((last-time (if (vector? res) (vector-ref res 0) #f))) (if last-time @@ -94,16 +92,17 @@ #f)))) (if useres (let ((result (vector-ref res 1))) (debug:print 4 "Using lazy value res: " result) result) - (let ((newres (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode))) + (let ((newres (rmt:get-prereqs-not-met run-id waitons ref-item-path mode: mode))) (hash-table-set! *pre-reqs-met-cache* key (vector (current-seconds) newres)) newres)))) +;; Get run stats from local access, move this ... but where? (define (mt:get-run-stats) - (cdb:remote-run db:get-run-stats #f)) + (db:get-run-stats #f)) (define (mt:discard-blocked-tests run-id failed-test tests test-records) (if (null? tests) tests (begin @@ -126,11 +125,11 @@ ;;====================================================================== ;; T R I G G E R S ;;====================================================================== (define (mt:process-triggers test-id newstate newstatus) - (let* ((test-dat (mt:lazy-get-test-info-by-id test-id)) + (let* ((test-dat (rmt:get-test-info-by-id test-id)) (test-rundir (db:test-get-rundir test-dat)) (test-name (db:test-get-testname test-dat)) (tconfig #f) (state (if newstate newstate (db:test-get-state test-dat))) (status (if newstatus newstatus (db:test-get-status test-dat)))) @@ -157,32 +156,21 @@ ;;====================================================================== ;; S T A T E A N D S T A T U S F O R T E S T S ;;====================================================================== -(define (mt:roll-up-pass-fail-counts run-id test-name item-path status) - (if (and (not (equal? item-path "")) - (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP"))) - (begin - (cdb:update-pass-fail-counts *runremote* run-id test-name) - (if (equal? status "RUNNING") - (cdb:top-test-set-running *runremote* run-id test-name) - (cdb:top-test-set-per-pf-counts *runremote* run-id test-name)) - #f) - #f)) - ;; speed up for common cases with a little logic (define (mt:test-set-state-status-by-id test-id newstate newstatus newcomment) (cond ((and newstate newstatus newcomment) - (cdb:client-call *runremote* 'state-status-msg #t *default-numtries* newstate newstatus newcomment test-id)) + (rmt:general-call 'state-status-msg newstate newstatus newcomment test-id)) ((and newstate newstatus) - (cdb:client-call *runremote* 'state-status #t *default-numtries* newstate newstatus test-id)) + (rmt:general-call 'state-status newstate newstatus test-id)) (else - (if newstate (cdb:client-call *runremote* 'set-test-state #t *default-numtries* newstate test-id)) - (if newstatus (cdb:client-call *runremote* 'set-test-status #t *default-numtries* newstatus test-id)) - (if newcomment (cdb:client-call *runremote* 'set-test-comment #t *default-numtries* newcomment test-id)))) + (if newstate (rmt:general-call 'set-test-state newstate test-id)) + (if newstatus (rmt:general-call 'set-test-status newstatus test-id)) + (if newcomment (rmt:general-call 'set-test-comment newcomment test-id)))) (mt:process-triggers test-id newstate newstatus) #t) (define (mt:lazy-get-test-info-by-id test-id) (let* ((tdat (hash-table-ref/default *test-info* test-id #f))) ADDED rmt.scm Index: rmt.scm ================================================================== --- /dev/null +++ rmt.scm @@ -0,0 +1,249 @@ +;;====================================================================== +;; Copyright 2006-2013, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + +(use json) + +(declare (unit rmt)) +(declare (uses api)) +(declare (uses tdb)) +(declare (uses http-transport)) + +;; +;; THESE ARE ALL CALLED ON THE CLIENT SIDE!!! +;; + +;; ;; For debugging add the following to ~/.megatestrc +;; +;; (require-library trace) +;; (import trace) +;; (trace +;; rmt:send-receive +;; api:execute-requests +;; ) + + +;;====================================================================== +;; S U P P O R T F U N C T I O N S +;;====================================================================== + +;; cmd is a symbol +;; vars is a json string encoding the parameters for the call +;; +(define (rmt:send-receive cmd params) + (case *transport-type* + ((fs-aint-here) + (debug:print 0 "ERROR: Not yet (re)supported") + (exit 1)) + ((fs http) + (let* ((jparams (db:obj->string params)) ;; (rmt:dat->json-str params)) + (res (http-transport:client-api-send-receive *runremote* cmd jparams))) + (if res + (db:string->obj res) ;; (rmt:json-str->dat res) + (begin + (debug:print 0 "ERROR: Bad value from http-transport:client-api-send-receive " res) + #f)) + )) + (else + (debug:print 0 "ERROR: Transport " *transport-type* " not yet (re)supported") + (exit 1)))) + +;; Wrap json library for strings (why the ports crap in the first place?) +(define (rmt:dat->json-str dat) + (with-output-to-string + (lambda () + (json-write dat)))) + +(define (rmt:json-str->dat json-str) + (with-input-from-string json-str + (lambda () + (json-read)))) + +;;====================================================================== +;; +;; A C T U A L A P I C A L L S +;; +;;====================================================================== + +;;====================================================================== +;; M I S C +;;====================================================================== + +(define (rmt:login) + (rmt:send-receive 'login (list *toppath* megatest-version *my-client-signature*))) + +(define (rmt:kill-server) + (rmt:send-receive 'kill-server '())) + +;; hand off a call to one of the db:queries statements +(define (rmt:general-call stmtname . params) + (rmt:send-receive 'general-call (append (list stmtname) params))) + +(define (rmt:sync-inmem->db) + (rmt:send-receive 'sync-inmem->db '())) + +;;====================================================================== +;; K E Y S +;;====================================================================== + +(define (rmt:get-key-val-pairs run-id) + (rmt:send-receive 'get-key-val-pairs (list run-id))) + +(define (rmt:get-keys) + (rmt:send-receive 'get-keys '())) + +;;====================================================================== +;; T E S T S +;;====================================================================== + +(define (rmt:get-test-id run-id testname item-path) + (rmt:send-receive 'get-test-id (list run-id testname item-path))) + +(define (rmt:get-test-info-by-id test-id) + (rmt:send-receive 'get-test-info-by-id (list test-id))) + +(define (rmt:test-get-rundir-from-test-id test-id) + (rmt:send-receive 'test-get-rundir-from-test-id (list test-id))) + +(define (rmt:open-test-db-by-test-id test-id #!key (work-area #f)) + (let* ((test-path (if (string? work-area) + work-area + (rmt:test-get-rundir-from-test-id test-id)))) + (debug:print 3 "TEST PATH: " test-path) + (open-test-db test-path))) + +;; WARNING: This currently bypasses the transaction wrapped writes system +(define (rmt:test-set-state-status-by-id test-id newstate newstatus newcomment) + (rmt:send-receive 'test-set-state-status-by-id (list test-id newstate newstatus newcomment))) + +(define (rmt:set-tests-state-status run-id testnames currstate currstatus newstate newstatus) + (rmt:send-receive 'set-tests-state-status (list run-id testnames currstate currstatus newstate newstatus))) + +(define (rmt:get-tests-for-run run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals) + (rmt:send-receive 'get-tests-for-run (list run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals))) + +(define (rmt:get-tests-for-runs-mindata run-ids testpatt states status not-in) + (rmt:send-receive 'get-tests-for-runs-mindata (list run-ids testpatt states status not-in))) + +(define (rmt:delete-test-records test-id) + (rmt:send-receive 'delete-test-records (list test-id))) + +(define (rmt:test-set-status-state test-id status state msg) + (rmt:send-receive 'test-set-status-state (list test-id status state msg))) + +(define (rmt:get-previous-test-run-record run-id test-name item-path) + (rmt:send-receive 'get-previous-test-run-record (list run-id test-name item-path))) + +(define (rmt:get-matching-previous-test-run-records run-id test-name item-path) + (rmt:send-receive 'get-matching-previous-test-run-records (list run-id test-name item-path))) + +(define (rmt:test-get-logfile-info run-id test-name) + (rmt:send-receive 'test-get-logfile-info (list run-id test-name))) + +(define (rmt:test-get-records-for-index-file run-id test-name) + (rmt:send-receive 'test-get-records-for-index-file (list run-id test-name))) + +(define (rmt:get-testinfo-state-status test-id) + (rmt:send-receive 'get-testinfo-state-status (list test-id))) + +(define (rmt:test-set-log! test-id logf) + (if (string? logf)(rmt:general-call 'test-set-log logf test-id))) + +(define (rmt:test-get-paths-matching-keynames-target-new keynames target res testpatt statepatt statuspatt runname) + (rmt:send-receive 'test-get-paths-matching-keynames-target-new (list keynames target res testpatt statepatt statuspatt runname))) + +(define (rmt:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal)) + (rmt:send-receive 'get-prereqs-not-met (list run-id waitons ref-item-path mode))) + +(define (rmt:get-count-tests-running-for-run-id run-id) + (rmt:send-receive 'get-count-tests-running-for-run-id (list run-id))) + +;; Statistical queries + +(define (rmt:get-count-tests-running) + (rmt:send-receive 'get-count-tests-running '())) + +(define (rmt:get-count-tests-running-in-jobgroup jobgroup) + (rmt:send-receive 'get-count-tests-running-in-jobgroup (list jobgroup))) + +(define (rmt:roll-up-pass-fail-counts run-id test-name item-path status) + (rmt:send-receive 'roll-up-pass-fail-counts (list run-id test-name item-path status))) + +(define (rmt:update-pass-fail-counts run-id test-name) + (rmt:general-call 'update-fail-pass-counts run-id test-name run-id test-name run-id test-name)) + +;;====================================================================== +;; R U N S +;;====================================================================== + +(define (rmt:get-run-info run-id) + (rmt:send-receive 'get-run-info (list run-id))) + +(define (rmt:register-run keyvals runname state status user) + (rmt:send-receive 'register-run (list keyvals runname state status user))) + +(define (rmt:get-run-name-from-id run-id) + (rmt:send-receive 'get-run-name-from-id (list run-id))) + +(define (rmt:delete-run run-id) + (rmt:send-receive 'delete-run (list run-id))) + +(define (rmt:delete-old-deleted-test-records) + (rmt:send-receive 'delete-old-deleted-test-records '())) + +(define (rmt:get-runs runpatt count offset keypatts) + (rmt:send-receive 'get-runs (list runpatt count offset keypatts))) + +(define (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit) + (rmt:send-receive 'get-runs-by-patt (list keys runnamepatt targpatt offset limit))) + +(define (rmt:lock/unlock-run run-id lock unlock user) + (rmt:send-receive 'lock/unlock-run (list run-id lock unlock user))) + +(define (rmt:update-run-event_time run-id) + (rmt:send-receive 'update-run-event_time (list run-id))) + +;;====================================================================== +;; S T E P S +;;====================================================================== + +;; Getting steps is more complicated. +;; +;; If given work area +;; 1. Find the testdat.db file +;; 2. Open the testdat.db file and do the query +;; If not given the work area +;; 1. Do a remote call to get the test path +;; 2. Continue as above +;; +(define (rmt:get-steps-for-test test-id #!key (work-area #f)) + (let* ((tdb (rmt:open-test-db-by-test-id test-id work-area: work-area))) + (if tdb + (tdb:get-steps-data tdb test-id) + '()))) + +;;====================================================================== +;; T E S T D A T A +;;====================================================================== + +(define (rmt:read-test-data test-id categorypatt #!key (work-area #f)) + (let ((tdb (rmt:open-test-db-by-test-id test-id work-area: work-area))) + (if tdb + (tdb:read-test-data tdb test-id categorypatt) + '()))) + +(define (rmt:testmeta-add-record testname) + (rmt:send-receive 'testmeta-add-record (list testname))) + +(define (rmt:testmeta-get-record testname) + (rmt:send-receive 'testmeta-get-record (list testname))) + +(define (rmt:testmeta-update-field test-name fld val) + (rmt:send-receive 'testmeta-update-field (list test-name fld val))) ADDED rmtdb.scm Index: rmtdb.scm ================================================================== --- /dev/null +++ rmtdb.scm @@ -0,0 +1,11 @@ +;;====================================================================== +;; Copyright 2006-2013, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -97,21 +97,21 @@ (define (set-megatest-env-vars run-id #!key (inkeys #f)(inrunname #f)(inkeyvals #f)) (let* ((target (or (args:get-arg "-reqtarg") (args:get-arg "-target") (get-environment-variable "MT_TARGET"))) - (keys (if inkeys inkeys (cdb:remote-run db:get-keys #f))) + (keys (if inkeys inkeys (rmt:get-keys))) (keyvals (if inkeyvals inkeyvals (keys:target->keyval keys target))) (vals (hash-table-ref/default *env-vars-by-run-id* run-id #f))) ;; get the info from the db and put it in the cache (if (not vals) (let ((ht (make-hash-table))) (hash-table-set! *env-vars-by-run-id* run-id ht) (set! vals ht) (for-each (lambda (key) - (hash-table-set! vals (car key) (cadr key))) ;; (cdb:remote-run db:get-run-key-val #f run-id (car key)))) + (hash-table-set! vals (car key) (cadr key))) keyvals))) ;; from the cached data set the vars (hash-table-for-each vals (lambda (key val) @@ -121,11 +121,14 @@ (setenv key val) (debug:print 0 "ERROR: Malformed environment variable definition: var=" var ", val=" val)))) (if (not (get-environment-variable "MT_TARGET"))(setenv "MT_TARGET" target)) (alist->env-vars (hash-table-ref/default *configdat* "env-override" '())) ;; Lets use this as an opportunity to put MT_RUNNAME in the environment - (setenv "MT_RUNNAME" (if inrunname inrunname (cdb:remote-run db:get-run-name-from-id #f run-id))) + (let ((runname (if inrunname inrunname (rmt:get-run-name-from-id run-id)))) + (if runname + (setenv "MT_RUNNAME" runname) + (debug:print 0 "ERROR: no value for runname for id " run-id))) (setenv "MT_RUN_AREA_HOME" *toppath*))) (define (set-item-env-vars itemdat) (for-each (lambda (item) (debug:print 2 "setenv " (car item) " " (cadr item)) @@ -136,19 +139,20 @@ ;; ;; NOTE: We run this server-side!! Do not use this global except in the runs:can-run-more-tests routine ;; (define *last-num-running-tests* 0) (define *runs:can-run-more-tests-count* 0) -(define (runs:shrink-can-run-more-tests-count) ;; the db is a dummy var so we can use cdb:remote-run +(define (runs:shrink-can-run-more-tests-count) (set! *runs:can-run-more-tests-count* 0)) ;; (/ *runs:can-run-more-tests-count* 2))) ;; Temporary globals. Move these into the logic or into common ;; (define *seen-cant-run-tests* (make-hash-table)) ;; use to track tests that we suspect cannot be run (define (runs:inc-cant-run-tests testname) (hash-table-set! *seen-cant-run-tests* testname (+ (hash-table-ref/default *seen-cant-run-tests* testname 0) 1))) + (define (runs:can-keep-running? testname n) (< (hash-table-ref/default *seen-cant-run-tests* testname 0) n)) (define *runs:denoise* (make-hash-table)) ;; key => last-time-ran @@ -163,12 +167,12 @@ (define (runs:can-run-more-tests jobgroup max-concurrent-jobs) (thread-sleep! (cond ((> *runs:can-run-more-tests-count* 20) 2);; obviously haven't had any work to do for a while (else 0))) - (let* ((num-running (cdb:remote-run db:get-count-tests-running #f)) - (num-running-in-jobgroup (cdb:remote-run db:get-count-tests-running-in-jobgroup #f jobgroup)) + (let* ((num-running (rmt:get-count-tests-running)) + (num-running-in-jobgroup (rmt:get-count-tests-running-in-jobgroup jobgroup)) (job-group-limit (config-lookup *configdat* "jobgroups" jobgroup))) (if (> (+ num-running num-running-in-jobgroup) 0) (set! *runs:can-run-more-tests-count* (+ *runs:can-run-more-tests-count* 1))) (if (not (eq? *last-num-running-tests* num-running)) (begin @@ -200,11 +204,11 @@ ;; (define (runs:run-tests target runname test-patts user flags) ;; test-names (common:clear-caches) ;; clear all caches (let* ((keys (keys:config-get-fields *configdat*)) (keyvals (keys:target->keyval keys target)) - (run-id (cdb:remote-run db:register-run #f keyvals runname "new" "n/a" user)) ;; test-name))) + (run-id (rmt:register-run keyvals runname "new" "n/a" user)) ;; test-name))) (deferred '()) ;; delay running these since they have a waiton clause (runconfigf (conc *toppath* "/runconfigs.config")) (required-tests '()) (test-records (make-hash-table)) (all-tests-registry (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names @@ -211,13 +215,12 @@ (all-test-names (hash-table-keys all-tests-registry)) (test-names (tests:filter-test-names all-test-names test-patts))) ;; Update the synchronous setting in the db based on the default or what is set by the user ;; This is done once here on a call to run tests rather than on every call to open-db - (cdb:remote-run db:set-sync #f) - (set-megatest-env-vars run-id inkeys: keys) ;; these may be needed by the launching process + (set-megatest-env-vars run-id inkeys: keys inrunname: runname) ;; these may be needed by the launching process (if (file-exists? runconfigf) (setup-env-defaults runconfigf run-id *already-seen-runconfig-info* keyvals "pre-launch-env-vars") (debug:print 0 "WARNING: You do not have a run config file: " runconfigf)) ;; look up all tests matching the comma separated list of globs in @@ -232,12 +235,12 @@ (begin ;; have to delete test records where NOT_STARTED since they can cause -keepgoing to ;; get stuck due to becoming inaccessible from a failed test. I.e. if test B depends ;; on test A but test B reached the point on being registered as NOT_STARTED and test ;; A failed for some reason then on re-run using -keepgoing the run can never complete. - (cdb:delete-tests-in-state *runremote* run-id "NOT_STARTED") - (cdb:remote-run db:set-tests-state-status #f run-id test-names #f "FAIL" "NOT_STARTED" "FAIL"))) + (rmt:general-call 'delete-tests-in-state run-id "NOT_STARTED") + (rmt:set-tests-state-status run-id test-names #f "FAIL" "NOT_STARTED" "FAIL"))) ;; Ensure all tests are registered in the test_meta table (runs:update-all-test_meta #f) ;; now add non-directly referenced dependencies (i.e. waiton) @@ -249,10 +252,11 @@ ;;====================================================================== (if (not (null? test-names)) (let loop ((hed (car test-names)) (tal (cdr test-names))) ;; 'return-procs tells the config reader to prep running system but return a proc (change-directory *toppath*) ;; PLEASE OPTIMIZE ME!!! I think this should be a no-op but there are several places where change-directories could be happening. + (setenv "MT_TEST_NAME" hed) ;; (let* ((config (tests:get-testconfig hed all-tests-registry 'return-procs)) (waitons (let ((instr (if config (config-lookup config "requirements" "waiton") (begin ;; No config means this is a non-existant test (debug:print 0 "ERROR: non-existent required test \"" hed "\"") @@ -380,11 +384,11 @@ (define runs:nothing-left-in-queue-count 0) (define (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records) (let* ((loop-list (list hed tal reg reruns)) - (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode)) + (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path mode: testmode)) ;; (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode)) (fails (runs:calc-fails prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met))) (debug:print-info 4 "START OF INNER COND #2 " "\n can-run-more: " can-run-more "\n testname: " hed @@ -585,11 +589,11 @@ (have-resources (car run-limits-info)) (num-running (list-ref run-limits-info 1)) (num-running-in-jobgroup (list-ref run-limits-info 2)) (max-concurrent-jobs (list-ref run-limits-info 3)) (job-group-limit (list-ref run-limits-info 4)) - (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode)) + (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path mode: testmode)) ;; (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode)) (fails (runs:calc-fails prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met)) (loop-list (list hed tal reg reruns))) (debug:print-info 4 "have-resources: " have-resources " prereqs-not-met: (" (string-intersperse @@ -625,20 +629,20 @@ ;; ((not (hash-table-ref/default test-registry (runs:make-full-test-name test-name item-path) #f)) (debug:print-info 4 "Pre-registering test " test-name "/" item-path " to create placeholder" ) (if (eq? *transport-type* 'fs) ;; no point in parallel registration if use fs (begin - (cdb:tests-register-test *runremote* run-id test-name item-path) + (rmt:general-call 'register-test run-id test-name item-path) (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'done)) (let ((th (make-thread (lambda () (mutex-lock! registry-mutex) (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'start) (mutex-unlock! registry-mutex) ;; If haven't done it before register a top level test if this is an itemized test (if (not (eq? (hash-table-ref/default test-registry (runs:make-full-test-name test-name "") #f) 'done)) - (cdb:tests-register-test *runremote* run-id test-name "")) - (cdb:tests-register-test *runremote* run-id test-name item-path) + (rmt:general-call 'register-test run-id test-name "")) + (rmt:general-call 'register-test run-id test-name item-path) (mutex-lock! registry-mutex) (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'done) (mutex-unlock! registry-mutex)) (conc test-name "/" item-path)))) (thread-start! th))) @@ -750,11 +754,11 @@ ;; Do mark-and-find clean up of db before starting runing of quue ;; ;; (cdb:remote-run db:find-and-mark-incomplete #f) - (let ((run-info (cdb:remote-run db:get-run-info #f run-id)) + (let ((run-info (rmt:get-run-info run-id)) (tests-info (mt:get-tests-for-run run-id #f '() '())) ;; qryvals: "id,testname,item_path")) (sorted-test-names (tests:sort-by-priority-and-waiton test-records)) (test-registry (make-hash-table)) (registry-mutex (make-mutex)) (num-retries 0) @@ -762,11 +766,12 @@ (max-concurrent-jobs (let ((mcj (config-lookup *configdat* "setup" "max_concurrent_jobs"))) (if (and mcj (string->number mcj)) (string->number mcj) 1))) ;; length of the register queue ahead (reglen (if (number? reglen-in) reglen-in 1)) - (last-time-incomplete (current-seconds))) + (last-time-incomplete (current-seconds)) + (last-time-some-running (current-seconds))) ;; Initialize the test-registery hash with tests that already have a record ;; convert state to symbol and use that as the hash value (for-each (lambda (trec) (let ((id (db:test-get-id trec)) @@ -802,20 +807,25 @@ (itemdat (tests:testqueue-get-itemdat test-record)) ;; itemdat can be a string, list or #f (items (tests:testqueue-get-items test-record)) (item-path (item-list->path itemdat)) (tfullname (runs:make-full-test-name test-name item-path)) (newtal (append tal (list hed))) - (regfull (>= (length reg) reglen))) + (regfull (>= (length reg) reglen)) + (num-running (rmt:get-count-tests-running-for-run-id run-id))) + + (if (> num-running 0) + (set! last-time-some-running (current-seconds))) - (hash-table-set! *max-tries-hash* tfullname (+ (hash-table-ref/default *max-tries-hash* tfullname 0) 1)) + (if (> (current-seconds)(+ last-time-some-running 60)) + (hash-table-set! *max-tries-hash* tfullname (+ (hash-table-ref/default *max-tries-hash* tfullname 0) 1))) ;; (debug:print 0 "max-tries-hash: " (hash-table->alist *max-tries-hash*)) ;; Ensure all top level tests get registered. This way they show up as "NOT_STARTED" on the dashboard ;; and it is clear they *should* have run but did not. (if (not (hash-table-ref/default test-registry (runs:make-full-test-name test-name "") #f)) (begin - (cdb:tests-register-test *runremote* run-id test-name "") + (rmt:general-call 'register-test run-id test-name "") (hash-table-set! test-registry (runs:make-full-test-name test-name "") 'done))) ;; Fast skip of tests that are already "COMPLETED" - NO! Cannot do that as the items may not have been expanded yet :( ;; (if (member (hash-table-ref/default test-registry tfullname #f) @@ -1033,12 +1043,12 @@ (hash-table-set! *test-meta-updated* test-name #t) (runs:update-test_meta test-name test-conf))) ;; itemdat => ((ripeness "overripe") (temperature "cool") (season "summer")) (let* ((new-test-path (string-intersperse (cons test-path (map cadr itemdat)) "/")) - (test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path)) - (testdat (if test-id (cdb:get-test-info-by-id *runremote* test-id) #f))) + (test-id (rmt:get-test-id run-id test-name item-path)) + (testdat (if test-id (rmt:get-test-info-by-id test-id) #f))) (if (not testdat) (let loop () ;; ensure that the path exists before registering the test ;; NOPE: Cannot! Don't know yet which disk area will be assigned.... ;; (system (conc "mkdir -p " new-test-path)) @@ -1045,18 +1055,18 @@ ;; ;; (open-run-close tests:register-test db run-id test-name item-path) ;; ;; NB// for the above line. I want the test to be registered long before this routine gets called! ;; - (if (not test-id)(set! test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path))) + (if (not test-id)(set! test-id (rmt:get-test-id-cached run-id test-name item-path))) (if (not test-id) (begin (debug:print 2 "WARN: Test not pre-created? test-name=" test-name ", item-path=" item-path ", run-id=" run-id) - (cdb:tests-register-test *runremote* run-id test-name item-path) - (set! test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path)))) + (rmt:general-call 'register-test run-id test-name item-path) + (set! test-id (rmt:get-test-id run-id test-name item-path)))) (debug:print-info 4 "test-id=" test-id ", run-id=" run-id ", test-name=" test-name ", item-path=\"" item-path "\"") - (set! testdat (cdb:get-test-info-by-id *runremote* test-id)) + (set! testdat (rmt:get-test-info-by-id test-id)) (if (not testdat) (begin (debug:print-info 0 "WARNING: server is overloaded, trying again in one second") (thread-sleep! 1) (loop))))) @@ -1122,11 +1132,11 @@ (cond ;; Have to check for skip conditions. This one skips if there are same-named tests ;; currently running ((and skip-check (configf:lookup test-conf "skip" "prevrunning")) - (let ((running-tests (cdb:remote-run db:get-tests-for-runs-mindata #f #f full-test-name '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED") '() #f))) + (let ((running-tests (rmt:get-tests-for-runs-mindata #f full-test-name '("RUNNING" "REMOTEHOSTSTART" "LAUNCHED") '() #f))) (if (not (null? running-tests)) ;; have to skip (set! skip-test "Skipping due to previous tests running")))) ((and skip-check (configf:lookup test-conf "skip" "fileexists")) (if (file-exists? (configf:lookup test-conf "skip" "fileexists")) @@ -1202,11 +1212,11 @@ ;; NB// should pass in keys? ;; (define (runs:operate-on action target runnamepatt testpatt #!key (state #f)(status #f)(new-state-status #f)) (common:clear-caches) ;; clear all caches (let* ((db #f) - (keys (cdb:remote-run db:get-keys db)) + (keys (rmt:get-keys)) (rundat (mt:get-runs-by-patt keys runnamepatt target)) (header (vector-ref rundat 0)) (runs (vector-ref rundat 1)) (states (if state (string-split state ",") '())) (statuses (if status (string-split status ",") '())) @@ -1257,11 +1267,11 @@ (test-retry-time (make-hash-table)) (allow-run-time 10)) ;; seconds to allow for killing tests before just brutally killing 'em (let loop ((test (car sorted-tests)) (tal (cdr sorted-tests))) (let* ((test-id (db:test-get-id test)) - (new-test-dat (cdb:get-test-info-by-id *runremote* test-id))) + (new-test-dat (rmt:get-test-info-by-id test-id))) (if (not new-test-dat) (begin (debug:print 0 "ERROR: We have a test-id of " test-id " but no record was found. NOTE: No locking of records is done between processes, do not simultaneously remove the same run from two processes!") (if (not (null? tal)) (loop (car tal)(cdr tal)))) @@ -1328,11 +1338,11 @@ (if run-dir (debug:print 0 "WARNING: not removing " run-dir " as it either doesn't exist or is not a symlink") (debug:print 0 "NOTE: the run dir for this test is undefined. Test may have already been deleted.")) )) ;; Only delete the records *after* removing the directory. If things fail we have a record - (cdb:remote-run db:delete-test-records db #f (db:test-get-id test)) + (rmt:delete-test-records (db:test-get-id test)) (if (not (null? tal)) (loop (car tal)(cdr tal)))))) ((set-state-status) (debug:print-info 2 "new state " (car state-status) ", new status " (cadr state-status)) (mt:test-set-state-status-by-id (db:test-get-id test) (car state-status)(cadr state-status) #f) @@ -1353,15 +1363,13 @@ (let* ((dparts (string-split lasttpath "/")) (runpath (conc "/" (string-intersperse (take dparts (- (length dparts) 1)) "/")))) (debug:print 1 "Removing run: " runkey " " (db:get-value-by-header run header "runname") " and related record") - (cdb:remote-run db:delete-run db run-id) - ;; This is a pretty good place to purge old DELETED tests - (cdb:remote-run db:delete-tests-for-run db run-id) - (cdb:remote-run db:delete-old-deleted-test-records db) - (cdb:remote-run db:set-var db "DELETED_TESTS" (current-seconds)) + (rmt:delete-run run-id) + (rmt:delete-old-deleted-test-records) + ;; (cdb:remote-run db:set-var db "DELETED_TESTS" (current-seconds)) ;; need to figure out the path to the run dir and remove it if empty ;; (if (null? (glob (conc runpath "/*"))) ;; (begin ;; (debug:print 1 "Removing run dir " runpath) ;; (system (conc "rmdir -p " runpath)))) @@ -1439,55 +1447,56 @@ (if (or lock (and unlock (begin (print "Do you really wish to unlock run " run-id "?\n y/n: ") (equal? "y" (read-line))))) - (cdb:remote-run db:lock/unlock-run db run-id lock unlock user) + (rmt:lock/unlock-run run-id lock unlock user) (debug:print-info 0 "Skipping lock/unlock on " run-id)))) runs))) ;;====================================================================== ;; Rollup runs ;;====================================================================== ;; Update the test_meta table for this test (define (runs:update-test_meta test-name test-conf) - (let ((currrecord (cdb:remote-run db:testmeta-get-record #f test-name))) + (let ((currrecord (rmt:testmeta-get-record test-name))) (if (not currrecord) (begin (set! currrecord (make-vector 10 #f)) - (cdb:remote-run db:testmeta-add-record #f test-name))) + (rmt:testmeta-add-record test-name))) (for-each (lambda (key) (let* ((idx (cadr key)) (fld (car key)) (val (config-lookup test-conf "test_meta" fld))) ;; (debug:print 5 "idx: " idx " fld: " fld " val: " val) (if (and val (not (equal? (vector-ref currrecord idx) val))) (begin (print "Updating " test-name " " fld " to " val) - (cdb:remote-run db:testmeta-update-field #f test-name fld val))))) + (rmt:testmeta-update-field test-name fld val))))) '(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9))))) ;; Update test_meta for all tests (define (runs:update-all-test_meta db) (let ((test-names (tests:get-all))) ;; (tests:get-valid-tests))) (for-each (lambda (test-name) (let* ((test-conf (mt:lazy-read-test-config test-name))) - ;; use the cdb:remote-run instead of passing in db (if test-conf (runs:update-test_meta test-name test-conf)))) (hash-table-keys test-names)))) ;; This could probably be refactored into one complex query ... +;; NOT PORTED - DO NOT USE YET +;; (define (runs:rollup-run keys runname user keyvals) (debug:print 4 "runs:rollup-run, keys: " keys " :runname " runname " user: " user) (let* ((db #f) - (new-run-id (cdb:remote-run db:register-run #f keyvals runname "new" "n/a" user)) - (prev-tests (cdb:remote-run test:get-matching-previous-test-run-records db new-run-id "%" "%")) + (new-run-id (rmt:register-run keyvals runname "new" "n/a" user)) + (prev-tests (rmt:get-matching-previous-test-run-records new-run-id "%" "%")) (curr-tests (mt:get-tests-for-run new-run-id "%/%" '() '())) (curr-tests-hash (make-hash-table))) - (cdb:remote-run db:update-run-event_time db new-run-id) + (rmt:update-run-event_time new-run-id) ;; index the already saved tests by testname and itemdat in curr-tests-hash (for-each (lambda (testdat) (let* ((testname (db:test-get-testname testdat)) (item-path (db:test-get-item-path testdat)) @@ -1501,11 +1510,11 @@ (lambda (testdat) (let* ((testname (db:test-get-testname testdat)) (item-path (db:test-get-item-path testdat)) (full-name (conc testname "/" item-path)) (prev-test-dat (hash-table-ref/default curr-tests-hash full-name #f)) - (test-steps (cdb:remote-run db:get-steps-for-test db (db:test-get-id testdat))) + (test-steps (rmt:get-steps-for-test (db:test-get-id testdat))) (new-test-record #f)) ;; replace these with insert ... select (apply sqlite3:execute db (conc "INSERT OR REPLACE INTO tests (run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment) " Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -67,28 +67,28 @@ ;; We don't want to flush the queue if it was just flushed (define *server:last-write-flush* (current-milliseconds)) ;; Flush the queue every third of a second. Can we assume that setup-for-run ;; has already been done? -(define (server:write-queue-handler) - (if (setup-for-run) - (let ((db (open-db))) - (let loop () - (let ((last-write-flush-time #f)) - (mutex-lock! *incoming-mutex*) - (set! last-write-flush-time *server:last-write-flush*) - (mutex-unlock! *incoming-mutex*) - (if (> (- (current-milliseconds) last-write-flush-time) 10) - (begin - (mutex-lock! *db:process-queue-mutex*) - (db:process-cached-writes db) - (mutex-unlock! *db:process-queue-mutex*) - (thread-sleep! 0.005)))) - (loop))) - (begin - (debug:print 0 "ERROR: failed to setup for Megatest in server:write-queue-handler") - (exit 1)))) +;; (define (server:write-queue-handler) +;; (if (setup-for-run) +;; (let ((db (open-db))) +;; (let loop () +;; (let ((last-write-flush-time #f)) +;; (mutex-lock! *incoming-mutex*) +;; (set! last-write-flush-time *server:last-write-flush*) +;; (mutex-unlock! *incoming-mutex*) +;; (if (> (- (current-milliseconds) last-write-flush-time) 10) +;; (begin +;; (mutex-lock! *db:process-queue-mutex*) +;; (db:process-cached-writes db) +;; (mutex-unlock! *db:process-queue-mutex*) +;; (thread-sleep! 0.005)))) +;; (loop))) +;; (begin +;; (debug:print 0 "ERROR: failed to setup for Megatest in server:write-queue-handler") +;; (exit 1)))) ;;====================================================================== ;; S E R V E R U T I L I T I E S ;;====================================================================== Index: tasks.scm ================================================================== --- tasks.scm +++ tasks.scm @@ -21,11 +21,11 @@ ;;====================================================================== ;; Tasks db ;;====================================================================== (define (tasks:open-db) - (let* ((dbpath (conc *toppath* "/monitor.db")) + (let* ((dbpath (conc *toppath* "/db/monitor.db")) (exists (file-exists? dbpath)) (write-access (file-write-access? dbpath)) (mdb (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) (handler (make-busy-timeout 36000))) (if (and exists @@ -64,10 +64,11 @@ priority INTEGER, state TEXT, mt_version TEXT, heartbeat TIMESTAMP, transport TEXT, + run_id INTEGER, CONSTRAINT servers_constraint UNIQUE (pid,hostname,port));") (sqlite3:execute mdb "CREATE TABLE IF NOT EXISTS clients (id INTEGER PRIMARY KEY, server_id INTEGER, pid INTEGER, hostname TEXT, @@ -425,11 +426,11 @@ ;; (define (tasks:start-monitor db mdb) (if (> (tasks:get-num-alive-monitors mdb) 2) ;; have two running, no need for more (debug:print-info 1 "Not starting monitor, already have more than two running") (let* ((megatestdb (conc *toppath* "/megatest.db")) - (monitordbf (conc *toppath* "/monitor.db")) + (monitordbf (conc *toppath* "/db/monitor.db")) (last-db-update 0)) ;; (file-modification-time megatestdb))) (task:register-monitor mdb) (let loop ((count 0) (next-touch 0)) ;; next-touch is the time where we need to update last_update ;; if the db has been modified we'd best look at the task queue ADDED tdb.scm Index: tdb.scm ================================================================== --- /dev/null +++ tdb.scm @@ -0,0 +1,539 @@ +;;====================================================================== +;; Copyright 2006-2013, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + +;;====================================================================== +;; Database access +;;====================================================================== + +(require-extension (srfi 18) extras tcp) ;; rpc) +;; (import (prefix rpc rpc:)) + +(use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64) +(import (prefix sqlite3 sqlite3:)) +(import (prefix base64 base64:)) + +;; Note, try to remove this dependency +;; (use zmq) + +(declare (unit tdb)) +(declare (uses common)) +(declare (uses keys)) +(declare (uses ods)) +(declare (uses fs-transport)) +(declare (uses client)) +(declare (uses mt)) + +(include "common_records.scm") +(include "db_records.scm") +(include "key_records.scm") +(include "run_records.scm") + +;;====================================================================== +;; +;; T E S T D A T A B A S E S +;; +;;====================================================================== + +;;====================================================================== +;; T E S T S P E C I F I C D B +;;====================================================================== + +;; Create the sqlite db for the individual test(s) +(define (open-test-db work-area) + (debug:print-info 11 "open-test-db " work-area) + (if (and work-area + (directory? work-area) + (file-read-access? work-area)) + (let* ((dbpath (conc work-area "/testdat.db")) + (tdb-writeable (file-write-access? dbpath)) + (dbexists (file-exists? dbpath)) + (handler (make-busy-timeout (if (args:get-arg "-override-timeout") + (string->number (args:get-arg "-override-timeout")) + 136000)))) + (handle-exceptions + exn + (begin + (debug:print 2 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test" + ((condition-property-accessor 'exn 'message) exn)) + (set! db (sqlite3:open-database ":memory:"))) ;; open an in-memory db to allow readonly access + (set! db (sqlite3:open-database dbpath))) + (if *db-write-access* (sqlite3:set-busy-handler! db handler)) + (if (not dbexists) + (begin + (sqlite3:execute db "PRAGMA synchronous = FULL;") + (debug:print-info 11 "Initialized test database " dbpath) + (tdb:testdb-initialize db))) + ;; (sqlite3:execute db "PRAGMA synchronous = 0;") + (debug:print-info 11 "open-test-db END (sucessful)" work-area) + ;; now let's test that everything is correct + (handle-exceptions + exn + (begin + (debug:print 0 "ERROR: problem accessing test db " work-area ", you probably should clean and re-run this test" + ((condition-property-accessor 'exn 'message) exn)) + #f) + ;; Is there a cheaper single line operation that will check for existance of a table + ;; and raise an exception ? + (sqlite3:execute db "SELECT id FROM test_data LIMIT 1;")) + db) + (let ((baddb (sqlite3:open-database ":memory:"))) + (debug:print-info 11 "open-test-db END (unsucessful)" work-area) + ;; provide an in-mem db (this is dangerous!) + (tdb:testdb-initialize baddb) + baddb))) + +;; find and open the testdat.db file for an existing test +(define (tdb:open-test-db-by-test-id test-id #!key (work-area #f)) + (let* ((test-path (if work-area + work-area + (rmt:test-get-rundir-from-test-id test-id)))) + (debug:print 3 "TEST PATH: " test-path) + (open-test-db test-path))) + +;; find and open the testdat.db file for an existing test +(define (tdb:open-test-db-by-test-id-local test-id #!key (work-area #f)) + (let* ((test-path (if work-area + work-area + (open-run-close db:test-get-rundir-from-test-id #f test-id)))) + (debug:print 3 "TEST PATH: " test-path) + (open-test-db test-path))) + +;; find and open the testdat.db file for an existing test +(define (tdb:open-run-close-db-by-test-id-local test-id work-area proc . params) + (let* ((test-path (if work-area + work-area + (open-run-close db:test-get-rundir-from-test-id #f test-id))) + (tdb (open-test-db test-path))) + (apply proc tdb params))) + +(define (tdb:testdb-initialize db) + (debug:print 11 "db:testdb-initialize START") + (for-each + (lambda (sqlcmd) + (sqlite3:execute db sqlcmd)) + (list "CREATE TABLE IF NOT EXISTS test_rundat ( + id INTEGER PRIMARY KEY, + update_time TIMESTAMP, + cpuload INTEGER DEFAULT -1, + diskfree INTEGER DEFAULT -1, + diskusage INTGER DEFAULT -1, + run_duration INTEGER DEFAULT 0);" + "CREATE TABLE IF NOT EXISTS test_data ( + id INTEGER PRIMARY KEY, + test_id INTEGER, + category TEXT DEFAULT '', + variable TEXT, + value REAL, + expected REAL, + tol REAL, + units TEXT, + comment TEXT DEFAULT '', + status TEXT DEFAULT 'n/a', + type TEXT DEFAULT '', + CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable));" + "CREATE TABLE IF NOT EXISTS test_steps ( + id INTEGER PRIMARY KEY, + test_id INTEGER, + stepname TEXT, + state TEXT DEFAULT 'NOT_STARTED', + status TEXT DEFAULT 'n/a', + event_time TIMESTAMP, + comment TEXT DEFAULT '', + logfile TEXT DEFAULT '', + CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state));" + ;; test_meta can be used for handing commands to the test + ;; e.g. KILLREQ + ;; the ackstate is set to 1 once the command has been completed + "CREATE TABLE IF NOT EXISTS test_meta ( + id INTEGER PRIMARY KEY, + var TEXT, + val TEXT, + ackstate INTEGER DEFAULT 0, + CONSTRAINT metadat_constraint UNIQUE (var));")) + (debug:print 11 "db:testdb-initialize END")) + +(define (tdb:get-steps-data tdb test-id) + (let ((res '())) + (sqlite3:for-each-row + (lambda (id test-id stepname state status event-time logfile) + (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res))) + tdb + "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC; + test-id) + (sqlite3:finalize! tdb) + (reverse res))) + +(define (tdb:read-test-data tdb test-id categorypatt) + (let ((res '())) + (sqlite3:for-each-row + (lambda (id test_id category variable value expected tol units comment status type) + (set! res (cons (vector id test_id category variable value expected tol units comment status type) res))) + tdb + "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt) + (sqlite3:finalize! tdb) + (reverse res))) + +;;====================================================================== +;; T E S T D A T A +;;====================================================================== + +(define (tdb:csv->test-data test-id csvdata #!key (work-area #f)) + (debug:print 4 "test-id " test-id ", csvdata: " csvdata) + (let ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area))) + (if (sqlite3:database? tdb) + (let ((csvlist (csv->list (make-csv-reader + (open-input-string csvdata) + '((strip-leading-whitespace? #t) + (strip-trailing-whitespace? #t)) )))) ;; (csv->list csvdata))) + (for-each + (lambda (csvrow) + (let* ((padded-row (take (append csvrow (list #f #f #f #f #f #f #f #f #f)) 9)) + (category (list-ref padded-row 0)) + (variable (list-ref padded-row 1)) + (value (any->number-if-possible (list-ref padded-row 2))) + (expected (any->number-if-possible (list-ref padded-row 3))) + (tol (any->number-if-possible (list-ref padded-row 4))) ;; >, <, >=, <=, or a number + (units (list-ref padded-row 5)) + (comment (list-ref padded-row 6)) + (status (let ((s (list-ref padded-row 7))) + (if (and (string? s)(or (string-match (regexp "^\\s*$") s) + (string-match (regexp "^n/a$") s))) + #f + s))) ;; if specified on the input then use, else calculate + (type (list-ref padded-row 8))) + ;; look up expected,tol,units from previous best fit test if they are all either #f or '' + (debug:print 4 "BEFORE: category: " category " variable: " variable " value: " value + ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment " type: " type) + + (if (and (or (not expected)(equal? expected "")) + (or (not tol) (equal? expected "")) + (or (not units) (equal? expected ""))) + (let-values (((new-expected new-tol new-units)(tdb:get-prev-tol-for-test tdb test-id category variable))) + (set! expected new-expected) + (set! tol new-tol) + (set! units new-units))) + + (debug:print 4 "AFTER: category: " category " variable: " variable " value: " value + ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) + ;; calculate status if NOT specified + (if (and (not status)(number? expected)(number? value)) ;; need expected and value to be numbers + (if (number? tol) ;; if tol is a number then we do the standard comparison + (let* ((max-val (+ expected tol)) + (min-val (- expected tol)) + (result (and (>= value min-val)(<= value max-val)))) + (debug:print 4 "max-val: " max-val " min-val: " min-val " result: " result) + (set! status (if result "pass" "fail"))) + (set! status ;; NB// need to assess each one (i.e. not return operator since need to act if not valid op. + (case (string->symbol tol) ;; tol should be >, <, >=, <= + ((>) (if (> value expected) "pass" "fail")) + ((<) (if (< value expected) "pass" "fail")) + ((>=) (if (>= value expected) "pass" "fail")) + ((<=) (if (<= value expected) "pass" "fail")) + (else (conc "ERROR: bad tol comparator " tol)))))) + (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value + ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) + (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);" + test-id category variable value expected tol units (if comment comment "") status type))) + csvlist) + (sqlite3:finalize! tdb))))) + +;; ;; get a list of test_data records matching categorypatt +;; (define (tdb:read-test-data test-id categorypatt #!key (work-area #f)) +;; (let ((tdb (tdb:open-test-db-by-test-id test-id work-area: work-area))) +;; (if (sqlite3:database? tdb) +;; (let ((res '())) +;; (sqlite3:for-each-row +;; (lambda (id test_id category variable value expected tol units comment status type) +;; (set! res (cons (vector id test_id category variable value expected tol units comment status type) res))) +;; tdb +;; "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt) +;; (sqlite3:finalize! tdb) +;; (reverse res)) +;; '()))) + +;; NOTE: Run this local with #f for db !!! +(define (tdb:load-test-data test-id #!key (work-area #f)) + (let loop ((lin (read-line))) + (if (not (eof-object? lin)) + (begin + (debug:print 4 lin) + (tdb:csv->test-data test-id lin work-area: work-area) + (loop (read-line))))) + ;; roll up the current results. + ;; FIXME: Add the status to + (tdb:test-data-rollup test-id #f work-area: work-area)) + +;; WARNING: Do NOT call this for the parent test on an iterated test +;; Roll up test_data pass/fail results +;; look at the test_data status field, +;; if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS. +;; if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored +(define (tdb:test-data-rollup test-id status #!key (work-area #f)) + (let ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area)) + (fail-count 0) + (pass-count 0)) + (if (sqlite3:database? tdb) + (begin + (sqlite3:for-each-row + (lambda (fcount pcount) + (set! fail-count fcount) + (set! pass-count pcount)) + tdb + "SELECT (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'fail') AS fail_count, + (SELECT count(id) FROM test_data WHERE test_id=? AND status like 'pass') AS pass_count;" + test-id test-id) + + ;; Now rollup the counts to the central megatest.db + (rmt:general-call 'pass-fail-counts fail-count pass-count test-id) + ;; (sqlite3:execute db "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;" + ;; fail-count pass-count test-id) + + ;; The flush is not needed with the transaction based write agregation enabled. Remove these commented lines + ;; next time you read this! + ;; + ;; (cdb:flush-queue *runremote*) + ;; (thread-sleep! 1) ;; play nice with the queue by ensuring the rollup is at least 10ms later than the set + + ;; if the test is not FAIL then set status based on the fail and pass counts. + (rmt:general-call 'test_data-pf-rollup test-id test-id test-id test-id) + ;; (sqlite3:execute + ;; db ;;; NOTE: Should this be WARN,FAIL? A WARN is not a FAIL????? BUG FIXME + ;; "UPDATE tests + ;; SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0 + ;; THEN 'FAIL' + ;; WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND + ;; (SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL') + ;; THEN 'PASS' + ;; ELSE status + ;; END WHERE id=?;" + ;; test-id test-id test-id test-id) + (sqlite3:finalize! tdb) + )))) + +(define (tdb:get-prev-tol-for-test tdb test-id category variable) + ;; Finish me? + (values #f #f #f)) + +;;====================================================================== +;; S T E P S +;;====================================================================== + +(define (tdb:step-get-time-as-string vec) + (seconds->time-string (tdb:step-get-event_time vec))) + +;; db-get-test-steps-for-run +(define (tdb:get-steps-for-test test-id #!key (work-area #f)) + (let* ((tdb (tdb:open-test-db-by-test-id-local test-id work-area: work-area)) + (res '())) + (if (sqlite3:database? tdb) + (handle-exceptions + exn + (begin + (debug:print 0 "ERROR: error on access to testdat for test with id " test-id) + '()) + (begin + (sqlite3:for-each-row + (lambda (id test-id stepname state status event-time logfile) + (set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "")) res))) + tdb + "SELECT id,test_id,stepname,state,status,event_time,logfile FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC; + test-id) + (sqlite3:finalize! tdb) + (reverse res))) + '()))) + +;; get a pretty table to summarize steps +;; +(define (tdb:get-steps-table test-id #!key (work-area #f)) + (let ((steps (tdb:get-steps-for-test test-id work-area: work-area))) + ;; organise the steps for better readability + (let ((res (make-hash-table))) + (for-each + (lambda (step) + (debug:print 6 "step=" step) + (let ((record (hash-table-ref/default + res + (tdb:step-get-stepname step) + ;; stepname start end status Duration Logfile + (vector (tdb:step-get-stepname step) "" "" "" "" "")))) + (debug:print 6 "record(before) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)) + (case (string->symbol (tdb:step-get-state step)) + ((start)(vector-set! record 1 (tdb:step-get-event_time step)) + (vector-set! record 3 (if (equal? (vector-ref record 3) "") + (tdb:step-get-status step))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + ((end) + (vector-set! record 2 (any->number (tdb:step-get-event_time step))) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (let ((startt (any->number (vector-ref record 1))) + (endt (any->number (vector-ref record 2)))) + (debug:print 4 "record[1]=" (vector-ref record 1) + ", startt=" startt ", endt=" endt + ", get-status: " (tdb:step-get-status step)) + (if (and (number? startt)(number? endt)) + (seconds->hr-min-sec (- endt startt)) "-1"))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + (else + (vector-set! record 2 (tdb:step-get-state step)) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (tdb:step-get-event_time step)))) + (hash-table-set! res (tdb:step-get-stepname step) record) + (debug:print 6 "record(after) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)))) + ;; (else (vector-set! record 1 (tdb:step-get-event_time step))) + (sort steps (lambda (a b) + (cond + ((< (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t) + ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) + (< (tdb:step-get-id a) (tdb:step-get-id b))) + (else #f))))) + res))) + +;; get a pretty table to summarize steps +;; +(define (tdb:get-steps-table-list test-id #!key (work-area #f)) + (let ((steps (tdb:get-steps-for-test test-id work-area: work-area))) + ;; organise the steps for better readability + (let ((res (make-hash-table))) + (for-each + (lambda (step) + (debug:print 6 "step=" step) + (let ((record (hash-table-ref/default + res + (tdb:step-get-stepname step) + ;; stepname start end status + (vector (tdb:step-get-stepname step) "" "" "" "" "")))) + (debug:print 6 "record(before) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)) + (case (string->symbol (tdb:step-get-state step)) + ((start)(vector-set! record 1 (tdb:step-get-event_time step)) + (vector-set! record 3 (if (equal? (vector-ref record 3) "") + (tdb:step-get-status step))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + ((end) + (vector-set! record 2 (any->number (tdb:step-get-event_time step))) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (let ((startt (any->number (vector-ref record 1))) + (endt (any->number (vector-ref record 2)))) + (debug:print 4 "record[1]=" (vector-ref record 1) + ", startt=" startt ", endt=" endt + ", get-status: " (tdb:step-get-status step)) + (if (and (number? startt)(number? endt)) + (seconds->hr-min-sec (- endt startt)) "-1"))) + (if (> (string-length (tdb:step-get-logfile step)) + 0) + (vector-set! record 5 (tdb:step-get-logfile step)))) + (else + (vector-set! record 2 (tdb:step-get-state step)) + (vector-set! record 3 (tdb:step-get-status step)) + (vector-set! record 4 (tdb:step-get-event_time step)))) + (hash-table-set! res (tdb:step-get-stepname step) record) + (debug:print 6 "record(after) = " record + "\nid: " (tdb:step-get-id step) + "\nstepname: " (tdb:step-get-stepname step) + "\nstate: " (tdb:step-get-state step) + "\nstatus: " (tdb:step-get-status step) + "\ntime: " (tdb:step-get-event_time step)))) + ;; (else (vector-set! record 1 (tdb:step-get-event_time step))) + (sort steps (lambda (a b) + (cond + ((< (tdb:step-get-event_time a)(tdb:step-get-event_time b)) #t) + ((eq? (tdb:step-get-event_time a)(tdb:step-get-event_time b)) + (< (tdb:step-get-id a) (tdb:step-get-id b))) + (else #f))))) + res))) + +(define (tdb:get-compressed-steps test-id #!key (work-area #f)(tdb #f)) + (if (or (not work-area) + (file-exists? (conc work-area "/testdat.db"))) + (let* ((comprsteps (tdb:get-steps-table test-id work-area: work-area))) + (map (lambda (x) + ;; take advantage of the \n on time->string + (vector + (vector-ref x 0) + (let ((s (vector-ref x 1))) + (if (number? s)(seconds->time-string s) s)) + (let ((s (vector-ref x 2))) + (if (number? s)(seconds->time-string s) s)) + (vector-ref x 3) ;; status + (vector-ref x 4) + (vector-ref x 5))) ;; time delta + (sort (hash-table-values comprsteps) + (lambda (a b) + (let ((time-a (vector-ref a 1)) + (time-b (vector-ref b 1))) + (if (and (number? time-a)(number? time-b)) + (if (< time-a time-b) + #t + (if (eq? time-a time-b) + (string (db:test-get-event_time testdat)(db:test-get-event_time stored-test)))) - ;; this test is younger, store it in the hash - (hash-table-set! tests-hash full-testname testdat)))) - results) - (if (null? tal) - (map cdr (hash-table->alist tests-hash)) ;; return a list of the most recent tests - (loop (car tal)(cdr tal)))))))))) - ;; Check for waiver eligibility ;; (define (tests:check-waiver-eligibility testdat prev-testdat) (let* ((test-registry (make-hash-table)) (testconfig (tests:get-testconfig (db:test-get-testname testdat) test-registry #f)) @@ -279,20 +191,18 @@ #f)))))) (pop-directory) result))))) (define (tests:test-force-state-status! test-id state status) - (cdb:test-set-status-state *runremote* test-id status state #f) + (rmt:test-set-status-state test-id status state #f) (mt:process-triggers test-id state status)) ;; Do not rpc this one, do the underlying calls!!! (define (tests:test-set-status! test-id state status comment dat #!key (work-area #f)) - (debug:print-info 4 "tests:test-set-status! test-id=" test-id ", state=" state ", status=" status ", dat=" dat) - (let* ((db #f) - (real-status status) + (let* ((real-status status) (otherdat (if dat dat (make-hash-table))) - (testdat (cdb:get-test-info-by-id *runremote* test-id)) + (testdat (rmt:get-test-info-by-id test-id)) (run-id (db:test-get-run_id testdat)) (test-name (db:test-get-testname testdat)) (item-path (db:test-get-item-path testdat)) ;; before proceeding we must find out if the previous test (where all keys matched except runname) ;; was WAIVED if this test is FAIL @@ -300,11 +210,11 @@ ;; NOTES: ;; 1. Is the call to test:get-previous-run-record remotified? ;; 2. Add test for testconfig waiver propagation control here ;; (prev-test (if (equal? status "FAIL") - (cdb:remote-run test:get-previous-test-run-record #f run-id test-name item-path) + (rmt:get-previous-test-run-record run-id test-name item-path) #f)) (waived (if prev-test (if prev-test ;; true if we found a previous test in this run series (let ((prev-status (db:test-get-status prev-test)) (prev-state (db:test-get-state prev-test)) @@ -325,17 +235,17 @@ (debug:print 4 "real-status " real-status ", waived " waived ", status " status) ;; update the primary record IF state AND status are defined (if (and state status) (begin - (cdb:test-set-status-state *runremote* test-id real-status state (if waived waived comment)) + (rmt:test-set-status-state test-id real-status state (if waived waived comment)) (mt:process-triggers test-id state real-status))) ;; if status is "AUTO" then call rollup (note, this one modifies data in test ;; run area, it does remote calls under the hood. (if (and test-id state status (equal? status "AUTO")) - (db:test-data-rollup #f test-id status work-area: work-area)) + (tdb:test-data-rollup test-id status work-area: work-area)) ;; add metadata (need to do this way to avoid SQL injection issues) ;; :first_err ;; (let ((val (hash-table-ref/default otherdat ":first_err" #f))) @@ -371,30 +281,28 @@ (db:csv->test-data #f test-id dat)))) ;; need to update the top test record if PASS or FAIL and this is a subtest (if (not (equal? item-path "")) - (mt:roll-up-pass-fail-counts run-id test-name item-path status)) + (rmt:roll-up-pass-fail-counts run-id test-name item-path status)) (if (or (and (string? comment) (string-match (regexp "\\S+") comment)) waived) (let ((cmt (if waived waived comment))) - (cdb:remote-run db:test-set-comment #f test-id cmt))) - )) - - -(define (tests:test-set-toplog! db run-id test-name logf) - (cdb:client-call *runremote* 'tests:test-set-toplog #t 2 logf run-id test-name)) - -(define (tests:summarize-items db run-id test-id test-name force) + (rmt:general-call 'set-test-comment cmt test-id))))) + +(define (tests:test-set-toplog! run-id test-name logf) + (rmt:general-call 'tests:test-set-toplog logf run-id test-name)) + +(define (tests:summarize-items run-id test-id test-name force) ;; if not force then only update the record if one of these is true: ;; 1. logf is "log/final.log ;; 2. logf is same as outputfilename (let* ((outputfilename (conc "megatest-rollup-" test-name ".html")) (orig-dir (current-directory)) - (logf-info (cdb:remote-run db:test-get-logfile-info #f run-id test-name)) + (logf-info (rmt:test-get-logfile-info run-id test-name)) (logf (if logf-info (cadr logf-info) #f)) (path (if logf-info (car logf-info) #f))) ;; This query finds the path and changes the directory to it for the test (if (and (string? path) (directory? path)) ;; can get #f here under some wierd conditions. why, unknown ... @@ -406,21 +314,20 @@ (debug:print 4 "summarize-items with logf " logf ", outputfilename " outputfilename " and force " force) (if (or (equal? logf "logs/final.log") (equal? logf outputfilename) force) (begin - (if ;; (not (obtain-dot-lock outputfilename 1 5 7)) ;; retry every second for 20 seconds, call it dead after 30 seconds and steal the lock - (not (lock-queue:wait-turn outputfilename test-id)) + (if (not (lock-queue:wait-turn outputfilename test-id)) (print "Not updating " outputfilename " as another test item has signed up for the job") (begin (print "Obtained lock for " outputfilename) (let ((oup (open-output-file outputfilename)) (counts (make-hash-table)) (statecounts (make-hash-table)) (outtxt "") (tot 0) - (testdat (cdb:remote-run db:test-get-records-for-index-file #f run-id test-name))) + (testdat (rmt:test-get-records-for-index-file run-id test-name))) (with-output-to-port oup (lambda () (set! outtxt (conc outtxt "Summary: " test-name "

Summary for " test-name "

")) @@ -467,16 +374,17 @@ (print "") (print "" "" outtxt "
ItemStateStatusComment
") - (release-dot-lock outputfilename))) + ;; (release-dot-lock outputfilename) + )) (close-output-port oup) (lock-queue:release-lock outputfilename test-id) (change-directory orig-dir) ;; NB// tests:test-set-toplog! is remote internal... - (tests:test-set-toplog! db run-id test-name outputfilename) + (tests:test-set-toplog! run-id test-name outputfilename) ))))))) ;;====================================================================== ;; Gather data from test/task specifications ;;====================================================================== @@ -555,12 +463,12 @@ (test-name (tests:testqueue-get-testname test-record)) (itemdat (tests:testqueue-get-itemdat test-record)) (item-path (tests:testqueue-get-item_path test-record)) (waitons (tests:testqueue-get-waitons test-record)) (keep-test #t) - (test-id (cdb:remote-run db:get-test-id-cached #f run-id test-name item-path)) - (tdat (cdb:remote-run db:get-testinfo-state-status #f test-id))) ;; (cdb:get-test-info-by-id *runremote* test-id))) + (test-id (rmt:get-test-id run-id test-name item-path)) + (tdat (rmt:get-testinfo-state-status test-id))) ;; (cdb:get-test-info-by-id *runremote* test-id))) (if tdat (begin ;; Look at the test state and status (if (or (and (member (db:test-get-status tdat) '("PASS" "WARN" "WAIVED" "CHECK" "SKIP")) @@ -572,12 +480,12 @@ ;; examine waitons for any fails. If it is FAIL or INCOMPLETE then eliminate this test ;; from the runnable list (if keep-test (for-each (lambda (waiton) ;; for now we are waiting only on the parent test - (let* ((parent-test-id (cdb:remote-run db:get-test-id-cached #f run-id waiton "")) - (wtdat (cdb:remote-run db:get-testinfo-state-status #f test-id))) ;; (cdb:get-test-info-by-id *runremote* test-id))) + (let* ((parent-test-id (rmt:get-test-id run-id waiton "")) + (wtdat (rmt:get-testinfo-state-status test-id))) ;; (cdb:get-test-info-by-id *runremote* test-id))) (if (or (and (equal? (db:test-get-state wtdat) "COMPLETED") (member (db:test-get-status wtdat) '("FAIL"))) (member (db:test-get-status wtdat) '("KILLED")) (member (db:test-get-state wtdat) '("INCOMPETE"))) ;; (if (or (member (db:test-get-status wtdat) @@ -680,11 +588,11 @@ ;;====================================================================== ;; teststep-set-status! used to be here (define (test-get-kill-request test-id) ;; run-id test-name itemdat) - (let* ((testdat (cdb:remote-run db:get-testinfo-state-status #f test-id))) ;; (cdb:get-test-info-by-id *runremote* test-id))) ;; run-id test-name item-path))) + (let* ((testdat (rmt:get-test-info-by-id test-id))) (and testdat (equal? (test:get-state testdat) "KILLREQ")))) (define (test:tdb-get-rundat-count tdb) (if tdb @@ -696,60 +604,30 @@ "SELECT count(id) FROM test_rundat;") res)) 0) (define (tests:update-central-meta-info test-id cpuload diskfree minutes uname hostname) - ;; This is a good candidate for threading the requests to enable - ;; transactionized write at the server - (cdb:tests-update-cpuload-diskfree *runremote* test-id cpuload diskfree) + (rmt:general-call 'update-cpuload-diskfree cpuload diskfree test-id) (if minutes - (cdb:tests-update-run-duration *runremote* test-id minutes)) + (rmt:general-call 'update-run-duration minutes test-id)) (if (and uname hostname) - (cdb:tests-update-uname-host *runremote* test-id uname hostname))) + (rmt:general-call 'update-uname-host uname hostname test-id))) -(define (tests:set-full-meta-info db test-id run-id minutes work-area) - ;; DOES cdb:remote-run under the hood! - (let* ((num-records 0) ;; (test:tdb-get-rundat-count tdb)) +(define (tests:set-full-meta-info test-id run-id minutes work-area) + (let* ((num-records 0) (cpuload (get-cpu-load)) (diskfree (get-df (current-directory))) (uname (get-uname "-srvpio")) (hostname (get-host-name))) - (tests:update-testdat-meta-info db test-id work-area cpuload diskfree minutes) + (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes) (tests:update-central-meta-info test-id cpuload diskfree minutes uname hostname))) -(define (tests:set-partial-meta-info db test-id run-id minutes work-area) - ;; DOES cdb:remote-run under the hood! +(define (tests:set-partial-meta-info test-id run-id minutes work-area) (let* ((cpuload (get-cpu-load)) (diskfree (get-df (current-directory)))) - (tests:update-testdat-meta-info db test-id work-area cpuload diskfree minutes) - ;; Update central with uname and hostname = #f - ;; Is this one of the performance problems? This info should come from testdat-meta anyway - ;; (tests:update-central-meta-info test-id cpuload diskfree minutes #f #f) - )) - -(define (tests:update-testdat-meta-info db test-id work-area cpuload diskfree minutes) - (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area))) - (if (sqlite3:database? tdb) - (begin - (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);" - cpuload diskfree minutes) - (sqlite3:finalize! tdb)) - (debug:print 2 "Can't update testdat.db for test " test-id " read-only or non-existant")))) - -(define (tests:testdat-get-testinfo db test-id work-area) - (let ((tdb (db:open-test-db-by-test-id db test-id work-area: work-area)) - (res '())) - (if (sqlite3:database? tdb) - (begin - (sqlite3:for-each-row - (lambda (update-time cpuload diskfree run-duration) - (set! res (cons (vector update-time cpuload diskfree run-duration) res))) - tdb - "SELECT update_time,cpuload,diskfree,run_duration FROM test_rundat ORDER BY update_time ASC;") - (sqlite3:finalize! tdb))) - res)) - + (tdb:update-testdat-meta-info test-id work-area cpuload diskfree minutes))) + ;;====================================================================== ;; A R C H I V I N G ;;====================================================================== (define (test:archive db test-id) Index: tests/Makefile ================================================================== --- tests/Makefile +++ tests/Makefile @@ -1,10 +1,11 @@ # # run some tests BINPATH=$(shell readlink -m $(PWD)/../bin) MEGATEST=$(BINPATH)/megatest +DASHBOARD=$(BINPATH)/dashboard PATH := $(BINPATH):$(PATH) RUNNAME := $(shell date +w%V.%u.%H.%M) IPADDR := "-" # Set SERVER to "-server -" SERVER = @@ -21,11 +22,11 @@ all : test1 test2 test3 test4 test5 test6 test7 test8 test9 server : cd ..;make;make install - cd fullrun;../../bin/megatest -server - -debug 22 & + cd fullrun;../../bin/megatest -server - -debug 22 stopserver : cd ..;make && make install cd fullrun;$(MEGATEST) -stop-server 0 @@ -35,15 +36,10 @@ test0 : cleanprep cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG) test1 : cleanprep - rm -f simplerun/megatest.db - rm -rf simplelinks/ simpleruns/ - mkdir -p simplelinks simpleruns - cd simplerun;cp ../../*_records.scm .;perl -pi.bak -e 's/define-inline/define/' *_records.scm - cd simplerun;echo '(load "../tests.scm")' | $(MEGATEST) -repl -debug $(DEBUG) test2 : fullprep cd fullrun;$(MEGATEST) -runtests ez_pass,runfirst/a/% -reqtarg ubuntu/nfs/none :runname $(RUNNAME) -debug $(DEBUG) $(LOGGING) cd fullrun;megatest -runtests % -target ubuntu/nfs/none :runname $(RUNNAME)_01 -testpatt %/,%/ai -debug $(DEBUG) cd fullrun;megatest -runtests %/,%/ai -target ubuntu/nfs/none :runname $(RUNAME)_02 -debug $(DEBUG) @@ -111,27 +107,27 @@ test9 : minsetup test9a test9b test9c test9d test9e test9a : @echo Run super-simple mintest e, no waitons. - cd mintest;megatest -runtests e -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) + cd mintest;$(MEGATEST) -runtests e -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) test9b : @echo Run simple mintest d with one waiton c - cd mintest;megatest -runtests d -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) + cd mintest;$(MEGATEST) -runtests d -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) test9c : @echo Run mintest a with full waiton chain a -> b -> c -> d -> e - cd mintest;megatest -runtests a -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) + cd mintest;$(MEGATEST) -runtests a -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) test9d : @echo Run an itemized test with no items - cd mintest;megatest -runtests g -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) + cd mintest;$(MEGATEST) -runtests g -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) test9e : @echo Run mintest a1 with full waiton chain with d1fail: a1 -> b1 -> c1 -> d1fail -> e1 - cd mintest;megatest -runtests a1 -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) + cd mintest;$(MEGATEST) -runtests a1 -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG) test10 : @echo Run a bunch of different targets simultaneously (cd fullrun;$(MEGATEST) -server - ;sleep 2)& for targ in mint/btrfs/mintdir sunos/sshfs/loc; do \ @@ -146,14 +142,14 @@ cd fullrun;time (for a in 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 ;do (megatest -test-paths -target %/%/% > /dev/null ) & done; wait; ) minsetup : cd ..;make && make install mkdir -p mintest/runs mintest/links - cd mintest;megatest -stop-server 0 - cd mintest;megatest -server - -debug $(DEBUG) > server.log 2> server.log & + cd mintest;$(MEGATEST) -stop-server 0 + cd mintest;$(MEGATEST) -server - -debug $(DEBUG) > server.log 2> server.log & sleep 3 - cd mintest;dashboard -rows 18 & + cd mintest;$(DASHBOARD) -rows 18 & cleanprep : ../*.scm Makefile */*.config mkdir -p fullrun/tmp/mt_runs fullrun/tmp/mt_links cd ..;make;make install rm -f */logging.db @@ -162,29 +158,26 @@ fullprep : cleanprep cd fullrun;$(MEGATEST) -remove-runs :runname $(RUNNAME)% -target %/%/% -testpatt %/% cd fullrun;$(BINPATH)/dashboard -rows 15 & dashboard : cleanprep - cd fullrun && $(BINPATH)/dashboard -transport fs -rows 20 & - -dashboard-http : cleanprep - cd fullrun && $(BINPATH)/dashboard -transport http -rows 20 & + cd fullrun && $(BINPATH)/dashboard -rows 20 & remove : cd fullrun;$(MEGATEST) -remove-runs :runname $(RUN) -testpatt % -itempatt % :sysname % :fsname % :datapath % clean : rm cleanprep kill : killall -v mtest main.sh dboard || true - rm -f */megatest.db */logging.db */monitor.db || true + rm -rf */megatest.db */logging.db */monitor.db fullrun/tmp/mt_*/* || true killall -v mtest dboard || true hardkill : kill - sleep 5;killall -v mtest main.sh dboard -9 + sleep 2;killall -v mtest main.sh dboard -9 listservers : cd fullrun;$(MEGATEST) -list-servers runforever : while(ls); do runname=`date +%F-%R:%S`;(cd fullrun;$(MEGATEST) -runall -target ubuntu/nfs/none :runname $$runname;/home/matt/data/megatest/megatest -runall -target ubuntu/nfs/none :runname $$runname;/home/matt/data/megatest/megatest -runall -target ubuntu/nfs/none :runname $$runname);done Index: tests/fdktestqa/testqa/megatest.config ================================================================== --- tests/fdktestqa/testqa/megatest.config +++ tests/fdktestqa/testqa/megatest.config @@ -1,9 +1,9 @@ [setup] testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log runqueue 20 -# transport http +transport http launchwait no [include ../fdk.config] [server] Index: tests/fullrun/megatest.config ================================================================== --- tests/fullrun/megatest.config +++ tests/fullrun/megatest.config @@ -19,10 +19,11 @@ [setup] # Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding # this may save a few milliseconds on launching tests # launchwait no +waivercommentpatt ^WW\d+ [a-z].* # Use http instead of direct filesystem access transport http # transport fs @@ -43,11 +44,11 @@ # It is possible (but not recommended) to override the rsync command used # to populate the test directories. For test development the following # example can be useful # -# testcopycmd cp --remove-destination -rsv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log +testcopycmd cp --remove-destination -rsv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log # or for hard links # testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/. Index: tests/fullrun/tests/all_toplevel/testconfig ================================================================== --- tests/fullrun/tests/all_toplevel/testconfig +++ tests/fullrun/tests/all_toplevel/testconfig @@ -1,8 +1,13 @@ [ezsteps] calcresults megatest -list-runs $MT_RUNNAME -target $MT_TARGET [requirements] -waiton all_toplevel exit_0 exit_1 ez_exit2_fail ez_fail ez_pass ezlog_fail ezlog_fail_then_pass ezlog_pass ezlog_warn lineitem_fail lineitem_pass logpro_required_fail manual_example neverrun priority_1 priority_10 priority_10_waiton_1 priority_2 priority_3 priority_4 priority_5 priority_6 priority_7 priority_8 priority_9 runfirst singletest singletest2 sqlitespeed test_mt_vars +waiton all_toplevel exit_0 exit_1 ez_exit2_fail ez_fail ez_pass ezlog_fail \ + ezlog_fail_then_pass ezlog_pass ezlog_warn lineitem_fail lineitem_pass logpro_required_fail \ + manual_example neverrun priority_1 priority_10 priority_10_waiton_1 priority_2 \ + priority_3 priority_4 priority_5 priority_6 priority_7 priority_8 \ + priority_9 runfirst singletest singletest2 sqlitespeed test_mt_vars \ + ez_fail_quick test1 test2 special blocktestxz # This is a "toplevel" test, it does not require waitons to be non-FAIL to run mode toplevel Index: tests/fullrun/tests/priority_8/main.sh ================================================================== --- tests/fullrun/tests/priority_8/main.sh +++ tests/fullrun/tests/priority_8/main.sh @@ -1,10 +1,14 @@ #!/bin/bash # a bunch of steps in 2 second increments for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17;do + echo "start step before $i: `date`" $MT_MEGATEST -step step$i :state start :status running -setlog results$i.html + echo "start step after $i: `date`" sleep 2 + echo "end step before $i: `date`" $MT_MEGATEST -step step$i :state end :status 0 + echo "end step after $i: `date`" done exit 0 Index: tests/fullrun/tests/runfirst/main.sh ================================================================== --- tests/fullrun/tests/runfirst/main.sh +++ tests/fullrun/tests/runfirst/main.sh @@ -30,7 +30,9 @@ if [[ `basename $PWD` == "mustfail" ]];then $MT_MEGATEST -test-status :state COMPLETED :status FAIL else $MT_MEGATEST -test-status :state COMPLETED :status $loadstatus -m "This is a test level comment" :value 10e6 :expected_value 1.1e6 :tol 100e3 :category nada :variable sillyvar :units mFarks :comment "This is the value/expected comment" fi + +env > envfile.txt # $MT_MEGATEST -test-status :state COMPLETED :status FAIL Index: tests/fullrun/tests/sqlitespeed/runscript.rb ================================================================== --- tests/fullrun/tests/sqlitespeed/runscript.rb +++ tests/fullrun/tests/sqlitespeed/runscript.rb @@ -1,8 +1,8 @@ #! /usr/bin/env ruby -require "#{ENV['MT_RUN_AREA_HOME']}/../supportfiles/ruby/librunscript.rb" +require "#{ENV['MT_RUN_AREA_HOME']}/../resources/ruby/librunscript.rb" # run_record(stepname, cmd) - will record in db if exit code of script was zero or not run_and_record('create db',"sqlite3 testing.db << EOF\ncreate table if not exists blah(id INTEGER PRIMARY KEY,name TEXT);\n.q\nEOF","") if (! File.exists?("../../runfirst/I_was_here")) Index: tests/mintest/megatest.config ================================================================== --- tests/mintest/megatest.config +++ tests/mintest/megatest.config @@ -2,10 +2,11 @@ X TEXT [setup] max_concurrent_jobs 50 linktree #{getenv PWD}/linktree +transport http [server] port 8090 [jobtools] ADDED tests/resources/ruby/librunscript.rb Index: tests/resources/ruby/librunscript.rb ================================================================== --- /dev/null +++ tests/resources/ruby/librunscript.rb @@ -0,0 +1,37 @@ +# This is the library of stuff for megatest + +def run_and_record(stepname, cmd, checks) + system "megatest -step #{stepname} :state start :status n/a" + system cmd + exitcode=$? + if exitcode==0 + exitcode='pass' + else + exitcode='fail' + end + system "megatest -step #{stepname} :state end :status #{exitcode}" +end + +def record_step(stepname,state,status) + system "megatest -step #{stepname} :state #{state} :status #{status}" +end + +def test_status(state,status) + system "megatest -test-status :state #{state} :status #{status}" +end + + +# WARNING: This example is deprecated. Don't use the -test-status command +# unless you know for sure what you are doing. +def file_size_checker(stepname,filename,minsize,maxsize) + fsize=File.size(filename) + if fsize > maxsize or fsize < minsize + system "megatest -test-status :state COMPLETED :status fail" + else + system "megatest -test-status :state COMPLETED :status pass" + end +end + + +def wait_for_step(testname,stepname) +end ADDED tests/rununittest.sh Index: tests/rununittest.sh ================================================================== --- /dev/null +++ tests/rununittest.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Usage: rununittest.sh testname debuglevel +# + +# Clean setup +# +rm -f simplerun/megatest.db simplerun/monitor.db simplerun/db/monitor.db +rm -rf simplelinks/ simpleruns/ +mkdir -p simplelinks simpleruns +(cd simplerun;cp ../../*_records.scm .;perl -pi.bak -e 's/define-inline/define/' *_records.scm) + +# Run the test $1 is the unit test to run +cd simplerun;echo '(load "../tests.scm")' | ../../bin/megatest -repl -debug $2 $1 Index: tests/simplerun/megatest.config ================================================================== --- tests/simplerun/megatest.config +++ tests/simplerun/megatest.config @@ -3,10 +3,15 @@ RELEASE TEXT [setup] # Adjust max_concurrent_jobs to limit how much you load your machines max_concurrent_jobs 50 + +# Uncomment this to make the in-mem db into a disk based db (slower but good for debug) +# be aware that some unit tests will fail with this due to persistent data +# +# tmpdb /tmp # This is your link path, you can move it but it is generally better to keep it stable linktree #{shell readlink -f #{getenv PWD}/../simplelinks} # Valid values for state and status for steps, NB// It is not recommended you use this Index: tests/tests.scm ================================================================== --- tests/tests.scm +++ tests/tests.scm @@ -11,12 +11,12 @@ (require-extension test) (require-extension regex) (require-extension srfi-18) (import srfi-18) -(require-extension zmq) -(import zmq) +;; (require-extension zmq) +;; (import zmq) (define test-work-dir (current-directory)) ;; read in all the _record files (let ((files (glob "*_records.scm"))) @@ -26,433 +26,11 @@ (load file)) files)) (define *runremote* #f) -;;====================================================================== -;; P R O C E S S E S -;;====================================================================== - -(test "cmd-run-with-stderr->list" '("No such file or directory") - (let ((reslst (cmd-run-with-stderr->list "ls" "/tmp/ihadbetternotexist"))) - (string-search (regexp "No such file or directory")(car reslst)))) - -;;====================================================================== -;; T E S T M A T C H I N G -;;====================================================================== - -;; tests:glob-like-match -(test #f '("abc") (tests:glob-like-match "abc" "abc")) -(for-each - (lambda (patt str expected) - (test (conc patt " " str "=>" expected) expected (tests:glob-like-match patt str))) - (list "abc" "~abc" "~abc" "a*c" "a%c") - (list "abc" "abcd" "abc" "ABC" "ABC") - (list '("abc") #t #f #f '("ABC")) - ) - -;; tests:match -(test #f #t (tests:match "abc/def" "abc" "def")) -(for-each - (lambda (patterns testname itempath expected) - (test (conc patterns " " testname "/" itempath "=>" expected) - expected - (tests:match patterns testname itempath))) - (list "abc" "abc/%" "ab%/c%" "~abc/c%" "abc/~c%" "a,b/c,%/d" "%/,%/a" "%/,%/a" "%/,%/a" "%" "%" "%/" "%/") - (list "abc" "abc" "abcd" "abc" "abc" "a" "abc" "def" "ghi" "a" "a" "a" "a") - (list "" "" "cde" "cde" "cde" "" "" "a" "b" "" "b" "" "b") - (list #t #t #t #f #f #t #t #t #f #t #t #t #f)) - -;; db:patt->like -(test #f "testname LIKE 't%'" (db:patt->like "testname" "t%" comparator: " AND ")) -(test #f "testname LIKE 't%' AND testname LIKE '%t'" (db:patt->like "testname" "t%,%t" comparator: " AND ")) -(test #f "item_path GLOB ''" (db:patt->like "item_path" "")) - -;; test:match->sqlqry -(test #f "(testname GLOB 'a' AND item_path GLOB 'b') OR (testname LIKE 'a%' AND item_path LIKE '%') OR (testname GLOB '' AND item_path LIKE 'b%')" - (tests:match->sqlqry "a/b,a%,/b%")) -(test #f "(testname GLOB 'a' AND item_path GLOB 'b') OR (testname LIKE 'a%' AND item_path LIKE '%') OR (testname LIKE '%' AND item_path LIKE 'b%')" - (tests:match->sqlqry "a/b,a%,%/b%")) - -;;====================================================================== -;; S E R V E R -;;====================================================================== - -(test "setup for run" #t (begin (setup-for-run) - (string? (getenv "MT_RUN_AREA_HOME")))) - -(test "server-register, get-best-server" #t (let ((res #f)) - (open-run-close tasks:server-register tasks:open-db 1 "bob" 1234 100 'live 'http) - (set! res (open-run-close tasks:get-best-server tasks:open-db)) - (number? (vector-ref res 3)))) - -(test "de-register server" #t (let ((res #f)) - (open-run-close tasks:server-deregister tasks:open-db "bob" port: 1234) - (vector? (open-run-close tasks:get-best-server tasks:open-db)))) - -(define server-pid #f) -(test "launch server" #t (let ((pid (process-fork (lambda () - ;; (daemon:ize) - (server:launch 'http))))) - (set! server-pid pid) - (number? pid))) - -(thread-sleep! 3) ;; need to wait for server to start. Yes, a better way is needed. -(test "get-best-server" #t (let ((dat (open-run-close tasks:get-best-server tasks:open-db))) - (set! *runremote* (list (vector-ref dat 1)(vector-ref dat 2))) ;; host ip pullport pubport - (and (string? (car *runremote*)) - (number? (cadr *runremote*))))) - -(test #f #t (car (cdb:login *runremote* *toppath* *my-client-signature*))) -(test #f #t (let ((res (client:login *runremote*))) - (car res))) - - -;;====================================================================== -;; C O N F I G F I L E S -;;====================================================================== - -(define conffile #f) -(test "Read a config" #t (hash-table? (read-config "test.config" #f #f))) -(test "Read a config that doesn't exist" #t (hash-table? (read-config "nada.config" #f #f))) - -(set! conffile (read-config "test.config" #f #f)) -(test "Get available diskspace" #t (number? (get-df "./"))) -(test "Get best dir" #t (let ((bestdir (get-best-disk conffile))) - (or (equal? "./" bestdir) - (equal? "/tmp" bestdir)))) -(test "Multiline variable" 4 (length (string-split (config-lookup conffile "metadata" "description") "\n"))) - -;; db -(define row (vector "a" "b" "c" "blah")) -(define header (list "col1" "col2" "col3" "col4")) -(test "Get row by header" "blah" (db:get-value-by-header row header "col4")) - -;; (define *toppath* "tests") -(define *db* #f) -(test "open-db" #t (begin - (set! *db* (open-db)) - (if *db* #t #f))) - -;; quit wasting time, I'm changing *db* to db -(define db *db*) - -(test "get cpu load" #t (number? (get-cpu-load))) -(test "get uname" #t (string? (get-uname))) - -(test "get validvalues as list" (list "start" "end" "completed") - (string-split (config-lookup *configdat* "validvalues" "state"))) - -(for-each (lambda (item) - (test (conc "get valid items (" item ")") - item (items:check-valid-items "state" item))) - (list "start" "end" "completed")) - -(for-each (lambda (item) - (test (conc "get valid items (" item ")") - item (items:check-valid-items "status" item))) - (list "pass" "fail" "n/a")) - -(test #f #f (items:check-valid-items "state" "blahfool")) - -(test "write env files" "nada.csh" (begin - (save-environment-as-files "nada") - (and (file-exists? "nada.sh") - (file-exists? "nada.csh")))) - -(test #f #t (cdb:client-call *runremote* 'immediate #t 1 (lambda ()(display "Got here eh!?") #t))) - -;; (set! *verbosity* 20) -(test #f *verbosity* (cadr (cdb:set-verbosity *runremote* *verbosity*))) -(test #f #f (cdb:roll-up-pass-fail-counts *runremote* 1 "test1" "" "PASS")) -;; (set! *verbosity* 1) -;; (cdb:set-verbosity *runremote* *verbosity*) - -(test "get all legal tests" (list "test1" "test2") (sort (get-all-legal-tests) string<=?)) - - -(test "get-keys" "SYSTEM" (car (db:get-keys *db*))) - -(define remargs (args:get-args - '("bar" "foo" ":runname" "bob" ":SYSTEM" "ubuntu" ":RELEASE" "v1.2" ":datapath" "blah/foo" "nada") - (list ":runname" ":state" ":status") - (list "-h") - args:arg-hash - 0)) - -(test "register-run" #t (number? - (db:register-run *db* - '(("SYSTEM" "key1")("RELEASE" "key2")) - "myrun" - "new" - "n/a" - "bob"))) - -(test #f #t (cdb:tests-register-test *runremote* 1 "nada" "")) -(test #f 1 (cdb:remote-run db:get-test-id #f 1 "nada" "")) -(test #f "NOT_STARTED" (vector-ref (open-run-close db:get-test-info #f 1 "nada" "") 3)) -(test #f "NOT_STARTED" (vector-ref (cdb:get-test-info *runremote* 1 "nada" "") 3)) - -(define keys (db:get-keys *db*)) - -;;====================================================================== -;; D B -;;====================================================================== - -(test #f "FOO LIKE 'abc%def'" (db:patt->like "FOO" "abc%def")) -(test #f "key2" (vector-ref (car (vector-ref (runs:get-runs-by-patt *db* '("SYSTEM" "RELEASE") "%" "key1/key2") 1)) 1)) - -(test #f "SYSTEM,RELEASE,id,runname,state,status,owner,event_time" (car (runs:get-std-run-fields keys '("id" "runname" "state" "status" "owner" "event_time")))) -(test #f #t (runs:operate-on 'print "%" "%" "%")) - -;;(test "update-test-info" #t (test-update-meta-info *db* 1 "nada" -(setenv "BLAHFOO" "1234") -(unsetenv "NADAFOO") -(test "env temp overrides" "xyz" (let ((prevvals (alist->env-vars '(("BLAHFOO" 4321)("NADAFOO" xyz)))) - (result (get-environment-variable "NADAFOO"))) - (alist->env-vars prevvals) - result)) - -(test "env restored" "1234" (get-environment-variable "BLAHFOO")) - - -(test "Items assoc" "Elephant" (cadar (cadr (item-assoc->item-list '(("ANIMAL" "Elephant Lion")("SEASON" "Spring Fall")))))) -(set! *verbosity* 6) -(test "Items assoc" '()(item-assoc->item-list '(("a" "a b c d")("b" "c d e")("c" "")("d")))) -(set! *verbosity* -1) -(test "Items assoc empty items" '() (item-assoc->item-list '(("A")))) -(set! *verbosity* 1) -(test "Items table" "SEASON" (caadar (item-table->item-list '(("ANIMAL" "Elephant Lion")("SEASON" "Spring Winter"))))) -(test "Items table empty items I" '() (item-table->item-list '(("A")))) -(test "Items table empty items II" '() (item-table->item-list '(("A" "")))) - -;; Test out the steps code - -(define test-id #f) - -;; force keepgoing -; (hash-table-set! args:arg-hash "-keepgoing" #t) -(hash-table-set! args:arg-hash "-itempatt" "%") -(hash-table-set! args:arg-hash "-testpatt" "%") -(hash-table-set! args:arg-hash "-target" "ubuntu/r1.2") -(test "Setup for a run" #t (begin (setup-for-run) #t)) - -(define *tdb* #f) -(define keyvals #f) -(test "target->keyval" #t (let ((kv (keys:target->keyval keys (args:get-arg "-target")))) - (set! keyvals kv)(list? keyvals))) - -(define testdbpath (conc "/tmp/" (getenv "USER") "/megatest_testing")) -(system (conc "rm -f " testdbpath "/testdat.db;mkdir -p " testdbpath)) - -(print "Using " testdbpath " for test db") -(test #f #t (let ((db (open-test-db testdbpath))) - (set! *tdb* db) - (sqlite3#database? db))) -(sqlite3#finalize! *tdb*) - -;; (test "Remove the rollup run" #t (begin (remove-runs) #t)) -(define tconfig #f) -(test "get a testconfig" #t (let ((tconf (tests:get-testconfig "test1" 'return-procs))) - (set! tconfig tconf) - (hash-table? tconf))) -(db:clean-all-caches) - -(test "set-megatest-env-vars" - "ubuntu" - (begin - (set-megatest-env-vars 1 inkeys: keys) - (get-environment-variable "SYSTEM"))) -(test "setup-env-defaults" - "see this variable" - (begin - (setup-env-defaults "runconfigs.config" 1 *already-seen-runconfig-info* keys keyvals "pre-launch-env-vars") - (get-environment-variable "ALLTESTS"))) - -(test #f "ubuntu" (car (keys:target-set-args keys (args:get-arg "-target") args:arg-hash))) - -(define rinfo #f) -(test "get-run-info" #f (vector? (vector-ref (let ((rinf (cdb:remote-run db:get-run-info #f 1))) - (set! rinfo rinf) - rinf) 0))) -(test "get-key-vals" "key1" (car (cdb:remote-run db:get-key-vals #f 1))) -(test "tests:sort-by" '() (tests:sort-by-priority-and-waiton (make-hash-table))) - -(test "update-test_meta" "test1" (begin - (runs:update-test_meta "test1" tconfig) - (let ((dat (cdb:remote-run db:testmeta-get-record #f "test1"))) - (vector-ref dat 1)))) - -(define test-path "tests/test1") -(define disk-path #f) -(test "get-best-disk" #t (string? (file-exists? (let ((d (get-best-disk *configdat*))) - (set! disk-path d) - d)))) -(test "create-work-area" #t (symbolic-link? (car (create-work-area 1 rinfo keyvals 1 test-path disk-path "test1" '())))) -(test #f "" (item-list->path '())) - -(test "launch-test" #t (string? (file-exists? (launch-test 1 1 rinfo keyvals "run1" tconfig "test1" test-path '() (make-hash-table))))) - - -(test "Run a test" #t (general-run-call - "-runtests" - "run a test" - (lambda (target runname keys keyvallst) - (let ((test-patts "test%")) - ;; (runs:run-tests target runname test-patts user (make-hash-table)) - ;; (run:test run-id run-info key-vals runname test-record flags parent-test) - ;; (set! *verbosity* 22) ;; (list 0 1 2)) - (run:test 1 ;; run-id - #f ;; run-info is yet only a dream - keyvallst ;; (keys:target->keyval keys target) - "run1" ;; runname - (vector ;; test_records.scm tests:testqueue - "test1" ;; testname - tconfig ;; testconfig - '() ;; waitons - 0 ;; priority - #f ;; items - #f ;; itemsdat - "" ;; itempath - ) - args:arg-hash ;; flags (e.g. -itemspatt) - #f) - ;; (set! *verbosity* 0) - )))) - - - - - -(test "server stop" #f (let ((hostname (car *runremote*)) - (port (cadr *runremote*))) - (tasks:kill-server #t hostname port server-pid 'http) - (open-run-close tasks:get-best-server tasks:open-db))) - -(exit 1) -;; (test "cache is coherent" #t (let ((cached-info (db:get-test-info-cached-by-id db 2)) -;; (non-cached (db:get-test-info-not-cached-by-id db 2))) -;; (print "\nCached: " cached-info) -;; (print "Noncached: " non-cached) -;; (equal? cached-info non-cached))) - -(change-directory test-work-dir) -(test "Add a step" #t - (begin - (db:teststep-set-status! db 2 "step1" "start" 0 "This is a comment" "mylogfile.html") - (sleep 2) - (db:teststep-set-status! db 2 "step1" "end" "pass" "This is a different comment" "finallogfile.html") - (set! test-id (db:test-get-id (car (cdb:remote-run db:get-tests-for-run #f 1 "test1" '() '())))) - (number? test-id))) - -(test "Get rundir" #t (let ((rundir (cdb:remote-run db:test-get-rundir-from-test-id #f test-id))) - (print "Rundir " rundir) - (system (conc "mkdir -p " rundir)) - (string? rundir))) -(test #f #t (sqlite3#database? (open-test-db "./"))) -(test "Create a test db" "../simpleruns/key1/key2/myrun/test1/testdat.db" - (let ((tdb (open-run-close db:open-test-db-by-test-id db test-id))) - (if tdb (sqlite3#finalize! tdb)) - (file-exists? "../simpleruns/key1/key2/myrun/test1/testdat.db"))) - -(test "Get steps for test" #t (let ((steps (cdb:remote-run db:get-steps-for-test #f test-id))) - (print steps) - (> (length steps) 0))) -(test "Get nice table for steps" "2.0s" - (begin - (vector-ref (hash-table-ref (open-run-close db:get-steps-table #f test-id) "step1") 4))) - -;; (exit) - -(test #f "myrun" (cdb:remote-run db:get-run-name-from-id #f 1)) - -(test #f #f (cdb:remote-run db:roll-up-pass-fail-counts #f 1 "nada" "" "PASS")) - -;;====================================================================== -;; R E M O T E C A L L S -;;====================================================================== - -(define start-wait (current-seconds)) -(print "Starting intensive cache and rpc test") -(for-each (lambda (params) - (print "Intensive: params=" params) - (cdb:tests-register-test *runremote* 1 (conc "test" (random 20)) "") - (apply cdb:test-set-status-state *runremote* test-id params) - (cdb:pass-fail-counts *runremote* test-id (random 100) (random 100)) - (cdb:test-rollup-test_data-pass-fail *runremote* test-id) - (cdb:roll-up-pass-fail-counts *runremote* 1 "test1" "" (cadr params)) - (thread-sleep! 0.01)) ;; cache ordering granularity is at the second level. Should really be at the ms level - '(("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("NOT_STARTED" "FAIL" "Just testing") - ("COMPLETED" "PASS" #f) - ("NOT_STARTED" "FAIL" "Just testing") - ("KILLED" "UNKNOWN" "More testing") - ("KILLED" "UNKNOWN" "More testing") - )) - -;; now set all tests to completed -(cdb:flush-queue *runremote*) -(let ((tests (cdb:remote-run db:get-tests-for-run #f 1 "%" '() '()))) - (print "Setting " (length tests) " to COMPLETED/PASS") - (for-each - (lambda (test) - (cdb:test-set-status-state *runremote* (db:test-get-id test) "COMPLETED" "PASS" "Forced pass")) - tests)) - -;; (process-wait server-pid) -;; (test "Server wait time" #t (let ((run-delta (- (current-seconds) start-wait))) -;; (print "Server ran for " run-delta " seconds") -;; (> run-delta 20))) - -(test "Rollup the run(s)" #t (begin - (runs:rollup-run keys (keys->alist keys "na") "rollup" "matt") - #t)) - -(hash-table-set! args:arg-hash ":runname" "%") - -(test "Remove the rollup run" #t (begin (operate-on 'remove-runs))) - -(print "Waiting for server to be done, should be about 20 seconds") -(test "server stop" #f (let ((hostname (car *runremote*)) - (port (cadr *runremote*))) - (tasks:kill-server #t hostname port server-pid 'http) - (open-run-close tasks:get-best-server tasks:open-db))) - -;; (cdb:kill-server *runremote*) - -;; (thread-join! th1 th2 th3) - -;; ADD ME!!!! (db:get-prereqs-not-met *db* 1 '("runfirst") "" mode: 'normal) -;; ADD ME!!!! (rdb:get-tests-for-run *db* 1 "runfirst" #f '() '()) +(let* ((unit-test-name (list-ref (argv) 4)) + (fname (conc "../unittests/" unit-test-name ".scm"))) + (if (file-exists? fname) + (load fname) + (print "ERROR: Unit test " unit-test-name " not found in unittests directory"))) + ADDED tests/unittests/configfiles.scm Index: tests/unittests/configfiles.scm ================================================================== --- /dev/null +++ tests/unittests/configfiles.scm @@ -0,0 +1,52 @@ +;;====================================================================== +;; C O N F I G F I L E S +;;====================================================================== + +(define conffile #f) +(test "Read a config" #t (hash-table? (read-config "test.config" #f #f))) +(test "Read a config that doesn't exist" #t (hash-table? (read-config "nada.config" #f #f))) + +(set! conffile (read-config "test.config" #f #f)) +(test "Get available diskspace" #t (number? (get-df "./"))) +(test "Get best dir" #t (let ((bestdir (get-best-disk conffile))) + (or (equal? "./" bestdir) + (equal? "/tmp" bestdir)))) +(test "Multiline variable" 4 (length (string-split (config-lookup conffile "metadata" "description") "\n"))) + +;; db +(define row (vector "a" "b" "c" "blah")) +(define header (list "col1" "col2" "col3" "col4")) +(test "Get row by header" "blah" (db:get-value-by-header row header "col4")) + +;; (define *toppath* "tests") +(define *db* #f) +(test "open-db" #t (begin + (set! *db* (open-db)) + (if *db* #t #f))) + +;; quit wasting time, I'm changing *db* to db +(define db *db*) + +(test "get cpu load" #t (number? (get-cpu-load))) +(test "get uname" #t (string? (get-uname))) + +(test "get validvalues as list" (list "start" "end" "completed") + (string-split (config-lookup *configdat* "validvalues" "state"))) + +(for-each (lambda (item) + (test (conc "get valid items (" item ")") + item (items:check-valid-items "state" item))) + (list "start" "end" "completed")) + +(for-each (lambda (item) + (test (conc "get valid items (" item ")") + item (items:check-valid-items "status" item))) + (list "pass" "fail" "n/a")) + +(test #f #f (items:check-valid-items "state" "blahfool")) + +(test "write env files" "nada.csh" (begin + (save-environment-as-files "nada") + (and (file-exists? "nada.sh") + (file-exists? "nada.csh")))) + ADDED tests/unittests/inmemdb.scm Index: tests/unittests/inmemdb.scm ================================================================== --- /dev/null +++ tests/unittests/inmemdb.scm @@ -0,0 +1,44 @@ +;;====================================================================== +;; S E R V E R +;;====================================================================== + +;; Run like this: +;; +;; (cd ..;make && make install) && ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) + +(set! *transport-type* 'http) + +(system "cp ../fullrun/megatest.db megatest.db") + +(test "open inmem db" 1 (begin (open-in-mem-db) 1)) + +(test "setup for run" #t (begin (setup-for-run) + (string? (getenv "MT_RUN_AREA_HOME")))) + +(system "megatest -server - -debug 0 &") + +(thread-sleep! 3) ;; need to wait for server to start. Yes, a better way is needed. + +(define *keys* (keys:config-get-fields *configdat*)) +(define *keyvals* (keys:target->keyval *keys* "a/b/c")) + +(test #f #t (string? (car *runremote*))) +(test #f '(#t "successful login") (rmt:login)) ;; *runremote* *toppath* *my-client-signature*))) + +(define inmem (open-in-mem-db)) + +(define (inmem-test t b) + (test "inmem sync to" t (db:sync-to *db* inmem)) + (test "inmem sync back" b (db:sync-to inmem *db*))) + +(inmem-test 0 0) + +(inmem-test 1 1) + +;;====================================================================== +;; D B +;;====================================================================== + +(test #f '(#t "exit process started") (rmt:kill-server)) ;; *toppath* *my-client-signature* #f))) + + ADDED tests/unittests/misc.scm Index: tests/unittests/misc.scm ================================================================== --- /dev/null +++ tests/unittests/misc.scm @@ -0,0 +1,45 @@ +;;====================================================================== +;; P R O C E S S E S +;;====================================================================== + +(test "cmd-run-with-stderr->list" '("No such file or directory") + (let ((reslst (cmd-run-with-stderr->list "ls" "/tmp/ihadbetternotexist"))) + (string-search (regexp "No such file or directory")(car reslst)))) + +;;====================================================================== +;; T E S T M A T C H I N G +;;====================================================================== + +;; tests:glob-like-match +(test #f '("abc") (tests:glob-like-match "abc" "abc")) +(for-each + (lambda (patt str expected) + (test (conc patt " " str "=>" expected) expected (tests:glob-like-match patt str))) + (list "abc" "~abc" "~abc" "a*c" "a%c") + (list "abc" "abcd" "abc" "ABC" "ABC") + (list '("abc") #t #f #f '("ABC")) + ) + +;; tests:match +(test #f #t (tests:match "abc/def" "abc" "def")) +(for-each + (lambda (patterns testname itempath expected) + (test (conc patterns " " testname "/" itempath "=>" expected) + expected + (tests:match patterns testname itempath))) + (list "abc" "abc/%" "ab%/c%" "~abc/c%" "abc/~c%" "a,b/c,%/d" "%/,%/a" "%/,%/a" "%/,%/a" "%" "%" "%/" "%/") + (list "abc" "abc" "abcd" "abc" "abc" "a" "abc" "def" "ghi" "a" "a" "a" "a") + (list "" "" "cde" "cde" "cde" "" "" "a" "b" "" "b" "" "b") + (list #t #t #t #f #f #t #t #t #f #t #t #t #f)) + +;; db:patt->like +(test #f "testname LIKE 't%'" (db:patt->like "testname" "t%" comparator: " AND ")) +(test #f "testname LIKE 't%' AND testname LIKE '%t'" (db:patt->like "testname" "t%,%t" comparator: " AND ")) +(test #f "item_path GLOB ''" (db:patt->like "item_path" "")) + +;; test:match->sqlqry +(test #f "(testname GLOB 'a' AND item_path GLOB 'b') OR (testname LIKE 'a%' AND item_path LIKE '%') OR (testname GLOB '' AND item_path LIKE 'b%')" + (tests:match->sqlqry "a/b,a%,/b%")) +(test #f "(testname GLOB 'a' AND item_path GLOB 'b') OR (testname LIKE 'a%' AND item_path LIKE '%') OR (testname LIKE '%' AND item_path LIKE 'b%')" + (tests:match->sqlqry "a/b,a%,%/b%")) + ADDED tests/unittests/runs.scm Index: tests/unittests/runs.scm ================================================================== --- /dev/null +++ tests/unittests/runs.scm @@ -0,0 +1,275 @@ +(define keys (db:get-keys *db*)) + +(test "get all legal tests" (list "test1" "test2") (sort (get-all-legal-tests) string<=?)) + +(test "register-run" #t (number? + (db:register-run *db* + '(("SYSTEM" "key1")("RELEASE" "key2")) + "myrun" + "new" + "n/a" + "bob"))) + +(test #f #t (cdb:tests-register-test *runremote* 1 "nada" "")) +(test #f 1 (cdb:remote-run db:get-test-id #f 1 "nada" "")) +(test #f "NOT_STARTED" (vector-ref (open-run-close db:get-test-info #f 1 "nada" "") 3)) +(test #f "NOT_STARTED" (vector-ref (cdb:get-test-info *runremote* 1 "nada" "") 3)) + +(test #f "FOO LIKE 'abc%def'" (db:patt->like "FOO" "abc%def")) +(test #f "key2" (vector-ref (car (vector-ref (runs:get-runs-by-patt *db* '("SYSTEM" "RELEASE") "%" "key1/key2") 1)) 1)) + +(test #f "SYSTEM,RELEASE,id,runname,state,status,owner,event_time" (car (runs:get-std-run-fields keys '("id" "runname" "state" "status" "owner" "event_time")))) +(test #f #t (runs:operate-on 'print "%" "%" "%")) + +;;(test "update-test-info" #t (test-update-meta-info *db* 1 "nada" +(setenv "BLAHFOO" "1234") +(unsetenv "NADAFOO") +(test "env temp overrides" "xyz" (let ((prevvals (alist->env-vars '(("BLAHFOO" 4321)("NADAFOO" xyz)))) + (result (get-environment-variable "NADAFOO"))) + (alist->env-vars prevvals) + result)) + +(test "env restored" "1234" (get-environment-variable "BLAHFOO")) + + +(test "Items assoc" "Elephant" (cadar (cadr (item-assoc->item-list '(("ANIMAL" "Elephant Lion")("SEASON" "Spring Fall")))))) +(set! *verbosity* 6) +(test "Items assoc" '()(item-assoc->item-list '(("a" "a b c d")("b" "c d e")("c" "")("d")))) +(set! *verbosity* -1) +(test "Items assoc empty items" '() (item-assoc->item-list '(("A")))) +(set! *verbosity* 1) +(test "Items table" "SEASON" (caadar (item-table->item-list '(("ANIMAL" "Elephant Lion")("SEASON" "Spring Winter"))))) +(test "Items table empty items I" '() (item-table->item-list '(("A")))) +(test "Items table empty items II" '() (item-table->item-list '(("A" "")))) + +;; Test out the steps code + +(define test-id #f) + +;; force keepgoing +; (hash-table-set! args:arg-hash "-keepgoing" #t) +(hash-table-set! args:arg-hash "-itempatt" "%") +(hash-table-set! args:arg-hash "-testpatt" "%") +(hash-table-set! args:arg-hash "-target" "ubuntu/r1.2") +(test "Setup for a run" #t (begin (setup-for-run) #t)) + +(define *tdb* #f) +(define keyvals #f) +(test "target->keyval" #t (let ((kv (keys:target->keyval keys (args:get-arg "-target")))) + (set! keyvals kv)(list? keyvals))) + +(define testdbpath (conc "/tmp/" (getenv "USER") "/megatest_testing")) +(system (conc "rm -f " testdbpath "/testdat.db;mkdir -p " testdbpath)) + +(print "Using " testdbpath " for test db") +(test #f #t (let ((db (open-test-db testdbpath))) + (set! *tdb* db) + (sqlite3#database? db))) +(sqlite3#finalize! *tdb*) + +;; (test "Remove the rollup run" #t (begin (remove-runs) #t)) +(define tconfig #f) +(test "get a testconfig" #t (let ((tconf (tests:get-testconfig "test1" 'return-procs))) + (set! tconfig tconf) + (hash-table? tconf))) +(db:clean-all-caches) + +(test "set-megatest-env-vars" + "ubuntu" + (begin + (set-megatest-env-vars 1 inkeys: keys) + (get-environment-variable "SYSTEM"))) +(test "setup-env-defaults" + "see this variable" + (begin + (setup-env-defaults "runconfigs.config" 1 *already-seen-runconfig-info* keys keyvals "pre-launch-env-vars") + (get-environment-variable "ALLTESTS"))) + +(test #f "ubuntu" (car (keys:target-set-args keys (args:get-arg "-target") args:arg-hash))) + +(define rinfo #f) +(test "get-run-info" #f (vector? (vector-ref (let ((rinf (cdb:remote-run db:get-run-info #f 1))) + (set! rinfo rinf) + rinf) 0))) +(test "get-key-vals" "key1" (car (cdb:remote-run db:get-key-vals #f 1))) +(test "tests:sort-by" '() (tests:sort-by-priority-and-waiton (make-hash-table))) + +(test "update-test_meta" "test1" (begin + (runs:update-test_meta "test1" tconfig) + (let ((dat (cdb:remote-run db:testmeta-get-record #f "test1"))) + (vector-ref dat 1)))) + +(define test-path "tests/test1") +(define disk-path #f) +(test "get-best-disk" #t (string? (file-exists? (let ((d (get-best-disk *configdat*))) + (set! disk-path d) + d)))) +(test "create-work-area" #t (symbolic-link? (car (create-work-area 1 rinfo keyvals 1 test-path disk-path "test1" '())))) +(test #f "" (item-list->path '())) + +(test "launch-test" #t (string? (file-exists? (launch-test 1 1 rinfo keyvals "run1" tconfig "test1" test-path '() (make-hash-table))))) + + +(test "Run a test" #t (general-run-call + "-runtests" + "run a test" + (lambda (target runname keys keyvallst) + (let ((test-patts "test%")) + ;; (runs:run-tests target runname test-patts user (make-hash-table)) + ;; (run:test run-id run-info key-vals runname test-record flags parent-test) + ;; (set! *verbosity* 22) ;; (list 0 1 2)) + (run:test 1 ;; run-id + #f ;; run-info is yet only a dream + keyvallst ;; (keys:target->keyval keys target) + "run1" ;; runname + (vector ;; test_records.scm tests:testqueue + "test1" ;; testname + tconfig ;; testconfig + '() ;; waitons + 0 ;; priority + #f ;; items + #f ;; itemsdat + "" ;; itempath + ) + args:arg-hash ;; flags (e.g. -itemspatt) + #f) + ;; (set! *verbosity* 0) + )))) + + + + + +(test "server stop" #f (let ((hostname (car *runremote*)) + (port (cadr *runremote*))) + (tasks:kill-server #t hostname port server-pid 'http) + (open-run-close tasks:get-best-server tasks:open-db))) + +(exit 1) +;; (test "cache is coherent" #t (let ((cached-info (db:get-test-info-cached-by-id db 2)) +;; (non-cached (db:get-test-info-not-cached-by-id db 2))) +;; (print "\nCached: " cached-info) +;; (print "Noncached: " non-cached) +;; (equal? cached-info non-cached))) + +(change-directory test-work-dir) +(test "Add a step" #t + (begin + (db:teststep-set-status! db 2 "step1" "start" 0 "This is a comment" "mylogfile.html") + (sleep 2) + (db:teststep-set-status! db 2 "step1" "end" "pass" "This is a different comment" "finallogfile.html") + (set! test-id (db:test-get-id (car (cdb:remote-run db:get-tests-for-run #f 1 "test1" '() '())))) + (number? test-id))) + +(test "Get rundir" #t (let ((rundir (cdb:remote-run db:test-get-rundir-from-test-id #f test-id))) + (print "Rundir " rundir) + (system (conc "mkdir -p " rundir)) + (string? rundir))) +(test #f #t (sqlite3#database? (open-test-db "./"))) +(test "Create a test db" "../simpleruns/key1/key2/myrun/test1/testdat.db" + (let ((tdb (open-run-close db:open-test-db-by-test-id db test-id))) + (if tdb (sqlite3#finalize! tdb)) + (file-exists? "../simpleruns/key1/key2/myrun/test1/testdat.db"))) + +(test "Get steps for test" #t (let ((steps (cdb:remote-run db:get-steps-for-test #f test-id))) + (print steps) + (> (length steps) 0))) +(test "Get nice table for steps" "2.0s" + (begin + (vector-ref (hash-table-ref (open-run-close db:get-steps-table #f test-id) "step1") 4))) + +;; (exit) + +(test #f "myrun" (cdb:remote-run db:get-run-name-from-id #f 1)) + +(test #f #f (cdb:remote-run db:roll-up-pass-fail-counts #f 1 "nada" "" "PASS")) + +;;====================================================================== +;; R E M O T E C A L L S +;;====================================================================== + +(define start-wait (current-seconds)) +(print "Starting intensive cache and rpc test") +(for-each (lambda (params) + (print "Intensive: params=" params) + (cdb:tests-register-test *runremote* 1 (conc "test" (random 20)) "") + (apply cdb:test-set-status-state *runremote* test-id params) + (cdb:pass-fail-counts *runremote* test-id (random 100) (random 100)) + (cdb:test-rollup-test_data-pass-fail *runremote* test-id) + (cdb:roll-up-pass-fail-counts *runremote* 1 "test1" "" (cadr params)) + (thread-sleep! 0.01)) ;; cache ordering granularity is at the second level. Should really be at the ms level + '(("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("NOT_STARTED" "FAIL" "Just testing") + ("COMPLETED" "PASS" #f) + ("NOT_STARTED" "FAIL" "Just testing") + ("KILLED" "UNKNOWN" "More testing") + ("KILLED" "UNKNOWN" "More testing") + )) + +;; now set all tests to completed +(cdb:flush-queue *runremote*) +(let ((tests (cdb:remote-run db:get-tests-for-run #f 1 "%" '() '()))) + (print "Setting " (length tests) " to COMPLETED/PASS") + (for-each + (lambda (test) + (cdb:test-set-status-state *runremote* (db:test-get-id test) "COMPLETED" "PASS" "Forced pass")) + tests)) + +;; (process-wait server-pid) +;; (test "Server wait time" #t (let ((run-delta (- (current-seconds) start-wait))) +;; (print "Server ran for " run-delta " seconds") +;; (> run-delta 20))) + +(test "Rollup the run(s)" #t (begin + (runs:rollup-run keys (keys->alist keys "na") "rollup" "matt") + #t)) + +(hash-table-set! args:arg-hash ":runname" "%") + +(test "Remove the rollup run" #t (begin (operate-on 'remove-runs))) + +(print "Waiting for server to be done, should be about 20 seconds") +(test "server stop" #f (let ((hostname (car *runremote*)) + (port (cadr *runremote*))) + (tasks:kill-server #t hostname port server-pid 'http) + (open-run-close tasks:get-best-server tasks:open-db))) + +;; (cdb:kill-server *runremote*) + +;; (thread-join! th1 th2 th3) + +;; ADD ME!!!! (db:get-prereqs-not-met *db* 1 '("runfirst") "" mode: 'normal) +;; ADD ME!!!! (rdb:get-tests-for-run *db* 1 "runfirst" #f '() '()) ADDED tests/unittests/server.scm Index: tests/unittests/server.scm ================================================================== --- /dev/null +++ tests/unittests/server.scm @@ -0,0 +1,88 @@ +;;====================================================================== +;; S E R V E R +;;====================================================================== + +;; Run like this: +;; +;; (cd ..;make && make install) && ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) + +(set! *transport-type* 'http) + +(test "setup for run" #t (begin (setup-for-run) + (string? (getenv "MT_RUN_AREA_HOME")))) + +(test "server-register, get-best-server" #t (let ((res #f)) + (open-run-close tasks:server-register tasks:open-db 1 "bob" 1234 100 'live 'http) + (set! res (open-run-close tasks:get-best-server tasks:open-db)) + (number? (vector-ref res 3)))) + +(test "de-register server" #f (let ((res #f)) + (open-run-close tasks:server-deregister tasks:open-db "bob" port: 1234) + (vector? (open-run-close tasks:get-best-server tasks:open-db)))) + +(define server-pid #f) + +;; Not sure how the following should work, replacing it with system of megatest -server +;; (test "launch server" #t (let ((pid (process-fork (lambda () +;; ;; (daemon:ize) +;; (server:launch 'http))))) +;; (set! server-pid pid) +;; (number? pid))) +(system "../../bin/megatest -server - -debug 22 > server.log 2> server.log &") + +(let loop ((n 10)) + (thread-sleep! 1) ;; need to wait for server to start. + (let ((res (open-run-close tasks:get-best-server tasks:open-db))) + (print "tasks:get-best-server returned " res) + (if (and (not res) + (> n 0)) + (loop (- n 1))))) + +(test "get-best-server" #t (begin + (client:launch) + (let ((dat (open-run-close tasks:get-best-server tasks:open-db))) + (vector? dat)))) + +(define *keys* (keys:config-get-fields *configdat*)) +(define *keyvals* (keys:target->keyval *keys* "a/b/c")) + +(test #f #t (string? (car *runremote*))) +(test #f '(#t "successful login") (rmt:login)) ;; *runremote* *toppath* *my-client-signature*))) + +(test #f #f (rmt:get-test-info-by-id 99)) ;; get non-existant test + +;; RUNS +(test #f 1 (rmt:register-run *keyvals* "firstrun" "new" "n/a" (current-user-name))) +(test "get run info" "firstrun" (let ((rinfo (rmt:get-run-info 1))) + (vector-ref (vector-ref rinfo 1) 3))) +(test "get runname from id" "firstrun" (rmt:get-run-name-from-id 1)) + +;; TESTS +(test "get tests (no data)" '() (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f)) +(test "register test" #t (rmt:general-call 'register-test 1 "test1" "")) +(test "get tests (some data)" 1 (length (rmt:get-tests-for-run 1 "%" '() '() #f #f #f #f #f #f))) +(test "get test id" 1 (rmt:get-test-id 1 "test1" "")) +(test "sync back" #t (> (rmt:sync-inmem->db) 0)) +(test "get test id from main" 1 (db:get-test-id *db* 1 "test1" "")) +(test "get keys" #t (list? (rmt:get-keys))) +(test "set comment" #t (begin (rmt:general-call 'set-test-comment "this is a comment" 1) #t)) +(test "get comment" "this is a comment" (let ((trec (rmt:get-test-info-by-id 1))) + (db:test-get-comment trec))) + +;; MORE RUNS +(test "get runs" #t (let* ((runs (rmt:get-runs "%" #f #f '())) + (header (vector-ref runs 0)) + (data (vector-ref runs 1))) + (and (list? header) + (list? data) + (vector? (car data))))) + +(test "get local testinfo" "test1" (vector-ref (db:get-testinfo-state-status *db* 1) 2)) +(test "get testinfo" "test1" (vector-ref (rmt:get-testinfo-state-status 1) 2)) + +;;====================================================================== +;; D B +;;====================================================================== + +(test #f '(#t "exit process started") (rmt:kill-server)) ;; *toppath* *my-client-signature* #f))) + ADDED tests/unittests/tests.scm Index: tests/unittests/tests.scm ================================================================== --- /dev/null +++ tests/unittests/tests.scm