Index: apimod.scm ================================================================== --- apimod.scm +++ apimod.scm @@ -39,10 +39,11 @@ chicken.process-context.posix chicken.string chicken.time chicken.condition chicken.process + chicken.pathname chicken.random chicken.file ;; (prefix sqlite3 sqlite3:) typed-records @@ -166,15 +167,20 @@ tasks-add tasks-set-state-given-param-key )) (define (api:run-server-process apath dbname) - (let* ((cmd (conc "nbfake megatest -server - -area "apath - " -db "dbname)) - (cleandbname (string-translate dbname "./" "_-")) - (logd (conc apath "/logs")) - (logf (conc logd "/server-"(current-seconds)cleandbname".log"))) + (let* ((cleandbname (pathname-strip-directory dbname)) ;; (string-translate dbname "./" "--")) + (logd (conc apath "/logs")) + (logf (conc logd "/server-launch-";;(current-process-id) + (seconds->year-work-week/day-time-fname (current-seconds)) + "-"cleandbname".log")) + (logf2 (conc logd "/server-" + (seconds->year-work-week/day-time-fname (current-seconds)) + "-"cleandbname"-")) + (cmd (conc "nbfake megatest -server - -area "apath + " -db "dbname" -autolog "logf2))) (if (not (directory-exists? logd)) (create-directory logd #t)) (system (conc "NBFAKE_LOG="logf" "cmd)))) ;; special function to get server @@ -344,11 +350,11 @@ ((get-prev-run-ids) (apply db:get-prev-run-ids dbstruct params)) ((get-run-ids-matching-target) (apply db:get-run-ids-matching-target dbstruct params)) ((get-runs-by-patt) (apply db:get-runs-by-patt dbstruct params)) ((get-run-name-from-id) (apply db:get-run-name-from-id dbstruct params)) ((get-main-run-stats) (apply db:get-main-run-stats dbstruct params)) - ((log-to-main) (apply debug:print params)) + ((log-to-main) (apply debug:print 0 *default-log-port* params)) ((get-var) (apply db:get-var dbstruct params)) ((get-run-stats) (apply db:get-run-stats dbstruct params)) ((get-run-times) (apply db:get-run-times dbstruct params)) ;; STEPS Index: build-assist/ck5-eggs.list ================================================================== --- build-assist/ck5-eggs.list +++ build-assist/ck5-eggs.list @@ -4,10 +4,11 @@ apropos base64 crypt csv-abnf directory-utils +dot-locking filepath fmt format http-client itemsmod Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -314,10 +314,11 @@ seconds->time-string seconds->work-week/day-time seconds->work-week/day seconds->year-work-week/day seconds->year-work-week/day-time +seconds->year-work-week/day-time-fname seconds->year-week/day-time seconds->quarter common:date-time->seconds common:find-start-mark-and-mark-delta common:expand-cron-slash @@ -3579,10 +3580,14 @@ (seconds->local-time sec) "ww%V.%u")) (define (seconds->year-work-week/day sec) (time->string (seconds->local-time sec) "%yww%V.%w")) + +(define (seconds->year-work-week/day-time-fname sec) + (time->string + (seconds->local-time sec) "%yww%V.%w.%H%M%S")) (define (seconds->year-work-week/day-time sec) (time->string (seconds->local-time sec) "%Yww%V.%w %H:%M")) Index: configfmod.scm ================================================================== --- configfmod.scm +++ configfmod.scm @@ -86,10 +86,11 @@ (prefix base64 base64:) (prefix dbi dbi:) (prefix sqlite3 sqlite3:) (srfi 18) directory-utils + dot-locking format matchable md5 message-digest regex @@ -112,10 +113,15 @@ ;; parameters ;;====================================================================== ;; while targets are Megatest specific they are a useful concept (define mytarget (make-parameter #f)) + +;; locking is optional, many environments don't care (e.g. running on one machine) +;; NOTE: the locker must follow the same syntax as with-dot-lock* +;; +(define my-with-lock (make-parameter with-dot-lock*)) ;;====================================================================== ;; move debug stuff to separate module then put these back where they belong ;;====================================================================== ;;====================================================================== @@ -1186,34 +1192,36 @@ ;; DO THE LOCKING AROUND THE CALL ;;====================================================================== ;; (define (configf:write-alist cdat fname) ;; (if (not (common:faux-lock fname)) - (debug:print 0 *default-log-port* "INFO: NEED LOCKING ADDED HERE " fname) - (let* ((dat (configf:config->alist cdat)) - (res - (begin - (with-output-to-file fname ;; first write out the file - (lambda () - (pp dat))) - ;; I don't like this. It makes write-alist opaque and complicated. -mrw- - (if (file-exists? fname) ;; now verify it is readable - (if (configf:read-alist fname) - #t ;; data is good. - (begin - (handle-exceptions + ;; (debug:print 0 *default-log-port* "INFO: NEED LOCKING ADDED HERE " fname) + ((my-with-lock) + fname + (lambda () + (let* ((dat (configf:config->alist cdat)) + (res + (begin + (with-output-to-file fname ;; first write out the file + (lambda () + (pp dat))) + ;; I don't like this. It makes write-alist opaque and complicated. -mrw- + (if (file-exists? fname) ;; now verify it is readable + (if (configf:read-alist fname) + #t ;; data is good. + (begin + (handle-exceptions exn - (begin - (debug:print 0 *default-log-port* "deleting " fname " failed, exn=" exn) - #f) - (debug:print 0 *default-log-port* "WARNING: content " dat " for cache " fname " is not readable. Deleting generated file.") - (delete-file fname)) - #f)) - #f)))) - ;; (common:faux-unlock fname) - res)) + (begin + (debug:print 0 *default-log-port* "deleting " fname " failed, exn=" exn) + #f) + (debug:print 0 *default-log-port* "WARNING: content " dat " for cache " fname " is not readable. Deleting generated file.") + (delete-file fname)) + #f)) + #f)))) + res)))) (define (common:get-fields cfgdat) (let ((fields (hash-table-ref/default cfgdat "fields" '()))) (map car fields))) ) Index: debugprint.scm ================================================================== --- debugprint.scm +++ debugprint.scm @@ -104,12 +104,14 @@ (list? n)) (member vb n)) (else #f)))) (define (debug:handle-remote-logging params) - (if (debug:print-logger) - (apply (debug:print-logger) "REMOTE ("(get-host-name)", pid="(current-process-id)") " params))) + (if (debug:print-logger) ;; NOTE: turn params into string to keep things simple for now + ((debug:print-logger)(conc "REMOTE ("(get-host-name)", pid="(current-process-id)") " + (string-intersperse (map conc params) " ") "; " + (string-intersperse (command-line-arguments) " "))))) (define (debug:print n e . params) (if (debug:debug-mode n) (with-output-to-port (or e (current-error-port)) (lambda () Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -437,10 +437,11 @@ - to automatically figure out hostname -adjutant C,M : start the server/adjutant with allocated cores C and Mem M (Gig), use 0,0 to auto use full machine -transport http|rpc : use http or rpc for transport (default is http) -log logfile : send stdout and stderr to logfile + -autolog logfilebase : appends pid and host to logfilebase for logfile -list-servers : list the servers -kill-servers : kill all servers -repl : start a repl (useful for extending megatest) -load file.scm : load and run file.scm -mark-incompletes : find and mark incomplete tests @@ -630,10 +631,11 @@ "-run-id" "-ping" "-refdb2dat" "-o" "-log" + "-autolog" "-sync-log" "-since" "-fields" "-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state "-sort" @@ -784,20 +786,24 @@ ;; this segment will run launch:setup only if -log is not set. This is fairly safe as servers are not ;; manually started and thus should never be started in a non-megatest area. Thus no need to handle situation ;; where (launch:setup) returns #f? ;; - (if (or (args:get-arg "-log")#;(args:get-arg "-server")) ;; redirect the log always when a server + (if (or (args:get-arg "-log") ;;(args:get-arg "-server") ;; redirect the log always when a server + (args:get-arg "-autolog")) (handle-exceptions exn (begin (print "ERROR: Failed to switch to log output. " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn) ) - (let* ((tl (or (args:get-arg "-log")(launch:setup))) ;; run launch:setup if -server, ensure we do NOT run launch:setup if -log specified - (logf (or (args:get-arg "-log") ;; use -log unless we are a server, then craft a logfile name - (conc tl "/logs/server-" (current-process-id) "-" (get-host-name) ".log"))) - (oup (open-logfile logf))) + (let* ((tl (or (args:get-arg "-log") + (args:get-arg "-autolog") ;; autolog provides the basename .../logs/something- for the logfile + (launch:setup))) ;; run launch:setup if -server, ensure we do NOT run launch:setup if -log specified + (logf (or (args:get-arg "-log") ;; use -log unless we are a server, then craft a logfile name + (conc tl (current-process-id)"-"(get-host-name)".log") + (conc tl "/logs/server-" (current-process-id) "-" (get-host-name) ".log"))) + (oup (open-logfile logf))) (if (not (args:get-arg "-log")) (hash-table-set! args:arg-hash "-log" logf)) ;; fake out future queries of -log (debug:print-info 0 *default-log-port* "Sending log output to " logf) (set! *default-log-port* oup)))) Index: rmtmod.scm ================================================================== --- rmtmod.scm +++ rmtmod.scm @@ -265,10 +265,15 @@ (assert (not (equal? dbname ".db/main.db")) "ERROR: general-open-connection should never be called with main as the db") (let* ((mdbname (db:run-id->dbname #f)) (fullname (db:dbname->path apath dbname)) (conns (remotedat-conns remdat)) (mconn (rmt:get-conn remdat apath mdbname))) + (if (and mconn + (not (debug:print-logger))) + (begin + (debug:print-info 0 *default-log-port* "Turning on logging to main, look in logs dir for main log.") + (debug:print-logger rmt:log-to-main))) (cond ((or (not mconn) ;; no channel open to main? (< (conndat-expires mconn)(+ (current-seconds) 2))) ;; restablish connection if less than 2 seconds on the lease (rmt:open-main-connection remdat apath) (rmt:general-open-connection remdat apath mdbname)) @@ -310,15 +315,11 @@ (debug:print-info 0 *default-log-port* "return data from starting server did not match host port servkey pid ipaddr apath dbname " res))) res) (begin (debug:print-info 0 *default-log-port* "Unexpected result: " res) res))))))) - (if (and mconn - (not (debug:print-logger))) - (begin - (debug:print-info 0 *default-log-port* "Turning on logging to main, look in logs dir for main log.") - (debug:print-logger rmt:log-to-main))) + #t)) ;;====================================================================== ;; FOR DEBUGGING SET TO #t @@ -365,12 +366,14 @@ (port (conndat-port conn)) (payload `((cmd . ,cmd) (key . ,(conndat-srvkey conn)) (params . ,params))) (res (send-receive-nn soc ;; (open-send-receive-nn (conc host":"port) - (sexpr->string payload)))) - (string->sexpr res)))) + (sexpr->string payload)))) + (if (member res '("#")) ;; TODO - fix this in string->sexpr + #f + (string->sexpr res))))) ;; db is at apath/.db/dbname, rid is an intermediary solution and will be removed ;; sometime in the future. ;; ;; Purpose - call the main.db server and request a server be started @@ -794,11 +797,11 @@ (define (rmt:get-main-run-stats run-id) (rmt:send-receive 'get-main-run-stats #f (list run-id))) (define (rmt:log-to-main . params) - (rmt:send-receive 'log-to-main #f (cons #f params))) + (rmt:send-receive 'log-to-main #f params)) (define (rmt:get-var run-id varname) (rmt:send-receive 'get-var run-id (list run-id varname))) (define (rmt:del-var run-id varname) @@ -1637,15 +1640,16 @@ ;; ====================================================================== (define (http-get-function fnkey) (hash-table-ref/default *http-functions* fnkey (lambda () "nothing here yet"))) +;; Main entry point to start a server. was start-server (define (rmt:run hostn) ;; ;; Configurations for server ;; (tcp-buffer-size 2048) ;; (max-connections 2048) - (debug:print 2 *default-log-port* "Attempting to start the server ...") + (debug:print 2 *default-log-port* "PID: "(current-process-id)". Attempting to start the server ...") (let* ((db #f) ;; (open-db)) ;; we don't want the server to be opening and closing the db unnecesarily (hostname (get-host-name)) (ipaddrstr (let ((ipstr (if (string=? "-" hostn) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") (server:get-best-guess-address hostname) ADDED tests/simplerun/Makefile Index: tests/simplerun/Makefile ================================================================== --- /dev/null +++ tests/simplerun/Makefile @@ -0,0 +1,3 @@ + +cleanup : + killall mtest -v -9;rm -rf .meta .db Index: tests/simplerun/megatest.config ================================================================== --- tests/simplerun/megatest.config +++ tests/simplerun/megatest.config @@ -36,11 +36,11 @@ state start end completed # Job tools are more advanced ways to control how your jobs are launched [jobtools] useshell yes -launcher nbfind +launcher nbfake # You can override environment variables for all your tests here [env-override] EXAMPLE_VAR example value Index: tests/unittests/server.scm ================================================================== --- tests/unittests/server.scm +++ tests/unittests/server.scm @@ -64,11 +64,11 @@ (test #f ".db/2.db" (list-ref (rmt:send-receive-real *remotedat* *toppath* ".db/main.db" 'get-server `(,apath ,dbname)) 6)) (thread-sleep! 2) (test #f #t (rmt:general-open-connection *remotedat* *toppath* ".db/2.db")) - +(test #f #t (list? (rmt:get-servers-info *toppath*))) (test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) (test #f 1 (rmt:send-receive 'register-run #f (list keyvals "run1" "new" "n/a" "justme" #f))) ;; (print "Got here.") @@ -78,10 +78,10 @@ ;; (test #f 2 (rmt:deregister-server *remotedat* *toppath* iface port server-key dbname (test #f 2 (rmt:get-count-servers *remotedat* *toppath*)) (test #f "run2" (rmt:get-run-name-from-id 2)) - -(test #f #t (list? (rmt:get-servers-info *toppath*))) +(test #f #f (rmt:send-receive 'get-test-info-by-id 2 '(2 1))) + +(test #f #t (rmt:general-call 'update-cpuload-diskfree 2 1.5 1e6 1)) (exit) -