Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -34,13 +34,17 @@ all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut tcmt # module source files MSRCFILES = dbmod.scm dbfile.scm debugprint.scm mtargs.scm commonmod.scm \ ducttape-lib.scm pkts.scm dbi.scm autoload.scm stml2.scm + # dbmod.import.o is just a hack here mofiles/dbfile.o : mofiles/debugprint.o dbmod.import.o -mofiles/debugprint.o : mofiles/mtargs.o +mofiles/debugprint.o : mofiles/margs.o + +# +common.o : mofiles/margs.o # ftail.scm rmtmod.scm commonmod.scm removed # MSRCFILES = ducttape-lib.scm pkts.scm stml2.scm cookie.scm mutils.scm \ # mtargs.scm commonmod.scm dbmod.scm adjutant.scm ulex.scm \ # rmtmod.scm apimod.scm Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -20,10 +20,11 @@ (declare (unit common)) (declare (uses commonmod)) (declare (uses pkts)) (declare (uses dbi)) +(declare (uses margs)) (import srfi-1 srfi-69 ;; data-structures posix @@ -64,10 +65,11 @@ ;; extras ;; tcp (prefix nanomsg nmsg:) (prefix sqlite3 sqlite3:) pkts (prefix dbi dbi:) + margs ) ;; (import posix-extras pathname-expand files) (import commonmod) @@ -254,10 +256,11 @@ (and (eq? 4 chicken-release-number) (> chicken-major-version 9))))) (if resolve-pathname-broken? (define ##sys#expand-home-path pathname-expand)))) ;; (define (realpath x) (resolve-pathname (pathname-expand (or x "/dev/null")) )) +(define (realpath x)(with-input-from-pipe (conc "realpath \""x"\"") read-line)) (define (common:get-this-exe-fullpath #!key (argv (argv))) (let* ((this-script (cond ((and (> (length argv) 2) @@ -343,24 +346,10 @@ (else "FAIL"))) (define (common:logpro-exit-code->test-status exit-code) (status-sym->string (common:logpro-exit-code->status-sym exit-code))) -(defstruct remote - (hh-dat (common:get-homehost)) ;; homehost record ( addr . hhflag ) - (server-url #f) ;; (server:check-if-running *toppath*) #f)) - (server-id #f) - (server-info (if *toppath* (server:check-if-running *toppath*) #f)) - (last-server-check 0) ;; last time we checked to see if the server was alive - (connect-time (current-seconds)) - (conndat #f) - (transport *transport-type*) - (server-timeout (server:expiration-timeout)) - (force-server #f) - (ro-mode #f) - (ro-mode-checked #f)) ;; flag that indicates we have checked for ro-mode - ;; launching and hosts (defstruct host (reachable #f) (last-update 0) (last-used 0) @@ -410,53 +399,10 @@ (conc megatest-version "-" megatest-fossil-hash)) (define (common:version-signature) (conc megatest-version "-" (substring megatest-fossil-hash 0 4))) -;;====================================================================== -;; from metadat lookup MEGATEST_VERSION -;; -(define (common:get-last-run-version) ;; RADT => How does this work in send-receive function??; assume it is the value saved in some DB - (rmt:get-var "MEGATEST_VERSION")) - -(define (common:get-last-run-version-number) - (string->number - (substring (common:get-last-run-version) 0 6))) - -(define (common:set-last-run-version) - (rmt:set-var "MEGATEST_VERSION" (common:version-signature))) - -;;====================================================================== -;; postive number if megatest version > db version -;; negative number if megatest version < db version -(define (common:version-db-delta) - (- megatest-version (common:get-last-run-version-number))) - -(define (common:version-changed?) - (not (equal? (common:get-last-run-version) - (common:version-signature)))) - -(define (common:api-changed?) - (not (equal? (substring (->string megatest-version) 0 4) - (substring (conc (common:get-last-run-version)) 0 4)))) - -;;====================================================================== -;; Move me elsewhere ... -;; RADT => Why do we meed the version check here, this is called only if version misma -;; -(define (common:cleanup-db dbstruct #!key (full #f)) - (apply db:multi-db-sync - dbstruct - 'schema - 'killservers - 'adj-target - 'new2old - '(dejunk) - ) - (if (common:api-changed?) - (common:set-last-run-version))) - (define (common:snapshot-file filepath #!key (subdir ".") ) (if (file-exists? filepath) (let* ((age-sec (lambda (file) (if (file-exists? file) (- (current-seconds) (file-modification-time file)) @@ -735,19 +681,19 @@ "")))) (define (common:alist-ref/default key alist default) (or (alist-ref key alist) default)) -(define (common:low-noise-print waitval . keys) - (let* ((key (string-intersperse (map conc keys) "-" )) - (lasttime (hash-table-ref/default *common:denoise* key 0)) - (currtime (current-seconds))) - (if (> (- currtime lasttime) waitval) - (begin - (hash-table-set! *common:denoise* key currtime) - #t) - #f))) +;; (define (common:low-noise-print waitval . keys) +;; (let* ((key (string-intersperse (map conc keys) "-" )) +;; (lasttime (hash-table-ref/default *common:denoise* key 0)) +;; (currtime (current-seconds))) +;; (if (> (- currtime lasttime) waitval) +;; (begin +;; (hash-table-set! *common:denoise* key currtime) +;; #t) +;; #f))) (define (common:get-megatest-exe) (or (getenv "MT_MEGATEST") "megatest")) (define (common:read-encoded-string instr) @@ -3506,10 +3452,43 @@ (debug:print 0 *default-log-port* "joining threads failed. exn=" exn) #t) ;; just ignore it, it might have died in the meantime so joining it will throw an exception (thread-join! thread)) ))) (hash-table-keys *common:thread-punchlist*))) + +;;====================================================================== +;; L O G G I N G D B +;;====================================================================== + +(define (open-logging-db) + (let* ((dbpath (conc (if *toppath* (conc *toppath* "/") "") "logging.db")) ;; fname) + (dbexists (common:file-exists? dbpath)) + (db (sqlite3:open-database dbpath)) + (handler (sqlite3:make-busy-timeout (if (args:get-arg "-override-timeout") + (string->number (args:get-arg "-override-timeout")) + 136000)))) ;; 136000))) + (sqlite3:set-busy-handler! db handler) + (if (not dbexists) + (begin + (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log (id INTEGER PRIMARY KEY,event_time TIMESTAMP DEFAULT (strftime('%s','now')),logline TEXT,pwd TEXT,cmdline TEXT,pid INTEGER);") + (db:set-sync db) ;; (sqlite3:execute db (conc "PRAGMA synchronous = 0;")) + )) + db)) + +(define (db:log-local-event . loglst) + (let ((logline (apply conc loglst))) + (db:log-event logline))) + +(define (db:log-event logline) + (let ((db (open-logging-db))) + (sqlite3:execute db "INSERT INTO log (logline,pwd,cmdline,pid) VALUES (?,?,?,?);" + logline + (current-directory) + (string-intersperse (argv) " ") + (current-process-id)) + (sqlite3:finalize! db) + logline)) ;;====================================================================== ;; (define *common:telemetry-log-state* 'startup) ;; (define *common:telemetry-log-socket* #f) ;; Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -1288,43 +1288,10 @@ ;; (db (dbr:dbdat-dbh dbdat)) ;; (res '()) ;; (blocks '())) ;; a block is an archive chunck that can be added too if there is space ;; (sqlite3:for-each-row #f) -;;====================================================================== -;; L O G G I N G D B -;;====================================================================== - -(define (open-logging-db) - (let* ((dbpath (conc (if *toppath* (conc *toppath* "/") "") "logging.db")) ;; fname) - (dbexists (common:file-exists? dbpath)) - (db (sqlite3:open-database dbpath)) - (handler (sqlite3:make-busy-timeout (if (args:get-arg "-override-timeout") - (string->number (args:get-arg "-override-timeout")) - 136000)))) ;; 136000))) - (sqlite3:set-busy-handler! db handler) - (if (not dbexists) - (begin - (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log (id INTEGER PRIMARY KEY,event_time TIMESTAMP DEFAULT (strftime('%s','now')),logline TEXT,pwd TEXT,cmdline TEXT,pid INTEGER);") - (db:set-sync db) ;; (sqlite3:execute db (conc "PRAGMA synchronous = 0;")) - )) - db)) - -(define (db:log-local-event . loglst) - (let ((logline (apply conc loglst))) - (db:log-event logline))) - -(define (db:log-event logline) - (let ((db (open-logging-db))) - (sqlite3:execute db "INSERT INTO log (logline,pwd,cmdline,pid) VALUES (?,?,?,?);" - logline - (current-directory) - (string-intersperse (argv) " ") - (current-process-id)) - (sqlite3:finalize! db) - logline)) - ;;====================================================================== ;; D B U T I L S ;;====================================================================== ;;====================================================================== @@ -5066,5 +5033,21 @@ ) ) 0) +;; PULLED FROM COMMON + +;;====================================================================== +;; +(define (common:cleanup-db dbstruct #!key (full #f)) + (apply db:multi-db-sync + dbstruct + 'schema + 'killservers + 'adj-target + 'new2old + '(dejunk) + ) + (if (common:api-changed?) + (common:set-last-run-version))) + Index: dbfile.scm ================================================================== --- dbfile.scm +++ dbfile.scm @@ -17,11 +17,11 @@ ;; along with Megatest. If not, see . ;;====================================================================== (declare (unit dbfile)) -;; (declare (uses debugprint)) +(declare (uses debugprint)) (declare (uses commonmod)) (module dbfile * @@ -31,10 +31,11 @@ chicken.base chicken.condition chicken.file chicken.file.posix chicken.io + chicken.pathname chicken.port chicken.process chicken.process-context.posix chicken.sort chicken.time @@ -53,10 +54,11 @@ system-information ;; files ;; ports commonmod + debugprint ) ;; (import debugprint) ;;====================================================================== @@ -486,21 +488,20 @@ ;; (db:multi-db-sync subdb 'old2new)) ;; migrate data from megatest.db automatically tmpdb)) (define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 50)) - - (let* ((busy-file (conc fname"-journal")) - (delay-time (* (- 51 tries-left) 1.1)) - (write-access (file-write-access? fname)) - (dir-access (file-write-access? (pathname-directory fname))) - (retry (lambda () - (thread-sleep! delay-time) - (if (> tries-left 0) - (dbfile:cautious-open-database fname init-proc (- tries-left 1)))))) + (let* ((busy-file (conc fname"-journal")) + (delay-time (* (- 51 tries-left) 1.1)) + (write-access (file-writable? fname)) + (dir-access (file-writable? (pathname-directory fname))) + (retry (lambda () + (thread-sleep! delay-time) + (if (> tries-left 0) + (dbfile:cautious-open-database fname init-proc (- tries-left 1)))))) (assert (>= tries-left 0) (conc "FATAL: too many attempts in dbfile:cautious-open-database of "fname", giving up.")) - (if (and (file-write-access? fname) + (if (and (file-writable? fname) (file-exists? busy-file)) (begin (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file " busy-file " exists, trying again in few seconds.") (thread-sleep! 1) (if (eq? tries-left 2) @@ -542,11 +543,11 @@ (retry)) (exn () (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn)) (retry))))) - #;(if (file-write-access? fname) + #;(if (file-writable? fname) (dbfile:simple-file-release-lock lock-file)) result)))) (define (dbfile:brute-force-salvage-db fname) (let* ((backupfname (conc fname"-"(current-process-id)".bak")) @@ -562,11 +563,11 @@ (retry (lambda () (thread-sleep! delay-time) (if (> tries-left 0) (dbfile:cautious-open-database fname init-proc (- tries-left 1)))))) (assert (>= tries-left 0) (conc "FATAL: too many attempts in dbfile:cautious-open-database of "fname", giving up.")) - (if (and (file-write-access? fname) (not (dbfile:simple-file-lock lock-file expire-time: 3))) + (if (and (file-writable? fname) (not (dbfile:simple-file-lock lock-file expire-time: 3))) (begin (dbfile:print-err "INFO: dbfile:cautious-open-database: lock file " lock-file " exists, trying again in few seconds.") (thread-sleep! 1) (if (eq? tries-left 2) (begin @@ -593,11 +594,11 @@ (retry)) (exn () (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn)) (retry))))) - (if (file-write-access? fname) + (if (file-writable? fname) (dbfile:simple-file-release-lock lock-file) ) result)))) Index: margs.scm ================================================================== --- margs.scm +++ margs.scm @@ -17,14 +17,23 @@ (declare (unit margs)) ;; (declare (uses common)) -(import chicken.process-context - srfi-1 - srfi-69 - ) +(module margs + * + +(import + scheme + chicken.base + chicken.process-context + srfi-1 + srfi-69 + + ) + +(define help #f) (define args:arg-hash (make-hash-table)) (define (args:get-arg arg . default) (if (null? default) @@ -94,5 +103,6 @@ (define (args:print-args remargs arg-hash) (print "ARGS: " remargs) (for-each (lambda (arg) (print " " arg " " (hash-table-ref/default arg-hash arg #f))) (hash-table-keys arg-hash))) +) Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -1106,5 +1106,35 @@ #;(set-functions rmt:send-receive remote-server-url-set! http-transport:close-connections remote-conndat-set! debug:print debug:print-info remote-ro-mode remote-ro-mode-set! remote-ro-mode-checked-set! remote-ro-mode-checked) + +;; PULLED FROM COMMON + +;;====================================================================== +;; from metadat lookup MEGATEST_VERSION +;; +(define (common:get-last-run-version) ;; RADT => How does this work in send-receive function??; assume it is the value saved in some DB + (rmt:get-var "MEGATEST_VERSION")) + +(define (common:get-last-run-version-number) + (string->number + (substring (common:get-last-run-version) 0 6))) + +(define (common:set-last-run-version) + (rmt:set-var "MEGATEST_VERSION" (common:version-signature))) + +;;====================================================================== +;; postive number if megatest version > db version +;; negative number if megatest version < db version +(define (common:version-db-delta) + (- megatest-version (common:get-last-run-version-number))) + +(define (common:version-changed?) + (not (equal? (common:get-last-run-version) + (common:version-signature)))) + +(define (common:api-changed?) + (not (equal? (substring (->string megatest-version) 0 4) + (substring (conc (common:get-last-run-version)) 0 4)))) + Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -65,10 +65,24 @@ (import commonmod) (include "common_records.scm") (include "db_records.scm") +(defstruct remote + (hh-dat (common:get-homehost)) ;; homehost record ( addr . hhflag ) + (server-url #f) ;; (server:check-if-running *toppath*) #f)) + (server-id #f) + (server-info (if *toppath* (server:check-if-running *toppath*) #f)) + (last-server-check 0) ;; last time we checked to see if the server was alive + (connect-time (current-seconds)) + (conndat #f) + (transport *transport-type*) + (server-timeout (server:expiration-timeout)) + (force-server #f) + (ro-mode #f) + (ro-mode-checked #f)) ;; flag that indicates we have checked for ro-mode + (define (server:make-server-url hostport) (if (not hostport) #f (conc "http://" (car hostport) ":" (cadr hostport))))