Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -43,11 +43,11 @@ (include "common_records.scm") (define (remove-files filespec) (let ((files (glob filespec))) - (for-each delete-file files))) + (for-each delete-file* files))) (define (stop-the-train) (thread-start! (make-thread (lambda () (let loop () (if (and *toppath* Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -3837,11 +3837,12 @@ (let* ((mtdb-path (conc *toppath* "/.mtdb/main.db")) (target (args:get-arg "-target")) (commondat (dboard:commondat-make))) (if target (begin - (args:remove-arg-from-ht "-target") + (hash-table-delete! args:arg-hash "-target") ;; workaround for the following commented out function + ;; (args:remove-arg-from-ht "-target") This function is in mtargs/mtargs.scm, but it's in an egg that is not in the current build of chicken 4.10 (dboard:commondat-target-set! commondat target) ) ) (if (not (launch:setup)) Index: dbfile.scm ================================================================== --- dbfile.scm +++ dbfile.scm @@ -494,11 +494,11 @@ (sqlite3:execute db (conc "PRAGMA journal_mode = "journal-mode";"))) (if (and init-proc (or force-init (not db-exists))) (init-proc db)) db)) - expire-time: 5) + expire-time: 30) (begin (if (file-exists? fname ) (let ((db (sqlite3:open-database fname))) ;; pragmas synchronous not needed because this db is used read-only ;; (sqlite3:execute db (conc "PRAGMA synchronous = "mode";") @@ -1572,11 +1572,11 @@ #f (with-input-from-file fname (lambda () (equal? key-string (read-line))))) (begin - (dbfile:print-err "dbfile:simple-file-lock created " fname " but it was gone 3 seconds later") + (dbfile:print-err "dbfile:simple-file-lock created " fname " but it was gone 0.25 seconds later") #f ) ) ) ) Index: dbmod.scm ================================================================== --- dbmod.scm +++ dbmod.scm @@ -508,11 +508,11 @@ ;; done (debug:print 2 *default-log-port* "Attaching "destdbfile" as auxdb") (handle-exceptions exn (begin - (debug:print 0 "ATTACH failed, exiting. exn="(condition->list exn)) + (debug:print 0 *default-log-port* "ATTACH failed, exiting. exn="(condition->list exn)) (exit 1)) (sqlite3:execute dbh (conc "ATTACH '"destdbfile"' AS auxdb;"))) (for-each (lambda (table) (let* ((dummy (debug:print 2 *default-log-port* "Doing table " table)) Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -934,11 +934,11 @@ (let ((tmpfile (conc fulldir "/.megatest.cfg." (current-seconds))) (targfile (conc fulldir "/.megatest.cfg-" megatest-version "-" megatest-fossil-hash)) (rconfig (conc fulldir "/.runconfig." megatest-version "-" megatest-fossil-hash))) (if (common:file-exists? rconfig) ;; only cache megatest.config AFTER runconfigs has been cached (begin - (debug:print-info 0 *default-log-port* "Caching megatest.config in " tmpfile) + (debug:print-info 2 *default-log-port* "Caching megatest.config in " tmpfile) (if (not (common:in-running-test?)) (configf:write-alist *configdat* tmpfile)) (system (conc "ln -sf " tmpfile " " targfile)))) ))) (debug:print-info 1 *default-log-port* "No linktree yet, no caching configs."))))) Index: megatest-version.scm ================================================================== --- megatest-version.scm +++ megatest-version.scm @@ -18,6 +18,6 @@ ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. ;; (declare (unit megatest-version)) -(define megatest-version 1.8028) +(define megatest-version 1.8031) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -111,11 +111,17 @@ ;; load the ~/.megatestrc file, put (use trace)(trace-call-sites #t)(trace function-you-want-to-trace) in this file ;; (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.megatestrc"))) (if (common:file-exists? debugcontrolf) - (load debugcontrolf))) + (begin + ;; for some reason, debug:print does not work here. Had to use print. + (print (conc "WARNING: loading " debugcontrolf)) + (load debugcontrolf) + ) + ) +) ;; usage logging, careful with this, it is not designed to deal with all real world challenges! ;; (if (and *usage-log-file* (file-write-access? *usage-log-file*)) @@ -2630,29 +2636,12 @@ 'old2new ) (set! *didsomething* #t))) (if (args:get-arg "-import-sexpr") - (let*( - (toppath (launch:setup)) - (tmppath (common:make-tmpdir-name toppath ""))) - (if (file-exists? (conc toppath "/.mtdb")) - (if (args:get-arg "-remove-dbs") - (let* ((dbfiles (conc toppath "/.mtdb/* " tmppath "/*"))) - (debug:print 0 *default-log-port* "Removing db files: " dbfiles) - (system (conc "rm -rvf " dbfiles)) - ) - (begin - (debug:print 0 *default-log-port* "ERROR: Cannot import sexpr with an existing DB present.") - (debug:print 0 *default-log-port* "Add '-remove-dbs all' to remove the current Megatest DBs.") - (set! *didsomething* #t) - (exit) - ) - ) - (debug:print 0 *default-log-port* "Did not find " (conc toppath "/.mtdb")) - ) - (db:setup) + (begin + (launch:setup) (rmt:import-sexpr (args:get-arg "-import-sexpr")) (set! *didsomething* #t))) (if (args:get-arg "-sync-to-megatest.db") (let* ((duh (launch:setup)) Index: mtargs/mtargs.scm ================================================================== --- mtargs/mtargs.scm +++ mtargs/mtargs.scm @@ -20,10 +20,11 @@ ( arg-hash get-arg get-arg-number get-arg-from + remove-arg-from-ht get-args usage print-args any-defined? ) @@ -64,10 +65,14 @@ (define (get-arg-from ht arg . default) (if (null? default) (hash-table-ref/default ht arg #f) (hash-table-ref/default ht arg (car default)))) + +(define (remove-arg-from-ht arg) + (hash-table-delete! arg-hash arg) +) (define (get-args args params switches arg-hash num-needed) (let* ((numtargs (length args)) (adj-num-needed (if num-needed (+ num-needed 2) #f))) (if (< numtargs (if adj-num-needed adj-num-needed 2)) Index: portlogger.scm ================================================================== --- portlogger.scm +++ portlogger.scm @@ -63,11 +63,12 @@ (import debugprint dbmod) ;; lsof -i (define (portlogger:open-db fname) - (let* ((avail (tasks:wait-on-journal fname 5 remove: #t)) ;; wait up to about 10 seconds for the journal to go away + (let* (;; (avail (tasks:wait-on-journal fname 5 remove: #t)) ;; wait up to about 10 seconds for the journal to go away + (avail #t) (exists (file-exists? fname)) (db (if avail (sqlite3:open-database fname) (begin (system (conc "rm -f " fname)) @@ -92,12 +93,12 @@ fail_count INTEGER DEFAULT 0, update_time TIMESTAMP DEFAULT (strftime('%s','now')) );") db)) (define (portlogger:open-run-close proc . params) - (let* ((fname (conc "/tmp/." (current-user-name) "-portlogger.db")) - (avail (tasks:wait-on-journal fname 10))) ;; wait up to about 10 seconds for the journal to go away + (let* ((fname (conc "/tmp/." (current-user-name) "-portlogger.db"))) + ;; (avail (tasks:wait-on-journal fname 10))) ;; wait up to about 10 seconds for the journal to go away (handle-exceptions exn (begin ;; (release-dot-lock fname) (debug:print-error 0 *default-log-port* "portlogger:open-run-close failed. " proc " " params) Index: rmtmod.scm ================================================================== --- rmtmod.scm +++ rmtmod.scm @@ -89,21 +89,28 @@ (tests-data (alist-ref "data" all-dat equal?)) (run-meta (alist-ref "meta" all-dat equal?)) (run-id (string->number (alist-ref "id" run-meta equal?)))) (rmt:insert-run run-id target runname run-meta) - (for-each - (lambda (test-dat) - (let* ((test-id (car test-dat)) + (if (list? tests-data) + (begin + (debug:print 0 *default-log-port* "import-run: inserting " (length tests-data) " tests") + (for-each + (lambda (test-dat) + (let* ((test-id (car test-dat)) (test-rec (cdr test-dat))) - (rmt:insert-test run-id test-rec))) - tests-data))) + (rmt:insert-test run-id test-rec))) + tests-data) + ) + (debug:print 0 *default-log-port* "import-run: run has no tests") + ) + ) +) ;; insert run if not there, return id either way (define (rmt:insert-run run-id target runname run-meta) ;; look for id, return if found - (debug:print 0 *default-log-port* "Insert run: "target"/"runname) (let* ((runs (rmtmod:send-receive 'simple-get-runs #f ;; runpatt count offset target last-update) (list runname #f #f target #f)))) (if (null? runs) (begin @@ -110,18 +117,27 @@ (debug:print 0 *default-log-port* "inserting run for runname " runname " target " target) (rmtmod:send-receive 'insert-run #f (list run-id target runname run-meta)) ) (begin (debug:print 0 *default-log-port* "Found run-id " (simple-run-id (car runs)) " for runname " runname " target " target) - (simple-run-id (car runs) - ) + (simple-run-id (car runs)) )))) (define (rmt:insert-test run-id test-rec) (let* ((testname (alist-ref "testname" test-rec equal?)) - (item-path (alist-ref "item_path" test-rec equal?))) - (rmtmod:send-receive 'insert-test run-id test-rec))) + (item-path (alist-ref "item_path" test-rec equal?)) + (test-id (rmt:get-test-id run-id testname item-path)) + ) + (if test-id + (debug:print 0 *default-log-port* "test "testname"/"item-path " already exists in run-id " run-id) + (begin + (debug:print 0 *default-log-port* " Insert test in run "run-id": "testname"/"item-path) + (rmtmod:send-receive 'insert-test run-id test-rec) + ) + ) + ) +) ;;====================================================================== ;; T E S T S ;;====================================================================== Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -2918,11 +2918,11 @@ (fld (car key)) (val (configf:lookup test-conf "test_meta" fld))) ;; (debug:print 5 *default-log-port* "idx: " idx " fld: " fld " val: " val) (if (and val (not (equal? (vector-ref currrecord idx) val))) (begin - (debug:print 0 *default-log-port* "Updating " test-name " " fld " to " val) + (debug:print 2 *default-log-port* "Updating " test-name " " fld " to " val) (rmt:testmeta-update-field test-name fld val))))) '(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9)("jobgroup" 10))))) ;; find tests with matching tags, tagpatt is a string "tagpatt1,tagpatt2%, ..." ;; Index: tcp-transportmod.scm ================================================================== --- tcp-transportmod.scm +++ tcp-transportmod.scm @@ -532,11 +532,12 @@ (result '())) (if (null? servrs) (reverse result) (let* ((servdat (car servrs))) (match servdat - ((host port startseconds server-id servinfofile) + ((host port startseconds server-id pid dbfilename servinfofile) + (debug:print-info 0 *default-log-port* "Good servinfo file: " servdat) (let* ((ping-res (tt:timed-ping host port server-id)) (good-ping (match ping-res ((result . ping-time) (not result)) ;; we couldn't reach the server or it was not a megatest server (else #f))) ;; the ping failed completely? @@ -546,10 +547,11 @@ (if keep-srv (loop (cdr servrs) host (cons servdat result)) (begin + ;; (debug:print-info 0 *default-log-port* "good-ping: " good-ping " same-host: " same-host "keep-srv: " keep-srv) (handle-exceptions exn (debug:print-info 0 *default-log-port* "Error removing server info file: "servinfofile", " (condition->list exn)) (delete-file* servinfofile)) @@ -716,11 +718,11 @@ ;; filter the files here by looking in processes table (if we are not main.db) ;; and or look at the time stamp on the servinfo file, a running server will ;; touch the file every minute (again, this will only apply for main.db) (for-each (lambda (fname) (let* ((age (- (current-seconds)(file-modification-time fname)))) - (if (> age 200) ;; can't trust it if over 200 seconds old + (if (> age (tt-server-timeout-param)) ;; can't trust it if over server timeout old. (begin (debug:print 0 *default-log-port* "WARNING: removing stale servinfo file "fname", it is "age" seconds old") (handle-exceptions exn (debug:print 0 *default-log-port* "WARNING: error attempting to remove stale servinfo file "fname) @@ -728,17 +730,15 @@ (set! goodfiles (cons fname goodfiles))))) sfiles) goodfiles)) ;; given a path to a server info file return: host port startseconds server-id pid dbfname logf -;; example of what it's looking for in the log file: +;; example of what it's looking for in the file: ;; SERVER STARTED: 10.38.175.67:50216 AT 1616502350.0 server-id: 4907e90fc55c7a09694e3f658c639cf4 ;; (define (tt:server-get-info logf) (let ((server-rx (regexp "^SERVER STARTED: (\\S+):(\\d+) AT ([\\d\\.]+) server-id: (\\S+) pid: (\\d+) dbfname: (\\S+)")) ;; SERVER STARTED: host:port AT timesecs server id - (dbprep-rx (regexp "^SERVER: dbprep")) - (dbprep-found 0) (bad-dat (list #f #f #f #f #f #f logf))) (let ((fdat (handle-exceptions exn (begin ;; BUG, TODO: add err checking, for now blanket ignore the errors? @@ -749,13 +749,11 @@ (if (null? fdat) ;; bad data, return bad-dat bad-dat (let loop ((inl (car fdat)) (tail (cdr fdat)) (lnum 0)) - (let ((mlst (string-match server-rx inl)) - (dbprep (string-match dbprep-rx inl))) - (if dbprep (set! dbprep-found 1)) + (let ((mlst (string-match server-rx inl))) (if (not mlst) (if (> lnum 500) ;; give up if more than 500 lines of server log read bad-dat (if (null? tail) bad-dat @@ -913,18 +911,20 @@ (debug:print 2 *default-log-port* "setup-listener-portlogger got port " port) (handle-exceptions exn (if (< port 65535) (begin + (debug:print 0 *default-log-port* "setup-listener-portlogger: exception finding port. Retrying") (portlogger:open-run-close portlogger:set-failed port) (thread-sleep! 0.25) (setup-listener-portlogger uconn)) (begin - (debug:print 0 *default-log-port* "setup-listener-portlogger: could not get a port") + (assert #t "setup-listener-portlogger: could not get a port") #f ) ) + (debug:print 2 *default-log-port* "setup-listener-portlogger: got port " port) (connect-listener uconn port)))) (define (connect-listener uconn port) ;; (tcp-listener-socket LISTENER)(socket-name so) ;; sockaddr-address, sockaddr-port, sockaddr->string Index: tests.scm ================================================================== --- tests.scm +++ tests.scm @@ -1638,11 +1638,11 @@ (if (and testexists cache-file (file-write-access? cache-path) allow-write-cache) (let ((tpath (conc cache-path "/.testconfig"))) - (debug:print-info 1 *default-log-port* "Caching testconfig for " test-name " in " tpath) + (debug:print-info 2 *default-log-port* "Caching testconfig for " test-name " in " tpath) (if (and tcfg (not (common:in-running-test?))) (configf:write-alist tcfg tpath)))) tcfg)))))) ;; sort tests by priority and waiton