Index: bigmod.scm ================================================================== --- bigmod.scm +++ bigmod.scm @@ -40,11 +40,12 @@ chicken.process chicken.process-context chicken.sort chicken.string chicken.time - + chicken.module + debugprint commonmod configfmod dbmod rmtmod Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -1116,14 +1116,14 @@ (string-split patts ",")) res) #t)) ;;====================================================================== -;; '(print (string-intersperse (map cadr (hash-table-ref/default (read-config "megatest.config" \#f \#t) "disks" '"'"'("none" ""))) "\n"))' +;; '(print (string-intersperse (map cadr (hash-table-ref/default (configf:read-config "megatest.config" \#f \#t) "disks" '"'"'("none" ""))) "\n"))' (define (common:get-disks #!key (configf #f)) (hash-table-ref/default - (or configf (read-config "megatest.config" #f #t)) + (or configf (configf:read-config "megatest.config" #f #t)) "disks" '("none" ""))) ;;====================================================================== ;; return first command that exists, else #f ;; @@ -1225,16 +1225,16 @@ ;; T A R G E T S , S T A T E , S T A T U S , ;; R U N N A M E A N D T E S T P A T T ;;====================================================================== ;;====================================================================== -;; (map print (map car (hash-table->alist (read-config "runconfigs.config" #f #t)))) +;; (map print (map car (hash-table->alist (configf:read-config "runconfigs.config" #f #t)))) ;; (define (common:get-runconfig-targets #!key (configf #f)) (let ((targs (sort (map car (hash-table->alist (or configf ;; NOTE: There is no value in using runconfig:read here. - (read-config (conc *toppath* "/runconfigs.config") + (configf:read-config (conc *toppath* "/runconfigs.config") #f #t) (make-hash-table)))) string get all, else list of sections to gather ;; post-section-procs alist of section-pattern => proc, where: (proc section-name next-section-name ht curr-path) ;; apply-wildcards: #t/#f - apply vars from targets with % wildcards to all matching sections ;; -(define (read-config path ht allow-system #!key (environ-patt #f) (curr-section #f) +(define (configf:read-config path ht allow-system #!key (environ-patt #f) (curr-section #f) (sections #f) (settings (make-hash-table)) (keep-filenames #f) (post-section-procs '()) (apply-wildcards #t) (env-to-use #f)) (debug:print 9 *default-log-port* "START: " path) ;; (if *configdat* ;; (common:save-pkt `((action . read-config) @@ -326,14 +326,14 @@ ;; (T . configf)) ;; *configdat* #t add-only: #t)) (if (and (not (port? path)) (not (file-exists? path))) ;; for case where we are handed a port (begin - (debug:print-info 1 *default-log-port* "read-config - file not found " path " current path: " (current-directory)) + (debug:print-info 1 *default-log-port* "configf:read-config - file not found " path " current path: " (current-directory)) ;; WARNING: This is a risky change but really, we should not return an empty hash table if no file read? #f) ;; (if (not ht)(make-hash-table) ht)) - (let ((env-to-use (if env-to-use env-to-use (module-environment 'configfmod))) + (let (;; (env-to-use (if env-to-use env-to-use (module-environment 'configfmod))) (inp (if (string? path) (open-input-file path) path)) ;; we can be handed a port (res (if (not ht)(make-hash-table) ht)) (metapath (if (or (debug:debug-mode 9) @@ -399,11 +399,11 @@ (debug:print 2 *default-log-port* " " full-conf)) (for-each (lambda (fpath) ;; (push-directory conf-dir) (debug:print 9 *default-log-port* "Including: " full-conf) - (read-config fpath res allow-system environ-patt: environ-patt + (configf:read-config fpath res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)) all-matches)) (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings env-to-use) curr-section-name #f #f)))) @@ -419,12 +419,12 @@ (common:with-env-vars env-delta (lambda () (open-input-pipe (conc include-script " " params)))))) (debug:print '(2 9) *default-log-port* "Including from script output: " include-script) - ;; (print "We got here, calling read-config next. Port is: " new-inp-port) - (read-config new-inp-port res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames) + ;; (print "We got here, calling configf:read-config next. Port is: " new-inp-port) + (configf:read-config new-inp-port res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames) (close-input-port new-inp-port) (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings env-to-use) curr-section-name #f #f)) (begin (debug:print 0 *default-log-port* "Script not found or not exectutable: " include-script) (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings env-to-use) curr-section-name #f #f))) @@ -507,11 +507,11 @@ ;; (string-match "^.*:.*:.*$" key) ;; ;; something:something:something reserved for triggers in runconfigs )) (realval (if envar (configf:eval-string-in-environment val) val))) - (debug:print-info 6 *default-log-port* "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name) + (debug:print-info 6 *default-log-port* "configf:read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name) (if envar (safe-setenv key realval)) (debug:print 10 *default-log-port* " setting: [" curr-section-name "] " key " = " val) (hash-table-set! res curr-section-name (configf:assoc-safe-add alist key realval metadata: metapath)) (loop (configf:read-line inp res @@ -612,11 +612,11 @@ ;;====================================================================== ;;====================================================================== (define (setup) (let* ((configf (find-config "megatest.config")) - (config (if configf (read-config configf #f #t) #f))) + (config (if configf (configf:read-config configf #f #t) #f))) (if config (setenv "RUN_AREA_HOME" (pathname-directory configf))) config)) (define (safe-setenv key val) @@ -877,11 +877,11 @@ (loop (read-line)(cons inl res))))))) (data '())) (for-each (lambda (sheet-name) (let* ((dat-path (conc refdb-path "/" sheet-name ".dat")) - (ref-dat (configf:read-file dat-path #f #t)) + (ref-dat (configf:read-config dat-path #f #t)) (ref-assoc (map (lambda (key) (list key (hash-table-ref ref-dat key))) (hash-table-keys ref-dat)))) ;; (hash-table->alist ref-dat))) ;; (set! data (append data (list (list sheet-name ref-assoc)))))) @@ -945,11 +945,11 @@ (hash-table->alist data))) (define (runconfig:read fname target environ-patt) (let ((ht (make-hash-table))) (if target (hash-table-set! ht target '())) - (read-config fname ht #t environ-patt: environ-patt sections: (if target (list "default" target) #f)))) + (configf:read-config fname ht #t environ-patt: environ-patt sections: (if target (list "default" target) #f)))) ;;====================================================================== ;; Config file handling ;;====================================================================== @@ -1000,11 +1000,14 @@ (set! result (conc "#{( " cmdtype ") " cmd "}, full expansion: " fullcmd))) (if (or allow-system (not (member cmdtype '("system" "shell" "sh")))) (with-input-from-string fullcmd (lambda () - (set! result ((eval (read) env-to-use) ht)))) + (set! result (if env-to-use + ((eval (read) env-to-use) ht) + ((eval (read)) ht) + )))) (set! result (conc "#{(" cmdtype ") " cmd "}")))) (case cmdsym ((system shell scheme) (let ((delta (- (current-seconds) start-time))) (if (> delta 2) @@ -1027,11 +1030,11 @@ (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht) (if (not (null? field-names))(keys:target-set-args field-names target #f)))))) (if toppath (change-directory toppath)) (if (and toppath pathenvvar)(setenv pathenvvar toppath)) (let ((configdat (if configfile - (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f)))) + (configf:read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f)))) (if toppath (change-directory curr-dir)) (list configdat toppath configfile fname)))) ;;====================================================================== ;; Non destructive writing of config file Index: dbmod.scm ================================================================== --- dbmod.scm +++ dbmod.scm @@ -5385,11 +5385,11 @@ (if (and (common:file-exists? tconfig-file) (file-readable? tconfig-file)) (let ((link-tree-path (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree")) (old-link-tree (get-environment-variable "MT_LINKTREE"))) (if link-tree-path (setenv "MT_LINKTREE" link-tree-path)) - (let ((newtcfg (read-config tconfig-file #f #f))) ;; NOTE: Does NOT run [system ...] + (let ((newtcfg (configf:read-config tconfig-file #f #f))) ;; NOTE: Does NOT run [system ...] (hash-table-set! *testconfigs* test-name newtcfg) (if old-link-tree (setenv "MT_LINKTREE" old-link-tree) (unsetenv "MT_LINKTREE")) newtcfg)) Index: ezstepsmod.scm ================================================================== --- ezstepsmod.scm +++ ezstepsmod.scm @@ -300,11 +300,11 @@ (define (ezsteps:run-from testdat start-step-name run-one) ;;# TODO - recapture item variables, debug repeated step eval; regen logpro from test (let* ((do-update-test-state-status #f) (test-run-dir ;; (filedb:get-path *fdb* (db:test-get-rundir testdat)) ;; ) - (testconfig (read-config (conc test-run-dir "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) + (testconfig (configf:read-config (conc test-run-dir "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) (ezstepslst (hash-table-ref/default testconfig "ezsteps" '())) (run-mutex (make-mutex)) (rollup-status 0) (rollup-status-string #f) (rollup-status-sym #f) @@ -464,11 +464,11 @@ ;; the comment can be set in the step record in launch.scm ;; (define (launch:load-logpro-dat run-id test-id stepname) (let ((cname (conc stepname ".dat"))) (if (common:file-exists? cname) - (let* ((dat (read-config cname #f #f)) + (let* ((dat (configf:read-config cname #f #f)) (csvr (db:logpro-dat->csv dat stepname)) (csvt (let-values (((fmt-cell fmt-record fmt-csv) (make-format ","))) (fmt-csv (map list->csv-record csvr)))) (status (configf:lookup dat "final" "exit-status")) (msg (configf:lookup dat "final" "message"))) Index: launchmod.scm ================================================================== --- launchmod.scm +++ launchmod.scm @@ -163,11 +163,11 @@ ))))) ;; then, if runscript ran ok (or did not get called) ;; do all the ezsteps (if any) (if (or ezsteps subrun) (let* ((test-run-dir (tests:get-test-path-from-environment)) - (testconfig ;; (read-config (conc work-area "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) ;; FIXME??? is allow-system ok here? + (testconfig ;; (configf:read-config (conc work-area "/testconfig") #f #t environ-patt: "pre-launch-env-vars")) ;; FIXME??? is allow-system ok here? ;; NOTE: it is tempting to turn off force-create of testconfig but dynamic ;; ezstep names need a full re-eval here. (tests:get-testconfig test-name item-path tconfigreg #t force-create: #t)) ;; 'return-procs))) (ezstepslst (if (hash-table? testconfig) (hash-table-ref/default testconfig "ezsteps" '()) @@ -544,12 +544,12 @@ ;; NOTE: Current order is to process runconfigs *before* setting the MT_ vars. This ;; seems non-ideal but could well break stuff ;; BUG? BUG? BUG? - (let ((rconfig (full-runconfigs-read)) ;; (read-config (conc *toppath* "/runconfigs.config") #f #t sections: (list "default" target)))) - (wconfig (read-config "waivers.config" #f #t sections: `( "default" ,target )))) ;; read the waivers config if it exists + (let ((rconfig (full-runconfigs-read)) ;; (configf:read-config (conc *toppath* "/runconfigs.config") #f #t sections: (list "default" target)))) + (wconfig (configf:read-config "waivers.config" #f #t sections: `( "default" ,target )))) ;; read the waivers config if it exists ;; (setup-env-defaults (conc *toppath* "/runconfigs.config") run-id (make-hash-table) keyvals target) ;; (set-run-config-vars run-id keyvals target) ;; (db:get-target db run-id)) ;; Now have runconfigs data loaded, set environment vars (for-each (lambda (section) @@ -895,11 +895,11 @@ given-toppath: toppath pathenvvar: "MT_RUN_AREA_HOME")) (first-rundat (let ((toppath (if toppath toppath (car first-pass)))) - (read-config ;; (conc toppath "/runconfigs.config") ;; this should be converted to runconfig:read but it is non-trivial, leaving it for now. + (configf:read-config ;; (conc toppath "/runconfigs.config") ;; this should be converted to runconfig:read but it is non-trivial, leaving it for now. (conc (if (string? toppath) toppath (get-environment-variable "MT_RUN_AREA_HOME")) "/runconfigs.config") *runconfigdat* #t @@ -933,11 +933,11 @@ pathenvvar: "MT_RUN_AREA_HOME")) (runconfigdat (begin ;; this read of the runconfigs will see any adjustments made by re-reading megatest.config (for-each (lambda (kt) (setenv (car kt) (cadr kt))) key-vals) - (read-config (conc toppath "/runconfigs.config") *runconfigdat* #t ;; consider using runconfig:read some day ... + (configf:read-config (conc toppath "/runconfigs.config") *runconfigdat* #t ;; consider using runconfig:read some day ... sections: sections))) (cachefiles (launch:get-cache-file-paths areapath toppath target mtconfig)) (mtcachef (car cachefiles)) (rccachef (cdr cachefiles))) ;; trap exception due to stale NFS handle -- Error: (open-output-file) cannot open file - Stale NFS file handle: "/p/fdk/gwa/lefkowit/mtTesting/qa/primbeqa/links/p1222/11/PDK_r1.1.1/prim/clean/pcell_testgen/.runconfigs.cfg-1.6427-7d1e789cb3f62f9cde719a4865bb51b3c17ea853" - ticket 220546342 @@ -970,11 +970,11 @@ given-toppath: (get-environment-variable "MT_RUN_AREA_HOME") pathenvvar: "MT_RUN_AREA_HOME"))) (if (and cfgdat (list? cfgdat) (> (length cfgdat) 0) (hash-table? (car cfgdat))) (let* ((toppath (or (get-environment-variable "MT_RUN_AREA_HOME")(cadr cfgdat))) - (rdat (read-config (conc toppath ;; convert this to use runconfig:read! + (rdat (configf:read-config (conc toppath ;; convert this to use runconfig:read! "/runconfigs.config") *runconfigdat* #t sections: sections))) (set! *configinfo* cfgdat) (set! *configdat* (car cfgdat)) (set! *runconfigdat* rdat) (set! *toppath* toppath) @@ -1044,11 +1044,11 @@ ;; if have -append-config then read and append here (let ((cfname (args:get-arg "-append-config"))) (if (and cfname (file-readable? cfname)) - (read-config cfname *configdat* #t))) ;; values are added to the hash, no need to do anything special. + (configf:read-config cfname *configdat* #t))) ;; values are added to the hash, no need to do anything special. *toppath*))) (define (get-best-disk confdat testconfig) (let* ((disks (or (and testconfig (hash-table-ref/default testconfig "disks" #f)) @@ -2255,11 +2255,11 @@ (setenv "MT_RUN_AREA_HOME" *toppath*) (if key-vals (for-each (lambda (kt) (setenv (car kt) (cadr kt))) key-vals)) - ;; (read-config (conc *toppath* "/runconfigs.config") #f #t sections: sections)))) + ;; (configf:read-config (conc *toppath* "/runconfigs.config") #f #t sections: sections)))) (runconfig:read (conc *toppath* "/runconfigs.config") target #f)))) (if (and rundir ;; have all needed variabless (directory-exists? rundir) (file-writable? rundir)) (begin Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -1210,11 +1210,11 @@ (set! *didsomething* #t)) (pop-directory))) (if (args:get-arg "-show-config") (let ((tl (launch:setup)) - (data *configdat*)) ;; (read-config "megatest.config" #f #t))) + (data *configdat*)) ;; (configf:read-config "megatest.config" #f #t))) (push-directory *toppath*) ;; keep this one local (cond ((and (args:get-arg "-section") (args:get-arg "-var")) @@ -2575,11 +2575,11 @@ (if (args:get-arg "-syscheck") (begin (mutils:syscheck common:raw-get-remote-host-load server:get-best-guess-address - read-config) + configf:read-config) (set! *didsomething* #t))) (if (args:get-arg "-extract-skeleton") (let* ((toppath (launch:setup))) (genexample:extract-skeleton-area (args:get-arg "-extract-skeleton")) Index: runsmod.scm ================================================================== --- runsmod.scm +++ runsmod.scm @@ -2531,11 +2531,11 @@ (set! keys (keys:config-get-fields *configdat*)) ;; have enough to process -target or -reqtarg here (if (args:get-arg "-reqtarg") (let* ((runconfigf (conc *toppath* "/runconfigs.config")) ;; DO NOT EVALUATE ALL - (runconfig (read-config runconfigf #f #t environ-patt: #f))) + (runconfig (configf:read-config runconfigf #f #t environ-patt: #f))) (if (hash-table-ref/default runconfig (args:get-arg "-reqtarg") #f) (keys:target-set-args keys (args:get-arg "-reqtarg") args:arg-hash) (begin (debug:print-error 0 *default-log-port* "[" (args:get-arg "-reqtarg") "] not found in " runconfigf) Index: testsmod.scm ================================================================== --- testsmod.scm +++ testsmod.scm @@ -1075,11 +1075,11 @@ (loopa (sub1 tries-left))) (else (debug:print 0 *default-log-port* "WARNING: testconfig file does not exist: "test-configf) ;; BB: this fires #f)))) (tcfg (if testexists - (read-config test-configf #f system-allowed + (configf:read-config test-configf #f system-allowed environ-patt: (if system-allowed "pre-launch-env-vars" #f)) #f))) (if (and tcfg cache-file) (hash-table-set! tcfg "have fulldata" #t)) ;; mark this as fully read data