Index: configf-inc.scm ================================================================== --- configf-inc.scm +++ configf-inc.scm @@ -113,11 +113,11 @@ " extra)))")) ((get g) (let* ((parts (string-split cmd)) (sect (car parts)) (var (cadr parts))) - (conc "(lambda (ht)(config-lookup ht \"" sect "\" \"" var "\"))"))) + (conc "(lambda (ht)(configf:lookup ht \"" sect "\" \"" var "\"))"))) ((runconfigs-get rget) (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))")) ;; ((rget) (conc "(lambda (ht)(runconfigs-get ht \"" cmd "\"))")) (else "(lambda (ht)(print \"ERROR\") \"ERROR\")")))) ;; (print "fullcmd=" fullcmd) (handle-exceptions @@ -455,11 +455,11 @@ ;; if a continued line (configf:cont-ln-rx ( x whsp val ) (let ((alist (hash-table-ref/default res curr-section-name '()))) (if var-flag ;; if set to a string then we have a continued var (let ((newval (conc - (config-lookup res curr-section-name var-flag) "\n" + (configf:lookup res curr-section-name var-flag) "\n" ;; trim lead from the incoming whsp to support some indenting. (if lead (string-substitute (regexp lead) "" whsp) "") val))) @@ -490,11 +490,11 @@ (let ((configdat (if configfile (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f)))) (if toppath (change-directory curr-dir)) (list configdat toppath configfile fname)))) -(define (config-lookup cfgdat section var) +#;(define (configf:lookup cfgdat section var) (if (hash-table? cfgdat) (let ((sectdat (hash-table-ref/default cfgdat section '()))) (if (null? sectdat) #f (let ((match (assoc var sectdat))) @@ -511,17 +511,16 @@ ;; (configf:var-is? cfgdat "foo" "var" "yes") => #t ;; (define (configf:var-is? cfgdat section var expected-val) (equal? (configf:lookup cfgdat section var) expected-val)) -(define configf:lookup config-lookup) (define configf:read-file read-config) ;; safely look up a value that is expected to be a number, return ;; a default (#f unless provided) ;; -(define (configf:lookup-number cfdat section varname #!key (default #f)) +#;(define (configf:lookup-number cfdat section varname #!key (default #f)) (let* ((val (configf:lookup *configdat* section varname)) (res (if val (string->number (string-substitute "\\s+" "" val #t)) #f))) (cond @@ -533,11 +532,11 @@ (let ((sectdat (hash-table-ref/default cfgdat section '()))) (if (null? sectdat) '() (map car sectdat)))) -(define (configf:get-section cfgdat section) +#;(define (configf:get-section cfgdat section) (hash-table-ref/default cfgdat section '())) (define (configf:set-section-var cfgdat section var val) (let ((sectdat (configf:get-section cfgdat section))) (hash-table-set! cfgdat section @@ -649,11 +648,11 @@ (set! new hed) ;; will append this at the bottom of the loop (set! secname section-name) )) ;; No need to process key cmd, let it fall though to key val (configf:key-val-pr ( x key val ) - (let ((newval (config-lookup indat secname key))) ;; was sec, bug or correct? + (let ((newval (configf:lookup indat secname key))) ;; was sec, bug or correct? ;; can handle newval == #f here => that means key is removed (cond ((equal? newval val) (set! res (append res (list hed)))) ((not newval) ;; key has been removed @@ -675,11 +674,11 @@ (lambda (section) (let ((sdat '()) ;; append needed bits here (svars (configf:section-vars indat section))) (for-each (lambda (var) - (let ((val (config-lookup refdat section var))) + (let ((val (configf:lookup refdat section var))) (if (not val) ;; this one is new (begin (if (null? sdat)(set! sdat (list (conc "[" section "]")))) (set! sdat (append sdat (list (conc var " " val)))))))) svars) Index: configfmod.scm ================================================================== --- configfmod.scm +++ configfmod.scm @@ -30,35 +30,35 @@ ;; (import commonmod) ;; (use (prefix ulex ulex:)) (include "common_records.scm") -;; (define (configf:lookup cfgdat section var) -;; (if (hash-table? cfgdat) -;; (let ((sectdat (hash-table-ref/default cfgdat section '()))) -;; (if (null? sectdat) -;; #f -;; (let ((match (assoc var sectdat))) -;; (if match ;; (and match (list? match)(> (length match) 1)) -;; (cadr match) -;; #f)) -;; )) -;; #f)) -;; -;; (define (configf:get-section cfgdat section) -;; (hash-table-ref/default cfgdat section '())) -;; -;; ;; safely look up a value that is expected to be a number, return -;; ;; a default (#f unless provided) -;; ;; -;; (define (configf:lookup-number cfgdat section varname #!key (default #f)) -;; (let* ((val (configf:lookup cfgdat section varname)) -;; (res (if val -;; (string->number (string-substitute "\\s+" "" val #t)) -;; #f))) -;; (cond -;; (res res) -;; (val (debug:print 0 *default-log-port* "ERROR: no number found for [" section "], " varname ", got: " val)) -;; (else default)))) -;; -;; +(define (configf:lookup cfgdat section var) + (if (hash-table? cfgdat) + (let ((sectdat (hash-table-ref/default cfgdat section '()))) + (if (null? sectdat) + #f + (let ((match (assoc var sectdat))) + (if match ;; (and match (list? match)(> (length match) 1)) + (cadr match) + #f)) + )) + #f)) + +(define (configf:get-section cfgdat section) + (hash-table-ref/default cfgdat section '())) + +;; safely look up a value that is expected to be a number, return +;; a default (#f unless provided) +;; +(define (configf:lookup-number cfgdat section varname #!key (default #f)) + (let* ((val (configf:lookup cfgdat section varname)) + (res (if val + (string->number (string-substitute "\\s+" "" val #t)) + #f))) + (cond + (res res) + (val (debug:print 0 *default-log-port* "ERROR: no number found for [" section "], " varname ", got: " val)) + (else default)))) + + ) Index: items-inc.scm ================================================================== --- items-inc.scm +++ items-inc.scm @@ -113,11 +113,11 @@ #f))) res))) ;; Nope, not now, return null as of 6/6/2011 (define (items:check-valid-items class item) - (let ((valid-values (let ((s (config-lookup *configdat* "validvalues" class))) + (let ((valid-values (let ((s (configf:lookup *configdat* "validvalues" class))) (if s (string-split s) #f)))) (if valid-values (if (member item valid-values) item #f) item))) Index: launch-inc.scm ================================================================== --- launch-inc.scm +++ launch-inc.scm @@ -1232,11 +1232,11 @@ (cons 1 (conc *toppath* "/runs")) (loop (car tail) (cdr tail)))))))))))))) ;; the code creates the necessary directories if it does not exist and returns the path. (define (launch:test-copy test-src-path test-path) - (let* ((ovrcmd (let ((cmd (config-lookup *configdat* "setup" "testcopycmd"))) + (let* ((ovrcmd (let ((cmd (configf:lookup *configdat* "setup" "testcopycmd"))) (if cmd ;; substitute the TEST_SRC_PATH and TEST_TARG_PATH (string-substitute "TEST_TARG_PATH" test-path (string-substitute "TEST_SRC_PATH" test-src-path cmd #t) #t) #f))) @@ -1285,11 +1285,11 @@ (toptest-path (conc disk-path (if contour (conc "/" contour) "") "/" testtop-base)) (test-path (conc disk-path (if contour (conc "/" contour) "") "/" test-base)) ;; ensure this exists first as links to subtests must be created there (linktree (common:get-linktree)) - ;; WAS: (let ((rd (config-lookup *configdat* "setup" "linktree"))) + ;; WAS: (let ((rd (configf:lookup *configdat* "setup" "linktree"))) ;; (if rd rd (conc *toppath* "/runs")))) ;; which seems wrong ... (lnkbase (conc linktree (if contour (conc "/" contour) "") "/" target "/" runname)) (lnkpath (conc lnkbase "/" testname)) @@ -1496,23 +1496,23 @@ ;; for tconfig, why do we allow fallback to test-conf? (tconfig (or (tests:get-testconfig test-name item-path tregistry #t force-create: #t) (begin (debug:print 0 *default-log-port* "WARNING: falling back to pre-calculated testconfig. This is likely not desired.") test-conf))) ;; force re-read now that all vars are set - (useshell (let ((ush (config-lookup *configdat* "jobtools" "useshell"))) + (useshell (let ((ush (configf:lookup *configdat* "jobtools" "useshell"))) (if ush (if (equal? ush "no") ;; must use "no" to NOT use shell #f ush) #t))) ;; default is yes - (runscript (config-lookup tconfig "setup" "runscript")) + (runscript (configf:lookup tconfig "setup" "runscript")) (ezsteps (> (length (hash-table-ref/default tconfig "ezsteps" '())) 0)) ;; don't send all the steps, could be big, just send a flag (subrun (> (length (hash-table-ref/default tconfig "subrun" '())) 0)) ;; send a flag to process a subrun - ;; (diskspace (config-lookup tconfig "requirements" "diskspace")) - ;; (memory (config-lookup tconfig "requirements" "memory")) - ;; (hosts (config-lookup *configdat* "jobtools" "workhosts")) ;; I'm pretty sure this was never completed - (remote-megatest (config-lookup *configdat* "setup" "executable")) + ;; (diskspace (configf:lookup tconfig "requirements" "diskspace")) + ;; (memory (configf:lookup tconfig "requirements" "memory")) + ;; (hosts (configf:lookup *configdat* "jobtools" "workhosts")) ;; I'm pretty sure this was never completed + (remote-megatest (configf:lookup *configdat* "setup" "executable")) (run-time-limit (or (configf:lookup tconfig "requirements" "runtimelim") (configf:lookup *configdat* "setup" "runtimelim"))) ;; FIXME SOMEDAY: not good how this is so obtuse, this hack is to ;; allow running from dashboard. Extract the path ;; from the called megatest and convert dashboard @@ -1524,11 +1524,11 @@ (case (string->symbol exe) ((dboard) "../megatest") ((mtest) "../megatest") ((dashboard) "megatest") (else exe))))) - (launcher (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools" "launcher")) + (launcher (common:get-launcher *configdat* test-name item-path)) ;; (configf:lookup *configdat* "jobtools" "launcher")) (test-sig (conc (common:get-testsuite-name) ":" test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path (work-area #f) (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all (diskpath #f) (cmdparms #f) Index: megamod.scm ================================================================== --- megamod.scm +++ megamod.scm @@ -26,11 +26,11 @@ ;; ;; (declare (uses rmtmod)) ;; (declare (uses commonmod)) ;; (declare (uses apimod)) ;; (declare (uses archivemod)) ;; (declare (uses clientmod)) -;; (declare (uses configfmod)) +(declare (uses configfmod)) ;; (declare (uses dbmod)) ;; (declare (uses dcommonmod)) ;; (declare (uses envmod)) ;; (declare (uses ezstepsmod)) ;; (declare (uses itemsmod)) @@ -106,11 +106,11 @@ ;; (import apimod) ;; (import archivemod) ;; (import clientmod) ;; (import commonmod) -;; (import configfmod) +(import configfmod) ;; (import dbmod) ;; (import dcommonmod) ;; (import envmod) ;; (import ezstepsmod) ;; (import ftail) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -51,10 +51,12 @@ (import commonmod) (declare (uses rmtmod)) (import rmtmod) (declare (uses dbmod)) (import dbmod) +(declare (uses configfmod)) +(import configfmod) (declare (uses megamod)) (import megamod) ;; (declare (uses tdb)) ;; (declare (uses mt)) Index: runs-inc.scm ================================================================== --- runs-inc.scm +++ runs-inc.scm @@ -224,11 +224,11 @@ );; obviously haven't had any work to do for a while (else 0))) (let* ((num-running (rmt:get-count-tests-running run-id)) (num-running-in-jobgroup (rmt:get-count-tests-running-in-jobgroup run-id jobgroup)) - (job-group-limit (let ((jobg-count (config-lookup *configdat* "jobgroups" jobgroup))) + (job-group-limit (let ((jobg-count (configf:lookup *configdat* "jobgroups" jobgroup))) (if (string? jobg-count) (string->number jobg-count) jobg-count)))) (if (> (+ num-running num-running-in-jobgroup) 0) (runs:inc-can-run-more-tests-count runsdat)) ;; (set! *runs:can-run-more-tests-count* (+ *runs:can-run-more-tests-count* 1))) @@ -525,11 +525,11 @@ (let*-values (((waitons waitors config) (tests:get-waitons hed all-tests-registry)) ;; NOTE: Have the config - can extract [waitons] section ((hed-mode) - (let ((m (config-lookup config "requirements" "mode"))) + (let ((m (configf:lookup config "requirements" "mode"))) (if m (map string->symbol (string-split m)) '(normal)))) ((hed-itemized-waiton) ;; are items in hed waiting on items of waiton? (not (null? (lset-intersection eq? hed-mode '(itemmatch itemwait))))) ) (debug:print-info 8 *default-log-port* "waitons: " waitons) @@ -546,11 +546,11 @@ (if (not (hash-table-ref/default test-records hed #f)) ;; waiton-tconfig below will be #f until that test is visted here at least once (hash-table-set! test-records ;; BB: we are doing a manual make-tests:testqueue hed (vector hed ;; 0 ;; testname config ;; 1 waitons ;; 2 - (config-lookup config "requirements" "priority") ;; priority 3 + (configf:lookup config "requirements" "priority") ;; priority 3 (tests:get-items config) ;; 4 ;; expand the [items] and or [itemstable] into explict items #f ;; itemsdat 5 #f ;; spare - used for item-path waitors ;; ))) @@ -1325,11 +1325,11 @@ (tests-info (mt:get-tests-for-run run-id #f '() '())) ;; qryvals: "id,testname,item_path")) (sorted-test-names (tests:sort-by-priority-and-waiton test-records)) (test-registry (make-hash-table)) (registry-mutex (make-mutex)) (num-retries 0) - (max-retries (config-lookup *configdat* "setup" "maxretries")) + (max-retries (configf:lookup *configdat* "setup" "maxretries")) (max-concurrent-jobs (configf:lookup-number *configdat* "setup" "max_concurrent_jobs" default: 50)) (reglen (if (number? reglen-in) reglen-in 1)) (last-time-incomplete (- (current-seconds) 900)) ;; force at least one clean up cycle (last-time-some-running (current-seconds)) ;; (tdbdat (tasks:open-db)) @@ -1395,12 +1395,12 @@ ;; (print "Top of loop, hed=" hed ", tal=" tal " ,reruns=" reruns) (let* ((test-record (hash-table-ref test-records hed)) (test-name (tests:testqueue-get-testname test-record)) (tconfig (tests:testqueue-get-testconfig test-record)) - (jobgroup (config-lookup tconfig "test_meta" "jobgroup")) - (testmode (let ((m (config-lookup tconfig "requirements" "mode"))) + (jobgroup (configf:lookup tconfig "test_meta" "jobgroup")) + (testmode (let ((m (configf:lookup tconfig "requirements" "mode"))) (if m (map string->symbol (string-split m)) '(normal)))) (itemmaps (tests:get-itemmaps tconfig)) ;; (configf:lookup tconfig "requirements" "itemmap")) (priority (tests:testqueue-get-priority test-record)) (itemdat (tests:testqueue-get-itemdat test-record)) ;; itemdat can be a string, list or #f (items (tests:testqueue-get-items test-record)) @@ -2523,11 +2523,11 @@ (rmt:testmeta-add-record test-name))) (for-each (lambda (key) (let* ((idx (cadr key)) (fld (car key)) - (val (config-lookup test-conf "test_meta" fld))) + (val (configf:lookup test-conf "test_meta" fld))) ;; (debug:print 5 *default-log-port* "idx: " idx " fld: " fld " val: " val) (if (and val (not (equal? (vector-ref currrecord idx) val))) (begin (print "Updating " test-name " " fld " to " val) (rmt:testmeta-update-field test-name fld val))))) Index: tests-inc.scm ================================================================== --- tests-inc.scm +++ tests-inc.scm @@ -149,16 +149,16 @@ ;; returns waitons waitors tconfigdat ;; (define (tests:get-waitons test-name all-tests-registry) (let* ((config (tests:get-testconfig test-name #f all-tests-registry 'return-procs))) ;; assuming no problems with immediate evaluation, this could be simplified ('return-procs -> #t) (let ((instr (if config - (config-lookup config "requirements" "waiton") + (configf:lookup config "requirements" "waiton") (begin ;; No config means this is a non-existant test (debug:print-error 0 *default-log-port* "non-existent required test \"" test-name "\"") (exit 1)))) (instr2 (if config - (config-lookup config "requirements" "waitor") + (configf:lookup config "requirements" "waitor") ""))) (debug:print-info 8 *default-log-port* "waitons string is " instr ", waitors string is " instr2) (let ((newwaitons (string-split (cond ((procedure? instr) ;; here @@ -1603,12 +1603,12 @@ (b-record (hash-table-ref test-records b)) (a-waitons (or (tests:testqueue-get-waitons a-record) '())) (b-waitons (or (tests:testqueue-get-waitons b-record) '())) (a-config (tests:testqueue-get-testconfig a-record)) (b-config (tests:testqueue-get-testconfig b-record)) - (a-raw-pri (config-lookup a-config "requirements" "priority")) - (b-raw-pri (config-lookup b-config "requirements" "priority")) + (a-raw-pri (configf:lookup a-config "requirements" "priority")) + (b-raw-pri (configf:lookup b-config "requirements" "priority")) (a-priority (mungepriority a-raw-pri)) (b-priority (mungepriority b-raw-pri))) (tests:testqueue-set-priority! a-record a-priority) (tests:testqueue-set-priority! b-record b-priority) ;; (debug:print 0 *default-log-port* "a=" a ", b=" b ", a-waitons=" a-waitons ", b-waitons=" b-waitons) @@ -1798,11 +1798,11 @@ (tal (cdr test-names))) ;; 'return-procs tells the config reader to prep running system but return a proc (debug:print-info 4 *default-log-port* "hed=" hed " at top of loop") ;; don't know item-path at this time, let the testconfig get the top level testconfig (let* ((config (tests:get-testconfig hed #f all-tests-registry 'return-procs)) (waitons (let ((instr (if config - (config-lookup config "requirements" "waiton") + (configf:lookup config "requirements" "waiton") (begin ;; No config means this is a non-existant test (debug:print-error 0 *default-log-port* "non-existent required test \"" hed "\", grep through your testconfigs to find and remove or create the test. Discarding and continuing.") "")))) (debug:print-info 8 *default-log-port* "waitons string is " instr) (string-split (cond @@ -1831,11 +1831,11 @@ (if (not (hash-table-ref/default test-records hed #f)) (hash-table-set! test-records hed (vector hed ;; 0 config ;; 1 waitons ;; 2 - (config-lookup config "requirements" "priority") ;; priority 3 + (configf:lookup config "requirements" "priority") ;; priority 3 (let ((items (hash-table-ref/default config "items" #f)) ;; items 4 (itemstable (hash-table-ref/default config "itemstable" #f))) ;; if either items or items table is a proc return it so test running ;; process can know to call items:get-items-from-config ;; if either is a list and none is a proc go ahead and call get-items