Megatest

Check-in [3200899a59]
Login
Overview
Comment:Factored out bunch of stuff to mtconfigf and mtcommon in src.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65-modularization
Files: files | file ages | folders
SHA1: 3200899a59caa8666a805583b2c0ea3228931542
User & Date: matt on 2018-04-09 22:31:41
Other Links: branch diff | manifest | tags
Context
2018-04-10
01:02
Fixed remaining fallout from the refactor check-in: 5fb5dbfbf7 user: matt tags: v1.65-modularization
2018-04-09
22:31
Factored out bunch of stuff to mtconfigf and mtcommon in src. check-in: 3200899a59 user: matt tags: v1.65-modularization
2018-04-04
14:47
Fixed few more missing license and copyright notices check-in: 39082cc602 user: mrwellan tags: v1.65-modularization
Changes

Modified common.scm from [6c5cb2ed66] to [aae9f9c538].

2841
2842
2843
2844
2845
2846
2847







2848
2849
2850
2851
2852

2853
2854
2855
2856

2857
2858
2859
2860
2861
2862
2863
2864
2865


2866
2867
2868
2869
2870
2871
2872
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858

2859
2860
2861
2862

2863





2864
2865
2866

2867
2868
2869
2870
2871
2872
2873
2874
2875







+
+
+
+
+
+
+




-
+



-
+
-
-
-
-
-



-
+
+







           (handle-exceptions
           exn
           #t ;; just ignore it, it might have died in the meantime so joining it will throw an exception
           (thread-join! thread))
           )))
   (hash-table-keys *common:thread-punchlist*)))

;; Use this when switching to mtconfigf module in find-and-read-config
;;
(define (common:set-fields curr-section next-section ht path)
  (let ((field-names (if ht (common:get-fields ht) '()))
	(target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
    (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
    (if (not (null? field-names))(keys:target-set-args field-names target #f))))

;; moved to common.scm as it is very megatest specific

;; pathenvvar will set the named var to the path of the config
(define (common:find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
(define (common:find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f)(set-fields common:set-fields))
  (let* ((curr-dir   (current-directory))
         (configinfo (find-config fname toppath: given-toppath))
	 (toppath    (car configinfo))
	 (configfile (cadr configinfo))
	 (configfile (cadr configinfo)))
	 (set-fields (lambda (curr-section next-section ht path)
		       (let ((field-names (if ht (common:get-fields ht) '()))
			     (target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
			 (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
			 (if (not (null? field-names))(keys:target-set-args field-names target #f))))))
    (if toppath (change-directory toppath)) 
    (if (and toppath pathenvvar)(setenv pathenvvar toppath))
    (let ((configdat  (if configfile 
			  (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f))))
			  (read-config configfile #f #t environ-patt: environ-patt
				       post-section-procs: (list (cons "^fields$" set-fields)) #f))))
      (if toppath (change-directory curr-dir))
      (list configdat toppath configfile fname))))

;;;; return list (path fullpath configname)
(define (common:find-config configname #!key (toppath #f))
  (if toppath
      (let ((cfname (conc toppath "/" configname)))

Modified mtut.scm from [175a9dda52] to [86a3c3c964].

554
555
556
557
558
559
560
561
562


563
564
565
566
567
568
569




570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585

586
587
588
589
590
591
592
554
555
556
557
558
559
560


561
562
563
564
565




566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584

585
586
587
588
589
590
591
592







-
-
+
+



-
-
-
-
+
+
+
+















-
+







;; collect, translate, collate and assemble a pkt from the command-line
;;
;; sched => force the run start time to be recorded as sched Unix
;; epoch. This aligns times properly for triggers in some cases.
;;
;;  extra-dat format is ( 'x xval 'y yval .... )
;;
(define (command-line->pkt action args-alist sched-in #!key (extra-dat '())(area-path #f)(new-ss #f))
   (let* ((sched     (cond
(define (command-line->pkt action args-alist sched-in #!key (extra-dat '())(area-path #f)(new-ss #f)(log-port (current-error-port)))
  (let* ((sched     (cond
		     ((vector? sched-in)(local-time->seconds sched-in)) ;; we recieved a time
		     ((number? sched-in) sched-in)
		     (else     (current-seconds))))
   (user  (if (and args-alist (hash-table? args-alist))
              (hash-table-ref/default args-alist "-override-user" (current-user-name))
						  (current-user-name)))
                    
	 (user  (if (and args-alist (hash-table? args-alist))
		    (hash-table-ref/default args-alist "-override-user" (current-user-name))
		    (current-user-name)))
	 
	 (args-data (if args-alist
			(if (hash-table? args-alist) ;; seriously?
			    (hash-table->alist args-alist)
			    args-alist)
			(hash-table->alist args:arg-hash))) ;; if no args-alist then we assume this is a call driven directly by commandline
	 (alldat    (apply append
			   (list 'A action
				 'U user
				 'D sched)
			   (if area-path
			       (list 'S area-path) ;; the area-path is mapped to the start-dir
			       '())
                           (if (list? extra-dat)
			       extra-dat
			       (begin
				 (debug:print 0 *default-log-port* "ERROR: command-line->pkt received bad extra-dat " extra-dat)
				 (debug-print 0 log-port "ERROR: command-line->pkt received bad extra-dat " extra-dat)
				 '()))
			   (map (lambda (x)
				  (let* ((param (car x))
					 (value (cdr x))
					 (pmeta (assoc param *arg-keys*))    ;; translate the card key to a megatest switch or parameter
					 (smeta (assoc param *switch-keys*)) ;; first lookup the key in arg-keys or switch-keys
					 (meta  (if (or pmeta smeta)
601
602
603
604
605
606
607
608

609
610
611
612
613
614
615
601
602
603
604
605
606
607

608
609
610
611
612
613
614
615







-
+







    ;(exit)
    (add-z-card
     (apply construct-sdat alldat))))

(define (simple-setup start-dir-in)
  (let* ((start-dir (or start-dir-in "."))
	 (mtconfig  (or (args:get-arg "-config") "megatest.config"))
	 (mtconfdat (find-and-read-config        ;; NB// sets MT_RUN_AREA_HOME as side effect
	 (mtconfdat (configf:find-and-read-config        ;; NB// sets MT_RUN_AREA_HOME as side effect
		     mtconfig
		     ;; environ-patt: "env-override"
		     given-toppath: start-dir
		     ;; pathenvvar: "MT_RUN_AREA_HOME"
		     ))
	 (mtconf    (if mtconfdat (car mtconfdat) #f)))
    ;; we set some dynamic data in a section called "scratchdata"
738
739
740
741
742
743
744
745



746
747


748
749

750
751
752
753
754
755
756
738
739
740
741
742
743
744

745
746
747
748

749
750
751

752
753
754
755
756
757
758
759







-
+
+
+

-
+
+

-
+







	  (print pkt))))))

;; (use trace)(trace create-run-pkt)

;; collect all needed data and create run pkts for contours with changed inputs
;;
(define (generate-run-pkts mtconf toppath)
  (let ((std-runname (conc "sched"  (time->string (seconds->local-time (current-seconds)) "%M%H%d"))))
  (let ((std-runname   (conc "sched"  (time->string (seconds->local-time (current-seconds)) "%M%H%d")))
	(pktsdir-str   (configf:lookup mtconf "scratchdat" "toppath"))
	(setup-pdbpath (configf:lookup mtconf "setup"  "pdbpath")))
    (common:with-queue-db
     mtconf
     pktsdir-str
     setup-pdbpath
     (lambda (pktsdirs pktsdir pdb)
       (let* ((rgconfdat (find-and-read-config (conc toppath "/runconfigs.config")))
       (let* ((rgconfdat (configf:find-and-read-config (conc toppath "/runconfigs.config")))
	      (rgconf    (car rgconfdat))
	      (all-areas (map car (configf:get-section mtconf "areas")))
	      (contours  (configf:get-section mtconf "contours"))
	      (torun     (make-hash-table)) ;; target => ( ... info ... )
	      (rgentargs (hash-table-keys rgconf))) ;; these are the targets registered for automatically triggering

	 (print "rgentargs: " rgentargs)
1101
1102
1103
1104
1105
1106
1107
1108

1109
1110
1111
1112
1113
1114
1115
1116
1117
1118

1119
1120
1121



1122
1123


1124
1125

1126
1127
1128
1129
1130
1131
1132
1104
1105
1106
1107
1108
1109
1110

1111
1112
1113
1114
1115
1116
1117
1118
1119
1120

1121
1122
1123

1124
1125
1126
1127

1128
1129
1130

1131
1132
1133
1134
1135
1136
1137
1138







-
+









-
+


-
+
+
+

-
+
+

-
+







	  (print pkt)))
      (print "ERROR: cannot process commands without a pkts directory")))

;; collect all needed data and create run pkts for contours with changed inputs
;;
(define (dispatch-commands mtconf toppath)
  ;; we are expecting a directory "logs", check and create it, create the log in /tmp if not able to create logs dir
  (let ((logdir
  (let* ((logdir
	 (if (if (not (directory? "logs"))
		 (handle-exceptions
		     exn
		     #f
		   (create-directory "logs")
		   #t)
		 #t)
	     "logs"
	     "/tmp"))
	(cpuload (alist-ref 'adj-proc-load (common:get-normalized-cpu-load #f)))
	(cpuload (alist-ref 'adj-proc-load (common:get-normalized-cpu-load logdir #f)))
	(maxload (string->number (or (configf:lookup mtconf "setup" "maxload")
				     (configf:lookup mtconf "jobtools" "maxload") ;; respect value used by Megatest calls
				     "1.1"))))
				     "1.1")))
	(pktsdir-str   (configf:lookup mtconf "scratchdat" "toppath"))
	(setup-pdbpath (configf:lookup mtconf "setup"  "pdbpath")))
    (common:with-queue-db
     mtconf
     pktsdir-str
     setup-pdbpath
     (lambda (pktsdirs pktsdir pdb)
       (let* ((rgconfdat (find-and-read-config (conc toppath "/runconfigs.config")))
       (let* ((rgconfdat (configf:find-and-read-config (conc toppath "/runconfigs.config")))
	      (rgconf    (car rgconfdat))
	      (areas     (configf:get-section mtconf "areas"))
	      (contours  (configf:get-section mtconf "contours"))
	      (pkts      (find-pkts pdb '(cmd) '()))
	      (torun     (make-hash-table)) ;; target => ( ... info ... )
	      (rgentargs (hash-table-keys rgconf))) ;; these are the targets registered for automatically triggering
	 (for-each
1252
1253
1254
1255
1256
1257
1258
1259



1260
1261
1262

1263
1264

1265
1266

1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287




1288
1289


1290
1291
1292
1293
1294
1295
1296
1258
1259
1260
1261
1262
1263
1264

1265
1266
1267
1268
1269

1270
1271

1272
1273

1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293


1294
1295
1296
1297
1298

1299
1300
1301
1302
1303
1304
1305
1306
1307







-
+
+
+


-
+

-
+

-
+



















-
-
+
+
+
+

-
+
+







	 ;;  (hash-table-keys adjargs))
	 (let-values (((uuid pkt)
		       (command-line->pkt *action* adjargs #f area-path: area-path new-ss: new-ss)))
	   (write-pkt pktsdir uuid pkt))))
      ((dispatch import rungen process)
       (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir")))
	      (mtconf    (car mtconfdat))
	      (toppath   (configf:lookup mtconf "scratchdat" "toppath")))
	      (toppath   (configf:lookup mtconf "scratchdat" "toppath"))
	      (pktsdir-str   (or (configf:lookup mtconf "scratchdat" "toppath")(configf:lookup mtconf "setup" "pktsdir")))
	      (setup-pdbpath (configf:lookup mtconf "setup"  "pdbpath")))
	 (case (string->symbol *action*)
	   ((process)  (begin
			 (common:load-pkts-to-db mtconf)
			 (common:load-pkts-to-db pktsdir-str setup-pdbpath)
			 (generate-run-pkts mtconf toppath)
			 (common:load-pkts-to-db mtconf)
			 (common:load-pkts-to-db pktsdir-str setup-pdbpath)
			 (dispatch-commands mtconf toppath)))
	   ((import)   (common:load-pkts-to-db mtconf)) ;; import pkts
	   ((import)   (common:load-pkts-to-db pktsdir-str setup-pdbpath)) ;; import pkts
	   ((rungen)   (generate-run-pkts mtconf toppath))
	   ((dispatch) (dispatch-commands mtconf toppath)))))
      ;; misc
      ((show)
       (if (> (length remargs) 0)
	   (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir")))
		  (mtconf    (car mtconfdat))
		  (sect-dat (configf:get-section mtconf (car remargs))))
	     (if sect-dat
		 (for-each
		  (lambda (entry)
		    (if (> (length entry) 1)
			(print (car entry) "   " (cadr entry))
			(print (car entry))))
		  sect-dat)
		 (print "No section \"" (car remargs) "\" found")))
	   (print "ERROR: list requires section parameter; areas, setup or contours")))
      ((gendot)
       (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir")))
	      (mtconf    (car mtconfdat)))
	 (common:load-pkts-to-db mtconf use-lt: #t) ;; need to NOT do this by default ...
	      (mtconf    (car mtconfdat))
	      (pktsdir-str   (configf:lookup mtconf "scratchdat" "toppath"))
	      (setup-pdbpath (configf:lookup mtconf "setup"  "pdbpath")))
	 (common:load-pkts-to-db pktsdir-str setup-pdbpath use-lt: #t) ;; need to NOT do this by default ...
	 (common:with-queue-db
	  mtconf
	  pktsdir-str
	  setup-pdbpath
	  (lambda (pktsdirs pktsdir conn)
	    ;;                       pktspec display-fields 
	    (make-report "out.dot" conn
			 '((cmd      . ((parent . P)
					(user   . M)
					(target . t)))
			   (runstart . ((parent . P)

Modified src/mtcommon.scm from [218dc9b1ef] to [a31207d0a5].

29
30
31
32
33
34
35








36
37
38
39

40
41
42
43
44
45
46
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54







+
+
+
+
+
+
+
+



-
+







         print-error
         print-info
         log-event
         debug-setup
         debug-mode
         check-verbosity
         calc-verbosity
	 ;; pkts stuff
	 load-pkts-to-db	 
	 get-pkt-alists
	 with-queue-db
	 ;; unix stuff
	 get-cached-info
	 write-cached-info
	 get-normalized-cpu-load
         )

(import scheme chicken data-structures extras posix ports)
(use (prefix sql-de-lite sql:) posix typed-records format srfi-1 srfi-69)
(use (prefix sql-de-lite sql:) posix typed-records format srfi-1 srfi-69 pkts regex (prefix dbi dbi:) regex-case)

(defstruct ctrldat
  (port      (current-error-port))
  (verbosity 1)
  (vcache    (make-hash-table))
  (logging   #f) ;; keep the flag and the db handle separate to enable overriding
  (logdb     #f) ;; might need to make this a stack of handles for threaded access
114
115
116
117
118
119
120




121
122
123
124
125
126
127
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139







+
+
+
+







      (with-output-to-port (or e (current-error-port))
	(lambda ()
	  (if (ctrldat-logging *log*)
	      (log-event (apply conc params))
	      (apply print params)
	      )))))

;; more betterer implementation above?
;; (define (print-info n e . params)
;;   (apply debug-print n e "INFO: " params))

;; ;; Brandon's debug printer shortcut (indulge me :)
;; (define *BB-process-starttime* (current-milliseconds))
;; (define (BB> . in-args)
;;   (let* ((stack (get-call-chain))
;;          (location "??"))
;;     (for-each
;;      (lambda (frame)
215
216
217
218
219
220
221





























































































222
223
224
225
226
227
228
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







	      )))))

;; if a value is printable (i.e. string or number) return the value
;; else return an empty string
(define-inline (printable val)
  (if (or (number? val)(string? val)) val ""))

;;======================================================================
;; Unix stuff
;;======================================================================

;; get values from cached info from dropping file in logs dir
;;  e.g. key is host and dtype is normalized-load
;;
(define (get-cached-info logdir key dtype #!key (age 5)(log-port (current-error-port)))
  (let* ((fullpath (conc logdir "/" key "-" dtype ".log")))
    (if (and (file-exists? fullpath)
	     (file-read-access? fullpath))
	(handle-exceptions
	 exn
	 #f
	 (debug-print 2 log-port "reading file " fullpath)
	 (let ((real-age (- (current-seconds)(file-change-time fullpath)))) 
	   (if (< real-age age)
	       (with-input-from-file fullpath read)
	       (begin
		 (debug-print 2 log-port "file " fullpath " is too old (" real-age" seconds)to trust, skipping reading it")
		 #f))))
	(begin
	  (debug-print 2 log-port "not reading file " fullpath)
	  #f))))

(define (write-cached-info logdir key dtype dat)
  (let* ((fullpath (conc logdir "/" key "-" dtype ".log")))
    (handle-exceptions
     exn
     #f
     (with-output-to-file fullpath (lambda ()(pp dat))))))


;; get normalized cpu load by reading from /proc/loadavg and /proc/cpuinfo return all three values and the number of real cpus and the number of threads
;; returns alist '((adj-cpu-load . normalized-proc-load) ... etc.
;;  keys: adj-proc-load, adj-core-load, 1m-load, 5m-load, 15m-load
;;
(define (get-normalized-cpu-load logdir remote-host)
  (let ((actual-host (or remote-host (get-host-name)))) ;; #f is localhost
    (or (get-cached-info logdir actual-host "normalized-load")
	(let ((data (if remote-host
			(with-input-from-pipe 
			    (conc "ssh " remote-host " cat /proc/loadavg;cat /proc/cpuinfo;echo end")
			  read-lines)
			(append 
			 (with-input-from-file "/proc/loadavg" 
			   read-lines)
			 (with-input-from-file "/proc/cpuinfo"
			   read-lines)
			 (list "end"))))
	      (load-rx  (regexp "^([\\d\\.]+)\\s+([\\d\\.]+)\\s+([\\d\\.]+)\\s+.*$"))
	      (proc-rx  (regexp "^processor\\s+:\\s+(\\d+)\\s*$"))
	      (core-rx  (regexp "^core id\\s+:\\s+(\\d+)\\s*$"))
	      (phys-rx  (regexp "^physical id\\s+:\\s+(\\d+)\\s*$"))
	      (max-num  (lambda (p n)(max (string->number p) n))))
	  ;; (print "data=" data)
	  (if (null? data) ;; something went wrong
	      #f
	      (let loop ((hed      (car data))
			 (tal      (cdr data))
			 (loads    #f)
			 (proc-num 0)  ;; processor includes threads
			 (phys-num 0)  ;; physical chip on motherboard
			 (core-num 0)) ;; core
		;; (print hed ", " loads ", " proc-num ", " phys-num ", " core-num)
		(if (null? tal) ;; have all our data, calculate normalized load and return result
		    (let* ((act-proc (+ proc-num 1))
			   (act-phys (+ phys-num 1))
			   (act-core (+ core-num 1))
			   (adj-proc-load (/ (car loads) act-proc))
			   (adj-core-load (/ (car loads) act-core))
			   (result
			    (append (list (cons 'adj-proc-load adj-proc-load)
					  (cons 'adj-core-load adj-core-load))
				    (list (cons '1m-load (car loads))
					  (cons '5m-load (cadr loads))
					  (cons '15m-load (caddr loads)))
				    (list (cons 'proc act-proc)
					  (cons 'core act-core)
					  (cons 'phys act-phys)))))
		      (write-cached-info logdir actual-host "normalized-load" result)
		      result)
		  (regex-case
		   hed
		   (load-rx  ( x l1 l5 l15 ) (loop (car tal)(cdr tal)(map string->number (list l1 l5 l15)) proc-num phys-num core-num))
		   (proc-rx  ( x p         ) (loop (car tal)(cdr tal) loads           (max-num p proc-num) phys-num core-num))
		   (phys-rx  ( x p         ) (loop (car tal)(cdr tal) loads           proc-num (max-num p phys-num) core-num))
		   (core-rx  ( x c         ) (loop (car tal)(cdr tal) loads           proc-num phys-num (max-num c core-num)))
		   (else 
		    (begin
		      ;; (print "NO MATCH: " hed)
		      (loop (car tal)(cdr tal) loads proc-num phys-num core-num)))))))))))

;;======================================================================
;; L O G G I N G    D B 
;;======================================================================

(define (open-logging-db toppath)
  (let* ((dbpath    (conc (if toppath (conc toppath "/") "") "logging.db")) ;; fname)
	 (dbexists  (file-exists? dbpath))
281
282
283
284
285
286
287
288



289









































































386
387
388
389
390
391
392
393
394
395
396

397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469








+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
(define (file-exists? path-string)
  ;; this avoids stack dumps. NOTE: The issues that triggered this approach might have been fixed TODO: test and remove if possible
  ;;;; TODO: catch permission denied exceptions and emit appropriate warnings, eg:  system error while trying to access file: "/nfs/pdx/disks/icf_env_disk001/bjbarcla/gwa/issues/mtdev/randy-slow/reproduce/q...
  (handle-exceptions
   exn
   #f
   (old-file-exists? path-string)))

;;======================================================================
;; pkts stuff
;;======================================================================
)

(define (load-pkts-to-db pktsdir-str setup-pdbpath #!key (use-lt #f)(log-port (current-error-port)))
  (with-queue-db
     pktsdir-str
     setup-pdbpath
     (lambda (pktsdirs pktsdir pdb)
       (for-each
	(lambda (pktsdir) ;; look at all
	  (cond
	   ((not (file-exists? pktsdir))
	    (debug-print 0 log-port "ERROR: packets directory " pktsdir " does not exist."))
	   ((not (directory? pktsdir))
	    (debug-print 0 log-port "ERROR: packets directory path " pktsdir " is not a directory."))
	   ((not (file-read-access? pktsdir))
	    (debug-print 0 log-port "ERROR: packets directory path " pktsdir " is not readable."))
	   (else
	    (print-info 0 log-port "Loading packets found in " pktsdir)
	    (let ((pkts (glob (conc pktsdir "/*.pkt"))))
	      (for-each
	       (lambda (pkt)
		 (let* ((uuid    (cadr (string-match ".*/([0-9a-f]+).pkt" pkt)))
			(exists  (lookup-by-uuid pdb uuid #f)))
		   (if (not exists)
		       (let* ((pktdat (string-intersperse
				       (with-input-from-file pkt read-lines)
				       "\n"))
			      (apkt   (pkt->alist pktdat))
			      (ptype  (alist-ref 'T apkt)))
			 (add-to-queue pdb pktdat uuid (or ptype 'cmd) #f 0)
			 (debug-print 4 log-port "Added " uuid " of type " ptype " to queue"))
		       (debug-print 4 log-port "pkt: " uuid " exists, skipping...")
		       )))
	       pkts)))))
	pktsdirs))
     use-lt: use-lt))
  
(define (get-pkt-alists pkts)
  (map (lambda (x)
	 (alist-ref 'apkt x)) ;; 'pkta pulls out the alist from the read pkt
       pkts))

(define (with-queue-db pktsdir-str setup-pdbpath proc #!key (use-lt #f)(toppath-in #f)(log-port (current-error-port)))
  (let* ((pktsdirs (get-pkts-dirs use-lt pktsdir-str))
	 (pktsdir  (if pktsdirs (car pktsdirs) #f))
	 (toppath  toppath-in) ;; (or (configf:lookup mtconf "scratchdat" "toppath") toppath-in))
	 (pdbpath  (or setup-pdbpath pktsdir))) ;; (configf:lookup mtconf "setup"  "pdbpath")
    (cond
     ((not (and  pktsdir toppath pdbpath))
      (debug-print 0 log-port "ERROR: settings are missing in your megatest.config for area management.")
      (debug-print 0 log-port "  you need to have pktsdir in the [setup] section."))
     ((not (file-exists? pktsdir))
      (debug-print 0 log-port "ERROR: pkts directory not found " pktsdir))
     ((not (equal? (file-owner pktsdir)(current-effective-user-id)))
      (debug-print 0 log-port "ERROR: directory " pktsdir " is not owned by " (current-effective-user-name)))
     (else
	(let* ((pdb  (open-queue-db pdbpath "pkts.db"
				    schema: '("CREATE TABLE groups (id INTEGER PRIMARY KEY,groupname TEXT, CONSTRAINT group_constraint UNIQUE (groupname));"))))
	  (proc pktsdirs pktsdir pdb)
	  (dbi:close pdb))))))

;; (configf:lookup mtconf "setup"  "pktsdirs")
(define (get-pkts-dirs use-lt #!key (top-path #f)(pktsdirs #f))
  (let* ((pktsdirs-str (or pktsdirs
			   (and use-lt
				(conc (or top-path
					  (current-directory))
				      "/lt/.pkts"))))
	 (pktsdirs  (if pktsdirs-str
			(string-split pktsdirs-str " ")
			#f)))
    pktsdirs))

)

Modified src/mtconfigf.scm from [61db1e25bb] to [8806606d06].

20
21
22
23
24
25
26
27










































28
29
30
31
32
33
34
35
36
37
38
39
40
41
















42
43
44



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

















78
79
80
81
82
83
84
20
21
22
23
24
25
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

















118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141







-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+














+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
















-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








;; NOTE: This is the configf module, long term it will replace configf.scm.

(declare (unit mtconfigf))

(module mtconfigf
        (

	 set-debug-printers
	 lazy-convert
	 assoc-safe-add
	 section-var-set!
	 safe-file-exists?
	 read-link-f
	 nice-path
	 eval-string-in-environment
	 safe-setenv
	 with-env-vars
	 cmd-run->list
	 port->list
	 configf:system
	 process-line
	 shell
	 configf:read-line
	 cfgdat->env-alist
	 calc-allow-system
	 apply-wildcards
	 val->alist
	 section->val-alist
	 read-config
	 find-config
	 find-and-read-config
	 lookup
	 var-is?
	 lookup-number
	 section-vars
	 get-section
	 set-section-var
	 compress-multi-lines
	 expand-multi-lines
	 file->list
	 write-config
	 read-refdb
	 map-all-hier-alist
	 config->alist
	 alist->config
	 read-alist
	 write-alist
	 config->ini
	 set-verbosity
         )

(import scheme chicken data-structures extras ports files)
(use posix typed-records srfi-18)
(use regex regex-case srfi-69 srfi-1 directory-utils extras srfi-13)
(import posix)

;; very wierd, the reference to pathname-directory here fixes a reference to possibly unbound identifier problem
;;
;; (define (dummy-function path)
;;   (pathname-directory path)
;;   (absolute-pathname? path)
;;   (normalize-pathname path))

;;======================================================================
;;
;; CONVERGE THIS WITH mtcommon.scm debug-print stuff
;;
;;======================================================================
(define *verbosity* 4)

(define (set-verbosity v)(set! *verbosity* v))

(define (tmp-debug-print n e . params)
  (if (cond
       ((list? n)(< (apply min n) *verbosity*))
       ((number? n) (< n *verbosity*))
       (else #f))
      (with-output-to-port (or e (current-error-port))
	(lambda ()(apply print params)))))
(define debug:print-error print)
(define debug:print       print)
(define debug:print-info  print)
(define debug:print-error tmp-debug-print)
(define debug:print       tmp-debug-print)
(define debug:print-info  tmp-debug-print)
(define *default-log-port* (current-error-port))

(define (set-debug-printers normal-fn info-fn error-fn default-port)
  (if error-fn  (set! debug:print-error error-fn))
  (if info-fn   (set! debug:print-info  info-fn))
  (if normal-fn (set! debug:print       normal-fn))
  (if default-port (set! *default-log-port* default-port)))
  
;; if it looks like a number -> convert it to a number, else return it
;;
(define (lazy-convert inval)
  (let* ((as-num (if (string? inval)(string->number inval) #f)))
    (or as-num inval)))

;; Moved to common
;;
;;;; return list (path fullpath configname)
;;(define (find-config configname #!key (toppath #f))
;;  (if toppath
;;      (let ((cfname (conc toppath "/" configname)))
;;	(if (common:file-exists? cfname)
;;	    (list toppath cfname configname)
;;	    (list #f      #f     #f)))
;;      (let* ((cwd (string-split (current-directory) "/")))
;;	(let loop ((dir cwd))
;;	  (let* ((path     (conc "/" (string-intersperse dir "/")))
;;		 (fullpath (conc path "/" configname)))
;;	    (if (common:file-exists? fullpath)
;;		(list path fullpath configname)
;;		(let ((remcwd (take dir (- (length dir) 1))))
;;		  (if (null? remcwd)
;;		      (list #f #f #f) ;;  #f #f) 
;;		  (loop remcwd)))))))))
;; return list (path fullpath configname)
(define (find-config configname #!key (toppath #f))
  (if toppath
      (let ((cfname (conc toppath "/" configname)))
	(if (safe-file-exists? cfname)
	    (list toppath cfname configname)
	    (list #f      #f     #f)))
      (let* ((cwd (string-split (current-directory) "/")))
	(let loop ((dir cwd))
	  (let* ((path     (conc "/" (string-intersperse dir "/")))
		 (fullpath (conc path "/" configname)))
	    (if (safe-file-exists? fullpath)
		(list path fullpath configname)
		(let ((remcwd (take dir (- (length dir) 1))))
		  (if (null? remcwd)
		      (list #f #f #f) ;;  #f #f) 
		  (loop remcwd)))))))))

(define (assoc-safe-add alist key val #!key (metadata #f))
  (let ((newalist (filter (lambda (x)(not (equal? key (car x)))) alist)))
    (append newalist (list (if metadata
			       (list key val metadata)
			       (list key val))))))

628
629
630
631
632
633
634
635


636
637
638
639
640
641
642





643
644
645
646
647
648
649
650
651
652
653








654
655
656
657
658
659
660
685
686
687
688
689
690
691

692
693
694






695
696
697
698
699











700
701
702
703
704
705
706
707
708
709
710
711
712
713
714







-
+
+

-
-
-
-
-
-
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+







                                         (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
	       (else (debug:print-error 0 *default-log-port* "problem parsing " path ",\n   \"" inl "\"")
		     (set! var-flag #f)
		     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
          ) ;; end loop
        )))

;; moved to common.scm as it is very megatest specific
;; look at common:set-fields for an example of how to use the set-fields proc
;; pathenvvar will set the named var to the path of the config
;;
;; ;; pathenvvar will set the named var to the path of the config
;; (define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
;;   (let* ((curr-dir   (current-directory))
;;          (configinfo (find-config fname toppath: given-toppath))
;; 	 (toppath    (car configinfo))
;; 	 (configfile (cadr configinfo))
(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f)(set-fields #f))
  (let* ((curr-dir   (current-directory))
         (configinfo (find-config fname toppath: given-toppath))
	 (toppath    (car configinfo))
	 (configfile (cadr configinfo)))
;; 	 (set-fields (lambda (curr-section next-section ht path)
;; 		       (let ((field-names (if ht (common:get-fields ht) '()))
;; 			     (target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
;; 			 (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
;; 			 (if (not (null? field-names))(keys:target-set-args field-names target #f))))))
;;     (if toppath (change-directory toppath)) 
;;     (if (and toppath pathenvvar)(setenv pathenvvar toppath))
;;     (let ((configdat  (if configfile 
;; 			  (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f))))
;;       (if toppath (change-directory curr-dir))
;;       (list configdat toppath configfile fname))))
    (if toppath (change-directory toppath)) 
    (if (and toppath pathenvvar)(setenv pathenvvar toppath))
    (let ((configdat  (if configfile 
			  (read-config configfile #f #t environ-patt: environ-patt
				       post-section-procs: (if set-fields (list (cons "^fields$" set-fields)) '())
				       #f))))
      (if toppath (change-directory curr-dir))
      (list configdat toppath configfile fname))))

(define (lookup cfgdat section var)
  (if (hash-table? cfgdat)
      (let ((sectdat (hash-table-ref/default cfgdat section '())))
	(if (null? sectdat)
	    #f
	    (let ((match (assoc var sectdat)))