Megatest

Check-in [47eef64ee9]
Login
Overview
Comment:Cherrypick 64826
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | rebase-envprocessing
Files: files | file ages | folders
SHA1: 47eef64ee93cee15b925d2c873db15af56215d13
User & Date: mrwellan on 2016-04-27 10:26:07
Other Links: branch diff | manifest | tags
Context
2016-04-27
10:46
Cherrypick 2972 check-in: 08dc0ff892 user: mrwellan tags: rebase-envprocessing
10:26
Cherrypick 64826 check-in: 47eef64ee9 user: mrwellan tags: rebase-envprocessing
10:24
Cherrypick 8269afbaa4e1a check-in: eaac938df2 user: mrwellan tags: rebase-envprocessing
Changes

Modified configf.scm from [dae73e7b9f] to [3ed4257dc9].

181
182
183
184
185
186
187
188
189








190
191
192
193
194
195
196
197
198
199
200
201
202
203


204
205
206

207
208
209
210
211
212
213
214
215
216
217
218
219

220
221

222
223
224
225

226
227
228
229
230
231
232
233
234
235

236
237
238
239
240
241

242
243
244
245
246
247
248
181
182
183
184
185
186
187


188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207


208
209
210
211

212
213
214
215
216
217
218
219
220
221
222
223
224

225
226

227
228
229
230

231
232
233
234
235
236
237
238
239
240

241
242
243
244
245
246

247
248
249
250
251
252
253
254







-
-
+
+
+
+
+
+
+
+












-
-
+
+


-
+












-
+

-
+



-
+









-
+





-
+







	(debug:print-info 1 "read-config - file not found " path " current path: " (current-directory))
	;; WARNING: This is a risky change but really, we should not return an empty hash table if no file read?
	#f) ;; (if (not ht)(make-hash-table) ht))
      (let ((inp        (open-input-file path))
	    (res        (if (not ht)(make-hash-table) ht))
	    (metapath   (if (or (debug:debug-mode 9)
				keep-filenames)
			    path #f)))
	(let loop ((inl               (configf:read-line inp res allow-system settings)) ;; (read-line inp))
			    path #f))
	    (calc-allow-system (lambda (allow-system section sections)
				 (if sections
				     (and (or (equal? "default" section)
					      (member section sections))
					  allow-system) ;; account for sections and return allow-system as it might be a symbol such as return-strings
				     allow-system))))
	(let loop ((inl               (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings)) ;; (read-line inp))
		   (curr-section-name (if curr-section curr-section "default"))
		   (var-flag #f);; turn on for key-var-pr and cont-ln-rx, turn off elsewhere
		   (lead     #f))
	  (debug:print-info 8 "curr-section-name: " curr-section-name " var-flag: " var-flag "\n   inl: \"" inl "\"")
	  (if (eof-object? inl) 
	      (begin
		(close-input-port inp)
		(hash-table-delete! res "") ;; we are using "" as a dumping ground and must remove it before returning the ht
		(debug:print 9 "END: " path)
		res)
	      (regex-case 
	       inl 
	       (configf:comment-rx _                  (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))
	       (configf:blank-l-rx _                  (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))
	       (configf:comment-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))
	       (configf:blank-l-rx _                  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))
	       (configf:settings   ( x setting val  ) (begin
							(hash-table-set! settings setting val)
							(loop (configf:read-line inp res allow-system settings) curr-section-name #f #f)))
							(loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f)))
	       (configf:include-rx ( x include-file ) (let* ((curr-conf-dir (pathname-directory path))
							     (full-conf     (if (absolute-pathname? include-file)
										include-file
										(nice-path 
										 (conc (if curr-conf-dir
											   curr-conf-dir
											   ".")
										       "/" include-file)))))
							(if (file-exists? full-conf)
							    (begin
							      ;; (push-directory conf-dir)
							      (debug:print 9 "Including: " full-conf)
							      (read-config full-conf res allow-system environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
							      (read-config full-conf res (calc-allow-system allow-system curr-section sections) environ-patt: environ-patt curr-section: curr-section-name sections: sections settings: settings keep-filenames: keep-filenames)
							      ;; (pop-directory)
							      (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))
							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))
							    (begin
							      (debug:print '(2 9) "INFO: include file " include-file " not found (called from " path ")")
							      (debug:print 2 "        " full-conf)
							      (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f)))))
							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f)))))
	       (configf:section-rx ( x section-name ) (begin
							;; call post-section-procs
							(for-each 
							 (lambda (dat)
							   (let ((patt (car dat))
								 (proc (cdr dat)))
							     (if (string-match patt curr-section-name)
								 (proc curr-section-name section-name res path))))
							 post-section-procs)
							(loop (configf:read-line inp res allow-system settings)
							(loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings)
							      ;; if we have the sections list then force all settings into "" and delete it later?
							      (if (or (not sections) 
								      (member section-name sections))
								  section-name "") ;; stick everything into ""
							      #f #f)))
	       (configf:key-sys-pr ( x key cmd      ) (if allow-system
	       (configf:key-sys-pr ( x key cmd      ) (if (calc-allow-system allow-system curr-section sections)
							  (let ((alist    (hash-table-ref/default res curr-section-name '()))
								(val-proc (lambda ()
									    (let* ((start-time (current-seconds))
										   (cmdres     (process:cmd-run->list cmd))
										   (delta      (- (current-seconds) start-time))
										   (status     (cadr cmdres))
										   (res        (car  cmdres)))
256
257
258
259
260
261
262
263

264
265
266
267
268
269


270
271
272
273
274
275
276
277
278
279
280

281
282
283
284
285

286
287
288
289
290
291
292
293
294
295
296
297
298
299
300


301
302
303

304
305
306
307
308
309
310
262
263
264
265
266
267
268

269
270
271
272
273


274
275
276
277
278
279
280
281
282
283
284
285

286
287
288
289
290

291
292
293
294
295
296
297
298
299
300
301
302
303
304


305
306
307
308

309
310
311
312
313
314
315
316







-
+




-
-
+
+










-
+




-
+













-
-
+
+


-
+







										  (debug:print-info 9 "for line \"" inl "\"\n  command: " cmd " took " delta " seconds to run with output:\n   " res))
									      (if (null? res)
										  ""
										  (string-intersperse res " "))))))
							    (hash-table-set! res curr-section-name 
									     (config:assoc-safe-add alist
									   			    key 
												    (case allow-system
												    (case (calc-allow-system allow-system curr-section sections)
												      ((return-procs) val-proc)
												      ((return-string) cmd)
												      (else (val-proc)))
												    metadata: metapath))
							    (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))
							  (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f)))
							    (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))
							  (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f)))
	       (configf:key-val-pr ( x key unk1 val unk2 ) (let* ((alist   (hash-table-ref/default res curr-section-name '()))
								  (envar   (and environ-patt (string-search (regexp environ-patt) curr-section-name)))
								  (realval (if envar
									       (config:eval-string-in-environment val)
									       val)))
							     (debug:print-info 6 "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name)
							     (if envar (safe-setenv key realval))
							     (debug:print 10 "   setting: [" curr-section-name "] " key " = " val)
							     (hash-table-set! res curr-section-name 
									      (config:assoc-safe-add alist key realval metadata: metapath))
							     (loop (configf:read-line inp res allow-system settings) curr-section-name key #f)))
							     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name key #f)))
	       (configf:key-no-val ( x key val)             (let* ((alist   (hash-table-ref/default res curr-section-name '())))
							      (debug:print 10 "   setting: [" curr-section-name "] " key " = #t")
							      (hash-table-set! res curr-section-name 
									       (config:assoc-safe-add alist key #t metadata: metapath))
							      (loop (configf:read-line inp res allow-system settings) curr-section-name key #f)))
							      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name key #f)))
	       ;; if a continued line
	       (configf:cont-ln-rx ( x whsp val     ) (let ((alist (hash-table-ref/default res curr-section-name '())))
						(if var-flag             ;; if set to a string then we have a continued var
						    (let ((newval (conc 
								   (config-lookup res curr-section-name var-flag) "\n"
								   ;; trim lead from the incoming whsp to support some indenting.
								   (if lead
								       (string-substitute (regexp lead) "" whsp)
								       "")
								   val)))
						      ;; (print "val: " val "\nnewval: \"" newval "\"\nvarflag: " var-flag)
						      (hash-table-set! res curr-section-name 
								       (config:assoc-safe-add alist var-flag newval metadata: metapath))
						      (loop (configf:read-line inp res allow-system settings) curr-section-name var-flag (if lead lead whsp)))
						    (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))))
						      (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name var-flag (if lead lead whsp)))
						    (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))))
	       (else (debug:print 0 "ERROR: problem parsing " path ",\n   \"" inl "\"")
		     (set! var-flag #f)
		     (loop (configf:read-line inp res allow-system settings) curr-section-name #f #f))))))))
		     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section sections) settings) curr-section-name #f #f))))))))
  
;; pathenvvar will set the named var to the path of the config
(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
  (let* ((curr-dir   (current-directory))
         (configinfo (find-config fname toppath: given-toppath))
	 (toppath    (car configinfo))
	 (configfile (cadr configinfo))

Modified launch.scm from [2b290f7cdb] to [a813a15787].

709
710
711
712
713
714
715
716

717
718
719

720
721
722
723
724
725

726
727

728
729
730
731


732
733
734
735

736
737
738


739
740


741
742
743
744






745
746
747
748
749

750
751
752


753
754
755
756
757







758
759
760
761
762






763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792










793
794




795
796
797
798
799
800
801
709
710
711
712
713
714
715

716
717
718
719
720
721
722
723
724
725

726
727

728
729
730


731
732
733
734
735
736
737
738
739
740
741
742


743
744
745



746
747
748
749
750
751
752
753
754
755
756
757
758


759
760





761
762
763
764
765
766
767





768
769
770
771
772
773
774
775
776
777









778
779
780
781
782
783
784
785
786








787
788
789
790
791
792
793
794
795
796


797
798
799
800
801
802
803
804
805
806
807







-
+



+





-
+

-
+


-
-
+
+




+



+
+
-
-
+
+

-
-
-
+
+
+
+
+
+





+

-
-
+
+
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
+




-
-
-
-
-
-
-
-
-









-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+







;;     *toppath*
;;   side effects:
;;     sets; *configdat*    (megatest.config info)
;;           *runconfigdat* (runconfigs.config info)
;;           *configstatus* (status of the read data)
;;
(define (launch:setup-new #!key (force #f))
  (let* ((toppath  (or *toppath* (getenv "MT_RUN_AREA_HOME")))
  (let* ((toppath  (or *toppath* (getenv "MT_RUN_AREA_HOME"))) ;; preserve toppath
	 (runname  (common:args-get-runname))
	 (target   (common:args-get-target))
	 (linktree (common:get-linktree))
	 (mtconfig (or (args:get-arg "-config") "megatest.config")) ;; allow overriding megatest.config 
	 (rundir   (if (and runname target linktree)(conc linktree "/" target "/" runname) #f))
	 (mtcachef (and rundir (conc rundir "/" ".megatest.cfg-"  megatest-version "-" megatest-fossil-hash)))
	 (rccachef (and rundir (conc rundir "/" ".runconfigs.cfg-"  megatest-version "-" megatest-fossil-hash)))
	 (cancreate (and rundir (file-exists? rundir)(file-write-access? rundir))))
    ;; (print "runname: " runname " target: " target " mtcachef: " mtcachef " rccachef: " rccachef)
    (if (not *toppath*)(set! *toppath* toppath)) ;; this probably is not needed?
    ;; (if (not *toppath*)(set! *toppath* toppath)) ;; this probably is not needed?
    (cond
     ;; data was read and cached and available in *configstatus*
     ;; data was read and cached and available in *configstatus*, toppath has already been set
     ((eq? *configstatus* 'fulldata)
      *toppath*)
     ;; if mtcachef exists just read it
     ((and mtcachef (file-exists? mtcachef))
     ;; if mtcachef exists just read it, however we need to assume toppath is available in $MT_RUN_AREA_HOME
     ((and mtcachef (file-exists? mtcachef) (get-environment-variable "MT_RUN_AREA_HOME"))
      (set! *configdat*    (configf:read-alist mtcachef))
      (set! *runconfigdat* (configf:read-alist rccachef))
      (set! *configinfo*   (list *configdat*  (get-environment-variable "MT_RUN_AREA_HOME")))
      (set! *configstatus* 'fulldata)
      (set! *toppath*      (get-environment-variable "MT_RUN_AREA_HOME"))
      *toppath*)
     ;; we have all the info needed to fully process runconfigs and megatest.config
     (mtcachef              
      (let* ((toppath    (get-environment-variable "MT_RUN_AREA_HOME")) ;; rely on MT_RUN_AREA_HOME if we are in a test environment
	     (sections   (list "default" target)) ;; for runconfigs
      (let ((first-pass (find-and-read-config 
				  (or (args:get-arg "-config") "megatest.config")
	     (first-pass (find-and-read-config        ;; NB// sets MT_RUN_AREA_HOME as side effect
				  mtconfig
				  environ-patt: "env-override"
				  given-toppath: (get-environment-variable "MT_RUN_AREA_HOME")
				  pathenvvar: "MT_RUN_AREA_HOME")))
	(if first-pass
				  given-toppath: toppath
				  pathenvvar: "MT_RUN_AREA_HOME"))
	     (first-rundat  (read-config (conc toppath "/runconfigs.config") *runconfigdat* #t 
					 sections: sections)))
	(set! *runconfigdat* first-rundat)
	(if first-pass  ;; 
	    (begin
	      (set! *configdat*  (car first-pass))
	      (set! *configinfo* first-pass)
	      (set! *toppath*    (or toppath (cadr first-pass))) ;; use the gathered data unless already have it
	      (set! toppath      *toppath*)
	      (setenv "MT_RUN_AREA_HOME" *toppath*)
	      ;; the seed read is done, now read runconfigs, cache it then read megatest.config one more time and cache it
	      (let* ((keys     (rmt:get-keys))
		     (key-vals (if target (keys:target->keyval keys target) #f))
	      (let* ((keys         (rmt:get-keys))
		     (key-vals     (keys:target->keyval keys target))
		     (sections (if target (list "default" target) #f)) ;; for runconfigs
		     (linktree (or (getenv "MT_LINKTREE")
				   (if *configdat* (configf:lookup *configdat* "setup" "linktree") #f)))
		     (runconfigdat (begin
				     (setenv "MT_RUN_AREA_HOME" *toppath*)
		     (linktree     (or (getenv "MT_LINKTREE")
				       (if *configdat* (configf:lookup *configdat* "setup" "linktree") #f)))
		     (second-pass  (find-and-read-config
				    mtconfig
				    environ-patt: "env-override"
				    given-toppath: toppath
				    pathenvvar: "MT_RUN_AREA_HOME"))
				     (if key-vals
					 (for-each (lambda (kt)
						     (setenv (car kt) (cadr kt)))
						   key-vals))
				     (read-config (conc *toppath* "/runconfigs.config") #f #t sections: sections))))
		     (runconfigdat (begin     ;; this read of the runconfigs will see any adjustments made by re-reading megatest.config
				     (for-each (lambda (kt)
						 (setenv (car kt) (cadr kt)))
					       key-vals)
				     (read-config (conc toppath "/runconfigs.config") *runconfigdat* #t 
						  sections: sections))))
		(if cancreate (configf:write-alist runconfigdat rccachef))
		(set! *runconfigdat* runconfigdat)
		(if cancreate (configf:write-alist *configdat* mtcachef))
		(if cancreate (set! *configstatus* 'fulldata))))
		;; (let ((second-pass (find-and-read-config
		;; 		    (or (args:get-arg "-config") "megatest.config")
		;; 		    environ-patt: "env-override"
		;; 		    given-toppath: (get-environment-variable "MT_RUN_AREA_HOME")
		;; 		    pathenvvar: "MT_RUN_AREA_HOME")))
		;;   (if cancreate (configf:write-alist (car second-pass) mtcachef))
		;;   (set! *configdat* (car second-pass))
		;;   (set! *toppath*   (or toppath (cadr second-pass))) ;; this should be a no-op, remove it later
		;;   (if cancreate (set! *configstatus* 'fulldata)))))
	    ;; no configs found? should not happen but let's try to recover gracefully, return an empty hash-table
	    (set! *configdat* (make-hash-table))
	    )))
     ;; else read what you can and set the flag accordingly
     (else
      (let* ((cfgdat   (find-and-read-config 
			(or (args:get-arg "-config") "megatest.config")
			environ-patt: "env-override"
			given-toppath: (get-environment-variable "MT_RUN_AREA_HOME")
			pathenvvar: "MT_RUN_AREA_HOME"))
	     (sections (if target (list "default" target) #f))
	     (rdat     (read-config (conc (or *toppath* (get-environment-variable "MT_RUN_AREA_HOME")(cadr cfgdat))
					  "/runconfigs.config") #f #t sections: sections)))
	(set! *configinfo*   cfgdat)
	(set! *configdat*    (car cfgdat))
	(set! *runconfigdat* rdat)
	(set! *toppath*      (or toppath (cadr cfgdat)))
			pathenvvar: "MT_RUN_AREA_HOME")))
	(if cfgdat
	    (let* ((sections (if target (list "default" target) #f))
		   (toppath  (or (get-environment-variable "MT_RUN_AREA_HOME")(cadr cfgdat)))
		   (rdat     (read-config (conc toppath
						"/runconfigs.config") *runconfigdat* #t sections: sections)))
	      (set! *configinfo*   cfgdat)
	      (set! *configdat*    (car cfgdat))
	      (set! *runconfigdat* rdat)
	      (set! *toppath*      toppath)
	(set! toppath        *toppath*)  ;; remove this sillyness later
	(set! *configstatus* 'partial))))
	      (set! *configstatus* 'partial))
	    (begin
	      (debug:print 0 "ERROR: No " mtconfig " file found. Giving up.")
	      (exit 2))))))
    ;; additional house keeping
    (let* ((linktree (or (getenv "MT_LINKTREE")
			 (if *configdat* (configf:lookup *configdat* "setup" "linktree") #f))))
      (if linktree
	  (if (not (file-exists? linktree))
	      (begin
		(handle-exceptions