Megatest

;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;;======================================================================

(declare (unit commonmod))
(declare (uses margsmod))
(declare (uses debugprint))
(declare (uses configfmod))

(module commonmod
	*
	
;;(import scheme chicken data-structures extras files ports)
(import scheme chicken)
(import margsmod)
(import debugprint)
(import configfmod)

(use data-structures extras files ports)
(use 
  (prefix base64 base64:)
  (prefix sqlite3 sqlite3:) 
  (srfi 18)
  (prefix dbi dbi:)
  directory-utils
  hostinfo
  format
  matchable
  md5
  message-digest
  pkts
  posix 
  regex
  regex-case
  sparse-vectors
  srfi-1
  srfi-4
  srfi-13
  srfi-69
  stack
  stml2
  typed-records 
  z3
 )

;; 
;; (define (exit . code)
;;   (if (null? code)
;;       (old-exit)
;;       (old-exit code)))

(define *numcpus-cache* (make-hash-table))



;;======================================================================
;; use to transition to area-name

(define (common:get-sync-lock-filepath)
  (let* ((tmp-area     (common:get-db-tmp-area))
         (lockfile     (conc tmp-area "/megatest.db.sync-lock")))
    lockfile))


(define (common:get-sync-lock-filepath)
  (let* ((tmp-area     (common:get-db-tmp-area))
         (lockfile     (conc tmp-area "/megatest.db.sync-lock")))
    lockfile))

;;======================================================================
;; (debug:print 0 *default-log-port* "Failed to find this executable! Using what can be found on the path")














(define (common:get-testsuite-name)
  (or (configf:lookup *configdat* "setup" "area-name") ;; megatest is a flexible tool, testsuite is too limiting a description.
      (configf:lookup *configdat* "setup" "testsuite" )
      (getenv "MT_TESTSUITE_NAME")
      (pathname-file (or (if (string? *toppath* )
			     (pathname-file *toppath*)
			     #f)
			 (common:get-toppath #f)))
      "please-set-setup-area-name")) ;; (pathname-file (current-directory)))))

(define common:get-area-name common:get-testsuite-name)


(define (get-area-name configdat toppath #!optional (short #f))
  ;; look up my area name in areas table (future)
  ;; generate auto name
  (conc (get-area-path-signature toppath short)
	"-"
	(common:get-testsuite-name toppath configdat)))

;; pathenvvar will set the named var to the path of the config
(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
  (let* ((curr-dir   (current-directory))
         (configinfo (find-config fname toppath: given-toppath))
	 (toppath    (car configinfo))
	 (configfile (cadr configinfo))
	 (set-fields (lambda (curr-section next-section ht path)
		       (let ((field-names (if ht (common:get-fields ht) '()))
			     (target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
			 (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
			 (if (not (null? field-names))(keys:target-set-args field-names target #f))))))
    (if toppath (change-directory toppath)) 
    (if (and toppath pathenvvar)(setenv pathenvvar toppath))
    (let ((configdat  (if configfile 
			  (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f))))
      (if toppath (change-directory curr-dir))
      (list configdat toppath configfile fname))))


;;======================================================================
;; L O C K I N G   M E C H A N I S M S 
;;======================================================================
(define (common:get-db-tmp-area . junk)
  (if *db-cache-path*
      *db-cache-path*
      (if *toppath* ;; common:get-create-writeable-dir
	  (handle-exceptions
	   exn
	   (begin
	     (debug:print-error 0 *default-log-port* "Couldn't create path to " *db-cache-path* ", exn=" exn)
	     (exit 1))
	   (let* ((tsname (common:get-testsuite-name))
		  (dbpath (common:get-create-writeable-dir
			   (list (conc "/tmp/" (current-user-name)
				       "/megatest_localdb/"
				       tsname "/"
				       (string-translate *toppath* "/" "."))
				 (conc "/tmp/" (current-process-id) ;; just in case we have an issue with the dir by own user name
				       "/megatest_localdb/"
				       tsname
				       (string-translate *toppath* "/" "."))
				 ))))
	     (set! *db-cache-path* dbpath)
	     dbpath))
	  #f)))



;;======================================================================
;; CONTENTS
;;
;;  config file utils
;;  misc conversion, data manipulation functions
;;  testsuite and area utilites
;;
;;======================================================================

(include "megatest-version.scm")
(include "megatest-fossil-hash.scm")

;; Move globals here

(define *toppath*           #f)
(define *configinfo*        #f)   ;; raw results from setup, includes toppath and table from megatest.config
(define *db-with-db-mutex*  (make-mutex))
(define *max-api-process-requests* 0)
(define *common:denoise*    (make-hash-table)) ;; for low noise printing
(define *api-process-request-count* 0)
(define *db-keys*           #f)

;;======================================================================
;; does the directory exist and do we have write access?
;;
;;    returns the directory or #f
;;
(define (common:directory-writable? path-string)
  (handle-exceptions
   exn
   (begin
     (debug:print 0 *default-log-port* "Failed to identify access to " path-string ", exn=" exn)
     #f)
   (if (and (directory-exists? path-string)
            (file-write-access? path-string))
       path-string
       #f)))

;;======================================================================
;; T I M E   A N D   D A T E

							      ((d) 86400)
							      ((w) 604800)
							      ((M) 2628000) ;; aproximately one month
							      ((y) 31536000)
							      (else #f))))))))))
	      parts)
    time-secs))

(define (seconds->hr-min-sec secs)
  (let* ((hrs (quotient secs 3600))
	 (min (quotient (- secs (* hrs 3600)) 60))
	 (sec (- secs (* hrs 3600)(* min 60))))
    (conc (if (> hrs 0)(conc hrs "hr ") "")
	  (if (> min 0)(conc min "m ")  "")
	  sec "s")))

	    cron-str ;; `(,cron-str)              ;; just return the string, something downstream will fix it
	    (let loop ((hed  (car cron-items))
		       (tal  (cdr cron-items))
		       (type 'min)
		       (type-tal '(hour dayofmonth month dayofweek))
		       (res  '()))
	      (regex-case
	       hed
	       (slash-rx ( _ base incr ) (let* ((basen          (string->number base))
						(incrn          (string->number incr))
						(expanded-vals  (common:expand-cron-slash basen incrn (alist-ref type max-vals)))
						(new-list-crons (fold (lambda (x myres)
									(cons (conc (if (null? res)
											""
											(conc (string-intersperse res " ") " "))
										    x " " (string-intersperse tal " "))
									      myres))
								      '() expanded-vals)))
					   ;; (print "new-list-crons: " new-list-crons)
					   ;; (fold (lambda (x res)
					   ;; 	    (if (list? x)
					   ;; 		(let ((newres (map common:cron-expand x)))
					   ;; 		  (append x newres))
					   ;; 		(cons x res)))
					   ;; 	  '()
					   (flatten (map common:cron-expand new-list-crons))))
	       ;;					    (map common:cron-expand (map common:cron-expand new-list-crons))))
	       (else (if (null? tal)
			 cron-str
			 (loop (car tal)(cdr tal)(car type-tal)(cdr type-tal)(append res (list hed)))))))))))

;;======================================================================
;; given a cron string and the last time event was processed return #t to run or #f to not run
;;
;;  min    hour   dayofmonth month  dayofweek
;; 0-59    0-23   1-31       1-12   0-6          ### NOTE: dayofweek does not include 7
;;

	(match-let (((     cmin chour cdayofmonth cmonth    cdayofweek)
		     cron-items)
		    ;; 0     1    2        3         4    5      6
		    ((nsec nmin nhour ndayofmonth nmonth nyr ndayofweek n7 n8 n9)
		     (vector->list now-time))
		    ((lsec lmin lhour ldayofmonth lmonth lyr ldayofweek l7 l8 l9)
		     (vector->list last-done-time)))
		   ;; create all possible time slots
		   ;; remove invalid slots due to (for example) day of week
		   ;; get the start and end entries for the ref-seconds (current) time
		   ;; if last-done > ref-seconds => this is an ERROR!
		   ;; does the last-done time fall in the legit region?
		   ;;    yes => #f  do not run again this command
		   ;;    no  => #t  ok to run the command
		   (for-each ;; month
		    (lambda (month)
		      (for-each ;; dayofmonth
		       (lambda (dom)
			 (for-each
			  (lambda (hr) ;; hour
			    (for-each
			     (lambda (minute) ;; minute
			       (let ((copy-now (apply vector (vector->list now-time))))
				 (vector-set! copy-now 0 0) ;; force seconds to zero
				 (vector-set! copy-now 1 minute)
				 (vector-set! copy-now 2 hr)
				 (vector-set! copy-now 3 dom)  ;; dom is already corrected for zero referenced
				 (vector-set! copy-now 4 month)
				 (let* ((copy-now-secs (local-time->seconds copy-now))
					(new-copy      (seconds->local-time copy-now-secs))) ;; remake the time vector
				   (if (or (not cdayofweek)
					   (equal? (vector-ref new-copy 6)
						   cdayofweek)) ;; if the day is specified and a match OR if the day is NOT specified
				       (if (or (not cdayofmonth)
					       (equal? (vector-ref new-copy 3)
						       (+ 1 cdayofmonth))) ;; if the month is specified and a match OR if the month is NOT specified
					   (hash-table-set! all-times copy-now-secs new-copy))))))
			     (if cmin
				 `(,cmin)  ;; if given cmin, have to use it
				 (list (- nmin 1) nmin (+ nmin 1))))) ;; minute
			  (if chour
			      `(,chour)
			      (list (- nhour 1) nhour (+ nhour 1))))) ;; hour
		       (if cdayofmonth
			   `(,cdayofmonth)
			   (list (- ndayofmonth 1) ndayofmonth (+ ndayofmonth 1)))))
		    (if cmonth
			`(,cmonth)
			(list (- nmonth 1) nmonth (+ nmonth 1))))
		   (let ((before #f)
			 (is-in  #f))
		     (for-each
		      (lambda (moment)
			(if (and before
				 (<= before now-seconds)
				 (>= moment now-seconds))
			    (begin
			      ;; (print)
			      ;; (print "Before: " (time->string (seconds->local-time before)))
			      ;; (print "Now:    " (time->string (seconds->local-time now-seconds)))
			      ;; (print "After:  " (time->string (seconds->local-time moment)))
			      ;; (print "Last:   " (time->string (seconds->local-time last-done)))
			      (if (<  last-done before)
				  (set! is-in before))
			      ))
			(set! before moment))
		      (sort (hash-table-keys all-times) <))
		     is-in)))))

(define (common:extended-cron  cron-str now-seconds-in last-done)
  (let ((expanded-cron (common:cron-expand cron-str)))
    (if (string? expanded-cron)
	(common:cron-event expanded-cron now-seconds-in last-done)
	(let loop ((hed (car expanded-cron))
		   (tal (cdr expanded-cron)))

;;======================================================================

;;======================================================================
;; lazy-safe get file mod time. on any error (file not existing etc.) return 0
;;
(define (common:lazy-modification-time fpath)
  (handle-exceptions
   exn
   (begin
     (debug:print 0 *default-log-port* "Failed to get modifcation time for " fpath ", treating it as zero. exn=" exn)
     0)
   (if (file-exists? fpath)
       (file-modification-time fpath)
       0)))

;;======================================================================
;; find timestamp of newest file associated with a sqlite db file
(define (common:lazy-sqlite-db-modification-time fpath)
  (let* ((glob-list (handle-exceptions
		     exn
		     (begin
		       (debug:print 0 *default-log-port* "Failed to glob " fpath "*, exn=" exn)
		       `(,(conc "/no/such/file, message: " ((condition-property-accessor 'exn 'message) exn))))
		     (glob (conc fpath "*"))))
         (file-list (if (eq? 0 (length glob-list))
			'("/no/such/file")
			glob-list)))
    (apply max
	   (map
	    common:lazy-modification-time 
	    file-list))))

;;======================================================================
;; execute thunk, return value.  If exception thrown, trap exception, return #f, and emit nonfatal condition note to *default-log-port* .
;; arguments - thunk, message
(define (common:fail-safe thunk warning-message-on-exception)
  (handle-exceptions
   exn
   (begin
     (debug:print-info 0 *default-log-port* "notable but nonfatal condition - "warning-message-on-exception", exn=" exn)
     (debug:print-info 0 *default-log-port*
                       (string-substitute "\n?Error:" "nonfatal condition:"
                                          (with-output-to-string
                                            (lambda ()
                                              (print-error-message exn) ))))
     (debug:print-info 0 *default-log-port* "    -- continuing after nonfatal condition...")
     #f)
   (thunk)))















(define home (getenv "HOME"))
(define user (getenv "USER"))


;;======================================================================
;; returns list of fd count, socket count
(define (get-file-descriptor-count #!key  (pid (current-process-id )))
  (list
   (length (glob (conc "/proc/" pid "/fd/*")))
   (length  (filter identity (map socket? (glob (conc "/proc/" pid "/fd/*")))))
   )
  )



;;======================================================================
;; GLOBALS

;; CONTEXTS
(defstruct cxt
  (taskdb #f)

;;         (set! cxt (let ((x (make-cxt)))(hash-table-set! *contexts* toppath x) x)))
;;     (let ((cxt-mutex (cxt-mutex cxt)))
;;       (mutex-unlock! *context-mutex*)
;;       (mutex-lock! cxt-mutex)
;;       (let ((res (proc cxt)))
;;         (mutex-unlock! cxt-mutex)
;;         res))))

;;======================================================================
;; A hash table that can be accessed by #{scheme ...} calls in
;; config files. Allows communicating between confgs
;;
(define *user-hash-data* (make-hash-table))

(define *pkts-info*    (make-hash-table)) ;; store stuff like the last parent here

                            (string-split (chicken-version) ".")
                            2)))))
  (let ((resolve-pathname-broken?
         (or (> chicken-release-number 4)
             (and (eq? 4 chicken-release-number) (> chicken-major-version 9)))))
    (if resolve-pathname-broken?
        (define ##sys#expand-home-path pathname-expand))))

(define (realpath x) (resolve-pathname  (pathname-expand (or x "/dev/null")) ))

(define (common:get-this-exe-fullpath #!key (argv (argv)))
  (let* ((this-script
          (cond
           ((and (> (length argv) 2)
                 (string-match "^(.*/csi|csi)$" (car argv))

(define (common:steps-can-proceed-given-status-sym status-sym)
  (if (member status-sym '(warn waived pass))
      #t
      #f))

(define (status-sym->string status-sym)
  (case status-sym
    ((pass) "PASS")
    ((fail) "FAIL")
    ((warn) "WARN")
    ((check) "CHECK")
    ((waived) "WAIVED")
    ((abort) "ABORT")
    ((skip) "SKIP")
    (else "FAIL")))

            (copy daysfile wksfile))


        ;; if age(<file>.mins.gz) >= 1h:
        ;;     copy <file>.mins.gz <file>.hrs.gz
        ;;     copy <prev file> <file>.mins.gz
        (when (>= (age-mins minsfile) 1)
	      (copy minsfile hrsfile)
	      (copy+zip filepath  minsfile))
        
        ;; if age(<file>.hrs.gz) >= 1d:
        ;;     copy <file>.hrs.gz <file>.days.gz
        ;;     copy <file>.mins.gz <file>.hrs.gz
        (when (>= (age-days hrsfile) 1)
	      (copy hrsfile daysfile)
	      (copy minsfile hrsfile))
        
        ;; if age(<file>.days.gz) >= 1w:
        ;;     copy <file>.days.gz <file>.weeks.gz
        ;;     copy <file>.hrs.gz <file>.days.gz
        (when (>= (age-wks daysfile) 1)
	      (copy daysfile wksfile)
	      (copy hrsfile daysfile))
        #t)
      #f))


;;======================================================================
;; S P A R S E   A R R A Y S
;;======================================================================

(define (make-sparse-array)
  (let ((a (make-sparse-vector)))
    (sparse-vector-set! a 0 (make-sparse-vector))

;;======================================================================
;; dot-locking egg seems not to work, using this for now
;; if lock is older than expire-time then remove it and try again
;; to get the lock
;;
(define (common:simple-file-lock fname #!key (expire-time 300))
  (let ((fmod-time (handle-exceptions
		    ext
		    (current-seconds)
		    (file-modification-time fname))))
    (if (common:file-exists? fname)
	(if (> (- (current-seconds) fmod-time) expire-time)
	    (begin
	      (handle-exceptions exn #f (delete-file* fname))	
	      (common:simple-file-lock fname expire-time: expire-time))
	    #f)
	(let ((key-string (conc (get-host-name) "-" (current-process-id))))
	  (with-output-to-file fname
	    (lambda ()
	      (print key-string)))
	  (thread-sleep! 0.25)
	  (if (common:file-exists? fname)
	      (handle-exceptions exn
				 #f 
				 (with-input-from-file fname
				   (lambda ()
				     (equal? key-string (read-line)))))
	      #f)))))

(define (common:simple-file-lock-and-wait fname #!key (expire-time 300))
  (let ((end-time (+ expire-time (current-seconds))))
    (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time)))
      (if got-lock
	  #t
	  (if (> end-time (current-seconds))
	      (begin
		(thread-sleep! 3)
		(loop (common:simple-file-lock fname expire-time: expire-time)))
	      #f)))))

(define (common:simple-file-release-lock fname)
  (handle-exceptions
   exn
   #f ;; I don't really care why this failed (at least for now)
   (delete-file* fname)))

;;======================================================================
;; U S E F U L   S T U F F
;;======================================================================

;;======================================================================
;; convert things to an alist or assoc list, #f gets converted to ""

(define *common:not-started-ok-statuses* ;; if not one of these statuses when in not_started state treat as dead
  '("n/a" "na" "PASS" "FAIL" "WARN" "CHECK" "WAIVED" "DEAD" "SKIP"))

;; group tests into buckets corresponding to rollup
;;; Running, completed-pass,  completed-non-pass + worst status, not started.
;; filter out 
					;(define (common:categorize-items-for-rollup in-tests)
					;  (

(define (common:special-sort items order comp)
  (let ((items-order (map reverse order))
        (acomp       (or comp >)))
    (sort items
	  (lambda (a b)
	    (let ((a-num (cadr (or (assoc a items-order) '(0 0))))
		  (b-num (cadr (or (assoc b items-order) '(0 0)))))
	      (acomp a-num b-num))))))

;;======================================================================
;; ;; given a toplevel with currstate, currstatus apply state and status
;; ;;  => (newstate . newstatus)
;; (define (common:apply-state-status currstate currstatus state status)
;;   (let* ((cstate  (string->symbol (string-downcase currstate)))
;;          (cstatus (string->symbol (string-downcase currstatus)))

;;                ((pass warn fail check n/a waived skip deleted stuck/dead stuck)  fail)
;;                ((abort)                                                         abort)
;;                (else                                                  unknown-error-6)))
;;             (else    unknown-error-7)))
;;     (cons 
;;      (if nstate  (symbol->string nstate)  nstate)
;;      (if nstatus (symbol->string nstatus) nstatus))))

;;======================================================================
;; D E B U G G I N G   S T U F F 
;;======================================================================

(define (common:debug-setup)
  (debug:setup (cond ;; debug arg
		((args:get-arg "-debug-noprop") 'noprop)

;;======================================================================
;; E X I T   H A N D L I N G
;;======================================================================

;;   (let ((ohh (common:on-homehost?))
;; 	(srv (args:get-arg "-server")))
;;     (and ohh srv)))
;; (debug:print-info 0 *default-log-port* "common:run-sync? ohh=" ohh ", srv=" srv)



(define *wdnum* 0)
(define *wdnum*mutex (make-mutex))

	(let ((res (with-input-from-pipe (conc "which " hed) read-line)))
	  (if (and (string? res)
		   (common:file-exists? res))
	      res
	      (if (null? tal)
		  #f
		  (loop (car tal)(cdr tal))))))))

(define (common:get-install-area)
  (let ((exe-path (car (argv))))
    (if (common:file-exists? exe-path)
	(handle-exceptions
	 exn
	 #f
	 (pathname-directory

      #f
      (let loop ((hed (car dirs))
		 (tal (cdr dirs)))
	(let ((res (or (and (directory? hed)
			    (file-write-access? hed)
			    hed)
		       (handle-exceptions
			exn
			(begin
			  (debug:print-info 0 *default-log-port* "could not create " hed
					    ", this might cause problems down the road. exn=" exn)
			  #f)
			(create-directory hed #t)))))
	  (if (and (string? res)
		   (directory? res))
	      res
	      (if (null? tal)
		  #f
		  (loop (car tal)(cdr tal))))))))

;;======================================================================
;; return the youngest timestamp . filename
;;
(define (common:get-youngest glob-list)
  (let ((all-files (apply append
			  (map (lambda (patt)
				 (handle-exceptions
				  exn
				  '()
				  (glob patt)))
			       glob-list))))
    (fold (lambda (fname res)
	    (let ((last-mod (car res))
		  (curmod   (handle-exceptions
			     exn
			     0
			     (file-modification-time fname))))
	      (if (> curmod last-mod)
		  (list curmod fname)
		  res)))
	  '(0 "n/a")
	  all-files)))

;;======================================================================
;; use bash to expand a glob. Does NOT handle paths with spaces!
;;
(define (common:bash-glob instr)
  (string-split
   (with-input-from-pipe
    (conc "/bin/bash -c \"echo " instr "\"")
    read-line)))

;;======================================================================
;; Some safety net stuff
;;======================================================================

;;======================================================================
;; return input if it is a list or return null
(define (common:list-or-null inlst #!key (ovrd #f)(message #f))
  (if (list? inlst)
      inlst
      (begin
	(if message (debug:print-error 0 *default-log-port* message))
	(or ovrd '()))))

(define (common:get-fields cfgdat)
  (let ((fields (hash-table-ref/default cfgdat "fields" '())))
    (map car fields)))

(define keys:config-get-fields common:get-fields)

;;======================================================================

	     val-list)
	'())))

;;======================================================================
;; S Y S T E M   S T U F F
;;======================================================================
























;;======================================================================
;; returns *effective load* (not normalized)
;;
(define (common:get-intercept onemin fivemin)
  (if (< onemin fivemin) ;; load is decreasing, just use the onemin load
      onemin
      (let* ((load-change (- onemin fivemin))

	     (delfile  (lambda (exn)
			 (debug:print-info 1 *default-log-port* " removing bad file " fullpath ", exn=" exn)
			 (delete-file* fullpath)
			 #f)))
	(if (and (file-exists? fullpath)
		 (file-read-access? fullpath))
	    (handle-exceptions
	     exn
	     (begin
	       (debug:print 0 *default-log-port* "failed to get cached info from " fullpath ", exn=" exn)
	       #f)
	     (debug:print 2 *default-log-port* "reading file " fullpath)
	     (let ((real-age (- (current-seconds)
				(handle-exceptions
				 exn
				 (begin
				   (debug:print 1 *default-log-port* "Failed to read mod time on file "
						fullpath ", using 0, exn=" exn)
				   0)
				 (file-change-time fullpath)))))
	       (if (< real-age age)
		   (handle-exceptions
		    exn
		    (delfile exn)
		    (let* ((res (with-input-from-file fullpath read)))
		      (if (eof-object? res)
			  (begin
			    (delfile "n/a")
			    #f)
			  res)))
		   (begin
		     (debug:print-info 2 *default-log-port* "file " fullpath
				       " is too old (" real-age" seconds) to trust, skipping reading it")
		     #f))))
	    (begin
	      (debug:print 2 *default-log-port* "not reading file " fullpath)
	      #f)))
      #f))

(define (common:write-cached-info key dtype dat)
  (if *toppath*
      (let* ((fulldir  (conc *toppath* "/.sysdata"))
	     (fullpath (conc fulldir "/" key "-" dtype ".log")))
	(if (not (file-exists? fulldir))(create-directory fulldir #t))
	(handle-exceptions
	 exn
	 (begin
	   (debug:print 0 *default-log-port* "failed to write file " fullpath ", exn=" exn)
	   #f)
	 (with-output-to-file fullpath (lambda ()(pp dat)))))
      #f))

(define (common:raw-get-remote-host-load remote-host)
  (handle-exceptions
   exn
   (begin
     (debug:print 0 *default-log-port* "failed to ssh to " remote-host " and get loadavg. exn=" exn)
     #f) ;; more specific handling of errors needed
   (with-input-from-pipe 
    (conc "ssh " remote-host " cat /proc/loadavg")
    (lambda ()(list (read)(read)(read))))))

;;======================================================================
;; get cpu load by reading from /proc/loadavg, return all three values
;;
(define (common:get-cpu-load remote-host)
  ;; (handle-exceptions
  ;;     exn
  ;;   (begin
  ;;     (debug:print 0 *default-log-port* "failed to ssh or read loadavg from host " remote-host ", exn=" exn)
  ;;     '(-99 -99 -99))
  (let* ((actual-hostname (or remote-host (get-host-name) "localhost")))
    (or (common:get-cached-info actual-hostname "cpu-load")
	(let ((result (if remote-host
			  (map (lambda (res)
				 (if (eof-object? res) 9e99 res))
			       (with-input-from-pipe 
				(conc "ssh " remote-host " cat /proc/loadavg")
				(lambda ()(list (read)(read)(read)))))
			  (with-input-from-file "/proc/loadavg" 
			    (lambda ()(list (read)(read)(read)))))))
	  (match
	   result
	   ((l1 l2 l3)
	    (if (and (number? l1)
		     (number? l2)
		     (number? l3))
		(begin
		  (common:write-cached-info actual-hostname "cpu-load" result)
		  result)
		'(-1 -1 -1))) ;; -1 is bad result
	   (else '(-2 -2 -2))))))) ;; )

;;======================================================================
;; get normalized cpu load by reading from /proc/loadavg and /proc/cpuinfo return all three values and the number of real cpus and the number of threads
;; returns alist '((adj-cpu-load . normalized-proc-load) ... etc.
;;  keys: adj-proc-load, adj-core-load, 1m-load, 5m-load, 15m-load
;;
(define (common:get-normalized-cpu-load remote-host)

     (else default))))

(define (common:get-normalized-cpu-load-raw remote-host)
  (let* ((actual-host (or remote-host (get-host-name)))) ;; #f is localhost
    (or (common:get-cached-info actual-host "normalized-load")
	(let ((data (if remote-host
			(with-input-from-pipe 
			 (conc "ssh " remote-host " \"cat /proc/loadavg;cat /proc/cpuinfo;echo end\"")
			 read-lines)
			(append 
			 (with-input-from-file "/proc/loadavg" 
			   read-lines)
			 (with-input-from-file "/proc/cpuinfo"
			   read-lines)
			 (list "end"))))
	      (load-rx  (regexp "^([\\d\\.]+)\\s+([\\d\\.]+)\\s+([\\d\\.]+)\\s+.*$"))

					  (cons '5m-load (cadr loads))
					  (cons '15m-load (caddr loads)))
				    (list (cons 'proc act-proc)
					  (cons 'core act-core)
					  (cons 'phys act-phys)))))
		      (common:write-cached-info actual-host "normalized-load" result)
		      result)
		    (regex-case
		     hed
		     (load-rx  ( x l1 l5 l15 ) (loop (car tal)(cdr tal)(map string->number (list l1 l5 l15)) proc-num phys-num core-num))
		     (proc-rx  ( x p         ) (loop (car tal)(cdr tal) loads           (max-num p proc-num) phys-num core-num))
		     (phys-rx  ( x p         ) (loop (car tal)(cdr tal) loads           proc-num (max-num p phys-num) core-num))
		     (core-rx  ( x c         ) (loop (car tal)(cdr tal) loads           proc-num phys-num (max-num c core-num)))
		     (else 
		      (begin
			;; (print "NO MATCH: " hed)
			(loop (car tal)(cdr tal) loads proc-num phys-num core-num)))))))))))

(define (common:unix-ping hostname)
  (let ((res (system (conc "ping -c 1 " hostname " > /dev/null"))))
    (eq? res 0)))

(define (common:get-num-cpus remote-host)
  (let* ((actual-host (or remote-host (get-host-name))))

	    (let ((m (string-match actions-regex s)))
	      (if m
		  (vector (regexp (cadr m))(string->symbol (caddr m))(cadr m))
		  (begin
		    (debug:print 0 *default-log-port* "WARNING: Unrecognised rule \"" s "\" in clean-up specification.")
		    #f))))
	  spec-strings))))

;;======================================================================
;; given a list of specs rx . rule and a file return the first matching rule
;; 
(define (common:file-find-rule fname rules) ;; rule is vector #( rx action rx-string)
  (let loop ((rule   (car rules))
	     (tail   (cdr rules)))
    (let ((rx (vector-ref rule 0))

(define (bb-check-path #!key (msg "check-path: "))
  (let ((path (or (get-environment-variable "PATH") "none")))
    (debug:print-info 0 *default-log-port* (conc msg" : $PATH="path))
    (if (string-match "^.*/isoenv-core/.*" path)
        (debug:print-error 0 *default-log-port* (conc msg" : !!ISOENV PRESENT!!")) ;; remove for prod
        (debug:print-info 1 *default-log-port* (conc msg" : **no isoenv present**")))))


(define (save-environment-as-files fname #!key (ignorevars (list "USER" "HOME" "DISPLAY" "LS_COLORS" "XKEYSYMDB" "EDITOR" "MAKEFLAGS" "MAKEF" "MAKEOVERRIDES")))
  ;;(bb-check-path msg: "save-environment-as-files entry")
  (let ((envvars (get-environment-variables))
        (whitesp (regexp "[^a-zA-Z0-9_\\-:,\\.\\/%$]"))
	(mungeval (lambda (val)
		    (cond
		     ((eq? val #t) "") ;; convert #t to empty string
		     ((eq? val #f) #f) ;; convert #f to itself (still thinking about this one
		     (else val)))))
    (with-output-to-file (conc fname ".csh")
      (lambda ()
	(for-each (lambda (keyval)
		    (let* ((key   (car keyval))
			   (val   (cdr keyval))
			   (delim (if (string-search whitesp val) 
				      "\""
				      "")))
		      (print (if (or (member key ignorevars)
				     (string-search whitesp key))
				 "# setenv "
				 "setenv ")
			     key " " delim (mungeval val) delim)))
		  envvars)))
    (with-output-to-file (conc fname ".sh")
      (lambda ()
	(for-each (lambda (keyval)
		    (let* ((key (car keyval))
			   (val (cdr keyval))
			   (delim (if (string-search whitesp val) 
				      "\""
				      "")))
		      (print (if (or (member key ignorevars)
				     (string-search whitesp key)
				     (string-search ":" key)) ;; internal only values to be skipped.
				 "# export "
				 "export ")
			     key "=" delim (mungeval val) delim)))
		  envvars)))))

;;======================================================================
;;
(define (common:propogate-mt-vars-to-subrun proc propogate-vars)
  (let ((vars      (make-hash-table))
        (var-patt  "^MT_.*"))
    (for-each
     (lambda (vardat) ;; each env var
					;(for-each
					;(lambda (var-patt)
       (if (string-match var-patt (car vardat))
	   (let ((var (car vardat))
		 (val (cdr vardat)))
	     (hash-table-set! vars var val)
	     (if (member var propogate-vars)
		 (begin
		   (print var "  " (string-substitute "MT_" "PARENT_" var))
		   (setenv (string-substitute "MT_" "PARENT_"  var) val)))
	     (unsetenv var))))
					;       var-patts))
     (get-environment-variables))
    (cond
     ((string? proc)(system proc))
     (proc          (proc)))
    (hash-table-for-each
     vars
     (lambda (var val)
       (if (member var propogate-vars)
	   (unsetenv (string-substitute "MT_" "PARENT_" var)))  
       (setenv var val)))
    vars))


(define (common:get-param-mapping #!key (flavor #f))
  "returns alist mapping string keys in testconfig/subrun to megatest command line switches; if flavor is switch-symbol, maps tcmt symbolic switches to megatest switches"
  (let ((default '(("tag-expr"  . "-tagexpr")

     (lambda (var val)
       (setenv var val)))
    vars))

;;======================================================================
;; C O L O R S
;;======================================================================

(define (common:name->iup-color name)
  (case (string->symbol (string-downcase name))
    ((red)    "223 33 49")
    ((grey)   "192 192 192")
    ((orange) "255 172 13")
    ((purple) "This is unfinished ...")))

		 (repl  (if (> (length parts) 1)(cadr parts) ""))

		 (newr  (if (and patt repl)
			    (begin
                              (handle-exceptions
                               exn
                               (begin
				 (debug:print 0 *default-log-port*
					      "WARNING: itemmap has problem \"" itemmap "\", patt: " patt ", repl: " repl ", exn=" exn)
                                 res)
			       (string-substitute patt repl res))


                              )
			    (begin
                              (debug:print 0 *default-log-port*
					   "WARNING: itemmap has problem \"" itemmap "\", patt: " patt ", repl: " repl)
			      res))))
	    (if (null? tal)
		newr
		(loop (car tal)(cdr tal) newr)))))))


(define (keys->valslots keys) ;; => ?,?,? ....

  (if (and host pid (not (equal? host "n/a")))
      (let* ((is-local (equal? host (get-host-name)))
	     (ssh-cmd   (if is-local " " (conc "ssh " host " ")))
	     (cmd (conc ssh-cmd "pstree -A " pid))
	     (output (with-input-from-pipe cmd read-lines)))
	(debug:print 2 *default-log-port* "Running " cmd " received " output)
	(if (eq? (length output) 0)
	    #f
	    #t))
      #t))

;;======================================================================
;; N A N O M S G   C L I E N T
;;======================================================================
;; 
;; 
;; 
;; (define (common:send-dboard-main-changed)

;; 
;; ;; get registered dashboards
;; ;;
;; (define (mddb:get-dashboards)
;;   (let ((db (mddb:open-db)))
;;     (query fetch-column
;; 	   (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;"))))

;;======================================================================
;; NMSG AND NEW API
;;======================================================================

;; nm based server experiment, keep around for now.
;;
#;(define (nm:start-server dbconn #!key (given-host-name #f))
(let* ((srvdat    (start-raw-server given-host-name: given-host-name))
       (host-name (srvdat-host srvdat))
       (soc       (srvdat-soc srvdat)))
  
  ;; start the queue processor (save for second round of development)
  ;;
  (thread-start! (make-thread! (lambda ()(queue-processor dbconn) "Queue processor")))
  ;; msg is an alist
  ;;  'r host:port  <== where to return the data
  ;;  'p params     <== data to apply the command to
  ;;  'e j|s|l      <== encoding of the params. default is s (sexp), if not specified is assumed to be default
  ;;  'c command    <== look up the function to call using this key
  ;;
  (let loop ((msg-in (nn-recv soc)))
    (if (not (equal? msg-in "quit"))
	(let* ((dat        (decode msg-in))
	       (host-port  (alist-ref 'r dat)) ;; this is for the reverse req rep where the server is a client of the original client
	       (params     (alist-ref 'p dat))
	       (command    (let ((c (alist-ref 'c dat)))(if c (string->symbol c) #f)))
	       (all-good   (and host-port params command (hash-table-exists? *commands* command))))
	  (if all-good
	      (let ((cmddat (make-qitem
			     command:   command
			     host-port: host-port
			     params:    params)))
		(queue-push cmddat) 		;; put request into the queue
		(nn-send soc "queued"))         ;; reply with "queued"
	      (print "ERROR: ["(common:human-time)"] BAD request " dat))
	  (loop (nn-recv soc)))))
  (nn-close soc)))

;;======================================================================
;; H I E R A R C H I C A L   H A S H   T A B L E S
;;======================================================================

;;======================================================================
;; Every element including top element is a vector:

		  (let ((new-sub-hh (hh:make-hh)))
		    (hash-table-set! sub-ht (car keys) new-sub-hh)
		    (apply hh:set! new-sub-hh value (cdr keys)))
		  (apply hh:set! sub-hh value (cdr keys))))    ;; call the sub-hierhash with remaining keys
	    (begin
	      (hh:set-ht! hh (make-hash-table))
	      (apply hh:set! hh value keys))))))

;;======================================================================
;; Manage pkts, used in servers, tests and likely other contexts so put
;; in common
;;======================================================================

(define common:pkts-spec
  '((default . ((parent    . P)
                (action    . a)
                (filename  . f)))
    (configf . ((parent    . P)
                (action    . a)
                (filename  . f)))
    (server  . ((action    . a)
		(pid       . d)
		(ipaddr    . i)
		(port      . p)
		(parent    . P)))
    
    (test    . ((cpuuse    . c)
		(diskuse   . d)
		(item-path . i)
		(runname   . r)
		(state     . s)
		(target    . t)
		(status    . u)

   (sort 
    (map (lambda (x)
	   `(,(alist-ref 't x) . ,(string->number (alist-ref 'D x))))
	 pkts)
    (lambda (a b)(> (cdr a)(cdr b))))      ;; sort descending
   (lambda (a b)(equal? (car a)(car b))))) ;; remove duplicates by target










































(define *common:thread-punchlist* (make-hash-table))
(define (common:send-thunk-to-background-thread thunk #!key (name #f))
  ;;(BB> "launched thread " name)

  ;; we need a unique name for the thread.
  (let* ((realname (if name

(define (common:join-backgrounded-threads)
  ;; may need to trap and ignore exceptions -- dunno how atomic threads are...
  (for-each
   (lambda (thread-name)
     (let* ((thread (hash-table-ref/default *common:thread-punchlist* thread-name #f)))
       (if thread
           (handle-exceptions
	    exn
	    (begin
	      (debug:print 0 *default-log-port* "joining threads failed. exn=" exn)
	      #t) ;; just ignore it, it might have died in the meantime so joining it will throw an exception
	    (thread-join! thread))
           )))
   (hash-table-keys *common:thread-punchlist*)))

;;======================================================================
;; (define *common:telemetry-log-state* 'startup)
;; (define *common:telemetry-log-socket* #f)
;; 

  (let loop ((res ""))
    (if (not (eof-object? (peek-char port)))
	(loop (conc res (read-char port)))
	res)))

(define (process:cmd-run-with-stderr->list cmd . params)
  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
  ;;  (handle-exceptions
  ;;   exn
  ;;   (begin
  ;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
  ;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
  ;;     #f)
  (let-values (((fh fho pid fhe) (if (null? params)
				     (process* cmd)
				     (process* cmd params))))
    (let loop ((curr (read-line fh))
	       (result  '()))
      (let ((errstr (process:conservative-read fhe)))
	(if (not (string=? errstr ""))
	    (set! result (append result (list errstr)))))
      (if (not (eof-object? curr))
	  (loop (read-line fh)
		(append result (list curr)))
	  (begin
	    (close-input-port fh)
	    (close-input-port fhe)
	    (close-output-port fho)
	    result))))) ;; )

(define (process:cmd-run-with-stderr-and-exitcode->list cmd . params)
  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
  ;;  (handle-exceptions
  ;;   exn
  ;;   (begin
  ;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
  ;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
  ;;     #f)
  (let-values (((fh fho pid fhe) (if (null? params)
				     (process* cmd)
				     (process* cmd params))))
    (let loop ((curr (read-line fh))
	       (result  '()))
      (let ((errstr (process:conservative-read fhe)))
	(if (not (string=? errstr ""))
	    (set! result (append result (list errstr)))))
      (if (not (eof-object? curr))
	  (loop (read-line fh)
		(append result (list curr)))
	  (begin
	    (let-values (((anotherpid normalexit? exitstatus)  (process-wait pid)))
	      (close-input-port fh)
	      (close-input-port fhe)
	      (close-output-port fho)
	      (list result (if normalexit? exitstatus -1))))))))

(define (process:cmd-run-proc-each-line cmd proc . params)
  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
  (handle-exceptions
   exn
   (begin
     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
     (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
     (debug:print 5 *default-log-port* "exn=" (condition->list exn))
     #f)
   (let-values (((fh fho pid) (if (null? params)
				  (process cmd)
				  (process cmd params))))
     (let loop ((curr (read-line fh))
		(result  '()))
       (if (not (eof-object? curr))
	   (loop (read-line fh)
		 (append result (list (proc curr))))
	   (begin
	     (close-input-port fh)
	     ;;(close-input-port fhe)
	     (close-output-port fho)
	     result))))))

(define (process:cmd-run-proc-each-line-alt cmd proc)
  (let* ((fh (open-input-pipe cmd))
         (res (port-proc->list fh proc))
         (status (close-input-pipe fh)))
    (if (eq? status 0) res #f)))



















(define (port-proc->list fh proc)
  (if (eof-object? fh) #f
      (let loop ((curr (proc (read-line fh)))
                 (result '()))
        (if (not (eof-object? curr))
            (loop (let ((l (read-line fh)))

	   (directory-exists? run-dir))
      (push-directory run-dir))
  (let ((pid (if params
		 (process-run cmdline params)
		 (process-run cmdline))))
    (let loop ((i 0))
      (let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
	(if (eq? pid-val 0)
	    (begin
	      (thread-sleep! 2)
	      (loop (+ i 1)))
	    (begin
	      (if (and run-dir
		       (directory-exists? run-dir))
		  (pop-directory))
	      (values pid-val exit-status exit-code)))))))

;;======================================================================
;; MISC PROCESS RELATED STUFF
;;======================================================================

(define (process:children proc)
  (with-input-from-pipe
   (conc "ps h --ppid " (current-process-id) " -o pid")

(define (process:alive? pid)
  (handle-exceptions
   exn
   ;; possibly pid is a process not a child, look in /proc to see if it is running still
   (common:file-exists? (conc "/proc/" pid))
   (let-values (((rpid exit-type exit-signal)(process-wait pid #t)))
     (and (number? rpid)
	  (equal? rpid pid)))))

(define (process:alive-on-host? host pid)
  (let ((cmd (conc "ssh " host " ps -o pid= -p " pid)))
    (handle-exceptions
     exn
     (begin
       (debug:print 0 *default-log-port* "failed to identify if process " pid ", on host " host " is alive. exn=" exn)
       #f) ;; anything goes wrong - assume the process in NOT running.
     (with-input-from-pipe 
      cmd
      (lambda ()
	(let loop ((inl (read-line)))
	  (if (eof-object? inl)
	      #f
	      (let* ((clean-str (string-substitute "^[^\\d]*([0-9]+)[^\\d]*$" "\\1" inl))

;;======================================================================
;; no elegance here ...
;;
(define (tasks:kill-server hostname pid #!key (kill-switch ""))
  (debug:print-info 0 *default-log-port* "Attempting to kill server process " pid " on host " hostname)
  (setenv "TARGETHOST" hostname)
  (let* ((logdir (if (directory-exists? "logs")
		     "logs/"
		     ""))
         (logfile (if logdir (conc "logs/server-"pid"-"hostname".log") #f))
         (gzfile  (if logfile (conc logfile ".gz"))))
    (setenv "TARGETHOST_LOGF" (conc logdir "server-kills.log"))

    (system (conc "nbfake kill "kill-switch" "pid))

    (when logfile
	  (thread-sleep! 0.5)
	  (if (common:file-exists? gzfile) (delete-file gzfile))
	  (system (conc "gzip " logfile))
	  
	  (unsetenv "TARGETHOST_LOGF")
	  (unsetenv "TARGETHOST"))))


;;======================================================================
;; hash-table tree to html list tree
;;
;;   tipfunc takes two parameters: y the tip value and path the path to that point
;;
(define (common:htree->html ht path tipfunc)
  (let ((datlist 	(sort (hash-table->alist ht)
			      ;;======================================================================
			      ;;======================================================================
                              (lambda (a b)
                                (string< (car a)(car b))))))
    (if (null? datlist)
    	(tipfunc #f path) ;; really shouldn't get here
	(s:ul
	 (map (lambda (x)
		(let* ((levelname (car x))

;;          logs directory you wish to log-rotate.
;;
(define (common:rotate-logs)
  (let* ((all-files (make-hash-table))
	 (stats     (make-hash-table))
	 (inc-stat  (lambda (key)
		      (hash-table-set! stats key (+ (hash-table-ref/default stats key 0) 1))))
	 (max-allowed (string->number (or (configf:lookup *configdat* "setup" "max-logfiles") "300")))) ;; name -> age
    (if (not (directory-exists? "logs"))(create-directory "logs"))
    (directory-fold 
     (lambda (file rem)
       (handle-exceptions
	exn
	(begin
	  (debug:print-info 2 *default-log-port* "unable to rotate log " file ", probably handled by another process, this is safe to ignore. exn=" exn)

		     (debug:print-info 0 *default-log-port* fullname " in logs directory is a directory! Cannot rotate it, it is best to not put subdirectories in the logs dir.")
		     (handle-exceptions
		      exn
		      (debug:print-error 0 *default-log-port* "failed to remove " fullname ", exn=" exn)
		      (delete-file* fullname)))))
	     files)
	    (debug:print-info 0 *default-log-port* "Deleted " (length files) " files from logs, keeping " max-allowed " files."))))))

;;======================================================================
;; calculate a delay number based on a droop curve
;;    inputs are:
;;      - load-in, load as from uptime, NOT normalized
;;      - numcpus, number of cpus, ideally use the real cpus, not threads
;;
(define (common:get-delay load-in numcpus)

		((and (>= ratio 0.5) (<= ratio .9))
		 (* ratio (/ 5 .9)))
		((and (> ratio .9) (<= ratio 1.1))
		 (+ 5 (* (- ratio .9) (/ 55 .2)))) 
		((> ratio 1.1)
		 60)))
	(match paramlst
	       ((r1 r2 s1 s2)
		(debug:print 3 *default-log-port* "Using params r1=" r1 " r2=" r2 " s1=" s1 " s2=" s2)
		(min (max (/ (expt r1 (* r2 s2 ratio)) s1) 0) 30))
	       (else
		(debug:print 0 *default-log-port* "BAD exp-params, should be \"r1 r2 s1 s2\" but got " paramstr)
		30)))))

(define (common:print-delay-table)
  (let loop ((x 0))
    (print x "," (common:get-delay x 1))
    (if (< x 2)
	(loop (+ x 0.1)))))

;;======================================================================
;; DO NOT CALL THIS DIRECTLY. It is called from common:wait-for-normalized-load
;;   count     - count down to zero, at some point we'd give up if the load never drops
;;   num-tries - count down to zero number tries to get numcpus
;;
(define (common:wait-for-cpuload maxnormload numcpus-in
				 #!key (count 1000)
				 (msg #f)(remote-host #f)(num-tries 5))
  (let* ((loadavg     (common:get-cpu-load remote-host))
	 ;; not possible to have zero.  If we get 1, it's possible that we got the previous default, and we should check again
	 (numcpus     (if (<= 1 numcpus-in)
			  (common:get-num-cpus remote-host)
			  numcpus-in))
	 (first       (car loadavg))
	 (next        (cadr loadavg))
	 (adjmaxload  (* maxnormload (max 1 numcpus))) ;; possible bug
	 ;; where numcpus
	 ;; (or could be
	 ;; maxload) is
	 ;; zero, crude
	 ;; fallback is to
	 ;; at least use 1
	 ;; effective load accounts for load jumps, this should elminate all the first-next-avg, adjwait, load-jump-limit
	 ;; etc.
	 (effective-load    (common:get-intercept first next))
	 (recommended-delay (common:get-delay effective-load numcpus))
	 (effective-host    (or remote-host "localhost"))
	 (normalized-effective-load (/ effective-load numcpus))
	 (will-wait                 (> normalized-effective-load maxnormload)))

	  dbspace
	  required
	  dirpath)))

(define (get-free-inodes path)
  (if (configf:lookup *configdat* "setup" "free-inodes-script")
      (with-input-from-pipe 
       (conc (configf:lookup *configdat* "setup" "free-inodes-script") " " path)
       ;;======================================================================
       ;;======================================================================
       ;;======================================================================
       ;;======================================================================
       ;;======================================================================

       (lambda ()
	 (let ((res (read-line)))
	   (if (string? res)
	       (string->number res)))))
      (get-unix-inodes path)))
;;======================================================================
;;======================================================================
;;======================================================================
;;======================================================================
;;======================================================================
;;======================================================================

      (debug:print 0 *default-log-port* "ERROR: settings are missing in your megatest.config for area management.")
      (debug:print  0 *default-log-port* "  you need to have pktsdirs in the [setup] section."))
     ((not (common:file-exists? pktsdir))
      (debug:print 0 *default-log-port* "ERROR: pkts directory not found " pktsdir))
     ((not (equal? (file-owner pktsdir)(current-effective-user-id)))
      (debug:print 0 *default-log-port* "ERROR: directory " pktsdir " is not owned by " (current-effective-user-name)))
     (else
      (let* ((pdb  (open-queue-db pdbpath "pkts.db"
				  schema: '("CREATE TABLE groups (id INTEGER PRIMARY KEY,groupname TEXT, CONSTRAINT group_constraint UNIQUE (groupname));"))))
	(proc pktsdirs pktsdir pdb)
	(dbi:close pdb))))))

;;======================================================================
;; check space in dbdir and in megatest dir
;; returns: ok/not dbspace required-space
;;
(define (common:check-db-dir-space)
  (let* ((required (string->number 

	  (exit 1)))))

;;======================================================================
;; paths is list of lists ((name path) ... )
;;
(define (common:get-disk-with-most-free-space disks minsize)
  (let* ((best     #f)
	 (bestsize 0)
	 (default-min-inodes-string "1000000")
	 (default-min-inodes (string->number default-min-inodes-string))
	 (min-inodes (or (string->number (if (configf:lookup *configdat* "setup" "min_inodes") (configf:lookup *configdat* "setup" "min_inodes") default-min-inodes-string)) default-min-inodes)))

    (for-each 
     (lambda (disk-num)
       (let* ((dirpath    (cadr (assoc disk-num disks)))
	      (freespc    (cond
			   ((not (directory? dirpath))
			    (if (common:low-noise-print 300 "disks not a dir " disk-num)
				(debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not a directory - ignoring it."))
			    -1)
			   ((not (file-write-access? dirpath))
			    (if (common:low-noise-print 300 "disks not writeable " disk-num)
				(debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not writeable - ignoring it."))
			    -1)
			   ((not (eq? (string-ref dirpath 0) #\/))
			    (if (common:low-noise-print 300 "disks not a proper path " disk-num)
				(debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not a fully qualified path - ignoring it."))
			    -1)
			   (else
			    (get-df dirpath))))
	      (free-inodes (cond
			    ((not (directory? dirpath))
			     (if (common:low-noise-print 300 "disks not a dir " disk-num)
				 (debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not a directory - ignoring it."))
			     -1)
			    ((not (file-write-access? dirpath))
			     (if (common:low-noise-print 300 "disks not writeable " disk-num)
				 (debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not writeable - ignoring it."))
			     -1)
			    ((not (eq? (string-ref dirpath 0) #\/))
			     (if (common:low-noise-print 300 "disks not a proper path " disk-num)
				 (debug:print 0 *default-log-port* "WARNING: disk " disk-num " at path \"" dirpath "\" is not a fully qualified path - ignoring it."))
			     -1)
			    (else
			     (get-free-inodes dirpath))))
	      ;;(free-inodes (get-free-inodes dirpath))
	      )
	 (debug:print 2 *default-log-port* "INFO: disk " disk-num " path " dirpath " free space " freespc " free inodes " free-inodes)
	 (if (and (> freespc bestsize)(> free-inodes min-inodes ))
	     (begin
	       (set! best     (cons disk-num dirpath))
	       (set! bestsize freespc)))
	 ;;(print "Processing: " disk-num " bestsize: " bestsize " best: " best " freespc: " freespc " min-inodes: " min-inodes " free-inodes: " free-inodes)
	 ))
     (map car disks))
    (if (and best (> bestsize minsize))
	best
	#f))) ;; #f means no disk candidate found

(define (common:load-pkts-to-db mtconf #!key (use-lt #f))
  (common:with-queue-db

             (if (not (file-exists? pktsdir))
                 (create-directory pktsdir #t))
             (with-output-to-file
                 (conc pktsdir "/" uuid ".pkt")
               (lambda ()
                 (print pkt)))))))))
;;        (set! *common:telemetry-log-socket* #f)))))


(define (common:get-linktree)
  (or (getenv "MT_LINKTREE")
      (if *configdat*
	  (configf:lookup *configdat* "setup" "linktree")
	  #f)
      (if (or *toppath* (getenv "MT_RUN_AREA_HOME"))
	  (conc (or *toppath* (getenv "MT_RUN_AREA_HOME")) "/lt")
	  #f)
      (let* ((tp (common:get-toppath #f))
	     (lt (conc tp "/lt")))
	(if (not tp)(debug:print 0 *default-log-port* "WARNING: cannot calculate best path for linktree, using " lt))
	lt)))


(define (tests:get-tests-search-path cfgdat)
  (let ((paths (let ((section (if cfgdat
				  (configf:get-section cfgdat "tests-paths")
				  #f)))
		 (if section
		     (map cadr section)
		     '()))))
    (filter (lambda (d)
	      (if (directory-exists? d)
		  d
		  (begin
		    ;; (if (common:low-noise-print 60 "tests:get-tests-search-path" d)
		    ;;	(debug:print 0 *default-log-port* "WARNING: problem with directory " d ", dropping it from tests path"))
		    #f)))
	    (append paths (list (conc *toppath* "/tests"))))))

(define (server:get-best-guess-address hostname)
  (let ((res #f))
    (for-each 
     (lambda (adr)
       (if (not (eq? (u8vector-ref adr 0) 127))
	   (set! res adr)))
     ;; NOTE: This can fail when there is no mention of the host in /etc/hosts. FIXME
     (vector->list (hostinfo-addresses (hostname->hostinfo hostname))))
    (string-intersperse 
     (map number->string
	  (u8vector->list
	   (if res res (hostname->ip hostname)))) ".")))
;;======================================================================
;; logic for getting homehost. Returns (host . at-home)
;; IF *toppath* is not set, wait up to five seconds trying every two seconds
;; (this is to accomodate the watchdog)
;;
(define (common:get-homehost #!key (trynum 5))
  ;; called often especially at start up. use mutex to eliminate collisions
  (mutex-lock! *homehost-mutex*)
  (cond
   (*home-host*
    (mutex-unlock! *homehost-mutex*)
    *home-host*)
   ((not *toppath*)
    (mutex-unlock! *homehost-mutex*)
    ((launch:setup) ;; safely mutexed now
     (if (> trynum 0)
	 (begin
	   (thread-sleep! 2)
	   (common:get-homehost trynum: (- trynum 1)))
	 #f))
    (else
     (let* ((currhost (get-host-name))
	    (bestadrs (server:get-best-guess-address currhost))
	    ;; first look in config, then look in file .homehost, create it if not found
	    (homehost (or (configf:lookup *configdat* "server" "homehost" )
			  (handle-exceptions
			   exn
			   (if (> trynum 0)
			       (let ((delay-time (* (- 5 trynum) 5)))
				 (mutex-unlock! *homehost-mutex*)
				 (debug:print 0 *default-log-port* "ERROR: ["(common:human-time)"] Failed to read .homehost file, delaying "
					      delay-time " seconds and trying again, message: "  ((condition-property-accessor 'exn 'message) exn)
					      ", exn=" exn)
				 (thread-sleep! delay-time)
				 (common:get-homehost trynum: (- trynum 1)))
			       (begin
				 (mutex-unlock! *homehost-mutex*)
				 (debug:print 0 *default-log-port* "ERROR: ["(common:human-time)
					      "] Failed to read .homehost file after trying five times. Giving up and exiting, message: "
					      ((condition-property-accessor 'exn 'message) exn))
				 ;;======================================================================
				 (exit 1)))
			   (let ((hhf (conc *toppath* "/.homehost")))
			     (if (common:file-exists? hhf)
				 (with-input-from-file hhf read-line)
				 (if (file-write-access? *toppath*)
				     (begin
				       (with-output-to-file hhf
					 (lambda ()
					   (print bestadrs)))
				       (begin
					 (mutex-unlock! *homehost-mutex*)
					 (car (common:get-homehost))))
				     #f))))))
	    (at-home  (or (equal? homehost currhost)
			  (equal? homehost bestadrs))))
       (set! *home-host* (cons homehost at-home))
       (mutex-unlock! *homehost-mutex*)
       *home-host*)))))

(define (common:wait-for-homehost-load maxnormload msg)
  (let* ((hh-dat (if (common:on-homehost?) ;; if we are on the homehost then pass in #f so the calls are local.
		     #f
		     (common:get-homehost)))
	 (hh     (if hh-dat (car hh-dat) #f)))
    (common:wait-for-normalized-load maxnormload msg hh)))

;;======================================================================
;; am I on the homehost?
;;======================================================================
;;======================================================================
;;======================================================================
;;
(define (common:on-homehost?)
  (let ((hh (common:get-homehost)))
    (if hh
	(cdr hh)
	#f)))

(define (common:run-sync?)
  (and (common:on-homehost?)
       (args:get-arg "-server")))


;; gather available information, if legit read configs in this order:
;;
;;   if have cache;
;;      read it a return it
;;   else
;;     megatest.config     (do not cache)
;;     runconfigs.config   (cache if all vars avail)
;;     megatest.config     (cache if all vars avail)
;;   returns:
;;     *toppath*
;;   side effects:
;;     sets; *configdat*    (megatest.config info)
;;           *runconfigdat* (runconfigs.config info)
;;           *configstatus* (status of the read data)
;;
(define (launch:setup #!key (force-reread #f) (areapath #f))
  (mutex-lock! *launch-setup-mutex*)
  (if (and *toppath*
	   (eq? *configstatus* 'fulldata) (not force-reread)) ;; got it all
      (begin
	(debug:print 2 *default-log-port* "NOTE: skipping launch:setup-body call since we have fulldata")
	(mutex-unlock! *launch-setup-mutex*)
	*toppath*)
      (let ((res (launch:setup-body force-reread: force-reread areapath: areapath)))
	(mutex-unlock! *launch-setup-mutex*)
	res)))

(define (launch:setup-body #!key (force-reread #f) (areapath #f))
  (if (and (eq? *configstatus* 'fulldata)
	   *toppath*
	   (not force-reread)) ;; no need to reprocess
      *toppath*   ;; return toppath
      (let* ((use-cache (common:use-cache?)) ;; BB- use-cache checks *configdat* for use-cache setting.  We do not have *configdat*.  Bootstrapping problem here.
	     (toppath  (common:get-toppath areapath))
	     (target   (common:args-get-target))
	     (sections (if target (list "default" target) #f)) ;; for runconfigs
	     (mtconfig (or (args:get-arg "-config") "megatest.config")) ;; allow overriding megatest.config 
	     (cachefiles (launch:get-cache-file-paths areapath toppath target mtconfig))
	     ;; checking for null cachefiles should not be necessary, I was seeing error car of '(), might be a chicken bug or a red herring ...
	     (mtcachef   (if (null? cachefiles)
			     #f
			     (car cachefiles))) ;; (and cachedir (conc cachedir "/" ".megatest.cfg-"  megatest-version "-" megatest-fossil-hash)))
	     (rccachef   (if (null? cachefiles)
			     #f
			     (cdr cachefiles)))) ;; (and cachedir (conc cachedir "/" ".runconfigs.cfg-"  megatest-version "-" megatest-fossil-hash)))
	;; (cancreate (and cachedir (common:file-exists? cachedir)(file-write-access? cachedir) (not (common:in-running-test?)))))
	(set! *toppath* toppath) ;; This is needed when we are running as a test using CMDINFO as a datasource
	;;(BB> "launch:setup-body -- cachefiles="cachefiles)
	(cond
	 ;; if mtcachef exists just read it, however we need to assume toppath is available in $MT_RUN_AREA_HOME
	 ((and (not force-reread)
	       mtcachef  rccachef
	       use-cache
	       (get-environment-variable "MT_RUN_AREA_HOME")
	       (common:file-exists? mtcachef)
	       (common:file-exists? rccachef))
	  ;;(BB> "launch:setup-body -- cond branch 1 - use-cache")
	  (set! *configdat*    (configf:read-alist mtcachef))
	  (set! *db-keys*      (common:get-fields *configdat*))
	  ;;(BB> "launch:setup-body -- 1 set! *configdat*="*configdat*)
	  (set! *runconfigdat* (configf:read-alist rccachef))
	  (set! *configinfo*   (list *configdat*  (get-environment-variable "MT_RUN_AREA_HOME")))
	  (set! *configstatus* 'fulldata)
	  (set! *toppath*      (get-environment-variable "MT_RUN_AREA_HOME"))
	  *toppath*)
	 ;; there are no existing cached configs, do full reads of the configs and cache them
	 ;; we have all the info needed to fully process runconfigs and megatest.config
	 ((and ;; (not force-reread) ;; force-reread is irrelevant in the AND, could however OR it?
	   mtcachef
	   rccachef) ;; BB- why are we doing this without asking if caching is desired?
	  ;;(BB> "launch:setup-body -- cond branch 2")
	  (let* ((first-pass    (find-and-read-config        ;; NB// sets MT_RUN_AREA_HOME as side effect
				 mtconfig
				 environ-patt: "env-override"
				 given-toppath: toppath
				 pathenvvar: "MT_RUN_AREA_HOME"))
		 (first-rundat  (let ((toppath (if toppath 
						   toppath
						   (car first-pass))))
				  (read-config ;; (conc toppath "/runconfigs.config") ;; this should be converted to runconfig:read but it is non-trivial, leaving it for now.
				   (conc (if (string? toppath)
					     toppath
					     (get-environment-variable "MT_RUN_AREA_HOME"))
					 "/runconfigs.config")
				   *runconfigdat* #t 
				   sections: sections))))
	    (set! *runconfigdat* first-rundat)
	    (if first-pass  ;; 
		(begin
		  ;;(BB> "launch:setup-body -- \"first-pass\"=first-pass")
		  (set! *configdat*  (car first-pass))
		  ;;(BB> "launch:setup-body -- 2 set! *configdat*="*configdat*)
		  (set! *configinfo* first-pass)
		  (set! *toppath*    (or toppath (cadr first-pass))) ;; use the gathered data unless already have it
		  (set! toppath      *toppath*)
		  (set! *db-keys*    (common:get-fields *configdat*))
		  (if (not *toppath*)
		      (begin
			(debug:print-error 0 *default-log-port* "you are not in a megatest area!")
			(exit 1)))
		  (setenv "MT_RUN_AREA_HOME" *toppath*)
		  ;; the seed read is done, now read runconfigs, cache it then read megatest.config one more time and cache it
		  (let* ((keys         (common:list-or-null *db-keys* ;; (common:get-fields (rmt:get-keys)
							    message: "Failed to retrieve keys in launch.scm. Please report this to the developers."))
			 (key-vals     (keys:target->keyval keys target))
			 (linktree     (common:get-linktree)) ;; (or (getenv "MT_LINKTREE")(if *configdat* (configf:lookup *configdat* "setup" "linktree") #f)))
					;     (if *configdat*
					; 	   (configf:lookup *configdat* "setup" "linktree")
					; 	   (conc *toppath* "/lt"))))
			 (second-pass  (find-and-read-config
					mtconfig
					environ-patt: "env-override"
					given-toppath: toppath
					pathenvvar: "MT_RUN_AREA_HOME"))
			 (runconfigdat (begin     ;; this read of the runconfigs will see any adjustments made by re-reading megatest.config
					 (for-each (lambda (kt)
						     (setenv (car kt) (cadr kt)))
						   key-vals)
					 (read-config (conc toppath "/runconfigs.config") *runconfigdat* #t ;; consider using runconfig:read some day ...
						      sections: sections)))
			 (cachefiles   (launch:get-cache-file-paths areapath toppath target mtconfig))
			 (mtcachef     (car cachefiles))
			 (rccachef     (cdr cachefiles)))
		    ;;  trap exception due to stale NFS handle -- Error: (open-output-file) cannot open file - Stale NFS file handle: "/p/fdk/gwa/lefkowit/mtTesting/qa/primbeqa/links/p1222/11/PDK_r1.1.1/prim/clean/pcell_testgen/.runconfigs.cfg-1.6427-7d1e789cb3f62f9cde719a4865bb51b3c17ea853" - ticket 220546342
		    ;; TODO - consider 1) using simple-lock to bracket cache write
		    ;;                 2) cache in hash on server, since need to do rmt: anyway to lock.

		    (if rccachef
			(common:fail-safe
			 (lambda ()
			   (configf:write-alist runconfigdat rccachef))
			 (conc "Could not write cache file - "rccachef)))
		    (if mtcachef
			(common:fail-safe
			 (lambda ()
			   (configf:write-alist *configdat* mtcachef))
			 (conc "Could not write cache file - "mtcachef)))
		    (set! *runconfigdat* runconfigdat)
		    (if (and rccachef mtcachef) (set! *configstatus* 'fulldata))))
		;; no configs found? should not happen but let's try to recover gracefully, return an empty hash-table
		(set! *configdat* (make-hash-table))
		)))

	 ;; else read what you can and set the flag accordingly
	 ;; here we don't have either mtconfig or rccachef
	 (else
	  ;;(BB> "launch:setup-body -- cond branch 3 - else")
	  (let* ((cfgdat   (find-and-read-config 
			    (or (args:get-arg "-config") "megatest.config")
			    environ-patt: "env-override"
			    given-toppath: (get-environment-variable "MT_RUN_AREA_HOME")
			    pathenvvar: "MT_RUN_AREA_HOME")))

	    (if (and cfgdat (list? cfgdat) (> (length cfgdat) 0) (hash-table? (car cfgdat)))
		(let* ((toppath  (or (get-environment-variable "MT_RUN_AREA_HOME")(cadr cfgdat)))
		       (rdat     (read-config (conc toppath  ;; convert this to use runconfig:read!
						    "/runconfigs.config") *runconfigdat* #t sections: sections)))
		  (set! *configinfo*   cfgdat)
		  (set! *configdat*    (car cfgdat))
		  (set! *db-keys*    (common:get-fields *configdat*))
		  (set! *runconfigdat* rdat)
		  (set! *toppath*      toppath)
		  (set! *configstatus* 'partial))
		(begin
		  (debug:print-error 0 *default-log-port* "No " mtconfig " file found. Giving up.")
		  (exit 2))))))
	;; COND ends here.
	
	;; additional house keeping
	(let* ((linktree (or (common:get-linktree)
			     (conc *toppath* "/lt"))))
	  (if linktree
	      (begin
		(if (not (common:file-exists? linktree))
		    (begin
		      (handle-exceptions
		       exn
		       (begin
			 (debug:print-error 0 *default-log-port* "Something went wrong when trying to create linktree dir at " linktree)
			 (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn)
			 (exit 1))
		       (create-directory linktree #t))))
		(handle-exceptions
		 exn
		 (begin
		   (debug:print-error 0 *default-log-port* "Something went wrong when trying to create link to linktree at " *toppath*)
		   (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
		 (let ((tlink (conc *toppath* "/lt")))
		   (if (not (common:file-exists? tlink))
		       (create-symbolic-link linktree tlink)))))
	      (begin
		(debug:print-error 0 *default-log-port* "linktree not defined in [setup] section of megatest.config")
		)))
	(if (and *toppath*
		 (directory-exists? *toppath*))
	    (begin
	      (setenv "MT_RUN_AREA_HOME" *toppath*)
	      (setenv "MT_TESTSUITENAME" (common:get-testsuite-name)))
	    (begin
	      (debug:print-error 0 *default-log-port* "failed to find the top path to your Megatest area.")
	      (set! *toppath* #f) ;; force it to be false so we return #f
	      #f))
	
	;; one more attempt to cache the configs for future reading
	(let* ((cachefiles   (launch:get-cache-file-paths areapath toppath target mtconfig))
	       (mtcachef     (car cachefiles))
	       (rccachef     (cdr cachefiles)))

	  ;; trap exception due to stale NFS handle -- Error: (open-output-file) cannot open file - Stale NFS file handle: "...somepath.../.runconfigs.cfg-1.6427-7d1e789cb3f62f9cde719a4865bb51b3c17ea853" - ticket 220546342
	  ;; TODO - consider 1) using simple-lock to bracket cache write
	  ;;                 2) cache in hash on server, since need to do rmt: anyway to lock.
	  (if (and rccachef *runconfigdat* (not (common:file-exists? rccachef)))
	      (common:fail-safe
	       (lambda ()
		 (configf:write-alist *runconfigdat* rccachef))
	       (conc "Could not write cache file - "rccachef))
	      )
	  (if (and mtcachef *configdat*    (not (common:file-exists? mtcachef)))
	      (common:fail-safe
	       (lambda ()
		 (configf:write-alist *configdat* mtcachef))
	       (conc "Could not write cache file - "mtcachef))
	      )
	  (if (and rccachef mtcachef *runconfigdat* *configdat*)
	      (set! *configstatus* 'fulldata)))

	;; if have -append-config then read and append here
	(let ((cfname (args:get-arg "-append-config")))
	  (if (and cfname
		   (file-read-access? cfname))
	      (read-config cfname *configdat* #t))) ;; values are added to the hash, no need to do anything special.
	*toppath*)))

;; return paths depending on what info is available.
;;
(define (launch:get-cache-file-paths areapath toppath target mtconfig)
  (let* ((use-cache (common:use-cache?))
         (runname  (common:args-get-runname))
         (linktree (common:get-linktree))
         (testname (common:get-full-test-name))
         (rundir   (if (and runname target linktree)
                       (common:directory-writable? (conc linktree "/" target "/" runname))
                       #f))
         (testdir  (if (and rundir testname)
                       (common:directory-writable? (conc rundir "/" testname))
                       #f))
         (cachedir (or testdir rundir))
         (mtcachef (and cachedir (conc cachedir "/" ".megatest.cfg-"  megatest-version "-" megatest-fossil-hash)))
         (rccachef (and cachedir (conc cachedir "/" ".runconfigs.cfg-"  megatest-version "-" megatest-fossil-hash))))
    (debug:print-info 6 *default-log-port* 
                      "runname=" runname 
                      "\n  linktree=" linktree
                      "\n  testname=" testname
                      "\n  rundir=" rundir 
                      "\n  testdir=" testdir 
                      "\n  cachedir=" cachedir
                      "\n  mtcachef=" mtcachef
                      "\n  rccachef=" rccachef)
    (cons mtcachef rccachef)))


;;======================================================================the end

)