Megatest

      (debug:print 0 *default-log-port* "failed to ssh or read loadavg from host " remote-host ", exn=" exn)
      '(-99 -99 -99))
    (let* ((actual-hostname (or remote-host (get-host-name) "localhost")))
      (or (common:get-cached-info actual-hostname "cpu-load")
	  (let ((result (if remote-host
			    (map (lambda (res)
				   (if (eof-object? res) 9e99 res))

			         (common:raw-get-remote-host-load remote-host))

			    (with-input-from-file "/proc/loadavg" 
			      (lambda ()(list (read)(read)(read)))))))
	    (match
		result
	      ((l1 l2 l3)
	       (if (and (number? l1)
		      (number? l2)

     ((eq? res #f) default)   ;; this would be the #eof
     (else default))))

(define (common:get-normalized-cpu-load-raw remote-host)
  (let* ((actual-host (or remote-host (get-host-name)))) ;; #f is localhost
    (or (common:get-cached-info actual-host "normalized-load")
	(let ((data (if remote-host
			(let ((inp #f))
			  (handle-exceptions
			      exn
			    (begin
			      (close-input-port inp)
			      '())
			    (set! inp (open-input-port (conc "ssh " remote-host " \"cat /proc/loadavg;cat /proc/cpuinfo;echo end\"")))
			    (let* ((res (read-lines inp)))
			      (close-input-port inp)
			      res)))
			(append 
			 (with-input-from-file "/proc/loadavg" 
			   read-lines)
			 (with-input-from-file "/proc/cpuinfo"
			   read-lines)
			 (list "end"))))
	      (load-rx  (regexp "^([\\d\\.]+)\\s+([\\d\\.]+)\\s+([\\d\\.]+)\\s+.*$"))

						       numcpu
						       #f) ;; if zero return #f so caller knows that things are not working
						   (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl)
							     (+ numcpu 1)
							     numcpu)
							 (read-line))))))
				   (result (if remote-host
					       (common:generic-ssh
						(conc "ssh " remote-host " cat /proc/cpuinfo")
						proc -1)
					       (with-input-from-file "/proc/cpuinfo" proc))))
			      (if (and (number? result)
				       (> result 0))
				  (common:write-cached-info actual-host "num-cpus" result))
			      result))))
	  (hash-table-set! *numcpus-cache* actual-host numcpus)
	  numcpus))))

(define (common:generic-ssh ssh-command proc default #!optional (msg-proc #f))
  (let ((inp #f))
    (handle-exceptions
	exn
      (begin
	(close-input-port inp)
	(if msg-proc
	    (msg-proc)
	    (debug:print 0 *default-log-port* "Command: \""ssh-command"\" failed. exn="exn))
	default)
      (set! inp (open-input-pipe ssh-command))
      (with-input-from-port inp
	(lambda ()
	  (let ((res (proc)))
	    (close-input-port inp)
	    res))))))

;;======================================================================
;; wait for normalized cpu load to drop below maxload
;;
(define (common:wait-for-normalized-load maxnormload msg remote-host #!optional (rem-tries 5))
  (let ((num-cpus (common:get-num-cpus remote-host)))
    (if num-cpus

	     (output (with-input-from-pipe cmd read-lines)))
	(debug:print 2 *default-log-port* "Running " cmd " received " output)
	(if (eq? (length output) 0)
	   #f
	   #t))
      #t))

;; this is a close duplicate of:
;;    process:alist-on-host?
;;    process:alive
;;
(define (launch:is-test-alive host pid)
  (let* ((same-host (equal? host (get-host-name)))
	 (cmd (conc 
	       (if same-host "" (conc "ssh "host" "))
	       "pstree -A "pid)))
    (if (and host pid
	     (not (equal? host "n/a")))
	
	(let* ((output (if same-host
			   (with-input-from-pipe cmd read-lines)
			   (common:generic-ssh cmd read-lines '())))) ;; (with-input-from-pipe cmd read-lines)))
	  (debug:print 2 *default-log-port* "Running " cmd " received " output)
	  (if (eq? (length output) 0)
	      #f
	      #t))
	#t))) ;; assuming bad query is about a live test is likely not the right thing to do?

(define (launch:kill-tests-if-dead run-id)

   (common:file-exists? (conc "/proc/" pid))
   (let-values (((rpid exit-type exit-signal)(process-wait pid #t)))
       (and (number? rpid)
	    (equal? rpid pid)))))

(define (process:alive-on-host? host pid)
  (let ((cmd (conc "ssh " host " ps -o pid= -p " pid)))
    (common:generic-ssh
     cmd
   ;; 
   ;; handle-exceptions
   ;; 	exn
   ;;  (begin
   ;; 	(debug:print 0 *default-log-port* "failed to identify if process " pid ", on host " host " is alive. exn=" exn)
   ;; 	#f) ;; anything goes wrong - assume the process in NOT running.
   ;;  (with-input-from-pipe 
   ;;   cmd
     (lambda ()
       (let loop ((inl (read-line)))
	 (if (eof-object? inl)
	     #f
	     (let* ((clean-str (string-substitute "^[^\\d]*([0-9]+)[^\\d]*$" "\\1" inl))
		    (innum     (string->number clean-str)))
	       (and innum
		    (eq? pid innum))))))
     #f
     (lambda ()
       (debug:print 0 *default-log-port* "failed to identify if process "
		    pid", on host "host" is alive. exn="exn)))))


(define (process:get-sub-pids pid)
  (with-input-from-pipe
   (conc "pstree -A -p " pid) ;; | tr 'a-z\\-+`()\\.' ' ' " pid)
   (lambda ()
     (let loop ((inl (read-line))
		(res '()))