Megatest

Check-in [127cbef166]
Login
Overview
Comment:reduce print noise. re-start server in long running loop
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.60
Files: files | file ages | folders
SHA1: 127cbef166921024552bb9bbf919eb73cad8deb7
User & Date: mrwellan on 2014-11-18 10:18:38
Other Links: branch diff | manifest | tags
Context
2014-11-18
22:29
Fixed couple minor bugs check-in: 4e02f67409 user: matt tags: v1.60
10:18
reduce print noise. re-start server in long running loop check-in: 127cbef166 user: mrwellan tags: v1.60
09:32
Force start on servers before doing write heavy stuff. The transition when doing writes from direct access -> server needs improving. This is a work-around (but a good feature anyway. check-in: c101fd0ec9 user: mrwellan tags: v1.60
Changes

Modified runs.scm from [a8e4b28ff5] to [380bd479a2].

883
884
885
886
887
888
889
890


891
892
893
894
895
896
897
883
884
885
886
887
888
889

890
891
892
893
894
895
896
897
898







-
+
+







	(max-retries           (config-lookup *configdat* "setup" "maxretries"))
	(max-concurrent-jobs   (let ((mcj (config-lookup *configdat* "setup"     "max_concurrent_jobs")))
				 (if (and mcj (string->number mcj))
				     (string->number mcj)
				     1))) ;; length of the register queue ahead
	(reglen                (if (number? reglen-in) reglen-in 1))
	(last-time-incomplete  (- (current-seconds) 900)) ;; force at least one clean up cycle
	(last-time-some-running (current-seconds)))
	(last-time-some-running (current-seconds))
	(tdbdat                (tasks:open-db)))

    ;; Initialize the test-registery hash with tests that already have a record
    ;; convert state to symbol and use that as the hash value
    (for-each (lambda (trec)
		(let ((id (db:test-get-id        trec))
		      (tn (db:test-get-testname  trec))
		      (ip (db:test-get-item-path trec))
930
931
932
933
934
935
936




937
938
939
940
941
942
943
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948







+
+
+
+







	     (itemdat     (tests:testqueue-get-itemdat    test-record)) ;; itemdat can be a string, list or #f
	     (items       (tests:testqueue-get-items      test-record))
	     (item-path   (item-list->path itemdat))
	     (tfullname   (runs:make-full-test-name test-name item-path))
	     (newtal      (append tal (list hed)))
	     (regfull     (>= (length reg) reglen))
	     (num-running (rmt:get-count-tests-running-for-run-id run-id)))

	;; every couple minutes verify the server is there for this run
	(if (common:low-noise-print 60 "try start server"  run-id)
	    (tasks:start-and-wait-for-server tdbdat run-id 10))

      (if (> num-running 0)
	  (set! last-time-some-running (current-seconds)))

      (if (> (current-seconds)(+ last-time-some-running 240))
	  (hash-table-set! *max-tries-hash* tfullname (+ (hash-table-ref/default *max-tries-hash* tfullname 0) 1)))
	;; (debug:print 0 "max-tries-hash: " (hash-table->alist *max-tries-hash*))

Modified tasks.scm from [792641d7ed] to [5a72285f95].

346
347
348
349
350
351
352
353
354
355
356
357
358
359
360

361
362
363
364
365
366
367
346
347
348
349
350
351
352

353
354
355
356
357
358

359
360
361
362
363
364
365
366







-






-
+







     mdb ;; NEEDS dbprep ADDED
     "SELECT id FROM servers WHERE run_id=? AND (state = 'running' OR (state = 'dbprep' AND  (strftime('%s','now') - start_time) < 60));" run-id)
    res))

;; try to start a server and wait for it to be available
;;
(define (tasks:start-and-wait-for-server tdbdat run-id delay-max-tries)
  (tdbdat             (tasks:open-db))
  ;; ensure a server is running for this run
  (let loop ((server-dat (tasks:get-server (db:delay-if-busy tdbdat) run-id))
	     (delay-time 0))
      (if (and (not server-dat)
	       (< delay-time delay-max-tries))
	  (begin
	    (debug:print 0 "Try starting server")
	    (if (common:low-noise-print 60 run-id)(debug:print 0 "Try starting server for run-id " run-id))
	    (server:kind-run run-id)
	    (thread-sleep! (min delay-time 5))
	    (loop (tasks:get-server (db:delay-if-busy tdbdat) run-id)(+ delay-time 1))))))

(define (tasks:get-all-servers mdb)
  (let ((res '()))
    (sqlite3:for-each-row