Megatest

Check-in [82ec591216]
Login
Overview
Comment:Added some instrumentation. Some cleanup for -O3
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.61
Files: files | file ages | folders
SHA1: 82ec59121607a5fff2675e60c808a149d33b1519
User & Date: matt on 2016-07-21 08:23:28
Other Links: branch diff | manifest | tags
Context
2016-07-21
21:08
more rework check-in: aab4163601 user: mrwellan tags: v1.61
08:23
Added some instrumentation. Some cleanup for -O3 check-in: 82ec591216 user: matt tags: v1.61
00:33
refactored rundat from vector to defstruct check-in: f52adb7de7 user: matt tags: v1.61
Changes

Modified archive.scm from [84c8e03d01] to [31c5249136].

66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  (let* ((existing-blocks (rmt:archive-get-allocations testname itempath dused))
	 (candidate-disks (map (lambda (block)
				 (list
				  (vector-ref block 1)   ;; archive-area-name
				  (vector-ref block 2))) ;; disk-path
			       existing-blocks)))
    (or (common:get-disk-with-most-free-space candidate-disks dused)
	(archive:allocate-new-archive-block testname itempath))))

;; allocate a new archive area
;;
(define (archive:allocate-new-archive-block run-area-home testsuite-name dneeded)
  (let* ((adisks    (archive:get-archive-disks))
	 (best-disk (common:get-disk-with-most-free-space adisks dneeded)))
    (if best-disk







|







66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  (let* ((existing-blocks (rmt:archive-get-allocations testname itempath dused))
	 (candidate-disks (map (lambda (block)
				 (list
				  (vector-ref block 1)   ;; archive-area-name
				  (vector-ref block 2))) ;; disk-path
			       existing-blocks)))
    (or (common:get-disk-with-most-free-space candidate-disks dused)
	(archive:allocate-new-archive-block #f #f #f)))) ;; BROKEN. testname itempath))))

;; allocate a new archive area
;;
(define (archive:allocate-new-archive-block run-area-home testsuite-name dneeded)
  (let* ((adisks    (archive:get-archive-disks))
	 (best-disk (common:get-disk-with-most-free-space adisks dneeded)))
    (if best-disk

Modified configf.scm from [cf3db9b475] to [b7009fc33b].

350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
	'()
	(map car sectdat))))

(define (configf:get-section cfgdat section)
  (hash-table-ref/default cfgdat section '()))

(define (setup)
  (let* ((configf (find-config))
	 (config  (if configf (read-config configf #f #t) #f)))
    (if config
	(setenv "RUN_AREA_HOME" (pathname-directory configf)))
    config))

;;======================================================================
;; Non destructive writing of config file







|







350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
	'()
	(map car sectdat))))

(define (configf:get-section cfgdat section)
  (hash-table-ref/default cfgdat section '()))

(define (setup)
  (let* ((configf (find-config "megatest.config"))
	 (config  (if configf (read-config configf #f #t) #f)))
    (if config
	(setenv "RUN_AREA_HOME" (pathname-directory configf)))
    config))

;;======================================================================
;; Non destructive writing of config file

Modified dashboard.scm from [fc69924fac] to [f4f45839ad].

461
462
463
464
465
466
467

468
469
470
471
472
473
474
475

476
477
478
479
480
481
482
    (print "prev-tests: " (length prev-tests) " tests: " (length tests))
    tests))

;; tmptests   - new tests data
;; prev-tests - old tests data
;;
(define (dashboard:merge-changed-tests tests tmptests use-new prev-tests) 

  (let ((newdat (filter
		 (lambda (x)
		   (not (equal? (db:test-get-state x) "DELETED"))) ;; remove deleted tests but do it after merging
		 (delete-duplicates (if use-new ;; (dboard:tabdat-filters-changed tabdat)
					tmptests
					(append tmptests prev-tests))
				    (lambda (a b)
				      (eq? (db:test-get-id a)(db:test-get-id b)))))))

    (if (eq? *tests-sort-reverse* 3) ;; +event_time
	(sort newdat dboard:compare-tests)
	newdat)))

;; this calls dboard:get-tests-for-run-duplicate for each run
;;
;; create a virtual table of all the tests







>
|
|
|
|
|
|
|
|
>







461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
    (print "prev-tests: " (length prev-tests) " tests: " (length tests))
    tests))

;; tmptests   - new tests data
;; prev-tests - old tests data
;;
(define (dashboard:merge-changed-tests tests tmptests use-new prev-tests) 
  (let ((start-time (current-seconds))
	(newdat     (filter
		     (lambda (x)
		       (not (equal? (db:test-get-state x) "DELETED"))) ;; remove deleted tests but do it after merging
		     (delete-duplicates (if use-new ;; (dboard:tabdat-filters-changed tabdat)
					    tmptests
					    (append tmptests prev-tests))
					(lambda (a b)
					  (eq? (db:test-get-id a)(db:test-get-id b)))))))
    (print "Time took: " (- (current-seconds) start-time))
    (if (eq? *tests-sort-reverse* 3) ;; +event_time
	(sort newdat dboard:compare-tests)
	newdat)))

;; this calls dboard:get-tests-for-run-duplicate for each run
;;
;; create a virtual table of all the tests

Modified db.scm from [9f0dd86e3d] to [bedbc0fe70].

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
(define *number-of-writes* 0)
(define *number-non-write-queries* 0)

;;======================================================================
;; SQLITE3 HELPERS
;;======================================================================

(define (db:general-sqlite-error-dump exn stmt run-id params)
  (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
    ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
    (print "err-status: " err-status)
    (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
    (print-call-chain (current-error-port))))

;; convert to -inline







|







34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
(define *number-of-writes* 0)
(define *number-non-write-queries* 0)

;;======================================================================
;; SQLITE3 HELPERS
;;======================================================================

(define (db:general-sqlite-error-dump exn stmt . params)
  (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
    ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
    (print "err-status: " err-status)
    (debug:print-error 0 *default-log-port* " query " stmt " failed, params: " params ", error: " ((condition-property-accessor 'exn 'message) exn))
    (print-call-chain (current-error-port))))

;; convert to -inline

Modified tasks.scm from [4d978918ac] to [98cb71820d].

527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
	     (last-db-update 0)) ;; (file-modification-time megatestdb)))
	(task:register-monitor mdb)
	(let loop ((count      0)
		   (next-touch 0)) ;; next-touch is the time where we need to update last_update
	  ;; if the db has been modified we'd best look at the task queue
	  (let ((modtime (file-modification-time megatestdbpath )))
	    (if (> modtime last-db-update)
		(tasks:process-queue db mdb last-db-update megatestdb next-touch))
	    ;; WARNING: Possible race conditon here!!
	    ;; should this update be immediately after the task-get-action call above?
	    (if (> (current-seconds) next-touch)
		(begin
		  (tasks:monitors-update mdb)
		  (loop (+ count 1)(+ (current-seconds) 240)))
		(loop (+ count 1) next-touch)))))))







|







527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
	     (last-db-update 0)) ;; (file-modification-time megatestdb)))
	(task:register-monitor mdb)
	(let loop ((count      0)
		   (next-touch 0)) ;; next-touch is the time where we need to update last_update
	  ;; if the db has been modified we'd best look at the task queue
	  (let ((modtime (file-modification-time megatestdbpath )))
	    (if (> modtime last-db-update)
		(tasks:process-queue db)) ;; BROKEN. mdb last-db-update megatestdb next-touch))
	    ;; WARNING: Possible race conditon here!!
	    ;; should this update be immediately after the task-get-action call above?
	    (if (> (current-seconds) next-touch)
		(begin
		  (tasks:monitors-update mdb)
		  (loop (+ count 1)(+ (current-seconds) 240)))
		(loop (+ count 1) next-touch)))))))