Megatest

Check-in [49e3212725]
Login
Overview
Comment:Better filter behavior in states/statuses
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.61 | v1.6102
Files: files | file ages | folders
SHA1: 49e321272584649f70c839ff5c67d64e01ba9889
User & Date: mrwellan on 2016-05-16 13:56:13
Other Links: branch diff | manifest | tags
Context
2016-05-16
17:13
Split show/hide to two buttons check-in: cf1f6d704a user: mrwellan tags: v1.61
13:56
Better filter behavior in states/statuses check-in: 49e3212725 user: mrwellan tags: v1.61, v1.6102
09:48
Removed the New View tab (for now). Added check for var in mt.scm - just guessing at it being a root cause of an archiving issue check-in: 36f8a6a72f user: mrwellan tags: v1.61
Changes

Modified dashboard.scm from [71676c3c43] to [4e5ce67413].

213
214
215
216
217
218
219

220
221
222
223
224
225
226
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227







+








(d:alldat-useserver-set! *alldat* (cond
				   ((args:get-arg "-use-local") #f)
				   ((configf:lookup *configdat* "dashboard" "use-server")
				    (let ((ans (config:lookup *configdat* "dashboard" "use-server")))
				      (if (equal? ans "yes") #t #f)))
				   (else #t)))
(define *dashboard-mode* (string->symbol (or (configf:lookup *configdat* "dashboard" "mode") "dashboard")))

(d:alldat-dbdir-set! *alldat* (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
(d:alldat-dblocal-set! *alldat* (make-dbr:dbstruct path:  (d:alldat-dbdir *alldat*)
						   local: #t))
(d:alldat-dbfpath-set! *alldat* (db:dbfile-path 0))

;; HACK ALERT: this is a hack, please fix.
347
348
349
350
351
352
353
354


355
356
357
358
359
360
361
362
363


364
365
366
367
368
369
370
348
349
350
351
352
353
354

355
356
357
358
359
360
361
362
363
364

365
366
367
368
369
370
371
372
373







-
+
+








-
+
+







						 #f #f
						 (d:alldat-hide-not-hide data)
						 sort-by
						 sort-order
						 'shortlist
						 (if (d:alldat-filters-changed data)
						     0
						     last-update))
						     last-update)
						 *dashboard-mode*) ;; use dashboard mode
			  (db:get-tests-for-run (d:alldat-dblocal data) run-id testnamepatt states statuses
						#f #f
						(d:alldat-hide-not-hide data)
						sort-by
						sort-order
						'shortlist
						(if (d:alldat-filters-changed data)
						    0
						    last-update))))
						    last-update)
						*dashboard-mode*)))
	 (tests       (let ((newdat (filter
				     (lambda (x)
				       (not (equal? (db:test-get-state x) "DELETED"))) ;; remove deleted tests but do it after merging
				     (delete-duplicates (if (d:alldat-filters-changed data)
							    tmptests
							    (append tmptests prev-tests))
							(lambda (a b)
399
400
401
402
403
404
405


406
407
408
409
410
411
412
413








414
415
416
417
418
419
420
402
403
404
405
406
407
408
409
410








411
412
413
414
415
416
417
418
419
420
421
422
423
424
425







+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+







					(db:get-key-vals (d:alldat-dblocal data) run-id)))
		       (tests       (dboard:get-tests-for-run-duplicate data run-id run testnamepatt key-vals)))
		  ;; NOTE: bubble-up also sets the global (d:alldat-item-test-names data)
		  ;; (tests       (bubble-up tmptests priority: bubble-type))
		  ;; NOTE: 11/01/2013 This routine is *NOT* getting called excessively.
		  ;; (debug:print 0 "Getting data for run " run-id " with key-vals=" key-vals)
		  ;; Not sure this is needed?
		  (if (not (null? tests))
		      (begin
		  (set! referenced-run-ids (cons run-id referenced-run-ids))
		  (if (> (length tests) maxtests)
		      (set! maxtests (length tests)))
		  (if (or (not (d:alldat-hide-empty-runs data)) ;; this reduces the data burden when set
			  (not (null? tests)))
		      (let ((dstruct (vector run tests key-vals (- (current-seconds) 10))))
			(hash-table-set! (d:alldat-allruns-by-id data) run-id dstruct)
			(set! result (cons dstruct result))))))
			(set! referenced-run-ids (cons run-id referenced-run-ids))
			(if (> (length tests) maxtests)
			    (set! maxtests (length tests)))
			(if (or (not (d:alldat-hide-empty-runs data)) ;; this reduces the data burden when set
				(not (null? tests)))
			    (let ((dstruct (vector run tests key-vals (- (current-seconds) 10))))
			      (hash-table-set! (d:alldat-allruns-by-id data) run-id dstruct)
			      (set! result (cons dstruct result))))))))
	      runs)

    (d:alldat-header-set! data header)
    (d:alldat-allruns-set! data result)
    (debug:print-info 6 "(d:alldat-allruns data) has " (length (d:alldat-allruns data)) " runs")
    maxtests))

678
679
680
681
682
683
684

685
686
687
688
689
690
691
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697







+








(define (update-search x val)
  (hash-table-set! (d:alldat-searchpatts *alldat*) x val)
  (d:alldat-filters-changed-set! *alldat* #t)
  (set-bg-on-filter))

(define (mark-for-update)
  (d:alldat-filters-changed-set! *alldat* #t)
  (d:alldat-last-db-update-set! *alldat* 0))

;;======================================================================
;; R U N C O N T R O L
;;======================================================================

;; target populating logic
1205
1206
1207
1208
1209
1210
1211
1212


1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223


1224
1225
1226
1227
1228
1229
1230
1211
1212
1213
1214
1215
1216
1217

1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229

1230
1231
1232
1233
1234
1235
1236
1237
1238







-
+
+










-
+
+







					     (hash-table-keys (d:alldat-status-ignore-hash data)) ;; '()
					     #f #f
					     (d:alldat-hide-not-hide data)
					     #f #f
					     "id,testname,item_path,state,status"
					     (if (d:alldat-filters-changed data)
						 0
						 last-update)) ;; get 'em all
						 last-update)
					     *dashboard-mode*) ;; get 'em all
		      (db:get-tests-for-run db run-id 
					    (hash-table-ref/default (d:alldat-searchpatts data) "test-name" "%/%")
					    (hash-table-keys (d:alldat-state-ignore-hash data)) ;; '()
					    (hash-table-keys (d:alldat-status-ignore-hash data)) ;; '()
					    #f #f
					    (d:alldat-hide-not-hide data)
					    #f #f
					    "id,testname,item_path,state,status"
					    (if (d:alldat-filters-changed data)
						0
						last-update)))
						last-update)
					    *dashboard-mode*))
		  '()))) ;; get 'em all
    (debug:print 0 "dboard:get-tests-dat: got " (length tdat) " test records for run " run-id)
    (sort tdat (lambda (a b)
		 (let* ((aval (vector-ref a 2))
			(bval (vector-ref b 2))
			(anum (string->number aval))
			(bnum (string->number bval)))
1593
1594
1595
1596
1597
1598
1599

1600
1601
1602
1603
1604
1605






1606
1607
1608
1609

1610
1611
1612
1613
1614
1615






1616
1617
1618
1619
1620
1621
1622
1601
1602
1603
1604
1605
1606
1607
1608






1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619






1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632







+
-
-
-
-
-
-
+
+
+
+
+
+




+
-
-
-
-
-
-
+
+
+
+
+
+







						 (mark-for-update))))))
	   (iup:frame 
	    #:title "state/status filter"
	    (iup:vbox
	     (apply 
	      iup:hbox
	      (map (lambda (status)
		     (iup:toggle (conc status "  ")
		     (iup:toggle status  #:action   (lambda (obj val)
						      (mark-for-update)
						      (if (eq? val 1)
							  (hash-table-set! (d:alldat-status-ignore-hash data) status #t)
							  (hash-table-delete! (d:alldat-status-ignore-hash data) status))
						      (set-bg-on-filter))))
				 #:action   (lambda (obj val)
					      (mark-for-update)
					      (if (eq? val 1)
						  (hash-table-set! (d:alldat-status-ignore-hash data) status #t)
						  (hash-table-delete! (d:alldat-status-ignore-hash data) status))
					      (set-bg-on-filter))))
		   (map cadr *common:std-statuses*))) ;; '("PASS" "FAIL" "WARN" "CHECK" "WAIVED" "STUCK/DEAD" "n/a" "SKIP")))
	     (apply 
	      iup:hbox
	      (map (lambda (state)
		     (iup:toggle (conc state "  ")
		     (iup:toggle state   #:action   (lambda (obj val)
						      (mark-for-update)
						      (if (eq? val 1)
							  (hash-table-set! (d:alldat-state-ignore-hash data) state #t)
							  (hash-table-delete! (d:alldat-state-ignore-hash data) state))
						      (set-bg-on-filter))))
				 #:action   (lambda (obj val)
					      (mark-for-update)
					      (if (eq? val 1)
						  (hash-table-set! (d:alldat-state-ignore-hash data) state #t)
						  (hash-table-delete! (d:alldat-state-ignore-hash data) state))
					      (set-bg-on-filter))))
		   (map cadr *common:std-states*))) ;; '("RUNNING" "COMPLETED" "INCOMPLETE" "LAUNCHED" "NOT_STARTED" "KILLED" "DELETED")))
	     (iup:valuator #:valuechanged_cb (lambda (obj)
					       (let ((val (inexact->exact (round (/ (string->number (iup:attribute obj "VALUE")) 10))))
						     (oldmax   (string->number (iup:attribute obj "MAX")))
						     (maxruns  (d:alldat-tot-runs data)))
						 (d:alldat-start-run-offset-set! data val)
						 (mark-for-update)

Modified db.scm from [a736cce216] to [b405cf0e93].

2160
2161
2162
2163
2164
2165
2166



2167

2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181


2182
2183
2184



2185
2186
2187
2188
2189


2190
2191
2192



2193
2194
2195
2196
2197


2198

2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210

2211
2212
2213
2214
2215
2216
2217
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169

2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186



2187
2188
2189
2190
2191
2192
2193
2194
2195
2196



2197
2198
2199
2200
2201
2202
2203
2204
2205
2206

2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218

2219
2220
2221
2222
2223
2224
2225
2226







+
+
+
-
+














+
+
-
-
-
+
+
+





+
+
-
-
-
+
+
+





+
+
-
+











-
+







;;  T E S T S
;;======================================================================

;; states and statuses are lists, turn them into ("PASS","FAIL"...) and use NOT IN
;; i.e. these lists define what to NOT show.
;; states and statuses are required to be lists, empty is ok
;; not-in #t = above behaviour, #f = must match
;; mode:
;;  'dashboard - use state = 'COMPLETED' AND status in ( statuses ) OR state in ( states )
;;
(define (db:get-tests-for-run dbstruct run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update)
(define (db:get-tests-for-run dbstruct run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update mode)
  (if (not (number? run-id))
      (begin ;; no need to treat this as an error by default
	(debug:print 4 "WARNING: call to db:get-tests-for-run with bad run-id=" run-id)
	;; (print-call-chain (current-error-port))
	'())
      (let* ((qryvalstr       (case qryvals
				((shortlist) "id,run_id,testname,item_path,state,status")
				((#f)        db:test-record-qry-selector) ;; "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment")
				(else        qryvals)))
	     (res            '())
	     ;; if states or statuses are null then assume match all when not-in is false
	     (states-qry      (if (null? states) 
				  #f
				  (conc " state "  
					(if (eq? mode 'dashboard)
					    " IN ('"
					(if not-in
					    " NOT IN ('"
					    " IN ('") 
					    (if not-in
						" NOT IN ('"
						" IN ('")) 
					(string-intersperse states   "','")
					"')")))
	     (statuses-qry    (if (null? statuses)
				  #f
				  (conc " status "
					(if (eq? mode 'dashboard)
					    " IN ('"
					(if not-in 
					    " NOT IN ('"
					    " IN ('") 
					    (if not-in 
						" NOT IN ('"
						" IN ('") )
					(string-intersperse statuses "','")
					"')")))
	     (states-statuses-qry 
	      (cond 
	       ((and states-qry statuses-qry)
		(case mode
		  ((dashboard)(conc " AND " (if not-in "NOT " "") "( ( state='COMPLETED' AND " statuses-qry " ) OR " states-qry " ) "))
		(conc " AND ( " states-qry " AND " statuses-qry " ) "))
		  (else       (conc " AND ( " states-qry " AND " statuses-qry " ) "))))
	       (states-qry  
		(conc " AND " states-qry))
	       (statuses-qry 
		(conc " AND " statuses-qry))
	       (else "")))
	     (tests-match-qry (tests:match->sqlqry testpatt))
	     (qry             (conc "SELECT " qryvalstr
				    " FROM tests WHERE run_id=? "
				    (if last-update " " " AND state != 'DELETED' ") ;; if using last-update we want deleted tests?
				    states-statuses-qry
				    (if tests-match-qry (conc " AND (" tests-match-qry ") ") "")
				    (if last-update (conc " AND last_update > " last-update " ") "")
				    (if last-update (conc " AND last_update >= " last-update " ") "")
				    (case sort-by
				      ((rundir)      " ORDER BY length(rundir) ")
				      ((testname)    (conc " ORDER BY testname " (if sort-order (conc sort-order ",") "") " item_path "))
				      ((statestatus) (conc " ORDER BY state " (if  sort-order (conc sort-order ",") "") " status "))
				      ((event_time)  " ORDER BY event_time ")
				      (else          (if (string? sort-by)
							 (conc " ORDER BY " sort-by " ")
2296
2297
2298
2299
2300
2301
2302
2303

2304
2305
2306
2307
2308
2309
2310
2305
2306
2307
2308
2309
2310
2311

2312
2313
2314
2315
2316
2317
2318
2319







-
+







(define (db:get-tests-for-runs dbstruct run-ids testpatt states statuses #!key (not-in #f)(qryvals #f))
  ;; (db:delay-if-busy)
  (let ((res '()))
    (for-each 
     (lambda (run-id)
       (set! res (append 
		  res 
		  (db:get-tests-for-run dbstruct run-id testpatt states statuses #f #f not-in #f #f qryvals))))
		  (db:get-tests-for-run dbstruct run-id testpatt states statuses #f #f not-in #f #f qryvals #f 'normal))))
     (if run-ids
	 run-ids
	 (db:get-all-run-ids dbstruct)))
    res))

;; Convert calling routines to get list of run-ids and loop, do not use the get-tests-for-runs
;;
3309
3310
3311
3312
3313
3314
3315
3316

3317
3318
3319
3320
3321
3322
3323
3318
3319
3320
3321
3322
3323
3324

3325
3326
3327
3328
3329
3330
3331
3332







-
+







	  ;; collect all matching tests for the runs then
	  ;; extract the most recent test and return that.
	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals 
		       ", previous run ids found: " prev-run-ids)
	  (if (null? prev-run-ids) '()  ;; no previous runs? return null
	      (let loop ((hed (car prev-run-ids))
			 (tal (cdr prev-run-ids)))
		(let ((results (db:get-tests-for-run dbstruct hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f #f)))
		(let ((results (db:get-tests-for-run dbstruct hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f #f 'normal)))
		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name 
			       ", item-path " item-path " results: " (intersperse results "\n"))
		  ;; Keep only the youngest of any test/item combination
		  (for-each 
		   (lambda (testdat)
		     (let* ((full-testname (conc (db:test-get-testname testdat) "/" (db:test-get-item-path testdat)))
			    (stored-test   (hash-table-ref/default tests-hash full-testname #f)))

Modified launch.scm from [daec9f5f05] to [24c723f779].

745
746
747
748
749
750
751




752
753
754
755
756
757
758
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762







+
+
+
+







	(set! *runconfigdat* first-rundat)
	(if first-pass  ;; 
	    (begin
	      (set! *configdat*  (car first-pass))
	      (set! *configinfo* first-pass)
	      (set! *toppath*    (or toppath (cadr first-pass))) ;; use the gathered data unless already have it
	      (set! toppath      *toppath*)
	      (if (not *toppath*)
		  (begin
		    (debug:print "ERROR: you are not in a megatest area!")
		    (exit 1)))
	      (setenv "MT_RUN_AREA_HOME" *toppath*)
	      ;; the seed read is done, now read runconfigs, cache it then read megatest.config one more time and cache it
	      (let* ((keys         (rmt:get-keys))
		     (key-vals     (keys:target->keyval keys target))
		     (linktree     (or (getenv "MT_LINKTREE")
				       (if *configdat* (configf:lookup *configdat* "setup" "linktree") #f)))
		     (second-pass  (find-and-read-config

Modified megatest.scm from [018155474b] to [d7706449e8].

1113
1114
1115
1116
1117
1118
1119
1120


1121
1122
1123
1124
1125
1126
1127
1113
1114
1115
1116
1117
1118
1119

1120
1121
1122
1123
1124
1125
1126
1127
1128







-
+
+







			  (tests   (if tests-spec
				       (rmt:get-tests-for-run run-id testpatt states statuses #f #f #f 'testname 'asc ;; (db:get-tests-for-run dbstruct run-id testpatt '() '() #f #f #f 'testname 'asc 
							     ;; use qryvals if test-spec provided
							     (if tests-spec
								 (string-intersperse adj-tests-spec ",")
								 ;; db:test-record-fields
								 #f)
							     #f)
							     #f
							     'normal)
				       '())))
		     (case dmode
		       ((json ods)
			(if runs-spec
			    (for-each 
			     (lambda (field-name)
			       (mutils:hierhash-set! data (conc (db:get-value-by-header run header field-name)) targetstr runname "meta" field-name))

Modified mt.scm from [1933da3585] to [4179497d10].

65
66
67
68
69
70
71
72

73
74
75
76
77
78
79
80
81

82
83
84
85
86
87
88
65
66
67
68
69
70
71

72
73
74
75
76
77
78
79
80

81
82
83
84
85
86
87
88







-
+








-
+







	 (vector header full-list)))))

;;======================================================================
;;  T E S T S
;;======================================================================

(define (mt:get-tests-for-run run-id testpatt states status #!key (not-in #t) (sort-by 'event_time) (sort-order "ASC") (qryvals #f)(last-update #f))
  (let loop ((testsdat (rmt:get-tests-for-run run-id testpatt states status 0 500 not-in sort-by sort-order qryvals last-update))
  (let loop ((testsdat (rmt:get-tests-for-run run-id testpatt states status 0 500 not-in sort-by sort-order qryvals last-update 'normal))
	     (res      '())
	     (offset   0)
	     (limit    500))
    (let* ((full-list (append res testsdat))
	   (have-more (eq? (length testsdat) limit)))
      (if have-more 
	  (let ((new-offset (+ offset limit)))
	    (debug:print-info 4 "More than " limit " tests, have " (length full-list) " tests so far.")
	    (loop (rmt:get-tests-for-run run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals last-update)
	    (loop (rmt:get-tests-for-run run-id testpatt states status new-offset limit not-in sort-by sort-order qryvals last-update 'normal)
		  full-list
		  new-offset
		  limit))
	  full-list))))

(define (mt:lazy-get-prereqs-not-met run-id waitons ref-item-path #!key (mode '(normal))(itemmaps #f) )
  (let* ((key    (list run-id waitons ref-item-path mode))

Modified rmt.scm from [b3e339430d] to [e1950b4244].

384
385
386
387
388
389
390
391

392
393

394
395
396
397
398
399
400
384
385
386
387
388
389
390

391
392

393
394
395
396
397
398
399
400







-
+

-
+







;; WARNING: This currently bypasses the transaction wrapped writes system
(define (rmt:test-set-state-status-by-id run-id test-id newstate newstatus newcomment)
  (rmt:send-receive 'test-set-state-status-by-id run-id (list run-id test-id newstate newstatus newcomment)))

(define (rmt:set-tests-state-status run-id testnames currstate currstatus newstate newstatus)
  (rmt:send-receive 'set-tests-state-status run-id (list run-id testnames currstate currstatus newstate newstatus)))

(define (rmt:get-tests-for-run run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update)
(define (rmt:get-tests-for-run run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update mode)
  (if (number? run-id)
      (rmt:send-receive 'get-tests-for-run run-id (list run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update))
      (rmt:send-receive 'get-tests-for-run run-id (list run-id testpatt states statuses offset limit not-in sort-by sort-order qryvals last-update mode))
      (begin
	(debug:print "ERROR: rmt:get-tests-for-run called with bad run-id=" run-id)
	(print-call-chain (current-error-port))
	'())))

;; get stuff via synchash 
(define (rmt:synchash-get run-id proc synckey keynum params)
617
618
619
620
621
622
623
624

625
626
627
628
629
630
631
617
618
619
620
621
622
623

624
625
626
627
628
629
630
631







-
+







	(let ((prev-run-ids (rmt:get-prev-run-ids run-id)))
	  ;; for each run starting with the most recent look to see if there is a matching test
	  ;; if found then return that matching test record
	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids)
	  (if (null? prev-run-ids) #f
	      (let loop ((hed (car prev-run-ids))
			 (tal (cdr prev-run-ids)))
		(let ((results (rmt:get-tests-for-run hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f #f)))
		(let ((results (rmt:get-tests-for-run hed (conc test-name "/" item-path) '() '() #f #f #f #f #f #f #f 'normal)))
		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results)
		  (if (and (null? results)
			   (not (null? tal)))
		      (loop (car tal)(cdr tal))
		      (if (null? results) #f
			  (car results))))))))))