Megatest

Check-in [dde2e2b857]
Login
Overview
Comment:Can't use stmt caching where string compiled changes with every query. Fixed wrong presevation of last-update for clean printing of pass/fail etc.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65
Files: files | file ages | folders
SHA1: dde2e2b857362e3390523f1fa4a92adaaa80acf6
User & Date: mrwellan on 2020-08-19 23:53:00
Other Links: branch diff | manifest | tags
Context
2020-08-20
11:11
changed version to 1.6564 ==3.39/1.0/PASS/1203/orion== v1.70 START check-in: cd0bb84cf2 user: mmgraham tags: v1.65, v1.6564
2020-08-19
23:53
Can't use stmt caching where string compiled changes with every query. Fixed wrong presevation of last-update for clean printing of pass/fail etc. check-in: dde2e2b857 user: mrwellan tags: v1.65
11:41
merged fork check-in: 4ff27cb772 user: mmgraham tags: v1.65
Changes

Modified db.scm from [12b84ccd5d] to [5d8ca5b259].

3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
				(if offset (conc " OFFSET " offset) " ")
				";"
				)))

    (debug:print-info 8 *default-log-port* "db:get-tests-for-run run-id=" run-id ", qry=" qry)
    (let* ((res (db:with-db dbstruct run-id #f
			    (lambda (db)
			      (let* ((stmth (db:get-cache-stmth dbstruct db qry)))
				(reverse
				 (sqlite3:fold-row
				  (lambda (res . row)
				    ;; id run-id testname state status event-time host cpuload
				    ;;     diskfree uname rundir item-path run-duration final-logf comment)
				    (cons (list->vector row) res))
				  '()
				  stmth
				  (or run-id 1) ;; 1 > 0 , for the case where we are seeking tests matching criteral for all runs
				  )))))))
      (case qryvals
	((shortlist)(map db:test-short-record->norm res))
	((#f)       res)
	(else       res)))))

(define (db:test-short-record->norm inrec)
  ;;  "id,run_id,testname,item_path,state,status"







|







|

|







3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
				(if offset (conc " OFFSET " offset) " ")
				";"
				)))

    (debug:print-info 8 *default-log-port* "db:get-tests-for-run run-id=" run-id ", qry=" qry)
    (let* ((res (db:with-db dbstruct run-id #f
			    (lambda (db)
			      ;; (let* ((stmth (db:get-cache-stmth dbstruct db qry))) ;; due to use of last-update we can't efficiently cache this query
				(reverse
				 (sqlite3:fold-row
				  (lambda (res . row)
				    ;; id run-id testname state status event-time host cpuload
				    ;;     diskfree uname rundir item-path run-duration final-logf comment)
				    (cons (list->vector row) res))
				  '()
				  db qry ;; stmth
				  (or run-id 1) ;; 1 > 0 , for the case where we are seeking tests matching criteral for all runs
				  ))))))
      (case qryvals
	((shortlist)(map db:test-short-record->norm res))
	((#f)       res)
	(else       res)))))

(define (db:test-short-record->norm inrec)
  ;;  "id,run_id,testname,item_path,state,status"
3628
3629
3630
3631
3632
3633
3634
3635

3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646

3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
	test-id)
       (reverse res)))))

;;======================================================================
;; T E S T  D A T A 
;;======================================================================

 (define (db:get-data-info-by-id dbstruct  test-data-id)

   (db:with-db
    dbstruct
    #f 
    #f
    (lambda (db)
      (let* ((res (vector #f #f #f #f #f #f #f #f #f #f #f #f)))
        (sqlite3:for-each-row 
       (lambda (id test-id  category variable value expected tol units comment status type last-update)
         (set! res (vector id test-id  category variable value expected tol units comment status type last-update)))
       db
       "SELECT id,test_id, category, variable, value, expected, tol, units, comment, status, type, last_update FROM test_data WHERE id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;

       test-data-id)
        res))))


;; WARNING: Do NOT call this for the parent test on an iterated test
;; Roll up test_data pass/fail results
;; look at the test_data status field, 
;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
(define (db:test-data-rollup dbstruct run-id test-id status)







|
>
|
|
|
|
|
|
|
|
|
<
|
>
|
|
<







3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645

3646
3647
3648
3649

3650
3651
3652
3653
3654
3655
3656
	test-id)
       (reverse res)))))

;;======================================================================
;; T E S T  D A T A 
;;======================================================================

(define (db:get-data-info-by-id dbstruct  test-data-id)
  (let* ((stmt        "SELECT id,test_id, category, variable, value, expected, tol, units, comment, status, type, last_update FROM test_data WHERE id=? ORDER BY id ASC;")) ;; event_time DESC,id ASC;
    (db:with-db
     dbstruct
     #f 
     #f
     (lambda (db)
       (let* ((stmth (db:get-cache-stmth dbstruct db stmt))
	      (res   (sqlite3:fold-row
		      (lambda (res id test-id  category variable value expected tol units comment status type last-update)
			(vector id test-id  category variable value expected tol units comment status type last-update))

		      (vector #f #f #f #f #f #f #f #f #f #f #f #f)
		      stmth
		      test-data-id)))
        res)))))


;; WARNING: Do NOT call this for the parent test on an iterated test
;; Roll up test_data pass/fail results
;; look at the test_data status field, 
;;    if all are pass (any case) and the test status is PASS or NULL or '' then set test status to PASS.
;;    if one or more are fail (any case) then set test status to PASS, non "pass" or "fail" are ignored
(define (db:test-data-rollup dbstruct run-id test-id status)

Modified runs.scm from [b44d99af31] to [fad1b4b96b].

1361
1362
1363
1364
1365
1366
1367
1368
1369

1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390

1391
1392
1393
1394
1395
1396
1397
   run-info: #f
   runname: #f
   target: #f
   )
  )

(define (runs:incremental-print-results run-id)
  (let ((curr-sec (current-seconds)))
    (if (> (- curr-sec (runs:gendat-inc-results-last-update *runs:general-data*)) 5) ;; at least five seconds since last update

	(let* ((run-dat  (or (runs:gendat-run-info *runs:general-data*)(rmt:get-run-info run-id)))
	       (runname  (or (runs:gendat-runname *runs:general-data*)
			     (db:get-value-by-header (db:get-rows run-dat)
						     (db:get-header run-dat) "runname")))
	       (target   (or (runs:gendat-target *runs:general-data*)(rmt:get-target run-id)))
	       (testsdat (let ((res (rmt:get-tests-for-run
				     run-id "%" '() '() ;; run-id testpatt states statuses
				     #f #f ;; offset limit
				     #f ;; not-in
				     #f ;; sort-by
				     #f ;; sort-order
				     #f ;; get full data (not 'shortlist)
				     (runs:gendat-inc-results-last-update *runs:general-data*) ;; last update time
				     'dashboard)))
			   (if (list? res)
			       res
			       (begin
				 (debug:print-error
				  0 *default-log-port*
				  "FAILED TO GET DATA using rmt:get-tests-for-run. Notify developers if you see this. result: " res)
				 '())))))

	  (if (not (runs:gendat-run-info *runs:general-data*))
	      (runs:gendat-run-info-set! *runs:general-data* run-dat))
	  (if (not (runs:gendat-runname  *runs:general-data*))
	      (runs:gendat-runname-set! *runs:general-data* runname))
	  (if (not (runs:gendat-target *runs:general-data*))
	      (runs:gendat-target-set! *runs:general-data* target))
	  (for-each







|
|
>












|








>







1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
   run-info: #f
   runname: #f
   target: #f
   )
  )

(define (runs:incremental-print-results run-id)
  (let ((curr-sec    (current-seconds))
	(last-update (runs:gendat-inc-results-last-update *runs:general-data*)))
    (if (> (- curr-sec last-update) 5) ;; at least five seconds since last update
	(let* ((run-dat  (or (runs:gendat-run-info *runs:general-data*)(rmt:get-run-info run-id)))
	       (runname  (or (runs:gendat-runname *runs:general-data*)
			     (db:get-value-by-header (db:get-rows run-dat)
						     (db:get-header run-dat) "runname")))
	       (target   (or (runs:gendat-target *runs:general-data*)(rmt:get-target run-id)))
	       (testsdat (let ((res (rmt:get-tests-for-run
				     run-id "%" '() '() ;; run-id testpatt states statuses
				     #f #f ;; offset limit
				     #f ;; not-in
				     #f ;; sort-by
				     #f ;; sort-order
				     #f ;; get full data (not 'shortlist)
				     last-update
				     'dashboard)))
			   (if (list? res)
			       res
			       (begin
				 (debug:print-error
				  0 *default-log-port*
				  "FAILED TO GET DATA using rmt:get-tests-for-run. Notify developers if you see this. result: " res)
				 '())))))
	  (runs:gendat-inc-results-last-update-set! *runs:general-data* (- curr-sec 1))
	  (if (not (runs:gendat-run-info *runs:general-data*))
	      (runs:gendat-run-info-set! *runs:general-data* run-dat))
	  (if (not (runs:gendat-runname  *runs:general-data*))
	      (runs:gendat-runname-set! *runs:general-data* runname))
	  (if (not (runs:gendat-target *runs:general-data*))
	      (runs:gendat-target-set! *runs:general-data* target))
	  (for-each
1419
1420
1421
1422
1423
1424
1425


1426


1427
1428
1429
1430
1431
1432
1433
			     state
			     status
			     dtime
			     (seconds->hr-min-sec duration)
			     (conc "lt/" target "/" runname "/" test-name (if (string-null? item-path) "" (conc "/" item-path))))
		     (hash-table-set! (runs:gendat-inc-results *runs:general-data*) (conc run-id "," test-id) testdat)))))
	   testsdat)))


    (runs:gendat-inc-results-last-update-set! *runs:general-data* (- curr-sec 10))))



;; every time though the loop increment the test/itempatt val.
;; when the min is > max-allowed and none running then force exit
;;
(define *max-tries-hash* (make-hash-table))

;;======================================================================







>
>
|
>
>







1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
			     state
			     status
			     dtime
			     (seconds->hr-min-sec duration)
			     (conc "lt/" target "/" runname "/" test-name (if (string-null? item-path) "" (conc "/" item-path))))
		     (hash-table-set! (runs:gendat-inc-results *runs:general-data*) (conc run-id "," test-id) testdat)))))
	   testsdat)))

    ;; I don't think this should be here? -- Matt
    #;(runs:gendat-inc-results-last-update-set! *runs:general-data* (- curr-sec 10))

    ))

;; every time though the loop increment the test/itempatt val.
;; when the min is > max-allowed and none running then force exit
;;
(define *max-tries-hash* (make-hash-table))

;;======================================================================