Megatest

Diff
Login

Differences From Artifact [9bee706896]:

To Artifact [db8f47231c]:


1973
1974
1975
1976
1977
1978
1979


1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
		   qrystr
		   )))
    (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " target: " target " offset: " offset " limit: " count)
    res))

;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db)
;;


(define (db:get-changed-run-ids since-time)
  (let* ((dbdir      (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir"))
	 (alldbs     (glob (conc dbdir "/.megatest/[0-9]*.db")))
	 (changed    (filter (lambda (dbfile)
			       (> (file-modification-time dbfile) since-time))
			     alldbs)))
    (delete-duplicates
     (map (lambda (dbfile)
	    (let* ((res (string-match ".*\\/(\\d)*\\.db" dbfile)))
	      (if res
		  (string->number (cadr res))
		  (begin
		    (debug:print 2 *default-log-port* "WARNING: Failed to process " dbfile " for run-id")
		    0))))
	  changed))))








>
>








|







1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
		   qrystr
		   )))
    (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " target: " target " offset: " offset " limit: " count)
    res))

;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db)
;;
;; NOTE: This DOESN'T (necessarily) get the real run ids, but the number of the <number>.db!!

(define (db:get-changed-run-ids since-time)
  (let* ((dbdir      (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir"))
	 (alldbs     (glob (conc dbdir "/.megatest/[0-9]*.db")))
	 (changed    (filter (lambda (dbfile)
			       (> (file-modification-time dbfile) since-time))
			     alldbs)))
    (delete-duplicates
     (map (lambda (dbfile)
	    (let* ((res (string-match ".*\\/(\\d\\d)\\.db" dbfile)))
	      (if res
		  (string->number (cadr res))
		  (begin
		    (debug:print 2 *default-log-port* "WARNING: Failed to process " dbfile " for run-id")
		    0))))
	  changed))))

4421
4422
4423
4424
4425
4426
4427
4428













4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
;;
(define (db:get-changed-record-ids dbstruct since-time)
  ;; no transaction, allow the db to be accessed between the big queries
  (let* ((backcons (lambda (lst item)(cons item lst)))
         (all_tests '())
         (all_test_steps '())
         (all_test_data '())
         (run_ids (db:get-changed-run-ids since-time))













         (run_stat_ids
          (db:with-db dbstruct #f #f 
            (lambda (dbdat db)
              (sqlite3:fold-row backcons '() db "SELECT id FROM run_stats  WHERE last_update>=?" since-time))
          )
         )
        )
        (for-each
          (lambda (run_id)
            (set! all_tests 
             (append 
               (map (lambda (x) (cons x run_id))                
                (db:with-db dbstruct run_id #f 
                  (lambda (dbdat db)
                    (sqlite3:fold-row backcons '() db "SELECT id FROM tests  WHERE last_update>=?" since-time)
                  )
                )
               ) all_tests
              )
            )
            (set! all_test_steps 
              (append 
                (map (lambda (x) (cons x run_id))
                  (db:with-db dbstruct run_id #f 
                    (lambda (dbdat db)
                      (sqlite3:fold-row backcons '() db "SELECT id FROM test_steps  WHERE last_update>=?" since-time)
                    )
                  )
                ) all_test_steps
              )
            )
            (set! all_test_data 
              (append 
                (map (lambda (x) (cons x run_id))
                  (db:with-db dbstruct run_id #f 
                    (lambda (dbdat db)
                      (sqlite3:fold-row backcons '() db "SELECT id FROM test_data  WHERE last_update>=?" since-time)
                    )
                  )
                ) all_test_data
              )
            )
          )
          run_ids
        )
        (debug:print 2 *default-log-port*  "run_ids = " run_ids)
        (debug:print 2 *default-log-port*  "all_tests = " all_tests)

      `((runs       . ,run_ids)
        (tests      . ,all_tests)
        (test_steps . ,all_test_steps)







|
>
>
>
>
>
>
>
>
>
>
>
>
>









|




|










|

















|







4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
;;
(define (db:get-changed-record-ids dbstruct since-time)
  ;; no transaction, allow the db to be accessed between the big queries
  (let* ((backcons (lambda (lst item)(cons item lst)))
         (all_tests '())
         (all_test_steps '())
         (all_test_data '())
         (changed_run_dbs (db:get-changed-run-ids since-time)) ;; gets the rundb numbers
         (all_run_ids 
          (db:with-db dbstruct #f #f 
            (lambda (dbdat db)
              (sqlite3:fold-row backcons '() db "SELECT id FROM runs"))
          )
         )
         (changed_run_ids (filter (lambda (run) (member (modulo run 100) changed_run_dbs)) all_run_ids))
         (run_ids 
          (db:with-db dbstruct #f #f 
            (lambda (dbdat db)
              (sqlite3:fold-row backcons '() db "SELECT id FROM runs  WHERE last_update>=?" since-time))
          )
         )
         (run_stat_ids
          (db:with-db dbstruct #f #f 
            (lambda (dbdat db)
              (sqlite3:fold-row backcons '() db "SELECT id FROM run_stats  WHERE last_update>=?" since-time))
          )
         )
        )
        (for-each
          (lambda (run_id)
           (set! all_tests 
             (append 
               (map (lambda (x) (cons x run_id))                
                (db:with-db dbstruct run_id #f 
                  (lambda (dbdat db)
                    (sqlite3:fold-row backcons '() db "SELECT id FROM tests  WHERE run_id=? and last_update>=?" run_id since-time)
                  )
                )
               ) all_tests
              )
            )
            (set! all_test_steps 
              (append 
                (map (lambda (x) (cons x run_id))
                  (db:with-db dbstruct run_id #f 
                    (lambda (dbdat db)
                      (sqlite3:fold-row backcons '() db "SELECT id FROM test_steps WHERE last_update>=?" since-time)
                    )
                  )
                ) all_test_steps
              )
            )
            (set! all_test_data 
              (append 
                (map (lambda (x) (cons x run_id))
                  (db:with-db dbstruct run_id #f 
                    (lambda (dbdat db)
                      (sqlite3:fold-row backcons '() db "SELECT id FROM test_data  WHERE last_update>=?" since-time)
                    )
                  )
                ) all_test_data
              )
            )
          )
          changed_run_ids
        )
        (debug:print 2 *default-log-port*  "run_ids = " run_ids)
        (debug:print 2 *default-log-port*  "all_tests = " all_tests)

      `((runs       . ,run_ids)
        (tests      . ,all_tests)
        (test_steps . ,all_test_steps)